mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-08-25 08:14:24 +00:00
experimental q3tts batch test
This commit is contained in:
Concedo
parent
4ede3dfea4
commit
c8d6546a14
2 changed files with 59 additions and 30 deletions
|
|
@ -798,6 +798,32 @@ bool AudioTokenizerDecoder::decode(const int32_t * codes, int32_t n_frames,
|
|||
return false;
|
||||
}
|
||||
|
||||
if (n_frames <= 0) {
|
||||
samples.clear();
|
||||
return true;
|
||||
}
|
||||
|
||||
const int32_t max_frames = 32;
|
||||
|
||||
if (max_frames > 0 && n_frames > max_frames) {
|
||||
samples.clear();
|
||||
for (int32_t offset = 0; offset < n_frames; offset += max_frames) {
|
||||
const int32_t chunk = std::min(max_frames, n_frames - offset);
|
||||
std::vector<float> chunk_samples;
|
||||
if (!decode_chunk(codes + (size_t)offset * model_.config.n_codebooks, chunk, offset, chunk_samples)) {
|
||||
return false;
|
||||
}
|
||||
samples.insert(samples.end(), chunk_samples.begin(), chunk_samples.end());
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
return decode_chunk(codes, n_frames, 0, samples);
|
||||
}
|
||||
|
||||
bool AudioTokenizerDecoder::decode_chunk(const int32_t * codes, int32_t n_frames,
|
||||
int32_t position_offset,
|
||||
std::vector<float> & samples) {
|
||||
const auto & cfg = model_.config;
|
||||
|
||||
codes_buf_.resize(n_frames * cfg.n_codebooks);
|
||||
|
|
@ -838,7 +864,7 @@ bool AudioTokenizerDecoder::decode(const int32_t * codes, int32_t n_frames,
|
|||
if (positions_tensor) {
|
||||
std::vector<int32_t> positions(n_frames);
|
||||
for (int i = 0; i < n_frames; ++i) {
|
||||
positions[i] = i;
|
||||
positions[i] = position_offset + i;
|
||||
}
|
||||
ggml_backend_tensor_set(positions_tensor, positions.data(), 0,
|
||||
n_frames * sizeof(int32_t));
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ struct pre_tfm_layer {
|
|||
struct ggml_tensor * attn_v_w = nullptr;
|
||||
struct ggml_tensor * attn_output_w = nullptr;
|
||||
struct ggml_tensor * attn_scale = nullptr; // layer_scale for attention
|
||||
|
||||
|
||||
// FFN (SwiGLU)
|
||||
struct ggml_tensor * ffn_norm_w = nullptr;
|
||||
struct ggml_tensor * ffn_gate_w = nullptr;
|
||||
|
|
@ -64,11 +64,11 @@ struct decoder_block {
|
|||
// Snake activation before conv transpose
|
||||
struct ggml_tensor * snake_alpha = nullptr;
|
||||
struct ggml_tensor * snake_beta = nullptr;
|
||||
|
||||
|
||||
// Transposed convolution for upsampling
|
||||
struct ggml_tensor * conv_t_w = nullptr;
|
||||
struct ggml_tensor * conv_t_b = nullptr;
|
||||
|
||||
|
||||
// Residual blocks (3 per decoder block)
|
||||
residual_block res[3];
|
||||
};
|
||||
|
|
@ -91,23 +91,23 @@ struct upsample_block {
|
|||
// Audio tokenizer decoder model weights
|
||||
struct audio_decoder_model {
|
||||
audio_decoder_config config;
|
||||
|
||||
|
||||
// VQ codebooks
|
||||
// vq_first: 1 codebook for first code
|
||||
struct ggml_tensor * vq_first_input_proj = nullptr; // [1, 512, 256]
|
||||
struct ggml_tensor * vq_first_output_proj = nullptr; // [1, 256, 512]
|
||||
struct ggml_tensor * vq_first_codebook = nullptr; // [256, 2048] embedding_sum
|
||||
struct ggml_tensor * vq_first_usage = nullptr; // [2048] cluster_usage
|
||||
|
||||
|
||||
// vq_rest: 15 codebooks for remaining codes
|
||||
struct ggml_tensor * vq_rest_input_proj = nullptr; // [1, 512, 256]
|
||||
struct ggml_tensor * vq_rest_output_proj = nullptr; // [1, 256, 512]
|
||||
struct ggml_tensor * vq_rest_codebook[15] = {nullptr}; // [256, 2048] embedding_sum each
|
||||
struct ggml_tensor * vq_rest_usage[15] = {nullptr}; // [2048] cluster_usage each
|
||||
|
||||
|
||||
// Upsample blocks (2 ConvNeXt-style blocks)
|
||||
upsample_block upsample[2];
|
||||
|
||||
|
||||
// Pre-transformer
|
||||
struct ggml_tensor * pre_tfm_input_proj_w = nullptr; // [1024, 512]
|
||||
struct ggml_tensor * pre_tfm_input_proj_b = nullptr;
|
||||
|
|
@ -115,33 +115,33 @@ struct audio_decoder_model {
|
|||
struct ggml_tensor * pre_tfm_norm_w = nullptr; // Final RMSNorm
|
||||
struct ggml_tensor * pre_tfm_output_proj_w = nullptr; // [512, 1024]
|
||||
struct ggml_tensor * pre_tfm_output_proj_b = nullptr;
|
||||
|
||||
|
||||
// Pre-conv: [3, 512, 1024]
|
||||
struct ggml_tensor * pre_conv_w = nullptr;
|
||||
struct ggml_tensor * pre_conv_b = nullptr;
|
||||
|
||||
|
||||
// Decoder blocks
|
||||
// Block 0: Initial conv [7, 1024, 1536]
|
||||
struct ggml_tensor * dec0_conv_w = nullptr;
|
||||
struct ggml_tensor * dec0_conv_b = nullptr;
|
||||
|
||||
|
||||
// Blocks 1-4: Snake + ConvTranspose + 3 residual blocks
|
||||
decoder_block dec_blocks[4];
|
||||
|
||||
|
||||
// Block 5: Final snake activation
|
||||
struct ggml_tensor * dec5_snake_alpha = nullptr;
|
||||
struct ggml_tensor * dec5_snake_beta = nullptr;
|
||||
|
||||
|
||||
// Block 6: Output conv [7, 96, 1]
|
||||
struct ggml_tensor * dec6_conv_w = nullptr;
|
||||
struct ggml_tensor * dec6_conv_b = nullptr;
|
||||
|
||||
|
||||
// GGML context for tensor metadata
|
||||
struct ggml_context * ctx = nullptr;
|
||||
|
||||
|
||||
// Backend buffer for weights
|
||||
ggml_backend_buffer_t buffer = nullptr;
|
||||
|
||||
|
||||
// Tensor name to tensor mapping
|
||||
std::map<std::string, struct ggml_tensor *> tensors;
|
||||
};
|
||||
|
|
@ -160,71 +160,74 @@ class AudioTokenizerDecoder {
|
|||
public:
|
||||
AudioTokenizerDecoder();
|
||||
~AudioTokenizerDecoder();
|
||||
|
||||
|
||||
// Load model from GGUF file (tokenizer model)
|
||||
bool load_model(const std::string & model_path);
|
||||
|
||||
// Release all model/runtime resources
|
||||
void unload_model();
|
||||
|
||||
|
||||
// Decode audio codes to waveform
|
||||
// codes: audio codes [n_frames, n_codebooks] as int32_t (row-major)
|
||||
// n_frames: number of frames
|
||||
// Returns: audio samples normalized to [-1, 1] at 24kHz
|
||||
bool decode(const int32_t * codes, int32_t n_frames,
|
||||
std::vector<float> & samples);
|
||||
|
||||
|
||||
bool decode_chunk(const int32_t * codes, int32_t n_frames, int32_t position_offset,
|
||||
std::vector<float> & samples);
|
||||
|
||||
const audio_decoder_config & get_config() const { return model_.config; }
|
||||
|
||||
|
||||
const std::string & get_error() const { return error_msg_; }
|
||||
|
||||
|
||||
private:
|
||||
// Build computation graph for decoding
|
||||
struct ggml_cgraph * build_graph(int32_t n_frames);
|
||||
|
||||
|
||||
// Apply Snake activation: x + (1/alpha) * sin^2(alpha * x)
|
||||
struct ggml_tensor * apply_snake(struct ggml_context * ctx,
|
||||
struct ggml_tensor * x,
|
||||
struct ggml_tensor * alpha,
|
||||
struct ggml_tensor * beta);
|
||||
|
||||
|
||||
// Apply RMSNorm
|
||||
struct ggml_tensor * apply_rms_norm(struct ggml_context * ctx,
|
||||
struct ggml_tensor * x,
|
||||
struct ggml_tensor * w,
|
||||
float eps);
|
||||
|
||||
|
||||
// Apply pre-transformer layer
|
||||
struct ggml_tensor * apply_pre_tfm_layer(struct ggml_context * ctx,
|
||||
struct ggml_tensor * x,
|
||||
const pre_tfm_layer & layer,
|
||||
int32_t n_frames,
|
||||
struct ggml_tensor * positions);
|
||||
|
||||
|
||||
// Apply upsample block (ConvNeXt-style)
|
||||
struct ggml_tensor * apply_upsample_block(struct ggml_context * ctx,
|
||||
struct ggml_tensor * x,
|
||||
const upsample_block & block,
|
||||
int block_idx);
|
||||
|
||||
|
||||
// Apply residual block
|
||||
struct ggml_tensor * apply_residual_block(struct ggml_context * ctx,
|
||||
struct ggml_tensor * x,
|
||||
const residual_block & block);
|
||||
|
||||
|
||||
// Apply decoder block (Snake + ConvTranspose + Residuals)
|
||||
struct ggml_tensor * apply_decoder_block(struct ggml_context * ctx,
|
||||
struct ggml_tensor * x,
|
||||
const decoder_block & block,
|
||||
int upsample_rate,
|
||||
int block_idx);
|
||||
|
||||
|
||||
void normalize_codebooks();
|
||||
|
||||
|
||||
audio_decoder_model model_;
|
||||
audio_decoder_state state_;
|
||||
std::string error_msg_;
|
||||
|
||||
|
||||
// Temporary storage for codes input
|
||||
std::vector<int32_t> codes_buf_;
|
||||
};
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue