From c8d6546a147ffa09a495dac0720e5bab989fe1bc Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Tue, 7 Apr 2026 20:49:46 +0800 Subject: [PATCH] experimental q3tts batch test --- .../qwen3tts/audio_tokenizer_decoder.cpp | 28 ++++++++- otherarch/qwen3tts/audio_tokenizer_decoder.h | 61 ++++++++++--------- 2 files changed, 59 insertions(+), 30 deletions(-) diff --git a/otherarch/qwen3tts/audio_tokenizer_decoder.cpp b/otherarch/qwen3tts/audio_tokenizer_decoder.cpp index ac1dd5986..c06925b6e 100644 --- a/otherarch/qwen3tts/audio_tokenizer_decoder.cpp +++ b/otherarch/qwen3tts/audio_tokenizer_decoder.cpp @@ -798,6 +798,32 @@ bool AudioTokenizerDecoder::decode(const int32_t * codes, int32_t n_frames, return false; } + if (n_frames <= 0) { + samples.clear(); + return true; + } + + const int32_t max_frames = 32; + + if (max_frames > 0 && n_frames > max_frames) { + samples.clear(); + for (int32_t offset = 0; offset < n_frames; offset += max_frames) { + const int32_t chunk = std::min(max_frames, n_frames - offset); + std::vector chunk_samples; + if (!decode_chunk(codes + (size_t)offset * model_.config.n_codebooks, chunk, offset, chunk_samples)) { + return false; + } + samples.insert(samples.end(), chunk_samples.begin(), chunk_samples.end()); + } + return true; + } + + return decode_chunk(codes, n_frames, 0, samples); +} + +bool AudioTokenizerDecoder::decode_chunk(const int32_t * codes, int32_t n_frames, + int32_t position_offset, + std::vector & samples) { const auto & cfg = model_.config; codes_buf_.resize(n_frames * cfg.n_codebooks); @@ -838,7 +864,7 @@ bool AudioTokenizerDecoder::decode(const int32_t * codes, int32_t n_frames, if (positions_tensor) { std::vector positions(n_frames); for (int i = 0; i < n_frames; ++i) { - positions[i] = i; + positions[i] = position_offset + i; } ggml_backend_tensor_set(positions_tensor, positions.data(), 0, n_frames * sizeof(int32_t)); diff --git a/otherarch/qwen3tts/audio_tokenizer_decoder.h b/otherarch/qwen3tts/audio_tokenizer_decoder.h index 03d734416..69030fc3f 100644 --- a/otherarch/qwen3tts/audio_tokenizer_decoder.h +++ b/otherarch/qwen3tts/audio_tokenizer_decoder.h @@ -37,7 +37,7 @@ struct pre_tfm_layer { struct ggml_tensor * attn_v_w = nullptr; struct ggml_tensor * attn_output_w = nullptr; struct ggml_tensor * attn_scale = nullptr; // layer_scale for attention - + // FFN (SwiGLU) struct ggml_tensor * ffn_norm_w = nullptr; struct ggml_tensor * ffn_gate_w = nullptr; @@ -64,11 +64,11 @@ struct decoder_block { // Snake activation before conv transpose struct ggml_tensor * snake_alpha = nullptr; struct ggml_tensor * snake_beta = nullptr; - + // Transposed convolution for upsampling struct ggml_tensor * conv_t_w = nullptr; struct ggml_tensor * conv_t_b = nullptr; - + // Residual blocks (3 per decoder block) residual_block res[3]; }; @@ -91,23 +91,23 @@ struct upsample_block { // Audio tokenizer decoder model weights struct audio_decoder_model { audio_decoder_config config; - + // VQ codebooks // vq_first: 1 codebook for first code struct ggml_tensor * vq_first_input_proj = nullptr; // [1, 512, 256] struct ggml_tensor * vq_first_output_proj = nullptr; // [1, 256, 512] struct ggml_tensor * vq_first_codebook = nullptr; // [256, 2048] embedding_sum struct ggml_tensor * vq_first_usage = nullptr; // [2048] cluster_usage - + // vq_rest: 15 codebooks for remaining codes struct ggml_tensor * vq_rest_input_proj = nullptr; // [1, 512, 256] struct ggml_tensor * vq_rest_output_proj = nullptr; // [1, 256, 512] struct ggml_tensor * vq_rest_codebook[15] = {nullptr}; // [256, 2048] embedding_sum each struct ggml_tensor * vq_rest_usage[15] = {nullptr}; // [2048] cluster_usage each - + // Upsample blocks (2 ConvNeXt-style blocks) upsample_block upsample[2]; - + // Pre-transformer struct ggml_tensor * pre_tfm_input_proj_w = nullptr; // [1024, 512] struct ggml_tensor * pre_tfm_input_proj_b = nullptr; @@ -115,33 +115,33 @@ struct audio_decoder_model { struct ggml_tensor * pre_tfm_norm_w = nullptr; // Final RMSNorm struct ggml_tensor * pre_tfm_output_proj_w = nullptr; // [512, 1024] struct ggml_tensor * pre_tfm_output_proj_b = nullptr; - + // Pre-conv: [3, 512, 1024] struct ggml_tensor * pre_conv_w = nullptr; struct ggml_tensor * pre_conv_b = nullptr; - + // Decoder blocks // Block 0: Initial conv [7, 1024, 1536] struct ggml_tensor * dec0_conv_w = nullptr; struct ggml_tensor * dec0_conv_b = nullptr; - + // Blocks 1-4: Snake + ConvTranspose + 3 residual blocks decoder_block dec_blocks[4]; - + // Block 5: Final snake activation struct ggml_tensor * dec5_snake_alpha = nullptr; struct ggml_tensor * dec5_snake_beta = nullptr; - + // Block 6: Output conv [7, 96, 1] struct ggml_tensor * dec6_conv_w = nullptr; struct ggml_tensor * dec6_conv_b = nullptr; - + // GGML context for tensor metadata struct ggml_context * ctx = nullptr; - + // Backend buffer for weights ggml_backend_buffer_t buffer = nullptr; - + // Tensor name to tensor mapping std::map tensors; }; @@ -160,71 +160,74 @@ class AudioTokenizerDecoder { public: AudioTokenizerDecoder(); ~AudioTokenizerDecoder(); - + // Load model from GGUF file (tokenizer model) bool load_model(const std::string & model_path); // Release all model/runtime resources void unload_model(); - + // Decode audio codes to waveform // codes: audio codes [n_frames, n_codebooks] as int32_t (row-major) // n_frames: number of frames // Returns: audio samples normalized to [-1, 1] at 24kHz bool decode(const int32_t * codes, int32_t n_frames, std::vector & samples); - + + bool decode_chunk(const int32_t * codes, int32_t n_frames, int32_t position_offset, + std::vector & samples); + const audio_decoder_config & get_config() const { return model_.config; } - + const std::string & get_error() const { return error_msg_; } - + private: // Build computation graph for decoding struct ggml_cgraph * build_graph(int32_t n_frames); - + // Apply Snake activation: x + (1/alpha) * sin^2(alpha * x) struct ggml_tensor * apply_snake(struct ggml_context * ctx, struct ggml_tensor * x, struct ggml_tensor * alpha, struct ggml_tensor * beta); - + // Apply RMSNorm struct ggml_tensor * apply_rms_norm(struct ggml_context * ctx, struct ggml_tensor * x, struct ggml_tensor * w, float eps); - + // Apply pre-transformer layer struct ggml_tensor * apply_pre_tfm_layer(struct ggml_context * ctx, struct ggml_tensor * x, const pre_tfm_layer & layer, int32_t n_frames, struct ggml_tensor * positions); - + // Apply upsample block (ConvNeXt-style) struct ggml_tensor * apply_upsample_block(struct ggml_context * ctx, struct ggml_tensor * x, const upsample_block & block, int block_idx); - + // Apply residual block struct ggml_tensor * apply_residual_block(struct ggml_context * ctx, struct ggml_tensor * x, const residual_block & block); - + // Apply decoder block (Snake + ConvTranspose + Residuals) struct ggml_tensor * apply_decoder_block(struct ggml_context * ctx, struct ggml_tensor * x, const decoder_block & block, int upsample_rate, int block_idx); - + void normalize_codebooks(); - + audio_decoder_model model_; audio_decoder_state state_; std::string error_msg_; - + // Temporary storage for codes input std::vector codes_buf_; };