support audio to video for ltx 2.3

expose.h

@@ -223,6 +223,7 @@ struct sd_generation_inputs
     const char * negative_prompt = nullptr;
     const char * init_images = "";
     const char * mask = "";
+    const char * audio_data = "";
     const int extra_images_len = 0;
     const char ** extra_images = nullptr;
     const bool reverse_refimg = false;

koboldcpp.py

@@ -412,6 +412,7 @@ class sd_generation_inputs(ctypes.Structure):
                 ("negative_prompt", ctypes.c_char_p),
                 ("init_images", ctypes.c_char_p),
                 ("mask", ctypes.c_char_p),
+                ("audio_data", ctypes.c_char_p),
                 ("extra_images_len", ctypes.c_int),
                 ("extra_images", ctypes.POINTER(ctypes.c_char_p)),
                 ("reverse_refimg", ctypes.c_bool),
@@ -1111,7 +1112,7 @@ def is_incomplete_utf8_sequence(byte_seq): #note, this will only flag INCOMPLETE
 def strip_base64_prefix(encoded_data):
     if not encoded_data:
         return ""
-    if encoded_data.startswith("data:image"):
+    if encoded_data.startswith("data:image") or encoded_data.startswith("data:audio"):
         encoded_data = encoded_data.split(',', 1)[-1]
     return encoded_data
 
@@ -2814,6 +2815,11 @@ def sd_generate(genparams):
     extra_images_arr = genparams.get("extra_images", [])
     extra_images_arr = ([] if not extra_images_arr else extra_images_arr)
     extra_images_arr = [img for img in extra_images_arr if img not in (None, "")]
+
+    audio_data = next((img for img in extra_images_arr if img.startswith("data:audio")), None)
+    extra_images_arr = [img for img in extra_images_arr if not img.startswith("data:audio")]
+    audio_data = strip_base64_prefix(audio_data)
+
     extra_images_arr = extra_images_arr[:extra_images_max]
     lora_filenames, lora_multipliers = prepare_lora_multipliers(genparams.get("lora", []))
 
@@ -2840,6 +2846,7 @@ def sd_generate(genparams):
     inputs.negative_prompt = negative_prompt.encode("UTF-8")
     inputs.init_images = init_images.encode("UTF-8")
     inputs.mask = "".encode("UTF-8") if not mask else mask.encode("UTF-8")
+    inputs.audio_data = "".encode("UTF-8") if not audio_data else audio_data.encode("UTF-8")
     inputs.extra_images_len = len(extra_images_arr)
     inputs.extra_images = (ctypes.c_char_p * inputs.extra_images_len)()
     for n, estr in enumerate(extra_images_arr):

otherarch/sdcpp/include/stable-diffusion.h

@@ -395,6 +395,7 @@ typedef struct {
     int64_t seed;
     int video_frames;
     int fps;
+    const sd_audio_t* input_audio;
     float vace_strength;
     sd_tiling_params_t vae_tiling_params;
     sd_cache_params_t cache;

otherarch/sdcpp/sdtype_adapter.cpp

@@ -939,6 +939,37 @@ static std::string raw_image_to_png_base64(const sd_image_t& img, std::string pa
     return result;
 }
 
+static sd_audio_t load_audio_from_b64(const std::string& b64audio) {
+    sd_audio_t audio = {0, 0, 0, nullptr};
+    if (b64audio.empty()) {
+        return audio;
+    }
+
+    std::vector<uint8_t> audio_data = kcpp_base64_decode(b64audio);
+    std::vector<float> decoded_samples;
+    int sample_rate = 0;
+    int channels = 0;
+    if (!kcpp_decode_audio_file_from_buf(audio_data.data(), audio_data.size(), sample_rate, channels, decoded_samples)) {
+        printf("KCPP SD: failed to decode input audio\n");
+        return audio;
+    }
+
+    uint64_t sample_count = static_cast<uint64_t>(decoded_samples.size() / static_cast<size_t>(channels));
+    size_t float_count = decoded_samples.size();
+    float* samples = (float*)malloc(float_count * sizeof(float));
+    if (samples == nullptr || sample_count == 0) {
+        free(samples);
+        return audio;
+    }
+
+    std::memcpy(samples, decoded_samples.data(), float_count * sizeof(float));
+    audio.sample_rate = static_cast<uint32_t>(sample_rate);
+    audio.channels = static_cast<uint32_t>(channels);
+    audio.sample_count = sample_count;
+    audio.data = samples;
+    return audio;
+}
+
 bool supports_reference_images(kcpp_sd::model_info info)
 {
     bool supported = (info.is_wan || info.is_ltx || info.is_qwenimg || info.is_flux2 || info.is_kontext || photomaker_enabled);
@@ -955,6 +986,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
 
     std::string img2img_data = std::string(inputs.init_images);
     std::string img2img_mask = std::string(inputs.mask);
+    std::string input_audio_data = std::string(inputs.audio_data ? inputs.audio_data : "");
     std::vector<std::string> extra_image_data;
     for(int i=0;i<inputs.extra_images_len;++i)
     {
@@ -1283,6 +1315,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
      //special case, is img2img and denoise strength is 0 and steps is 1, do a passthru
     bool is_passthrough = (sd_params->sample_steps<=1 && sd_params->strength<=0 && is_img2img && vid_req_frames<=1 && extra_image_data.size()==0);
     sd_audio_t* generated_audio = nullptr;
+    sd_audio_t input_audio = {0, 0, 0, nullptr};
 
     if(is_vid_model)
     {
@@ -1302,6 +1335,13 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
         vid_gen_params.video_frames = vid_req_frames;
         vid_gen_params.fps = vid_fps;
         vid_gen_params.vae_tiling_params = params.vae_tiling_params;
+        if (!input_audio_data.empty()) {
+            input_audio = load_audio_from_b64(input_audio_data);
+            if (input_audio.data == nullptr) {
+                return sd_generation.error("KCPP SD: load audio from base64 failed!");
+            }
+            vid_gen_params.input_audio = &input_audio;
+        }
         if (wan_imgs.size() > 0) {
             if (wan_imgs.size() >= 2) {
                 vid_gen_params.init_image = wan_imgs[0];
@@ -1327,6 +1367,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
             << "\nFRAMES:"   << vid_gen_params.video_frames
             << "\nCTRL_FRM:" << vid_gen_params.control_frames_size
             << "\nINIT_IMGS:" << wan_imgs.size()
+            << "\nINPUT_AUDIO:" << (vid_gen_params.input_audio ? "true" : "false")
             << "\n\n";
             printf("%s", ss.str().c_str());
         }
@@ -1429,6 +1470,10 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
     }
 
     if (!is_passthrough && results == NULL) {
+        if (input_audio.data) {
+            free(input_audio.data);
+            input_audio.data = nullptr;
+        }
         return sd_generation.error("KCPP SD generate failed!");
     }
 
@@ -1576,6 +1621,10 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
         free_sd_audio(generated_audio);
         generated_audio = nullptr;
     }
+    if (input_audio.data) {
+        free(input_audio.data);
+        input_audio.data = nullptr;
+    }
 
     free(results);
 

otherarch/sdcpp/src/model/vae/ltx_audio_vae.hpp

@@ -1,6 +1,7 @@
 #ifndef __SD_MODEL_VAE_LTX_AUDIO_VAE_HPP__
 #define __SD_MODEL_VAE_LTX_AUDIO_VAE_HPP__
 
+#include <algorithm>
 #include <cmath>
 #include <limits>
 #include <numeric>
@@ -21,6 +22,10 @@ namespace LTXV {
         int latent_channels                                        = 8;
         int latent_frequency_bins                                  = 16;
         int audio_channels                                         = 2;
+        bool has_encoder                                           = false;
+        int encoder_channels                                       = 128;
+        std::vector<int> encoder_channel_multipliers               = {1, 2, 4};
+        int encoder_num_res_blocks                                 = 2;
         int decoder_channels                                       = 128;
         std::vector<int> decoder_channel_multipliers               = {1, 2, 4};
         int decoder_num_res_blocks                                 = 2;
@@ -74,6 +79,7 @@ namespace LTXV {
 
             const TensorStorage* decoder_conv_in   = require("audio_vae.decoder.conv_in.conv.weight");
             const TensorStorage* decoder_conv_out  = require("audio_vae.decoder.conv_out.conv.weight");
+            const TensorStorage* encoder_conv_in   = require("audio_vae.encoder.conv_in.conv.weight");
             const TensorStorage* latent_std        = require("audio_vae.per_channel_statistics.std-of-means");
             const TensorStorage* vocoder_conv_pre  = require("vocoder.vocoder.conv_pre.weight");
             const TensorStorage* vocoder_conv_post = require("vocoder.vocoder.conv_post.weight");
@@ -98,6 +104,40 @@ namespace LTXV {
                 return config;
             }
 
+            if (encoder_conv_in != nullptr) {
+                config.has_encoder      = true;
+                config.audio_channels   = static_cast<int>(encoder_conv_in->ne[2]);
+                config.encoder_channels = static_cast<int>(encoder_conv_in->ne[3]);
+
+                std::vector<std::pair<int, int>> encoder_level_channels;
+                for (const auto& pair : tensor_storage_map) {
+                    const std::string& name  = pair.first;
+                    const std::string prefix = "audio_vae.encoder.down.";
+                    const std::string suffix = ".block.0.conv1.conv.weight";
+                    if (!starts_with(name, prefix) || !ends_with(name, suffix)) {
+                        continue;
+                    }
+                    std::string level_str = name.substr(prefix.size(), name.size() - prefix.size() - suffix.size());
+                    int level             = std::stoi(level_str);
+                    encoder_level_channels.push_back({level, static_cast<int>(pair.second.ne[3])});
+                }
+                std::sort(encoder_level_channels.begin(), encoder_level_channels.end());
+                if (!encoder_level_channels.empty()) {
+                    config.encoder_channel_multipliers.clear();
+                    for (const auto& level_channel : encoder_level_channels) {
+                        config.encoder_channel_multipliers.push_back(level_channel.second / std::max(1, config.encoder_channels));
+                    }
+                }
+
+                int encoder_block_count = 0;
+                while (tensor_storage_map.find("audio_vae.encoder.down.0.block." + std::to_string(encoder_block_count) + ".conv1.conv.weight") != tensor_storage_map.end()) {
+                    ++encoder_block_count;
+                }
+                if (encoder_block_count > 0) {
+                    config.encoder_num_res_blocks = encoder_block_count;
+                }
+            }
+
             std::vector<std::pair<int, int>> level_channels;
             for (const auto& pair : tensor_storage_map) {
                 const std::string& name  = pair.first;
@@ -171,16 +211,89 @@ namespace LTXV {
             if (config.audio_channels != 2 || config.latent_channels != 8 || config.mel_bins != 64) {
                 return config;
             }
-            LOG_DEBUG("ltx_audio_vae: sample_rate = %d, mel_bins = %d, latent_channels = %d, latent_frequency_bins = %d, has_bwe = %s",
+            LOG_DEBUG("ltx_audio_vae: sample_rate = %d, mel_bins = %d, latent_channels = %d, latent_frequency_bins = %d, has_encoder = %s, has_bwe = %s",
                       config.sample_rate,
                       config.mel_bins,
                       config.latent_channels,
                       config.latent_frequency_bins,
+                      config.has_encoder ? "true" : "false",
                       config.has_bwe ? "true" : "false");
             return config;
         }
     };
 
+    static double ltx_audio_hz_to_mel(double freq) {
+        constexpr double min_log_hz  = 1000.0;
+        constexpr double min_log_mel = 15.0;
+        constexpr double logstep     = 0.06875177742094912;  // log(6.4) / 27
+        constexpr double f_sp        = 200.0 / 3.0;
+        if (freq < min_log_hz) {
+            return freq / f_sp;
+        }
+        return min_log_mel + std::log(freq / min_log_hz) / logstep;
+    }
+
+    static double ltx_audio_mel_to_hz(double mel) {
+        constexpr double min_log_hz  = 1000.0;
+        constexpr double min_log_mel = 15.0;
+        constexpr double logstep     = 0.06875177742094912;  // log(6.4) / 27
+        constexpr double f_sp        = 200.0 / 3.0;
+        if (mel < min_log_mel) {
+            return mel * f_sp;
+        }
+        return min_log_hz * std::exp(logstep * (mel - min_log_mel));
+    }
+
+    static sd::Tensor<float> build_encoder_stft_basis(int n_fft) {
+        constexpr double kPi = 3.14159265358979323846;
+        const int n_freqs    = n_fft / 2 + 1;
+        sd::Tensor<float> basis({n_fft, 1, n_freqs * 2});
+        for (int k = 0; k < n_freqs; ++k) {
+            for (int n = 0; n < n_fft; ++n) {
+                double window          = 0.5 - 0.5 * std::cos(2.0 * kPi * n / static_cast<double>(n_fft));
+                double phase           = 2.0 * kPi * k * n / static_cast<double>(n_fft);
+                basis.index(n, 0, k)   = static_cast<float>(std::cos(phase) * window);
+                basis.index(n, 0, k + n_freqs) = static_cast<float>(-std::sin(phase) * window);
+            }
+        }
+        return basis;
+    }
+
+    static sd::Tensor<float> build_encoder_mel_basis(int sample_rate, int n_fft, int n_mels) {
+        const int n_freqs = n_fft / 2 + 1;
+        sd::Tensor<float> basis({n_freqs, n_mels});
+        std::vector<double> fft_freqs(n_freqs);
+        for (int i = 0; i < n_freqs; ++i) {
+            fft_freqs[i] = (static_cast<double>(sample_rate) * 0.5) * static_cast<double>(i) / static_cast<double>(n_freqs - 1);
+        }
+
+        std::vector<double> mel_f(n_mels + 2);
+        const double min_mel = ltx_audio_hz_to_mel(0.0);
+        const double max_mel = ltx_audio_hz_to_mel(static_cast<double>(sample_rate) * 0.5);
+        for (int i = 0; i < n_mels + 2; ++i) {
+            double mel = min_mel + (max_mel - min_mel) * static_cast<double>(i) / static_cast<double>(n_mels + 1);
+            mel_f[i]   = ltx_audio_mel_to_hz(mel);
+        }
+
+        for (int m = 0; m < n_mels; ++m) {
+            double lower = mel_f[m];
+            double center = mel_f[m + 1];
+            double upper = mel_f[m + 2];
+            double enorm = 2.0 / std::max(upper - lower, 1e-12);
+            for (int f = 0; f < n_freqs; ++f) {
+                double freq = fft_freqs[f];
+                double value = 0.0;
+                if (freq > lower && freq <= center) {
+                    value = (freq - lower) / std::max(center - lower, 1e-12);
+                } else if (freq > center && freq < upper) {
+                    value = (upper - freq) / std::max(upper - center, 1e-12);
+                }
+                basis.index(f, m) = static_cast<float>(value * enorm);
+            }
+        }
+        return basis;
+    }
+
     static ggml_tensor* compute_log_mel_spectrogram(GGMLRunnerContext* runner_ctx,
                                                     ggml_tensor* waveform,
                                                     ggml_tensor* forward_basis,
@@ -478,6 +591,31 @@ namespace LTXV {
         }
     };
 
+    struct AudioDownsample2D : public GGMLBlock {
+        AudioDownsample2D(int64_t channels) {
+            blocks["conv"] = std::make_shared<Conv2d>(channels,
+                                                      channels,
+                                                      std::pair<int, int>{3, 3},
+                                                      std::pair<int, int>{2, 2},
+                                                      std::pair<int, int>{0, 0});
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) {
+            auto conv = std::dynamic_pointer_cast<Conv2d>(blocks["conv"]);
+            x         = ggml_ext_pad_ext(ctx->ggml_ctx,
+                                         x,
+                                         0,
+                                         1,
+                                         2,
+                                         0,
+                                         0,
+                                         0,
+                                         0,
+                                         0);
+            return conv->forward(ctx, x);
+        }
+    };
+
     struct AudioResnetBlock2D : public GGMLBlock {
         int64_t in_channels;
         int64_t out_channels;
@@ -514,6 +652,86 @@ namespace LTXV {
         }
     };
 
+    struct AudioEncoder : public GGMLBlock {
+        LTXAudioVAEConfig config;
+
+        explicit AudioEncoder(const LTXAudioVAEConfig& config)
+            : config(config) {
+            int block_in      = config.encoder_channels;
+            blocks["conv_in"] = std::make_shared<HeightCausalConv2D>(config.audio_channels, block_in, std::pair<int, int>{3, 3});
+
+            for (int level = 0; level < static_cast<int>(config.encoder_channel_multipliers.size()); ++level) {
+                int block_out = config.encoder_channels * config.encoder_channel_multipliers[level];
+                for (int block_idx = 0; block_idx < config.encoder_num_res_blocks; ++block_idx) {
+                    blocks["down." + std::to_string(level) + ".block." + std::to_string(block_idx)] =
+                        std::make_shared<AudioResnetBlock2D>(block_in, block_out);
+                    block_in = block_out;
+                }
+                if (level != static_cast<int>(config.encoder_channel_multipliers.size()) - 1) {
+                    blocks["down." + std::to_string(level) + ".downsample"] = std::make_shared<AudioDownsample2D>(block_in);
+                }
+            }
+
+            blocks["mid.block_1"] = std::make_shared<AudioResnetBlock2D>(block_in, block_in);
+            blocks["mid.block_2"] = std::make_shared<AudioResnetBlock2D>(block_in, block_in);
+            blocks["norm_out"]    = std::make_shared<PixelNorm2D>();
+            blocks["conv_out"]    = std::make_shared<HeightCausalConv2D>(block_in, config.latent_channels * 2, std::pair<int, int>{3, 3});
+        }
+
+        ggml_tensor* normalize_latent(GGMLRunnerContext* ctx,
+                                      ggml_tensor* latent,
+                                      ggml_tensor* mean,
+                                      ggml_tensor* stddev) {
+            latent = ggml_ext_slice(ctx->ggml_ctx, latent, 2, 0, config.latent_channels);
+            latent = ggml_permute(ctx->ggml_ctx, latent, 0, 2, 1, 3);
+            latent = ggml_cont(ctx->ggml_ctx, latent);
+            latent = ggml_reshape_4d(ctx->ggml_ctx, latent, config.latent_frequency_bins * config.latent_channels, latent->ne[2], 1, latent->ne[3]);
+
+            mean   = ggml_reshape_4d(ctx->ggml_ctx, mean, mean->ne[0], 1, 1, 1);
+            stddev = ggml_reshape_4d(ctx->ggml_ctx, stddev, stddev->ne[0], 1, 1, 1);
+            latent = ggml_div(ctx->ggml_ctx, ggml_sub(ctx->ggml_ctx, latent, mean), stddev);
+
+            latent = ggml_reshape_4d(ctx->ggml_ctx,
+                                     latent,
+                                     config.latent_frequency_bins,
+                                     config.latent_channels,
+                                     latent->ne[1],
+                                     latent->ne[3]);
+            latent = ggml_cont(ctx->ggml_ctx, ggml_permute(ctx->ggml_ctx, latent, 0, 2, 1, 3));
+            return latent;
+        }
+
+        ggml_tensor* forward(GGMLRunnerContext* ctx,
+                             ggml_tensor* spectrogram,
+                             ggml_tensor* mean,
+                             ggml_tensor* stddev) {
+            auto conv_in     = std::dynamic_pointer_cast<HeightCausalConv2D>(blocks["conv_in"]);
+            auto mid_block_1 = std::dynamic_pointer_cast<AudioResnetBlock2D>(blocks["mid.block_1"]);
+            auto mid_block_2 = std::dynamic_pointer_cast<AudioResnetBlock2D>(blocks["mid.block_2"]);
+            auto norm_out    = std::dynamic_pointer_cast<PixelNorm2D>(blocks["norm_out"]);
+            auto conv_out    = std::dynamic_pointer_cast<HeightCausalConv2D>(blocks["conv_out"]);
+
+            auto x = conv_in->forward(ctx, spectrogram);
+            for (int level = 0; level < static_cast<int>(config.encoder_channel_multipliers.size()); ++level) {
+                for (int block_idx = 0; block_idx < config.encoder_num_res_blocks; ++block_idx) {
+                    auto block = std::dynamic_pointer_cast<AudioResnetBlock2D>(blocks["down." + std::to_string(level) + ".block." + std::to_string(block_idx)]);
+                    x          = block->forward(ctx, x);
+                }
+                if (level != static_cast<int>(config.encoder_channel_multipliers.size()) - 1) {
+                    auto downsample = std::dynamic_pointer_cast<AudioDownsample2D>(blocks["down." + std::to_string(level) + ".downsample"]);
+                    x               = downsample->forward(ctx, x);
+                }
+            }
+
+            x = mid_block_1->forward(ctx, x);
+            x = mid_block_2->forward(ctx, x);
+            x = norm_out->forward(ctx, x);
+            x = ggml_silu_inplace(ctx->ggml_ctx, x);
+            x = conv_out->forward(ctx, x);
+            return normalize_latent(ctx, x, mean, stddev);
+        }
+    };
+
     struct Conv1D : public UnaryBlock {
         int64_t in_channels;
         int64_t out_channels;
@@ -914,6 +1132,9 @@ namespace LTXV {
 
         explicit LTXAudioVAE(const LTXAudioVAEConfig& config)
             : config(config) {
+            if (config.has_encoder) {
+                blocks["audio_vae.encoder"] = std::make_shared<AudioEncoder>(config);
+            }
             blocks["audio_vae.decoder"] = std::make_shared<AudioDecoder>(config);
             blocks["vocoder.vocoder"]   = std::make_shared<Vocoder>(config);
             if (config.has_bwe) {
@@ -993,6 +1214,18 @@ namespace LTXV {
 
             return waveform;
         }
+
+        ggml_tensor* encode(GGMLRunnerContext* ctx,
+                            ggml_tensor* waveform,
+                            ggml_tensor* stft_basis,
+                            ggml_tensor* mel_basis) {
+            GGML_ASSERT(config.has_encoder);
+            auto encoder = std::dynamic_pointer_cast<AudioEncoder>(blocks["audio_vae.encoder"]);
+            auto mean    = params["audio_vae.per_channel_statistics.mean-of-means"];
+            auto stddev  = params["audio_vae.per_channel_statistics.std-of-means"];
+            auto mel     = compute_log_mel_spectrogram(ctx, waveform, stft_basis, mel_basis, config.mel_hop_length);
+            return encoder->forward(ctx, mel, mean, stddev);
+        }
     };
 
     struct LTXAudioVAERunner : public GGMLRunner {
@@ -1000,6 +1233,8 @@ namespace LTXV {
         LTXAudioVAE model;
         std::string weight_prefix;
         sd::Tensor<float> bwe_skip_filter_tensor;
+        sd::Tensor<float> encoder_stft_basis_tensor;
+        sd::Tensor<float> encoder_mel_basis_tensor;
 
         LTXAudioVAERunner(ggml_backend_t backend,
                           const String2TensorStorage& tensor_storage_map,
@@ -1014,6 +1249,10 @@ namespace LTXV {
                 const int bwe_ratio    = config.bwe_output_sample_rate / config.bwe_input_sample_rate;
                 bwe_skip_filter_tensor = sd::Tensor<float>::from_vector(build_hann_resample_filter(bwe_ratio));
             }
+            if (config.has_encoder) {
+                encoder_stft_basis_tensor = build_encoder_stft_basis(config.n_fft);
+                encoder_mel_basis_tensor  = build_encoder_mel_basis(config.sample_rate, config.n_fft, config.mel_bins);
+            }
         }
 
         void get_param_tensors(std::map<std::string, ggml_tensor*>& tensors) {
@@ -1046,6 +1285,29 @@ namespace LTXV {
             return result;
         }
 
+        sd::Tensor<float> encode(int n_threads,
+                                 const sd::Tensor<float>& waveform_tensor) {
+            if (!config.has_encoder || waveform_tensor.empty() ||
+                encoder_stft_basis_tensor.empty() || encoder_mel_basis_tensor.empty()) {
+                return {};
+            }
+            int64_t t0     = ggml_time_ms();
+            auto get_graph = [&]() -> ggml_cgraph* {
+                auto waveform   = make_input(waveform_tensor);
+                auto stft_basis = make_input(encoder_stft_basis_tensor);
+                auto mel_basis  = make_input(encoder_mel_basis_tensor);
+                ggml_cgraph* gf = new_graph_custom(655360);
+                auto runner_ctx = GGMLRunner::get_context();
+                auto latent     = model.encode(&runner_ctx, waveform, stft_basis, mel_basis);
+                ggml_build_forward_expand(gf, latent);
+                return gf;
+            };
+            auto result = restore_trailing_singleton_dims(GGMLRunner::compute<float>(get_graph, n_threads, false, false, false), 4);
+            int64_t t1  = ggml_time_ms();
+            LOG_INFO("ltx audio vae encode completed, taking %.2fs", (t1 - t0) * 1.0f / 1000);
+            return result;
+        }
+
         void test(const std::string& input_path) {
             auto z = sd::load_tensor_from_file_as_tensor<float>(input_path);
             GGML_ASSERT(!z.empty());

otherarch/sdcpp/src/stable-diffusion.cpp

@@ -2,6 +2,7 @@
 #include <cmath>
 #include <cstdlib>
 #include <filesystem>
+#include <limits>
 #include <set>
 #include <unordered_set>
 #include <vector>
@@ -1342,9 +1343,6 @@ public:
         ignore_tensors.insert("model.diffusion_model.__32x32__");
         ignore_tensors.insert("model.diffusion_model.__index_timestep_zero__");
 
-        if (audio_vae_model) {
-            ignore_tensors.insert("audio_vae.encoder");
-        }
         if (version == VERSION_OVIS_IMAGE) {
             ignore_tensors.insert("text_encoders.llm.vision_model.");
             ignore_tensors.insert("text_encoders.llm.visual_tokenizer.");
@@ -3205,6 +3203,7 @@ void sd_vid_gen_params_init(sd_vid_gen_params_t* sd_vid_gen_params) {
     sd_vid_gen_params->seed                                  = -1;
     sd_vid_gen_params->video_frames                          = 6;
     sd_vid_gen_params->fps                                   = 16;
+    sd_vid_gen_params->input_audio                           = nullptr;
     sd_vid_gen_params->moe_boundary                          = 0.875f;
     sd_vid_gen_params->vace_strength                         = 1.f;
     sd_vid_gen_params->vae_tiling_params                     = {false, false, 0, 0, 0.5f, 0.0f, 0.0f, nullptr};
@@ -3304,6 +3303,77 @@ static sd_audio_t* waveform_to_sd_audio(const StableDiffusionGGML* sd,
     return audio;
 }
 
+static sd_audio_t* clone_sd_audio(const sd_audio_t* src) {
+    if (src == nullptr || src->data == nullptr || src->sample_rate == 0 || src->channels == 0 || src->sample_count == 0) {
+        return nullptr;
+    }
+
+    sd_audio_t* audio = (sd_audio_t*)malloc(sizeof(sd_audio_t));
+    if (audio == nullptr) {
+        return nullptr;
+    }
+
+    audio->sample_rate  = src->sample_rate;
+    audio->channels     = src->channels;
+    audio->sample_count = src->sample_count;
+    size_t sample_bytes = static_cast<size_t>(src->sample_count) * static_cast<size_t>(src->channels) * sizeof(float);
+    audio->data         = (float*)malloc(sample_bytes);
+    if (audio->data == nullptr) {
+        free(audio);
+        return nullptr;
+    }
+
+    std::memcpy(audio->data, src->data, sample_bytes);
+    return audio;
+}
+
+static sd::Tensor<float> sd_audio_to_ltx_waveform_tensor(const sd_audio_t* audio,
+                                                         int target_sample_rate,
+                                                         int target_channels) {
+    if (audio == nullptr || audio->data == nullptr || audio->sample_rate == 0 ||
+        audio->channels == 0 || audio->sample_count == 0 || target_sample_rate <= 0 ||
+        target_channels <= 0) {
+        return {};
+    }
+
+    uint64_t out_samples_u64 = (audio->sample_count * static_cast<uint64_t>(target_sample_rate) +
+                                static_cast<uint64_t>(audio->sample_rate) - 1) /
+                               static_cast<uint64_t>(audio->sample_rate);
+    if (out_samples_u64 == 0 || out_samples_u64 > static_cast<uint64_t>(std::numeric_limits<int64_t>::max())) {
+        return {};
+    }
+
+    int64_t out_samples = static_cast<int64_t>(out_samples_u64);
+    sd::Tensor<float> waveform({out_samples, target_channels, 1, 1});
+    const double src_rate = static_cast<double>(audio->sample_rate);
+    const double dst_rate = static_cast<double>(target_sample_rate);
+    const int src_channels = static_cast<int>(audio->channels);
+
+    auto src_value = [&](uint64_t sample, int channel) -> float {
+        int src_channel = channel;
+        if (src_channels == 1) {
+            src_channel = 0;
+        } else if (channel >= src_channels) {
+            src_channel = src_channels - 1;
+        }
+        return audio->data[static_cast<size_t>(sample) * static_cast<size_t>(src_channels) + static_cast<size_t>(src_channel)];
+    };
+
+    for (int64_t t = 0; t < out_samples; ++t) {
+        double src_pos = static_cast<double>(t) * src_rate / dst_rate;
+        uint64_t i0    = static_cast<uint64_t>(std::floor(src_pos));
+        uint64_t i1    = std::min<uint64_t>(i0 + 1, audio->sample_count - 1);
+        float frac     = static_cast<float>(src_pos - static_cast<double>(i0));
+        for (int ch = 0; ch < target_channels; ++ch) {
+            float v0 = src_value(i0, ch);
+            float v1 = src_value(i1, ch);
+            waveform.index(t, ch, 0, 0) = v0 + (v1 - v0) * frac;
+        }
+    }
+
+    return waveform;
+}
+
 void free_sd_audio(sd_audio_t* audio) {
     if (audio == nullptr) {
         return;
@@ -3872,7 +3942,8 @@ static sd::Tensor<float> pack_ltxav_audio_and_video_latents(const sd::Tensor<flo
 
 static sd::Tensor<float> pack_ltxav_audio_and_video_denoise_mask(const sd::Tensor<float>& video_mask,
                                                                  const sd::Tensor<float>& video_latent,
-                                                                 const sd::Tensor<float>& audio_latent) {
+                                                                 const sd::Tensor<float>& audio_latent,
+                                                                 float audio_mask_value = 1.f) {
     if (video_mask.empty() || audio_latent.empty()) {
         return video_mask;
     }
@@ -3915,7 +3986,7 @@ static sd::Tensor<float> pack_ltxav_audio_and_video_denoise_mask(const sd::Tenso
 
     std::vector<int64_t> audio_mask_shape = video_latent.shape();
     audio_mask_shape[3]                   = extra_ch;
-    auto audio_mask                       = sd::Tensor<float>::ones(audio_mask_shape);
+    auto audio_mask                       = sd::full<float>(audio_mask_shape, audio_mask_value);
     return sd::ops::concat(video_mask_full, audio_mask, 3);
 }
 
@@ -4957,6 +5028,44 @@ static std::optional<ImageGenerationLatents> prepare_video_generation_latents(sd
     if (sd_version_is_ltxav(sd_ctx->sd->version)) {
         latents.audio_length = get_ltxav_num_audio_latents(request->frames, request->fps);
         latents.audio_latent = make_ltxav_empty_audio_latent(latents.audio_length);
+        if (sd_vid_gen_params->input_audio != nullptr &&
+            sd_vid_gen_params->input_audio->data != nullptr &&
+            sd_vid_gen_params->input_audio->sample_count > 0) {
+            if (sd_ctx->sd->audio_vae_model == nullptr || !sd_ctx->sd->audio_vae_model->config.has_encoder) {
+                LOG_ERROR("LTX A2V requires an audio VAE with encoder weights");
+                return std::nullopt;
+            }
+
+            int64_t audio_encode_start = ggml_time_ms();
+            auto waveform = sd_audio_to_ltx_waveform_tensor(sd_vid_gen_params->input_audio,
+                                                            sd_ctx->sd->audio_vae_model->config.sample_rate,
+                                                            sd_ctx->sd->audio_vae_model->config.audio_channels);
+            if (waveform.empty()) {
+                LOG_ERROR("failed to convert source audio for LTX A2V encoding");
+                return std::nullopt;
+            }
+
+            auto encoded_audio_latent = sd_ctx->sd->audio_vae_model->encode(sd_ctx->sd->n_threads, waveform);
+            if (encoded_audio_latent.empty()) {
+                LOG_ERROR("LTX A2V audio latent encoding failed");
+                return std::nullopt;
+            }
+
+            latents.audio_latent = resize_ltxav_audio_latent(encoded_audio_latent, latents.audio_length);
+            if (latents.audio_latent.empty()) {
+                LOG_ERROR("failed to resize encoded LTX A2V audio latent");
+                return std::nullopt;
+            }
+
+            int64_t audio_encode_end = ggml_time_ms();
+            LOG_INFO("encoded LTX A2V source audio latent %dx%dx%dx%d -> length %d, taking %.2fs",
+                     (int)encoded_audio_latent.shape()[0],
+                     (int)encoded_audio_latent.shape()[1],
+                     (int)encoded_audio_latent.shape()[2],
+                     (int)encoded_audio_latent.shape()[3],
+                     latents.audio_length,
+                     (audio_encode_end - audio_encode_start) * 1.0f / 1000);
+        }
     }
 
     if (sd_version_is_ltxav(sd_ctx->sd->version)) {
@@ -5234,10 +5343,17 @@ static std::optional<ImageGenerationLatents> prepare_video_generation_latents(sd
     }
 
     if (sd_version_is_ltxav(sd_ctx->sd->version) && !latents.audio_latent.empty()) {
+        bool has_input_audio = sd_vid_gen_params->input_audio != nullptr &&
+                               sd_vid_gen_params->input_audio->data != nullptr &&
+                               sd_vid_gen_params->input_audio->sample_count > 0;
+        if (has_input_audio && latents.denoise_mask.empty()) {
+            latents.denoise_mask = make_ltxav_video_denoise_mask(latents.init_latent, 1.f);
+        }
         if (!latents.denoise_mask.empty()) {
             latents.denoise_mask = pack_ltxav_audio_and_video_denoise_mask(latents.denoise_mask,
                                                                            latents.init_latent,
-                                                                           latents.audio_latent);
+                                                                           latents.audio_latent,
+                                                                           has_input_audio ? 0.f : 1.f);
         }
         latents.init_latent = pack_ltxav_audio_and_video_latents(latents.init_latent, latents.audio_latent);
     }
@@ -5509,8 +5625,14 @@ static bool apply_ltxv_refine_image_conditioning(sd_ctx_t* sd_ctx,
     }
 
     if (!audio_latent.empty()) {
+        bool has_input_audio = sd_vid_gen_params->input_audio != nullptr &&
+                               sd_vid_gen_params->input_audio->data != nullptr &&
+                               sd_vid_gen_params->input_audio->sample_count > 0;
         *latent       = pack_ltxav_audio_and_video_latents(video_latent, audio_latent);
-        *denoise_mask = pack_ltxav_audio_and_video_denoise_mask(video_mask, video_latent, audio_latent);
+        *denoise_mask = pack_ltxav_audio_and_video_denoise_mask(video_mask,
+                                                                video_latent,
+                                                                audio_latent,
+                                                                has_input_audio ? 0.f : 1.f);
     } else {
         *latent       = std::move(video_latent);
         *denoise_mask = std::move(video_mask);
@@ -5542,6 +5664,9 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
     int64_t t0                    = ggml_time_ms();
     sd_ctx->sd->vae_tiling_params = sd_vid_gen_params->vae_tiling_params;
     GenerationRequest request(sd_ctx, sd_vid_gen_params);
+    bool has_input_audio = sd_vid_gen_params->input_audio != nullptr &&
+                           sd_vid_gen_params->input_audio->data != nullptr &&
+                           sd_vid_gen_params->input_audio->sample_count > 0;
     bool latent_upscale_enabled     = request.hires.enabled;
     GenerationRequest hires_request = request;
     if (latent_upscale_enabled) {
@@ -5759,6 +5884,17 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
                                                   &hires_video_positions)) {
             return false;
         }
+        if (has_input_audio && hires_denoise_mask.empty() && x_t.shape()[3] > sd_ctx->sd->get_latent_channel()) {
+            int latent_channels = sd_ctx->sd->get_latent_channel();
+            auto video_latent   = sd::ops::slice(x_t, 3, 0, latent_channels);
+            auto audio_latent   = unpack_ltxav_audio_latent(x_t, latents.audio_length, latent_channels);
+            if (!audio_latent.empty()) {
+                hires_denoise_mask = pack_ltxav_audio_and_video_denoise_mask(make_ltxav_video_denoise_mask(video_latent, 1.f),
+                                                                              video_latent,
+                                                                              audio_latent,
+                                                                              0.f);
+            }
+        }
         noise = sd::Tensor<float>::randn_like(x_t, sd_ctx->sd->rng);
 
         W                                   = hires_request.width / hires_request.vae_scale_factor;
@@ -5827,6 +5963,9 @@ SD_API bool generate_video(sd_ctx_t* sd_ctx,
 
     sd_audio_t* generated_audio = nullptr;
     if (sd_version_is_ltxav(sd_ctx->sd->version) &&
+        has_input_audio) {
+        generated_audio = clone_sd_audio(sd_vid_gen_params->input_audio);
+    } else if (sd_version_is_ltxav(sd_ctx->sd->version) &&
         latents.audio_length > 0 &&
         sd_ctx->sd->audio_vae_model != nullptr) {
         if (sd_ctx->sd->get_cancel_flag() == SD_CANCEL_ALL) {

otherarch/utils.cpp

@@ -949,6 +949,55 @@ bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int ta
     return true;
 }
 
+bool kcpp_decode_audio_file_from_buf(const unsigned char * buf_in, size_t len, int & sample_rate, int & channels, std::vector<float> & pcmf32_interleaved) {
+    sample_rate = 0;
+    channels = 0;
+    pcmf32_interleaved.clear();
+
+    if (!buf_is_audio_file((const char *)buf_in, len))
+    {
+        return false;
+    }
+
+    ma_result result;
+    ma_decoder_config decoder_config = ma_decoder_config_init(ma_format_f32, 0, 0);
+    ma_decoder decoder;
+
+    result = ma_decoder_init_memory(buf_in, len, &decoder_config, &decoder);
+    if (result != MA_SUCCESS) {
+        return false;
+    }
+
+    ma_uint32 decoded_channels = 0;
+    ma_uint32 decoded_sample_rate = 0;
+    result = ma_decoder_get_data_format(&decoder, nullptr, &decoded_channels, &decoded_sample_rate, nullptr, 0);
+    if (result != MA_SUCCESS || decoded_channels == 0 || decoded_sample_rate == 0) {
+        ma_decoder_uninit(&decoder);
+        return false;
+    }
+
+    ma_uint64 frame_count;
+    ma_uint64 frames_read;
+    result = ma_decoder_get_length_in_pcm_frames(&decoder, &frame_count);
+    if (result != MA_SUCCESS) {
+        ma_decoder_uninit(&decoder);
+        return false;
+    }
+
+    pcmf32_interleaved.resize(static_cast<size_t>(frame_count) * static_cast<size_t>(decoded_channels));
+    result = ma_decoder_read_pcm_frames(&decoder, pcmf32_interleaved.data(), frame_count, &frames_read);
+    ma_decoder_uninit(&decoder);
+    if (result != MA_SUCCESS) {
+        pcmf32_interleaved.clear();
+        return false;
+    }
+
+    pcmf32_interleaved.resize(static_cast<size_t>(frames_read) * static_cast<size_t>(decoded_channels));
+    sample_rate = static_cast<int>(decoded_sample_rate);
+    channels = static_cast<int>(decoded_channels);
+    return !pcmf32_interleaved.empty();
+}
+
 //this version is specifically required for ace-step
 bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector<float> & pcm, int & T_audio) {
     ma_result result;

otherarch/utils.h

@@ -61,6 +61,7 @@ int32_t kcpp_quick_sample(float * logits, const int n_logits, const std::vector<
 
 std::vector<std::string> split_string(const std::string& input, const std::string& separator);
 bool kcpp_decode_audio_from_buf(const unsigned char * buf_in, size_t len, int target_sampler_rate, std::vector<float> & pcmf32_mono);
+bool kcpp_decode_audio_file_from_buf(const unsigned char * buf_in, size_t len, int & sample_rate, int & channels, std::vector<float> & pcmf32_interleaved);
 bool kcpp_decode_audio_to_f32_stereo_48k(const uint8_t * data, size_t data_size, std::vector<float> & pcm, int & T_audio);
 
 typedef struct ggml_backend_device * ggml_backend_dev_t;
@@ -112,4 +113,4 @@ struct wav_ulaw_header {
 std::string save_ulaw_wav8_base64(const std::vector<float> &data, int sample_rate);
 std::string save_wav16_base64(const std::vector<float> &data, int sample_rate);
 std::string save_stereo_wav16_base64(const std::vector<float> & raw_audio, int T_audio, int sample_rate);
-std::string save_stereo_mp3_base64(const std::vector<float> & raw_audio,int T_audio,int sample_rate);
+std::string save_stereo_mp3_base64(const std::vector<float> & raw_audio,int T_audio,int sample_rate);