diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml new file mode 100644 index 000000000..fed9c877c --- /dev/null +++ b/.github/workflows/make-release.yml @@ -0,0 +1,46 @@ +name: Make Release + +on: + workflow_dispatch: + inputs: + dry_run: + description: 'Dry run - validate without creating the tag' + required: true + type: boolean + default: true + +env: + GH_TOKEN: ${{ github.token }} + +permissions: + contents: write + +jobs: + make-release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Run release checks + id: checks + run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }} + env: + GITHUB_REPOSITORY: ${{ github.repository }} + + - name: Create release tag + if: ${{ github.event.inputs.dry_run == 'false' }} + run: | + VERSION="${{ steps.checks.outputs.version }}" + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag -a "${VERSION}" -m "Release ${VERSION}" + git push origin "${VERSION}" + echo "Created and pushed tag ${VERSION}" + + - name: Dry run summary + if: ${{ github.event.inputs.dry_run == 'true' }} + run: | + echo "Dry run complete - all checks passed." + echo "Would have created tag: ${{ steps.checks.outputs.version }}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 7cfa63d2d..337940f93 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -478,6 +478,8 @@ add_library(common2 tools/mtmd/mtmd-helper.cpp tools/mtmd/mtmd-image.cpp tools/mtmd/mtmd-helper.h + tools/mtmd/mtmd-helper-gen.cpp + tools/mtmd/mtmd-helper-common.h tools/mtmd/clip.cpp tools/mtmd/clip.h src/unicode.h diff --git a/Makefile b/Makefile index 64e6684a4..cd4a03009 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,8 @@ ifdef KCPP_SANITIZE CFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error CXXFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error endif -CFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC -CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC +CFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./tools/mtmd -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC +CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./tools/mtmd -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC ifndef KCPP_DEBUG CFLAGS += -DNDEBUG -s @@ -579,6 +579,8 @@ mtmd.o: tools/mtmd/mtmd.cpp tools/mtmd/mtmd.h $(CXX) $(CXXFLAGS) -c $< -o $@ mtmd-helper.o: tools/mtmd/mtmd-helper.cpp tools/mtmd/mtmd-helper.h $(CXX) $(CXXFLAGS) -c $< -o $@ +mtmd-helper-gen.o: tools/mtmd/mtmd-helper-gen.cpp tools/mtmd/mtmd-helper-common.h + $(CXX) $(CXXFLAGS) -c $< -o $@ mtmd-image.o: tools/mtmd/mtmd-image.cpp tools/mtmd/mtmd-image.h $(CXX) $(CXXFLAGS) -c $< -o $@ unicode-common.o: common/unicode.cpp common/unicode.h @@ -667,7 +669,7 @@ ggml-vulkan-shaders-noext.o: ggml/src/ggml-vulkan-shaders-noext.cpp ggml/include $(CXX) $(CXXFLAGS) $(VKGEN_NOEXT_FORCE) $(VULKAN_FLAGS) -c $< -o $@ # intermediate objects -llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h +llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-kv-cache-msa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h $(CXX) $(CXXFLAGS) -c $< -o $@ llama-model.o: src/llama-model.cpp src/llama-model.h src/models/models.h ggml/include/ggml.h include/llama.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -688,7 +690,7 @@ budget.o: common/reasoning-budget.cpp common/reasoning-budget.h chat.o: common/chat.cpp common/chat.h $(CXX) $(CXXFLAGS) -c $< -o $@ -SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h +SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minimax_h3.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/audio_vae.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/minimax_h3_audio_vae.hpp src/model/vae/minimax_h3_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp @@ -764,35 +766,35 @@ clean: rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o # useful tools -main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) ggml/src/ggml-vulkan-shaders.cpp: ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -892,15 +894,18 @@ else @echo 'Vulkan Shaders (no extensions) Rebuilt for Linux...' endif +# common object files for all libraries +KOBOLDCPP_COMMON_OBJS = expose.o chat.o ggml-binops.o ggml-unops.o ggml-backend.o ggml-backend-meta.o ggml-repack.o llama.o llama-model.o embeddings_default.o music_default.o tts_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o $(OBJS) $(OBJS_SDTYPE) + #generated libraries -koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) $(DEFAULT_BUILD) -koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) $(DEFAULT_BUILD) ifdef FAILSAFE_BUILD -koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) +koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o gpttype_adapter_failsafe.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FAILSAFE) $(FAILSAFE_BUILD) else koboldcpp_failsafe: @@ -908,7 +913,7 @@ koboldcpp_failsafe: endif ifdef NOAVX2_BUILD -koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o gpttype_adapter_failsafe.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLE) $(NOAVX2_BUILD) else koboldcpp_noavx2: @@ -916,7 +921,7 @@ koboldcpp_noavx2: endif ifdef CUBLAS_BUILD -koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o gpttype_adapter_cublas.o whispercpp_cublas.o clip_cublas.o ggml-backend-reg_cublas.o $(CUBLAS_OBJS) $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) $(CUBLAS_BUILD) else koboldcpp_cublas: @@ -924,7 +929,7 @@ koboldcpp_cublas: endif ifdef HIPBLAS_BUILD -koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o gpttype_adapter_cublas.o whispercpp_cublas.o clip_cublas.o ggml-backend-reg_cublas.o $(HIP_OBJS) $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) $(HIPBLAS_BUILD) else koboldcpp_hipblas: @@ -932,12 +937,12 @@ koboldcpp_hipblas: endif ifdef VULKAN_BUILD -koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o llama.o chat.o llama-model.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) $(VULKAN_BUILD) ifdef NOAVX2_BUILD -koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLE) $(VULKAN_BUILD) -koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) +koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLER) $(VULKAN_BUILD) else koboldcpp_vulkan_noavx2: @@ -955,15 +960,15 @@ koboldcpp_vulkan_failsafe: endif # tools -quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) diff --git a/cmake/arm64-windows-msvc-cuda.cmake b/cmake/arm64-windows-msvc-cuda.cmake new file mode 100644 index 000000000..370f2b3d2 --- /dev/null +++ b/cmake/arm64-windows-msvc-cuda.cmake @@ -0,0 +1,26 @@ +# Used to cross-compile ggml-cuda for Windows ARM64 on an x64 Windows host. +set( CMAKE_SYSTEM_NAME Windows ) +set( CMAKE_SYSTEM_PROCESSOR arm64 ) + +if ( DEFINED CUDAToolkit_ROOT ) + file( TO_CMAKE_PATH "${CUDAToolkit_ROOT}" CUDA_ROOT ) +elseif ( DEFINED ENV{CUDA_PATH} ) + file( TO_CMAKE_PATH "$ENV{CUDA_PATH}" CUDA_ROOT ) +else() + message( FATAL_ERROR "Set CUDAToolkit_ROOT or CUDA_PATH to a Windows CUDA Toolkit with ARM64 target libraries" ) +endif() + +if ( DEFINED ENV{VCToolsInstallDir} ) + file( TO_CMAKE_PATH "$ENV{VCToolsInstallDir}" MSVC_TOOLS_ROOT ) + set( CMAKE_CUDA_HOST_COMPILER "${MSVC_TOOLS_ROOT}/bin/Hostx64/arm64/cl.exe" CACHE FILEPATH "" ) +endif() + +set( CMAKE_CUDA_COMPILER "${CUDA_ROOT}/bin/nvcc.exe" CACHE FILEPATH "" ) +set( CMAKE_CUDA_FLAGS_INIT "-target-dir=arm64" ) + +# FindCUDAToolkit selects lib/x64 from the host architecture on Windows. +set( CUDA_CUDART "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" ) +set( CUDA_cudart_LIBRARY "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" ) +set( CUDA_cublas_LIBRARY "${CUDA_ROOT}/lib/arm64/cublas.lib" CACHE FILEPATH "" ) +set( CUDA_cublasLt_LIBRARY "${CUDA_ROOT}/lib/arm64/cublasLt.lib" CACHE FILEPATH "" ) +set( CUDA_cuda_driver_LIBRARY "${CUDA_ROOT}/lib/arm64/cuda.lib" CACHE FILEPATH "" ) diff --git a/common/arg.cpp b/common/arg.cpp index 3da048a63..e1e9ef36e 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,7 @@ #include #include #include +#include #include // for hardware_concurrency #include @@ -61,6 +63,7 @@ static std::initializer_list mmproj_examples = { LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, + LLAMA_EXAMPLE_TTS, }; static std::string read_file(const std::string & fname) { @@ -360,7 +363,6 @@ static bool spec_types_is_default(const common_params & params) { common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) { common_download_hf_plan plan; common_download_hf_plan plan_spec; - common_download_hf_plan plan_voc; common_download_opts opts; const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), @@ -375,6 +377,10 @@ common_models_handler common_models_handler_init(const common_params & params, l params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end(); + const bool spec_type_draft_dspark = std::find(params.speculative.types.begin(), + params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) != params.speculative.types.end(); + // only download mmproj if the current example is using it bool use_mmproj = false; for (const auto & ex : mmproj_examples) { @@ -389,6 +395,7 @@ common_models_handler common_models_handler_init(const common_params & params, l opts.download_mtp = spec_type_draft_mtp; opts.download_eagle3 = spec_type_draft_eagle3; opts.download_dflash = spec_type_draft_dflash; + opts.download_dspark = spec_type_draft_dspark; opts.download_mmproj = use_mmproj && !params.no_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty(); @@ -403,15 +410,12 @@ common_models_handler common_models_handler_init(const common_params & params, l opts_spec.download_mtp = true; opts_spec.download_dflash = true; opts_spec.download_eagle3 = true; + opts_spec.download_dspark = true; } plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec); } - if (!params.vocoder.model.hf_repo.empty()) { - plan_voc = common_download_get_hf_plan(params.vocoder.model, opts); - } - - return common_models_handler{plan, plan_spec, plan_voc, opts}; + return common_models_handler{plan, plan_spec, opts}; } bool common_models_handler_is_preset_repo(const common_models_handler & handler) { @@ -461,7 +465,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params auto & plan = handler.plan; auto & plan_spec = handler.plan_spec; - auto & plan_voc = handler.plan_voc; auto opts = handler.opts; // copy opts.callback = callback; @@ -476,7 +479,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params }; handle_url(params.model); handle_url(params.mmproj); - handle_url(params.vocoder.model); handle_url(params.speculative.draft.mparams); // optionally, if docker repo is set, resolve it @@ -504,14 +506,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params task.opts = opts; tasks.push_back(task); } - if (!params.vocoder.model.url.empty()) { - common_download_task task; - task.url = params.vocoder.model.url; - task.local_path = params.vocoder.model.path; - task.opts = opts; - tasks.push_back(task); - } - bool had_spec_url = false; if (!params.speculative.draft.mparams.url.empty()) { common_download_task task; @@ -545,12 +539,19 @@ void common_models_handler_apply(common_models_handler & handler, common_params plan_spec.mtp = {}; plan_spec.dflash = {}; plan_spec.eagle3 = {}; + plan_spec.dspark = {}; } // infer the speculative type from the sidecar shipped by the draft repo when none is requested if (spec_types_is_default(params)) { if (!plan_spec.mtp.local_path.empty()) { params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP }; + plan_spec.dspark = {}; + plan_spec.dflash = {}; + plan_spec.eagle3 = {}; + } else if (!plan_spec.dspark.local_path.empty()) { + // dspark outranks dflash, its sidecar carries the extra Markov head + params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK }; plan_spec.dflash = {}; plan_spec.eagle3 = {}; } else if (!plan_spec.dflash.local_path.empty()) { @@ -561,10 +562,20 @@ void common_models_handler_apply(common_models_handler & handler, common_params } } + // infer the speculative type from the draft GGUF metadata when none is requested + // note: reads only the first split - sharded drafts need an explicit --spec-type + if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) { + const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path); + if (!types_gguf.empty()) { + params.speculative.types = types_gguf; + } + } + // when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() || !plan_spec.dflash.local_path.empty() || - !plan_spec.eagle3.local_path.empty(); + !plan_spec.eagle3.local_path.empty() || + !plan_spec.dspark.local_path.empty(); if (!plan_spec.mtp.local_path.empty() && !had_spec_url) { tasks.emplace_back(plan_spec.mtp, opts, [&]() { // only use the discovered MTP head when no draft path is set yet @@ -595,6 +606,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }); } + if (!plan_spec.dspark.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan_spec.dspark, opts, [&]() { + // only use the discovered DSpark sidecar when no draft path is set yet + if (params.speculative.draft.mparams.path.empty()) { + params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark); + } else { + hf_cache::finalize_file(plan_spec.dspark); + } + }); + } // a wired draft sidecar counts as an explicit draft for the main plan fallback below if (spec_sidecar_found) { @@ -607,11 +628,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params had_spec_url = true; } - // handle vocoder plan (e.g. --hf-repo-v) - if (!plan_voc.model_files.empty()) { - add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model); - } - if (!plan.model_files.empty()) { add_tasks(plan.model_files, plan.primary, params.model); } @@ -650,6 +666,16 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }); } + if (!plan.dspark.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan.dspark, opts, [&]() { + // only fall back to the discovered DSpark sidecar when no draft was explicitly provided + if (params.speculative.draft.mparams.empty()) { + params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark); + } else { + hf_cache::finalize_file(plan.dspark); + } + }); + } if (!plan.preset.local_path.empty()) { tasks.emplace_back(plan.preset, opts, [&]() { // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file @@ -689,12 +715,61 @@ void common_models_handler_apply(common_models_handler & handler, common_params // CLI argument parsing functions // +// apply config files (if present), a later file overrides an earlier one: +// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows) +// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows) +static void common_params_apply_system_config(common_params & params, llama_example ex) { + std::vector paths; + +#if defined(_WIN32) + const std::string program_data = common_get_env("PROGRAMDATA"); + if (!program_data.empty()) { + paths.push_back(program_data + "\\llama.cpp\\config.ini"); + } +#else + paths.push_back("/etc/llama.cpp/config.ini"); +#endif + + try { + paths.push_back(fs_get_config_directory() + "config.ini"); + } catch (const std::exception & e) { + LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what()); + } + + std::vector found; + for (const auto & path : paths) { + std::error_code ec; + if (std::filesystem::exists(path, ec)) { + found.push_back(path); + } + } + if (found.empty()) { + return; + } + + common_preset_context ctx(ex); + ctx.ignore_unknown_keys = true; // the same config file is shared by all programs + for (const auto & path : found) { + LOG_INF("using config file: %s\n", path.c_str()); + common_preset global; + common_presets presets = ctx.load_from_ini(path, global); + global.apply_to_params(params); + auto it = presets.find(COMMON_PRESET_DEFAULT_NAME); + if (it != presets.end()) { + it->second.apply_to_params(params); + } + } +} + static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) { common_params & params = ctx_arg.params; // setup log directly from params.verbosity: see tools/cli/cli.cpp common_log_set_verbosity_thold(params.verbosity); + // config file applies first, so env variables and CLI arguments override it + common_params_apply_system_config(params, ctx_arg.ex); + std::unordered_map> arg_to_options; for (auto & opt : ctx_arg.options) { for (const auto & arg : opt.args) { @@ -1327,6 +1402,10 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.n_parallel = -1; // auto by default } else if (ex == LLAMA_EXAMPLE_TOKENIZE) { params.parse_special = true; // parse special tokens by default, like the old tokenize tool + } else if (ex == LLAMA_EXAMPLE_TTS) { + params.out_file = "output.wav"; + params.sampling.penalty_repeat = 1.05f; + params.sampling.penalty_last_n = -1; } params.use_color = tty_can_use_colors(); @@ -1371,8 +1450,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--version"}, "show version and build info", [](common_params &) { - fprintf(stderr, "version: %d (%s)\n", llama_build_number(), llama_commit()); - fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target()); + llama_print_build_info(llama_version()); exit(0); } )); @@ -1989,9 +2067,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_sampling()); add_opt(common_arg( {"--repeat-last-n"}, "N", - string_format("last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", params.sampling.penalty_last_n), + string_format("last n tokens to consider for penalize (default: %d, 0 = disabled)", params.sampling.penalty_last_n), [](common_params & params, int value) { - if (value < -1) { + if (value < 0) { throw std::runtime_error(string_format("error: invalid repeat-last-n = %d\n", value)); } params.sampling.penalty_last_n = value; @@ -2003,7 +2081,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--repeat-penalty"}, "N", string_format("penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)", (double)params.sampling.penalty_repeat), [](common_params & params, const std::string & value) { - params.sampling.penalty_repeat = std::stof(value); + const float penalty_repeat = std::stof(value); + if (!std::isfinite(penalty_repeat) || + penalty_repeat <= 0.0f || + !std::isfinite(1.0f/penalty_repeat)) { + throw std::runtime_error("error: repeat-penalty must be finite and greater than 0\n"); + } + params.sampling.penalty_repeat = penalty_repeat; params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT; } ).set_sampling()); @@ -2011,14 +2095,22 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--presence-penalty"}, "N", string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present), [](common_params & params, const std::string & value) { - params.sampling.penalty_present = std::stof(value); + const float penalty_present = std::stof(value); + if (!std::isfinite(penalty_present)) { + throw std::runtime_error("error: presence-penalty must be finite\n"); + } + params.sampling.penalty_present = penalty_present; } ).set_sampling()); add_opt(common_arg( {"--frequency-penalty"}, "N", string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq), [](common_params & params, const std::string & value) { - params.sampling.penalty_freq = std::stof(value); + const float penalty_freq = std::stof(value); + if (!std::isfinite(penalty_freq)) { + throw std::runtime_error("error: frequency-penalty must be finite\n"); + } + params.sampling.penalty_freq = penalty_freq; } ).set_sampling()); add_opt(common_arg( @@ -2048,9 +2140,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_sampling()); add_opt(common_arg( {"--dry-penalty-last-n"}, "N", - string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable, -1 = context size)", params.sampling.dry_penalty_last_n), + string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable)", params.sampling.dry_penalty_last_n), [](common_params & params, int value) { - if (value < -1) { + if (value < 0) { throw std::runtime_error(string_format("error: invalid dry-penalty-last-n = %d\n", value)); } params.sampling.dry_penalty_last_n = value; @@ -2534,7 +2626,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.mtmd_batch_max_tokens = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS")); - if (llama_supports_rpc()) { + if (params.is_gen_docs || llama_supports_rpc()) { add_opt(common_arg( {"--rpc"}, "SERVERS", "comma-separated list of RPC servers (host:port)", @@ -2572,14 +2664,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_env("LLAMA_ARG_DIO")); add_opt(common_arg( {"-lm", "--load-mode"}, "MODE", - "model loading mode (default: mmap)\n" + "model loading mode (default: auto)\n" + "- auto: mmap, unless a device does not support it\n" "- none: no special loading mode\n" "- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n" "- mlock: force system to keep model in RAM rather than swapping or compressing\n" "- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n" "- dio: use DirectIO if available\n", [](common_params & params, const std::string & value) { - /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; } + /**/ if (value == "auto") { params.load_mode = LLAMA_LOAD_MODE_AUTO; } + else if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; } else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; } else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; } else if (value == "mmap+mlock") { params.load_mode = LLAMA_LOAD_MODE_MMAP_MLOCK; } @@ -2935,20 +3029,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.model.hf_file = value; } ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE")); - add_opt(common_arg( - {"-hfv", "-hfrv", "--hf-repo-v"}, "/[:quant]", - "Hugging Face model repository for the vocoder model (default: unused)", - [](common_params & params, const std::string & value) { - params.vocoder.model.hf_repo = value; - } - ).set_env("LLAMA_ARG_HF_REPO_V")); - add_opt(common_arg( - {"-hffv", "--hf-file-v"}, "FILE", - "Hugging Face model file for the vocoder model (default: unused)", - [](common_params & params, const std::string & value) { - params.vocoder.model.hf_file = value; - } - ).set_env("LLAMA_ARG_HF_FILE_V")); add_opt(common_arg( {"-hft", "--hf-token"}, "TOKEN", "Hugging Face access token (default: value from HF_TOKEN environment variable)", @@ -3283,12 +3363,23 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--tools"}, "TOOL1,TOOL2,...", "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n" "specify \"all\" to enable all tools\n" - "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n" + "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n" "note: for security reasons, this will limit --cors-origins to localhost by default", [](common_params & params, const std::string & value) { params.server_tools = parse_csv_row(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS")); + add_opt(common_arg( + {"--tools-runtime"}, "OPTION", + "experimental: run tools in a separate runtime environment (default: none, use host environment)\n" + "available options:\n" + " 'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit\n" + " 'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit\n" + " 'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required\n", + [](common_params & params, const std::string & value) { + params.server_tools_runtime = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME")); add_opt(common_arg( {"--mcp-servers-config"}, "PATH", "experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n" @@ -3556,6 +3647,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING")); + add_opt(common_arg( + {"--reasoning-effort"}, "LEVEL", + "reasoning effort level given to the chat template: 'default' to keep the template default,\n" + "or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)", + [](common_params & params, const std::string & value) { + if (value == "default") { + params.default_template_kwargs.erase("reasoning_effort"); + } else { + params.default_template_kwargs["reasoning_effort"] = json(value).dump(); + } + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_EFFORT")); add_opt(common_arg( {"--reasoning-budget"}, "N", "token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)", @@ -3975,6 +4078,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--spec-draft-n-max"}, "N", string_format("number of tokens to draft for speculative decoding (default: %d)", params.speculative.draft.n_max), [](common_params & params, int value) { + if (value < 0) { + throw std::invalid_argument("invalid value"); + } params.speculative.draft.n_max = value; } ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MAX")); @@ -4224,24 +4330,18 @@ common_params_context common_params_parser_init(common_params & params, llama_ex // add_opt(common_arg( - {"-mv", "--model-vocoder"}, "FNAME", - "vocoder model for audio generation (default: unused)", + {"--tts-lang"}, "FNAME", + "language (ISO 639-1) for audio generation\n" + "see tts/README.md for per-model usage notes", [](common_params & params, const std::string & value) { - params.vocoder.model.path = value; + params.tts_lang = value; } - ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); - add_opt(common_arg( - {"--tts-use-guide-tokens"}, - "Use guide tokens to improve TTS word recall", - [](common_params & params) { - params.vocoder.use_guide_tokens = true; - } - ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); + ).set_examples({LLAMA_EXAMPLE_TTS})); add_opt(common_arg( {"--tts-speaker-file"}, "FNAME", "speaker file path for audio generation", [](common_params & params, const std::string & value) { - params.vocoder.speaker_file = value; + params.tts_speaker_file = value; } ).set_examples({LLAMA_EXAMPLE_TTS})); @@ -4361,16 +4461,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_DEBUG})); // presets - add_opt(common_arg( - {"--tts-oute-default"}, - string_format("use default OuteTTS models (note: can download weights from the internet)"), - [](common_params & params) { - params.model.hf_repo = "OuteAI/OuteTTS-0.2-500M-GGUF"; - params.model.hf_file = "OuteTTS-0.2-500M-Q8_0.gguf"; - params.vocoder.model.hf_repo = "ggml-org/WavTokenizer"; - params.vocoder.model.hf_file = "WavTokenizer-Large-75-F16.gguf"; - } - ).set_examples({LLAMA_EXAMPLE_TTS})); add_opt(common_arg( {"--embd-gemma-default"}, diff --git a/common/arg.h b/common/arg.h index 8f609e356..44b9e887c 100644 --- a/common/arg.h +++ b/common/arg.h @@ -137,7 +137,6 @@ void common_params_add_preset_options(std::vector & args); struct common_models_handler { common_download_hf_plan plan; common_download_hf_plan plan_spec; - common_download_hf_plan plan_voc; common_download_opts opts; }; diff --git a/common/build-info.cpp.in b/common/build-info.cpp.in index f888fd079..4ec339708 100644 --- a/common/build-info.cpp.in +++ b/common/build-info.cpp.in @@ -29,7 +29,7 @@ const char * llama_build_info(void) { return s.c_str(); } -void llama_print_build_info(void) { - fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit()); - fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target()); +void llama_print_build_info(const char * llama_version) { + fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit()); + fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target()); } diff --git a/common/build-info.h b/common/build-info.h index afc7b49a1..e6a696acb 100644 --- a/common/build-info.h +++ b/common/build-info.h @@ -2,6 +2,7 @@ #define BUILD_INFO_H #define LLAMA_BUILD_NUMBER 999 +#define LLAMA_VERSION "1.0" #define LLAMA_COMMIT "KOBOLDCPP" #define LLAMA_COMPILER "KCPP" #define LLAMA_TARGET "KCPP" @@ -32,7 +33,7 @@ static inline const char * llama_build_info(void) { return s.c_str(); } -static inline void llama_print_build_info(void) { +static inline void llama_print_build_info(const char *) { fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit()); fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target()); } diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index f786f5ff2..06737b165 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -6,6 +6,9 @@ #include +#include +#include + using ordered_json = nlohmann::ordered_json; static std::string_view trim_trailing_space(std::string_view sv, int max = -1) { @@ -235,6 +238,43 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri return zero_or_more(choice({ p, content_chunk })); } +common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix, + const std::vector & parsers) { + if (parsers.empty()) { + return eps(); + } + + if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) { + return sequence(parsers); + } + + std::map rules; + std::function remaining_of; + + remaining_of = [&](uint32_t remaining) -> common_peg_parser { + if (remaining == 0) { + return eps(); + } + + auto cached = rules.find(remaining); + if (cached != rules.end()) { + return cached->second; + } + + auto alternatives = choice(); + for (size_t i = 0; i < parsers.size(); i++) { + const uint32_t bit = 1u << i; + if (remaining & bit) { + alternatives |= parsers[i] + remaining_of(remaining & ~bit); + } + } + + return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second; + }; + + return remaining_of((1u << parsers.size()) - 1); +} + std::string & common_chat_peg_mapper::args_target() { return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer; } @@ -554,9 +594,7 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls( // Full argument: name="value" or name=value auto arg_rule = tool_arg( - tool_arg_open(eps()) + - tool_arg_name(arg_name_parser) + - literal("=") + + tool_arg_open(tool_arg_name(arg_name_parser) + literal("=")) + arg_value_parser + tool_arg_close(eps()) ); diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index cd14f2c11..5d764dbaa 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -55,6 +55,8 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper { struct content_structure; struct tool_call_structure; +constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6; + class common_chat_peg_builder : public common_peg_parser_builder { public: // Tag constants (from former common_chat_peg_base_builder) @@ -105,6 +107,9 @@ class common_chat_peg_builder : public common_peg_parser_builder { common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); } + // Matches every parser exactly once, in any order. + common_peg_parser permute(const std::string & rule_prefix, const std::vector & parsers); + // Return a parser that parses the prefix of a string, up to a given delimiter. common_peg_parser prefix(const std::string & s, const std::string & delimiter = {}); diff --git a/common/chat.cpp b/common/chat.cpp index 5c38b0578..803b4021b 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -934,6 +934,10 @@ static std::string common_chat_template_direct_apply_impl( bool enabled = inp["preserve_reasoning"].get(); jinja::caps_apply_preserve_reasoning(ctx, enabled); } + if (inp.contains("reasoning_effort") && inp["reasoning_effort"].is_string() && !inp["reasoning_effort"].empty()) { + std::string reasoning_effort = inp["reasoning_effort"].get(); + jinja::caps_apply_reasoning_effort(ctx, reasoning_effort); + } jinja::global_from_json(ctx, inp, inputs.mark_input); @@ -1124,6 +1128,179 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ return data; } +static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string GEN_PREFIX = "<|im_start|>assistant\n"; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + + auto supports_reasoning = tmpl.source().find("") != std::string::npos; + + data.supports_thinking = supports_reasoning; + data.preserved_tokens = { + "", + "", + }; + + if (supports_reasoning) { + data.thinking_start_tag = ""; + // Support both and as reasoning end sequences. + // ", "" }; + data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX; + if (supports_reasoning) { + data.generation_prompt += "\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n\n\n"; + } + } + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + std::vector tool_call_starts = { "" }; + + // Match complete opener for Qwen3-Coder models that occasionally omit the + // starting . The model may hallucinate a tool name, but it is preferable over + // constraining on + foreach_function(inputs.tools, [&](const json & tool) { + const std::string name = tool.at("function").at("name"); + tool_call_starts.push_back(""); + }); + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + + auto reasoning = p.eps(); + if (supports_reasoning && extract_reasoning) { + reasoning = p.optional("" + p.space() + + p.reasoning(p.until_one_of({ "", "" })) + + (p.literal("") | p.peek(p.literal("")))); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); + } + + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto arg_close = p.tool_arg_close(p.literal("\n\n")); + auto arg_string = p.rule("xml-arg-string", + p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(parameters); + + std::vector required_args; + std::vector optional_args; + + foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { + auto rule_name = "tool-" + name + "-arg-" + param_name; + + auto arg_open = p.tool_arg_open("\n"); + + auto arg_value = schema_info.resolves_to_string(param_schema) ? + arg_string : + p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; + + auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); + + (is_required ? required_args : optional_args).push_back(arg_rule); + }); + + // Accept required arguments in any order, as Qwen does not always adhere to the + // order provided. + auto args = p.permute("tool-" + name + "-args", required_args); + if (!optional_args.empty()) { + args = args + p.zero_or_more(p.choice(optional_args)); + } + + auto func = p.tool(p.tool_open("\n") + + p.tool_args(args) + + p.tool_close(p.literal("\n"))); + + tool_choice |= p.rule("tool-" + name, func); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + + // Qwen3-Coder models may occasionally omit the token. + auto tool_call_body = tool_choice + "" + p.space(); + auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body); + auto tool_call = p.rule("tool-call", "\n" + tool_call_body); + + auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); + + return generation_prompt + + (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls); + } + + // Content only parser + return generation_prompt + (reasoning << p.content(p.rest())); + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + if (has_response_format) { + auto schema = inputs.json_schema; + builder.resolve_refs(schema); + } + parser.build_grammar(builder, data.grammar_lazy); + }); + + if (data.grammar_lazy) { + for (const auto & start : tool_call_starts) { + data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start }); + } + } + } + + return data; +} + static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { common_chat_params data; @@ -1957,23 +2134,16 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); } - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.thinking_start_tag = ""; - data.thinking_end_tags = {""}; - data.preserved_tokens = { - "｜DSML｜", - "", - "", - }; - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + std::optional additional_context; + if (is_v4 && has_response_format) { + additional_context = json{ { "response_format", inputs.json_schema } }; + } + const std::string DSML = "｜DSML｜"; const std::string THINK_START = ""; const std::string THINK_END = ""; @@ -1985,25 +2155,137 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha const std::string PARAM_START = "<" + DSML + "parameter"; const std::string PARAM_END = ""; const std::string GEN_PROMPT = "<｜Assistant｜>"; + const std::string TC_SEPARATOR = "\n\n"; + + data.prompt = common_chat_template_direct_apply_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); + data.generation_prompt = common_chat_template_generation_prompt_impl( + tmpl, inputs, adjusted_messages, std::nullopt, additional_context); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = THINK_START; + data.thinking_end_tags = {THINK_END, FC_START}; + data.preserved_tokens = { + DSML, + THINK_START, + THINK_END, + }; if (inputs.has_continuation()) { const auto & msg = inputs.continue_msg; - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); + if (is_v4 && msg.reasoning_content.empty()) { + data.generation_prompt = GEN_PROMPT + THINK_END; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + } else { + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); + } } data.prompt += data.generation_prompt; } + bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PROMPT); - auto end = p.end(); + auto end = p.end(); + + // build tool call section first since we might need it in reasoning + auto tool_choice = p.choice(); + if (has_tool_calls) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + const auto & props = params.contains("properties") ? params.at("properties") : json::object(); + + std::set required; + if (params.contains("required")) { + params.at("required").get_to(required); + } + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + std::vector required_parsers; + std::vector optional_parsers; + for (const auto & [param_name, param_schema] : props.items()) { + bool is_required = required.find(param_name) != required.end(); + bool is_string = schema_info.resolves_to_string(param_schema); + + auto arg = p.tool_arg( + p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) + + p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + + (is_string ? + p.tool_arg_string_value(p.until(PARAM_END)) : + p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", + param_schema, false))) + + p.tool_arg_close(p.literal(PARAM_END))); + + auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); + if (is_required) { + required_parsers.push_back(named_arg); + } else { + optional_parsers.push_back(named_arg); + } + } + + common_peg_parser args_seq = p.eps(); + for (size_t i = 0; i < required_parsers.size(); i++) { + if (i > 0) { + args_seq = args_seq + p.space(); + } + args_seq = args_seq + required_parsers[i]; + } + + if (!optional_parsers.empty()) { + common_peg_parser any_opt = p.choice(); + for (const auto & opt : optional_parsers) { + any_opt |= opt; + } + args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); + } + + common_peg_parser invoke_body = args_seq; + auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") + + p.tool_name(p.literal(name)) + p.literal("\">\n")) + + invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END))); + + tool_choice |= p.rule("tool-" + name, func_parser); + }); + } + + common_peg_parser tool_calls = p.eps(); + if (inputs.parallel_tool_calls) { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + + p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); + } else { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); + } auto reasoning = p.eps(); + auto reasoning_with_tc = p.eps(); + auto obligatory_tool_calls = tool_calls; + bool allow_reasoning_with_tc = false; + + if (!require_tools) { + tool_calls = p.optional(tool_calls); + } + if (extract_reasoning && inputs.enable_thinking) { reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); + reasoning_with_tc = THINK_START + + p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + + p.space() + obligatory_tool_calls; + allow_reasoning_with_tc = true; } else if (extract_reasoning) { // Thinking disabled but reasoning extraction requested: the generation prompt // contains an empty pair (V3.2) or a bare (V4) that @@ -2021,101 +2303,21 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha return generation_prompt + reasoning + response_format + end; } - if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + if (!has_tool_calls) { return generation_prompt + reasoning + p.content(p.rest()) + end; } - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - const auto & props = params.contains("properties") ? params.at("properties") : json::object(); - - std::set required; - if (params.contains("required")) { - params.at("required").get_to(required); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - std::vector required_parsers; - std::vector optional_parsers; - for (const auto & [param_name, param_schema] : props.items()) { - bool is_required = required.find(param_name) != required.end(); - bool is_string = schema_info.resolves_to_string(param_schema); - - auto arg = p.tool_arg( - p.tool_arg_open( - p.literal(PARAM_START + " name=\"") + - p.tool_arg_name(p.literal(param_name)) + - p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + - (is_string - ? p.tool_arg_string_value(p.until(PARAM_END)) - : p.tool_arg_json_value(p.schema(p.json(), - "tool-" + name + "-arg-" + param_name + "-schema", - param_schema, false))) + - p.tool_arg_close(p.literal(PARAM_END))); - - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { - required_parsers.push_back(named_arg); - } else { - optional_parsers.push_back(named_arg); - } - } - - common_peg_parser args_seq = p.eps(); - for (size_t i = 0; i < required_parsers.size(); i++) { - if (i > 0) { - args_seq = args_seq + p.space(); - } - args_seq = args_seq + required_parsers[i]; - } - - if (!optional_parsers.empty()) { - common_peg_parser any_opt = p.choice(); - for (const auto & opt : optional_parsers) { - any_opt |= opt; - } - args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); - } - - common_peg_parser invoke_body = args_seq; - auto func_parser = p.tool( - p.tool_open(p.literal(INVOKE_START + " name=\"") + - p.tool_name(p.literal(name)) + p.literal("\">\n")) + - invoke_body + p.space() + - p.tool_close(p.literal(INVOKE_END))); - - tool_choice |= p.rule("tool-" + name, func_parser); - }); - - auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - - common_peg_parser tool_calls = p.eps(); - if (inputs.parallel_tool_calls) { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + - p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); - } else { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); - } - - if (!require_tools) { - tool_calls = p.optional(tool_calls); - } - - auto content_before_tools = p.content(p.until(FC_START)); - return generation_prompt + reasoning + content_before_tools + tool_calls + end; + auto content_before_tools = p.negate(p.literal(THINK_START)) + + p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) + + p.space(); + return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : + generation_prompt + reasoning + content_before_tools + tool_calls + end; }); data.parser = parser.save(); if (include_grammar) { - data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); + data.grammar_lazy = has_tools && !require_tools; data.grammar = build_grammar([&](const common_grammar_builder & builder) { foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); @@ -2909,6 +3111,153 @@ static common_chat_params common_chat_params_init_minicpm5(const common_chat_tem return data; } +// An assistant turn is rendered as one or more messages, each +// "<|start|>assistant to=<|message|>{content}{END}" where END is +// <|eom|> (more messages follow) or <|eot|> (end of turn): +// - chain-of-thought: to=self, terminated by <|eom|> +// - final answer: to=user, terminated by <|eot|> +// The generation prompt is just "<|start|>assistant"; the model emits its own +// " to=...<|message|>". +static common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = "<|start|>assistant"; + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + + data.preserved_tokens = { + "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", + // ATEM tool-call markup emitted on " to=" turns. + "", "", + "", "", + }; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" }, + }; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + // Constrained grammar whenever tools are offered. + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto start = p.rule("start", p.literal("<|start|>assistant")); + + if (!extract_reasoning && !include_grammar) { + return start + p.content(p.rest()); + } + + if (extract_reasoning) { + p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); + } else { + p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); + } + auto analysis = p.ref("analysis"); + + auto recipient = p.optional(p.literal(" to=user")); + auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + + p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto string_value = p.ac( + p.tool_arg_string_value(p.until("")) + p.tool_arg_close(p.literal("")), + ""); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto args = p.eps(); + if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + auto arg_choice = p.choice(); + for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { + auto value_parser = p.eps(); + if (schema_info.resolves_to_string(prop_schema)) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)) + + p.tool_arg_close(p.literal("")); + } + + auto arg_rule = p.tool_arg( + p.tool_arg_open(p.literal("")) + + value_parser); + + arg_choice |= arg_rule; + } + args = p.zero_or_more(arg_choice + p.space()); + } + + auto tool_parser = p.tool( + p.tool_open(p.literal(" to=") + p.until("<|message|>") + + p.literal("<|message|>") + p.space() + + p.literal("") + p.space()) + << p.tool_args(args) + << p.tool_close(p.literal("") + p.space() + p.literal(""))); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto tool_calls = inputs.parallel_tool_calls + ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) + : p.trigger_rule("tool-call", tool_choice); + + + if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { + return p.zero_or_more(start + analysis) + start + tool_calls; + } + auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls); + return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls)); + } + + return p.zero_or_more(start + analysis) + start + final_msg; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, + "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, + }; + } + + return data; +} + static json common_chat_extra_context() { json ctx = json::object(); std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); @@ -2937,6 +3286,12 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_gpt_oss(tmpl, params); } + // Muse Glimmer format using " to=" recipients and <|eom|>/<|eot|> message terminators. + if (src.find("") != std::string::npos && src.find("<|eom|>") != std::string::npos) { + LOG_DBG("Using specialized template: Muse Glimmer\n"); + return common_chat_params_init_muse_glimmer(tmpl, params); + } + // Functionary v3.2 - uses recipient-based format with >>>recipient\n{content} // Detection: template has ">>>all" for content and ">>>" prefix for tool calls if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) { @@ -3020,6 +3375,14 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_minicpm5(tmpl, params); } + // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash + if (src.find("") != std::string::npos && + src.find("pw_dir)) { + throw std::runtime_error("Failed to find $HOME directory"); + } + + config_directory = std::string(pw->pw_dir) + std::string("/.config/"); +#else + throw std::runtime_error("Failed to find $HOME directory"); +#endif + } +#elif defined(_WIN32) + config_directory = common_get_env("APPDATA"); + if (config_directory.empty()) { + throw std::runtime_error("Failed to find %APPDATA% directory"); + } +#elif defined(__EMSCRIPTEN__) + // caller decides what to do when there is no config directory + throw std::runtime_error("not implemented on this platform"); +#else +# error Unknown architecture +#endif + config_directory = ensure_trailing_slash(config_directory); + config_directory += "llama.cpp"; + return ensure_trailing_slash(config_directory); +} + std::string fs_get_cache_file(const std::string & filename) { GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos); std::string cache_directory = fs_get_cache_directory(); @@ -1211,6 +1281,8 @@ struct common_init_result::impl { // note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top + common_threadpools threadpools; + llama_model_ptr model; llama_context_ptr context; @@ -1291,16 +1363,6 @@ common_init_result::common_init_result(common_params & params, bool model_only) params.sampling.logit_bias_eog.begin(), params.sampling.logit_bias_eog.end()); } - //if (params.sampling.penalty_last_n == -1) { - // LOG_TRC("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx)); - // params.sampling.penalty_last_n = llama_n_ctx(lctx); - //} - - //if (params.sampling.dry_penalty_last_n == -1) { - // LOG_TRC("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx)); - // params.sampling.dry_penalty_last_n = llama_n_ctx(lctx); - //} - // init the backend samplers as part of the context creation pimpl->samplers.resize(cparams.n_seq_max); pimpl->samplers_seq_config.resize(cparams.n_seq_max); @@ -1322,6 +1384,10 @@ common_init_result::common_init_result(common_params & params, bool model_only) } pimpl->context.reset(lctx); + + set_process_priority(params.cpuparams.priority); + + pimpl->threadpools.init(lctx, params); } llama_model * common_init_result::model() { @@ -1468,18 +1534,18 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode common_init_result::~common_init_result() = default; std::string common_get_model_endpoint() { - const char * model_endpoint_env = getenv("MODEL_ENDPOINT"); - // We still respect the use of environment-variable "HF_ENDPOINT" for backward-compatibility. - const char * hf_endpoint_env = getenv("HF_ENDPOINT"); - const char * endpoint_env = model_endpoint_env ? model_endpoint_env : hf_endpoint_env; - std::string model_endpoint = "https://huggingface.co/"; - if (endpoint_env) { - model_endpoint = endpoint_env; - if (model_endpoint.back() != '/') { - model_endpoint += '/'; - } + std::string endpoint = common_get_env("MODEL_ENDPOINT"); + if (endpoint.empty()) { + // the HF_ENDPOINT variable is respected for backward compatibility + endpoint = common_get_env("HF_ENDPOINT"); } - return model_endpoint; + if (endpoint.empty()) { + return "https://huggingface.co/"; + } + if (endpoint.back() != '/') { + endpoint += '/'; + } + return endpoint; } char * common_get_model_or_exit(int argc, char * argv[]) { @@ -1626,6 +1692,7 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { mparams.progress_callback = params.load_progress_callback; mparams.progress_callback_user_data = params.load_progress_callback_user_data; mparams.no_alloc = params.no_alloc; + mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); return mparams; } @@ -1637,6 +1704,7 @@ struct llama_context_params common_context_params_to_llama(const common_params & cparams.n_seq_max = params.n_parallel; cparams.n_rs_seq = params.speculative.need_n_rs_seq(); cparams.n_outputs_max = std::max(params.n_outputs_max, 0); + cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0); cparams.n_batch = params.n_batch; cparams.n_ubatch = params.n_ubatch; cparams.n_threads = params.cpuparams.n_threads; @@ -1668,6 +1736,10 @@ struct llama_context_params common_context_params_to_llama(const common_params & return cparams; } +// +// Threadpool utils +// + struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) { struct ggml_threadpool_params tpp; @@ -1684,6 +1756,56 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo return tpp; } +common_threadpools::~common_threadpools() { + if (!free_fn) { + return; + } + free_fn(threadpool); + free_fn(threadpool_batch); +} + +void common_threadpools::init(llama_context * ctx, const common_params & params) { + GGML_ASSERT(!threadpool); + GGML_ASSERT(!threadpool_batch); + + COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads); + + auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + if (!cpu_dev) { + COM_WRN("%s", "no CPU backend found\n"); + return; + } + auto * reg = ggml_backend_dev_backend_reg(cpu_dev); + auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); + free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); + + struct ggml_threadpool_params tpp_batch = + ggml_threadpool_params_from_cpu_params(params.cpuparams_batch); + struct ggml_threadpool_params tpp = + ggml_threadpool_params_from_cpu_params(params.cpuparams); + + if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) { + threadpool_batch = ggml_threadpool_new_fn(&tpp_batch); + if (!threadpool_batch) { + COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads); + return; + } + + // start the non-batch threadpool in the paused state + tpp.paused = true; + } + + threadpool = ggml_threadpool_new_fn(&tpp); + if (!threadpool) { + COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads); + free_fn(threadpool_batch); + threadpool_batch = nullptr; + return; + } + + llama_attach_threadpool(ctx, threadpool, threadpool_batch); +} + // // Batch utils // diff --git a/common/common.h b/common/common.h index e6d5d892e..ebdc23d18 100644 --- a/common/common.h +++ b/common/common.h @@ -236,14 +236,14 @@ struct common_params_sampling { float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities float dynatemp_range = 0.00f; // 0.0 = disabled float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler - int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size) + int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty) float penalty_repeat = 1.00f; // 1.0 = disabled float penalty_freq = 0.00f; // 0.0 = disabled float penalty_present = 0.00f; // 0.0 = disabled float dry_multiplier = 0.0f; // 0.0 = disabled; DRY repetition penalty for tokens extending repetition: float dry_base = 1.75f; // 0.0 = disabled; multiplier * base ^ (length of sequence before token - allowed length) int32_t dry_allowed_length = 2; // tokens extending repetitions beyond this receive penalty - int32_t dry_penalty_last_n = -1; // how many tokens to scan for repetitions (0 = disable penalty, -1 = context size) + int32_t dry_penalty_last_n = 64; // how many tokens to scan for repetitions (0 = disable penalty) float adaptive_target = -1.0f; // select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) float adaptive_decay = 0.90f; // EMA decay for adaptation; history ≈ 1/(1-decay) tokens (0.0 - 0.99) int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0 @@ -393,14 +393,6 @@ struct common_params_speculative { } }; -struct common_params_vocoder { - struct common_params_model model; - - std::string speaker_file; // speaker file path - - bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy -}; - struct common_params_diffusion { int32_t steps = 128; bool visual_mode = false; @@ -456,6 +448,7 @@ struct common_params { int32_t n_parallel = 1; // number of parallel sequences to decode int32_t n_sequences = 1; // number of sequences to decode int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) + int32_t n_outputs_max_per_seq = 1; // max outputs per sequence int32_t grp_attn_n = 1; // group-attention factor int32_t grp_attn_w = 512; // group-attention width int32_t n_print = -1; // print token count every n tokens (-1 = disabled) @@ -481,7 +474,7 @@ struct common_params { std::vector fit_params_target = std::vector(llama_max_devices(), 1024 * 1024*1024); enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs - enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the model + enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model common_cpu_params cpuparams; common_cpu_params cpuparams_batch; @@ -498,7 +491,6 @@ struct common_params { struct common_params_sampling sampling; struct common_params_speculative speculative; - struct common_params_vocoder vocoder; struct common_params_diffusion diffusion; struct common_params_model model; @@ -665,6 +657,7 @@ struct common_params { // enable built-in tools std::vector server_tools; + std::string server_tools_runtime; // MCP server configs (Cursor-compatible JSON) std::string mcp_servers_config; // path to JSON file with MCP server definitions @@ -740,6 +733,12 @@ struct common_params { llama_progress_callback load_progress_callback = NULL; void * load_progress_callback_user_data = NULL; bool no_alloc = false; // Don't allocate model buffers + + // TTS params + std::string tts_lang = ""; + std::string tts_speaker_file = ""; + + bool is_gen_docs = false; // whether we are running inside llama-gen-docs }; // call once at the start of a program if it uses libcommon @@ -864,6 +863,15 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat bool glob_match(const std::string & pattern, const std::string & str); +// +// Environment utils +// + +// portable environment access, an unset variable reads as an empty string +// and setting an empty value unsets the variable +std::string common_get_env(const std::string & name); +void common_set_env(const std::string & name, const std::string & value); + // // Filesystem utils // @@ -874,6 +882,7 @@ bool fs_is_directory(const std::string & path); std::string fs_get_cache_directory(); std::string fs_get_cache_file(const std::string & filename); +std::string fs_get_config_directory(); struct common_file_info { std::string path; @@ -921,9 +930,8 @@ using common_init_result_ptr = std::unique_ptr; common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false); -struct llama_model_params common_model_params_to_llama ( common_params & params); -struct llama_context_params common_context_params_to_llama(const common_params & params); -struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params); +struct llama_model_params common_model_params_to_llama ( common_params & params); +struct llama_context_params common_context_params_to_llama(const common_params & params); // clear LoRA adapters from context, then apply new list of adapters void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora); @@ -934,6 +942,28 @@ std::string common_get_model_endpoint(); // for testing purposes char * common_get_model_or_exit(int, char*[]); +// +// Threadpool utils +// + +struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params); + +struct common_threadpools { + common_threadpools() = default; + ~common_threadpools(); + + common_threadpools(const common_threadpools &) = delete; + common_threadpools & operator=(const common_threadpools &) = delete; + + void init(llama_context * ctx, const common_params & params); + +private: + ggml_threadpool * threadpool = nullptr; + ggml_threadpool * threadpool_batch = nullptr; + + decltype(ggml_threadpool_free) * free_fn = nullptr; +}; + // // Context utils // diff --git a/common/download.cpp b/common/download.cpp index 3776c6c7e..44c6cea42 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -656,6 +656,12 @@ static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, return find_best_sibling(files, model, "dflash-", tag); } +static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files, + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "dspark-", tag); +} + static bool gguf_filename_is_model(const std::string & filepath) { if (!string_ends_with(filepath, ".gguf")) { return false; @@ -670,7 +676,8 @@ static bool gguf_filename_is_model(const std::string & filepath) { filename.find("imatrix") == std::string::npos && filename.find("mtp-") == std::string::npos && filename.find("eagle3-") == std::string::npos && - filename.find("dflash-") == std::string::npos; + filename.find("dflash-") == std::string::npos && + filename.find("dspark-") == std::string::npos; } static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, @@ -763,7 +770,7 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } else { primary = find_best_model(all, tag); // a requested sidecar can resolve on its own, without a full model of the same tag - if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) { + if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; @@ -787,9 +794,12 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & if (opts.download_eagle3) { plan.eagle3 = find_best_eagle3(all, primary.path, tag); } + if (opts.download_dspark) { + plan.dspark = find_best_dspark(all, primary.path, tag); + } if (primary.path.empty() && - plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) { + plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); } @@ -967,7 +977,8 @@ std::vector common_list_cached_models() { split.prefix.find("mmproj") != std::string::npos || split.prefix.find("mtp-") != std::string::npos || split.prefix.find("eagle3-") != std::string::npos || - split.prefix.find("dflash-") != std::string::npos) { + split.prefix.find("dflash-") != std::string::npos || + split.prefix.find("dspark-") != std::string::npos) { continue; } if (seen.insert(f.repo_id + ":" + split.tag).second) { diff --git a/common/download.h b/common/download.h index 6007c37fc..9da595d1f 100644 --- a/common/download.h +++ b/common/download.h @@ -60,6 +60,7 @@ struct common_download_opts { bool download_mtp = false; bool download_eagle3 = false; bool download_dflash = false; + bool download_dspark = false; common_download_callback * callback = nullptr; }; @@ -111,6 +112,7 @@ struct common_download_hf_plan { hf_cache::hf_file mtp; hf_cache::hf_file eagle3; hf_cache::hf_file dflash; + hf_cache::hf_file dspark; hf_cache::hf_file preset; // if set, only this file is downloaded }; common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); diff --git a/common/fit.cpp b/common/fit.cpp index c82d066ad..dd1f3ef76 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -136,7 +136,10 @@ static std::vector common_get_device_memory_data_impl( devs.push_back(llama_model_get_device(model, i)); } - hp_ngl = llama_model_n_layer(model) + llama_model_n_layer_nextn(model); + hp_ngl = llama_model_n_layer(model); + if (mparams->load_mtp) { + hp_ngl += llama_model_n_layer_nextn(model); + } hp_n_ctx_train = llama_model_n_ctx_train(model); hp_n_expert = llama_model_n_expert(model); diff --git a/common/imatrix-loader.cpp b/common/imatrix-loader.cpp index efe9aecee..71d3b500f 100644 --- a/common/imatrix-loader.cpp +++ b/common/imatrix-loader.cpp @@ -102,7 +102,8 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) { const int64_t chunk_count_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT); const int64_t chunk_size_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE); - if (datasets_key != -1 && gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) { + if (datasets_key != -1 && gguf_get_kv_type(ctx_gguf, datasets_key) == GGUF_TYPE_ARRAY && + gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) { const int64_t n = gguf_get_arr_n(ctx_gguf, datasets_key); imatrix.datasets.reserve(imatrix.datasets.size() + n); for (int64_t i = 0; i < n; ++i) { @@ -143,6 +144,13 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) { return false; } + if (in_sum2->type != GGML_TYPE_F32 || counts->type != GGML_TYPE_F32) { + LOG_ERR("%s: sums and counts for %s must be F32\n", __func__, name.c_str()); + gguf_free(ctx_gguf); + ggml_free(ctx); + return false; + } + auto & e = imatrix.entries[name]; const int64_t nval = ggml_nelements(in_sum2); diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index 26306bd91..00b02119a 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -20,7 +20,7 @@ namespace jinja { using caps_json_fn = std::function; using caps_ctx_fn = std::function; -using caps_analyze_fn = std::function; +using caps_analyze_fn = std::function; void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { ctx.set_val("preserve_thinking", mk_val(enabled)); @@ -29,6 +29,12 @@ void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { ctx.set_val("drop_thinking", mk_val(!enabled)); } +void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort) { + value var = mk_val(effort); // bind to the same value for stats + ctx.set_val("reasoning_effort", var); + ctx.set_val("reasoning_strength", var); +} + static void caps_try_execute(jinja::program & prog, const caps_json_fn & messages_fn, const caps_ctx_fn & ctx_fn, @@ -65,7 +71,7 @@ static void caps_try_execute(jinja::program & prog, // ignore exceptions during capability analysis } - analyze_fn(success, messages, tools, result); + analyze_fn(ctx, success, messages, tools, result); } // for debugging only @@ -90,6 +96,7 @@ std::map caps::to_map() const { {"supports_parallel_tool_calls", supports_parallel_tool_calls}, {"supports_system_role", supports_system_role}, {"supports_preserve_reasoning", supports_preserve_reasoning}, + {"supports_reasoning_effort", supports_reasoning_effort}, {"supports_object_arguments", supports_object_arguments}, }; } @@ -127,7 +134,7 @@ caps caps_get(jinja::program & prog) { }, nullptr, // ctx_fn nullptr, // tools_fn - [&](bool success, value & messages, value &, const std::string &) { + [&](context &, bool success, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (has_op(content, "selectattr") || has_op(content, "array_access")) { @@ -161,7 +168,7 @@ caps caps_get(jinja::program & prog) { }, nullptr, // ctx_fn nullptr, // tools_fn - [&](bool, value & messages, value &, const std::string &) { + [&](context &, bool, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (!content->stats.used) { @@ -237,7 +244,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](context &, bool success, value & messages, value & tools, const std::string &) { if (!success) { return; // Nothing can be inferred } @@ -330,7 +337,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](context &, bool success, value & messages, value & tools, const std::string &) { if (!success) { result.supports_tool_calls = false; result.supports_tools = false; @@ -432,7 +439,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value &, const std::string &) { + [&](context &, bool success, value & messages, value &, const std::string &) { if (!success) { result.supports_parallel_tool_calls = false; return; @@ -485,10 +492,11 @@ caps caps_get(jinja::program & prog) { }); }, [&](context & ctx) { + ctx.set_val("enable_thinking", mk_val(true)); caps_apply_preserve_reasoning(ctx, true); }, nullptr, // tools_fn - [&](bool, value &, value &, const std::string & output) { + [&](context &, bool, value &, value &, const std::string & output) { // note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result if (output.find(reasoning_placeholder) != std::string::npos) { result.supports_preserve_reasoning = true; @@ -496,6 +504,32 @@ caps caps_get(jinja::program & prog) { } ); + JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort"); + + // case: reasoning effort level + caps_try_execute( + prog, + [&]() { + // messages + return json::array({ + { + {"role", "user"}, + {"content", "User message"} + }, + }); + }, + [&](context & ctx) { + ctx.set_val("enable_thinking", mk_val(true)); + caps_apply_reasoning_effort(ctx, "low"); + }, + nullptr, // tools_fn + [&](context & ctx, bool, value &, value &, const std::string &) { + value effort = ctx.get_val("reasoning_effort"); + caps_print_stats(effort, "reasoning_effort"); + result.supports_reasoning_effort = effort->stats.used; + } + ); + JJ_DEBUG("%s\n", result.to_string().c_str()); return result; diff --git a/common/jinja/caps.h b/common/jinja/caps.h index a290cd7da..b81dd95f2 100644 --- a/common/jinja/caps.h +++ b/common/jinja/caps.h @@ -16,6 +16,9 @@ struct caps { // supports preserve reasoning trace in the full history, not just the last assistant message bool supports_preserve_reasoning = false; + // supports reasoning effort levels + bool supports_reasoning_effort = false; + // one of the 2 content capabilities must be true bool supports_string_content = true; bool supports_typed_content = false; @@ -32,5 +35,6 @@ struct caps { caps caps_get(jinja::program & prog); void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled); +void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort); } // namespace jinja diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index afba0025f..9cc5cfa46 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -266,7 +266,7 @@ value binary_expression::execute_impl(context & ctx) { return res; } for (int64_t i = 0; i < repeat; ++i) { - res->val_str = res->val_str.append(str); + res->val_str.append(str); } return res; } diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 0884a1592..69bd683c6 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -763,14 +763,22 @@ struct runtime { gather_string_parts_recursive(val, parts); // join consecutive parts with the same type auto & p = parts->val_str.parts; - for (size_t i = 1; i < p.size(); ) { - if (p[i].is_input == p[i - 1].is_input) { - p[i - 1].val += p[i].val; - p.erase(p.begin() + i); + if (p.empty()) { + return parts; + } + size_t w = 0; + for (size_t r = 1; r < p.size(); r++) { + if (p[w].is_input == p[r].is_input) { + p[w].val += p[r].val; } else { - i++; + w++; + if (w != r) { + // the guard is needed, self-move leaves the string in an unspecified state + p[w] = std::move(p[r]); + } } } + p.resize(w + 1); return parts; } diff --git a/common/jinja/string.cpp b/common/jinja/string.cpp index 8087e15b3..bde679e4e 100644 --- a/common/jinja/string.cpp +++ b/common/jinja/string.cpp @@ -103,7 +103,7 @@ void string::mark_input_based_on(const string & other) { } } -string string::append(const string & other) { +string & string::append(const string & other) { for (const auto & part : other.parts) { parts.push_back(part); } diff --git a/common/jinja/string.h b/common/jinja/string.h index c4963000a..669afb8f1 100644 --- a/common/jinja/string.h +++ b/common/jinja/string.h @@ -47,7 +47,7 @@ struct string { // mark this string as input if other has ALL parts as input void mark_input_based_on(const string & other); - string append(const string & other); + string & append(const string & other); // in-place transformations diff --git a/common/llguidance.cpp b/common/llguidance.cpp index d58f147a7..500bb0914 100644 --- a/common/llguidance.cpp +++ b/common/llguidance.cpp @@ -116,6 +116,8 @@ static llama_sampler_i llama_sampler_llg_i = { /* .backend_accept = */ NULL, /* .backend_apply = */ NULL, /* .backend_set_input = */ NULL, + /* .backend_reset = */ NULL, + /* .copy_state = */ NULL, }; static size_t llama_sampler_llg_tokenize_fn(const void * user_data, const uint8_t * bytes, size_t bytes_len, diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index ef290ed7c..4a4be7cf7 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -570,23 +570,34 @@ struct parser_executor { } static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos, const char delimiter) { + auto save = pos; + ++pos; // consume '\' if (pos >= ctx.input.size()) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } + pos = save; // suppress unmatched '\' return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } char c = ctx.input[pos]; + if (c == delimiter || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') { ++pos; return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos); - } else if (c == 'u') { - return handle_unicode_escape(ctx, start, pos); - } else { - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } + + if (c == 'u') { + auto result = handle_unicode_escape(ctx, start, pos); + if (result.need_more_input()) { + pos = save; // suppress incomplete sequence + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); + } + return result; + } + + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } static common_peg_parse_result handle_unicode_escape(common_peg_parse_context & ctx, size_t start, size_t & pos) { diff --git a/common/preset.cpp b/common/preset.cpp index eb0c60b09..4c61e93ee 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -322,6 +322,8 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co preset.options[opt] = value; } LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str()); + } else if (ignore_unknown_keys) { + LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.c_str()); } else { throw std::runtime_error(string_format( "option '%s' not recognized in preset '%s'", @@ -363,8 +365,25 @@ struct local_model { std::string name; std::string path; std::string path_mmproj; + std::string path_draft; }; +// TODO @ngxson: handle "eagle3-" when it's supported by common_speculative_types_from_gguf() +static const char * draft_prefixes[] = { "mtp-", "dspark-", "dflash-" }; + +static bool is_mmproj_file(const std::string & fname) { + return fname.find("mmproj") != std::string::npos; +} + +static bool is_draft_file(const std::string & fname) { + for (const auto & prefix : draft_prefixes) { + if (fname.rfind(prefix, 0) == 0) { + return true; + } + } + return false; +} + common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const { if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str())); @@ -376,10 +395,15 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m common_file_info model_file; common_file_info first_shard_file; common_file_info mmproj_file; + common_file_info draft_file; for (const auto & file : files) { if (string_ends_with(file.name, ".gguf")) { - if (file.name.find("mmproj") != std::string::npos) { + if (is_mmproj_file(file.name)) { mmproj_file = file; + } else if (is_draft_file(file.name)) { + if (draft_file.path.empty()) { + draft_file = file; // first sidecar found wins + } } else if (file.name.find("-00001-of-") != std::string::npos) { first_shard_file = file; } else { @@ -391,7 +415,8 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m local_model model{ /* name */ name, /* path */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path, - /* path_mmproj */ mmproj_file.path // can be empty + /* path_mmproj */ mmproj_file.path, // can be empty + /* path_draft */ draft_file.path // can be empty }; if (!model.path.empty()) { models.push_back(model); @@ -403,13 +428,17 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m if (file.is_dir) { scan_subdir(file.path, file.name); } else if (string_ends_with(file.name, ".gguf")) { + if (is_mmproj_file(file.name) || is_draft_file(file.name)) { + continue; // companion file, cannot be loaded as a model on its own + } // single file model std::string name = file.name; string_replace_all(name, ".gguf", ""); local_model model{ /* name */ name, /* path */ file.path, - /* path_mmproj */ "" + /* path_mmproj */ "", + /* path_draft */ "" }; models.push_back(model); } @@ -424,6 +453,9 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m if (!model.path_mmproj.empty()) { preset.set_option(*this, "LLAMA_ARG_MMPROJ", model.path_mmproj); } + if (!model.path_draft.empty()) { + preset.set_option(*this, "LLAMA_ARG_SPEC_DRAFT_MODEL", model.path_draft); + } out[preset.name] = preset; } diff --git a/common/preset.h b/common/preset.h index 52935ebde..d8fc3915b 100644 --- a/common/preset.h +++ b/common/preset.h @@ -59,6 +59,10 @@ struct common_preset_context { bool filter_allowed_keys = false; std::set allowed_keys; + // if true, options unknown to the current example are skipped instead of being an error + // used for config files shared by all binaries, where each binary only knows a subset of options + bool ignore_unknown_keys = false; + // if only_remote_allowed is true, only accept whitelisted keys common_preset_context(llama_example ex); diff --git a/common/reasoning-budget.cpp b/common/reasoning-budget.cpp index 1fe242d06..4884299f3 100644 --- a/common/reasoning-budget.cpp +++ b/common/reasoning-budget.cpp @@ -217,6 +217,8 @@ static struct llama_sampler_i common_reasoning_budget_i = { /* .backend_accept = */ nullptr, /* .backend_apply = */ nullptr, /* .backend_set_input = */ nullptr, + /* .backend_reset = */ nullptr, + /* .copy_state = */ nullptr, }; static struct llama_sampler * common_reasoning_budget_clone(const struct llama_sampler * smpl) { diff --git a/common/sampling.cpp b/common/sampling.cpp index 256ac161e..06dea1e1c 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -184,9 +184,21 @@ std::string common_params_sampling::print() const { return std::string(result); } -struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params) { +struct common_sampler * common_sampler_init( + const struct llama_model * model, + struct common_params_sampling & params) { + if (!std::isfinite(params.penalty_repeat) || + params.penalty_repeat <= 0.0f || + !std::isfinite(1.0f/params.penalty_repeat)) { + throw std::invalid_argument("penalty_repeat must be finite and greater than 0"); + } + if (!std::isfinite(params.penalty_freq)) { + throw std::invalid_argument("penalty_freq must be finite"); + } + if (!std::isfinite(params.penalty_present)) { + throw std::invalid_argument("penalty_present must be finite"); + } const llama_vocab * vocab = llama_model_get_vocab(model); - llama_sampler_chain_params lparams = llama_sampler_chain_default_params(); lparams.no_perf = params.no_perf; @@ -338,7 +350,7 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st for (const auto & str : params.dry_sequence_breakers) { c_breakers.push_back(str.c_str()); } - samplers.push_back(llama_sampler_init_dry(vocab, llama_model_n_ctx_train(model), params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size())); + samplers.push_back(llama_sampler_init_dry(vocab, params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size())); } break; case COMMON_SAMPLER_TYPE_TOP_K: @@ -366,7 +378,7 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st samplers.push_back(llama_sampler_init_infill(vocab)); break; case COMMON_SAMPLER_TYPE_PENALTIES: - samplers.push_back(llama_sampler_init_penalties(params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present)); + samplers.push_back(llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present)); break; case COMMON_SAMPLER_TYPE_ADAPTIVE_P: // the `adaptive-p` sampler is like `dist` and `mirostat` in that it selects @@ -506,6 +518,26 @@ struct common_sampler * common_sampler_clone(common_sampler * gsmpl) { }; } +void common_sampler_copy(const common_sampler * src, common_sampler * dst) { + if (!src || !dst || src == dst) { + return; + } + + GGML_ASSERT((src->grmr == nullptr) == (dst->grmr == nullptr)); + GGML_ASSERT((src->rbudget == nullptr) == (dst->rbudget == nullptr)); + + llama_sampler_copy(src->grmr, dst->grmr); + llama_sampler_copy(src->rbudget, dst->rbudget); + llama_sampler_copy(src->chain, dst->chain); + + dst->params = src->params; + dst->prev = src->prev; + dst->cur = src->cur; + dst->cur_p = src->cur_p; + dst->cur_p.data = src->cur_p.data ? dst->cur.data() : nullptr; // re-point to dst's buffer + dst->t_total_us = src->t_total_us; +} + void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl) { // TODO: measure grammar performance diff --git a/common/sampling.h b/common/sampling.h index 4191988bb..ced3c8364 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -37,7 +37,9 @@ struct common_sampler; // llama_sampler API overloads // note: can mutate params in some cases -struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params); +struct common_sampler * common_sampler_init( + const struct llama_model * model, + struct common_params_sampling & params); void common_sampler_free(struct common_sampler * gsmpl); @@ -45,6 +47,7 @@ void common_sampler_free(struct common_sampler * gsmpl); void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool is_generated); void common_sampler_reset (struct common_sampler * gsmpl); struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl); +void common_sampler_copy (const struct common_sampler * src, struct common_sampler * dst); // arguments can be nullptr to skip printing void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl); diff --git a/common/speculative.cpp b/common/speculative.cpp index b91974c11..aec94199b 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2,6 +2,7 @@ #include "common.h" #include "ggml.h" +#include "ggml-cpp.h" #include "llama.h" #include "log.h" #include "ngram-cache.cpp" @@ -171,12 +172,6 @@ struct common_speculative_impl { // (optional) serialize/restore per-seq internal state (e.g. eagle3's deferred boundary). virtual bool get_state(llama_seq_id /*seq_id*/, std::vector & /*data*/) const { return false; } virtual void set_state(llama_seq_id /*seq_id*/, const std::vector & /*data*/) {} - - // true if this implementation requires the target context to extract post-norm embeddings - virtual bool need_embd() const = 0; - - // true if this implementation requires the target context to extract pre-norm embeddings - virtual bool need_embd_nextn() const { return false; } }; struct common_speculative_impl_draft_simple : public common_speculative_impl { @@ -193,6 +188,10 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; auto * ctx_tgt = this->params.ctx_tgt; + if (!ctx_dft) { + throw std::runtime_error("draft-simple requires a draft context"); + } + SPC_TRC("%s", "adding speculative implementation 'draft-simple'\n"); SPC_TRC("- n_max=%d, n_min=%d, p_min=%f\n", this->params.n_max, this->params.n_min, this->params.p_min); SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", @@ -385,10 +384,6 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } - - bool need_embd() const override { - return false; - } }; @@ -907,10 +902,6 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { pending_g_last[seq_id].resize(n_embd_dec); std::memcpy(pending_g_last[seq_id].data(), data.data() + sizeof(llama_pos), (size_t) n_embd_dec * sizeof(float)); } - - bool need_embd() const override { - return false; - } }; // DFlash: block-diffusion drafting with a draft-side KV cache injection @@ -922,6 +913,9 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { std::vector smpls; + // backend sampler chain per seq, attached to ctx_dft + std::vector backend_chains; + int32_t n_embd_dec = 0; // draft hidden size int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size int32_t n_embd_tgt = 0; // target model hidden size @@ -995,6 +989,22 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { s.reset(common_sampler_init(model_dft, sparams)); } + // offload draft sampling to the backend + backend_chains.assign(n_seq, nullptr); + if (this->params.backend_sampling) { + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); + llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); + + if (!llama_set_sampler(ctx_dft, seq_id, chain)) { + SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); + llama_sampler_free(chain); + chain = nullptr; + } + backend_chains[seq_id] = chain; + } + } + // turn on extraction of the target layers' input embeddings for (uint32_t k = 0; k < target_layer_ids_n; ++k) { llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); @@ -1005,6 +1015,18 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { } ~common_speculative_impl_draft_dflash() override { + auto * ctx_dft = this->params.ctx_dft; + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) { + if (backend_chains[seq_id] == nullptr) { + continue; + } + if (ctx_dft) { + llama_set_sampler(ctx_dft, seq_id, nullptr); + } + llama_sampler_free(backend_chains[seq_id]); + } + backend_chains.clear(); + llama_batch_free(batch); llama_batch_free(batch_inject); } @@ -1032,7 +1054,14 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { return true; } - if (batch_in.token == nullptr || batch_in.embd != nullptr) { + // Target prefill may contain token IDs or multimodal embeddings. Both + // produce the target-layer features used to seed the draft KV cache, so + // skipping the embedding batches leaves a hole in the draft's cache and + // the next injection fails to initialize. + // TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged + const bool has_tokens = batch_in.token != nullptr; + const bool has_embeddings = batch_in.embd != nullptr; + if (has_tokens == has_embeddings) { return true; } @@ -1240,10 +1269,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } - - bool need_embd() const override { - return false; - } }; struct common_speculative_impl_draft_mtp : public common_speculative_impl { @@ -1291,7 +1316,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { GGML_ASSERT(ctx_tgt && ctx_dft && "MTP requires ctx_tgt and ctx_dft to be set"); n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft)); - GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) && + GGML_ASSERT(n_embd == llama_model_n_embd_out(llama_get_model(ctx_tgt)) && "MTP input row width must match the target h_nextn width"); n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); @@ -1682,14 +1707,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const size_t row_bytes = (size_t) n_embd * sizeof(float); std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes); } - - bool need_embd() const override { - return false; - } - - bool need_embd_nextn() const override { - return true; - } }; // state of self-speculation (simple implementation, not ngram-map) @@ -1736,10 +1753,6 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } - - bool need_embd() const override { - return false; - } }; struct common_speculative_impl_ngram_map_k : public common_speculative_impl { @@ -1794,10 +1807,6 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl { common_ngram_map_accept(config[seq_id], n_accepted); } - - bool need_embd() const override { - return false; - } }; struct common_speculative_impl_ngram_mod : public common_speculative_impl { @@ -1973,10 +1982,6 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { } } } - - bool need_embd() const override { - return false; - } }; struct common_speculative_impl_ngram_cache : public common_speculative_impl { @@ -2116,10 +2121,6 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } - - bool need_embd() const override { - return false; - } }; struct common_speculative { @@ -2227,6 +2228,43 @@ common_speculative_type common_speculative_type_from_name(const std::string & na return it->second; } +std::vector common_speculative_types_from_gguf(const std::string & path) { + struct gguf_init_params gguf_params = { + /* .no_alloc = */ true, + /* .ctx = */ nullptr, + }; + + gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params)); + if (!gguf_ctx) { + return {}; + } + + const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture"); + if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) { + return {}; + } + + const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id); + if (arch != "dflash") { + const uint32_t block_count = gguf_get_val_u32(gguf_ctx.get(), gguf_find_key(gguf_ctx.get(), (arch + ".block_count").c_str())); + + if (gguf_find_tensor(gguf_ctx.get(), ("blk." + std::to_string(block_count - 1) + ".nextn.eh_proj.weight").c_str()) >= 0) { + return { COMMON_SPECULATIVE_TYPE_DRAFT_MTP }; + } + + return {}; + } + + // the Markov head distinguishes draft-dspark from draft-dflash + const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0 + ? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK + : COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH; + + SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str()); + + return { type }; +} + static uint32_t common_get_enabled_speculative_configs(const std::vector & configs) { uint32_t result = 0; for (size_t i = 0; i < configs.size(); i++) { @@ -2292,6 +2330,24 @@ common_params common_base_params_to_speculative(const common_params & params) { result.cache_type_k = params_spec.cache_type_k; result.cache_type_v = params_spec.cache_type_v; result.n_outputs_max = params.n_parallel; + result.n_outputs_max_per_seq = 1; + + // dflash/dspark decode the whole noise block in a single pass and sample every block position on the backend + // TODO: refactor such properties to be announced by the speculative types + // something like `struct common_speculative_type_props common_speculative_type_get_props(...);` + const bool has_block_draft = std::any_of( + params.speculative.types.begin(), params.speculative.types.end(), + [](common_speculative_type t) { + return t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; + }); + if (has_block_draft) { + // per-seq output positions: DFlash decodes anchor + n_max masks (n_max + 1); DSpark n_max -> +1 covers both + const int32_t per_seq = std::max(1, params_spec.n_max + 1); + result.n_outputs_max = params.n_parallel * per_seq; + if (params_spec.backend_sampling) { + result.n_outputs_max_per_seq = per_seq; + } + } return result; } @@ -2314,7 +2370,6 @@ common_speculative_init_result::common_speculative_init_result( const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); - GGML_ASSERT(has_draft || spec_mtp); auto mparams = common_model_params_to_llama(params); auto cparams = common_context_params_to_llama(params); @@ -2377,6 +2432,17 @@ common_speculative_init_result_ptr common_speculative_init_from_params(common_pa return std::make_unique(params, model_tgt, ctx_tgt); } +common_speculative_output_limits common_speculative_get_output_limits( + int32_t n_batch, int32_t n_parallel, int32_t n_draft) { + const int64_t per_seq = 1 + (int64_t) std::max(0, n_draft); + const int64_t total = (int64_t) n_parallel * per_seq; + + return { + /* .total = */ (int32_t) std::min(n_batch, total), + /* .per_seq = */ (int32_t) std::min(n_batch, per_seq), + }; +} + // initialization of the speculative decoding system // common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) { @@ -2385,57 +2451,28 @@ common_speculative * common_speculative_init(common_params_speculative & params, { uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types); - bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); - bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr; - bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; - bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr; - bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr; - - - - bool has_ngram_cache = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_CACHE)); - bool has_ngram_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE)); - bool has_ngram_map_k = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K)); - bool has_ngram_map_k4v = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V)); - bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); + auto add_config_if_enabled = [&](common_speculative_type type, bool available = true) { + if (available && (enabled_configs & (1u << type))) { + configs.emplace_back(type, params); + } + }; // when adding a new type - update here the logic above static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11); // this list here defines the priority of the speculators // the one with highest priority are listed first - if (has_ngram_simple) { - // This implementation can guess a lot of tokens without any draft model. - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, params)); - } - if (has_ngram_map_k) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, params)); - } - if (has_ngram_map_k4v) { - // This implementation can guess tokens with high acceptance rate but is more expensive. - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, params)); - } - if (has_ngram_mod) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, params)); - } - if (has_ngram_cache) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params)); - } - if (has_draft_simple) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params)); - } - if (has_draft_eagle3) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params)); - } - if (has_draft_mtp) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); - } - if (has_draft_dflash) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params)); - } - if (has_draft_dspark) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params)); - } + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MOD); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE); + + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params.draft.ctx_dft != nullptr); + add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params.draft.ctx_dft != nullptr); } std::vector> impls = {}; @@ -2570,34 +2607,6 @@ bool common_speculative_process(common_speculative * spec, const llama_batch & b return result; } -bool common_speculative_need_embd(common_speculative * spec) { - if (spec == nullptr) { - return false; - } - - for (auto & impl : spec->impls) { - if (impl->need_embd()) { - return true; - } - } - - return false; -} - -bool common_speculative_need_embd_nextn(common_speculative * spec) { - if (spec == nullptr) { - return false; - } - - for (auto & impl : spec->impls) { - if (impl->need_embd_nextn()) { - return true; - } - } - - return false; -} - void common_speculative_draft(common_speculative * spec) { if (spec == nullptr) { return; @@ -2682,7 +2691,10 @@ void common_speculative_draft(common_speculative * spec) { void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, uint16_t n_accepted) { common_speculative_impl * impl = spec->impl_last[seq_id]; - GGML_ASSERT(impl); + if (impl == nullptr) { + GGML_ASSERT(n_accepted == 0); + return; + } { common_time_meas tm(impl->t_accept_us, !impl->gen_perf); diff --git a/common/speculative.h b/common/speculative.h index 062bf2093..12ae31b7d 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -14,6 +14,9 @@ const char * common_speculative_all_types_str(); // parse user provided types std::vector common_speculative_types_from_names(const std::vector & names); +// infer the spec types from the GGUF metadata of a draft model; empty if unknown +std::vector common_speculative_types_from_gguf(const std::string & path); + // convert string to type enum common_speculative_type common_speculative_type_from_name(const std::string & name); @@ -25,6 +28,15 @@ int32_t common_speculative_n_max(const common_params_speculative * spec); common_params common_base_params_to_speculative(const common_params & params); +struct common_speculative_output_limits { + int32_t total; + int32_t per_seq; +}; + +// return the output limits needed for speculative decoding +common_speculative_output_limits common_speculative_get_output_limits( + int32_t n_batch, int32_t n_parallel, int32_t n_draft); + common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq); void common_speculative_free(common_speculative * spec); @@ -58,12 +70,6 @@ void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, co // process the batch and update the internal state of the speculative context bool common_speculative_process(common_speculative * spec, const llama_batch & batch); -// true if any implementation requires target post-norm embeddings to be extracted -bool common_speculative_need_embd(common_speculative * spec); - -// true if any implementation requires target nextn embeddings to be extracted -bool common_speculative_need_embd_nextn(common_speculative * spec); - // generate drafts for the sequences specified with `common_speculative_get_draft_params` void common_speculative_draft(common_speculative * spec); diff --git a/conversion/__init__.py b/conversion/__init__.py index 1a47b851a..f4d475de7 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -55,6 +55,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "DFlashDraftModel": "qwen", "Qwen3DSparkModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", + "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", @@ -69,6 +70,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "Exaone4ForCausalLM": "exaone", "ExaoneForCausalLM": "exaone", "ExaoneMoEForCausalLM": "exaone", + "ExaoneMoeForCausalLM": "exaone", "FalconForCausalLM": "falcon", "FalconH1ForCausalLM": "falcon_h1", "FalconMambaForCausalLM": "mamba", @@ -101,6 +103,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "GraniteMoeForCausalLM": "granite", "GraniteMoeHybridForCausalLM": "granite", "GraniteMoeSharedForCausalLM": "granite", + "GraniteSwitchForCausalLM": "granite", "GraniteSpeechForConditionalGeneration": "granite", "GraniteSpeechPlusForConditionalGeneration": "granite", "Grok1ForCausalLM": "grok", @@ -158,6 +161,8 @@ TEXT_MODEL_MAP: dict[str, str] = { "MiniCPM3ForCausalLM": "minicpm", "MiniCPMForCausalLM": "minicpm", "MiniCPMV4_6ForConditionalGeneration": "minicpm", + "MiniMaxText01ForCausalLM": "minimax", + "MiniMaxM1ForCausalLM": "minimax", "MiniMaxM2ForCausalLM": "minimax", "MiniMaxM3SparseForCausalLM": "minimax", "MiniMaxM3SparseForConditionalGeneration": "minimax", @@ -180,6 +185,8 @@ TEXT_MODEL_MAP: dict[str, str] = { "Olmo3ForCausalLM": "olmo", "OlmoForCausalLM": "olmo", "OlmoeForCausalLM": "olmo", + "MuseGlimmerAssistantModel": "muse_glimmer", + "MuseGlimmerForConditionalGeneration": "muse_glimmer", "OpenELMForCausalLM": "openelm", "OrionForCausalLM": "orion", "PLMForCausalLM": "plm", @@ -209,6 +216,8 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3MoeForCausalLM": "qwen", "Qwen3NextForCausalLM": "qwen", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", + "PocketTTSModel": "pockettts", + "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForCausalLM": "qwen", @@ -294,6 +303,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "MiniCPMV4_6ForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "NemotronH_Nano_VL_V2": "nemotron", + "MuseGlimmerForConditionalGeneration": "muse_glimmer", "PaddleOCRVisionModel": "ernie", "Phi4ForCausalLMV": "phi", "Qwen2AudioForConditionalGeneration": "ultravox", @@ -303,6 +313,8 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen2_5_VLForConditionalGeneration": "qwenvl", "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", + "PocketTTSModel": "pockettts", + "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForConditionalGeneration": "qwen3vl", diff --git a/conversion/base.py b/conversion/base.py index a7cd3fd90..718d53944 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -58,6 +58,11 @@ logger = logging.getLogger("hf-to-gguf") AnyModel = TypeVar("AnyModel", bound="type[ModelBase]") +# for checkpoints that ship no config.json, we will try to provide a synthetic one +HparamsMatcher = Callable[[Path], bool] +HparamsLoader = Callable[[Path], dict[str, Any]] + + class SentencePieceTokenTypes(IntEnum): NORMAL = 1 UNKNOWN = 2 @@ -77,6 +82,7 @@ class ModelBase: ModelType.TEXT: {}, ModelType.MMPROJ: {}, } + _hparams_loaders: list[tuple[HparamsMatcher, HparamsLoader]] = [] dir_model: Path ftype: gguf.LlamaFileType @@ -823,7 +829,7 @@ class ModelBase: elif any(str(v.get("quant_algo")).endswith("NVFP4") for v in quant_layers.values() if isinstance(v, dict)): quant_algo = "NVFP4" - self._is_nvfp4 = quant_algo == "NVFP4" + self._is_nvfp4 = quant_algo in ("NVFP4", "W4A16_NVFP4") self._is_mxfp4 = quant_method == "mxfp4" # NVFP4 weights are repacked and written directly to gguf_writer. @@ -1040,6 +1046,24 @@ class ModelBase: return part_names + @staticmethod + def load_hparams_guess(dir_model: Path) -> dict[str, Any] | None: + # some models ship no config.json, will try to guess them + from conversion import load_all_models + load_all_models() + + for matcher, loader in ModelBase._hparams_loaders: + if matcher(dir_model): + return loader(dir_model) + return None + + @classmethod + def register_hparams_loader(cls, matcher: HparamsMatcher) -> Callable[[HparamsLoader], HparamsLoader]: + def inner(loader: HparamsLoader) -> HparamsLoader: + cls._hparams_loaders.append((matcher, loader)) + return loader + return inner + @staticmethod def load_hparams(dir_model: Path, is_mistral_format: bool): if is_mistral_format: @@ -1053,6 +1077,10 @@ class ModelBase: config = AutoConfig.from_pretrained(dir_model, trust_remote_code=False).to_dict() except Exception as e: logger.warning(f"Failed to load model config from {dir_model}: {e}") + if not (dir_model / "config.json").is_file(): + config = ModelBase.load_hparams_guess(dir_model) + if config is not None: + return config logger.warning("Trying to load config.json instead") with open(dir_model / "config.json", "r", encoding="utf-8") as f: config = json.load(f) diff --git a/conversion/chatglm.py b/conversion/chatglm.py index 801913075..d63855038 100644 --- a/conversion/chatglm.py +++ b/conversion/chatglm.py @@ -81,7 +81,7 @@ class ChatGLMModel(TextModel): @staticmethod def token_bytes_to_string(b): - from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] + from transformers.convert_slow_tokenizer import bytes_to_unicode byte_encoder = bytes_to_unicode() return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index ea6ae23d5..1846ca401 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -17,8 +17,11 @@ from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logg from .qwen import QwenModel -@ModelBase.register("DeepseekOCRForCausalLM", "UnlimitedOCRForCausalLM") +@ModelBase.register("DeepseekOCRForCausalLM") class DeepseekOCRVisionModel(MmprojModel): + # HF dynamic_preprocess() max_num, which differs per model + preproc_max_tiles = 9 + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR @@ -43,6 +46,9 @@ class DeepseekOCRVisionModel(MmprojModel): # @bluebread: there's no window_size in config but just add it here anyway self.gguf_writer.add_vision_window_size(self.hparams.get("window_size", 14)) + self.gguf_writer.add_vision_preproc_min_tiles(2) + self.gguf_writer.add_vision_preproc_max_tiles(self.preproc_max_tiles) + # SAM configuration sam_hparams = hparams['sam'] self.gguf_writer.add_vision_sam_layers_count(sam_hparams['layers']) @@ -93,8 +99,15 @@ class DeepseekOCRVisionModel(MmprojModel): return super().filter_tensors((name, gen)) +@ModelBase.register("UnlimitedOCRForCausalLM") +class UnlimitedOCRVisionModel(DeepseekOCRVisionModel): + preproc_max_tiles = 32 + + @ModelBase.register("DeepseekOCR2ForCausalLM") class DeepseekOCR2VisionModel(DeepseekOCRVisionModel): + preproc_max_tiles = 6 + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2 @@ -447,12 +460,43 @@ class DeepseekV2Model(TextModel): class DeepseekV32Model(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.DEEPSEEK32 skip_mtp = False + supports_mtp_export = True + _n_main_layers: int | None = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0) + self.block_count = self.hparams["num_hidden_layers"] + if not self.no_mtp: + self.block_count += self.hparams.get("num_nextn_predict_layers", 0) self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + def index_tensors(self, remote_hf_model_id: str | None = None): + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + # DeepSeek V3.2 appends the NextN/MTP block past num_hidden_layers + # (model.layers.61 -> blk.61 in the 62-block file). + assert cls._n_main_layers is not None + is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers + + # --no-mtp: drop the appended NextN block entirely. + if is_mtp and cls.no_mtp: + return None + # --mtp: keep ONLY NextN-block tensors plus the shared embeddings/ + # norm/lm_head (so the resulting GGUF carries just the draft head). + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) @@ -463,7 +507,7 @@ class DeepseekV32Model(DeepseekV2Model): super().set_gguf_parameters() # NextN/MTP prediction layers - if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: + if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) # DSA indexer parameters @@ -475,7 +519,10 @@ class DeepseekV32Model(DeepseekV2Model): @ModelBase.register("DeepseekV4ForCausalLM") class DeepseekV4Model(TextModel): model_arch = gguf.MODEL_ARCH.DEEPSEEK4 + supports_mtp_export = True _skipped_mtp_tensors = 0 + _dsv4_main_layers: int | None = None + _dsv4_nextn_layers: int = 0 def __init__(self, *args, **kwargs): type(self)._skipped_mtp_tensors = 0 @@ -486,7 +533,16 @@ class DeepseekV4Model(TextModel): for key, value in raw_hparams.items(): self.hparams.setdefault(key, value) + # workaround for special rope_parameters (main/compress) in transformers 5.x + if self.rope_parameters.get("full_attention", self.rope_parameters).get("rope_type") is None: + if (rope_scaling := raw_hparams.get("rope_scaling")) is not None: + if "rope_type" not in rope_scaling and (rope_type := rope_scaling.get("type")) is not None: + rope_scaling["rope_type"] = rope_type + self.rope_parameters.update(**rope_scaling) + self.block_count = self.hparams["num_hidden_layers"] + if self.mtp_only: + self.block_count += self.hparams.get("num_nextn_predict_layers", 0) self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) self._dsv4_fp8_dequantized: set[str] = set() @@ -499,18 +555,71 @@ class DeepseekV4Model(TextModel): logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors) # add a default chat template; if the model has a built-in template, it will be overridden later - template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja" + model_id_hint = self.remote_hf_model_id or self.dir_model.name + is_0731 = "0731" in model_id_hint + template_name = "deepseek-ai-DeepSeek-V4-Flash-0731.jinja" if is_0731 else "deepseek-ai-DeepSeek-V4.jinja" + template_path = Path(__file__).parent.parent / "models" / "templates" / template_name if template_path.is_file(): with open(template_path, "r", encoding="utf-8") as f: self.gguf_writer.add_chat_template(f.read()) + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + type(self)._dsv4_main_layers = self.hparams["num_hidden_layers"] + type(self)._dsv4_nextn_layers = self.hparams.get("num_nextn_predict_layers", 0) + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, _ = item + name, gen = item if name.startswith("mtp."): - cls._skipped_mtp_tensors += 1 - return None - return super().filter_tensors(item) + if not cls.mtp_only: + cls._skipped_mtp_tensors += 1 + return None + + assert cls._dsv4_main_layers is not None + parts = name.split(".", 2) + if len(parts) < 3 or not parts[1].isdecimal(): + raise ValueError(f"Unexpected DeepSeek-V4 MTP tensor {name!r}") + + mtp_idx = int(parts[1]) + if mtp_idx >= cls._dsv4_nextn_layers: + raise ValueError(f"Unexpected DeepSeek-V4 MTP layer {mtp_idx}") + + bid = cls._dsv4_main_layers + mtp_idx + suffix = parts[2] + root_hc_head = { + "hc_head_fn", + "hc_head_base", + "hc_head_scale", + } + if suffix in root_hc_head: + name = suffix + elif suffix in ( + "e_proj.weight", "e_proj.scale", + "h_proj.weight", "h_proj.scale", + ): + name = f"layers.{bid}.nextn.{suffix}" + elif suffix == "enorm.weight": + name = f"layers.{bid}.nextn.enorm.weight" + elif suffix == "hnorm.weight": + name = f"layers.{bid}.nextn.hnorm.weight" + elif suffix == "norm.weight": + name = f"layers.{bid}.nextn.shared_head_norm.weight" + else: + name = f"layers.{bid}.{suffix}" + return name, gen + + if cls.mtp_only: + keep = name in ( + "embed.weight", + "norm.weight", + "head.weight", + "head.scale", + ) + if not keep: + return None + + return super().filter_tensors((name, gen)) @staticmethod def _float8_dtypes() -> tuple[torch.dtype, ...]: @@ -565,6 +674,10 @@ class DeepseekV4Model(TextModel): self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) + if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4: + self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"]) + if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0: + self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) def dequant_model(self): fp8_dtypes = self._float8_dtypes() @@ -669,12 +782,37 @@ class DeepseekV4Model(TextModel): if self._dsv4_mxfp4_generated: return () - consumed: list[str] = self._write_hash_routing_tensors() + consumed: list[str] = [] + main_layers = self.hparams["num_hidden_layers"] + if not self.mtp_only: + consumed.extend(self._write_hash_routing_tensors()) + elif self.hparams["num_hash_layers"] > 0: + for bid in range(self.hparams["num_hash_layers"]): + name = f"layers.{bid}.ffn.gate.tid2eid" + if name in self.model_tensors: + consumed.extend(self._write_hash_routing_tensors()) + break + for bid in range(self.block_count): + if self.mtp_only and bid < main_layers: + continue consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)) consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)) consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)) + for bid in range(main_layers, self.block_count): + e_name = f"layers.{bid}.nextn.e_proj.weight" + h_name = f"layers.{bid}.nextn.h_proj.weight" + if e_name not in self.model_tensors and h_name not in self.model_tensors: + continue + if e_name not in self.model_tensors or h_name not in self.model_tensors: + raise KeyError(f"Missing DeepSeek-V4 MTP e/h projection pair for block {bid}") + + e_proj = LazyTorchTensor.to_eager(self.model_tensors[e_name]()) + h_proj = LazyTorchTensor.to_eager(self.model_tensors[h_name]()) + yield (f"layers.{bid}.nextn.eh_proj.weight", torch.cat((e_proj, h_proj), dim=1).contiguous()) + consumed.extend((e_name, h_name)) + for name in consumed: del self.model_tensors[name] @@ -737,6 +875,12 @@ class DeepseekV4Model(TextModel): "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), "ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"), + "nextn.eh_proj.weight": (gguf.MODEL_TENSOR.NEXTN_EH_PROJ, ".weight"), + "nextn.enorm.weight": (gguf.MODEL_TENSOR.NEXTN_ENORM, ".weight"), + "nextn.hnorm.weight": (gguf.MODEL_TENSOR.NEXTN_HNORM, ".weight"), + "nextn.shared_head_norm.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ".weight"), + "nextn.embed_tokens.weight": (gguf.MODEL_TENSOR.NEXTN_EMBED_TOKENS, ".weight"), + "nextn.shared_head_head.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, ".weight"), } tensor_name = match.group(2) @@ -759,10 +903,12 @@ class DeepseekV4Model(TextModel): return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)] def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: - del new_name, bid # unused + del bid # unused if name in self._dsv4_fp8_dequantized and n_dims >= 2: return gguf.GGMLQuantizationType.Q8_0 + if new_name.endswith(".nextn.eh_proj.weight"): + return gguf.GGMLQuantizationType.Q8_0 if name in self._dsv4_f32_tensors: return gguf.GGMLQuantizationType.F32 if name in self._dsv4_bf16_tensors and n_dims >= 2: @@ -770,7 +916,122 @@ class DeepseekV4Model(TextModel): return False + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + def prepare_tensors(self): super().prepare_tensors() self._is_mxfp4 = True self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE + + +@ModelBase.register("DeepseekV4DSparkModel") +class DeepseekV4DSparkModel(DeepseekV4Model): + model_arch = gguf.MODEL_ARCH.DFLASH + + _DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { + "main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"), + "main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"), + "markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"), + "markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"), + "confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"), + } + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.block_count = 1 + max( + int(match.group(1)) for name in self.model_tensors + if (match := re.match(r"layers\.(\d+)\.", name)) + ) + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + self.hparams["compress_ratios"] = [0] * self.block_count + self.hparams["num_hash_layers"] = 0 + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + if remote_hf_model_id is None: + return super().index_tensors() + + with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: + weight_map = json.load(f)["weight_map"] + + part_names = sorted({ + part_name for name, part_name in weight_map.items() + if name.startswith("mtp.") + }) + tensors: dict[str, Callable[[], Tensor]] = {} + + for part_name in part_names: + from huggingface_hub import hf_hub_download + + logger.info("gguf: caching remote DSpark part '%s'", part_name) + part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name)) + with gguf.utility.SafetensorsLocal(part_path) as model_part: + for name in model_part: + data = model_part[name] + data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731 + if titem := self.filter_tensors((name, data_gen)): + tensor_name, tensor_gen = titem + tensors[tensor_name] = tensor_gen + + return tensors + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if not name.startswith("mtp."): + return None + return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen)) + + @staticmethod + def _rekey_mtp_tensor_name(name: str) -> str: + match = re.match(r"mtp\.(\d+)\.(.+)$", name) + if match is None: + raise ValueError(f"Unexpected DSpark tensor {name!r}") + + stage, rest = match.group(1), match.group(2) + root_names = ( + "main_proj.scale", + "norm.weight", + "hc_head_fn", + "hc_head_base", + "hc_head_scale", + ) + if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names: + return rest + return f"layers.{stage}.{rest}" + + def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: + if name in self._DSPARK_ROOT_MAP: + return self._DSPARK_ROOT_MAP[name] + return super()._map_dsv4_tensor_name(name, bid) + + def set_vocab(self): + if self.target_model_dir is None: + raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer") + + original_dir = self.dir_model + try: + self.dir_model = self.target_model_dir + super().set_vocab() + finally: + self.dir_model = original_dir + + self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"]) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + self.gguf_writer.add_block_size(self.hparams["dspark_block_size"]) + self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]]) diff --git a/conversion/exaone.py b/conversion/exaone.py index bc4fb3f1b..1cd2244db 100644 --- a/conversion/exaone.py +++ b/conversion/exaone.py @@ -123,7 +123,9 @@ class Exaone4Model(TextModel): yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32)) -@ModelBase.register("ExaoneMoEForCausalLM") +# note: transformers >= 5.1 renamed the class to "ExaoneMoeForCausalLM" (lowercase 'e'), +# so accept both spellings - LG AI have updated the configs of already-released models +@ModelBase.register("ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM") class ExaoneMoEModel(Exaone4Model): model_arch = gguf.MODEL_ARCH.EXAONE_MOE diff --git a/conversion/gemma.py b/conversion/gemma.py index c552df732..f15a10a38 100644 --- a/conversion/gemma.py +++ b/conversion/gemma.py @@ -665,7 +665,18 @@ class Gemma4Model(Gemma3Model): swa_layers = [t == "sliding_attention" for t in self.hparams["layer_types"]] self.gguf_writer.add_sliding_window_pattern(swa_layers) - head_dim_full = self.hparams["global_head_dim"] + per_layer_config = self.hparams.get("per_layer_config") + layer_types = self.hparams.get("layer_types", []) + if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None: + for layer_idx, layer_config in per_layer_config.items(): + layer_idx = int(layer_idx) + if layer_idx < len(layer_types): + if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config: + head_dim_full = layer_config["head_dim"] + break + + assert head_dim_full is not None + head_dim_swa = self.hparams["head_dim"] # correct the head dim for global/swa layers self.gguf_writer.add_key_length(head_dim_full) @@ -685,8 +696,14 @@ class Gemma4Model(Gemma3Model): n_ff_arr = [n_ff if il < first_kv_shared_layer_idx else n_ff * 2 for il in range(self.block_count)] self.gguf_writer.add_feed_forward_length(n_ff_arr) - # handle num_global_key_value_heads - num_key_value_heads_full = self.hparams.get("num_global_key_value_heads") + if (num_key_value_heads_full := self.hparams.get("num_global_key_value_heads")) is None and per_layer_config is not None: + for layer_idx, layer_config in per_layer_config.items(): + layer_idx = int(layer_idx) + if layer_idx < len(layer_types): + if layer_types[layer_idx] == "full_attention" and "num_key_value_heads" in layer_config: + num_key_value_heads_full = layer_config["num_key_value_heads"] + break + num_key_value_heads_swa = self.hparams.get("num_key_value_heads") if num_key_value_heads_full is not None and num_key_value_heads_swa is not None: value_arr = [num_key_value_heads_swa if is_swa else num_key_value_heads_full for is_swa in swa_layers] @@ -708,7 +725,19 @@ class Gemma4Model(Gemma3Model): # IMPORTANT: this ROPE_FREQS tensor is ONLY used by the full_attention layers rope_params_full = self.hparams["rope_parameters"]["full_attention"] assert rope_params_full["rope_type"] == "proportional" - head_dim_full = (self.hparams["global_head_dim"]) + + per_layer_config = self.hparams.get("per_layer_config") + if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None: + layer_types = self.hparams.get("layer_types", []) + for layer_idx, layer_config in per_layer_config.items(): + layer_idx = int(layer_idx) + if layer_idx < len(layer_types): + if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config: + head_dim_full = layer_config["head_dim"] + break + + assert head_dim_full is not None + partial_rotary_factor_full = rope_params_full["partial_rotary_factor"] n_rot_full = int(head_dim_full * partial_rotary_factor_full / 2) n_unrot_full = int(head_dim_full / 2) - n_rot_full diff --git a/conversion/glm.py b/conversion/glm.py index cc34cddbf..e28f54574 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -206,10 +206,70 @@ class Glm4MoeModel(TextModel): @ModelBase.register("Glm4MoeLiteForCausalLM") class Glm4MoeLiteModel(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.DEEPSEEK2 + skip_mtp = False + supports_mtp_export = True + _n_main_layers: int | None = None def set_vocab(self): return self._set_vocab_glm() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + num_hidden_layers = self.hparams["num_hidden_layers"] + self.num_nextn_predict_layers = self.hparams.get("num_nextn_predict_layers", 0) + self.skip_mtp = self.no_mtp or self.num_nextn_predict_layers == 0 + + if self.skip_mtp: + self.block_count = num_hidden_layers + else: + self.block_count = num_hidden_layers + self.num_nextn_predict_layers + + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + if self.skip_mtp: + return + + self.gguf_writer.add_nextn_predict_layers(self.num_nextn_predict_layers) + + def index_tensors(self, remote_hf_model_id: str | None = None): + type(self)._n_main_layers = self.hparams["num_hidden_layers"] + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item): + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + if cls._n_main_layers is not None: + match = re.match(r"model\.layers\.(\d+)\.", name) + is_mtp = match is not None and int(match.group(1)) >= cls._n_main_layers + if is_mtp and cls.no_mtp: + return None + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + @ModelBase.register("GlmMoeDsaForCausalLM") class GlmMoeDsaModel(DeepseekV2Model): diff --git a/conversion/granite.py b/conversion/granite.py index 8367ed225..956342e6d 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -123,6 +123,166 @@ class GraniteMoeModel(GraniteModel): yield from super().modify_tensors(data_torch, name, bid) +@ModelBase.register("GraniteSwitchForCausalLM") +class GraniteSwitchModel(GraniteMoeModel): + """Dense, all-attention Granite with N per-token embedded LoRA adapters, stacked + over the adapter dim with a zero adapter at slot 0 (N = num_adapters + 1).""" + model_arch = gguf.MODEL_ARCH.GRANITE_SWITCH + + # permute q/k per-slice below (NORM-rope layout), not via the parent's auto-permute + undo_permute = False + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # the weightless switch reserves one cache slot: one fewer block than num_hidden_layers + self.block_count = self.block_count - 1 + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + self._n_adapters = int(self.hparams["num_adapters"]) + self._max_lora_rank = int(self.hparams["max_lora_rank"]) + self._n_slots = self._n_adapters + 1 # +1 for the zero slot at index 0 + + n_head = int(self.hparams["num_attention_heads"]) + n_kv_head = int(self.hparams["num_key_value_heads"]) + head_dim = ( + self.hparams.get("projection_head_dim") + or self.hparams.get("head_dim") + or (self.hparams["hidden_size"] // n_head) + ) + self._n_head = n_head + self._n_kv_head = n_kv_head + self._head_dim = int(head_dim) + self._q_size = n_head * self._head_dim + self._kv_size = n_kv_head * self._head_dim + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + # dense: pin expert_used_count to 0 (config carries a leftover num_experts_per_tok) + if not self.hparams.get("num_local_experts"): + self.gguf_writer.add_expert_used_count(0) + + self.gguf_writer.add_adapter_count(self._n_adapters) + self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank) + self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"]) + self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"]) + router_gain = float(self.hparams.get("control_token_gain", 15.0)) + self.gguf_writer.add_adapter_router_gain(router_gain) + logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain) + + def _lora_a(self, data: Tensor) -> Tensor: + # on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in] + a = data.squeeze(1) + zero = torch.zeros_like(a[:1]) + return torch.cat([zero, a], dim=0).contiguous() + + def _lora_b(self, data: Tensor, permute_n_head: int | None = None) -> Tensor: + # on-disk B: [n_adapters, 1, out, max_rank] -> [n_adapters+1, out, max_rank] + b = data.squeeze(1) + if permute_n_head is not None: + # permute each adapter's B output rows to match the permuted q/k base + b = torch.stack([self.permute(b[i], permute_n_head, permute_n_head) for i in range(b.shape[0])], dim=0) + zero = torch.zeros_like(b[:1]) + return torch.cat([zero, b], dim=0).contiguous() + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + + # skip the weightless switch + control-token buffers (rebuilt at load time) + bare = name.split(".")[-1] + if ( + name.startswith("model.switch.") or name.startswith("switch.") + or bare in ("adapter_token_ids", "control_to_substitute_lut") + ): + return + + if "self_attn.qkv_proj" in name: + if name.endswith("base_layer.weight"): + # fused [q|k|v] rows: permute q/k row-blocks for ggml's NORM-rope layout + q, k, v = data_torch.split([self._q_size, self._kv_size, self._kv_size], dim=0) + q = self.permute(q, self._n_head, self._n_head) + k = self.permute(k, self._n_kv_head, self._n_kv_head) + fused = torch.cat([q, k, v], dim=0) + yield (self.format_tensor_name(T.ATTN_QKV, bid), fused) + return + if "lora_A_slices." in name: + slot = int(name.rsplit(".", 1)[1]) + key = {0: T.ATTN_Q, 1: T.ATTN_K, 2: T.ATTN_V}[slot] + yield (self.format_tensor_name(key, bid, suffix=".lora_a"), self._lora_a(data_torch)) + return + if "lora_B_slices." in name: + slot = int(name.rsplit(".", 1)[1]) + key, ph = { + 0: (T.ATTN_Q, self._n_head), + 1: (T.ATTN_K, self._n_kv_head), + 2: (T.ATTN_V, None), + }[slot] + yield (self.format_tensor_name(key, bid, suffix=".lora_b"), self._lora_b(data_torch, ph)) + return + raise ValueError(f"Unexpected qkv_proj tensor: {name}") + + if "self_attn.o_proj" in name: + if name.endswith("base_layer.weight"): + yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch) + return + if name.endswith("lora_A"): + yield (self.format_tensor_name(T.ATTN_OUT, bid, suffix=".lora_a"), self._lora_a(data_torch)) + return + if name.endswith("lora_B"): + yield (self.format_tensor_name(T.ATTN_OUT, bid, suffix=".lora_b"), self._lora_b(data_torch)) + return + raise ValueError(f"Unexpected o_proj tensor: {name}") + + if "shared_mlp.input_linear" in name: + ffn = self.hparams["shared_intermediate_size"] + if name.endswith("base_layer.weight"): + gate, up = data_torch.split([ffn, ffn], dim=0) + yield (self.format_tensor_name(T.FFN_GATE, bid), gate) + yield (self.format_tensor_name(T.FFN_UP, bid), up) + return + if "lora_A_slices." in name: + slot = int(name.rsplit(".", 1)[1]) + key = {0: T.FFN_GATE, 1: T.FFN_UP}[slot] + yield (self.format_tensor_name(key, bid, suffix=".lora_a"), self._lora_a(data_torch)) + return + if "lora_B_slices." in name: + slot = int(name.rsplit(".", 1)[1]) + key = {0: T.FFN_GATE, 1: T.FFN_UP}[slot] + yield (self.format_tensor_name(key, bid, suffix=".lora_b"), self._lora_b(data_torch)) + return + raise ValueError(f"Unexpected shared_mlp.input_linear tensor: {name}") + + if "shared_mlp.output_linear" in name: + if name.endswith("base_layer.weight"): + yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch) + return + if name.endswith("lora_A"): + yield (self.format_tensor_name(T.FFN_DOWN, bid, suffix=".lora_a"), self._lora_a(data_torch)) + return + if name.endswith("lora_B"): + yield (self.format_tensor_name(T.FFN_DOWN, bid, suffix=".lora_b"), self._lora_b(data_torch)) + return + raise ValueError(f"Unexpected shared_mlp.output_linear tensor: {name}") + + if bid is not None and ".layers." in name and ( + "input_layernorm" in name or "post_attention_layernorm" in name + ): + key = T.ATTN_NORM if "input_layernorm" in name else T.FFN_NORM + yield (self.format_tensor_name(key, bid), data_torch) + return + + if name in ("model.embed_tokens.weight", "embed_tokens.weight"): + yield (self.format_tensor_name(T.TOKEN_EMBD), data_torch) + return + if name in ("model.norm.weight", "norm.weight"): + yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch) + return + if name == "lm_head.weight": + return # tied to token_embd + + raise ValueError(f"graniteswitch: unhandled tensor {name!r} (bid={bid})") + + @ModelBase.register("GraniteMoeHybridForCausalLM", "BambaForCausalLM") class GraniteHybridModel(Mamba2Model, GraniteMoeModel): """GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM diff --git a/conversion/llama.py b/conversion/llama.py index 9b3373f91..1aced49c5 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -119,7 +119,7 @@ class LlamaModel(TextModel): path_tekken_json = self.dir_model / "tekken.json" path_tokenizer_json = self.dir_model / "tokenizer.json" if path_tekken_json.is_file() and not path_tokenizer_json.is_file(): - self._set_vocab_mistral() + return self._set_vocab_mistral() tokenizer_config_file = self.dir_model / 'tokenizer_config.json' if tokenizer_config_file.is_file(): diff --git a/conversion/minicpm.py b/conversion/minicpm.py index e31b26a00..bf3fa8142 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -137,6 +137,15 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel): class MiniCPMV4_6VisionModel(MmprojModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) + self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x") + if self.downsample_mode not in {"4x", "16x"}: + raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}") + if self.downsample_mode == "4x": + self.model_tensors = { + name: tensor for name, tensor in self.model_tensors.items() + if ".vit_merger." not in name + } + if self.hparams_vision is not None: # In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP # positional embedding bucket grid (70 x 70), while the per-slice processing @@ -156,8 +165,8 @@ class MiniCPMV4_6VisionModel(MmprojModel): # (mapped to PROJECTOR_TYPE_MINICPMV4_6). self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6) - # ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment - self.gguf_writer.add_vision_projector_scale_factor(4) + self.gguf_writer.add_vision_projector_scale_factor( + 2 if self.downsample_mode == "4x" else 4) # borrow wa_layer_indexes for vit_merger insertion point insert_layer_id = int(self.global_config.get( diff --git a/conversion/minimax.py b/conversion/minimax.py index c2175cc93..d7a00bac9 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -1,13 +1,121 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import Iterable, Sequence, TYPE_CHECKING import torch if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, MmprojModel, gguf +from .base import ModelBase, TextModel, MmprojModel, gguf, logger + + +@ModelBase.register("MiniMaxText01ForCausalLM") +@ModelBase.register("MiniMaxM1ForCausalLM") +class MiniMaxText01Model(TextModel): + model_arch = gguf.MODEL_ARCH.MINIMAX01 + + def _get_suppress_tokens(self) -> Sequence[int] | None: + import json + from transformers import AutoTokenizer + from .base import LazyTorchTensor + + # check added tokens embeddings in embeddings tensor for zero-valued embeddings + # they get in the way of the token sampling process and must be suppressed + + tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) + tokenizer_vocab_size = tokenizer.vocab_size + + with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: + weight_map = json.load(f)["weight_map"] + + embeddings_tensor_name = "model.embed_tokens.weight" + embeddings_shard_name = weight_map[embeddings_tensor_name] + with gguf.utility.SafetensorsLocal(self.dir_model / embeddings_shard_name) as model_shard: + embeddings_data = model_shard[embeddings_tensor_name] + + embeddings_weights_dtype = LazyTorchTensor._dtype_str_map[embeddings_data.dtype] + embeddings_weights = torch.from_numpy(embeddings_data.mmap_bytes()).view(embeddings_weights_dtype).reshape(embeddings_data.shape) + embeddings_vocab_size = embeddings_weights.shape[0] + + embeddings_added_tokens = embeddings_weights[tokenizer_vocab_size:embeddings_vocab_size] + embeddings_zero_rows = torch.all(embeddings_added_tokens == 0, dim=1) + tokens_zero_embeddings_ids = (torch.nonzero(embeddings_zero_rows, as_tuple=False).flatten() + tokenizer_vocab_size).tolist() + + return tokens_zero_embeddings_ids + + def set_vocab(self) -> None: + from pathlib import Path + + self._set_vocab_gpt2() + + for tmpl_file in [ + self.dir_model / "chat_template.jinja", + Path(__file__).parent.parent / "models" / "templates" / "MiniMax-M1.jinja" + ]: + if tmpl_file.is_file(): + self.gguf_writer.add_chat_template(tmpl_file.read_text(encoding="utf-8")) + logger.info(f"Chat template overridden with {tmpl_file}.") + break + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + suppress_tokens = self._get_suppress_tokens() + if suppress_tokens: + logger.info(f"Suppressing tokens with zero embeddings {suppress_tokens}") + self.gguf_writer.add_suppress_tokens(suppress_tokens) + + layernorm_full_attention_alpha = self.hparams["layernorm_full_attention_alpha"] + layernorm_full_attention_beta = self.hparams["layernorm_full_attention_beta"] + layernorm_linear_attention_alpha = self.hparams["layernorm_linear_attention_alpha"] + layernorm_linear_attention_beta = self.hparams["layernorm_linear_attention_beta"] + layernorm_mlp_alpha = self.hparams["layernorm_mlp_alpha"] + layernorm_mlp_beta = self.hparams["layernorm_mlp_beta"] + assert layernorm_full_attention_alpha == layernorm_linear_attention_alpha == layernorm_mlp_alpha + assert layernorm_full_attention_beta == layernorm_linear_attention_beta == layernorm_mlp_beta == 1.0 + # we do not store the layernorm betas as they are all 1.0 + # layernorm alphas are stored as single residual_scale hparam + self.gguf_writer.add_residual_scale(layernorm_full_attention_alpha) + + self.gguf_writer.add_rope_dimension_count(self.hparams["rotary_dim"]) + + _experts: list[dict[str, Tensor]] | None = None + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # process the experts separately + if name.find("block_sparse_moe.experts") != -1: + n_experts = self.hparams["num_local_experts"] + + assert bid is not None + + if self._experts is None: + self._experts = [{} for _ in range(self.block_count)] + + self._experts[bid][name] = data_torch + + if len(self._experts[bid]) >= n_experts * 3: + # merge the experts into a single 3d tensor + for wid in ["w1", "w2", "w3"]: + datas: list[Tensor] = [] + + for xid in range(n_experts): + ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight" + datas.append(self._experts[bid][ename]) + del self._experts[bid][ename] + + data_torch = torch.stack(datas, dim=0) + + merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight" + + new_name = self.map_tensor_name(merged_name) + + yield from super().modify_tensors(data_torch, new_name, bid) + return + else: + return + + yield from super().modify_tensors(data_torch, name, bid) @ModelBase.register("MiniMaxM2ForCausalLM") diff --git a/conversion/muse_glimmer.py b/conversion/muse_glimmer.py new file mode 100644 index 000000000..cc588e832 --- /dev/null +++ b/conversion/muse_glimmer.py @@ -0,0 +1,179 @@ +from __future__ import annotations + +import json +from typing import Any, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import MmprojModel, ModelBase, TextModel, gguf + + +def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor": + """Invert transformers' `_permute_for_rope`: HF stores Q/K in rotate_half layout, + llama.cpp consumes the interleaved (NORM) layout.""" + if tensor.ndim == 2: + dim1, dim2 = tensor.shape + return tensor.view(n_heads, 2, dim1 // n_heads // 2, dim2).transpose(1, 2).reshape(dim1, dim2) + if tensor.ndim == 1: + (dim1,) = tensor.shape + return tensor.view(n_heads, 2, dim1 // n_heads // 2).transpose(1, 2).reshape(dim1) + raise ValueError(f"_unpermute_for_rope: unexpected shape {tuple(tensor.shape)}") + + +@ModelBase.register("MuseGlimmerForConditionalGeneration") +class MuseGlimmerModel(TextModel): + model_arch = gguf.MODEL_ARCH.MUSE_GLIMMER + + def norm_shift(self, name: str) -> float: + # All four layer norms use 1, the final norm uses 0. + return 1.0 if name.endswith("layernorm.weight") else 0.0 + + def set_vocab(self): + self._set_vocab_gpt2() + + from transformers import AutoTokenizer + tok = AutoTokenizer.from_pretrained(self.dir_model) + eot_id = tok.convert_tokens_to_ids("<|eot|>") + if isinstance(eot_id, int) and eot_id >= 0: + self.gguf_writer.add_eot_token_id(eot_id) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hparams = self.hparams + + self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"]) + self.gguf_writer.add_logit_scale(hparams["output_multiplier"]) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]]) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + shift = self.norm_shift(name) + if shift != 0.0: + data_torch = data_torch + shift + + # Invert transformers' `_permute_for_rope` on Q/K, we keep ggml's NORM (interleaved) rope + if ".self_attn.q_proj." in name: + data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_attention_heads"])) + elif ".self_attn.k_proj." in name: + data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_key_value_heads"])) + + # Synthesize QK-norm weights to absorb qk_scale_factor. + # MuseGlimmer implementation: scaleless RMSNorm followed by qk_scale_factor.. + if bid is not None and name.endswith(f"model.layers.{bid}.self_attn.q_proj.weight"): + head_dim = self.hparams["head_dim"] + q_scale = float(self.hparams["qk_scale_factor"]) + yield ( + self.map_tensor_name(f"model.layers.{bid}.self_attn.q_norm.weight"), + torch.full((head_dim,), q_scale, dtype=torch.float32), + ) + yield ( + self.map_tensor_name(f"model.layers.{bid}.self_attn.k_norm.weight"), + torch.ones((head_dim,), dtype=torch.float32), + ) + + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("MuseGlimmerForConditionalGeneration") +class MuseGlimmerVisionModel(MmprojModel): + def get_vision_config(self) -> dict[str, Any] | None: + c = self.global_config.get("vision_config") + if not c: + return None + # MuseGlimmer actually uses dynamic size, initialize with nominal size + image_size = c["pos_emb_height"] * c["patch_size"] * c["merge_size"] + return {**c, "image_size": image_size} + + def set_gguf_parameters(self): + super().set_gguf_parameters() + assert self.hparams_vision is not None + c = self.hparams_vision # enriched vision_config from get_vision_config() + + self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER) + self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"])) + self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"])) + + @classmethod + def filter_tensors(cls, item): + name, gen = item + keep = ("model.vision_tower.", "model.vision_adapter.", "model.vision_projection.") + if not any(name.startswith(k) for k in keep): + return None + return super().filter_tensors((name, gen)) + + # 3-layer projector MLP + _MM_MLP_MAP = { + "model.vision_adapter.fc1": (gguf.MODEL_TENSOR.V_MMPROJ, 0), + "model.vision_adapter.fc2": (gguf.MODEL_TENSOR.V_MMPROJ, 1), + "model.vision_projection": (gguf.MODEL_TENSOR.V_MMPROJ, 2), + } + + def modify_tensors(self, data_torch, name, bid): + assert self.hparams_vision is not None + if ".attn.q_proj." in name or ".attn.k_proj." in name: + n_heads = int(self.hparams_vision["num_attention_heads"]) + data_torch = _unpermute_for_rope(data_torch, n_heads) + # Lay out the pt=2 temporal slabs of the patch embedding as a conv2d for build_inp() + if name.endswith("patch_embedder.patch_embedding.weight"): + n_embd = data_torch.shape[0] + pt = int(self.hparams_vision["patch_temporal"]) + ps = int(self.hparams_vision["patch_size"]) + data_torch = data_torch.view(n_embd, pt, 3, ps, ps).sum(dim=1) # (n_embd, 3, ps, ps) + stem, _, suffix = name.rpartition(".") + if stem in self._MM_MLP_MAP: + tensor_key, idx = self._MM_MLP_MAP[stem] + yield (self.format_tensor_name(tensor_key, bid=idx, suffix="." + suffix), data_torch) + return + yield (self.map_tensor_name(name), data_torch) + + +@ModelBase.register("MuseGlimmerAssistantModel") +class MuseGlimmerAssistantModel(TextModel): + model_arch = gguf.MODEL_ARCH.DFLASH + + def set_vocab(self): + if self.target_model_dir is None: + raise ValueError( + "MuseGlimmerAssistant (DFlash drafter) requires --target-model-dir pointing to the " + "target MuseGlimmer HF directory" + ) + + original_dir = self.dir_model + self.dir_model = self.target_model_dir + + from . import get_model_class + with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f: + target_arch = json.load(f)["architectures"][0] + target_cls = get_model_class(target_arch) + if target_cls is not type(self): + target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] + else: + super().set_vocab() + + self.dir_model = original_dir + + mask_token_id = self.hparams.get("mask_token_id") + if mask_token_id is not None: + self.gguf_writer.add_mask_token_id(int(mask_token_id)) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + h = self.hparams + + self.gguf_writer.add_block_size(int(h["block_size"])) + + # dflash.target_layers[k] refers to the inputs going into the ith layer, which come from the (i-1)th layer's output. + # The transformers configuration refers to the outputs being recorded. + self.gguf_writer.add_target_layers([int(x) + 1 for x in h["target_layer_ids"]]) + + if h.get("sliding_window") and h.get("layer_types"): + self.gguf_writer.add_sliding_window(int(h["sliding_window"])) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in h["layer_types"]]) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # DFlash defaults to NEOX (rotate_half) rope, matching transformers HF layout for Q/K, QK-norms + # no permutation needed. + yield (self.map_tensor_name(name), data_torch) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index 0572b42ca..c46cec143 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -197,6 +197,7 @@ class NemotronHModel(GraniteHybridModel): """Hybrid mamba2/attention model from NVIDIA""" model_arch = gguf.MODEL_ARCH.NEMOTRON_H is_moe: bool = False + supports_mtp_export = True def __init__(self, *args, **kwargs): # We have to determine the correct model architecture (MoE vs non-MoE) before @@ -236,6 +237,25 @@ class NemotronHModel(GraniteHybridModel): self._ssm_layers = [i for i, val in enumerate(pattern) if val == "mamba"] self._mlp_layers = [i for i, val in enumerate(pattern) if val == "moe"] + # `--no-mtp` drops it entirely; `--mtp` exports only the MTP head + self._mtp_bid: int | None = None + if self.is_moe and not self.no_mtp: + n_nextn = self.hparams.get("num_nextn_predict_layers", 0) or 0 + if n_nextn > 0: + assert n_nextn == 1, ( + "NemotronH MTP conversion currently supports num_nextn_predict_layers == 1" + ) + self._mtp_bid = self.block_count + self.block_count += 1 + # The folded MTP block carries both an attention sub-layer and a + # MoE sub-layer, so register it as both so the per-layer metadata arrays cover it + self._attn_layers.append(self._mtp_bid) + self._mlp_layers.append(self._mtp_bid) + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + if self.mtp_only and self._mtp_bid is None: + raise ValueError("--mtp was requested, but this model does not contain a supported MTP head") + def get_attn_layers(self): pattern = self.hparams.get("hybrid_override_pattern") or self.hparams.get("layers_block_type") if pattern is None: @@ -246,6 +266,44 @@ class NemotronHModel(GraniteHybridModel): return [i for i, val in enumerate(pattern) if val == "attention"] + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if name.startswith("mtp."): + # --no-mtp: drop the MTP head entirely + if cls.no_mtp: + return None + elif cls.mtp_only: + # --mtp: export the MTP head plus the tensors it shares with the target model + # Include lm_head scale sidecars so NVFP4 packing sees them. + keep = name in ( + "backbone.embeddings.weight", + "backbone.norm_f.weight", + "lm_head.weight", + "lm_head.weight_scale", + "lm_head.weight_scale_2", + "lm_head.weight_scale_inv", + "lm_head.input_scale", + "lm_head.input_global_scale", + "lm_head.weight_global_scale", + "lm_head.weight_packed", + ) + if not keep: + return None + return super().filter_tensors((name, gen)) + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) + + if not self.mtp_only or not from_dir: + return + output_type: str = self.ftype.name.partition("_")[2] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, + self.metadata.version, size_label=None, output_type=output_type, model_type=None) + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + def set_gguf_parameters(self): super().set_gguf_parameters() @@ -284,6 +342,10 @@ class NemotronHModel(GraniteHybridModel): if (latent_size := self.hparams.get("moe_latent_size")) is not None: self.gguf_writer.add_moe_latent_size(latent_size) + # MTP head: number of trailing NextN blocks + if self._mtp_bid is not None: + self.gguf_writer.add_nextn_predict_layers(self.hparams["num_nextn_predict_layers"]) + def set_vocab(self): # The NemotronH config uses pattern characters (e.g. '-') that may not # be supported by the installed transformers version. AutoTokenizer @@ -350,15 +412,24 @@ class NemotronHModel(GraniteHybridModel): if not self.is_moe: self.gguf_writer.add_add_bos_token(True) - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - if self.is_moe and bid is not None: - # Skip Multi-Token Prediction (MTP) tensors. These are used for - # for speculative decoding but we don't include them in this model - # conversion. See https://github.com/ggml-org/llama.cpp/pull/18886 - if name.startswith("mtp."): - logger.info(f"gguf: Skipping MTP (Speculative) layer: {name}") - return + _MTP_SPECIAL_RENAMES = { + "mtp.layers.0.enorm.weight": "model.layers.{bid}.enorm.weight", + "mtp.layers.0.hnorm.weight": "model.layers.{bid}.hnorm.weight", + "mtp.layers.0.eh_proj.weight": "model.layers.{bid}.eh_proj.weight", + "mtp.layers.1.norm.weight": "model.layers.{bid}.post_attention_layernorm.weight", + "mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight", + } + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # mtp.layers.0: NextN input fusion + attention + # mtp.layers.1: MoE + final head norm + if self._mtp_bid is not None and name.startswith(("mtp.layers.0.", "mtp.layers.1.")): + suffix = name.split(".", 3)[3] + bid = self._mtp_bid + renamed = self._MTP_SPECIAL_RENAMES.get(name) + name = renamed.format(bid=bid) if renamed else f"backbone.layers.{bid}.{suffix}" + + if self.is_moe and bid is not None: if name.endswith("mixer.gate.e_score_correction.bias"): yield from ModelBase.modify_tensors(self, data_torch, name, bid) return diff --git a/conversion/pockettts.py b/conversion/pockettts.py new file mode 100644 index 000000000..62ecb5acd --- /dev/null +++ b/conversion/pockettts.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any, Iterable, TYPE_CHECKING + +import torch + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, MmprojModel, SentencePieceTokenTypes, TextModel, gguf, logger + +# Pocket TTS is a CALM: the backbone conditions a flow-matching decoder that generates one +# continuous 32-d latent per frame. There is no codebook in this model. +# The checkpoint ships no config.json, hparams come from _load_hparams() below. +# +# Tricks being used to support this model via existing llama.cpp code paths: +# - bos_before_voice and bos_emb are learned input vectors, not tokens +# they are appended to the embedding table as extra tokens, to be looked up like any other row +# - bos_emb lives in latent space, so input_linear is folded into it here +# - the backbone has no lm_head, the embedding table is reused as output for the unused logits +# +# pipeline stage mapping: +# mimi encoder + speaker_proj --> mapped to normal mtmd audio encoder +# flow_lm.transformer --> mapped to normal libllama text model (autoregressive) +# flow_lm.flow_net + out_eos --> MTMD_GEN_PROCESS_TYPE_GEN_CODE +# mimi decoder --> MTMD_GEN_PROCESS_TYPE_GEN_WAV + +# indices into mimi.encoder.model / mimi.decoder.model for stage i, see SEANetEncoder/SEANetDecoder +_ENC_RES_IDX = lambda i: 1 + 3 * i # noqa: E731 +_ENC_SCALE_IDX = lambda i: 3 + 3 * i # noqa: E731 +_DEC_SCALE_IDX = lambda i: 2 + 3 * i # noqa: E731 +_DEC_RES_IDX = lambda i: 3 + 3 * i # noqa: E731 + +_N_SEANET_STAGES = 3 +_SAMPLE_RATE = 24000 + + +def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]: + part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors") + if len(part_names) != 1: + return {} + with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part: + return {name: tuple(part[name].shape) for name in part.keys()} + + +@ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model)) +def _load_hparams(dir_model: Path) -> dict[str, Any]: + logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes") + shapes = _tensor_shapes(dir_model) + n_vocab, n_embd = shapes["flow_lm.conditioner.embed.weight"] + n_layer = sum(1 for name in shapes if re.fullmatch(r"flow_lm\.transformer\.layers\.\d+\.norm1\.weight", name)) + n_layer_a = sum(1 for name in shapes if re.fullmatch(r"mimi\.encoder_transformer\.transformer\.layers\.\d+\.norm1\.weight", name)) + n_embd_a = shapes["mimi.encoder_transformer.transformer.layers.0.norm1.weight"][0] + return { + "architectures": ["PocketTTSModel"], + "model_type": "pockettts", + "num_hidden_layers": n_layer, + "hidden_size": n_embd, + "intermediate_size": shapes["flow_lm.transformer.layers.0.linear1.weight"][0], + # the transformer is fully causal with no context limit, this only bounds the KV cache + "max_position_embeddings": 4096, + # not in the checkpoint, but every released variant uses head_dim 64 + "num_attention_heads": n_embd // 64, + # extra rows for the learned input vectors, see _embd_table() + "vocab_size": n_vocab + (2 if "flow_lm.bos_before_voice" in shapes else 1), + "rope_theta": 10000.0, + "layer_norm_eps": 1e-5, + "audio_config": { + "num_hidden_layers": n_layer_a, + "hidden_size": n_embd_a, + "intermediate_size": shapes["mimi.encoder_transformer.transformer.layers.0.linear1.weight"][0], + "num_attention_heads": n_embd_a // 64, + }, + } + + +@ModelBase.register("PocketTTSModel") +class PocketTTSModel(TextModel): + model_arch = gguf.MODEL_ARCH.POCKETTTS + + _LAYER_TENSOR_MAP = { + "norm1": gguf.MODEL_TENSOR.ATTN_NORM, + "norm2": gguf.MODEL_TENSOR.FFN_NORM, + "self_attn.out_proj": gguf.MODEL_TENSOR.ATTN_OUT, + "linear1": gguf.MODEL_TENSOR.FFN_UP, + "linear2": gguf.MODEL_TENSOR.FFN_DOWN, + } + + def set_vocab(self): + # this is a unigram sentencepiece model, llama.cpp's SPM tokenizer cannot do + # unigram segmentation, so use the UGM tokenizer instead + from sentencepiece import sentencepiece_model_pb2 as model + + proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + proto.ParseFromString(open(self.dir_model / "tokenizer.model", "rb").read()) + assert proto.trainer_spec.model_type == 1, "expected a unigram tokenizer" + + tokens, scores, toktypes = self._create_vocab_sentencepiece() + + # the last rows of the embedding table are not sentencepiece pieces + extra = self._extra_tokens() + for i, name in enumerate(extra): + tokens[len(tokens) - len(extra) + i] = name.encode("utf-8") + toktypes[len(tokens) - len(extra) + i] = SentencePieceTokenTypes.CONTROL + scores[len(tokens) - len(extra) + i] = -1000.0 + + self.gguf_writer.add_tokenizer_model("t5") + self.gguf_writer.add_tokenizer_pre("default") + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + self.gguf_writer.add_add_space_prefix(proto.normalizer_spec.add_dummy_prefix) + self.gguf_writer.add_remove_extra_whitespaces(proto.normalizer_spec.remove_extra_whitespaces) + if proto.normalizer_spec.precompiled_charsmap: + self.gguf_writer.add_precompiled_charsmap(proto.normalizer_spec.precompiled_charsmap) + self.gguf_writer.add_add_bos_token(False) + self.gguf_writer.add_add_eos_token(False) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if not name.startswith("flow_lm."): + return # mimi and the flow net go to the mmproj + + if name == "flow_lm.conditioner.embed.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), self._embd_table(data_torch)) + return + + if name.startswith("flow_lm.out_norm."): + suffix = "." + name.rsplit(".", 1)[1] + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT_NORM, suffix=suffix), data_torch) + return + + if name.startswith("flow_lm.transformer.layers."): + assert bid is not None + key_with_suffix = name.split(f"layers.{bid}.", 1)[1] + key, suffix = key_with_suffix.rsplit(".", 1) + + if key == "self_attn.in_proj": + q, k, v = data_torch.chunk(3, dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), q) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), k) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v) + return + + tensor = self._LAYER_TENSOR_MAP.get(key) + if tensor is not None: + yield (self.format_tensor_name(tensor, bid, suffix="." + suffix), data_torch) + return + + return + + def _extra_tokens(self) -> list[str]: + # the conditioner's padding row, then the learned vectors appended by _embd_table(). + # bos_before_voice only exists when the pack sets insert_bos_before_voice + names = ["<|pad|>"] + if "flow_lm.bos_before_voice" in self.model_tensors: + names.append("<|bos_before_voice|>") + names.append("<|audio_bos|>") + return names + + def _embd_table(self, embed: Tensor) -> Tensor: + rows = [embed] + if "flow_lm.bos_before_voice" in self.model_tensors: + rows.append(self.model_tensors["flow_lm.bos_before_voice"]().reshape(1, -1).to(embed.dtype)) + + # bos_emb is a latent, it only enters the backbone through input_linear + bos_emb = self.model_tensors["flow_lm.bos_emb"]() + input_linear = self.model_tensors["flow_lm.input_linear.weight"]() + audio_bos = torch.nn.functional.linear(bos_emb.float(), input_linear.float()).reshape(1, -1) + rows.append(audio_bos.to(embed.dtype)) + + return torch.cat(rows, dim=0) + + +@ModelBase.register("PocketTTSModel") +class PocketTTSMmprojModel(MmprojModel): + has_audio_encoder = True + has_vision_encoder = False + + _MIMI_TFM_MAP = { + "norm1": (gguf.MODEL_TENSOR.A_ENC_INPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM), + "norm2": (gguf.MODEL_TENSOR.A_ENC_OUTPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM), + "self_attn.out_proj": (gguf.MODEL_TENSOR.A_ENC_OUTPUT, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT), + "linear1": (gguf.MODEL_TENSOR.A_ENC_FFN_UP, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP), + "linear2": (gguf.MODEL_TENSOR.A_ENC_FFN_DOWN, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN), + "layer_scale_1.scale": (gguf.MODEL_TENSOR.A_ENC_ATTN_SCALE, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE), + "layer_scale_2.scale": (gguf.MODEL_TENSOR.A_ENC_FFN_SCALE_LS, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE), + } + _MIMI_TFM_QKV = ( + (gguf.MODEL_TENSOR.A_ENC_ATTN_Q, gguf.MODEL_TENSOR.A_ENC_ATTN_K, gguf.MODEL_TENSOR.A_ENC_ATTN_V), + (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V), + ) + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + assert self.hparams_audio is not None + + # voice-prompt encoder: mimi encoder + speaker_proj + self.gguf_writer.add_clip_has_audio_encoder(True) + # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC) + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"]) + self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"]) + self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"]) + self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"]) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + # mimi convolves the waveform directly, it is passed around as a 1-row "mel" + self.gguf_writer.add_audio_num_mel_bins(1) + + # generation: flow-matching decoder + mimi decoder + # the SEANet and flow net hparams are constant across the family, clip.cpp holds them + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_GEN) + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(self.hparams_audio["hidden_size"]) + self.gguf_writer.add_gen_audio_feed_forward_length(self.hparams_audio["intermediate_size"]) + self.gguf_writer.add_gen_audio_block_count(self.hparams_audio["num_hidden_layers"]) + self.gguf_writer.add_gen_audio_head_count(self.hparams_audio["num_attention_heads"]) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5) + + self.gguf_writer.add_gen_audio_model_variant(self.dir_model.name) + + def tensor_force_quant(self, name, new_name, bid, n_dims): + del name, bid, n_dims + # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path + if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"): + return gguf.GGMLQuantizationType.F16 + return False + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + del bid # the block index of the mimi transformers is parsed here, not by the base class + T = gguf.MODEL_TENSOR + + if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"): + return # folded into the backbone embedding table + if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."): + return # backbone + + if name == "flow_lm.speaker_proj_weight": + yield (self.format_tensor_name(T.A_ENC_SPEAKER_PROJ), data_torch) + return + if name == "flow_lm.input_linear.weight": + yield (self.format_tensor_name(T.A_GEN_INPUT_LINEAR), data_torch) + return + if name == "flow_lm.emb_mean": + yield (self.format_tensor_name(T.A_GEN_EMB_MEAN, suffix=""), data_torch) + return + if name == "flow_lm.emb_std": + yield (self.format_tensor_name(T.A_GEN_EMB_STD, suffix=""), data_torch) + return + if name.startswith("flow_lm.out_eos."): + suffix = "." + name.rsplit(".", 1)[1] + yield (self.format_tensor_name(T.A_GEN_OUT_EOS, suffix=suffix), data_torch) + return + + if name.startswith("flow_lm.flow_net."): + yield from self._flow_net_tensor(name, data_torch) + return + + if name == "mimi.downsample.conv.conv.weight": + yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), data_torch) + return + if name == "mimi.upsample.convtr.convtr.weight": + yield (self.format_tensor_name(T.A_GEN_WAV_UPSAMPLE), data_torch) + return + if name == "mimi.quantizer.output_proj.weight": + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_OUT), data_torch.squeeze(-1)) + return + + if "_transformer.transformer.layers." in name: + yield from self._mimi_tfm_tensor(name, data_torch) + return + + if name.startswith("mimi.encoder.model.") or name.startswith("mimi.decoder.model."): + yield from self._seanet_tensor(name, data_torch) + return + + return + + def _flow_net_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + key = name.split("flow_lm.flow_net.", 1)[1] + suffix = "." + key.rsplit(".", 1)[1] + + simple = { + "input_proj": T.A_GEN_FLOW_INPUT_PROJ, + "cond_embed": T.A_GEN_FLOW_COND_EMBD, + "final_layer.linear": T.A_GEN_FLOW_FINAL_PROJ, + "final_layer.adaLN_modulation.1": T.A_GEN_FLOW_FINAL_ADA, + } + tensor = simple.get(key.rsplit(".", 1)[0]) + if tensor is not None: + yield (self.format_tensor_name(tensor, suffix=suffix), data_torch) + return + + if key.startswith("time_embed."): + bid = int(key.split(".")[1]) + rest = key.split(f"time_embed.{bid}.", 1)[1] + time_map = { + "freqs": (T.A_GEN_FLOW_TIME_FREQS, ""), + "mlp.0": (T.A_GEN_FLOW_TIME_UP, suffix), + "mlp.2": (T.A_GEN_FLOW_TIME_DOWN, suffix), + "mlp.3.alpha": (T.A_GEN_FLOW_TIME_NORM, ""), + } + entry = time_map.get(rest) or time_map.get(rest.rsplit(".", 1)[0]) + if entry is not None: + yield (self.format_tensor_name(entry[0], bid, suffix=entry[1]), data_torch) + return + + if key.startswith("res_blocks."): + bid = int(key.split(".")[1]) + rest = key.split(f"res_blocks.{bid}.", 1)[1].rsplit(".", 1)[0] + blk_map = { + "in_ln": T.A_GEN_FLOW_BLK_NORM, + "mlp.0": T.A_GEN_FLOW_BLK_UP, + "mlp.2": T.A_GEN_FLOW_BLK_DOWN, + "adaLN_modulation.1": T.A_GEN_FLOW_BLK_ADA, + } + tensor = blk_map.get(rest) + if tensor is not None: + yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch) + return + + def _mimi_tfm_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]: + is_decoder = name.startswith("mimi.decoder_transformer.") + bid = int(name.split("_transformer.transformer.layers.", 1)[1].split(".")[0]) + key_with_suffix = name.split(f".layers.{bid}.", 1)[1] + + if key_with_suffix == "self_attn.in_proj.weight": + q, k, v = data_torch.chunk(3, dim=0) + names = self._MIMI_TFM_QKV[1 if is_decoder else 0] + for tensor, part in zip(names, (q, k, v)): + yield (self.format_tensor_name(tensor, bid), part) + return + + key, suffix = key_with_suffix.rsplit(".", 1) + entry = self._MIMI_TFM_MAP.get(key) or self._MIMI_TFM_MAP.get(key_with_suffix) + if entry is None: + return + tensor = entry[1 if is_decoder else 0] + suffix = ".weight" if key_with_suffix.endswith(".scale") else "." + suffix + yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch) + + def _seanet_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]: + T = gguf.MODEL_TENSOR + is_decoder = name.startswith("mimi.decoder.") + idx = int(name.split(".model.", 1)[1].split(".")[0]) + suffix = "." + name.rsplit(".", 1)[1] + + conv_in, conv_out, res1, res2, scale = ( + (T.A_GEN_WAV_SEANET_CONV_IN, T.A_GEN_WAV_SEANET_CONV_OUT, T.A_GEN_WAV_SEANET_RES_CONV1, + T.A_GEN_WAV_SEANET_RES_CONV2, T.A_GEN_WAV_SEANET_SCALE_CONV) + if is_decoder else + (T.A_ENC_SEANET_CONV_IN, T.A_ENC_SEANET_CONV_OUT, T.A_ENC_SEANET_RES_CONV1, + T.A_ENC_SEANET_RES_CONV2, T.A_ENC_SEANET_SCALE_CONV) + ) + + if idx == 0: + yield (self.format_tensor_name(conv_in, suffix=suffix), data_torch) + return + if idx == 3 * _N_SEANET_STAGES + 2: + yield (self.format_tensor_name(conv_out, suffix=suffix), data_torch) + return + + for stage in range(_N_SEANET_STAGES): + res_idx = _DEC_RES_IDX(stage) if is_decoder else _ENC_RES_IDX(stage) + scale_idx = _DEC_SCALE_IDX(stage) if is_decoder else _ENC_SCALE_IDX(stage) + if idx == scale_idx: + yield (self.format_tensor_name(scale, stage, suffix=suffix), data_torch) + return + if idx == res_idx: + # block.1 is the dilated conv, block.3 the pointwise one (0 and 2 are ELU) + inner = int(name.split(".block.", 1)[1].split(".")[0]) + tensor = res1 if inner == 1 else res2 + yield (self.format_tensor_name(tensor, stage, suffix=suffix), data_torch) + return diff --git a/conversion/qwen.py b/conversion/qwen.py index d1127f743..ead435455 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -18,7 +18,7 @@ class QwenModel(TextModel): @staticmethod def token_bytes_to_string(b): - from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] + from transformers.convert_slow_tokenizer import bytes_to_unicode byte_encoder = bytes_to_unicode() return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) @@ -268,8 +268,101 @@ class Qwen3MoeModel(Qwen2MoeModel): super().set_vocab() +class _QwenMtpMixin: + """Shared MTP wiring for Qwen3-Next and Qwen3.5/3.6 text variants. The HF + config carries the MTP block under `mtp_num_hidden_layers` (computed from + the checkpoint when absent, e.g. Qwen3-Next) and the tensors under + `mtp.*`; we extend block_count, emit the nextn metadata key, and remap + `mtp.*` to the standard layer-indexed nextn naming so the existing + tensor_map handles them.""" + + supports_mtp_export = True + hparams: dict[str, Any] + model_arch: gguf.MODEL_ARCH + gguf_writer: gguf.GGUFWriter + block_count: int + tensor_map: gguf.TensorNameMap + no_mtp: bool + mtp_only: bool + _original_block_count: int | None = None + opt_num_mtp_layers: int = 0 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.block_count = self.hparams["num_hidden_layers"] + if not self.no_mtp: + n_mtp = self.hparams.get("mtp_num_hidden_layers", 0) + # Qwen-3-Next doesn't include `mtp_num_hidden_layers` in config. + if n_mtp == 0: + assert self.opt_num_mtp_layers != 0 + n_mtp = self.opt_num_mtp_layers + self.block_count += n_mtp + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + hparams = {**self.hparams, **self.hparams.get("text_config", {})} + key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None) + type(self)._original_block_count = hparams.get(key) + type(self).opt_num_mtp_layers = 0 + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute] + + @classmethod + def filter_tensors(cls, item): + assert cls._original_block_count is not None + # TODO: change TextModel to super() + if (titem := TextModel.filter_tensors(item)) is None: + return None + name, gen = titem + if name.startswith("model.mtp."): + name = name.replace("model.", "", 1) + if name.startswith("mtp."): + if cls.no_mtp: + return None + remapper = { + "fc": "eh_proj", + "pre_fc_norm_embedding": "enorm", + "pre_fc_norm_hidden": "hnorm", + "norm": "shared_head.norm", + } + parts = name.split(".", 3) + if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal(): + mtp_idx = int(parts[2]) + name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}" + cls.opt_num_mtp_layers = max(cls.opt_num_mtp_layers, mtp_idx + 1) + elif len(parts) == 3 and parts[1] in remapper: + name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}" + elif cls.mtp_only: + keep = name in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + "embed_tokens.weight", "norm.weight", + ) + if not keep: + return None + return name, gen + + def set_gguf_parameters(self): + super().set_gguf_parameters() # ty: ignore[unresolved-attribute] + if self.no_mtp: + return + if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0: + self.gguf_writer.add_nextn_predict_layers(n) + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute] + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + + @ModelBase.register("Qwen3NextForCausalLM") -class Qwen3NextModel(Qwen2MoeModel): +class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel): model_arch = gguf.MODEL_ARCH.QWEN3NEXT def set_gguf_parameters(self): @@ -284,16 +377,6 @@ class Qwen3NextModel(Qwen2MoeModel): rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.25))) - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - - if name.startswith("mtp"): - # ignore MTP layers for now - return None - - return super().filter_tensors(item) - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name.endswith(".A_log"): data_torch = -torch.exp(data_torch) @@ -536,97 +619,13 @@ class _Qwen35MRopeMixin: self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION) -class _Qwen35MtpMixin: - """Shared MTP wiring for Qwen3.5/3.6 text variants. The HF config carries - the MTP block under `mtp_num_hidden_layers` and the tensors under - `mtp.*`; we extend block_count, emit the nextn metadata key, and remap - `mtp.*` to the standard layer-indexed nextn naming so the existing - tensor_map handles them.""" - - supports_mtp_export = True - hparams: dict[str, Any] - model_arch: gguf.MODEL_ARCH - gguf_writer: gguf.GGUFWriter - block_count: int - tensor_map: gguf.TensorNameMap - no_mtp: bool - mtp_only: bool - _original_block_count: int | None = None - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] - if not self.no_mtp: - self.block_count += self.hparams.get("mtp_num_hidden_layers", 0) - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: - hparams = {**self.hparams, **self.hparams.get("text_config", {})} - key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None) - type(self)._original_block_count = hparams.get(key) - return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute] - - @classmethod - def filter_tensors(cls, item): - assert cls._original_block_count is not None - # TODO: change TextModel to super() - if (titem := TextModel.filter_tensors(item)) is None: - return None - name, gen = titem - if name.startswith("model.mtp."): - name = name.replace("model.", "", 1) - if name.startswith("mtp."): - if cls.no_mtp: - return None - remapper = { - "fc": "eh_proj", - "pre_fc_norm_embedding": "enorm", - "pre_fc_norm_hidden": "hnorm", - "norm": "shared_head.norm", - } - parts = name.split(".", 3) - if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal(): - mtp_idx = int(parts[2]) - name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}" - elif len(parts) == 3 and parts[1] in remapper: - name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}" - elif cls.mtp_only: - keep = name in ( - "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", - "embed_tokens.weight", "norm.weight", - ) - if not keep: - return None - return name, gen - - def set_gguf_parameters(self): - super().set_gguf_parameters() # ty: ignore[unresolved-attribute] - if self.no_mtp: - return - if (n := self.hparams.get("mtp_num_hidden_layers", 0)) > 0: - self.gguf_writer.add_nextn_predict_layers(n) - - def prepare_metadata(self, vocab_only: bool): - from_dir = self.fname_out.is_dir() - super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute] - - if not self.mtp_only or not from_dir: - return - - output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - fname_default: str = gguf.naming_convention( - self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" - - @ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM") -class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): +class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35 @ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM") -class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): +class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35MOE @@ -648,10 +647,13 @@ class DFlashModel(Qwen3Model): # own tokenizer logic, not the Qwen default). from . import get_model_class with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f: - target_arch = json.load(f)["architectures"][0] + target_hparams = json.load(f) + target_arch = target_hparams["architectures"][0] target_cls = get_model_class(target_arch) if target_cls is not type(self): + if target_arch == "NemotronHForCausalLM": + setattr(self, "is_moe", "num_experts_per_tok" in target_hparams) target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] else: super().set_vocab() @@ -689,6 +691,12 @@ class DFlashModel(Qwen3Model): name = "model." + name return super().filter_tensors((name, gen)) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name == "model.embed_tokens.weight" and not self.hparams.get("has_embed_tokens", True): + return + + yield from super().modify_tensors(data_torch, name, bid) + @ModelBase.register("Qwen3DSparkModel") class DSparkModel(DFlashModel): diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py new file mode 100644 index 000000000..d21a50595 --- /dev/null +++ b/conversion/qwen3tts.py @@ -0,0 +1,471 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Callable, Iterable, TYPE_CHECKING + +import torch +import torch.nn.functional as F + +if TYPE_CHECKING: + from torch import Tensor + +from .base import ModelBase, MmprojModel, TextModel, gguf + +# Tricks being used to support this model via existing llama.cpp code paths: +# - Text projection MLP is folded into the embedding table +# - codec_embedding is concat to the text embedding table, vocab is extended +# example: codec_bos_id(2149) --> "<|codec_bos|>" +# codec_eos_token_id(2150) --> "<|codec_eos_token|>" +# codec_language_id.chinese(2055) --> "<|codec_language_chinese|>" +# other rows --> "<|codec_0|>", "<|codec_1|>", ..., "<|codec_1023|>" +# - output tensor codec_head is smaller than vocab, so logits will be padded at inference time +# - suppress_tokens is used to limit the backbone to only sample either semantic or EOS (stop) token + +# pipeline stage mapping: +# speaker reference encoder --> mapped to normal mtmd audio encoder +# backbone --> mapped to normal libllama text model (autoregressive) +# code_predictor --> MTMD_GEN_PROCESS_TYPE_GEN_CODE +# code2wav --> MTMD_GEN_PROCESS_TYPE_GEN_WAV + +# torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act) +_ACT2FN = { + "silu": F.silu, + "gelu": F.gelu, + "relu": F.relu, +} + + +@ModelBase.register("Qwen3TTSForConditionalGeneration") +class Qwen3TTSTalkerModel(TextModel): + model_arch = gguf.MODEL_ARCH.QWEN3TTS + + _TEXT_PROJ_KEYS = ( + "model.text_embedding.weight", + "text_projection.linear_fc1.weight", + "text_projection.linear_fc1.bias", + "text_projection.linear_fc2.weight", + "text_projection.linear_fc2.bias", + ) + + _text_proj_buffer: dict[str, Tensor] + _folded_text_embed: Tensor | None + _codec_embed: Tensor | None + + def __init__(self, dir_model: Path, *args, **kwargs): + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + raw_talker_config = dict(hparams["talker_config"]) + self._talker_config = raw_talker_config + self.n_codec_vocab = raw_talker_config["vocab_size"] + talker_config = dict(raw_talker_config) + talker_config["vocab_size"] = talker_config["text_vocab_size"] + hparams["text_config"] = talker_config + super().__init__(dir_model, *args, hparams=hparams, **kwargs) + self._text_proj_buffer = {} + self._folded_text_embed = None + self._codec_embed = None + + def _codec_token_names(self) -> list[str]: + # start every row with a generic name, then override the ones with a + # known meaning (bos/eos/language/etc, derived from the *_id fields + # of talker_config) with a more descriptive one + names = [f"<|codec_{i}|>" for i in range(self.n_codec_vocab)] + for key, val in self._talker_config.items(): + if not key.endswith("_id"): + continue + prefix = key[:-len("_id")] + if isinstance(val, int): + names[val] = f"<|{prefix}|>" + elif isinstance(val, dict): + for subkey, subval in val.items(): + names[subval] = f"<|{prefix}_{subkey}|>" + return names + + def set_vocab(self): + codec_tokens = self._codec_token_names() + codec_toktypes = [gguf.TokenType.CONTROL] * len(codec_tokens) + + try: + tokens, scores, toktypes = self._create_vocab_sentencepiece() + self.gguf_writer.add_tokenizer_model("llama") + self.gguf_writer.add_tokenizer_pre("default") + tokens += [t.encode("utf-8") for t in codec_tokens] + scores += [0.0] * len(codec_tokens) + toktypes += codec_toktypes + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_scores(scores) + self.gguf_writer.add_token_types(toktypes) + special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) + special_vocab.add_to_gguf(self.gguf_writer) + return + except FileNotFoundError: + pass + + tokens, toktypes, tokpre = self.get_vocab_base() + tokens += codec_tokens + toktypes += codec_toktypes + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab.add_to_gguf(self.gguf_writer) + + # make sure that the model has no chat template, so chat will be disabled + self.gguf_writer.add_chat_template(None) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + # note: final vocab layout is [text_vocab | codec_vocab], with text_vocab is actually padded with -inf in cgraph + # for codec_vocab, only first 2048 rows can be sampled for semantic code + # plus codec_eos_token_id that used for signaling end of generation + # ref: https://github.com/QwenLM/Qwen3-TTS/blob/022e286b98fbec7e1e916cb940cdf532cd9f488e/qwen_tts/core/models/modeling_qwen3_tts.py#L2059-L2063 + + vocab_size = self.hparams["vocab_size"] + self.n_codec_vocab + codec_eos_token_id = self.hparams["vocab_size"] + self._talker_config["codec_eos_token_id"] + self.gguf_writer.add_suppress_tokens([ + i for i in range(vocab_size - 1024, vocab_size) + if i != codec_eos_token_id + ]) + self.gguf_writer.add_eos_token_id(codec_eos_token_id) + self.gguf_writer.add_add_eos_token(False) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if not name.startswith("talker.") or name.startswith("talker.code_predictor."): + return None + + name = name[len("talker."):] + return super().filter_tensors((name, gen)) + + def _maybe_emit_token_embd(self) -> Iterable[tuple[str, Tensor]]: + if self._folded_text_embed is None or self._codec_embed is None: + return + combined = torch.cat([self._folded_text_embed, self._codec_embed], dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), combined) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # codec_embedding rows are appended after the text vocab, extending the embedding table + if name == "model.codec_embedding.weight": + self._codec_embed = data_torch + yield from self._maybe_emit_token_embd() + return + + # codec_head is the output head for the (smaller) codec vocab; logits get padded to + # the extended vocab size at inference time + if name == "codec_head.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch) + return + + if name in self._TEXT_PROJ_KEYS: + self._text_proj_buffer[name] = data_torch + if len(self._text_proj_buffer) < len(self._TEXT_PROJ_KEYS): + return + + # fold MLP into the embedding table at conversion time, MLP won't be used at inference time anyway + act_fn = _ACT2FN[self.hparams["hidden_act"]] + embed = self._text_proj_buffer["model.text_embedding.weight"] + hidden = act_fn(F.linear(embed, + self._text_proj_buffer["text_projection.linear_fc1.weight"], + self._text_proj_buffer["text_projection.linear_fc1.bias"])) + folded = F.linear(hidden, + self._text_proj_buffer["text_projection.linear_fc2.weight"], + self._text_proj_buffer["text_projection.linear_fc2.bias"]) + self._folded_text_embed = folded + yield from self._maybe_emit_token_embd() + return + + yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Qwen3TTSForConditionalGeneration") +class Qwen3TTSSpeakerEncoderModel(MmprojModel): + has_vision_encoder = False + has_audio_encoder = True + + # talker.code_predictor.model.layers.{bid}. -> A_GEN_CODE_* + # bypass tensor_mapping.py for now to make it simple + _CODE_LAYER_TENSOR_MAP = { + "input_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, + "self_attn.q_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q, + "self_attn.q_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, + "self_attn.k_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K, + "self_attn.k_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, + "self_attn.v_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V, + "self_attn.o_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, + "post_attention_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, + "mlp.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, + "mlp.up_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, + "mlp.down_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, + } + + # note: codebook pages will be stacked to 3D + _CODE_GEN_N_CODEBOOKS = 15 + _code_embed_buffer: dict[int, Tensor] = {} + _code_head_buffer: dict[int, Tensor] = {} + _wav_config_cache: dict[str, Any] | None = None + + def __init__(self, dir_model: Path, *args, **kwargs): + hparams = kwargs.pop("hparams", None) + if hparams is None: + hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) + hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]} + # ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys + hparams["speaker_encoder_config"]["n_layers"] = 4 + super().__init__(dir_model, *args, hparams=hparams, **kwargs) + self._wav_config_cache = None + + def get_audio_config(self) -> dict[str, Any] | None: + return self.global_config.get("speaker_encoder_config") + + def set_gguf_parameters(self): + self.gguf_writer.add_file_type(self.ftype) + self.gguf_writer.add_clip_has_audio_encoder(True) + self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC) + + # handle speaker encoder config + self.gguf_writer.add_audio_projection_dim(self.n_embd_text) + # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default) + self.gguf_writer.add_audio_num_mel_bins(128) + # 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here + self.gguf_writer.add_audio_block_count(3) + # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it + self.gguf_writer.add_audio_embedding_length(1536) + self.gguf_writer.add_audio_head_count(1) + self.gguf_writer.add_audio_feed_forward_length(1536) + self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) + + # handle code predictor config + self.gguf_writer.add_clip_has_gen_audio_encoder(True) + self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_GEN) + code_predictor_config = self.global_config["talker_config"]["code_predictor_config"] + self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) + self.gguf_writer.add_gen_audio_embedding_length(code_predictor_config["hidden_size"]) + self.gguf_writer.add_gen_audio_feed_forward_length(code_predictor_config["intermediate_size"]) + self.gguf_writer.add_gen_audio_block_count(code_predictor_config["num_hidden_layers"]) + self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"]) + self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"]) + self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"]) + # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here + + def _wav_decoder_config(self) -> dict[str, Any] | None: + # code2wav has its own config.json, inside the speech_tokenizer dir + if self._wav_config_cache is None: + path = self.dir_model / "speech_tokenizer" / "config.json" + with open(path, "r", encoding="utf-8") as f: + cfg = json.load(f) + self._wav_config_cache = cfg["decoder_config"] + return self._wav_config_cache + + def tensor_force_quant(self, name, new_name, bid, n_dims): + # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path + if new_name.endswith(".weight") and ( + new_name in ("a.gen.wav.pre_conv.weight", "a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight") + or (".up.blk." in new_name and new_name.endswith(".dwconv.weight")) + or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight"))) + ): + return gguf.GGMLQuantizationType.F16 + # ConvTranspose1d kernels: only F16/F32 are implemented, no BF16 + if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name): + return gguf.GGMLQuantizationType.F32 + return super().tensor_force_quant(name, new_name, bid, n_dims) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if not ( + name.startswith("speaker_encoder.") + or name.startswith("talker.code_predictor.") + or name == "talker.model.codec_embedding.weight" + ): + return None + + return super().filter_tensors((name, gen)) + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + # code2wav tensors are already named by generate_extra_tensors(), pass them through + if name.startswith("a.gen.wav."): + yield (name, data_torch) + return + + # codebook-0 embedding, fed back to the talker backbone (codebooks 1-15 live in code_predictor) + if name == "talker.model.codec_embedding.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD), data_torch) + return + + if name == "talker.code_predictor.model.norm.weight": + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM), data_torch) + return + + if name.startswith("talker.code_predictor.small_to_mtp_projection."): + suffix = "." + name.rsplit(".", 1)[1] + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_PROJ_IN, suffix=suffix), data_torch) + return + + if name.startswith("talker.code_predictor.model.codec_embedding."): + idx = int(name.split("codec_embedding.")[1].split(".")[0]) + self._code_embed_buffer[idx] = data_torch + if len(self._code_embed_buffer) < self._CODE_GEN_N_CODEBOOKS: + return + stacked = torch.stack([self._code_embed_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_EMBD), stacked) + return + + if name.startswith("talker.code_predictor.lm_head."): + idx = int(name.split("lm_head.")[1].split(".")[0]) + self._code_head_buffer[idx] = data_torch + if len(self._code_head_buffer) < self._CODE_GEN_N_CODEBOOKS: + return + stacked = torch.stack([self._code_head_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0) + yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_HEAD), stacked) + return + + if name.startswith("talker.code_predictor.model.layers."): + rest = name.split("model.layers.")[1] # "{bid}..weight" + _, key_with_suffix = rest.split(".", 1) # ".weight" + key = key_with_suffix.rsplit(".", 1)[0] # "" + tensor = self._CODE_LAYER_TENSOR_MAP.get(key) + if tensor is not None: + yield (self.format_tensor_name(tensor, bid), data_torch) + return + + if "res2net_block.blocks." in name: + assert bid is not None # the outer stage index, picked up from the tensor name automatically + xid = int(name.split("res2net_block.blocks.")[1].split(".")[0]) + suffix = "." + name.rsplit(".", 1)[1] + new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CONV_RES2].format(bid=bid, xid=xid) + suffix + yield (new_name, data_torch) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + yield from self._generate_code2wav_tensors() + + def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]: + # code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors + from safetensors.torch import load_file + + wav_config = self._wav_decoder_config() + state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors") + + def get(name: str) -> Tensor: + return state_dict[name] + + def snake_fold(alpha: Tensor, beta: Tensor) -> tuple[Tensor, Tensor]: + # fold SnakeBeta's exp()/reciprocal here, so the graph is only mul/sin/sqr/mul/add + return torch.exp(alpha), 1.0 / (torch.exp(beta) + 1e-9) + + def rvq_codebook(prefix: str, n_layers: int) -> Tensor: + # checkpoint has EMA accumulators, so codebook[i] = embedding_sum[i] / cluster_usage[i] + books = [] + for i in range(n_layers): + embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum") + cluster_usage = get(f"{prefix}.vq.layers.{i}._codebook.cluster_usage") + books.append(embedding_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)) + return torch.stack(books, dim=0) if n_layers > 1 else books[0] + + T = gguf.MODEL_TENSOR + + # --- quantizer: RVQ codebook decode --- + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_IN), get("decoder.quantizer.rvq_first.input_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT), get("decoder.quantizer.rvq_first.output_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), rvq_codebook("decoder.quantizer.rvq_first", 1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_IN), get("decoder.quantizer.rvq_rest.input_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_OUT), get("decoder.quantizer.rvq_rest.output_proj.weight").squeeze(-1)) + yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB), rvq_codebook("decoder.quantizer.rvq_rest", self._CODE_GEN_N_CODEBOOKS)) + + # --- pre_conv --- + yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".weight"), get("decoder.pre_conv.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".bias"), get("decoder.pre_conv.conv.bias")) + + # --- pre_transformer --- + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".weight"), get("decoder.pre_transformer.input_proj.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".bias"), get("decoder.pre_transformer.input_proj.bias")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".weight"), get("decoder.pre_transformer.output_proj.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".bias"), get("decoder.pre_transformer.output_proj.bias")) + yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get("decoder.pre_transformer.norm.weight")) + + tfm_layer_map = { + "input_layernorm.weight": T.A_GEN_WAV_TFM_ATTN_NORM, + "self_attn.q_proj.weight": T.A_GEN_WAV_TFM_ATTN_Q, + "self_attn.k_proj.weight": T.A_GEN_WAV_TFM_ATTN_K, + "self_attn.v_proj.weight": T.A_GEN_WAV_TFM_ATTN_V, + "self_attn.o_proj.weight": T.A_GEN_WAV_TFM_ATTN_OUT, + "self_attn_layer_scale.scale": T.A_GEN_WAV_TFM_ATTN_SCALE, + "post_attention_layernorm.weight": T.A_GEN_WAV_TFM_FFN_NORM, + "mlp.gate_proj.weight": T.A_GEN_WAV_TFM_FFN_GATE, + "mlp.up_proj.weight": T.A_GEN_WAV_TFM_FFN_UP, + "mlp.down_proj.weight": T.A_GEN_WAV_TFM_FFN_DOWN, + "mlp_layer_scale.scale": T.A_GEN_WAV_TFM_FFN_SCALE, + } + assert wav_config is not None + for bid in range(wav_config["num_hidden_layers"]): + for key, tensor_id in tfm_layer_map.items(): + yield (self.format_tensor_name(tensor_id, bid), get(f"decoder.pre_transformer.layers.{bid}.{key}")) + + # --- upsample: 2x (causal ConvTranspose1d + ConvNeXt block) --- + up_map = { + "0.conv.weight": (T.A_GEN_WAV_UP_CONV, ".weight"), + "0.conv.bias": (T.A_GEN_WAV_UP_CONV, ".bias"), + "1.dwconv.conv.weight": (T.A_GEN_WAV_UP_DWCONV, ".weight"), + "1.dwconv.conv.bias": (T.A_GEN_WAV_UP_DWCONV, ".bias"), + "1.norm.weight": (T.A_GEN_WAV_UP_NORM, ".weight"), + "1.norm.bias": (T.A_GEN_WAV_UP_NORM, ".bias"), + "1.pwconv1.weight": (T.A_GEN_WAV_UP_PW1, ".weight"), + "1.pwconv1.bias": (T.A_GEN_WAV_UP_PW1, ".bias"), + "1.pwconv2.weight": (T.A_GEN_WAV_UP_PW2, ".weight"), + "1.pwconv2.bias": (T.A_GEN_WAV_UP_PW2, ".bias"), + "1.gamma": (T.A_GEN_WAV_UP_GAMMA, ""), + } + for bid in range(len(wav_config["upsampling_ratios"])): + for key, (tensor_id, suffix) in up_map.items(): + yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), get(f"decoder.upsample.{bid}.{key}")) + + # --- DAC decoder --- + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), get("decoder.decoder.0.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get("decoder.decoder.0.conv.bias")) + + n_dac_blocks = len(wav_config["upsample_rates"]) + for bid in range(n_dac_blocks): + py = bid + 1 # decoder.decoder.0 is the entry conv, blocks start at 1 + + a, b = snake_fold(get(f"decoder.decoder.{py}.block.0.alpha"), get(f"decoder.decoder.{py}.block.0.beta")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), get(f"decoder.decoder.{py}.block.1.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(f"decoder.decoder.{py}.block.1.conv.bias")) + + for xid in range(3): + ridx = xid + 2 # block.2/3/4 are the 3 residual units + + a1, b1 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act1.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act1.beta")) + name1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT1].format(bid=bid, xid=xid) + yield (name1 + ".alpha", a1) + yield (name1 + ".beta", b1) + + name_conv1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV1].format(bid=bid, xid=xid) + yield (name_conv1 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.weight")) + yield (name_conv1 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.bias")) + + a2, b2 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act2.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act2.beta")) + name2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT2].format(bid=bid, xid=xid) + yield (name2 + ".alpha", a2) + yield (name2 + ".beta", b2) + + name_conv2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV2].format(bid=bid, xid=xid) + yield (name_conv2 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.weight")) + yield (name_conv2 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.bias")) + + a5, b5 = snake_fold(get("decoder.decoder.5.alpha"), get("decoder.decoder.5.beta")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a5) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b5) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), get("decoder.decoder.6.conv.weight")) + yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get("decoder.decoder.6.conv.bias")) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index d95311ffc..6e74c8764 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -126,8 +126,12 @@ def parse_args() -> argparse.Namespace: help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.", ) parser.add_argument( - "--no-mtp", action="store_true", - help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.", + "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true", + help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.", + ) + parser.add_argument( + "--dspark", action="store_true", + help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.", ) parser.add_argument( "--mistral-format", action="store_true", @@ -258,13 +262,20 @@ def main() -> None: from conversion.mistral import MistralModel model_class = MistralModel - if args.mtp and args.no_mtp: - logger.error("--mtp and --no-mtp are mutually exclusive") + if sum((args.mtp, args.no_mtp, args.dspark)) > 1: + logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive") sys.exit(1) + if args.dspark: + if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM": + logger.error("--dspark is only supported for DeepseekV4ForCausalLM") + sys.exit(1) + from conversion.deepseek import DeepseekV4DSparkModel + model_class = DeepseekV4DSparkModel + if args.mtp or args.no_mtp: if not model_class.supports_mtp_export: - logger.error("--mtp / --no-mtp are not supported for %s", model_architecture) + logger.error("--mtp / --no-nextn are not supported for %s", model_architecture) sys.exit(1) if args.no_mtp: model_class.no_mtp = True diff --git a/docs/release.md b/docs/release.md new file mode 100644 index 000000000..4335ef9d4 --- /dev/null +++ b/docs/release.md @@ -0,0 +1,49 @@ +# Release process + +llama.cpp uses [semantic versioning](https://semver.org) (`MAJOR.MINOR.PATCH`). + +## Version bump guidelines + +| Change type | Version component | +|---|---| +| Breaking change to the public C API (`include/llama.h`) | `MAJOR` | +| Backward-compatible features, model support, or API addition | `MINOR` | +| Bug fix with no API change | `PATCH` | + +The version is set in the three variables at the top of the root `CMakeLists.txt`: + +```cmake +set(LLAMA_VERSION_MAJOR 0) +set(LLAMA_VERSION_MINOR 1) +set(LLAMA_VERSION_PATCH 0) +``` + +_A version bump should be included in the PR that introduces the change, or in a +dedicated bump commit merged before the release is cut._ + +_TODO: add PR labels (`semver: patch`, `semver: minor`, `semver: major`) to help +identify which PRs require a version bump before cutting a release._ + +## Making a release + +Releases are created by running the [make-release](.github/workflows/make-release.yml) +which is a manual workflow. + +The workflow creates an annotated git tag (e.g. `v0.1.0`) and pushes it to the +remote. No GitHub Release object is created, the tag is the release artifact. + +## Building a release + +By default, `LLAMA_BUILD_IS_DEV=ON` which appends a `-dev` suffix to `LLAMA_VERSION`, +marking the build as a nightly/development build. Distributors building from a +release tag must pass `-DLLAMA_BUILD_IS_DEV=OFF` to produce a clean version string +(e.g. `0.1.0` instead of `0.1.0-dev`). + +## How releases reach users +Currently releases are not published to github releases, only nightly/development +builds are available there. The way users can access releases are using the following +channels: + +- **llama-install.sh** — downloads pre-built binaries built from the release tag. +- **Package managers** — consume the git tag directly. +- **Build from source** — users clone the repo and check out the tag. diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd index 09f7a0400..d6bc85f44 100644 --- a/embd_res/kcpp_sdui.embd +++ b/embd_res/kcpp_sdui.embd @@ -5,18 +5,18 @@ Stable UI for KoboldCpp - - diff --git a/embd_res/klite.embd b/embd_res/klite.embd index 8da59c10d..a5d03f444 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -12,7 +12,7 @@ Current version indicated by LITEVER below. --> {#each modalities as modality (modality)} diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte index e74bd8456..36895c8e7 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte @@ -28,18 +28,18 @@ } let { - class: className = '', - style = '', + activeModelId, attachments = [], - readonly = false, - onFileRemove, - uploadedFiles = $bindable([]), + class: className = '', // Default to small size for form previews imageClass = '', imageHeight = 'h-24', imageWidth = 'w-auto', limitToSingleRow = false, - activeModelId + onFileRemove, + readonly = false, + style = '', + uploadedFiles = $bindable([]) }: Props = $props(); let carouselRef: HorizontalScrollCarousel | undefined = $state(); @@ -48,7 +48,7 @@ let previewFocusIndex = $state(0); let viewAllDialogOpen = $state(false); - let displayItems = $derived(getAttachmentDisplayItems({ uploadedFiles, attachments })); + let displayItems = $derived(getAttachmentDisplayItems({ attachments, uploadedFiles })); function openPreview(item: ChatAttachmentDisplayItem, event?: MouseEvent) { event?.stopPropagation(); diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index 143621cd9..ba06e1815 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -2,8 +2,8 @@ import { ChatAttachmentsListItemMcpPrompt, ChatAttachmentsListItemMcpResource, - ChatAttachmentsListItemThumbnailImage, - ChatAttachmentsListItemThumbnailFile + ChatAttachmentsListItemThumbnailFile, + ChatAttachmentsListItemThumbnailImage } from '$lib/components/app'; import { AttachmentType } from '$lib/enums'; import type { @@ -49,10 +49,10 @@ return { id, resource: { - uri: extra.uri, name: extra.name, + serverName: extra.serverName, title: extra.name, - serverName: extra.serverName + uri: extra.uri } }; } @@ -64,12 +64,12 @@ ? (item.attachment as DatabaseMessageExtraMcpPrompt) : item.uploadedFile?.mcpPrompt ? { - type: AttachmentType.MCP_PROMPT as const, - name: item.name, - serverName: item.uploadedFile.mcpPrompt.serverName, - promptName: item.uploadedFile.mcpPrompt.promptName, + arguments: item.uploadedFile.mcpPrompt.arguments, content: item.textContent ?? '', - arguments: item.uploadedFile.mcpPrompt.arguments + name: item.name, + promptName: item.uploadedFile.mcpPrompt.promptName, + serverName: item.uploadedFile.mcpPrompt.serverName, + type: AttachmentType.MCP_PROMPT as const } : null} {#if mcpPrompt} diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte index 636e93f22..f5452aade 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte @@ -1,8 +1,8 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte index 7c7cf5120..4be156edb 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte @@ -1,13 +1,13 @@ {#if show} diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte index 8a85df7d0..366c8372b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte @@ -1,7 +1,7 @@ @@ -34,7 +36,7 @@ {#each items as item, index (item.id)}
+ + {#if toolsStore.hasEnabledCwdTools} + + {/if} - import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes'; import { Plus } from '@lucide/svelte'; import { Button } from '$lib/components/ui/button'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { ATTACHMENT_TOOLTIP_TEXT } from '$lib/constants'; + import { ATTACHMENT_TOOLTIP_TEXT, ICON_CLASS_DEFAULT } from '$lib/constants'; interface Props { disabled?: boolean; diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index f81dcf09c..02bfadb7e 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -1,51 +1,30 @@ {#if isMobile.current} - + {#snippet trigger({ disabled, onclick })} {/snippet} {:else} - + {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index 998e8dcb4..9f163c6d6 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -1,15 +1,6 @@
{#if showAddButton}
- goto(ROUTES.MCP_SERVERS)} - /> +
{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte index ff6d39fdd..606fa6cd3 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte @@ -1,32 +1,36 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte index fdec5aca5..eaaba69de 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte @@ -1,8 +1,9 @@ - + Token usage details - + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte index 6e2616d36..67d705ae4 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte @@ -1,6 +1,6 @@ {#if modelId !== null && !isLoading} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte index af9ad010e..e6abb4a3e 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte @@ -1,15 +1,10 @@ + + + + + + + event.preventDefault()} + onCloseAutoFocus={(event) => event.preventDefault()} + class="w-[var(--bits-popover-anchor-width)] max-w-none rounded-xl border-border/50 p-0 shadow-xl" + > +
+ + + {#if !fileSearchEnabled} +
{searchUnavailableMessage}
+ {:else if query.trim() && (search.isSearching || queryResults.length > 0 || searchError)} + nav.setHover(index)} + /> + {/if} + + {#if pickerSupported && fileSearchEnabled} + + {/if} + + {#if homeBase && fileSearchEnabled} + + + + Searching in: + + {abbreviateHome(searchScope, homeBase)} + + {/if} +
+ + + + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryChip.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryChip.svelte new file mode 100644 index 000000000..23661d223 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryChip.svelte @@ -0,0 +1,70 @@ + + + +
+ + + {#if showTooltip && displayLabelTitle} + + + {#snippet child({ props })} + {displayLabel} + {/snippet} + + +

{displayLabelTitle}

+ + + {:else} + {displayLabel} + {/if} +
+ + {#if directory} +
+ +
+ {/if} + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryResultsList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryResultsList.svelte new file mode 100644 index 000000000..e8087d967 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryResultsList.svelte @@ -0,0 +1,73 @@ + + +
+ {#if isSearching && results.length === 0} +
Searching...
+ {:else if error} +
{error}
+ {:else if results.length === 0} +
No matching folders
+ {:else} + {#each results as path, index (path)} + + {/each} + {/if} +
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte new file mode 100644 index 000000000..b708ae047 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte @@ -0,0 +1,78 @@ + + +{#if useRichInput} + +{:else} + +{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte similarity index 74% rename from tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte rename to tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte index 3e683389f..e0c08c721 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte @@ -1,5 +1,5 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormFileInputInvisible.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputFileInputInvisible.svelte similarity index 100% rename from tools/ui/src/lib/components/app/chat/ChatForm/ChatFormFileInputInvisible.svelte rename to tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputFileInputInvisible.svelte diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte new file mode 100644 index 000000000..d87817adb --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte @@ -0,0 +1,867 @@ + + +
+
+
+ + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte index 36c82224a..3f178da18 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte @@ -1,13 +1,9 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte index 6647928b2..160c14ce8 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte @@ -1,8 +1,9 @@ {#if showSearchInput}
- +
{/if} -
+
{#if isLoading} {#if skeleton} {@render skeleton()} + {:else} +
+ {#each { length: skeletonCount } as _, rowIndex (rowIndex)} +
+
+
+
+
+
+
+ {/each} +
+ {/if} + {:else if items && items.length === 0} + {#if emptyMessage} +
{emptyMessage}
{/if} - {:else if items.length === 0} -
{emptyMessage}
{:else} {#each items as itemData, index (itemKey(itemData, index))} {@render item(itemData, index, index === selectedIndex)} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte index 4d82c6b58..045534f48 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte @@ -1,23 +1,37 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte index 5a2ab26fc..cbf7b972e 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte @@ -4,7 +4,7 @@ showBadge?: boolean; } - let { titleWidth = 'w-48', showBadge = false }: Props = $props(); + let { showBadge = false, titleWidth = 'w-48' }: Props = $props();
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte index c43a002e6..b09d346f1 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte @@ -1,6 +1,6 @@ @@ -42,6 +42,7 @@ align="start" sideOffset={12} class="w-[var(--bits-popover-anchor-width)] max-w-none rounded-xl border-border/50 p-0 shadow-xl {className}" + preventScroll={false} onkeydown={onKeydown} onOpenAutoFocus={(event) => event.preventDefault()} > diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte new file mode 100644 index 000000000..df654b25b --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte @@ -0,0 +1,142 @@ + + + + command.name} + scrollTrigger={nav.scrollTrigger} + > + {#snippet item(command, index, isSelected)} + {@const Icon = commandIcon[command.action]} + handleSelect(command)} + onmouseenter={() => { + if (!command.disabled) nav.setHover(index); + }} + > + +
+ /{command.name} + + {command.description} + +
+ + {/snippet} + + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte index f35d816de..9b5a57b9b 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte @@ -1,19 +1,18 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte index 638d10eef..074c69b84 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte @@ -1,8 +1,8 @@ @@ -66,7 +66,7 @@ {#if isAutocompleteActive && suggestions.length > 0}
{#each suggestions as suggestion, i (suggestion)} - {/if} - {/snippet} - - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte new file mode 100644 index 000000000..1c7c8f7d4 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte @@ -0,0 +1,278 @@ + + + { + if (!open) onClose(); + }} +> + + + + event.preventDefault()} + onCloseAutoFocus={(event) => event.preventDefault()} + class={[ + 'w-[var(--bits-popover-anchor-width)] max-w-none rounded-xl border-border/50 p-0 shadow-xl', + className + ]} + > + entry.type + ':' + entry.path} + scrollTrigger={nav.scrollTrigger} + > + {#snippet item(entry, index, isSelected)} + handleSelect(entry)} + onmouseenter={() => nav.setHover(index)} + > + {@const Icon = entry.type === FileMentionEntryType.DIRECTORY ? Folder : File} + +
+
+ {#if showTooltip} + + + {#snippet child({ props })} + {entry.name} + {/snippet} + + +

{entry.path}

+ + + {:else} + {entry.name} + {/if} + + {entry.type} + +
+ + + +
+ + {/snippet} + + + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte index 7c5dc85b2..dbe03e2e0 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte @@ -1,16 +1,30 @@ + {})} + onSelect={onCommandSelect ?? (() => {})} +/> + - {})} + onOpened={onMentionOpened} + onSelect={onMentionSelect ?? (() => {})} /> diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index b8068f790..78cb88721 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -1,26 +1,28 @@ -
+
{#if message.role === MessageRole.SYSTEM} - + {:else if mcpPromptExtra} - + + {:else if isSynthetic} + {:else if message.role === MessageRole.USER} - + {:else} {/if}
@@ -422,7 +410,17 @@ * once known; 500px sizes messages that have never been rendered. */ .chat-message { + --chat-message-intrinsic-size: 500px; content-visibility: auto; - contain-intrinsic-size: auto 500px; + contain-intrinsic-size: auto var(--chat-message-intrinsic-size); + } + + /* + * Synthetic rows (e.g. the working-directory change) are small, so an + * accurate placeholder keeps the injected row from inflating the + * auto-scroll offset; the 500px default is for ordinary bubbles. + */ + .chat-message--synthetic { + --chat-message-intrinsic-size: 40px; } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index 199d75fce..b92be9fbd 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -1,84 +1,55 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte index 30ce16be9..d69337960 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte @@ -1,5 +1,5 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte index 4cc4080c3..e6e18ae08 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte @@ -2,6 +2,7 @@ import { ChatMessageStatistics } from '$lib/components/app'; import { ChatMessageStatisticsMode } from '$lib/enums'; import type { UseProcessingStateReturn } from '$lib/hooks/use-processing-state.svelte'; + import { agenticStore } from '$lib/stores'; interface Props { message: DatabaseMessage; @@ -10,10 +11,27 @@ showMessageStats: boolean; } - let { message, isLoading, processingState, showMessageStats }: Props = $props(); + let { isLoading, message, processingState, showMessageStats }: Props = $props(); + + // A running agentic flow stamps per-turn timings on its root message at each + // turn boundary and the cumulative agentic totals only on exit; while it runs, + // show the session's live totals on the root message instead. + const liveLlm = $derived(agenticStore.getLiveLlmTotals(message.convId)); + const isLiveFlowRoot = $derived( + liveLlm !== null && agenticStore.getFlowRootMessageId(message.convId) === message.id + ); -{#if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms} +{#if showMessageStats && isLiveFlowRoot && liveLlm} + +{:else if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms} {@const agentic = message.timings.agentic} + import { Folder, FolderX } from '@lucide/svelte'; + import type { DatabaseMessage } from '$lib/types'; + import { parseCwdMessage } from '$lib/utils'; + + interface Props { + class?: string; + message: DatabaseMessage; + } + + let { class: className = '', message }: Props = $props(); + + // Parse the synthetic message content in the UI so the row reuses the + // exact same text the model saw, including any guidance suffix. + let info = $derived(parseCwdMessage(message.content)); + + +{#if info} +
+ {#if info.path === null} + + Working directory cleared + {:else} + + Set working directory to  + + {info.display} + + {/if} +
+{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte index 2dcb36baf..4563b1fa8 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte @@ -4,47 +4,20 @@ ChatMessageEditForm, ChatMessageMcpPromptContent } from '$lib/components/app'; - import { getMessageEditContext } from '$lib/contexts'; - import { MessageRole, McpPromptVariant } from '$lib/enums'; + import { getChatMessageEditContext } from '$lib/contexts'; + import { McpPromptVariant, MessageRole } from '$lib/enums'; import type { DatabaseMessageExtraMcpPrompt } from '$lib/types'; interface Props { class?: string; message: DatabaseMessage; mcpPrompt: DatabaseMessageExtraMcpPrompt; - siblingInfo?: ChatMessageSiblingInfo | null; - showDeleteDialog: boolean; - deletionInfo: { - totalCount: number; - userMessages: number; - assistantMessages: number; - messageTypes: string[]; - } | null; - onCopy: () => void; - onEdit: () => void; - onDelete: () => void; - onConfirmDelete: () => void; - onNavigateToSibling?: (siblingId: string) => void; - onShowDeleteDialogChange: (show: boolean) => void; } - let { - class: className = '', - message, - mcpPrompt, - siblingInfo = null, - showDeleteDialog, - deletionInfo, - onCopy, - onEdit, - onDelete, - onConfirmDelete, - onNavigateToSibling, - onShowDeleteDialogChange - }: Props = $props(); + let { class: className = '', mcpPrompt, message }: Props = $props(); // Get edit context - const editCtx = getMessageEditContext(); + const editCtx = getChatMessageEditContext();
- +
{/if} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte index 3d5dec3b6..9190c7e62 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte @@ -1,11 +1,11 @@ + +{#if isCwdChange} + +{:else} + {message.content} +{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte index 24b3be4c5..c6222f568 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte @@ -4,47 +4,20 @@ import { Button } from '$lib/components/ui/button'; import { Card } from '$lib/components/ui/card'; import { INPUT_CLASSES } from '$lib/constants'; - import { getMessageEditContext } from '$lib/contexts'; + import { getChatMessageEditContext } from '$lib/contexts'; import { KeyboardKey, MessageRole } from '$lib/enums'; - import { config } from '$lib/stores/settings.svelte'; + import { settingsStore } from '$lib/stores'; import { autoResizeTextarea, isIMEComposing } from '$lib/utils'; interface Props { class?: string; message: DatabaseMessage; - siblingInfo?: ChatMessageSiblingInfo | null; - showDeleteDialog: boolean; - deletionInfo: { - totalCount: number; - userMessages: number; - assistantMessages: number; - messageTypes: string[]; - } | null; - onCopy: () => void; - onEdit: () => void; - onDelete: () => void; - onConfirmDelete: () => void; - onNavigateToSibling?: (siblingId: string) => void; - onShowDeleteDialogChange: (show: boolean) => void; textareaElement?: HTMLTextAreaElement; } - let { - class: className = '', - message, - siblingInfo = null, - showDeleteDialog, - deletionInfo, - onCopy, - onEdit, - onDelete, - onConfirmDelete, - onNavigateToSibling, - onShowDeleteDialogChange, - textareaElement = $bindable() - }: Props = $props(); + let { class: className = '', message, textareaElement = $bindable() }: Props = $props(); - const editCtx = getMessageEditContext(); + const editCtx = getChatMessageEditContext(); function handleEditKeydown(event: KeyboardEvent) { if (event.key === KeyboardKey.ENTER && !event.shiftKey && !isIMEComposing(event)) { @@ -64,7 +37,7 @@ let contentHeight = $state(0); const MAX_HEIGHT = 200; // pixels - const currentConfig = config(); + const currentConfig = settingsStore.config; let showExpandButton = $derived(contentHeight > MAX_HEIGHT); @@ -218,20 +191,7 @@ {#if message.timestamp}
- +
{/if} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index b1daedfc8..a6fa2e250 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -1,22 +1,19 @@ @@ -103,13 +103,26 @@
{line.text}
- {#if line.image} - {line.image.name} + {#if line.media} + {#if line.media.type === AttachmentType.AUDIO} + {@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG} +
+ +
+ {:else} + {line.media.name} + {/if} {/if} {/each}
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte index b990c3898..6545cc39f 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte @@ -1,9 +1,11 @@ {#snippet execShellTitle()} + {#if cwd} + {wdDisplay} + $ + {/if} + {#if highlightedCommandHtml} {@html highlightedCommandHtml} {:else} @@ -200,10 +220,10 @@ > {#each outputLines as line, i (i)}
{line.text}
- {#if line.image} + {#if line.media?.type === AttachmentType.IMAGE} {line.image.name} @@ -232,6 +252,23 @@ ${markup}` : ''; } diff --git a/tools/ui/src/lib/utils/svg-to-png.ts b/tools/ui/src/lib/utils/svg-to-png.ts index d5a7f7d83..07b84b3f8 100644 --- a/tools/ui/src/lib/utils/svg-to-png.ts +++ b/tools/ui/src/lib/utils/svg-to-png.ts @@ -20,6 +20,7 @@ export function svgBase64UrlToPngDataURL( if (!ctx) { reject(new Error('Failed to get 2D canvas context.')); + return; } @@ -33,6 +34,7 @@ export function svgBase64UrlToPngDataURL( ctx.fillStyle = backgroundColor; ctx.fillRect(0, 0, canvas.width, canvas.height); } + ctx.drawImage(img, 0, 0, targetWidth, targetHeight); resolve(canvas.toDataURL(MimeTypeImage.PNG)); @@ -46,6 +48,7 @@ export function svgBase64UrlToPngDataURL( } catch (error) { const message = error instanceof Error ? error.message : String(error); const errorMessage = `Error converting SVG to PNG: ${message}`; + console.error(errorMessage, error); reject(new Error(errorMessage)); } diff --git a/tools/ui/src/lib/utils/text-files.ts b/tools/ui/src/lib/utils/text-files.ts index 3f7a55ebc..f77094047 100644 --- a/tools/ui/src/lib/utils/text-files.ts +++ b/tools/ui/src/lib/utils/text-files.ts @@ -4,8 +4,8 @@ */ import { DEFAULT_BINARY_DETECTION_OPTIONS } from '$lib/constants'; -import type { BinaryDetectionOptions } from '$lib/types'; import { FileExtensionText } from '$lib/enums'; +import type { BinaryDetectionOptions } from '$lib/types'; /** * Check if a filename indicates a text file based on its extension diff --git a/tools/ui/src/lib/utils/text.ts b/tools/ui/src/lib/utils/text.ts index 18a36eb8a..32bf1f38f 100644 --- a/tools/ui/src/lib/utils/text.ts +++ b/tools/ui/src/lib/utils/text.ts @@ -15,6 +15,7 @@ export function getPreviewText(content: string, max = 150): string { export function generateConversationTitle(content: string, useFirstLine: boolean = false): string { if (useFirstLine) { const firstLine = content.split(NEWLINE).find((line) => line.trim().length > 0); + return firstLine ? firstLine.trim() : content.trim(); } diff --git a/tools/ui/src/lib/utils/tool-call-meta.ts b/tools/ui/src/lib/utils/tool-call-meta.ts index 798ba7b25..b64bca786 100644 --- a/tools/ui/src/lib/utils/tool-call-meta.ts +++ b/tools/ui/src/lib/utils/tool-call-meta.ts @@ -15,11 +15,14 @@ export function tryParseToolResultObject( toolResultString: string | undefined ): Record | null { if (!toolResultString) return null; + try { const parsed: unknown = JSON.parse(toolResultString); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { return parsed as Record; } + return null; } catch { return null; diff --git a/tools/ui/src/lib/utils/uri-template.ts b/tools/ui/src/lib/utils/uri-template.ts index eb8dbfb36..4ba82719b 100644 --- a/tools/ui/src/lib/utils/uri-template.ts +++ b/tools/ui/src/lib/utils/uri-template.ts @@ -1,11 +1,10 @@ import { + LEADING_SLASHES_REGEX, TEMPLATE_EXPRESSION_REGEX, URI_SCHEME_SEPARATOR, - URI_TEMPLATE_OPERATORS, - URI_TEMPLATE_SEPARATORS, + URI_TEMPLATE_SYMBOLS, VARIABLE_EXPLODE_MODIFIER_REGEX, - VARIABLE_PREFIX_MODIFIER_REGEX, - LEADING_SLASHES_REGEX + VARIABLE_PREFIX_MODIFIER_REGEX } from '../constants'; /** @@ -25,6 +24,7 @@ import { */ export function normalizeResourceUri(uri: string): string { const schemeEnd = uri.indexOf(URI_SCHEME_SEPARATOR); + if (schemeEnd === -1) return uri; const scheme = uri.substring(0, schemeEnd); @@ -65,6 +65,7 @@ export function extractTemplateVariables(template: string): UriTemplateVariable[ const seen = new Set(); let match; + TEMPLATE_EXPRESSION_REGEX.lastIndex = 0; while ((match = TEMPLATE_EXPRESSION_REGEX.exec(template)) !== null) { @@ -117,7 +118,6 @@ export function expandTemplate(template: string, values: Record) .replace(VARIABLE_PREFIX_MODIFIER_REGEX, '') .trim() ); - const expandedParts = varNames .map((name: string) => values[name] ?? '') .filter((v: string) => v !== ''); @@ -125,60 +125,59 @@ export function expandTemplate(template: string, values: Record) if (expandedParts.length === 0) return ''; switch (operator) { - case URI_TEMPLATE_OPERATORS.RESERVED: + case URI_TEMPLATE_SYMBOLS.RESERVED: // Reserved expansion: no encoding - return expandedParts.join(URI_TEMPLATE_SEPARATORS.COMMA); - case URI_TEMPLATE_OPERATORS.FRAGMENT: + return expandedParts.join(URI_TEMPLATE_SYMBOLS.COMMA); + case URI_TEMPLATE_SYMBOLS.FRAGMENT: // Fragment expansion - return ( - URI_TEMPLATE_OPERATORS.FRAGMENT + expandedParts.join(URI_TEMPLATE_SEPARATORS.COMMA) - ); - case URI_TEMPLATE_OPERATORS.PATH_SEGMENT: + return URI_TEMPLATE_SYMBOLS.FRAGMENT + expandedParts.join(URI_TEMPLATE_SYMBOLS.COMMA); + case URI_TEMPLATE_SYMBOLS.PATH_SEGMENT: // Path segments - return URI_TEMPLATE_SEPARATORS.SLASH + expandedParts.join(URI_TEMPLATE_SEPARATORS.SLASH); - case URI_TEMPLATE_OPERATORS.LABEL: - // Label expansion return ( - URI_TEMPLATE_SEPARATORS.PERIOD + expandedParts.join(URI_TEMPLATE_SEPARATORS.PERIOD) + URI_TEMPLATE_SYMBOLS.PATH_SEGMENT + + expandedParts.join(URI_TEMPLATE_SYMBOLS.PATH_SEGMENT) ); - case URI_TEMPLATE_OPERATORS.PATH_PARAM: + case URI_TEMPLATE_SYMBOLS.LABEL: + // Label expansion + return URI_TEMPLATE_SYMBOLS.LABEL + expandedParts.join(URI_TEMPLATE_SYMBOLS.LABEL); + case URI_TEMPLATE_SYMBOLS.PATH_PARAM: // Path-style parameters return varNames .filter((_: string, i: number) => expandedParts[i]) .map( (name: string, i: number) => - `${URI_TEMPLATE_SEPARATORS.SEMICOLON}${name}=${expandedParts[i]}` + `${URI_TEMPLATE_SYMBOLS.PATH_PARAM}${name}=${expandedParts[i]}` ) .join(''); - case URI_TEMPLATE_OPERATORS.FORM_QUERY: + case URI_TEMPLATE_SYMBOLS.FORM_QUERY: // Form-style query return ( - URI_TEMPLATE_SEPARATORS.QUERY_PREFIX + + URI_TEMPLATE_SYMBOLS.FORM_QUERY + varNames .filter((_: string, i: number) => expandedParts[i]) .map( (name: string, i: number) => `${encodeURIComponent(name)}=${encodeURIComponent(expandedParts[i])}` ) - .join(URI_TEMPLATE_SEPARATORS.QUERY_CONTINUATION) + .join(URI_TEMPLATE_SYMBOLS.FORM_CONTINUATION) ); - case URI_TEMPLATE_OPERATORS.FORM_CONTINUATION: + case URI_TEMPLATE_SYMBOLS.FORM_CONTINUATION: // Form-style query continuation return ( - URI_TEMPLATE_SEPARATORS.QUERY_CONTINUATION + + URI_TEMPLATE_SYMBOLS.FORM_CONTINUATION + varNames .filter((_: string, i: number) => expandedParts[i]) .map( (name: string, i: number) => `${encodeURIComponent(name)}=${encodeURIComponent(expandedParts[i])}` ) - .join(URI_TEMPLATE_SEPARATORS.COMMA) + .join(URI_TEMPLATE_SYMBOLS.COMMA) ); default: // Simple string expansion (default operator) return expandedParts .map((v: string) => encodeURIComponent(v)) - .join(URI_TEMPLATE_SEPARATORS.COMMA); + .join(URI_TEMPLATE_SYMBOLS.COMMA); } } ); diff --git a/tools/ui/src/lib/utils/url.ts b/tools/ui/src/lib/utils/url.ts index f1bf9ecb8..1d44720e1 100644 --- a/tools/ui/src/lib/utils/url.ts +++ b/tools/ui/src/lib/utils/url.ts @@ -28,6 +28,7 @@ function isIpAddress(hostname: string): boolean { */ export function extractRootDomain(url: URL): string | null { const hostname = url.hostname.toLowerCase(); + if (!hostname || isIpAddress(hostname)) return null; const parts = hostname.split('.'); @@ -95,7 +96,6 @@ export function canonicalizeServerUrl(raw: string): string { try { const parsed = new URL(trimmed); const pathname = parsed.pathname.replace(TRAILING_SLASHES_REGEX, ''); - // Aggressive: drop the port unconditionally. We only use this for // equality checks between user-typed URLs and a hard-coded list of // recommendations, where the port can never carry distinguishing diff --git a/tools/ui/src/lib/utils/webp-to-png.ts b/tools/ui/src/lib/utils/webp-to-png.ts index ea5183802..8c61ecf85 100644 --- a/tools/ui/src/lib/utils/webp-to-png.ts +++ b/tools/ui/src/lib/utils/webp-to-png.ts @@ -20,6 +20,7 @@ export function webpBase64UrlToPngDataURL( if (!ctx) { reject(new Error('Failed to get 2D canvas context.')); + return; } @@ -33,6 +34,7 @@ export function webpBase64UrlToPngDataURL( ctx.fillStyle = backgroundColor; ctx.fillRect(0, 0, canvas.width, canvas.height); } + ctx.drawImage(img, 0, 0, targetWidth, targetHeight); resolve(canvas.toDataURL(MimeTypeImage.PNG)); @@ -46,6 +48,7 @@ export function webpBase64UrlToPngDataURL( } catch (error) { const message = error instanceof Error ? error.message : String(error); const errorMessage = `Error converting WebP to PNG: ${message}`; + console.error(errorMessage, error); reject(new Error(errorMessage)); } diff --git a/tools/ui/src/lib/utils/working-directory.ts b/tools/ui/src/lib/utils/working-directory.ts new file mode 100644 index 000000000..906142d1c --- /dev/null +++ b/tools/ui/src/lib/utils/working-directory.ts @@ -0,0 +1,167 @@ +/** + * Pure helpers for the working-directory picker search, backed by the + * server's `file_glob_search` tool. Queries starting from a root (`/`, + * `C:\`, `\\host\share`) or `~` navigate the tree (search the parent for + * the last segment); anything else glob-matches home-relative entries. + */ + +import { lastPathSegment } from './path-display'; +import { + GLOB, + HOME_TILDE, + LEADING_SLASHES_REGEX, + PATH_SEPARATOR, + SEARCH, + TRAILING_SLASHES_REGEX +} from '$lib/constants'; +import type { GlobEntry, GlobSearchArgs } from '$lib/types/glob'; + +export interface PathQuery { + parent: string; + last: string; +} + +/** + * Rewrite `\` into `/` when the query carries a Windows root. Elsewhere the + * backslash is left alone: it is a legal filename character on POSIX. + */ +function toPosixSeparators(query: string): string { + if (!GLOB.DRIVE_PREFIX_REGEX.test(query) && !query.startsWith(GLOB.WINDOWS_SEPARATOR)) + return query; + + return query.split(GLOB.WINDOWS_SEPARATOR).join(PATH_SEPARATOR); +} + +export function rootPrefixLength(path: string): number { + const unc = path.match(GLOB.UNC_ROOT_REGEX); + + if (unc) return unc[0].length; + + const drive = path.match(GLOB.DRIVE_ROOT_REGEX); + + if (drive) return drive[0].length; + + return path.startsWith(PATH_SEPARATOR) ? PATH_SEPARATOR.length : 0; +} + +/** A query starting from a root or from `~` is path navigation, not a home-relative glob. */ +export function splitPathQuery(query: string): PathQuery | null { + const normalized = toPosixSeparators(query); + const rootLength = rootPrefixLength(normalized); + + if (rootLength === 0 && !normalized.startsWith(HOME_TILDE)) return null; + + // a root keeps its trailing separator so it stays absolute on its own + const root = + rootLength > 0 + ? normalized.slice(0, rootLength).replace(TRAILING_SLASHES_REGEX, '') + PATH_SEPARATOR + : HOME_TILDE; + const rest = normalized + .slice(rootLength > 0 ? rootLength : HOME_TILDE.length) + .replace(LEADING_SLASHES_REGEX, '') + .replace(TRAILING_SLASHES_REGEX, ''); + const parentOf = (dirs: string) => + rootLength > 0 ? root + dirs : HOME_TILDE + PATH_SEPARATOR + dirs; + + if (!rest) return { last: '', parent: root }; + + const idx = rest.lastIndexOf(PATH_SEPARATOR); + + if (idx === -1) return { last: rest, parent: root }; + + return { last: rest.slice(idx + 1), parent: parentOf(rest.slice(0, idx)) }; +} + +export function buildCaseInsensitiveGlob(query: string): string { + let out = GLOB.WILDCARD; + + for (const c of query) { + const lo = c.toLowerCase(); + const up = c.toUpperCase(); + + if (lo !== up) out += GLOB.RANGE_OPEN + lo + up + GLOB.RANGE_CLOSE; + // glob metacharacters are escaped into a literal character class so a + // query like "a*b" matches a literal '*' instead of becoming "ab" + else if (GLOB.SPECIAL_CHARS.includes(c)) out += GLOB.RANGE_OPEN + c + GLOB.RANGE_CLOSE; + else out += c; + } + + return out + GLOB.WILDCARD; +} + +export function buildGlobSearchArgs( + query: string, + scopePath: string, + searchDepth: number +): GlobSearchArgs { + const pathQuery = splitPathQuery(query); + const path = pathQuery ? pathQuery.parent : scopePath; + const include = pathQuery + ? pathQuery.last + ? buildCaseInsensitiveGlob(pathQuery.last) + : GLOB.WILDCARD + : buildCaseInsensitiveGlob(query); + const maxDepth = pathQuery ? SEARCH.PATH_NAV_MAX_DEPTH : searchDepth; + + return { include, last: pathQuery?.last, maxDepth, path, rankQuery: pathQuery?.last ?? query }; +} + +const RANK_EXACT = 0; +const RANK_PREFIX = 1; +const RANK_SUBSTRING = 2; +const RANK_OTHER = 3; + +function rankScore(path: string, query: string): number { + const name = lastPathSegment(path).toLowerCase(); + const q = query.toLowerCase(); + + if (name === q) return RANK_EXACT; + + if (name.startsWith(q)) return RANK_PREFIX; + + if (name.includes(q)) return RANK_SUBSTRING; + + return RANK_OTHER; +} + +export function rankEntries(entries: GlobEntry[], query: string): GlobEntry[] { + return [...entries].sort( + (a, b) => + rankScore(a.path, query) - rankScore(b.path, query) || + a.path.length - b.path.length || + a.path.localeCompare(b.path) + ); +} + +export function joinPath(base: string, rel: string): string { + if (!base) return rel; + + return base.replace(TRAILING_SLASHES_REGEX, '') + PATH_SEPARATOR + rel; +} + +export function highlightMatch(text: string, query: string): { text: string; match: boolean }[] { + if (!query) return [{ match: false, text }]; + + const segments: { text: string; match: boolean }[] = []; + const lowerText = text.toLowerCase(); + const lowerQuery = query.toLowerCase(); + + let i = 0; + + while (i < text.length) { + const idx = lowerText.indexOf(lowerQuery, i); + + if (idx < 0) { + segments.push({ match: false, text: text.slice(i) }); + + break; + } + + if (idx > i) segments.push({ match: false, text: text.slice(i, idx) }); + + segments.push({ match: true, text: text.slice(idx, idx + query.length) }); + i = idx + query.length; + } + + return segments; +} diff --git a/tools/ui/src/routes/(chat)/+page.svelte b/tools/ui/src/routes/(chat)/+page.svelte index 9db1d445f..c4958b7c1 100644 --- a/tools/ui/src/routes/(chat)/+page.svelte +++ b/tools/ui/src/routes/(chat)/+page.svelte @@ -1,21 +1,20 @@ - {activeConversation()?.name || 'Chat'} - {APP_NAME} + {conversationsStore.activeConversation?.name || 'Chat'} - {APP_NAME} - import { page } from '$app/stores'; import { goto } from '$app/navigation'; + import { page } from '$app/stores'; import { ServerErrorSplash } from '$lib/components/app'; - import { ROUTES } from '$lib/constants/routes'; - import { APP_NAME } from '$lib/constants'; + import { APP_NAME, ROUTES } from '$lib/constants'; let error = $derived($page.error); let status = $derived($page.status); diff --git a/tools/ui/src/routes/+layout.svelte b/tools/ui/src/routes/+layout.svelte index ad2dd1560..f938c7edf 100644 --- a/tools/ui/src/routes/+layout.svelte +++ b/tools/ui/src/routes/+layout.svelte @@ -1,36 +1,39 @@ diff --git a/tools/ui/tests/client/components/ChatFormInputRichHarness.svelte b/tools/ui/tests/client/components/ChatFormInputRichHarness.svelte new file mode 100644 index 000000000..58768a1e8 --- /dev/null +++ b/tools/ui/tests/client/components/ChatFormInputRichHarness.svelte @@ -0,0 +1,27 @@ + + + diff --git a/tools/ui/tests/client/components/ChatFormPickersHarness.svelte b/tools/ui/tests/client/components/ChatFormPickersHarness.svelte new file mode 100644 index 000000000..e8ef6465b --- /dev/null +++ b/tools/ui/tests/client/components/ChatFormPickersHarness.svelte @@ -0,0 +1,51 @@ + diff --git a/tools/ui/tests/client/components/ChatFormTestWrapper.svelte b/tools/ui/tests/client/components/ChatFormTestWrapper.svelte new file mode 100644 index 000000000..7ec8bf7f8 --- /dev/null +++ b/tools/ui/tests/client/components/ChatFormTestWrapper.svelte @@ -0,0 +1,12 @@ + + + + + diff --git a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte index 14d938978..504f68597 100644 --- a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte +++ b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte @@ -2,8 +2,8 @@ // Mounts the real ChatMessages list against the real conversations store, so // the harness exercises `displayMessages` (which rebuilds every message's // toolMessages array) rather than a single message subtree. - import * as Tooltip from '$lib/components/ui/tooltip'; import ChatMessages from '$lib/components/app/chat/ChatMessages/ChatMessages.svelte'; + import * as Tooltip from '$lib/components/ui/tooltip'; import { conversationsStore } from '$lib/stores/conversations.svelte'; diff --git a/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte index 53d3b9cfd..b11ea0f10 100644 --- a/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte +++ b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte @@ -7,7 +7,7 @@ open: boolean; } - let { variant, open }: Props = $props(); + let { open, variant }: Props = $props(); {#if variant === 'content'} diff --git a/tools/ui/tests/client/components/McpServerFormWrapper.svelte b/tools/ui/tests/client/components/McpServerFormWrapper.svelte index fe2cc958b..7bbabbc82 100644 --- a/tools/ui/tests/client/components/McpServerFormWrapper.svelte +++ b/tools/ui/tests/client/components/McpServerFormWrapper.svelte @@ -1,6 +1,6 @@ + +
conversation
+ +{#if open} +
+ it.id} + > + {#snippet item(it, index, isSelected)} + {}}> + {it.label} + + {/snippet} + +
+{/if} diff --git a/tools/ui/tests/client/components/TestWrapper.svelte b/tools/ui/tests/client/components/TestWrapper.svelte index 1380ec851..3c874276a 100644 --- a/tools/ui/tests/client/components/TestWrapper.svelte +++ b/tools/ui/tests/client/components/TestWrapper.svelte @@ -1,6 +1,6 @@