diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml deleted file mode 100644 index fed9c877c..000000000 --- a/.github/workflows/make-release.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Make Release - -on: - workflow_dispatch: - inputs: - dry_run: - description: 'Dry run - validate without creating the tag' - required: true - type: boolean - default: true - -env: - GH_TOKEN: ${{ github.token }} - -permissions: - contents: write - -jobs: - make-release: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Run release checks - id: checks - run: bash scripts/make-release-checks.sh ${{ github.event.inputs.dry_run == 'true' && '--dry-run' || '' }} - env: - GITHUB_REPOSITORY: ${{ github.repository }} - - - name: Create release tag - if: ${{ github.event.inputs.dry_run == 'false' }} - run: | - VERSION="${{ steps.checks.outputs.version }}" - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - git tag -a "${VERSION}" -m "Release ${VERSION}" - git push origin "${VERSION}" - echo "Created and pushed tag ${VERSION}" - - - name: Dry run summary - if: ${{ github.event.inputs.dry_run == 'true' }} - run: | - echo "Dry run complete - all checks passed." - echo "Would have created tag: ${{ steps.checks.outputs.version }}" diff --git a/CMakeLists.txt b/CMakeLists.txt index 337940f93..7cfa63d2d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -478,8 +478,6 @@ add_library(common2 tools/mtmd/mtmd-helper.cpp tools/mtmd/mtmd-image.cpp tools/mtmd/mtmd-helper.h - tools/mtmd/mtmd-helper-gen.cpp - tools/mtmd/mtmd-helper-common.h tools/mtmd/clip.cpp tools/mtmd/clip.h src/unicode.h diff --git a/Makefile b/Makefile index cd4a03009..64e6684a4 100644 --- a/Makefile +++ b/Makefile @@ -55,8 +55,8 @@ ifdef KCPP_SANITIZE CFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error CXXFLAGS += -fsanitize=undefined -fsanitize-undefined-trap-on-error endif -CFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./tools/mtmd -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC -CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./tools/mtmd -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC +CFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c11 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC +CXXFLAGS += -I. -Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -I./common -I./vendor -I./vendor/stb -I./include -I./otherarch -I./otherarch/tools -I./otherarch/sdcpp -I./otherarch/ttscpp/include -I./otherarch/ttscpp/src -I./otherarch/qwen3tts -I./otherarch/sdcpp/thirdparty -I./include/vulkan -O3 -fno-finite-math-only -std=c++17 -fPIC -DLOG_DISABLE_LOGS -D_GNU_SOURCE -DGGML_USE_CPU -DGGML_USE_CPU_REPACK -DGGML_USE_RPC ifndef KCPP_DEBUG CFLAGS += -DNDEBUG -s @@ -579,8 +579,6 @@ mtmd.o: tools/mtmd/mtmd.cpp tools/mtmd/mtmd.h $(CXX) $(CXXFLAGS) -c $< -o $@ mtmd-helper.o: tools/mtmd/mtmd-helper.cpp tools/mtmd/mtmd-helper.h $(CXX) $(CXXFLAGS) -c $< -o $@ -mtmd-helper-gen.o: tools/mtmd/mtmd-helper-gen.cpp tools/mtmd/mtmd-helper-common.h - $(CXX) $(CXXFLAGS) -c $< -o $@ mtmd-image.o: tools/mtmd/mtmd-image.cpp tools/mtmd/mtmd-image.h $(CXX) $(CXXFLAGS) -c $< -o $@ unicode-common.o: common/unicode.cpp common/unicode.h @@ -669,7 +667,7 @@ ggml-vulkan-shaders-noext.o: ggml/src/ggml-vulkan-shaders-noext.cpp ggml/include $(CXX) $(CXXFLAGS) $(VKGEN_NOEXT_FORCE) $(VULKAN_FLAGS) -c $< -o $@ # intermediate objects -llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-kv-cache-msa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h +llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h $(CXX) $(CXXFLAGS) -c $< -o $@ llama-model.o: src/llama-model.cpp src/llama-model.h src/models/models.h ggml/include/ggml.h include/llama.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -690,7 +688,7 @@ budget.o: common/reasoning-budget.cpp common/reasoning-budget.h chat.o: common/chat.cpp common/chat.h $(CXX) $(CXXFLAGS) -c $< -o $@ -SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minimax_h3.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/audio_vae.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/minimax_h3_audio_vae.hpp src/model/vae/minimax_h3_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h +SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp @@ -766,35 +764,35 @@ clean: rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o # useful tools -main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) ggml/src/ggml-vulkan-shaders.cpp: ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -894,18 +892,15 @@ else @echo 'Vulkan Shaders (no extensions) Rebuilt for Linux...' endif -# common object files for all libraries -KOBOLDCPP_COMMON_OBJS = expose.o chat.o ggml-binops.o ggml-unops.o ggml-backend.o ggml-backend-meta.o ggml-repack.o llama.o llama-model.o embeddings_default.o music_default.o tts_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o $(OBJS) $(OBJS_SDTYPE) - #generated libraries -koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) +koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(DEFAULT_BUILD) -koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) +koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(DEFAULT_BUILD) ifdef FAILSAFE_BUILD -koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o gpttype_adapter_failsafe.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FAILSAFE) +koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) $(FAILSAFE_BUILD) else koboldcpp_failsafe: @@ -913,7 +908,7 @@ koboldcpp_failsafe: endif ifdef NOAVX2_BUILD -koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o gpttype_adapter_failsafe.o whispercpp_default.o clip_default.o ggml-backend-reg_default.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLE) +koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(NOAVX2_BUILD) else koboldcpp_noavx2: @@ -921,7 +916,7 @@ koboldcpp_noavx2: endif ifdef CUBLAS_BUILD -koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o gpttype_adapter_cublas.o whispercpp_cublas.o clip_cublas.o ggml-backend-reg_cublas.o $(CUBLAS_OBJS) $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) +koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) $(CUBLAS_BUILD) else koboldcpp_cublas: @@ -929,7 +924,7 @@ koboldcpp_cublas: endif ifdef HIPBLAS_BUILD -koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o gpttype_adapter_cublas.o whispercpp_cublas.o clip_cublas.o ggml-backend-reg_cublas.o $(HIP_OBJS) $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) +koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) $(HIPBLAS_BUILD) else koboldcpp_hipblas: @@ -937,12 +932,12 @@ koboldcpp_hipblas: endif ifdef VULKAN_BUILD -koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml_v3.o ggml_v2.o ggml_v1.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_FULL) +koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o llama.o chat.o llama-model.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(VULKAN_BUILD) ifdef NOAVX2_BUILD -koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLE) +koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(VULKAN_BUILD) -koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o whispercpp_vulkan.o clip_vulkan.o ggml-backend-reg_vulkan.o $(KOBOLDCPP_COMMON_OBJS) $(OBJS_SIMPLER) +koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) $(VULKAN_BUILD) else koboldcpp_vulkan_noavx2: @@ -960,15 +955,15 @@ koboldcpp_vulkan_failsafe: endif # tools -quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-helper-gen.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) diff --git a/cmake/arm64-windows-msvc-cuda.cmake b/cmake/arm64-windows-msvc-cuda.cmake deleted file mode 100644 index 370f2b3d2..000000000 --- a/cmake/arm64-windows-msvc-cuda.cmake +++ /dev/null @@ -1,26 +0,0 @@ -# Used to cross-compile ggml-cuda for Windows ARM64 on an x64 Windows host. -set( CMAKE_SYSTEM_NAME Windows ) -set( CMAKE_SYSTEM_PROCESSOR arm64 ) - -if ( DEFINED CUDAToolkit_ROOT ) - file( TO_CMAKE_PATH "${CUDAToolkit_ROOT}" CUDA_ROOT ) -elseif ( DEFINED ENV{CUDA_PATH} ) - file( TO_CMAKE_PATH "$ENV{CUDA_PATH}" CUDA_ROOT ) -else() - message( FATAL_ERROR "Set CUDAToolkit_ROOT or CUDA_PATH to a Windows CUDA Toolkit with ARM64 target libraries" ) -endif() - -if ( DEFINED ENV{VCToolsInstallDir} ) - file( TO_CMAKE_PATH "$ENV{VCToolsInstallDir}" MSVC_TOOLS_ROOT ) - set( CMAKE_CUDA_HOST_COMPILER "${MSVC_TOOLS_ROOT}/bin/Hostx64/arm64/cl.exe" CACHE FILEPATH "" ) -endif() - -set( CMAKE_CUDA_COMPILER "${CUDA_ROOT}/bin/nvcc.exe" CACHE FILEPATH "" ) -set( CMAKE_CUDA_FLAGS_INIT "-target-dir=arm64" ) - -# FindCUDAToolkit selects lib/x64 from the host architecture on Windows. -set( CUDA_CUDART "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" ) -set( CUDA_cudart_LIBRARY "${CUDA_ROOT}/lib/arm64/cudart.lib" CACHE FILEPATH "" ) -set( CUDA_cublas_LIBRARY "${CUDA_ROOT}/lib/arm64/cublas.lib" CACHE FILEPATH "" ) -set( CUDA_cublasLt_LIBRARY "${CUDA_ROOT}/lib/arm64/cublasLt.lib" CACHE FILEPATH "" ) -set( CUDA_cuda_driver_LIBRARY "${CUDA_ROOT}/lib/arm64/cuda.lib" CACHE FILEPATH "" ) diff --git a/common/arg.cpp b/common/arg.cpp index e1e9ef36e..3da048a63 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -36,7 +35,6 @@ #include #include #include -#include #include // for hardware_concurrency #include @@ -63,7 +61,6 @@ static std::initializer_list mmproj_examples = { LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, - LLAMA_EXAMPLE_TTS, }; static std::string read_file(const std::string & fname) { @@ -363,6 +360,7 @@ static bool spec_types_is_default(const common_params & params) { common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) { common_download_hf_plan plan; common_download_hf_plan plan_spec; + common_download_hf_plan plan_voc; common_download_opts opts; const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), @@ -377,10 +375,6 @@ common_models_handler common_models_handler_init(const common_params & params, l params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3) != params.speculative.types.end(); - const bool spec_type_draft_dspark = std::find(params.speculative.types.begin(), - params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK) != params.speculative.types.end(); - // only download mmproj if the current example is using it bool use_mmproj = false; for (const auto & ex : mmproj_examples) { @@ -395,7 +389,6 @@ common_models_handler common_models_handler_init(const common_params & params, l opts.download_mtp = spec_type_draft_mtp; opts.download_eagle3 = spec_type_draft_eagle3; opts.download_dflash = spec_type_draft_dflash; - opts.download_dspark = spec_type_draft_dspark; opts.download_mmproj = use_mmproj && !params.no_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty(); @@ -410,12 +403,15 @@ common_models_handler common_models_handler_init(const common_params & params, l opts_spec.download_mtp = true; opts_spec.download_dflash = true; opts_spec.download_eagle3 = true; - opts_spec.download_dspark = true; } plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec); } - return common_models_handler{plan, plan_spec, opts}; + if (!params.vocoder.model.hf_repo.empty()) { + plan_voc = common_download_get_hf_plan(params.vocoder.model, opts); + } + + return common_models_handler{plan, plan_spec, plan_voc, opts}; } bool common_models_handler_is_preset_repo(const common_models_handler & handler) { @@ -465,6 +461,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params auto & plan = handler.plan; auto & plan_spec = handler.plan_spec; + auto & plan_voc = handler.plan_voc; auto opts = handler.opts; // copy opts.callback = callback; @@ -479,6 +476,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params }; handle_url(params.model); handle_url(params.mmproj); + handle_url(params.vocoder.model); handle_url(params.speculative.draft.mparams); // optionally, if docker repo is set, resolve it @@ -506,6 +504,14 @@ void common_models_handler_apply(common_models_handler & handler, common_params task.opts = opts; tasks.push_back(task); } + if (!params.vocoder.model.url.empty()) { + common_download_task task; + task.url = params.vocoder.model.url; + task.local_path = params.vocoder.model.path; + task.opts = opts; + tasks.push_back(task); + } + bool had_spec_url = false; if (!params.speculative.draft.mparams.url.empty()) { common_download_task task; @@ -539,19 +545,12 @@ void common_models_handler_apply(common_models_handler & handler, common_params plan_spec.mtp = {}; plan_spec.dflash = {}; plan_spec.eagle3 = {}; - plan_spec.dspark = {}; } // infer the speculative type from the sidecar shipped by the draft repo when none is requested if (spec_types_is_default(params)) { if (!plan_spec.mtp.local_path.empty()) { params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_MTP }; - plan_spec.dspark = {}; - plan_spec.dflash = {}; - plan_spec.eagle3 = {}; - } else if (!plan_spec.dspark.local_path.empty()) { - // dspark outranks dflash, its sidecar carries the extra Markov head - params.speculative.types = { COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK }; plan_spec.dflash = {}; plan_spec.eagle3 = {}; } else if (!plan_spec.dflash.local_path.empty()) { @@ -562,20 +561,10 @@ void common_models_handler_apply(common_models_handler & handler, common_params } } - // infer the speculative type from the draft GGUF metadata when none is requested - // note: reads only the first split - sharded drafts need an explicit --spec-type - if (spec_types_is_default(params) && !params.speculative.draft.mparams.path.empty()) { - const auto types_gguf = common_speculative_types_from_gguf(params.speculative.draft.mparams.path); - if (!types_gguf.empty()) { - params.speculative.types = types_gguf; - } - } - // when a sidecar type is requested, the draft repo resolves to its sidecar instead of a full model const bool spec_sidecar_found = !plan_spec.mtp.local_path.empty() || !plan_spec.dflash.local_path.empty() || - !plan_spec.eagle3.local_path.empty() || - !plan_spec.dspark.local_path.empty(); + !plan_spec.eagle3.local_path.empty(); if (!plan_spec.mtp.local_path.empty() && !had_spec_url) { tasks.emplace_back(plan_spec.mtp, opts, [&]() { // only use the discovered MTP head when no draft path is set yet @@ -606,16 +595,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }); } - if (!plan_spec.dspark.local_path.empty() && !had_spec_url) { - tasks.emplace_back(plan_spec.dspark, opts, [&]() { - // only use the discovered DSpark sidecar when no draft path is set yet - if (params.speculative.draft.mparams.path.empty()) { - params.speculative.draft.mparams.path = hf_cache::finalize_file(plan_spec.dspark); - } else { - hf_cache::finalize_file(plan_spec.dspark); - } - }); - } // a wired draft sidecar counts as an explicit draft for the main plan fallback below if (spec_sidecar_found) { @@ -628,6 +607,11 @@ void common_models_handler_apply(common_models_handler & handler, common_params had_spec_url = true; } + // handle vocoder plan (e.g. --hf-repo-v) + if (!plan_voc.model_files.empty()) { + add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model); + } + if (!plan.model_files.empty()) { add_tasks(plan.model_files, plan.primary, params.model); } @@ -666,16 +650,6 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }); } - if (!plan.dspark.local_path.empty() && !had_spec_url) { - tasks.emplace_back(plan.dspark, opts, [&]() { - // only fall back to the discovered DSpark sidecar when no draft was explicitly provided - if (params.speculative.draft.mparams.empty()) { - params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.dspark); - } else { - hf_cache::finalize_file(plan.dspark); - } - }); - } if (!plan.preset.local_path.empty()) { tasks.emplace_back(plan.preset, opts, [&]() { // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file @@ -715,61 +689,12 @@ void common_models_handler_apply(common_models_handler & handler, common_params // CLI argument parsing functions // -// apply config files (if present), a later file overrides an earlier one: -// 1. system-wide: /etc/llama.cpp/config.ini (%PROGRAMDATA%\llama.cpp\config.ini on windows) -// 2. user-level: ${XDG_CONFIG_HOME:-~/.config}/llama.cpp/config.ini (%APPDATA%\llama.cpp\config.ini on windows) -static void common_params_apply_system_config(common_params & params, llama_example ex) { - std::vector paths; - -#if defined(_WIN32) - const std::string program_data = common_get_env("PROGRAMDATA"); - if (!program_data.empty()) { - paths.push_back(program_data + "\\llama.cpp\\config.ini"); - } -#else - paths.push_back("/etc/llama.cpp/config.ini"); -#endif - - try { - paths.push_back(fs_get_config_directory() + "config.ini"); - } catch (const std::exception & e) { - LOG_DBG("cannot read user-level config file, skipping: %s\n", e.what()); - } - - std::vector found; - for (const auto & path : paths) { - std::error_code ec; - if (std::filesystem::exists(path, ec)) { - found.push_back(path); - } - } - if (found.empty()) { - return; - } - - common_preset_context ctx(ex); - ctx.ignore_unknown_keys = true; // the same config file is shared by all programs - for (const auto & path : found) { - LOG_INF("using config file: %s\n", path.c_str()); - common_preset global; - common_presets presets = ctx.load_from_ini(path, global); - global.apply_to_params(params); - auto it = presets.find(COMMON_PRESET_DEFAULT_NAME); - if (it != presets.end()) { - it->second.apply_to_params(params); - } - } -} - static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) { common_params & params = ctx_arg.params; // setup log directly from params.verbosity: see tools/cli/cli.cpp common_log_set_verbosity_thold(params.verbosity); - // config file applies first, so env variables and CLI arguments override it - common_params_apply_system_config(params, ctx_arg.ex); - std::unordered_map> arg_to_options; for (auto & opt : ctx_arg.options) { for (const auto & arg : opt.args) { @@ -1402,10 +1327,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.n_parallel = -1; // auto by default } else if (ex == LLAMA_EXAMPLE_TOKENIZE) { params.parse_special = true; // parse special tokens by default, like the old tokenize tool - } else if (ex == LLAMA_EXAMPLE_TTS) { - params.out_file = "output.wav"; - params.sampling.penalty_repeat = 1.05f; - params.sampling.penalty_last_n = -1; } params.use_color = tty_can_use_colors(); @@ -1450,7 +1371,8 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--version"}, "show version and build info", [](common_params &) { - llama_print_build_info(llama_version()); + fprintf(stderr, "version: %d (%s)\n", llama_build_number(), llama_commit()); + fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target()); exit(0); } )); @@ -2067,9 +1989,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_sampling()); add_opt(common_arg( {"--repeat-last-n"}, "N", - string_format("last n tokens to consider for penalize (default: %d, 0 = disabled)", params.sampling.penalty_last_n), + string_format("last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)", params.sampling.penalty_last_n), [](common_params & params, int value) { - if (value < 0) { + if (value < -1) { throw std::runtime_error(string_format("error: invalid repeat-last-n = %d\n", value)); } params.sampling.penalty_last_n = value; @@ -2081,13 +2003,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--repeat-penalty"}, "N", string_format("penalize repeat sequence of tokens (default: %.2f, 1.0 = disabled)", (double)params.sampling.penalty_repeat), [](common_params & params, const std::string & value) { - const float penalty_repeat = std::stof(value); - if (!std::isfinite(penalty_repeat) || - penalty_repeat <= 0.0f || - !std::isfinite(1.0f/penalty_repeat)) { - throw std::runtime_error("error: repeat-penalty must be finite and greater than 0\n"); - } - params.sampling.penalty_repeat = penalty_repeat; + params.sampling.penalty_repeat = std::stof(value); params.sampling.user_sampling_config |= common_params_sampling_config::COMMON_PARAMS_SAMPLING_CONFIG_PENALTY_REPEAT; } ).set_sampling()); @@ -2095,22 +2011,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--presence-penalty"}, "N", string_format("repeat alpha presence penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_present), [](common_params & params, const std::string & value) { - const float penalty_present = std::stof(value); - if (!std::isfinite(penalty_present)) { - throw std::runtime_error("error: presence-penalty must be finite\n"); - } - params.sampling.penalty_present = penalty_present; + params.sampling.penalty_present = std::stof(value); } ).set_sampling()); add_opt(common_arg( {"--frequency-penalty"}, "N", string_format("repeat alpha frequency penalty (default: %.2f, 0.0 = disabled)", (double)params.sampling.penalty_freq), [](common_params & params, const std::string & value) { - const float penalty_freq = std::stof(value); - if (!std::isfinite(penalty_freq)) { - throw std::runtime_error("error: frequency-penalty must be finite\n"); - } - params.sampling.penalty_freq = penalty_freq; + params.sampling.penalty_freq = std::stof(value); } ).set_sampling()); add_opt(common_arg( @@ -2140,9 +2048,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_sampling()); add_opt(common_arg( {"--dry-penalty-last-n"}, "N", - string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable)", params.sampling.dry_penalty_last_n), + string_format("set DRY penalty for the last n tokens (default: %d, 0 = disable, -1 = context size)", params.sampling.dry_penalty_last_n), [](common_params & params, int value) { - if (value < 0) { + if (value < -1) { throw std::runtime_error(string_format("error: invalid dry-penalty-last-n = %d\n", value)); } params.sampling.dry_penalty_last_n = value; @@ -2626,7 +2534,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.mtmd_batch_max_tokens = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS")); - if (params.is_gen_docs || llama_supports_rpc()) { + if (llama_supports_rpc()) { add_opt(common_arg( {"--rpc"}, "SERVERS", "comma-separated list of RPC servers (host:port)", @@ -2664,16 +2572,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_env("LLAMA_ARG_DIO")); add_opt(common_arg( {"-lm", "--load-mode"}, "MODE", - "model loading mode (default: auto)\n" - "- auto: mmap, unless a device does not support it\n" + "model loading mode (default: mmap)\n" "- none: no special loading mode\n" "- mmap: memory-map model (if mmap disabled, slower load but may reduce pageouts if not using mlock)\n" "- mlock: force system to keep model in RAM rather than swapping or compressing\n" "- mmap+mlock: mmap + force system to keep model in RAM rather than swapping or compressing\n" "- dio: use DirectIO if available\n", [](common_params & params, const std::string & value) { - /**/ if (value == "auto") { params.load_mode = LLAMA_LOAD_MODE_AUTO; } - else if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; } + /**/ if (value == "none") { params.load_mode = LLAMA_LOAD_MODE_NONE; } else if (value == "mmap") { params.load_mode = LLAMA_LOAD_MODE_MMAP; } else if (value == "mlock") { params.load_mode = LLAMA_LOAD_MODE_MLOCK; } else if (value == "mmap+mlock") { params.load_mode = LLAMA_LOAD_MODE_MMAP_MLOCK; } @@ -3029,6 +2935,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.model.hf_file = value; } ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE")); + add_opt(common_arg( + {"-hfv", "-hfrv", "--hf-repo-v"}, "/[:quant]", + "Hugging Face model repository for the vocoder model (default: unused)", + [](common_params & params, const std::string & value) { + params.vocoder.model.hf_repo = value; + } + ).set_env("LLAMA_ARG_HF_REPO_V")); + add_opt(common_arg( + {"-hffv", "--hf-file-v"}, "FILE", + "Hugging Face model file for the vocoder model (default: unused)", + [](common_params & params, const std::string & value) { + params.vocoder.model.hf_file = value; + } + ).set_env("LLAMA_ARG_HF_FILE_V")); add_opt(common_arg( {"-hft", "--hf-token"}, "TOKEN", "Hugging Face access token (default: value from HF_TOKEN environment variable)", @@ -3363,23 +3283,12 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--tools"}, "TOOL1,TOOL2,...", "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n" "specify \"all\" to enable all tools\n" - "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n" + "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime\n" "note: for security reasons, this will limit --cors-origins to localhost by default", [](common_params & params, const std::string & value) { params.server_tools = parse_csv_row(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS")); - add_opt(common_arg( - {"--tools-runtime"}, "OPTION", - "experimental: run tools in a separate runtime environment (default: none, use host environment)\n" - "available options:\n" - " 'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit\n" - " 'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit\n" - " 'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required\n", - [](common_params & params, const std::string & value) { - params.server_tools_runtime = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS_RUNTIME")); add_opt(common_arg( {"--mcp-servers-config"}, "PATH", "experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)\n" @@ -3647,18 +3556,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING")); - add_opt(common_arg( - {"--reasoning-effort"}, "LEVEL", - "reasoning effort level given to the chat template: 'default' to keep the template default,\n" - "or a level such as 'minimal', 'low', 'medium', 'high', 'xhigh' or 'max' (default: default)", - [](common_params & params, const std::string & value) { - if (value == "default") { - params.default_template_kwargs.erase("reasoning_effort"); - } else { - params.default_template_kwargs["reasoning_effort"] = json(value).dump(); - } - } - ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_EFFORT")); add_opt(common_arg( {"--reasoning-budget"}, "N", "token budget for thinking: -1 for unrestricted, 0 for immediate end, N>0 for token budget (default: -1)", @@ -4078,9 +3975,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--spec-draft-n-max"}, "N", string_format("number of tokens to draft for speculative decoding (default: %d)", params.speculative.draft.n_max), [](common_params & params, int value) { - if (value < 0) { - throw std::invalid_argument("invalid value"); - } params.speculative.draft.n_max = value; } ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_LOOKUP, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_N_MAX")); @@ -4330,18 +4224,24 @@ common_params_context common_params_parser_init(common_params & params, llama_ex // add_opt(common_arg( - {"--tts-lang"}, "FNAME", - "language (ISO 639-1) for audio generation\n" - "see tts/README.md for per-model usage notes", + {"-mv", "--model-vocoder"}, "FNAME", + "vocoder model for audio generation (default: unused)", [](common_params & params, const std::string & value) { - params.tts_lang = value; + params.vocoder.model.path = value; } - ).set_examples({LLAMA_EXAMPLE_TTS})); + ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); + add_opt(common_arg( + {"--tts-use-guide-tokens"}, + "Use guide tokens to improve TTS word recall", + [](common_params & params) { + params.vocoder.use_guide_tokens = true; + } + ).set_examples({LLAMA_EXAMPLE_TTS, LLAMA_EXAMPLE_SERVER})); add_opt(common_arg( {"--tts-speaker-file"}, "FNAME", "speaker file path for audio generation", [](common_params & params, const std::string & value) { - params.tts_speaker_file = value; + params.vocoder.speaker_file = value; } ).set_examples({LLAMA_EXAMPLE_TTS})); @@ -4461,6 +4361,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_DEBUG})); // presets + add_opt(common_arg( + {"--tts-oute-default"}, + string_format("use default OuteTTS models (note: can download weights from the internet)"), + [](common_params & params) { + params.model.hf_repo = "OuteAI/OuteTTS-0.2-500M-GGUF"; + params.model.hf_file = "OuteTTS-0.2-500M-Q8_0.gguf"; + params.vocoder.model.hf_repo = "ggml-org/WavTokenizer"; + params.vocoder.model.hf_file = "WavTokenizer-Large-75-F16.gguf"; + } + ).set_examples({LLAMA_EXAMPLE_TTS})); add_opt(common_arg( {"--embd-gemma-default"}, diff --git a/common/arg.h b/common/arg.h index 44b9e887c..8f609e356 100644 --- a/common/arg.h +++ b/common/arg.h @@ -137,6 +137,7 @@ void common_params_add_preset_options(std::vector & args); struct common_models_handler { common_download_hf_plan plan; common_download_hf_plan plan_spec; + common_download_hf_plan plan_voc; common_download_opts opts; }; diff --git a/common/build-info.cpp.in b/common/build-info.cpp.in index 4ec339708..f888fd079 100644 --- a/common/build-info.cpp.in +++ b/common/build-info.cpp.in @@ -29,7 +29,7 @@ const char * llama_build_info(void) { return s.c_str(); } -void llama_print_build_info(const char * llama_version) { - fprintf(stderr, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit()); - fprintf(stderr, "built with %s for %s\n", llama_compiler(), llama_build_target()); +void llama_print_build_info(void) { + fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit()); + fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target()); } diff --git a/common/build-info.h b/common/build-info.h index e6a696acb..afc7b49a1 100644 --- a/common/build-info.h +++ b/common/build-info.h @@ -2,7 +2,6 @@ #define BUILD_INFO_H #define LLAMA_BUILD_NUMBER 999 -#define LLAMA_VERSION "1.0" #define LLAMA_COMMIT "KOBOLDCPP" #define LLAMA_COMPILER "KCPP" #define LLAMA_TARGET "KCPP" @@ -33,7 +32,7 @@ static inline const char * llama_build_info(void) { return s.c_str(); } -static inline void llama_print_build_info(const char *) { +static inline void llama_print_build_info(void) { fprintf(stderr, "%s: build = %d (%s)\n", __func__, llama_build_number(), llama_commit()); fprintf(stderr, "%s: built with %s for %s\n", __func__, llama_compiler(), llama_build_target()); } diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 06737b165..f786f5ff2 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -6,9 +6,6 @@ #include -#include -#include - using ordered_json = nlohmann::ordered_json; static std::string_view trim_trailing_space(std::string_view sv, int max = -1) { @@ -238,43 +235,6 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri return zero_or_more(choice({ p, content_chunk })); } -common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix, - const std::vector & parsers) { - if (parsers.empty()) { - return eps(); - } - - if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) { - return sequence(parsers); - } - - std::map rules; - std::function remaining_of; - - remaining_of = [&](uint32_t remaining) -> common_peg_parser { - if (remaining == 0) { - return eps(); - } - - auto cached = rules.find(remaining); - if (cached != rules.end()) { - return cached->second; - } - - auto alternatives = choice(); - for (size_t i = 0; i < parsers.size(); i++) { - const uint32_t bit = 1u << i; - if (remaining & bit) { - alternatives |= parsers[i] + remaining_of(remaining & ~bit); - } - } - - return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second; - }; - - return remaining_of((1u << parsers.size()) - 1); -} - std::string & common_chat_peg_mapper::args_target() { return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer; } @@ -594,7 +554,9 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls( // Full argument: name="value" or name=value auto arg_rule = tool_arg( - tool_arg_open(tool_arg_name(arg_name_parser) + literal("=")) + + tool_arg_open(eps()) + + tool_arg_name(arg_name_parser) + + literal("=") + arg_value_parser + tool_arg_close(eps()) ); diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index 5d764dbaa..cd14f2c11 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -55,8 +55,6 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper { struct content_structure; struct tool_call_structure; -constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6; - class common_chat_peg_builder : public common_peg_parser_builder { public: // Tag constants (from former common_chat_peg_base_builder) @@ -107,9 +105,6 @@ class common_chat_peg_builder : public common_peg_parser_builder { common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); } - // Matches every parser exactly once, in any order. - common_peg_parser permute(const std::string & rule_prefix, const std::vector & parsers); - // Return a parser that parses the prefix of a string, up to a given delimiter. common_peg_parser prefix(const std::string & s, const std::string & delimiter = {}); diff --git a/common/chat.cpp b/common/chat.cpp index 803b4021b..5c38b0578 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -934,10 +934,6 @@ static std::string common_chat_template_direct_apply_impl( bool enabled = inp["preserve_reasoning"].get(); jinja::caps_apply_preserve_reasoning(ctx, enabled); } - if (inp.contains("reasoning_effort") && inp["reasoning_effort"].is_string() && !inp["reasoning_effort"].empty()) { - std::string reasoning_effort = inp["reasoning_effort"].get(); - jinja::caps_apply_reasoning_effort(ctx, reasoning_effort); - } jinja::global_from_json(ctx, inp, inputs.mark_input); @@ -1128,179 +1124,6 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ return data; } -static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - const std::string GEN_PREFIX = "<|im_start|>assistant\n"; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - - auto supports_reasoning = tmpl.source().find("") != std::string::npos; - - data.supports_thinking = supports_reasoning; - data.preserved_tokens = { - "", - "", - }; - - if (supports_reasoning) { - data.thinking_start_tag = ""; - // Support both and as reasoning end sequences. - // ", "" }; - data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); - } - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash - { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = GEN_PREFIX; - if (supports_reasoning) { - data.generation_prompt += "\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "\n\n\n"; - } - } - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - std::vector tool_call_starts = { "" }; - - // Match complete opener for Qwen3-Coder models that occasionally omit the - // starting . The model may hallucinate a tool name, but it is preferable over - // constraining on - foreach_function(inputs.tools, [&](const json & tool) { - const std::string name = tool.at("function").at("name"); - tool_call_starts.push_back(""); - }); - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal(GEN_PREFIX); - - auto reasoning = p.eps(); - if (supports_reasoning && extract_reasoning) { - reasoning = p.optional("" + p.space() + - p.reasoning(p.until_one_of({ "", "" })) + - (p.literal("") | p.peek(p.literal("")))); - } - - // Response format parser - if (has_response_format) { - return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); - } - - // Tool call parser - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto arg_close = p.tool_arg_close(p.literal("\n\n")); - auto arg_string = p.rule("xml-arg-string", - p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(parameters); - - std::vector required_args; - std::vector optional_args; - - foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { - auto rule_name = "tool-" + name + "-arg-" + param_name; - - auto arg_open = p.tool_arg_open("\n"); - - auto arg_value = schema_info.resolves_to_string(param_schema) ? - arg_string : - p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; - - auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); - - (is_required ? required_args : optional_args).push_back(arg_rule); - }); - - // Accept required arguments in any order, as Qwen does not always adhere to the - // order provided. - auto args = p.permute("tool-" + name + "-args", required_args); - if (!optional_args.empty()) { - args = args + p.zero_or_more(p.choice(optional_args)); - } - - auto func = p.tool(p.tool_open("\n") + - p.tool_args(args) + - p.tool_close(p.literal("\n"))); - - tool_choice |= p.rule("tool-" + name, func); - }); - - auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; - - // Qwen3-Coder models may occasionally omit the token. - auto tool_call_body = tool_choice + "" + p.space(); - auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body); - auto tool_call = p.rule("tool-call", "\n" + tool_call_body); - - auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; - auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); - - return generation_prompt + - (reasoning << p.content(p.until_one_of(tool_call_starts)) << tool_calls); - } - - // Content only parser - return generation_prompt + (reasoning << p.content(p.rest())); - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; - - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - if (has_response_format) { - auto schema = inputs.json_schema; - builder.resolve_refs(schema); - } - parser.build_grammar(builder, data.grammar_lazy); - }); - - if (data.grammar_lazy) { - for (const auto & start : tool_call_starts) { - data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, start }); - } - } - } - - return data; -} - static common_chat_params common_chat_params_init_gpt_oss(const common_chat_template & tmpl, const autoparser::generation_params & inputs) { common_chat_params data; @@ -2134,16 +1957,23 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha adjusted_messages = deepseek_v4_sort_tool_results(inputs.messages); } + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs, adjusted_messages); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, adjusted_messages); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = ""; + data.thinking_end_tags = {""}; + data.preserved_tokens = { + "๏ฝœDSML๏ฝœ", + "", + "", + }; + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); auto has_response_format = !inputs.json_schema.is_null() && inputs.json_schema.is_object(); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - std::optional additional_context; - if (is_v4 && has_response_format) { - additional_context = json{ { "response_format", inputs.json_schema } }; - } - const std::string DSML = "๏ฝœDSML๏ฝœ"; const std::string THINK_START = ""; const std::string THINK_END = ""; @@ -2155,137 +1985,25 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha const std::string PARAM_START = "<" + DSML + "parameter"; const std::string PARAM_END = ""; const std::string GEN_PROMPT = "<๏ฝœAssistant๏ฝœ>"; - const std::string TC_SEPARATOR = "\n\n"; - - data.prompt = common_chat_template_direct_apply_impl( - tmpl, inputs, adjusted_messages, std::nullopt, additional_context); - data.generation_prompt = common_chat_template_generation_prompt_impl( - tmpl, inputs, adjusted_messages, std::nullopt, additional_context); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.thinking_start_tag = THINK_START; - data.thinking_end_tags = {THINK_END, FC_START}; - data.preserved_tokens = { - DSML, - THINK_START, - THINK_END, - }; if (inputs.has_continuation()) { const auto & msg = inputs.continue_msg; - if (is_v4 && msg.reasoning_content.empty()) { - data.generation_prompt = GEN_PROMPT + THINK_END; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += msg.render_content(); - } - } else { - data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += THINK_END + msg.render_content(); - } + data.generation_prompt = GEN_PROMPT + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + msg.render_content(); } data.prompt += data.generation_prompt; } - bool require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; - bool has_tool_calls = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { auto generation_prompt = p.literal(GEN_PROMPT); - auto end = p.end(); - - // build tool call section first since we might need it in reasoning - auto tool_choice = p.choice(); - if (has_tool_calls) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - const auto & props = params.contains("properties") ? params.at("properties") : json::object(); - - std::set required; - if (params.contains("required")) { - params.at("required").get_to(required); - } - - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - std::vector required_parsers; - std::vector optional_parsers; - for (const auto & [param_name, param_schema] : props.items()) { - bool is_required = required.find(param_name) != required.end(); - bool is_string = schema_info.resolves_to_string(param_schema); - - auto arg = p.tool_arg( - p.tool_arg_open(p.literal(PARAM_START + " name=\"") + p.tool_arg_name(p.literal(param_name)) + - p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + - (is_string ? - p.tool_arg_string_value(p.until(PARAM_END)) : - p.tool_arg_json_value(p.schema(p.json(), "tool-" + name + "-arg-" + param_name + "-schema", - param_schema, false))) + - p.tool_arg_close(p.literal(PARAM_END))); - - auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); - if (is_required) { - required_parsers.push_back(named_arg); - } else { - optional_parsers.push_back(named_arg); - } - } - - common_peg_parser args_seq = p.eps(); - for (size_t i = 0; i < required_parsers.size(); i++) { - if (i > 0) { - args_seq = args_seq + p.space(); - } - args_seq = args_seq + required_parsers[i]; - } - - if (!optional_parsers.empty()) { - common_peg_parser any_opt = p.choice(); - for (const auto & opt : optional_parsers) { - any_opt |= opt; - } - args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); - } - - common_peg_parser invoke_body = args_seq; - auto func_parser = p.tool(p.tool_open(p.literal(INVOKE_START + " name=\"") + - p.tool_name(p.literal(name)) + p.literal("\">\n")) + - invoke_body + p.space() + p.tool_close(p.literal(INVOKE_END))); - - tool_choice |= p.rule("tool-" + name, func_parser); - }); - } - - common_peg_parser tool_calls = p.eps(); - if (inputs.parallel_tool_calls) { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + - p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); - } else { - tool_calls = p.trigger_rule("tool-call", - p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); - } + auto end = p.end(); auto reasoning = p.eps(); - auto reasoning_with_tc = p.eps(); - auto obligatory_tool_calls = tool_calls; - bool allow_reasoning_with_tc = false; - - if (!require_tools) { - tool_calls = p.optional(tool_calls); - } - if (extract_reasoning && inputs.enable_thinking) { reasoning = p.optional(THINK_START + p.reasoning(p.until(THINK_END)) + THINK_END); - reasoning_with_tc = THINK_START + - p.reasoning(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START, THINK_END })) + - p.space() + obligatory_tool_calls; - allow_reasoning_with_tc = true; } else if (extract_reasoning) { // Thinking disabled but reasoning extraction requested: the generation prompt // contains an empty pair (V3.2) or a bare (V4) that @@ -2303,21 +2021,101 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha return generation_prompt + reasoning + response_format + end; } - if (!has_tool_calls) { + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { return generation_prompt + reasoning + p.content(p.rest()) + end; } - auto content_before_tools = p.negate(p.literal(THINK_START)) + - p.content(p.until_one_of({ TC_SEPARATOR + FC_START, FC_START })) + - p.space(); - return allow_reasoning_with_tc ? generation_prompt + (reasoning_with_tc | (reasoning + content_before_tools + tool_calls)) + end : - generation_prompt + reasoning + content_before_tools + tool_calls + end; + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + const auto & props = params.contains("properties") ? params.at("properties") : json::object(); + + std::set required; + if (params.contains("required")) { + params.at("required").get_to(required); + } + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + std::vector required_parsers; + std::vector optional_parsers; + for (const auto & [param_name, param_schema] : props.items()) { + bool is_required = required.find(param_name) != required.end(); + bool is_string = schema_info.resolves_to_string(param_schema); + + auto arg = p.tool_arg( + p.tool_arg_open( + p.literal(PARAM_START + " name=\"") + + p.tool_arg_name(p.literal(param_name)) + + p.literal("\" string=\"" + std::string(is_string ? "true" : "false") + "\">")) + + (is_string + ? p.tool_arg_string_value(p.until(PARAM_END)) + : p.tool_arg_json_value(p.schema(p.json(), + "tool-" + name + "-arg-" + param_name + "-schema", + param_schema, false))) + + p.tool_arg_close(p.literal(PARAM_END))); + + auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); + if (is_required) { + required_parsers.push_back(named_arg); + } else { + optional_parsers.push_back(named_arg); + } + } + + common_peg_parser args_seq = p.eps(); + for (size_t i = 0; i < required_parsers.size(); i++) { + if (i > 0) { + args_seq = args_seq + p.space(); + } + args_seq = args_seq + required_parsers[i]; + } + + if (!optional_parsers.empty()) { + common_peg_parser any_opt = p.choice(); + for (const auto & opt : optional_parsers) { + any_opt |= opt; + } + args_seq = args_seq + p.repeat(p.space() + any_opt, 0, -1); + } + + common_peg_parser invoke_body = args_seq; + auto func_parser = p.tool( + p.tool_open(p.literal(INVOKE_START + " name=\"") + + p.tool_name(p.literal(name)) + p.literal("\">\n")) + + invoke_body + p.space() + + p.tool_close(p.literal(INVOKE_END))); + + tool_choice |= p.rule("tool-" + name, func_parser); + }); + + auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + common_peg_parser tool_calls = p.eps(); + if (inputs.parallel_tool_calls) { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + + p.zero_or_more(p.space() + tool_choice) + p.space() + p.literal(FC_END)); + } else { + tool_calls = p.trigger_rule("tool-call", + p.literal(FC_START) + p.space() + tool_choice + p.space() + p.literal(FC_END)); + } + + if (!require_tools) { + tool_calls = p.optional(tool_calls); + } + + auto content_before_tools = p.content(p.until(FC_START)); + return generation_prompt + reasoning + content_before_tools + tool_calls + end; }); data.parser = parser.save(); if (include_grammar) { - data.grammar_lazy = has_tools && !require_tools; + data.grammar_lazy = !(has_response_format || (has_tools && inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED)); data.grammar = build_grammar([&](const common_grammar_builder & builder) { foreach_function(inputs.tools, [&](const json & tool) { const auto & function = tool.at("function"); @@ -3111,153 +2909,6 @@ static common_chat_params common_chat_params_init_minicpm5(const common_chat_tem return data; } -// An assistant turn is rendered as one or more messages, each -// "<|start|>assistant to=<|message|>{content}{END}" where END is -// <|eom|> (more messages follow) or <|eot|> (end of turn): -// - chain-of-thought: to=self, terminated by <|eom|> -// - final answer: to=user, terminated by <|eot|> -// The generation prompt is just "<|start|>assistant"; the model emits its own -// " to=...<|message|>". -static common_chat_params common_chat_params_init_muse_glimmer(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = "<|start|>assistant"; - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - - data.preserved_tokens = { - "<|start|>", "<|message|>", "<|eom|>", "<|eot|>", - // ATEM tool-call markup emitted on " to=" turns. - "", "", - "", "", - }; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, - { COMMON_CHAT_ROLE_USER, "<|start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, - { COMMON_CHAT_ROLE_TOOL, "<|start|>tool" }, - }; - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|start|>assistant to=self<|message|>" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "<|eom|><|start|>assistant to=user<|message|>" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - // Constrained grammar whenever tools are offered. - auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto start = p.rule("start", p.literal("<|start|>assistant")); - - if (!extract_reasoning && !include_grammar) { - return start + p.content(p.rest()); - } - - if (extract_reasoning) { - p.rule("analysis", p.literal(" to=self<|message|>") + p.reasoning(p.until("<|eom|>")) + p.literal("<|eom|>")); - } else { - p.rule("analysis", p.literal(" to=self<|message|>") + p.content(p.until("<|eom|>")) + p.literal("<|eom|>")); - } - auto analysis = p.ref("analysis"); - - auto recipient = p.optional(p.literal(" to=user")); - auto final_msg = p.rule("final", recipient + p.literal("<|message|>") + - p.content(p.until_one_of({ "<|eot|>", "<|eom|>" }))); - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - auto string_value = p.ac( - p.tool_arg_string_value(p.until("")) + p.tool_arg_close(p.literal("")), - ""); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false)) - + p.tool_arg_close(p.literal("")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("")) + - value_parser); - - arg_choice |= arg_rule; - } - args = p.zero_or_more(arg_choice + p.space()); - } - - auto tool_parser = p.tool( - p.tool_open(p.literal(" to=") + p.until("<|message|>") + - p.literal("<|message|>") + p.space() + - p.literal("") + p.space()) - << p.tool_args(args) - << p.tool_close(p.literal("") + p.space() + p.literal(""))); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto tool_calls = inputs.parallel_tool_calls - ? p.trigger_rule("tool-call", tool_choice + p.zero_or_more(p.literal("<|eom|>") + start + tool_choice)) - : p.trigger_rule("tool-call", tool_choice); - - - if (inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED) { - return p.zero_or_more(start + analysis) + start + tool_calls; - } - auto trailing_calls = p.optional(p.literal("<|eom|>") + start + tool_calls); - return p.zero_or_more(start + analysis) + start + (tool_calls | (final_msg + trailing_calls)); - } - - return p.zero_or_more(start + analysis) + start + final_msg; - }); - - data.parser = parser.save(); - - if (include_grammar) { - data.grammar_lazy = inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_REQUIRED; - data.grammar = build_grammar([&](const common_grammar_builder & builder) { - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - auto schema = function.contains("parameters") ? function.at("parameters") : json::object(); - builder.resolve_refs(schema); - }); - parser.build_grammar(builder, data.grammar_lazy); - }); - data.grammar_triggers = { - { COMMON_GRAMMAR_TRIGGER_TYPE_PATTERN, - "<\\|start\\|>assistant( to=(?!self<\\|message\\|>)(?!user<\\|message\\|>)[^<]*?<\\|message\\|>)" }, - }; - } - - return data; -} - static json common_chat_extra_context() { json ctx = json::object(); std::chrono::system_clock::time_point now = std::chrono::system_clock::now(); @@ -3286,12 +2937,6 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_gpt_oss(tmpl, params); } - // Muse Glimmer format using " to=" recipients and <|eom|>/<|eot|> message terminators. - if (src.find("") != std::string::npos && src.find("<|eom|>") != std::string::npos) { - LOG_DBG("Using specialized template: Muse Glimmer\n"); - return common_chat_params_init_muse_glimmer(tmpl, params); - } - // Functionary v3.2 - uses recipient-based format with >>>recipient\n{content} // Detection: template has ">>>all" for content and ">>>" prefix for tool calls if (src.find(">>>all") != std::string::npos && src.find(">>>${recipient}") != std::string::npos) { @@ -3375,14 +3020,6 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_minicpm5(tmpl, params); } - // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash - if (src.find("") != std::string::npos && - src.find("pw_dir)) { - throw std::runtime_error("Failed to find $HOME directory"); - } - - config_directory = std::string(pw->pw_dir) + std::string("/.config/"); -#else - throw std::runtime_error("Failed to find $HOME directory"); -#endif - } -#elif defined(_WIN32) - config_directory = common_get_env("APPDATA"); - if (config_directory.empty()) { - throw std::runtime_error("Failed to find %APPDATA% directory"); - } -#elif defined(__EMSCRIPTEN__) - // caller decides what to do when there is no config directory - throw std::runtime_error("not implemented on this platform"); -#else -# error Unknown architecture -#endif - config_directory = ensure_trailing_slash(config_directory); - config_directory += "llama.cpp"; - return ensure_trailing_slash(config_directory); -} - std::string fs_get_cache_file(const std::string & filename) { GGML_ASSERT(filename.find(DIRECTORY_SEPARATOR) == std::string::npos); std::string cache_directory = fs_get_cache_directory(); @@ -1281,8 +1211,6 @@ struct common_init_result::impl { // note: the order in which model, context, etc. are declared matters because their destructors will be called bottom-to-top - common_threadpools threadpools; - llama_model_ptr model; llama_context_ptr context; @@ -1363,6 +1291,16 @@ common_init_result::common_init_result(common_params & params, bool model_only) params.sampling.logit_bias_eog.begin(), params.sampling.logit_bias_eog.end()); } + //if (params.sampling.penalty_last_n == -1) { + // LOG_TRC("%s: setting penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx)); + // params.sampling.penalty_last_n = llama_n_ctx(lctx); + //} + + //if (params.sampling.dry_penalty_last_n == -1) { + // LOG_TRC("%s: setting dry_penalty_last_n to ctx_size = %d\n", __func__, llama_n_ctx(lctx)); + // params.sampling.dry_penalty_last_n = llama_n_ctx(lctx); + //} + // init the backend samplers as part of the context creation pimpl->samplers.resize(cparams.n_seq_max); pimpl->samplers_seq_config.resize(cparams.n_seq_max); @@ -1384,10 +1322,6 @@ common_init_result::common_init_result(common_params & params, bool model_only) } pimpl->context.reset(lctx); - - set_process_priority(params.cpuparams.priority); - - pimpl->threadpools.init(lctx, params); } llama_model * common_init_result::model() { @@ -1534,18 +1468,18 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode common_init_result::~common_init_result() = default; std::string common_get_model_endpoint() { - std::string endpoint = common_get_env("MODEL_ENDPOINT"); - if (endpoint.empty()) { - // the HF_ENDPOINT variable is respected for backward compatibility - endpoint = common_get_env("HF_ENDPOINT"); + const char * model_endpoint_env = getenv("MODEL_ENDPOINT"); + // We still respect the use of environment-variable "HF_ENDPOINT" for backward-compatibility. + const char * hf_endpoint_env = getenv("HF_ENDPOINT"); + const char * endpoint_env = model_endpoint_env ? model_endpoint_env : hf_endpoint_env; + std::string model_endpoint = "https://huggingface.co/"; + if (endpoint_env) { + model_endpoint = endpoint_env; + if (model_endpoint.back() != '/') { + model_endpoint += '/'; + } } - if (endpoint.empty()) { - return "https://huggingface.co/"; - } - if (endpoint.back() != '/') { - endpoint += '/'; - } - return endpoint; + return model_endpoint; } char * common_get_model_or_exit(int argc, char * argv[]) { @@ -1692,7 +1626,6 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { mparams.progress_callback = params.load_progress_callback; mparams.progress_callback_user_data = params.load_progress_callback_user_data; mparams.no_alloc = params.no_alloc; - mparams.load_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); return mparams; } @@ -1704,7 +1637,6 @@ struct llama_context_params common_context_params_to_llama(const common_params & cparams.n_seq_max = params.n_parallel; cparams.n_rs_seq = params.speculative.need_n_rs_seq(); cparams.n_outputs_max = std::max(params.n_outputs_max, 0); - cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0); cparams.n_batch = params.n_batch; cparams.n_ubatch = params.n_ubatch; cparams.n_threads = params.cpuparams.n_threads; @@ -1736,10 +1668,6 @@ struct llama_context_params common_context_params_to_llama(const common_params & return cparams; } -// -// Threadpool utils -// - struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params) { struct ggml_threadpool_params tpp; @@ -1756,56 +1684,6 @@ struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const commo return tpp; } -common_threadpools::~common_threadpools() { - if (!free_fn) { - return; - } - free_fn(threadpool); - free_fn(threadpool_batch); -} - -void common_threadpools::init(llama_context * ctx, const common_params & params) { - GGML_ASSERT(!threadpool); - GGML_ASSERT(!threadpool_batch); - - COM_INF("llama threadpool init, n_threads = %d\n", (int) params.cpuparams.n_threads); - - auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); - if (!cpu_dev) { - COM_WRN("%s", "no CPU backend found\n"); - return; - } - auto * reg = ggml_backend_dev_backend_reg(cpu_dev); - auto * ggml_threadpool_new_fn = (decltype(ggml_threadpool_new) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_new"); - free_fn = (decltype(ggml_threadpool_free) *) ggml_backend_reg_get_proc_address(reg, "ggml_threadpool_free"); - - struct ggml_threadpool_params tpp_batch = - ggml_threadpool_params_from_cpu_params(params.cpuparams_batch); - struct ggml_threadpool_params tpp = - ggml_threadpool_params_from_cpu_params(params.cpuparams); - - if (!ggml_threadpool_params_match(&tpp, &tpp_batch)) { - threadpool_batch = ggml_threadpool_new_fn(&tpp_batch); - if (!threadpool_batch) { - COM_WRN("batch threadpool create failed : n_threads %d\n", tpp_batch.n_threads); - return; - } - - // start the non-batch threadpool in the paused state - tpp.paused = true; - } - - threadpool = ggml_threadpool_new_fn(&tpp); - if (!threadpool) { - COM_WRN("threadpool create failed : n_threads %d\n", tpp.n_threads); - free_fn(threadpool_batch); - threadpool_batch = nullptr; - return; - } - - llama_attach_threadpool(ctx, threadpool, threadpool_batch); -} - // // Batch utils // diff --git a/common/common.h b/common/common.h index ebdc23d18..e6d5d892e 100644 --- a/common/common.h +++ b/common/common.h @@ -236,14 +236,14 @@ struct common_params_sampling { float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities float dynatemp_range = 0.00f; // 0.0 = disabled float dynatemp_exponent = 1.00f; // controls how entropy maps to temperature in dynamic temperature sampler - int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty) + int32_t penalty_last_n = 64; // last n tokens to penalize (0 = disable penalty, -1 = context size) float penalty_repeat = 1.00f; // 1.0 = disabled float penalty_freq = 0.00f; // 0.0 = disabled float penalty_present = 0.00f; // 0.0 = disabled float dry_multiplier = 0.0f; // 0.0 = disabled; DRY repetition penalty for tokens extending repetition: float dry_base = 1.75f; // 0.0 = disabled; multiplier * base ^ (length of sequence before token - allowed length) int32_t dry_allowed_length = 2; // tokens extending repetitions beyond this receive penalty - int32_t dry_penalty_last_n = 64; // how many tokens to scan for repetitions (0 = disable penalty) + int32_t dry_penalty_last_n = -1; // how many tokens to scan for repetitions (0 = disable penalty, -1 = context size) float adaptive_target = -1.0f; // select tokens near this probability (valid range 0.0 to 1.0; negative = disabled) float adaptive_decay = 0.90f; // EMA decay for adaptation; history โ‰ˆ 1/(1-decay) tokens (0.0 - 0.99) int32_t mirostat = 0; // 0 = disabled, 1 = mirostat, 2 = mirostat 2.0 @@ -393,6 +393,14 @@ struct common_params_speculative { } }; +struct common_params_vocoder { + struct common_params_model model; + + std::string speaker_file; // speaker file path + + bool use_guide_tokens = false; // enable guide tokens to improve TTS accuracy +}; + struct common_params_diffusion { int32_t steps = 128; bool visual_mode = false; @@ -448,7 +456,6 @@ struct common_params { int32_t n_parallel = 1; // number of parallel sequences to decode int32_t n_sequences = 1; // number of sequences to decode int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) - int32_t n_outputs_max_per_seq = 1; // max outputs per sequence int32_t grp_attn_n = 1; // group-attention factor int32_t grp_attn_w = 512; // group-attention width int32_t n_print = -1; // print token count every n tokens (-1 = disabled) @@ -474,7 +481,7 @@ struct common_params { std::vector fit_params_target = std::vector(llama_max_devices(), 1024 * 1024*1024); enum llama_split_mode split_mode = LLAMA_SPLIT_MODE_LAYER; // how to split the model across GPUs - enum llama_load_mode load_mode = LLAMA_LOAD_MODE_AUTO; // how to load the model + enum llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP; // how to load the model common_cpu_params cpuparams; common_cpu_params cpuparams_batch; @@ -491,6 +498,7 @@ struct common_params { struct common_params_sampling sampling; struct common_params_speculative speculative; + struct common_params_vocoder vocoder; struct common_params_diffusion diffusion; struct common_params_model model; @@ -657,7 +665,6 @@ struct common_params { // enable built-in tools std::vector server_tools; - std::string server_tools_runtime; // MCP server configs (Cursor-compatible JSON) std::string mcp_servers_config; // path to JSON file with MCP server definitions @@ -733,12 +740,6 @@ struct common_params { llama_progress_callback load_progress_callback = NULL; void * load_progress_callback_user_data = NULL; bool no_alloc = false; // Don't allocate model buffers - - // TTS params - std::string tts_lang = ""; - std::string tts_speaker_file = ""; - - bool is_gen_docs = false; // whether we are running inside llama-gen-docs }; // call once at the start of a program if it uses libcommon @@ -863,15 +864,6 @@ std::string string_from(const struct llama_context * ctx, const struct llama_bat bool glob_match(const std::string & pattern, const std::string & str); -// -// Environment utils -// - -// portable environment access, an unset variable reads as an empty string -// and setting an empty value unsets the variable -std::string common_get_env(const std::string & name); -void common_set_env(const std::string & name, const std::string & value); - // // Filesystem utils // @@ -882,7 +874,6 @@ bool fs_is_directory(const std::string & path); std::string fs_get_cache_directory(); std::string fs_get_cache_file(const std::string & filename); -std::string fs_get_config_directory(); struct common_file_info { std::string path; @@ -930,8 +921,9 @@ using common_init_result_ptr = std::unique_ptr; common_init_result_ptr common_init_from_params(common_params & params, bool model_only = false); -struct llama_model_params common_model_params_to_llama ( common_params & params); -struct llama_context_params common_context_params_to_llama(const common_params & params); +struct llama_model_params common_model_params_to_llama ( common_params & params); +struct llama_context_params common_context_params_to_llama(const common_params & params); +struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params); // clear LoRA adapters from context, then apply new list of adapters void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora); @@ -942,28 +934,6 @@ std::string common_get_model_endpoint(); // for testing purposes char * common_get_model_or_exit(int, char*[]); -// -// Threadpool utils -// - -struct ggml_threadpool_params ggml_threadpool_params_from_cpu_params(const common_cpu_params & params); - -struct common_threadpools { - common_threadpools() = default; - ~common_threadpools(); - - common_threadpools(const common_threadpools &) = delete; - common_threadpools & operator=(const common_threadpools &) = delete; - - void init(llama_context * ctx, const common_params & params); - -private: - ggml_threadpool * threadpool = nullptr; - ggml_threadpool * threadpool_batch = nullptr; - - decltype(ggml_threadpool_free) * free_fn = nullptr; -}; - // // Context utils // diff --git a/common/download.cpp b/common/download.cpp index 44c6cea42..3776c6c7e 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -656,12 +656,6 @@ static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, return find_best_sibling(files, model, "dflash-", tag); } -static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "dspark-", tag); -} - static bool gguf_filename_is_model(const std::string & filepath) { if (!string_ends_with(filepath, ".gguf")) { return false; @@ -676,8 +670,7 @@ static bool gguf_filename_is_model(const std::string & filepath) { filename.find("imatrix") == std::string::npos && filename.find("mtp-") == std::string::npos && filename.find("eagle3-") == std::string::npos && - filename.find("dflash-") == std::string::npos && - filename.find("dspark-") == std::string::npos; + filename.find("dflash-") == std::string::npos; } static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, @@ -770,7 +763,7 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & } else { primary = find_best_model(all, tag); // a requested sidecar can resolve on its own, without a full model of the same tag - if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3 && !opts.download_dspark) { + if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); return plan; @@ -794,12 +787,9 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & if (opts.download_eagle3) { plan.eagle3 = find_best_eagle3(all, primary.path, tag); } - if (opts.download_dspark) { - plan.dspark = find_best_dspark(all, primary.path, tag); - } if (primary.path.empty() && - plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty() && plan.dspark.local_path.empty()) { + plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) { LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); list_available_gguf_files(all); } @@ -977,8 +967,7 @@ std::vector common_list_cached_models() { split.prefix.find("mmproj") != std::string::npos || split.prefix.find("mtp-") != std::string::npos || split.prefix.find("eagle3-") != std::string::npos || - split.prefix.find("dflash-") != std::string::npos || - split.prefix.find("dspark-") != std::string::npos) { + split.prefix.find("dflash-") != std::string::npos) { continue; } if (seen.insert(f.repo_id + ":" + split.tag).second) { diff --git a/common/download.h b/common/download.h index 9da595d1f..6007c37fc 100644 --- a/common/download.h +++ b/common/download.h @@ -60,7 +60,6 @@ struct common_download_opts { bool download_mtp = false; bool download_eagle3 = false; bool download_dflash = false; - bool download_dspark = false; common_download_callback * callback = nullptr; }; @@ -112,7 +111,6 @@ struct common_download_hf_plan { hf_cache::hf_file mtp; hf_cache::hf_file eagle3; hf_cache::hf_file dflash; - hf_cache::hf_file dspark; hf_cache::hf_file preset; // if set, only this file is downloaded }; common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); diff --git a/common/fit.cpp b/common/fit.cpp index dd1f3ef76..c82d066ad 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -136,10 +136,7 @@ static std::vector common_get_device_memory_data_impl( devs.push_back(llama_model_get_device(model, i)); } - hp_ngl = llama_model_n_layer(model); - if (mparams->load_mtp) { - hp_ngl += llama_model_n_layer_nextn(model); - } + hp_ngl = llama_model_n_layer(model) + llama_model_n_layer_nextn(model); hp_n_ctx_train = llama_model_n_ctx_train(model); hp_n_expert = llama_model_n_expert(model); diff --git a/common/imatrix-loader.cpp b/common/imatrix-loader.cpp index 71d3b500f..efe9aecee 100644 --- a/common/imatrix-loader.cpp +++ b/common/imatrix-loader.cpp @@ -102,8 +102,7 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) { const int64_t chunk_count_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT); const int64_t chunk_size_key = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE); - if (datasets_key != -1 && gguf_get_kv_type(ctx_gguf, datasets_key) == GGUF_TYPE_ARRAY && - gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) { + if (datasets_key != -1 && gguf_get_arr_type(ctx_gguf, datasets_key) == GGUF_TYPE_STRING) { const int64_t n = gguf_get_arr_n(ctx_gguf, datasets_key); imatrix.datasets.reserve(imatrix.datasets.size() + n); for (int64_t i = 0; i < n; ++i) { @@ -144,13 +143,6 @@ bool common_imatrix_load(const std::string & fname, common_imatrix & imatrix) { return false; } - if (in_sum2->type != GGML_TYPE_F32 || counts->type != GGML_TYPE_F32) { - LOG_ERR("%s: sums and counts for %s must be F32\n", __func__, name.c_str()); - gguf_free(ctx_gguf); - ggml_free(ctx); - return false; - } - auto & e = imatrix.entries[name]; const int64_t nval = ggml_nelements(in_sum2); diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index 00b02119a..26306bd91 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -20,7 +20,7 @@ namespace jinja { using caps_json_fn = std::function; using caps_ctx_fn = std::function; -using caps_analyze_fn = std::function; +using caps_analyze_fn = std::function; void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { ctx.set_val("preserve_thinking", mk_val(enabled)); @@ -29,12 +29,6 @@ void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { ctx.set_val("drop_thinking", mk_val(!enabled)); } -void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort) { - value var = mk_val(effort); // bind to the same value for stats - ctx.set_val("reasoning_effort", var); - ctx.set_val("reasoning_strength", var); -} - static void caps_try_execute(jinja::program & prog, const caps_json_fn & messages_fn, const caps_ctx_fn & ctx_fn, @@ -71,7 +65,7 @@ static void caps_try_execute(jinja::program & prog, // ignore exceptions during capability analysis } - analyze_fn(ctx, success, messages, tools, result); + analyze_fn(success, messages, tools, result); } // for debugging only @@ -96,7 +90,6 @@ std::map caps::to_map() const { {"supports_parallel_tool_calls", supports_parallel_tool_calls}, {"supports_system_role", supports_system_role}, {"supports_preserve_reasoning", supports_preserve_reasoning}, - {"supports_reasoning_effort", supports_reasoning_effort}, {"supports_object_arguments", supports_object_arguments}, }; } @@ -134,7 +127,7 @@ caps caps_get(jinja::program & prog) { }, nullptr, // ctx_fn nullptr, // tools_fn - [&](context &, bool success, value & messages, value &, const std::string &) { + [&](bool success, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (has_op(content, "selectattr") || has_op(content, "array_access")) { @@ -168,7 +161,7 @@ caps caps_get(jinja::program & prog) { }, nullptr, // ctx_fn nullptr, // tools_fn - [&](context &, bool, value & messages, value &, const std::string &) { + [&](bool, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (!content->stats.used) { @@ -244,7 +237,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](context &, bool success, value & messages, value & tools, const std::string &) { + [&](bool success, value & messages, value & tools, const std::string &) { if (!success) { return; // Nothing can be inferred } @@ -337,7 +330,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](context &, bool success, value & messages, value & tools, const std::string &) { + [&](bool success, value & messages, value & tools, const std::string &) { if (!success) { result.supports_tool_calls = false; result.supports_tools = false; @@ -439,7 +432,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](context &, bool success, value & messages, value &, const std::string &) { + [&](bool success, value & messages, value &, const std::string &) { if (!success) { result.supports_parallel_tool_calls = false; return; @@ -492,11 +485,10 @@ caps caps_get(jinja::program & prog) { }); }, [&](context & ctx) { - ctx.set_val("enable_thinking", mk_val(true)); caps_apply_preserve_reasoning(ctx, true); }, nullptr, // tools_fn - [&](context &, bool, value &, value &, const std::string & output) { + [&](bool, value &, value &, const std::string & output) { // note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result if (output.find(reasoning_placeholder) != std::string::npos) { result.supports_preserve_reasoning = true; @@ -504,32 +496,6 @@ caps caps_get(jinja::program & prog) { } ); - JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort"); - - // case: reasoning effort level - caps_try_execute( - prog, - [&]() { - // messages - return json::array({ - { - {"role", "user"}, - {"content", "User message"} - }, - }); - }, - [&](context & ctx) { - ctx.set_val("enable_thinking", mk_val(true)); - caps_apply_reasoning_effort(ctx, "low"); - }, - nullptr, // tools_fn - [&](context & ctx, bool, value &, value &, const std::string &) { - value effort = ctx.get_val("reasoning_effort"); - caps_print_stats(effort, "reasoning_effort"); - result.supports_reasoning_effort = effort->stats.used; - } - ); - JJ_DEBUG("%s\n", result.to_string().c_str()); return result; diff --git a/common/jinja/caps.h b/common/jinja/caps.h index b81dd95f2..a290cd7da 100644 --- a/common/jinja/caps.h +++ b/common/jinja/caps.h @@ -16,9 +16,6 @@ struct caps { // supports preserve reasoning trace in the full history, not just the last assistant message bool supports_preserve_reasoning = false; - // supports reasoning effort levels - bool supports_reasoning_effort = false; - // one of the 2 content capabilities must be true bool supports_string_content = true; bool supports_typed_content = false; @@ -35,6 +32,5 @@ struct caps { caps caps_get(jinja::program & prog); void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled); -void caps_apply_reasoning_effort(jinja::context & ctx, const std::string & effort); } // namespace jinja diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index 9cc5cfa46..afba0025f 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -266,7 +266,7 @@ value binary_expression::execute_impl(context & ctx) { return res; } for (int64_t i = 0; i < repeat; ++i) { - res->val_str.append(str); + res->val_str = res->val_str.append(str); } return res; } diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 69bd683c6..0884a1592 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -763,22 +763,14 @@ struct runtime { gather_string_parts_recursive(val, parts); // join consecutive parts with the same type auto & p = parts->val_str.parts; - if (p.empty()) { - return parts; - } - size_t w = 0; - for (size_t r = 1; r < p.size(); r++) { - if (p[w].is_input == p[r].is_input) { - p[w].val += p[r].val; + for (size_t i = 1; i < p.size(); ) { + if (p[i].is_input == p[i - 1].is_input) { + p[i - 1].val += p[i].val; + p.erase(p.begin() + i); } else { - w++; - if (w != r) { - // the guard is needed, self-move leaves the string in an unspecified state - p[w] = std::move(p[r]); - } + i++; } } - p.resize(w + 1); return parts; } diff --git a/common/jinja/string.cpp b/common/jinja/string.cpp index bde679e4e..8087e15b3 100644 --- a/common/jinja/string.cpp +++ b/common/jinja/string.cpp @@ -103,7 +103,7 @@ void string::mark_input_based_on(const string & other) { } } -string & string::append(const string & other) { +string string::append(const string & other) { for (const auto & part : other.parts) { parts.push_back(part); } diff --git a/common/jinja/string.h b/common/jinja/string.h index 669afb8f1..c4963000a 100644 --- a/common/jinja/string.h +++ b/common/jinja/string.h @@ -47,7 +47,7 @@ struct string { // mark this string as input if other has ALL parts as input void mark_input_based_on(const string & other); - string & append(const string & other); + string append(const string & other); // in-place transformations diff --git a/common/llguidance.cpp b/common/llguidance.cpp index 500bb0914..d58f147a7 100644 --- a/common/llguidance.cpp +++ b/common/llguidance.cpp @@ -116,8 +116,6 @@ static llama_sampler_i llama_sampler_llg_i = { /* .backend_accept = */ NULL, /* .backend_apply = */ NULL, /* .backend_set_input = */ NULL, - /* .backend_reset = */ NULL, - /* .copy_state = */ NULL, }; static size_t llama_sampler_llg_tokenize_fn(const void * user_data, const uint8_t * bytes, size_t bytes_len, diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 4a4be7cf7..ef290ed7c 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -570,34 +570,23 @@ struct parser_executor { } static common_peg_parse_result handle_escape_sequence(common_peg_parse_context & ctx, size_t start, size_t & pos, const char delimiter) { - auto save = pos; - ++pos; // consume '\' if (pos >= ctx.input.size()) { if (!ctx.is_lenient()) { return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } - pos = save; // suppress unmatched '\' return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); } char c = ctx.input[pos]; - if (c == delimiter || c == '\\' || c == '/' || c == 'b' || c == 'f' || c == 'n' || c == 'r' || c == 't') { ++pos; return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_SUCCESS, start, pos); + } else if (c == 'u') { + return handle_unicode_escape(ctx, start, pos); + } else { + return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } - - if (c == 'u') { - auto result = handle_unicode_escape(ctx, start, pos); - if (result.need_more_input()) { - pos = save; // suppress incomplete sequence - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_NEED_MORE_INPUT, start, pos); - } - return result; - } - - return common_peg_parse_result(COMMON_PEG_PARSE_RESULT_FAIL, start); } static common_peg_parse_result handle_unicode_escape(common_peg_parse_context & ctx, size_t start, size_t & pos) { diff --git a/common/preset.cpp b/common/preset.cpp index 4c61e93ee..eb0c60b09 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -322,8 +322,6 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co preset.options[opt] = value; } LOG_DBG("accepted option: %s = %s\n", key.c_str(), preset.options[opt].c_str()); - } else if (ignore_unknown_keys) { - LOG_WRN("ignoring option '%s' from %s: not supported by this program\n", key.c_str(), path.c_str()); } else { throw std::runtime_error(string_format( "option '%s' not recognized in preset '%s'", @@ -365,25 +363,8 @@ struct local_model { std::string name; std::string path; std::string path_mmproj; - std::string path_draft; }; -// TODO @ngxson: handle "eagle3-" when it's supported by common_speculative_types_from_gguf() -static const char * draft_prefixes[] = { "mtp-", "dspark-", "dflash-" }; - -static bool is_mmproj_file(const std::string & fname) { - return fname.find("mmproj") != std::string::npos; -} - -static bool is_draft_file(const std::string & fname) { - for (const auto & prefix : draft_prefixes) { - if (fname.rfind(prefix, 0) == 0) { - return true; - } - } - return false; -} - common_presets common_preset_context::load_from_models_dir(const std::string & models_dir) const { if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { throw std::runtime_error(string_format("error: '%s' does not exist or is not a directory\n", models_dir.c_str())); @@ -395,15 +376,10 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m common_file_info model_file; common_file_info first_shard_file; common_file_info mmproj_file; - common_file_info draft_file; for (const auto & file : files) { if (string_ends_with(file.name, ".gguf")) { - if (is_mmproj_file(file.name)) { + if (file.name.find("mmproj") != std::string::npos) { mmproj_file = file; - } else if (is_draft_file(file.name)) { - if (draft_file.path.empty()) { - draft_file = file; // first sidecar found wins - } } else if (file.name.find("-00001-of-") != std::string::npos) { first_shard_file = file; } else { @@ -415,8 +391,7 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m local_model model{ /* name */ name, /* path */ first_shard_file.path.empty() ? model_file.path : first_shard_file.path, - /* path_mmproj */ mmproj_file.path, // can be empty - /* path_draft */ draft_file.path // can be empty + /* path_mmproj */ mmproj_file.path // can be empty }; if (!model.path.empty()) { models.push_back(model); @@ -428,17 +403,13 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m if (file.is_dir) { scan_subdir(file.path, file.name); } else if (string_ends_with(file.name, ".gguf")) { - if (is_mmproj_file(file.name) || is_draft_file(file.name)) { - continue; // companion file, cannot be loaded as a model on its own - } // single file model std::string name = file.name; string_replace_all(name, ".gguf", ""); local_model model{ /* name */ name, /* path */ file.path, - /* path_mmproj */ "", - /* path_draft */ "" + /* path_mmproj */ "" }; models.push_back(model); } @@ -453,9 +424,6 @@ common_presets common_preset_context::load_from_models_dir(const std::string & m if (!model.path_mmproj.empty()) { preset.set_option(*this, "LLAMA_ARG_MMPROJ", model.path_mmproj); } - if (!model.path_draft.empty()) { - preset.set_option(*this, "LLAMA_ARG_SPEC_DRAFT_MODEL", model.path_draft); - } out[preset.name] = preset; } diff --git a/common/preset.h b/common/preset.h index d8fc3915b..52935ebde 100644 --- a/common/preset.h +++ b/common/preset.h @@ -59,10 +59,6 @@ struct common_preset_context { bool filter_allowed_keys = false; std::set allowed_keys; - // if true, options unknown to the current example are skipped instead of being an error - // used for config files shared by all binaries, where each binary only knows a subset of options - bool ignore_unknown_keys = false; - // if only_remote_allowed is true, only accept whitelisted keys common_preset_context(llama_example ex); diff --git a/common/reasoning-budget.cpp b/common/reasoning-budget.cpp index 4884299f3..1fe242d06 100644 --- a/common/reasoning-budget.cpp +++ b/common/reasoning-budget.cpp @@ -217,8 +217,6 @@ static struct llama_sampler_i common_reasoning_budget_i = { /* .backend_accept = */ nullptr, /* .backend_apply = */ nullptr, /* .backend_set_input = */ nullptr, - /* .backend_reset = */ nullptr, - /* .copy_state = */ nullptr, }; static struct llama_sampler * common_reasoning_budget_clone(const struct llama_sampler * smpl) { diff --git a/common/sampling.cpp b/common/sampling.cpp index 06dea1e1c..256ac161e 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -184,21 +184,9 @@ std::string common_params_sampling::print() const { return std::string(result); } -struct common_sampler * common_sampler_init( - const struct llama_model * model, - struct common_params_sampling & params) { - if (!std::isfinite(params.penalty_repeat) || - params.penalty_repeat <= 0.0f || - !std::isfinite(1.0f/params.penalty_repeat)) { - throw std::invalid_argument("penalty_repeat must be finite and greater than 0"); - } - if (!std::isfinite(params.penalty_freq)) { - throw std::invalid_argument("penalty_freq must be finite"); - } - if (!std::isfinite(params.penalty_present)) { - throw std::invalid_argument("penalty_present must be finite"); - } +struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params) { const llama_vocab * vocab = llama_model_get_vocab(model); + llama_sampler_chain_params lparams = llama_sampler_chain_default_params(); lparams.no_perf = params.no_perf; @@ -350,7 +338,7 @@ struct common_sampler * common_sampler_init( for (const auto & str : params.dry_sequence_breakers) { c_breakers.push_back(str.c_str()); } - samplers.push_back(llama_sampler_init_dry(vocab, params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size())); + samplers.push_back(llama_sampler_init_dry(vocab, llama_model_n_ctx_train(model), params.dry_multiplier, params.dry_base, params.dry_allowed_length, params.dry_penalty_last_n, c_breakers.data(), c_breakers.size())); } break; case COMMON_SAMPLER_TYPE_TOP_K: @@ -378,7 +366,7 @@ struct common_sampler * common_sampler_init( samplers.push_back(llama_sampler_init_infill(vocab)); break; case COMMON_SAMPLER_TYPE_PENALTIES: - samplers.push_back(llama_sampler_init_penalties(llama_vocab_n_tokens(vocab), params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present)); + samplers.push_back(llama_sampler_init_penalties(params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present)); break; case COMMON_SAMPLER_TYPE_ADAPTIVE_P: // the `adaptive-p` sampler is like `dist` and `mirostat` in that it selects @@ -518,26 +506,6 @@ struct common_sampler * common_sampler_clone(common_sampler * gsmpl) { }; } -void common_sampler_copy(const common_sampler * src, common_sampler * dst) { - if (!src || !dst || src == dst) { - return; - } - - GGML_ASSERT((src->grmr == nullptr) == (dst->grmr == nullptr)); - GGML_ASSERT((src->rbudget == nullptr) == (dst->rbudget == nullptr)); - - llama_sampler_copy(src->grmr, dst->grmr); - llama_sampler_copy(src->rbudget, dst->rbudget); - llama_sampler_copy(src->chain, dst->chain); - - dst->params = src->params; - dst->prev = src->prev; - dst->cur = src->cur; - dst->cur_p = src->cur_p; - dst->cur_p.data = src->cur_p.data ? dst->cur.data() : nullptr; // re-point to dst's buffer - dst->t_total_us = src->t_total_us; -} - void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl) { // TODO: measure grammar performance diff --git a/common/sampling.h b/common/sampling.h index ced3c8364..4191988bb 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -37,9 +37,7 @@ struct common_sampler; // llama_sampler API overloads // note: can mutate params in some cases -struct common_sampler * common_sampler_init( - const struct llama_model * model, - struct common_params_sampling & params); +struct common_sampler * common_sampler_init(const struct llama_model * model, struct common_params_sampling & params); void common_sampler_free(struct common_sampler * gsmpl); @@ -47,7 +45,6 @@ void common_sampler_free(struct common_sampler * gsmpl); void common_sampler_accept(struct common_sampler * gsmpl, llama_token token, bool is_generated); void common_sampler_reset (struct common_sampler * gsmpl); struct common_sampler * common_sampler_clone (struct common_sampler * gsmpl); -void common_sampler_copy (const struct common_sampler * src, struct common_sampler * dst); // arguments can be nullptr to skip printing void common_perf_print(const struct llama_context * ctx, const struct common_sampler * gsmpl); diff --git a/common/speculative.cpp b/common/speculative.cpp index aec94199b..b91974c11 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2,7 +2,6 @@ #include "common.h" #include "ggml.h" -#include "ggml-cpp.h" #include "llama.h" #include "log.h" #include "ngram-cache.cpp" @@ -172,6 +171,12 @@ struct common_speculative_impl { // (optional) serialize/restore per-seq internal state (e.g. eagle3's deferred boundary). virtual bool get_state(llama_seq_id /*seq_id*/, std::vector & /*data*/) const { return false; } virtual void set_state(llama_seq_id /*seq_id*/, const std::vector & /*data*/) {} + + // true if this implementation requires the target context to extract post-norm embeddings + virtual bool need_embd() const = 0; + + // true if this implementation requires the target context to extract pre-norm embeddings + virtual bool need_embd_nextn() const { return false; } }; struct common_speculative_impl_draft_simple : public common_speculative_impl { @@ -188,10 +193,6 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; auto * ctx_tgt = this->params.ctx_tgt; - if (!ctx_dft) { - throw std::runtime_error("draft-simple requires a draft context"); - } - SPC_TRC("%s", "adding speculative implementation 'draft-simple'\n"); SPC_TRC("- n_max=%d, n_min=%d, p_min=%f\n", this->params.n_max, this->params.n_min, this->params.p_min); SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", @@ -384,6 +385,10 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } + + bool need_embd() const override { + return false; + } }; @@ -902,6 +907,10 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { pending_g_last[seq_id].resize(n_embd_dec); std::memcpy(pending_g_last[seq_id].data(), data.data() + sizeof(llama_pos), (size_t) n_embd_dec * sizeof(float)); } + + bool need_embd() const override { + return false; + } }; // DFlash: block-diffusion drafting with a draft-side KV cache injection @@ -913,9 +922,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { std::vector smpls; - // backend sampler chain per seq, attached to ctx_dft - std::vector backend_chains; - int32_t n_embd_dec = 0; // draft hidden size int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size int32_t n_embd_tgt = 0; // target model hidden size @@ -989,22 +995,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { s.reset(common_sampler_init(model_dft, sparams)); } - // offload draft sampling to the backend - backend_chains.assign(n_seq, nullptr); - if (this->params.backend_sampling) { - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); - llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); - - if (!llama_set_sampler(ctx_dft, seq_id, chain)) { - SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); - llama_sampler_free(chain); - chain = nullptr; - } - backend_chains[seq_id] = chain; - } - } - // turn on extraction of the target layers' input embeddings for (uint32_t k = 0; k < target_layer_ids_n; ++k) { llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); @@ -1015,18 +1005,6 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { } ~common_speculative_impl_draft_dflash() override { - auto * ctx_dft = this->params.ctx_dft; - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) { - if (backend_chains[seq_id] == nullptr) { - continue; - } - if (ctx_dft) { - llama_set_sampler(ctx_dft, seq_id, nullptr); - } - llama_sampler_free(backend_chains[seq_id]); - } - backend_chains.clear(); - llama_batch_free(batch); llama_batch_free(batch_inject); } @@ -1054,14 +1032,7 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { return true; } - // Target prefill may contain token IDs or multimodal embeddings. Both - // produce the target-layer features used to seed the draft KV cache, so - // skipping the embedding batches leaves a hole in the draft's cache and - // the next injection fails to initialize. - // TODO: revisit after https://github.com/ggml-org/llama.cpp/pull/24669 is merged - const bool has_tokens = batch_in.token != nullptr; - const bool has_embeddings = batch_in.embd != nullptr; - if (has_tokens == has_embeddings) { + if (batch_in.token == nullptr || batch_in.embd != nullptr) { return true; } @@ -1269,6 +1240,10 @@ struct common_speculative_impl_draft_dflash : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } + + bool need_embd() const override { + return false; + } }; struct common_speculative_impl_draft_mtp : public common_speculative_impl { @@ -1316,7 +1291,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { GGML_ASSERT(ctx_tgt && ctx_dft && "MTP requires ctx_tgt and ctx_dft to be set"); n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft)); - GGML_ASSERT(n_embd == llama_model_n_embd_out(llama_get_model(ctx_tgt)) && + GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) && "MTP input row width must match the target h_nextn width"); n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); @@ -1707,6 +1682,14 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const size_t row_bytes = (size_t) n_embd * sizeof(float); std::memcpy(pending_h[seq_id].data(), verify_h[seq_id].data() + (size_t) i_h * n_embd, row_bytes); } + + bool need_embd() const override { + return false; + } + + bool need_embd_nextn() const override { + return true; + } }; // state of self-speculation (simple implementation, not ngram-map) @@ -1753,6 +1736,10 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } + + bool need_embd() const override { + return false; + } }; struct common_speculative_impl_ngram_map_k : public common_speculative_impl { @@ -1807,6 +1794,10 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl { common_ngram_map_accept(config[seq_id], n_accepted); } + + bool need_embd() const override { + return false; + } }; struct common_speculative_impl_ngram_mod : public common_speculative_impl { @@ -1982,6 +1973,10 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { } } } + + bool need_embd() const override { + return false; + } }; struct common_speculative_impl_ngram_cache : public common_speculative_impl { @@ -2121,6 +2116,10 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { // noop } + + bool need_embd() const override { + return false; + } }; struct common_speculative { @@ -2228,43 +2227,6 @@ common_speculative_type common_speculative_type_from_name(const std::string & na return it->second; } -std::vector common_speculative_types_from_gguf(const std::string & path) { - struct gguf_init_params gguf_params = { - /* .no_alloc = */ true, - /* .ctx = */ nullptr, - }; - - gguf_context_ptr gguf_ctx(gguf_init_from_file(path.c_str(), gguf_params)); - if (!gguf_ctx) { - return {}; - } - - const int64_t arch_id = gguf_find_key(gguf_ctx.get(), "general.architecture"); - if (arch_id < 0 || gguf_get_kv_type(gguf_ctx.get(), arch_id) != GGUF_TYPE_STRING) { - return {}; - } - - const std::string arch = gguf_get_val_str(gguf_ctx.get(), arch_id); - if (arch != "dflash") { - const uint32_t block_count = gguf_get_val_u32(gguf_ctx.get(), gguf_find_key(gguf_ctx.get(), (arch + ".block_count").c_str())); - - if (gguf_find_tensor(gguf_ctx.get(), ("blk." + std::to_string(block_count - 1) + ".nextn.eh_proj.weight").c_str()) >= 0) { - return { COMMON_SPECULATIVE_TYPE_DRAFT_MTP }; - } - - return {}; - } - - // the Markov head distinguishes draft-dspark from draft-dflash - const auto type = gguf_find_tensor(gguf_ctx.get(), "markov_w1.weight") >= 0 - ? COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK - : COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH; - - SPC_INF("auto-detected speculative type '%s' from the draft model metadata\n", common_speculative_type_to_str(type).c_str()); - - return { type }; -} - static uint32_t common_get_enabled_speculative_configs(const std::vector & configs) { uint32_t result = 0; for (size_t i = 0; i < configs.size(); i++) { @@ -2330,24 +2292,6 @@ common_params common_base_params_to_speculative(const common_params & params) { result.cache_type_k = params_spec.cache_type_k; result.cache_type_v = params_spec.cache_type_v; result.n_outputs_max = params.n_parallel; - result.n_outputs_max_per_seq = 1; - - // dflash/dspark decode the whole noise block in a single pass and sample every block position on the backend - // TODO: refactor such properties to be announced by the speculative types - // something like `struct common_speculative_type_props common_speculative_type_get_props(...);` - const bool has_block_draft = std::any_of( - params.speculative.types.begin(), params.speculative.types.end(), - [](common_speculative_type t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH || t == COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK; - }); - if (has_block_draft) { - // per-seq output positions: DFlash decodes anchor + n_max masks (n_max + 1); DSpark n_max -> +1 covers both - const int32_t per_seq = std::max(1, params_spec.n_max + 1); - result.n_outputs_max = params.n_parallel * per_seq; - if (params_spec.backend_sampling) { - result.n_outputs_max_per_seq = per_seq; - } - } return result; } @@ -2370,6 +2314,7 @@ common_speculative_init_result::common_speculative_init_result( const bool spec_mtp = std::find(params.speculative.types.begin(), params.speculative.types.end(), COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + GGML_ASSERT(has_draft || spec_mtp); auto mparams = common_model_params_to_llama(params); auto cparams = common_context_params_to_llama(params); @@ -2432,17 +2377,6 @@ common_speculative_init_result_ptr common_speculative_init_from_params(common_pa return std::make_unique(params, model_tgt, ctx_tgt); } -common_speculative_output_limits common_speculative_get_output_limits( - int32_t n_batch, int32_t n_parallel, int32_t n_draft) { - const int64_t per_seq = 1 + (int64_t) std::max(0, n_draft); - const int64_t total = (int64_t) n_parallel * per_seq; - - return { - /* .total = */ (int32_t) std::min(n_batch, total), - /* .per_seq = */ (int32_t) std::min(n_batch, per_seq), - }; -} - // initialization of the speculative decoding system // common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq) { @@ -2451,28 +2385,57 @@ common_speculative * common_speculative_init(common_params_speculative & params, { uint32_t enabled_configs = common_get_enabled_speculative_configs(params.types); - auto add_config_if_enabled = [&](common_speculative_type type, bool available = true) { - if (available && (enabled_configs & (1u << type))) { - configs.emplace_back(type, params); - } - }; + bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); + bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr; + bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; + bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr; + bool has_draft_dspark = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK)) && params.draft.ctx_dft != nullptr; + + + + bool has_ngram_cache = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_CACHE)); + bool has_ngram_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE)); + bool has_ngram_map_k = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K)); + bool has_ngram_map_k4v = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V)); + bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); // when adding a new type - update here the logic above static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 11); // this list here defines the priority of the speculators // the one with highest priority are listed first - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_MOD); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE); - - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params.draft.ctx_dft != nullptr); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params.draft.ctx_dft != nullptr); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params.draft.ctx_dft != nullptr); - add_config_if_enabled(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params.draft.ctx_dft != nullptr); + if (has_ngram_simple) { + // This implementation can guess a lot of tokens without any draft model. + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, params)); + } + if (has_ngram_map_k) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, params)); + } + if (has_ngram_map_k4v) { + // This implementation can guess tokens with high acceptance rate but is more expensive. + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, params)); + } + if (has_ngram_mod) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_MOD, params)); + } + if (has_ngram_cache) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_NGRAM_CACHE, params)); + } + if (has_draft_simple) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, params)); + } + if (has_draft_eagle3) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params)); + } + if (has_draft_mtp) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); + } + if (has_draft_dflash) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params)); + } + if (has_draft_dspark) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DSPARK, params)); + } } std::vector> impls = {}; @@ -2607,6 +2570,34 @@ bool common_speculative_process(common_speculative * spec, const llama_batch & b return result; } +bool common_speculative_need_embd(common_speculative * spec) { + if (spec == nullptr) { + return false; + } + + for (auto & impl : spec->impls) { + if (impl->need_embd()) { + return true; + } + } + + return false; +} + +bool common_speculative_need_embd_nextn(common_speculative * spec) { + if (spec == nullptr) { + return false; + } + + for (auto & impl : spec->impls) { + if (impl->need_embd_nextn()) { + return true; + } + } + + return false; +} + void common_speculative_draft(common_speculative * spec) { if (spec == nullptr) { return; @@ -2691,10 +2682,7 @@ void common_speculative_draft(common_speculative * spec) { void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, uint16_t n_accepted) { common_speculative_impl * impl = spec->impl_last[seq_id]; - if (impl == nullptr) { - GGML_ASSERT(n_accepted == 0); - return; - } + GGML_ASSERT(impl); { common_time_meas tm(impl->t_accept_us, !impl->gen_perf); diff --git a/common/speculative.h b/common/speculative.h index 12ae31b7d..062bf2093 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -14,9 +14,6 @@ const char * common_speculative_all_types_str(); // parse user provided types std::vector common_speculative_types_from_names(const std::vector & names); -// infer the spec types from the GGUF metadata of a draft model; empty if unknown -std::vector common_speculative_types_from_gguf(const std::string & path); - // convert string to type enum common_speculative_type common_speculative_type_from_name(const std::string & name); @@ -28,15 +25,6 @@ int32_t common_speculative_n_max(const common_params_speculative * spec); common_params common_base_params_to_speculative(const common_params & params); -struct common_speculative_output_limits { - int32_t total; - int32_t per_seq; -}; - -// return the output limits needed for speculative decoding -common_speculative_output_limits common_speculative_get_output_limits( - int32_t n_batch, int32_t n_parallel, int32_t n_draft); - common_speculative * common_speculative_init(common_params_speculative & params, uint32_t n_seq); void common_speculative_free(common_speculative * spec); @@ -70,6 +58,12 @@ void common_speculative_begin(common_speculative * spec, llama_seq_id seq_id, co // process the batch and update the internal state of the speculative context bool common_speculative_process(common_speculative * spec, const llama_batch & batch); +// true if any implementation requires target post-norm embeddings to be extracted +bool common_speculative_need_embd(common_speculative * spec); + +// true if any implementation requires target nextn embeddings to be extracted +bool common_speculative_need_embd_nextn(common_speculative * spec); + // generate drafts for the sequences specified with `common_speculative_get_draft_params` void common_speculative_draft(common_speculative * spec); diff --git a/conversion/__init__.py b/conversion/__init__.py index f4d475de7..1a47b851a 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -55,7 +55,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "DFlashDraftModel": "qwen", "Qwen3DSparkModel": "qwen", "DeepseekV4ForCausalLM": "deepseek", - "DeepseekV4DSparkModel": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", @@ -70,7 +69,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "Exaone4ForCausalLM": "exaone", "ExaoneForCausalLM": "exaone", "ExaoneMoEForCausalLM": "exaone", - "ExaoneMoeForCausalLM": "exaone", "FalconForCausalLM": "falcon", "FalconH1ForCausalLM": "falcon_h1", "FalconMambaForCausalLM": "mamba", @@ -103,7 +101,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "GraniteMoeForCausalLM": "granite", "GraniteMoeHybridForCausalLM": "granite", "GraniteMoeSharedForCausalLM": "granite", - "GraniteSwitchForCausalLM": "granite", "GraniteSpeechForConditionalGeneration": "granite", "GraniteSpeechPlusForConditionalGeneration": "granite", "Grok1ForCausalLM": "grok", @@ -161,8 +158,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "MiniCPM3ForCausalLM": "minicpm", "MiniCPMForCausalLM": "minicpm", "MiniCPMV4_6ForConditionalGeneration": "minicpm", - "MiniMaxText01ForCausalLM": "minimax", - "MiniMaxM1ForCausalLM": "minimax", "MiniMaxM2ForCausalLM": "minimax", "MiniMaxM3SparseForCausalLM": "minimax", "MiniMaxM3SparseForConditionalGeneration": "minimax", @@ -185,8 +180,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "Olmo3ForCausalLM": "olmo", "OlmoForCausalLM": "olmo", "OlmoeForCausalLM": "olmo", - "MuseGlimmerAssistantModel": "muse_glimmer", - "MuseGlimmerForConditionalGeneration": "muse_glimmer", "OpenELMForCausalLM": "openelm", "OrionForCausalLM": "orion", "PLMForCausalLM": "plm", @@ -216,8 +209,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "Qwen3MoeForCausalLM": "qwen", "Qwen3NextForCausalLM": "qwen", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", - "PocketTTSModel": "pockettts", - "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForCausalLM": "qwen", @@ -303,7 +294,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "MiniCPMV4_6ForConditionalGeneration": "minicpm", "Mistral3ForConditionalGeneration": "llava", "NemotronH_Nano_VL_V2": "nemotron", - "MuseGlimmerForConditionalGeneration": "muse_glimmer", "PaddleOCRVisionModel": "ernie", "Phi4ForCausalLMV": "phi", "Qwen2AudioForConditionalGeneration": "ultravox", @@ -313,8 +303,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen2_5_VLForConditionalGeneration": "qwenvl", "Qwen3ASRForConditionalGeneration": "qwen3vl", "Qwen3OmniMoeForConditionalGeneration": "qwen3vl", - "PocketTTSModel": "pockettts", - "Qwen3TTSForConditionalGeneration": "qwen3tts", "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "Qwen3_5ForConditionalGeneration": "qwen3vl", diff --git a/conversion/base.py b/conversion/base.py index 718d53944..a7cd3fd90 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -58,11 +58,6 @@ logger = logging.getLogger("hf-to-gguf") AnyModel = TypeVar("AnyModel", bound="type[ModelBase]") -# for checkpoints that ship no config.json, we will try to provide a synthetic one -HparamsMatcher = Callable[[Path], bool] -HparamsLoader = Callable[[Path], dict[str, Any]] - - class SentencePieceTokenTypes(IntEnum): NORMAL = 1 UNKNOWN = 2 @@ -82,7 +77,6 @@ class ModelBase: ModelType.TEXT: {}, ModelType.MMPROJ: {}, } - _hparams_loaders: list[tuple[HparamsMatcher, HparamsLoader]] = [] dir_model: Path ftype: gguf.LlamaFileType @@ -829,7 +823,7 @@ class ModelBase: elif any(str(v.get("quant_algo")).endswith("NVFP4") for v in quant_layers.values() if isinstance(v, dict)): quant_algo = "NVFP4" - self._is_nvfp4 = quant_algo in ("NVFP4", "W4A16_NVFP4") + self._is_nvfp4 = quant_algo == "NVFP4" self._is_mxfp4 = quant_method == "mxfp4" # NVFP4 weights are repacked and written directly to gguf_writer. @@ -1046,24 +1040,6 @@ class ModelBase: return part_names - @staticmethod - def load_hparams_guess(dir_model: Path) -> dict[str, Any] | None: - # some models ship no config.json, will try to guess them - from conversion import load_all_models - load_all_models() - - for matcher, loader in ModelBase._hparams_loaders: - if matcher(dir_model): - return loader(dir_model) - return None - - @classmethod - def register_hparams_loader(cls, matcher: HparamsMatcher) -> Callable[[HparamsLoader], HparamsLoader]: - def inner(loader: HparamsLoader) -> HparamsLoader: - cls._hparams_loaders.append((matcher, loader)) - return loader - return inner - @staticmethod def load_hparams(dir_model: Path, is_mistral_format: bool): if is_mistral_format: @@ -1077,10 +1053,6 @@ class ModelBase: config = AutoConfig.from_pretrained(dir_model, trust_remote_code=False).to_dict() except Exception as e: logger.warning(f"Failed to load model config from {dir_model}: {e}") - if not (dir_model / "config.json").is_file(): - config = ModelBase.load_hparams_guess(dir_model) - if config is not None: - return config logger.warning("Trying to load config.json instead") with open(dir_model / "config.json", "r", encoding="utf-8") as f: config = json.load(f) diff --git a/conversion/chatglm.py b/conversion/chatglm.py index d63855038..801913075 100644 --- a/conversion/chatglm.py +++ b/conversion/chatglm.py @@ -81,7 +81,7 @@ class ChatGLMModel(TextModel): @staticmethod def token_bytes_to_string(b): - from transformers.convert_slow_tokenizer import bytes_to_unicode + from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] byte_encoder = bytes_to_unicode() return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 1846ca401..ea6ae23d5 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -17,11 +17,8 @@ from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logg from .qwen import QwenModel -@ModelBase.register("DeepseekOCRForCausalLM") +@ModelBase.register("DeepseekOCRForCausalLM", "UnlimitedOCRForCausalLM") class DeepseekOCRVisionModel(MmprojModel): - # HF dynamic_preprocess() max_num, which differs per model - preproc_max_tiles = 9 - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR @@ -46,9 +43,6 @@ class DeepseekOCRVisionModel(MmprojModel): # @bluebread: there's no window_size in config but just add it here anyway self.gguf_writer.add_vision_window_size(self.hparams.get("window_size", 14)) - self.gguf_writer.add_vision_preproc_min_tiles(2) - self.gguf_writer.add_vision_preproc_max_tiles(self.preproc_max_tiles) - # SAM configuration sam_hparams = hparams['sam'] self.gguf_writer.add_vision_sam_layers_count(sam_hparams['layers']) @@ -99,15 +93,8 @@ class DeepseekOCRVisionModel(MmprojModel): return super().filter_tensors((name, gen)) -@ModelBase.register("UnlimitedOCRForCausalLM") -class UnlimitedOCRVisionModel(DeepseekOCRVisionModel): - preproc_max_tiles = 32 - - @ModelBase.register("DeepseekOCR2ForCausalLM") class DeepseekOCR2VisionModel(DeepseekOCRVisionModel): - preproc_max_tiles = 6 - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.clip_projector_type = gguf.VisionProjectorType.DEEPSEEKOCR2 @@ -460,43 +447,12 @@ class DeepseekV2Model(TextModel): class DeepseekV32Model(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.DEEPSEEK32 skip_mtp = False - supports_mtp_export = True - _n_main_layers: int | None = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] - if not self.no_mtp: - self.block_count += self.hparams.get("num_nextn_predict_layers", 0) + self.block_count = self.hparams["num_hidden_layers"] + self.hparams.get("num_nextn_predict_layers", 0) self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - def index_tensors(self, remote_hf_model_id: str | None = None): - type(self)._n_main_layers = self.hparams["num_hidden_layers"] - return super().index_tensors(remote_hf_model_id=remote_hf_model_id) - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - if (titem := super().filter_tensors(item)) is None: - return None - name, gen = titem - - # DeepSeek V3.2 appends the NextN/MTP block past num_hidden_layers - # (model.layers.61 -> blk.61 in the 62-block file). - assert cls._n_main_layers is not None - is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers - - # --no-mtp: drop the appended NextN block entirely. - if is_mtp and cls.no_mtp: - return None - # --mtp: keep ONLY NextN-block tensors plus the shared embeddings/ - # norm/lm_head (so the resulting GGUF carries just the draft head). - if cls.mtp_only and not is_mtp and name not in ( - "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", - ): - return None - - return name, gen - def set_vocab(self): from transformers import AutoTokenizer tokenizer = AutoTokenizer.from_pretrained(self.dir_model) @@ -507,7 +463,7 @@ class DeepseekV32Model(DeepseekV2Model): super().set_gguf_parameters() # NextN/MTP prediction layers - if not self.no_mtp and (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: + if (num_nextn_predict_layers := self.hparams.get("num_nextn_predict_layers")) is not None: self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) # DSA indexer parameters @@ -519,10 +475,7 @@ class DeepseekV32Model(DeepseekV2Model): @ModelBase.register("DeepseekV4ForCausalLM") class DeepseekV4Model(TextModel): model_arch = gguf.MODEL_ARCH.DEEPSEEK4 - supports_mtp_export = True _skipped_mtp_tensors = 0 - _dsv4_main_layers: int | None = None - _dsv4_nextn_layers: int = 0 def __init__(self, *args, **kwargs): type(self)._skipped_mtp_tensors = 0 @@ -533,16 +486,7 @@ class DeepseekV4Model(TextModel): for key, value in raw_hparams.items(): self.hparams.setdefault(key, value) - # workaround for special rope_parameters (main/compress) in transformers 5.x - if self.rope_parameters.get("full_attention", self.rope_parameters).get("rope_type") is None: - if (rope_scaling := raw_hparams.get("rope_scaling")) is not None: - if "rope_type" not in rope_scaling and (rope_type := rope_scaling.get("type")) is not None: - rope_scaling["rope_type"] = rope_type - self.rope_parameters.update(**rope_scaling) - self.block_count = self.hparams["num_hidden_layers"] - if self.mtp_only: - self.block_count += self.hparams.get("num_nextn_predict_layers", 0) self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) self._dsv4_fp8_dequantized: set[str] = set() @@ -555,71 +499,18 @@ class DeepseekV4Model(TextModel): logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors) # add a default chat template; if the model has a built-in template, it will be overridden later - model_id_hint = self.remote_hf_model_id or self.dir_model.name - is_0731 = "0731" in model_id_hint - template_name = "deepseek-ai-DeepSeek-V4-Flash-0731.jinja" if is_0731 else "deepseek-ai-DeepSeek-V4.jinja" - template_path = Path(__file__).parent.parent / "models" / "templates" / template_name + template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja" if template_path.is_file(): with open(template_path, "r", encoding="utf-8") as f: self.gguf_writer.add_chat_template(f.read()) - def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: - type(self)._dsv4_main_layers = self.hparams["num_hidden_layers"] - type(self)._dsv4_nextn_layers = self.hparams.get("num_nextn_predict_layers", 0) - return super().index_tensors(remote_hf_model_id=remote_hf_model_id) - @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item + name, _ = item if name.startswith("mtp."): - if not cls.mtp_only: - cls._skipped_mtp_tensors += 1 - return None - - assert cls._dsv4_main_layers is not None - parts = name.split(".", 2) - if len(parts) < 3 or not parts[1].isdecimal(): - raise ValueError(f"Unexpected DeepSeek-V4 MTP tensor {name!r}") - - mtp_idx = int(parts[1]) - if mtp_idx >= cls._dsv4_nextn_layers: - raise ValueError(f"Unexpected DeepSeek-V4 MTP layer {mtp_idx}") - - bid = cls._dsv4_main_layers + mtp_idx - suffix = parts[2] - root_hc_head = { - "hc_head_fn", - "hc_head_base", - "hc_head_scale", - } - if suffix in root_hc_head: - name = suffix - elif suffix in ( - "e_proj.weight", "e_proj.scale", - "h_proj.weight", "h_proj.scale", - ): - name = f"layers.{bid}.nextn.{suffix}" - elif suffix == "enorm.weight": - name = f"layers.{bid}.nextn.enorm.weight" - elif suffix == "hnorm.weight": - name = f"layers.{bid}.nextn.hnorm.weight" - elif suffix == "norm.weight": - name = f"layers.{bid}.nextn.shared_head_norm.weight" - else: - name = f"layers.{bid}.{suffix}" - return name, gen - - if cls.mtp_only: - keep = name in ( - "embed.weight", - "norm.weight", - "head.weight", - "head.scale", - ) - if not keep: - return None - - return super().filter_tensors((name, gen)) + cls._skipped_mtp_tensors += 1 + return None + return super().filter_tensors(item) @staticmethod def _float8_dtypes() -> tuple[torch.dtype, ...]: @@ -674,10 +565,6 @@ class DeepseekV4Model(TextModel): self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) - if self.model_arch == gguf.MODEL_ARCH.DEEPSEEK4: - self.gguf_writer.add_embedding_length_out(hparams["hidden_size"] * hparams["hc_mult"]) - if self.mtp_only and (num_nextn_predict_layers := hparams.get("num_nextn_predict_layers", 0)) > 0: - self.gguf_writer.add_nextn_predict_layers(num_nextn_predict_layers) def dequant_model(self): fp8_dtypes = self._float8_dtypes() @@ -782,37 +669,12 @@ class DeepseekV4Model(TextModel): if self._dsv4_mxfp4_generated: return () - consumed: list[str] = [] - main_layers = self.hparams["num_hidden_layers"] - if not self.mtp_only: - consumed.extend(self._write_hash_routing_tensors()) - elif self.hparams["num_hash_layers"] > 0: - for bid in range(self.hparams["num_hash_layers"]): - name = f"layers.{bid}.ffn.gate.tid2eid" - if name in self.model_tensors: - consumed.extend(self._write_hash_routing_tensors()) - break - + consumed: list[str] = self._write_hash_routing_tensors() for bid in range(self.block_count): - if self.mtp_only and bid < main_layers: - continue consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)) consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)) consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)) - for bid in range(main_layers, self.block_count): - e_name = f"layers.{bid}.nextn.e_proj.weight" - h_name = f"layers.{bid}.nextn.h_proj.weight" - if e_name not in self.model_tensors and h_name not in self.model_tensors: - continue - if e_name not in self.model_tensors or h_name not in self.model_tensors: - raise KeyError(f"Missing DeepSeek-V4 MTP e/h projection pair for block {bid}") - - e_proj = LazyTorchTensor.to_eager(self.model_tensors[e_name]()) - h_proj = LazyTorchTensor.to_eager(self.model_tensors[h_name]()) - yield (f"layers.{bid}.nextn.eh_proj.weight", torch.cat((e_proj, h_proj), dim=1).contiguous()) - consumed.extend((e_name, h_name)) - for name in consumed: del self.model_tensors[name] @@ -875,12 +737,6 @@ class DeepseekV4Model(TextModel): "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), "ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"), - "nextn.eh_proj.weight": (gguf.MODEL_TENSOR.NEXTN_EH_PROJ, ".weight"), - "nextn.enorm.weight": (gguf.MODEL_TENSOR.NEXTN_ENORM, ".weight"), - "nextn.hnorm.weight": (gguf.MODEL_TENSOR.NEXTN_HNORM, ".weight"), - "nextn.shared_head_norm.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_NORM, ".weight"), - "nextn.embed_tokens.weight": (gguf.MODEL_TENSOR.NEXTN_EMBED_TOKENS, ".weight"), - "nextn.shared_head_head.weight": (gguf.MODEL_TENSOR.NEXTN_SHARED_HEAD_HEAD, ".weight"), } tensor_name = match.group(2) @@ -903,12 +759,10 @@ class DeepseekV4Model(TextModel): return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)] def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: - del bid # unused + del new_name, bid # unused if name in self._dsv4_fp8_dequantized and n_dims >= 2: return gguf.GGMLQuantizationType.Q8_0 - if new_name.endswith(".nextn.eh_proj.weight"): - return gguf.GGMLQuantizationType.Q8_0 if name in self._dsv4_f32_tensors: return gguf.GGMLQuantizationType.F32 if name in self._dsv4_bf16_tensors and n_dims >= 2: @@ -916,122 +770,7 @@ class DeepseekV4Model(TextModel): return False - def prepare_metadata(self, vocab_only: bool): - from_dir = self.fname_out.is_dir() - super().prepare_metadata(vocab_only=vocab_only) - - if not self.mtp_only or not from_dir: - return - - output_type: str = self.ftype.name.partition("_")[2] - fname_default: str = gguf.naming_convention( - self.metadata.name, self.metadata.basename, self.metadata.finetune, - self.metadata.version, size_label=None, output_type=output_type, model_type=None) - self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" - def prepare_tensors(self): super().prepare_tensors() self._is_mxfp4 = True self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE - - -@ModelBase.register("DeepseekV4DSparkModel") -class DeepseekV4DSparkModel(DeepseekV4Model): - model_arch = gguf.MODEL_ARCH.DFLASH - - _DSPARK_ROOT_MAP: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { - "main_proj.weight": (gguf.MODEL_TENSOR.FC, ".weight"), - "main_norm.weight": (gguf.MODEL_TENSOR.ENC_OUTPUT_NORM, ".weight"), - "markov_head.markov_w1.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W1, ".weight"), - "markov_head.markov_w2.weight": (gguf.MODEL_TENSOR.DSPARK_MARKOV_W2, ".weight"), - "confidence_head.proj.weight": (gguf.MODEL_TENSOR.DSPARK_CONF_PROJ, ".weight"), - } - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.block_count = 1 + max( - int(match.group(1)) for name in self.model_tensors - if (match := re.match(r"layers\.(\d+)\.", name)) - ) - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - self.hparams["compress_ratios"] = [0] * self.block_count - self.hparams["num_hash_layers"] = 0 - - def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: - if remote_hf_model_id is None: - return super().index_tensors() - - with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: - weight_map = json.load(f)["weight_map"] - - part_names = sorted({ - part_name for name, part_name in weight_map.items() - if name.startswith("mtp.") - }) - tensors: dict[str, Callable[[], Tensor]] = {} - - for part_name in part_names: - from huggingface_hub import hf_hub_download - - logger.info("gguf: caching remote DSpark part '%s'", part_name) - part_path = Path(hf_hub_download(repo_id=remote_hf_model_id, filename=part_name)) - with gguf.utility.SafetensorsLocal(part_path) as model_part: - for name in model_part: - data = model_part[name] - data_gen = lambda data=data: LazyTorchTensor.from_local_tensor(data) # noqa: E731 - if titem := self.filter_tensors((name, data_gen)): - tensor_name, tensor_gen = titem - tensors[tensor_name] = tensor_gen - - return tensors - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - if not name.startswith("mtp."): - return None - return super().filter_tensors((cls._rekey_mtp_tensor_name(name), gen)) - - @staticmethod - def _rekey_mtp_tensor_name(name: str) -> str: - match = re.match(r"mtp\.(\d+)\.(.+)$", name) - if match is None: - raise ValueError(f"Unexpected DSpark tensor {name!r}") - - stage, rest = match.group(1), match.group(2) - root_names = ( - "main_proj.scale", - "norm.weight", - "hc_head_fn", - "hc_head_base", - "hc_head_scale", - ) - if rest in DeepseekV4DSparkModel._DSPARK_ROOT_MAP or rest in root_names: - return rest - return f"layers.{stage}.{rest}" - - def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: - if name in self._DSPARK_ROOT_MAP: - return self._DSPARK_ROOT_MAP[name] - return super()._map_dsv4_tensor_name(name, bid) - - def set_vocab(self): - if self.target_model_dir is None: - raise ValueError("DeepSeek-V4 DSpark requires --target-model-dir with the target tokenizer") - - original_dir = self.dir_model - try: - self.dir_model = self.target_model_dir - super().set_vocab() - finally: - self.dir_model = original_dir - - self.gguf_writer.add_mask_token_id(self.hparams["dspark_noise_token_id"]) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - - self.gguf_writer.add_block_size(self.hparams["dspark_block_size"]) - self.gguf_writer.add_target_layers([layer + 1 for layer in self.hparams["dspark_target_layer_ids"]]) diff --git a/conversion/exaone.py b/conversion/exaone.py index 1cd2244db..bc4fb3f1b 100644 --- a/conversion/exaone.py +++ b/conversion/exaone.py @@ -123,9 +123,7 @@ class Exaone4Model(TextModel): yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), torch.tensor(rope_factors, dtype=torch.float32)) -# note: transformers >= 5.1 renamed the class to "ExaoneMoeForCausalLM" (lowercase 'e'), -# so accept both spellings - LG AI have updated the configs of already-released models -@ModelBase.register("ExaoneMoEForCausalLM", "ExaoneMoeForCausalLM") +@ModelBase.register("ExaoneMoEForCausalLM") class ExaoneMoEModel(Exaone4Model): model_arch = gguf.MODEL_ARCH.EXAONE_MOE diff --git a/conversion/gemma.py b/conversion/gemma.py index f15a10a38..c552df732 100644 --- a/conversion/gemma.py +++ b/conversion/gemma.py @@ -665,18 +665,7 @@ class Gemma4Model(Gemma3Model): swa_layers = [t == "sliding_attention" for t in self.hparams["layer_types"]] self.gguf_writer.add_sliding_window_pattern(swa_layers) - per_layer_config = self.hparams.get("per_layer_config") - layer_types = self.hparams.get("layer_types", []) - if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None: - for layer_idx, layer_config in per_layer_config.items(): - layer_idx = int(layer_idx) - if layer_idx < len(layer_types): - if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config: - head_dim_full = layer_config["head_dim"] - break - - assert head_dim_full is not None - + head_dim_full = self.hparams["global_head_dim"] head_dim_swa = self.hparams["head_dim"] # correct the head dim for global/swa layers self.gguf_writer.add_key_length(head_dim_full) @@ -696,14 +685,8 @@ class Gemma4Model(Gemma3Model): n_ff_arr = [n_ff if il < first_kv_shared_layer_idx else n_ff * 2 for il in range(self.block_count)] self.gguf_writer.add_feed_forward_length(n_ff_arr) - if (num_key_value_heads_full := self.hparams.get("num_global_key_value_heads")) is None and per_layer_config is not None: - for layer_idx, layer_config in per_layer_config.items(): - layer_idx = int(layer_idx) - if layer_idx < len(layer_types): - if layer_types[layer_idx] == "full_attention" and "num_key_value_heads" in layer_config: - num_key_value_heads_full = layer_config["num_key_value_heads"] - break - + # handle num_global_key_value_heads + num_key_value_heads_full = self.hparams.get("num_global_key_value_heads") num_key_value_heads_swa = self.hparams.get("num_key_value_heads") if num_key_value_heads_full is not None and num_key_value_heads_swa is not None: value_arr = [num_key_value_heads_swa if is_swa else num_key_value_heads_full for is_swa in swa_layers] @@ -725,19 +708,7 @@ class Gemma4Model(Gemma3Model): # IMPORTANT: this ROPE_FREQS tensor is ONLY used by the full_attention layers rope_params_full = self.hparams["rope_parameters"]["full_attention"] assert rope_params_full["rope_type"] == "proportional" - - per_layer_config = self.hparams.get("per_layer_config") - if (head_dim_full := self.hparams.get("global_head_dim")) is None and per_layer_config is not None: - layer_types = self.hparams.get("layer_types", []) - for layer_idx, layer_config in per_layer_config.items(): - layer_idx = int(layer_idx) - if layer_idx < len(layer_types): - if layer_types[layer_idx] == "full_attention" and "head_dim" in layer_config: - head_dim_full = layer_config["head_dim"] - break - - assert head_dim_full is not None - + head_dim_full = (self.hparams["global_head_dim"]) partial_rotary_factor_full = rope_params_full["partial_rotary_factor"] n_rot_full = int(head_dim_full * partial_rotary_factor_full / 2) n_unrot_full = int(head_dim_full / 2) - n_rot_full diff --git a/conversion/glm.py b/conversion/glm.py index e28f54574..cc34cddbf 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -206,70 +206,10 @@ class Glm4MoeModel(TextModel): @ModelBase.register("Glm4MoeLiteForCausalLM") class Glm4MoeLiteModel(DeepseekV2Model): model_arch = gguf.MODEL_ARCH.DEEPSEEK2 - skip_mtp = False - supports_mtp_export = True - _n_main_layers: int | None = None def set_vocab(self): return self._set_vocab_glm() - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - num_hidden_layers = self.hparams["num_hidden_layers"] - self.num_nextn_predict_layers = self.hparams.get("num_nextn_predict_layers", 0) - self.skip_mtp = self.no_mtp or self.num_nextn_predict_layers == 0 - - if self.skip_mtp: - self.block_count = num_hidden_layers - else: - self.block_count = num_hidden_layers + self.num_nextn_predict_layers - - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - - if self.skip_mtp: - return - - self.gguf_writer.add_nextn_predict_layers(self.num_nextn_predict_layers) - - def index_tensors(self, remote_hf_model_id: str | None = None): - type(self)._n_main_layers = self.hparams["num_hidden_layers"] - return super().index_tensors(remote_hf_model_id=remote_hf_model_id) - - @classmethod - def filter_tensors(cls, item): - if (titem := super().filter_tensors(item)) is None: - return None - name, gen = titem - - if cls._n_main_layers is not None: - match = re.match(r"model\.layers\.(\d+)\.", name) - is_mtp = match is not None and int(match.group(1)) >= cls._n_main_layers - if is_mtp and cls.no_mtp: - return None - if cls.mtp_only and not is_mtp and name not in ( - "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", - ): - return None - - return name, gen - - def prepare_metadata(self, vocab_only: bool): - from_dir = self.fname_out.is_dir() - super().prepare_metadata(vocab_only=vocab_only) - - if not self.mtp_only or not from_dir: - return - - output_type: str = self.ftype.name.partition("_")[2] - fname_default: str = gguf.naming_convention( - self.metadata.name, self.metadata.basename, self.metadata.finetune, - self.metadata.version, size_label=None, output_type=output_type, model_type=None) - self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" - @ModelBase.register("GlmMoeDsaForCausalLM") class GlmMoeDsaModel(DeepseekV2Model): diff --git a/conversion/granite.py b/conversion/granite.py index 956342e6d..8367ed225 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -123,166 +123,6 @@ class GraniteMoeModel(GraniteModel): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("GraniteSwitchForCausalLM") -class GraniteSwitchModel(GraniteMoeModel): - """Dense, all-attention Granite with N per-token embedded LoRA adapters, stacked - over the adapter dim with a zero adapter at slot 0 (N = num_adapters + 1).""" - model_arch = gguf.MODEL_ARCH.GRANITE_SWITCH - - # permute q/k per-slice below (NORM-rope layout), not via the parent's auto-permute - undo_permute = False - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # the weightless switch reserves one cache slot: one fewer block than num_hidden_layers - self.block_count = self.block_count - 1 - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - self._n_adapters = int(self.hparams["num_adapters"]) - self._max_lora_rank = int(self.hparams["max_lora_rank"]) - self._n_slots = self._n_adapters + 1 # +1 for the zero slot at index 0 - - n_head = int(self.hparams["num_attention_heads"]) - n_kv_head = int(self.hparams["num_key_value_heads"]) - head_dim = ( - self.hparams.get("projection_head_dim") - or self.hparams.get("head_dim") - or (self.hparams["hidden_size"] // n_head) - ) - self._n_head = n_head - self._n_kv_head = n_kv_head - self._head_dim = int(head_dim) - self._q_size = n_head * self._head_dim - self._kv_size = n_kv_head * self._head_dim - - def set_gguf_parameters(self): - super().set_gguf_parameters() - - # dense: pin expert_used_count to 0 (config carries a leftover num_experts_per_tok) - if not self.hparams.get("num_local_experts"): - self.gguf_writer.add_expert_used_count(0) - - self.gguf_writer.add_adapter_count(self._n_adapters) - self.gguf_writer.add_adapter_lora_rank(self._max_lora_rank) - self.gguf_writer.add_adapter_token_ids_activate(self.hparams["adapter_token_ids"]) - self.gguf_writer.add_adapter_token_ids_substitute(self.hparams["adapter_substitute_token_ids"]) - router_gain = float(self.hparams.get("control_token_gain", 15.0)) - self.gguf_writer.add_adapter_router_gain(router_gain) - logger.info("gguf: (graniteswitch) num_adapters=%s max_lora_rank=%s n_slots=%s router_gain=%s", self._n_adapters, self._max_lora_rank, self._n_slots, router_gain) - - def _lora_a(self, data: Tensor) -> Tensor: - # on-disk A: [n_adapters, 1, max_rank, in] -> [n_adapters+1, max_rank, in] - a = data.squeeze(1) - zero = torch.zeros_like(a[:1]) - return torch.cat([zero, a], dim=0).contiguous() - - def _lora_b(self, data: Tensor, permute_n_head: int | None = None) -> Tensor: - # on-disk B: [n_adapters, 1, out, max_rank] -> [n_adapters+1, out, max_rank] - b = data.squeeze(1) - if permute_n_head is not None: - # permute each adapter's B output rows to match the permuted q/k base - b = torch.stack([self.permute(b[i], permute_n_head, permute_n_head) for i in range(b.shape[0])], dim=0) - zero = torch.zeros_like(b[:1]) - return torch.cat([zero, b], dim=0).contiguous() - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - T = gguf.MODEL_TENSOR - - # skip the weightless switch + control-token buffers (rebuilt at load time) - bare = name.split(".")[-1] - if ( - name.startswith("model.switch.") or name.startswith("switch.") - or bare in ("adapter_token_ids", "control_to_substitute_lut") - ): - return - - if "self_attn.qkv_proj" in name: - if name.endswith("base_layer.weight"): - # fused [q|k|v] rows: permute q/k row-blocks for ggml's NORM-rope layout - q, k, v = data_torch.split([self._q_size, self._kv_size, self._kv_size], dim=0) - q = self.permute(q, self._n_head, self._n_head) - k = self.permute(k, self._n_kv_head, self._n_kv_head) - fused = torch.cat([q, k, v], dim=0) - yield (self.format_tensor_name(T.ATTN_QKV, bid), fused) - return - if "lora_A_slices." in name: - slot = int(name.rsplit(".", 1)[1]) - key = {0: T.ATTN_Q, 1: T.ATTN_K, 2: T.ATTN_V}[slot] - yield (self.format_tensor_name(key, bid, suffix=".lora_a"), self._lora_a(data_torch)) - return - if "lora_B_slices." in name: - slot = int(name.rsplit(".", 1)[1]) - key, ph = { - 0: (T.ATTN_Q, self._n_head), - 1: (T.ATTN_K, self._n_kv_head), - 2: (T.ATTN_V, None), - }[slot] - yield (self.format_tensor_name(key, bid, suffix=".lora_b"), self._lora_b(data_torch, ph)) - return - raise ValueError(f"Unexpected qkv_proj tensor: {name}") - - if "self_attn.o_proj" in name: - if name.endswith("base_layer.weight"): - yield (self.format_tensor_name(T.ATTN_OUT, bid), data_torch) - return - if name.endswith("lora_A"): - yield (self.format_tensor_name(T.ATTN_OUT, bid, suffix=".lora_a"), self._lora_a(data_torch)) - return - if name.endswith("lora_B"): - yield (self.format_tensor_name(T.ATTN_OUT, bid, suffix=".lora_b"), self._lora_b(data_torch)) - return - raise ValueError(f"Unexpected o_proj tensor: {name}") - - if "shared_mlp.input_linear" in name: - ffn = self.hparams["shared_intermediate_size"] - if name.endswith("base_layer.weight"): - gate, up = data_torch.split([ffn, ffn], dim=0) - yield (self.format_tensor_name(T.FFN_GATE, bid), gate) - yield (self.format_tensor_name(T.FFN_UP, bid), up) - return - if "lora_A_slices." in name: - slot = int(name.rsplit(".", 1)[1]) - key = {0: T.FFN_GATE, 1: T.FFN_UP}[slot] - yield (self.format_tensor_name(key, bid, suffix=".lora_a"), self._lora_a(data_torch)) - return - if "lora_B_slices." in name: - slot = int(name.rsplit(".", 1)[1]) - key = {0: T.FFN_GATE, 1: T.FFN_UP}[slot] - yield (self.format_tensor_name(key, bid, suffix=".lora_b"), self._lora_b(data_torch)) - return - raise ValueError(f"Unexpected shared_mlp.input_linear tensor: {name}") - - if "shared_mlp.output_linear" in name: - if name.endswith("base_layer.weight"): - yield (self.format_tensor_name(T.FFN_DOWN, bid), data_torch) - return - if name.endswith("lora_A"): - yield (self.format_tensor_name(T.FFN_DOWN, bid, suffix=".lora_a"), self._lora_a(data_torch)) - return - if name.endswith("lora_B"): - yield (self.format_tensor_name(T.FFN_DOWN, bid, suffix=".lora_b"), self._lora_b(data_torch)) - return - raise ValueError(f"Unexpected shared_mlp.output_linear tensor: {name}") - - if bid is not None and ".layers." in name and ( - "input_layernorm" in name or "post_attention_layernorm" in name - ): - key = T.ATTN_NORM if "input_layernorm" in name else T.FFN_NORM - yield (self.format_tensor_name(key, bid), data_torch) - return - - if name in ("model.embed_tokens.weight", "embed_tokens.weight"): - yield (self.format_tensor_name(T.TOKEN_EMBD), data_torch) - return - if name in ("model.norm.weight", "norm.weight"): - yield (self.format_tensor_name(T.OUTPUT_NORM), data_torch) - return - if name == "lm_head.weight": - return # tied to token_embd - - raise ValueError(f"graniteswitch: unhandled tensor {name!r} (bid={bid})") - - @ModelBase.register("GraniteMoeHybridForCausalLM", "BambaForCausalLM") class GraniteHybridModel(Mamba2Model, GraniteMoeModel): """GraniteHybrid is a hybrid SSM + Attention model that uses Mamba2 SSM diff --git a/conversion/llama.py b/conversion/llama.py index 1aced49c5..9b3373f91 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -119,7 +119,7 @@ class LlamaModel(TextModel): path_tekken_json = self.dir_model / "tekken.json" path_tokenizer_json = self.dir_model / "tokenizer.json" if path_tekken_json.is_file() and not path_tokenizer_json.is_file(): - return self._set_vocab_mistral() + self._set_vocab_mistral() tokenizer_config_file = self.dir_model / 'tokenizer_config.json' if tokenizer_config_file.is_file(): diff --git a/conversion/minicpm.py b/conversion/minicpm.py index bf3fa8142..e31b26a00 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -137,15 +137,6 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel): class MiniCPMV4_6VisionModel(MmprojModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x") - if self.downsample_mode not in {"4x", "16x"}: - raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}") - if self.downsample_mode == "4x": - self.model_tensors = { - name: tensor for name, tensor in self.model_tensors.items() - if ".vit_merger." not in name - } - if self.hparams_vision is not None: # In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP # positional embedding bucket grid (70 x 70), while the per-slice processing @@ -165,8 +156,8 @@ class MiniCPMV4_6VisionModel(MmprojModel): # (mapped to PROJECTOR_TYPE_MINICPMV4_6). self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6) - self.gguf_writer.add_vision_projector_scale_factor( - 2 if self.downsample_mode == "4x" else 4) + # ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment + self.gguf_writer.add_vision_projector_scale_factor(4) # borrow wa_layer_indexes for vit_merger insertion point insert_layer_id = int(self.global_config.get( diff --git a/conversion/minimax.py b/conversion/minimax.py index d7a00bac9..c2175cc93 100644 --- a/conversion/minimax.py +++ b/conversion/minimax.py @@ -1,121 +1,13 @@ from __future__ import annotations -from typing import Iterable, Sequence, TYPE_CHECKING +from typing import TYPE_CHECKING import torch if TYPE_CHECKING: from torch import Tensor -from .base import ModelBase, TextModel, MmprojModel, gguf, logger - - -@ModelBase.register("MiniMaxText01ForCausalLM") -@ModelBase.register("MiniMaxM1ForCausalLM") -class MiniMaxText01Model(TextModel): - model_arch = gguf.MODEL_ARCH.MINIMAX01 - - def _get_suppress_tokens(self) -> Sequence[int] | None: - import json - from transformers import AutoTokenizer - from .base import LazyTorchTensor - - # check added tokens embeddings in embeddings tensor for zero-valued embeddings - # they get in the way of the token sampling process and must be suppressed - - tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True) - tokenizer_vocab_size = tokenizer.vocab_size - - with open(self.dir_model / "model.safetensors.index.json", "r", encoding="utf-8") as f: - weight_map = json.load(f)["weight_map"] - - embeddings_tensor_name = "model.embed_tokens.weight" - embeddings_shard_name = weight_map[embeddings_tensor_name] - with gguf.utility.SafetensorsLocal(self.dir_model / embeddings_shard_name) as model_shard: - embeddings_data = model_shard[embeddings_tensor_name] - - embeddings_weights_dtype = LazyTorchTensor._dtype_str_map[embeddings_data.dtype] - embeddings_weights = torch.from_numpy(embeddings_data.mmap_bytes()).view(embeddings_weights_dtype).reshape(embeddings_data.shape) - embeddings_vocab_size = embeddings_weights.shape[0] - - embeddings_added_tokens = embeddings_weights[tokenizer_vocab_size:embeddings_vocab_size] - embeddings_zero_rows = torch.all(embeddings_added_tokens == 0, dim=1) - tokens_zero_embeddings_ids = (torch.nonzero(embeddings_zero_rows, as_tuple=False).flatten() + tokenizer_vocab_size).tolist() - - return tokens_zero_embeddings_ids - - def set_vocab(self) -> None: - from pathlib import Path - - self._set_vocab_gpt2() - - for tmpl_file in [ - self.dir_model / "chat_template.jinja", - Path(__file__).parent.parent / "models" / "templates" / "MiniMax-M1.jinja" - ]: - if tmpl_file.is_file(): - self.gguf_writer.add_chat_template(tmpl_file.read_text(encoding="utf-8")) - logger.info(f"Chat template overridden with {tmpl_file}.") - break - - def set_gguf_parameters(self): - super().set_gguf_parameters() - - suppress_tokens = self._get_suppress_tokens() - if suppress_tokens: - logger.info(f"Suppressing tokens with zero embeddings {suppress_tokens}") - self.gguf_writer.add_suppress_tokens(suppress_tokens) - - layernorm_full_attention_alpha = self.hparams["layernorm_full_attention_alpha"] - layernorm_full_attention_beta = self.hparams["layernorm_full_attention_beta"] - layernorm_linear_attention_alpha = self.hparams["layernorm_linear_attention_alpha"] - layernorm_linear_attention_beta = self.hparams["layernorm_linear_attention_beta"] - layernorm_mlp_alpha = self.hparams["layernorm_mlp_alpha"] - layernorm_mlp_beta = self.hparams["layernorm_mlp_beta"] - assert layernorm_full_attention_alpha == layernorm_linear_attention_alpha == layernorm_mlp_alpha - assert layernorm_full_attention_beta == layernorm_linear_attention_beta == layernorm_mlp_beta == 1.0 - # we do not store the layernorm betas as they are all 1.0 - # layernorm alphas are stored as single residual_scale hparam - self.gguf_writer.add_residual_scale(layernorm_full_attention_alpha) - - self.gguf_writer.add_rope_dimension_count(self.hparams["rotary_dim"]) - - _experts: list[dict[str, Tensor]] | None = None - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # process the experts separately - if name.find("block_sparse_moe.experts") != -1: - n_experts = self.hparams["num_local_experts"] - - assert bid is not None - - if self._experts is None: - self._experts = [{} for _ in range(self.block_count)] - - self._experts[bid][name] = data_torch - - if len(self._experts[bid]) >= n_experts * 3: - # merge the experts into a single 3d tensor - for wid in ["w1", "w2", "w3"]: - datas: list[Tensor] = [] - - for xid in range(n_experts): - ename = f"model.layers.{bid}.block_sparse_moe.experts.{xid}.{wid}.weight" - datas.append(self._experts[bid][ename]) - del self._experts[bid][ename] - - data_torch = torch.stack(datas, dim=0) - - merged_name = f"layers.{bid}.feed_forward.experts.{wid}.weight" - - new_name = self.map_tensor_name(merged_name) - - yield from super().modify_tensors(data_torch, new_name, bid) - return - else: - return - - yield from super().modify_tensors(data_torch, name, bid) +from .base import ModelBase, TextModel, MmprojModel, gguf @ModelBase.register("MiniMaxM2ForCausalLM") diff --git a/conversion/muse_glimmer.py b/conversion/muse_glimmer.py deleted file mode 100644 index cc588e832..000000000 --- a/conversion/muse_glimmer.py +++ /dev/null @@ -1,179 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any, Iterable, TYPE_CHECKING - -import torch - -if TYPE_CHECKING: - from torch import Tensor - -from .base import MmprojModel, ModelBase, TextModel, gguf - - -def _unpermute_for_rope(tensor: "Tensor", n_heads: int) -> "Tensor": - """Invert transformers' `_permute_for_rope`: HF stores Q/K in rotate_half layout, - llama.cpp consumes the interleaved (NORM) layout.""" - if tensor.ndim == 2: - dim1, dim2 = tensor.shape - return tensor.view(n_heads, 2, dim1 // n_heads // 2, dim2).transpose(1, 2).reshape(dim1, dim2) - if tensor.ndim == 1: - (dim1,) = tensor.shape - return tensor.view(n_heads, 2, dim1 // n_heads // 2).transpose(1, 2).reshape(dim1) - raise ValueError(f"_unpermute_for_rope: unexpected shape {tuple(tensor.shape)}") - - -@ModelBase.register("MuseGlimmerForConditionalGeneration") -class MuseGlimmerModel(TextModel): - model_arch = gguf.MODEL_ARCH.MUSE_GLIMMER - - def norm_shift(self, name: str) -> float: - # All four layer norms use 1, the final norm uses 0. - return 1.0 if name.endswith("layernorm.weight") else 0.0 - - def set_vocab(self): - self._set_vocab_gpt2() - - from transformers import AutoTokenizer - tok = AutoTokenizer.from_pretrained(self.dir_model) - eot_id = tok.convert_tokens_to_ids("<|eot|>") - if isinstance(eot_id, int) and eot_id >= 0: - self.gguf_writer.add_eot_token_id(eot_id) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - hparams = self.hparams - - self.gguf_writer.add_final_logit_softcapping(hparams["final_logit_softcapping"]) - self.gguf_writer.add_logit_scale(hparams["output_multiplier"]) - self.gguf_writer.add_sliding_window(hparams["sliding_window"]) - self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]]) - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - shift = self.norm_shift(name) - if shift != 0.0: - data_torch = data_torch + shift - - # Invert transformers' `_permute_for_rope` on Q/K, we keep ggml's NORM (interleaved) rope - if ".self_attn.q_proj." in name: - data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_attention_heads"])) - elif ".self_attn.k_proj." in name: - data_torch = _unpermute_for_rope(data_torch, int(self.hparams["num_key_value_heads"])) - - # Synthesize QK-norm weights to absorb qk_scale_factor. - # MuseGlimmer implementation: scaleless RMSNorm followed by qk_scale_factor.. - if bid is not None and name.endswith(f"model.layers.{bid}.self_attn.q_proj.weight"): - head_dim = self.hparams["head_dim"] - q_scale = float(self.hparams["qk_scale_factor"]) - yield ( - self.map_tensor_name(f"model.layers.{bid}.self_attn.q_norm.weight"), - torch.full((head_dim,), q_scale, dtype=torch.float32), - ) - yield ( - self.map_tensor_name(f"model.layers.{bid}.self_attn.k_norm.weight"), - torch.ones((head_dim,), dtype=torch.float32), - ) - - yield from super().modify_tensors(data_torch, name, bid) - - -@ModelBase.register("MuseGlimmerForConditionalGeneration") -class MuseGlimmerVisionModel(MmprojModel): - def get_vision_config(self) -> dict[str, Any] | None: - c = self.global_config.get("vision_config") - if not c: - return None - # MuseGlimmer actually uses dynamic size, initialize with nominal size - image_size = c["pos_emb_height"] * c["patch_size"] * c["merge_size"] - return {**c, "image_size": image_size} - - def set_gguf_parameters(self): - super().set_gguf_parameters() - assert self.hparams_vision is not None - c = self.hparams_vision # enriched vision_config from get_vision_config() - - self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MUSE_GLIMMER) - self.gguf_writer.add_vision_attention_layernorm_eps(float(c["layer_norm_eps"])) - self.gguf_writer.add_vision_spatial_merge_size(int(c["merge_size"])) - - @classmethod - def filter_tensors(cls, item): - name, gen = item - keep = ("model.vision_tower.", "model.vision_adapter.", "model.vision_projection.") - if not any(name.startswith(k) for k in keep): - return None - return super().filter_tensors((name, gen)) - - # 3-layer projector MLP - _MM_MLP_MAP = { - "model.vision_adapter.fc1": (gguf.MODEL_TENSOR.V_MMPROJ, 0), - "model.vision_adapter.fc2": (gguf.MODEL_TENSOR.V_MMPROJ, 1), - "model.vision_projection": (gguf.MODEL_TENSOR.V_MMPROJ, 2), - } - - def modify_tensors(self, data_torch, name, bid): - assert self.hparams_vision is not None - if ".attn.q_proj." in name or ".attn.k_proj." in name: - n_heads = int(self.hparams_vision["num_attention_heads"]) - data_torch = _unpermute_for_rope(data_torch, n_heads) - # Lay out the pt=2 temporal slabs of the patch embedding as a conv2d for build_inp() - if name.endswith("patch_embedder.patch_embedding.weight"): - n_embd = data_torch.shape[0] - pt = int(self.hparams_vision["patch_temporal"]) - ps = int(self.hparams_vision["patch_size"]) - data_torch = data_torch.view(n_embd, pt, 3, ps, ps).sum(dim=1) # (n_embd, 3, ps, ps) - stem, _, suffix = name.rpartition(".") - if stem in self._MM_MLP_MAP: - tensor_key, idx = self._MM_MLP_MAP[stem] - yield (self.format_tensor_name(tensor_key, bid=idx, suffix="." + suffix), data_torch) - return - yield (self.map_tensor_name(name), data_torch) - - -@ModelBase.register("MuseGlimmerAssistantModel") -class MuseGlimmerAssistantModel(TextModel): - model_arch = gguf.MODEL_ARCH.DFLASH - - def set_vocab(self): - if self.target_model_dir is None: - raise ValueError( - "MuseGlimmerAssistant (DFlash drafter) requires --target-model-dir pointing to the " - "target MuseGlimmer HF directory" - ) - - original_dir = self.dir_model - self.dir_model = self.target_model_dir - - from . import get_model_class - with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f: - target_arch = json.load(f)["architectures"][0] - target_cls = get_model_class(target_arch) - if target_cls is not type(self): - target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] - else: - super().set_vocab() - - self.dir_model = original_dir - - mask_token_id = self.hparams.get("mask_token_id") - if mask_token_id is not None: - self.gguf_writer.add_mask_token_id(int(mask_token_id)) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - h = self.hparams - - self.gguf_writer.add_block_size(int(h["block_size"])) - - # dflash.target_layers[k] refers to the inputs going into the ith layer, which come from the (i-1)th layer's output. - # The transformers configuration refers to the outputs being recorded. - self.gguf_writer.add_target_layers([int(x) + 1 for x in h["target_layer_ids"]]) - - if h.get("sliding_window") and h.get("layer_types"): - self.gguf_writer.add_sliding_window(int(h["sliding_window"])) - self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in h["layer_types"]]) - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # DFlash defaults to NEOX (rotate_half) rope, matching transformers HF layout for Q/K, QK-norms - # no permutation needed. - yield (self.map_tensor_name(name), data_torch) diff --git a/conversion/nemotron.py b/conversion/nemotron.py index c46cec143..0572b42ca 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -197,7 +197,6 @@ class NemotronHModel(GraniteHybridModel): """Hybrid mamba2/attention model from NVIDIA""" model_arch = gguf.MODEL_ARCH.NEMOTRON_H is_moe: bool = False - supports_mtp_export = True def __init__(self, *args, **kwargs): # We have to determine the correct model architecture (MoE vs non-MoE) before @@ -237,25 +236,6 @@ class NemotronHModel(GraniteHybridModel): self._ssm_layers = [i for i, val in enumerate(pattern) if val == "mamba"] self._mlp_layers = [i for i, val in enumerate(pattern) if val == "moe"] - # `--no-mtp` drops it entirely; `--mtp` exports only the MTP head - self._mtp_bid: int | None = None - if self.is_moe and not self.no_mtp: - n_nextn = self.hparams.get("num_nextn_predict_layers", 0) or 0 - if n_nextn > 0: - assert n_nextn == 1, ( - "NemotronH MTP conversion currently supports num_nextn_predict_layers == 1" - ) - self._mtp_bid = self.block_count - self.block_count += 1 - # The folded MTP block carries both an attention sub-layer and a - # MoE sub-layer, so register it as both so the per-layer metadata arrays cover it - self._attn_layers.append(self._mtp_bid) - self._mlp_layers.append(self._mtp_bid) - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - if self.mtp_only and self._mtp_bid is None: - raise ValueError("--mtp was requested, but this model does not contain a supported MTP head") - def get_attn_layers(self): pattern = self.hparams.get("hybrid_override_pattern") or self.hparams.get("layers_block_type") if pattern is None: @@ -266,44 +246,6 @@ class NemotronHModel(GraniteHybridModel): return [i for i, val in enumerate(pattern) if val == "attention"] - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - if name.startswith("mtp."): - # --no-mtp: drop the MTP head entirely - if cls.no_mtp: - return None - elif cls.mtp_only: - # --mtp: export the MTP head plus the tensors it shares with the target model - # Include lm_head scale sidecars so NVFP4 packing sees them. - keep = name in ( - "backbone.embeddings.weight", - "backbone.norm_f.weight", - "lm_head.weight", - "lm_head.weight_scale", - "lm_head.weight_scale_2", - "lm_head.weight_scale_inv", - "lm_head.input_scale", - "lm_head.input_global_scale", - "lm_head.weight_global_scale", - "lm_head.weight_packed", - ) - if not keep: - return None - return super().filter_tensors((name, gen)) - - def prepare_metadata(self, vocab_only: bool): - from_dir = self.fname_out.is_dir() - super().prepare_metadata(vocab_only=vocab_only) - - if not self.mtp_only or not from_dir: - return - output_type: str = self.ftype.name.partition("_")[2] - fname_default: str = gguf.naming_convention( - self.metadata.name, self.metadata.basename, self.metadata.finetune, - self.metadata.version, size_label=None, output_type=output_type, model_type=None) - self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" - def set_gguf_parameters(self): super().set_gguf_parameters() @@ -342,10 +284,6 @@ class NemotronHModel(GraniteHybridModel): if (latent_size := self.hparams.get("moe_latent_size")) is not None: self.gguf_writer.add_moe_latent_size(latent_size) - # MTP head: number of trailing NextN blocks - if self._mtp_bid is not None: - self.gguf_writer.add_nextn_predict_layers(self.hparams["num_nextn_predict_layers"]) - def set_vocab(self): # The NemotronH config uses pattern characters (e.g. '-') that may not # be supported by the installed transformers version. AutoTokenizer @@ -412,24 +350,15 @@ class NemotronHModel(GraniteHybridModel): if not self.is_moe: self.gguf_writer.add_add_bos_token(True) - _MTP_SPECIAL_RENAMES = { - "mtp.layers.0.enorm.weight": "model.layers.{bid}.enorm.weight", - "mtp.layers.0.hnorm.weight": "model.layers.{bid}.hnorm.weight", - "mtp.layers.0.eh_proj.weight": "model.layers.{bid}.eh_proj.weight", - "mtp.layers.1.norm.weight": "model.layers.{bid}.post_attention_layernorm.weight", - "mtp.layers.1.final_layernorm.weight": "model.layers.{bid}.shared_head.norm.weight", - } - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # mtp.layers.0: NextN input fusion + attention - # mtp.layers.1: MoE + final head norm - if self._mtp_bid is not None and name.startswith(("mtp.layers.0.", "mtp.layers.1.")): - suffix = name.split(".", 3)[3] - bid = self._mtp_bid - renamed = self._MTP_SPECIAL_RENAMES.get(name) - name = renamed.format(bid=bid) if renamed else f"backbone.layers.{bid}.{suffix}" - if self.is_moe and bid is not None: + # Skip Multi-Token Prediction (MTP) tensors. These are used for + # for speculative decoding but we don't include them in this model + # conversion. See https://github.com/ggml-org/llama.cpp/pull/18886 + if name.startswith("mtp."): + logger.info(f"gguf: Skipping MTP (Speculative) layer: {name}") + return + if name.endswith("mixer.gate.e_score_correction.bias"): yield from ModelBase.modify_tensors(self, data_torch, name, bid) return diff --git a/conversion/pockettts.py b/conversion/pockettts.py deleted file mode 100644 index 62ecb5acd..000000000 --- a/conversion/pockettts.py +++ /dev/null @@ -1,378 +0,0 @@ -from __future__ import annotations - -import re -from pathlib import Path -from typing import Any, Iterable, TYPE_CHECKING - -import torch - -if TYPE_CHECKING: - from torch import Tensor - -from .base import ModelBase, MmprojModel, SentencePieceTokenTypes, TextModel, gguf, logger - -# Pocket TTS is a CALM: the backbone conditions a flow-matching decoder that generates one -# continuous 32-d latent per frame. There is no codebook in this model. -# The checkpoint ships no config.json, hparams come from _load_hparams() below. -# -# Tricks being used to support this model via existing llama.cpp code paths: -# - bos_before_voice and bos_emb are learned input vectors, not tokens -# they are appended to the embedding table as extra tokens, to be looked up like any other row -# - bos_emb lives in latent space, so input_linear is folded into it here -# - the backbone has no lm_head, the embedding table is reused as output for the unused logits -# -# pipeline stage mapping: -# mimi encoder + speaker_proj --> mapped to normal mtmd audio encoder -# flow_lm.transformer --> mapped to normal libllama text model (autoregressive) -# flow_lm.flow_net + out_eos --> MTMD_GEN_PROCESS_TYPE_GEN_CODE -# mimi decoder --> MTMD_GEN_PROCESS_TYPE_GEN_WAV - -# indices into mimi.encoder.model / mimi.decoder.model for stage i, see SEANetEncoder/SEANetDecoder -_ENC_RES_IDX = lambda i: 1 + 3 * i # noqa: E731 -_ENC_SCALE_IDX = lambda i: 3 + 3 * i # noqa: E731 -_DEC_SCALE_IDX = lambda i: 2 + 3 * i # noqa: E731 -_DEC_RES_IDX = lambda i: 3 + 3 * i # noqa: E731 - -_N_SEANET_STAGES = 3 -_SAMPLE_RATE = 24000 - - -def _tensor_shapes(dir_model: Path) -> dict[str, tuple[int, ...]]: - part_names = ModelBase.get_model_part_names(dir_model, "model", ".safetensors") - if len(part_names) != 1: - return {} - with gguf.utility.SafetensorsLocal(dir_model / part_names[0]) as part: - return {name: tuple(part[name].shape) for name in part.keys()} - - -@ModelBase.register_hparams_loader(lambda dir_model: "flow_lm.bos_emb" in _tensor_shapes(dir_model)) -def _load_hparams(dir_model: Path) -> dict[str, Any]: - logger.info("gguf: detected pocket-tts checkpoint, deriving hparams from tensor shapes") - shapes = _tensor_shapes(dir_model) - n_vocab, n_embd = shapes["flow_lm.conditioner.embed.weight"] - n_layer = sum(1 for name in shapes if re.fullmatch(r"flow_lm\.transformer\.layers\.\d+\.norm1\.weight", name)) - n_layer_a = sum(1 for name in shapes if re.fullmatch(r"mimi\.encoder_transformer\.transformer\.layers\.\d+\.norm1\.weight", name)) - n_embd_a = shapes["mimi.encoder_transformer.transformer.layers.0.norm1.weight"][0] - return { - "architectures": ["PocketTTSModel"], - "model_type": "pockettts", - "num_hidden_layers": n_layer, - "hidden_size": n_embd, - "intermediate_size": shapes["flow_lm.transformer.layers.0.linear1.weight"][0], - # the transformer is fully causal with no context limit, this only bounds the KV cache - "max_position_embeddings": 4096, - # not in the checkpoint, but every released variant uses head_dim 64 - "num_attention_heads": n_embd // 64, - # extra rows for the learned input vectors, see _embd_table() - "vocab_size": n_vocab + (2 if "flow_lm.bos_before_voice" in shapes else 1), - "rope_theta": 10000.0, - "layer_norm_eps": 1e-5, - "audio_config": { - "num_hidden_layers": n_layer_a, - "hidden_size": n_embd_a, - "intermediate_size": shapes["mimi.encoder_transformer.transformer.layers.0.linear1.weight"][0], - "num_attention_heads": n_embd_a // 64, - }, - } - - -@ModelBase.register("PocketTTSModel") -class PocketTTSModel(TextModel): - model_arch = gguf.MODEL_ARCH.POCKETTTS - - _LAYER_TENSOR_MAP = { - "norm1": gguf.MODEL_TENSOR.ATTN_NORM, - "norm2": gguf.MODEL_TENSOR.FFN_NORM, - "self_attn.out_proj": gguf.MODEL_TENSOR.ATTN_OUT, - "linear1": gguf.MODEL_TENSOR.FFN_UP, - "linear2": gguf.MODEL_TENSOR.FFN_DOWN, - } - - def set_vocab(self): - # this is a unigram sentencepiece model, llama.cpp's SPM tokenizer cannot do - # unigram segmentation, so use the UGM tokenizer instead - from sentencepiece import sentencepiece_model_pb2 as model - - proto = model.ModelProto() # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - proto.ParseFromString(open(self.dir_model / "tokenizer.model", "rb").read()) - assert proto.trainer_spec.model_type == 1, "expected a unigram tokenizer" - - tokens, scores, toktypes = self._create_vocab_sentencepiece() - - # the last rows of the embedding table are not sentencepiece pieces - extra = self._extra_tokens() - for i, name in enumerate(extra): - tokens[len(tokens) - len(extra) + i] = name.encode("utf-8") - toktypes[len(tokens) - len(extra) + i] = SentencePieceTokenTypes.CONTROL - scores[len(tokens) - len(extra) + i] = -1000.0 - - self.gguf_writer.add_tokenizer_model("t5") - self.gguf_writer.add_tokenizer_pre("default") - self.gguf_writer.add_token_list(tokens) - self.gguf_writer.add_token_scores(scores) - self.gguf_writer.add_token_types(toktypes) - self.gguf_writer.add_add_space_prefix(proto.normalizer_spec.add_dummy_prefix) - self.gguf_writer.add_remove_extra_whitespaces(proto.normalizer_spec.remove_extra_whitespaces) - if proto.normalizer_spec.precompiled_charsmap: - self.gguf_writer.add_precompiled_charsmap(proto.normalizer_spec.precompiled_charsmap) - self.gguf_writer.add_add_bos_token(False) - self.gguf_writer.add_add_eos_token(False) - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - if not name.startswith("flow_lm."): - return # mimi and the flow net go to the mmproj - - if name == "flow_lm.conditioner.embed.weight": - yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), self._embd_table(data_torch)) - return - - if name.startswith("flow_lm.out_norm."): - suffix = "." + name.rsplit(".", 1)[1] - yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT_NORM, suffix=suffix), data_torch) - return - - if name.startswith("flow_lm.transformer.layers."): - assert bid is not None - key_with_suffix = name.split(f"layers.{bid}.", 1)[1] - key, suffix = key_with_suffix.rsplit(".", 1) - - if key == "self_attn.in_proj": - q, k, v = data_torch.chunk(3, dim=0) - yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_Q, bid), q) - yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_K, bid), k) - yield (self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_V, bid), v) - return - - tensor = self._LAYER_TENSOR_MAP.get(key) - if tensor is not None: - yield (self.format_tensor_name(tensor, bid, suffix="." + suffix), data_torch) - return - - return - - def _extra_tokens(self) -> list[str]: - # the conditioner's padding row, then the learned vectors appended by _embd_table(). - # bos_before_voice only exists when the pack sets insert_bos_before_voice - names = ["<|pad|>"] - if "flow_lm.bos_before_voice" in self.model_tensors: - names.append("<|bos_before_voice|>") - names.append("<|audio_bos|>") - return names - - def _embd_table(self, embed: Tensor) -> Tensor: - rows = [embed] - if "flow_lm.bos_before_voice" in self.model_tensors: - rows.append(self.model_tensors["flow_lm.bos_before_voice"]().reshape(1, -1).to(embed.dtype)) - - # bos_emb is a latent, it only enters the backbone through input_linear - bos_emb = self.model_tensors["flow_lm.bos_emb"]() - input_linear = self.model_tensors["flow_lm.input_linear.weight"]() - audio_bos = torch.nn.functional.linear(bos_emb.float(), input_linear.float()).reshape(1, -1) - rows.append(audio_bos.to(embed.dtype)) - - return torch.cat(rows, dim=0) - - -@ModelBase.register("PocketTTSModel") -class PocketTTSMmprojModel(MmprojModel): - has_audio_encoder = True - has_vision_encoder = False - - _MIMI_TFM_MAP = { - "norm1": (gguf.MODEL_TENSOR.A_ENC_INPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM), - "norm2": (gguf.MODEL_TENSOR.A_ENC_OUTPUT_NORM, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM), - "self_attn.out_proj": (gguf.MODEL_TENSOR.A_ENC_OUTPUT, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT), - "linear1": (gguf.MODEL_TENSOR.A_ENC_FFN_UP, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP), - "linear2": (gguf.MODEL_TENSOR.A_ENC_FFN_DOWN, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN), - "layer_scale_1.scale": (gguf.MODEL_TENSOR.A_ENC_ATTN_SCALE, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_SCALE), - "layer_scale_2.scale": (gguf.MODEL_TENSOR.A_ENC_FFN_SCALE_LS, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE), - } - _MIMI_TFM_QKV = ( - (gguf.MODEL_TENSOR.A_ENC_ATTN_Q, gguf.MODEL_TENSOR.A_ENC_ATTN_K, gguf.MODEL_TENSOR.A_ENC_ATTN_V), - (gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K, gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V), - ) - - def set_gguf_parameters(self): - self.gguf_writer.add_file_type(self.ftype) - assert self.hparams_audio is not None - - # voice-prompt encoder: mimi encoder + speaker_proj - self.gguf_writer.add_clip_has_audio_encoder(True) - # note: the 24kHz sample rate is hardcoded on the clip.cpp side, like the other audio models - self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_SPKENC) - self.gguf_writer.add_audio_projection_dim(self.n_embd_text) - self.gguf_writer.add_audio_block_count(self.hparams_audio["num_hidden_layers"]) - self.gguf_writer.add_audio_embedding_length(self.hparams_audio["hidden_size"]) - self.gguf_writer.add_audio_feed_forward_length(self.hparams_audio["intermediate_size"]) - self.gguf_writer.add_audio_head_count(self.hparams_audio["num_attention_heads"]) - self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) - # mimi convolves the waveform directly, it is passed around as a 1-row "mel" - self.gguf_writer.add_audio_num_mel_bins(1) - - # generation: flow-matching decoder + mimi decoder - # the SEANet and flow net hparams are constant across the family, clip.cpp holds them - self.gguf_writer.add_clip_has_gen_audio_encoder(True) - self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.POCKETTTS_GEN) - self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) - self.gguf_writer.add_gen_audio_embedding_length(self.hparams_audio["hidden_size"]) - self.gguf_writer.add_gen_audio_feed_forward_length(self.hparams_audio["intermediate_size"]) - self.gguf_writer.add_gen_audio_block_count(self.hparams_audio["num_hidden_layers"]) - self.gguf_writer.add_gen_audio_head_count(self.hparams_audio["num_attention_heads"]) - self.gguf_writer.add_gen_audio_attention_layernorm_eps(1e-5) - - self.gguf_writer.add_gen_audio_model_variant(self.dir_model.name) - - def tensor_force_quant(self, name, new_name, bid, n_dims): - del name, bid, n_dims - # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path - if ".seanet." in new_name or new_name in ("a.downsample.conv.weight", "a.gen.wav.upsample.weight"): - return gguf.GGMLQuantizationType.F16 - return False - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - del bid # the block index of the mimi transformers is parsed here, not by the base class - T = gguf.MODEL_TENSOR - - if name in ("flow_lm.bos_emb", "flow_lm.bos_before_voice", "flow_lm.conditioner.embed.weight"): - return # folded into the backbone embedding table - if name.startswith("flow_lm.transformer.") or name.startswith("flow_lm.out_norm."): - return # backbone - - if name == "flow_lm.speaker_proj_weight": - yield (self.format_tensor_name(T.A_ENC_SPEAKER_PROJ), data_torch) - return - if name == "flow_lm.input_linear.weight": - yield (self.format_tensor_name(T.A_GEN_INPUT_LINEAR), data_torch) - return - if name == "flow_lm.emb_mean": - yield (self.format_tensor_name(T.A_GEN_EMB_MEAN, suffix=""), data_torch) - return - if name == "flow_lm.emb_std": - yield (self.format_tensor_name(T.A_GEN_EMB_STD, suffix=""), data_torch) - return - if name.startswith("flow_lm.out_eos."): - suffix = "." + name.rsplit(".", 1)[1] - yield (self.format_tensor_name(T.A_GEN_OUT_EOS, suffix=suffix), data_torch) - return - - if name.startswith("flow_lm.flow_net."): - yield from self._flow_net_tensor(name, data_torch) - return - - if name == "mimi.downsample.conv.conv.weight": - yield (self.format_tensor_name(T.A_ENC_DOWNSAMPLE_CONV), data_torch) - return - if name == "mimi.upsample.convtr.convtr.weight": - yield (self.format_tensor_name(T.A_GEN_WAV_UPSAMPLE), data_torch) - return - if name == "mimi.quantizer.output_proj.weight": - yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_OUT), data_torch.squeeze(-1)) - return - - if "_transformer.transformer.layers." in name: - yield from self._mimi_tfm_tensor(name, data_torch) - return - - if name.startswith("mimi.encoder.model.") or name.startswith("mimi.decoder.model."): - yield from self._seanet_tensor(name, data_torch) - return - - return - - def _flow_net_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]: - T = gguf.MODEL_TENSOR - key = name.split("flow_lm.flow_net.", 1)[1] - suffix = "." + key.rsplit(".", 1)[1] - - simple = { - "input_proj": T.A_GEN_FLOW_INPUT_PROJ, - "cond_embed": T.A_GEN_FLOW_COND_EMBD, - "final_layer.linear": T.A_GEN_FLOW_FINAL_PROJ, - "final_layer.adaLN_modulation.1": T.A_GEN_FLOW_FINAL_ADA, - } - tensor = simple.get(key.rsplit(".", 1)[0]) - if tensor is not None: - yield (self.format_tensor_name(tensor, suffix=suffix), data_torch) - return - - if key.startswith("time_embed."): - bid = int(key.split(".")[1]) - rest = key.split(f"time_embed.{bid}.", 1)[1] - time_map = { - "freqs": (T.A_GEN_FLOW_TIME_FREQS, ""), - "mlp.0": (T.A_GEN_FLOW_TIME_UP, suffix), - "mlp.2": (T.A_GEN_FLOW_TIME_DOWN, suffix), - "mlp.3.alpha": (T.A_GEN_FLOW_TIME_NORM, ""), - } - entry = time_map.get(rest) or time_map.get(rest.rsplit(".", 1)[0]) - if entry is not None: - yield (self.format_tensor_name(entry[0], bid, suffix=entry[1]), data_torch) - return - - if key.startswith("res_blocks."): - bid = int(key.split(".")[1]) - rest = key.split(f"res_blocks.{bid}.", 1)[1].rsplit(".", 1)[0] - blk_map = { - "in_ln": T.A_GEN_FLOW_BLK_NORM, - "mlp.0": T.A_GEN_FLOW_BLK_UP, - "mlp.2": T.A_GEN_FLOW_BLK_DOWN, - "adaLN_modulation.1": T.A_GEN_FLOW_BLK_ADA, - } - tensor = blk_map.get(rest) - if tensor is not None: - yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch) - return - - def _mimi_tfm_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]: - is_decoder = name.startswith("mimi.decoder_transformer.") - bid = int(name.split("_transformer.transformer.layers.", 1)[1].split(".")[0]) - key_with_suffix = name.split(f".layers.{bid}.", 1)[1] - - if key_with_suffix == "self_attn.in_proj.weight": - q, k, v = data_torch.chunk(3, dim=0) - names = self._MIMI_TFM_QKV[1 if is_decoder else 0] - for tensor, part in zip(names, (q, k, v)): - yield (self.format_tensor_name(tensor, bid), part) - return - - key, suffix = key_with_suffix.rsplit(".", 1) - entry = self._MIMI_TFM_MAP.get(key) or self._MIMI_TFM_MAP.get(key_with_suffix) - if entry is None: - return - tensor = entry[1 if is_decoder else 0] - suffix = ".weight" if key_with_suffix.endswith(".scale") else "." + suffix - yield (self.format_tensor_name(tensor, bid, suffix=suffix), data_torch) - - def _seanet_tensor(self, name: str, data_torch: Tensor) -> Iterable[tuple[str, Tensor]]: - T = gguf.MODEL_TENSOR - is_decoder = name.startswith("mimi.decoder.") - idx = int(name.split(".model.", 1)[1].split(".")[0]) - suffix = "." + name.rsplit(".", 1)[1] - - conv_in, conv_out, res1, res2, scale = ( - (T.A_GEN_WAV_SEANET_CONV_IN, T.A_GEN_WAV_SEANET_CONV_OUT, T.A_GEN_WAV_SEANET_RES_CONV1, - T.A_GEN_WAV_SEANET_RES_CONV2, T.A_GEN_WAV_SEANET_SCALE_CONV) - if is_decoder else - (T.A_ENC_SEANET_CONV_IN, T.A_ENC_SEANET_CONV_OUT, T.A_ENC_SEANET_RES_CONV1, - T.A_ENC_SEANET_RES_CONV2, T.A_ENC_SEANET_SCALE_CONV) - ) - - if idx == 0: - yield (self.format_tensor_name(conv_in, suffix=suffix), data_torch) - return - if idx == 3 * _N_SEANET_STAGES + 2: - yield (self.format_tensor_name(conv_out, suffix=suffix), data_torch) - return - - for stage in range(_N_SEANET_STAGES): - res_idx = _DEC_RES_IDX(stage) if is_decoder else _ENC_RES_IDX(stage) - scale_idx = _DEC_SCALE_IDX(stage) if is_decoder else _ENC_SCALE_IDX(stage) - if idx == scale_idx: - yield (self.format_tensor_name(scale, stage, suffix=suffix), data_torch) - return - if idx == res_idx: - # block.1 is the dilated conv, block.3 the pointwise one (0 and 2 are ELU) - inner = int(name.split(".block.", 1)[1].split(".")[0]) - tensor = res1 if inner == 1 else res2 - yield (self.format_tensor_name(tensor, stage, suffix=suffix), data_torch) - return diff --git a/conversion/qwen.py b/conversion/qwen.py index ead435455..d1127f743 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -18,7 +18,7 @@ class QwenModel(TextModel): @staticmethod def token_bytes_to_string(b): - from transformers.convert_slow_tokenizer import bytes_to_unicode + from transformers.models.gpt2.tokenization_gpt2 import bytes_to_unicode # ty: ignore[unresolved-import] byte_encoder = bytes_to_unicode() return ''.join([byte_encoder[ord(char)] for char in b.decode('latin-1')]) @@ -268,101 +268,8 @@ class Qwen3MoeModel(Qwen2MoeModel): super().set_vocab() -class _QwenMtpMixin: - """Shared MTP wiring for Qwen3-Next and Qwen3.5/3.6 text variants. The HF - config carries the MTP block under `mtp_num_hidden_layers` (computed from - the checkpoint when absent, e.g. Qwen3-Next) and the tensors under - `mtp.*`; we extend block_count, emit the nextn metadata key, and remap - `mtp.*` to the standard layer-indexed nextn naming so the existing - tensor_map handles them.""" - - supports_mtp_export = True - hparams: dict[str, Any] - model_arch: gguf.MODEL_ARCH - gguf_writer: gguf.GGUFWriter - block_count: int - tensor_map: gguf.TensorNameMap - no_mtp: bool - mtp_only: bool - _original_block_count: int | None = None - opt_num_mtp_layers: int = 0 - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.block_count = self.hparams["num_hidden_layers"] - if not self.no_mtp: - n_mtp = self.hparams.get("mtp_num_hidden_layers", 0) - # Qwen-3-Next doesn't include `mtp_num_hidden_layers` in config. - if n_mtp == 0: - assert self.opt_num_mtp_layers != 0 - n_mtp = self.opt_num_mtp_layers - self.block_count += n_mtp - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: - hparams = {**self.hparams, **self.hparams.get("text_config", {})} - key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None) - type(self)._original_block_count = hparams.get(key) - type(self).opt_num_mtp_layers = 0 - return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute] - - @classmethod - def filter_tensors(cls, item): - assert cls._original_block_count is not None - # TODO: change TextModel to super() - if (titem := TextModel.filter_tensors(item)) is None: - return None - name, gen = titem - if name.startswith("model.mtp."): - name = name.replace("model.", "", 1) - if name.startswith("mtp."): - if cls.no_mtp: - return None - remapper = { - "fc": "eh_proj", - "pre_fc_norm_embedding": "enorm", - "pre_fc_norm_hidden": "hnorm", - "norm": "shared_head.norm", - } - parts = name.split(".", 3) - if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal(): - mtp_idx = int(parts[2]) - name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}" - cls.opt_num_mtp_layers = max(cls.opt_num_mtp_layers, mtp_idx + 1) - elif len(parts) == 3 and parts[1] in remapper: - name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}" - elif cls.mtp_only: - keep = name in ( - "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", - "embed_tokens.weight", "norm.weight", - ) - if not keep: - return None - return name, gen - - def set_gguf_parameters(self): - super().set_gguf_parameters() # ty: ignore[unresolved-attribute] - if self.no_mtp: - return - if (n := self.block_count - self.hparams["num_hidden_layers"]) > 0: - self.gguf_writer.add_nextn_predict_layers(n) - - def prepare_metadata(self, vocab_only: bool): - from_dir = self.fname_out.is_dir() - super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute] - - if not self.mtp_only or not from_dir: - return - - output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - fname_default: str = gguf.naming_convention( - self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] - self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" - - @ModelBase.register("Qwen3NextForCausalLM") -class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel): +class Qwen3NextModel(Qwen2MoeModel): model_arch = gguf.MODEL_ARCH.QWEN3NEXT def set_gguf_parameters(self): @@ -377,6 +284,16 @@ class Qwen3NextModel(_QwenMtpMixin, Qwen2MoeModel): rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.25))) + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + + if name.startswith("mtp"): + # ignore MTP layers for now + return None + + return super().filter_tensors(item) + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if name.endswith(".A_log"): data_torch = -torch.exp(data_torch) @@ -619,13 +536,97 @@ class _Qwen35MRopeMixin: self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION) +class _Qwen35MtpMixin: + """Shared MTP wiring for Qwen3.5/3.6 text variants. The HF config carries + the MTP block under `mtp_num_hidden_layers` and the tensors under + `mtp.*`; we extend block_count, emit the nextn metadata key, and remap + `mtp.*` to the standard layer-indexed nextn naming so the existing + tensor_map handles them.""" + + supports_mtp_export = True + hparams: dict[str, Any] + model_arch: gguf.MODEL_ARCH + gguf_writer: gguf.GGUFWriter + block_count: int + tensor_map: gguf.TensorNameMap + no_mtp: bool + mtp_only: bool + _original_block_count: int | None = None + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.block_count = self.hparams["num_hidden_layers"] + if not self.no_mtp: + self.block_count += self.hparams.get("mtp_num_hidden_layers", 0) + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: + hparams = {**self.hparams, **self.hparams.get("text_config", {})} + key = next((k for k in ["n_layers", "num_hidden_layers", "n_layer", "num_layers"] if k in hparams), None) + type(self)._original_block_count = hparams.get(key) + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) # ty: ignore[unresolved-attribute] + + @classmethod + def filter_tensors(cls, item): + assert cls._original_block_count is not None + # TODO: change TextModel to super() + if (titem := TextModel.filter_tensors(item)) is None: + return None + name, gen = titem + if name.startswith("model.mtp."): + name = name.replace("model.", "", 1) + if name.startswith("mtp."): + if cls.no_mtp: + return None + remapper = { + "fc": "eh_proj", + "pre_fc_norm_embedding": "enorm", + "pre_fc_norm_hidden": "hnorm", + "norm": "shared_head.norm", + } + parts = name.split(".", 3) + if len(parts) == 4 and parts[1] == "layers" and parts[2].isdecimal(): + mtp_idx = int(parts[2]) + name = f"model.layers.{cls._original_block_count + mtp_idx}.{parts[3]}" + elif len(parts) == 3 and parts[1] in remapper: + name = f"model.layers.{cls._original_block_count}.{remapper[parts[1]]}.{parts[2]}" + elif cls.mtp_only: + keep = name in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + "embed_tokens.weight", "norm.weight", + ) + if not keep: + return None + return name, gen + + def set_gguf_parameters(self): + super().set_gguf_parameters() # ty: ignore[unresolved-attribute] + if self.no_mtp: + return + if (n := self.hparams.get("mtp_num_hidden_layers", 0)) > 0: + self.gguf_writer.add_nextn_predict_layers(n) + + def prepare_metadata(self, vocab_only: bool): + from_dir = self.fname_out.is_dir() + super().prepare_metadata(vocab_only=vocab_only) # ty: ignore[unresolved-attribute] + + if not self.mtp_only or not from_dir: + return + + output_type: str = self.ftype.name.partition("_")[2] # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + fname_default: str = gguf.naming_convention( + self.metadata.name, self.metadata.basename, self.metadata.finetune, # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + self.metadata.version, size_label=None, output_type=output_type, model_type=None) # pyright: ignore[reportAttributeAccessIssue] # ty: ignore[unresolved-attribute] + self.fname_out = self.fname_out.parent / f"mtp-{fname_default}.gguf" + + @ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM") -class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): +class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35 @ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM") -class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase): +class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35MOE @@ -647,13 +648,10 @@ class DFlashModel(Qwen3Model): # own tokenizer logic, not the Qwen default). from . import get_model_class with open(self.target_model_dir / "config.json", "r", encoding="utf-8") as f: - target_hparams = json.load(f) - target_arch = target_hparams["architectures"][0] + target_arch = json.load(f)["architectures"][0] target_cls = get_model_class(target_arch) if target_cls is not type(self): - if target_arch == "NemotronHForCausalLM": - setattr(self, "is_moe", "num_experts_per_tok" in target_hparams) target_cls.set_vocab(self) # ty: ignore[unresolved-attribute] else: super().set_vocab() @@ -691,12 +689,6 @@ class DFlashModel(Qwen3Model): name = "model." + name return super().filter_tensors((name, gen)) - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - if name == "model.embed_tokens.weight" and not self.hparams.get("has_embed_tokens", True): - return - - yield from super().modify_tensors(data_torch, name, bid) - @ModelBase.register("Qwen3DSparkModel") class DSparkModel(DFlashModel): diff --git a/conversion/qwen3tts.py b/conversion/qwen3tts.py deleted file mode 100644 index d21a50595..000000000 --- a/conversion/qwen3tts.py +++ /dev/null @@ -1,471 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any, Callable, Iterable, TYPE_CHECKING - -import torch -import torch.nn.functional as F - -if TYPE_CHECKING: - from torch import Tensor - -from .base import ModelBase, MmprojModel, TextModel, gguf - -# Tricks being used to support this model via existing llama.cpp code paths: -# - Text projection MLP is folded into the embedding table -# - codec_embedding is concat to the text embedding table, vocab is extended -# example: codec_bos_id(2149) --> "<|codec_bos|>" -# codec_eos_token_id(2150) --> "<|codec_eos_token|>" -# codec_language_id.chinese(2055) --> "<|codec_language_chinese|>" -# other rows --> "<|codec_0|>", "<|codec_1|>", ..., "<|codec_1023|>" -# - output tensor codec_head is smaller than vocab, so logits will be padded at inference time -# - suppress_tokens is used to limit the backbone to only sample either semantic or EOS (stop) token - -# pipeline stage mapping: -# speaker reference encoder --> mapped to normal mtmd audio encoder -# backbone --> mapped to normal libllama text model (autoregressive) -# code_predictor --> MTMD_GEN_PROCESS_TYPE_GEN_CODE -# code2wav --> MTMD_GEN_PROCESS_TYPE_GEN_WAV - -# torch activation functions used by Qwen3TTSTalkerResizeMLP (config's hidden_act) -_ACT2FN = { - "silu": F.silu, - "gelu": F.gelu, - "relu": F.relu, -} - - -@ModelBase.register("Qwen3TTSForConditionalGeneration") -class Qwen3TTSTalkerModel(TextModel): - model_arch = gguf.MODEL_ARCH.QWEN3TTS - - _TEXT_PROJ_KEYS = ( - "model.text_embedding.weight", - "text_projection.linear_fc1.weight", - "text_projection.linear_fc1.bias", - "text_projection.linear_fc2.weight", - "text_projection.linear_fc2.bias", - ) - - _text_proj_buffer: dict[str, Tensor] - _folded_text_embed: Tensor | None - _codec_embed: Tensor | None - - def __init__(self, dir_model: Path, *args, **kwargs): - hparams = kwargs.pop("hparams", None) - if hparams is None: - hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) - raw_talker_config = dict(hparams["talker_config"]) - self._talker_config = raw_talker_config - self.n_codec_vocab = raw_talker_config["vocab_size"] - talker_config = dict(raw_talker_config) - talker_config["vocab_size"] = talker_config["text_vocab_size"] - hparams["text_config"] = talker_config - super().__init__(dir_model, *args, hparams=hparams, **kwargs) - self._text_proj_buffer = {} - self._folded_text_embed = None - self._codec_embed = None - - def _codec_token_names(self) -> list[str]: - # start every row with a generic name, then override the ones with a - # known meaning (bos/eos/language/etc, derived from the *_id fields - # of talker_config) with a more descriptive one - names = [f"<|codec_{i}|>" for i in range(self.n_codec_vocab)] - for key, val in self._talker_config.items(): - if not key.endswith("_id"): - continue - prefix = key[:-len("_id")] - if isinstance(val, int): - names[val] = f"<|{prefix}|>" - elif isinstance(val, dict): - for subkey, subval in val.items(): - names[subval] = f"<|{prefix}_{subkey}|>" - return names - - def set_vocab(self): - codec_tokens = self._codec_token_names() - codec_toktypes = [gguf.TokenType.CONTROL] * len(codec_tokens) - - try: - tokens, scores, toktypes = self._create_vocab_sentencepiece() - self.gguf_writer.add_tokenizer_model("llama") - self.gguf_writer.add_tokenizer_pre("default") - tokens += [t.encode("utf-8") for t in codec_tokens] - scores += [0.0] * len(codec_tokens) - toktypes += codec_toktypes - self.gguf_writer.add_token_list(tokens) - self.gguf_writer.add_token_scores(scores) - self.gguf_writer.add_token_types(toktypes) - special_vocab = gguf.SpecialVocab(self.dir_model, n_vocab=len(tokens)) - special_vocab.add_to_gguf(self.gguf_writer) - return - except FileNotFoundError: - pass - - tokens, toktypes, tokpre = self.get_vocab_base() - tokens += codec_tokens - toktypes += codec_toktypes - self.gguf_writer.add_tokenizer_model("gpt2") - self.gguf_writer.add_tokenizer_pre(tokpre) - self.gguf_writer.add_token_list(tokens) - self.gguf_writer.add_token_types(toktypes) - - special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) - special_vocab.add_to_gguf(self.gguf_writer) - - # make sure that the model has no chat template, so chat will be disabled - self.gguf_writer.add_chat_template(None) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - - # note: final vocab layout is [text_vocab | codec_vocab], with text_vocab is actually padded with -inf in cgraph - # for codec_vocab, only first 2048 rows can be sampled for semantic code - # plus codec_eos_token_id that used for signaling end of generation - # ref: https://github.com/QwenLM/Qwen3-TTS/blob/022e286b98fbec7e1e916cb940cdf532cd9f488e/qwen_tts/core/models/modeling_qwen3_tts.py#L2059-L2063 - - vocab_size = self.hparams["vocab_size"] + self.n_codec_vocab - codec_eos_token_id = self.hparams["vocab_size"] + self._talker_config["codec_eos_token_id"] - self.gguf_writer.add_suppress_tokens([ - i for i in range(vocab_size - 1024, vocab_size) - if i != codec_eos_token_id - ]) - self.gguf_writer.add_eos_token_id(codec_eos_token_id) - self.gguf_writer.add_add_eos_token(False) - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - - if not name.startswith("talker.") or name.startswith("talker.code_predictor."): - return None - - name = name[len("talker."):] - return super().filter_tensors((name, gen)) - - def _maybe_emit_token_embd(self) -> Iterable[tuple[str, Tensor]]: - if self._folded_text_embed is None or self._codec_embed is None: - return - combined = torch.cat([self._folded_text_embed, self._codec_embed], dim=0) - yield (self.format_tensor_name(gguf.MODEL_TENSOR.TOKEN_EMBD), combined) - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # codec_embedding rows are appended after the text vocab, extending the embedding table - if name == "model.codec_embedding.weight": - self._codec_embed = data_torch - yield from self._maybe_emit_token_embd() - return - - # codec_head is the output head for the (smaller) codec vocab; logits get padded to - # the extended vocab size at inference time - if name == "codec_head.weight": - yield (self.format_tensor_name(gguf.MODEL_TENSOR.OUTPUT), data_torch) - return - - if name in self._TEXT_PROJ_KEYS: - self._text_proj_buffer[name] = data_torch - if len(self._text_proj_buffer) < len(self._TEXT_PROJ_KEYS): - return - - # fold MLP into the embedding table at conversion time, MLP won't be used at inference time anyway - act_fn = _ACT2FN[self.hparams["hidden_act"]] - embed = self._text_proj_buffer["model.text_embedding.weight"] - hidden = act_fn(F.linear(embed, - self._text_proj_buffer["text_projection.linear_fc1.weight"], - self._text_proj_buffer["text_projection.linear_fc1.bias"])) - folded = F.linear(hidden, - self._text_proj_buffer["text_projection.linear_fc2.weight"], - self._text_proj_buffer["text_projection.linear_fc2.bias"]) - self._folded_text_embed = folded - yield from self._maybe_emit_token_embd() - return - - yield from super().modify_tensors(data_torch, name, bid) - - -@ModelBase.register("Qwen3TTSForConditionalGeneration") -class Qwen3TTSSpeakerEncoderModel(MmprojModel): - has_vision_encoder = False - has_audio_encoder = True - - # talker.code_predictor.model.layers.{bid}. -> A_GEN_CODE_* - # bypass tensor_mapping.py for now to make it simple - _CODE_LAYER_TENSOR_MAP = { - "input_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM, - "self_attn.q_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q, - "self_attn.q_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM, - "self_attn.k_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K, - "self_attn.k_norm": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM, - "self_attn.v_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V, - "self_attn.o_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT, - "post_attention_layernorm": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM, - "mlp.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE, - "mlp.up_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP, - "mlp.down_proj": gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN, - } - - # note: codebook pages will be stacked to 3D - _CODE_GEN_N_CODEBOOKS = 15 - _code_embed_buffer: dict[int, Tensor] = {} - _code_head_buffer: dict[int, Tensor] = {} - _wav_config_cache: dict[str, Any] | None = None - - def __init__(self, dir_model: Path, *args, **kwargs): - hparams = kwargs.pop("hparams", None) - if hparams is None: - hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False) - hparams["text_config"] = {"hidden_size": hparams["talker_config"]["hidden_size"]} - # ECAPA-TDNN has a fixed 4-stage backbone, but MmprojModel.__init__ needs a n_block_keys - hparams["speaker_encoder_config"]["n_layers"] = 4 - super().__init__(dir_model, *args, hparams=hparams, **kwargs) - self._wav_config_cache = None - - def get_audio_config(self) -> dict[str, Any] | None: - return self.global_config.get("speaker_encoder_config") - - def set_gguf_parameters(self): - self.gguf_writer.add_file_type(self.ftype) - self.gguf_writer.add_clip_has_audio_encoder(True) - self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_SPKENC) - - # handle speaker encoder config - self.gguf_writer.add_audio_projection_dim(self.n_embd_text) - # mel_spectrogram() front-end: sr=24000, n_fft=1024, hop=256, n_mels=128, fmin=0, fmax=12000 (=sr/2, the clip.cpp default) - self.gguf_writer.add_audio_num_mel_bins(128) - # 3 SE-Res2Net stages; the stem conv, mfa, asp and fc are not counted here - self.gguf_writer.add_audio_block_count(3) - # ECAPA-TDNN has no attention/FFN, these are dummy to allow clip.cpp to load it - self.gguf_writer.add_audio_embedding_length(1536) - self.gguf_writer.add_audio_head_count(1) - self.gguf_writer.add_audio_feed_forward_length(1536) - self.gguf_writer.add_audio_attention_layernorm_eps(1e-5) - - # handle code predictor config - self.gguf_writer.add_clip_has_gen_audio_encoder(True) - self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.QWEN3TTS_GEN) - code_predictor_config = self.global_config["talker_config"]["code_predictor_config"] - self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text) - self.gguf_writer.add_gen_audio_embedding_length(code_predictor_config["hidden_size"]) - self.gguf_writer.add_gen_audio_feed_forward_length(code_predictor_config["intermediate_size"]) - self.gguf_writer.add_gen_audio_block_count(code_predictor_config["num_hidden_layers"]) - self.gguf_writer.add_gen_audio_head_count(code_predictor_config["num_attention_heads"]) - self.gguf_writer.add_gen_audio_head_count_kv(code_predictor_config["num_key_value_heads"]) - self.gguf_writer.add_gen_audio_attention_layernorm_eps(code_predictor_config["rms_norm_eps"]) - # note: code2wav hparams are hardcoded on the mtmd/clip.cpp side for now, not written here - - def _wav_decoder_config(self) -> dict[str, Any] | None: - # code2wav has its own config.json, inside the speech_tokenizer dir - if self._wav_config_cache is None: - path = self.dir_model / "speech_tokenizer" / "config.json" - with open(path, "r", encoding="utf-8") as f: - cfg = json.load(f) - self._wav_config_cache = cfg["decoder_config"] - return self._wav_config_cache - - def tensor_force_quant(self, name, new_name, bid, n_dims): - # conv1d/conv1d_dw kernels must be F16, ggml_conv_1d(_dw) has no BF16 path - if new_name.endswith(".weight") and ( - new_name in ("a.gen.wav.pre_conv.weight", "a.gen.wav.dac.entry.weight", "a.gen.wav.dac.post_conv.weight") - or (".up.blk." in new_name and new_name.endswith(".dwconv.weight")) - or (".dac.blk." in new_name and (new_name.endswith(".conv1.weight") or new_name.endswith(".conv2.weight"))) - ): - return gguf.GGMLQuantizationType.F16 - # ConvTranspose1d kernels: only F16/F32 are implemented, no BF16 - if new_name.endswith(".conv.weight") and (".up.blk." in new_name or ".dac.blk." in new_name): - return gguf.GGMLQuantizationType.F32 - return super().tensor_force_quant(name, new_name, bid, n_dims) - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - - if not ( - name.startswith("speaker_encoder.") - or name.startswith("talker.code_predictor.") - or name == "talker.model.codec_embedding.weight" - ): - return None - - return super().filter_tensors((name, gen)) - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - # code2wav tensors are already named by generate_extra_tensors(), pass them through - if name.startswith("a.gen.wav."): - yield (name, data_torch) - return - - # codebook-0 embedding, fed back to the talker backbone (codebooks 1-15 live in code_predictor) - if name == "talker.model.codec_embedding.weight": - yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUT_EMBD), data_torch) - return - - if name == "talker.code_predictor.model.norm.weight": - yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM), data_torch) - return - - if name.startswith("talker.code_predictor.small_to_mtp_projection."): - suffix = "." + name.rsplit(".", 1)[1] - yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_PROJ_IN, suffix=suffix), data_torch) - return - - if name.startswith("talker.code_predictor.model.codec_embedding."): - idx = int(name.split("codec_embedding.")[1].split(".")[0]) - self._code_embed_buffer[idx] = data_torch - if len(self._code_embed_buffer) < self._CODE_GEN_N_CODEBOOKS: - return - stacked = torch.stack([self._code_embed_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0) - yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_EMBD), stacked) - return - - if name.startswith("talker.code_predictor.lm_head."): - idx = int(name.split("lm_head.")[1].split(".")[0]) - self._code_head_buffer[idx] = data_torch - if len(self._code_head_buffer) < self._CODE_GEN_N_CODEBOOKS: - return - stacked = torch.stack([self._code_head_buffer.pop(i) for i in range(self._CODE_GEN_N_CODEBOOKS)], dim=0) - yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_GEN_CODE_HEAD), stacked) - return - - if name.startswith("talker.code_predictor.model.layers."): - rest = name.split("model.layers.")[1] # "{bid}..weight" - _, key_with_suffix = rest.split(".", 1) # ".weight" - key = key_with_suffix.rsplit(".", 1)[0] # "" - tensor = self._CODE_LAYER_TENSOR_MAP.get(key) - if tensor is not None: - yield (self.format_tensor_name(tensor, bid), data_torch) - return - - if "res2net_block.blocks." in name: - assert bid is not None # the outer stage index, picked up from the tensor name automatically - xid = int(name.split("res2net_block.blocks.")[1].split(".")[0]) - suffix = "." + name.rsplit(".", 1)[1] - new_name = gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.A_ENC_CONV_RES2].format(bid=bid, xid=xid) + suffix - yield (new_name, data_torch) - return - - yield from super().modify_tensors(data_torch, name, bid) - - def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - yield from self._generate_code2wav_tensors() - - def _generate_code2wav_tensors(self) -> Iterable[tuple[str, Tensor]]: - # code2wav weights live in speech_tokenizer/model.safetensors, not the main safetensors - from safetensors.torch import load_file - - wav_config = self._wav_decoder_config() - state_dict = load_file(self.dir_model / "speech_tokenizer" / "model.safetensors") - - def get(name: str) -> Tensor: - return state_dict[name] - - def snake_fold(alpha: Tensor, beta: Tensor) -> tuple[Tensor, Tensor]: - # fold SnakeBeta's exp()/reciprocal here, so the graph is only mul/sin/sqr/mul/add - return torch.exp(alpha), 1.0 / (torch.exp(beta) + 1e-9) - - def rvq_codebook(prefix: str, n_layers: int) -> Tensor: - # checkpoint has EMA accumulators, so codebook[i] = embedding_sum[i] / cluster_usage[i] - books = [] - for i in range(n_layers): - embedding_sum = get(f"{prefix}.vq.layers.{i}._codebook.embedding_sum") - cluster_usage = get(f"{prefix}.vq.layers.{i}._codebook.cluster_usage") - books.append(embedding_sum / cluster_usage.clamp_min(1e-5).unsqueeze(-1)) - return torch.stack(books, dim=0) if n_layers > 1 else books[0] - - T = gguf.MODEL_TENSOR - - # --- quantizer: RVQ codebook decode --- - yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_IN), get("decoder.quantizer.rvq_first.input_proj.weight").squeeze(-1)) - yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT), get("decoder.quantizer.rvq_first.output_proj.weight").squeeze(-1)) - yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), rvq_codebook("decoder.quantizer.rvq_first", 1)) - yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_IN), get("decoder.quantizer.rvq_rest.input_proj.weight").squeeze(-1)) - yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_OUT), get("decoder.quantizer.rvq_rest.output_proj.weight").squeeze(-1)) - yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_REST_CB), rvq_codebook("decoder.quantizer.rvq_rest", self._CODE_GEN_N_CODEBOOKS)) - - # --- pre_conv --- - yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".weight"), get("decoder.pre_conv.conv.weight")) - yield (self.format_tensor_name(T.A_GEN_WAV_PRE_CONV, suffix=".bias"), get("decoder.pre_conv.conv.bias")) - - # --- pre_transformer --- - yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".weight"), get("decoder.pre_transformer.input_proj.weight")) - yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=".bias"), get("decoder.pre_transformer.input_proj.bias")) - yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".weight"), get("decoder.pre_transformer.output_proj.weight")) - yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=".bias"), get("decoder.pre_transformer.output_proj.bias")) - yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), get("decoder.pre_transformer.norm.weight")) - - tfm_layer_map = { - "input_layernorm.weight": T.A_GEN_WAV_TFM_ATTN_NORM, - "self_attn.q_proj.weight": T.A_GEN_WAV_TFM_ATTN_Q, - "self_attn.k_proj.weight": T.A_GEN_WAV_TFM_ATTN_K, - "self_attn.v_proj.weight": T.A_GEN_WAV_TFM_ATTN_V, - "self_attn.o_proj.weight": T.A_GEN_WAV_TFM_ATTN_OUT, - "self_attn_layer_scale.scale": T.A_GEN_WAV_TFM_ATTN_SCALE, - "post_attention_layernorm.weight": T.A_GEN_WAV_TFM_FFN_NORM, - "mlp.gate_proj.weight": T.A_GEN_WAV_TFM_FFN_GATE, - "mlp.up_proj.weight": T.A_GEN_WAV_TFM_FFN_UP, - "mlp.down_proj.weight": T.A_GEN_WAV_TFM_FFN_DOWN, - "mlp_layer_scale.scale": T.A_GEN_WAV_TFM_FFN_SCALE, - } - assert wav_config is not None - for bid in range(wav_config["num_hidden_layers"]): - for key, tensor_id in tfm_layer_map.items(): - yield (self.format_tensor_name(tensor_id, bid), get(f"decoder.pre_transformer.layers.{bid}.{key}")) - - # --- upsample: 2x (causal ConvTranspose1d + ConvNeXt block) --- - up_map = { - "0.conv.weight": (T.A_GEN_WAV_UP_CONV, ".weight"), - "0.conv.bias": (T.A_GEN_WAV_UP_CONV, ".bias"), - "1.dwconv.conv.weight": (T.A_GEN_WAV_UP_DWCONV, ".weight"), - "1.dwconv.conv.bias": (T.A_GEN_WAV_UP_DWCONV, ".bias"), - "1.norm.weight": (T.A_GEN_WAV_UP_NORM, ".weight"), - "1.norm.bias": (T.A_GEN_WAV_UP_NORM, ".bias"), - "1.pwconv1.weight": (T.A_GEN_WAV_UP_PW1, ".weight"), - "1.pwconv1.bias": (T.A_GEN_WAV_UP_PW1, ".bias"), - "1.pwconv2.weight": (T.A_GEN_WAV_UP_PW2, ".weight"), - "1.pwconv2.bias": (T.A_GEN_WAV_UP_PW2, ".bias"), - "1.gamma": (T.A_GEN_WAV_UP_GAMMA, ""), - } - for bid in range(len(wav_config["upsampling_ratios"])): - for key, (tensor_id, suffix) in up_map.items(): - yield (self.format_tensor_name(tensor_id, bid, suffix=suffix), get(f"decoder.upsample.{bid}.{key}")) - - # --- DAC decoder --- - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".weight"), get("decoder.decoder.0.conv.weight")) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_ENTRY, suffix=".bias"), get("decoder.decoder.0.conv.bias")) - - n_dac_blocks = len(wav_config["upsample_rates"]) - for bid in range(n_dac_blocks): - py = bid + 1 # decoder.decoder.0 is the entry conv, blocks start at 1 - - a, b = snake_fold(get(f"decoder.decoder.{py}.block.0.alpha"), get(f"decoder.decoder.{py}.block.0.beta")) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".alpha"), a) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_SNAKE, bid, suffix=".beta"), b) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".weight"), get(f"decoder.decoder.{py}.block.1.conv.weight")) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_UP_CONV, bid, suffix=".bias"), get(f"decoder.decoder.{py}.block.1.conv.bias")) - - for xid in range(3): - ridx = xid + 2 # block.2/3/4 are the 3 residual units - - a1, b1 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act1.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act1.beta")) - name1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT1].format(bid=bid, xid=xid) - yield (name1 + ".alpha", a1) - yield (name1 + ".beta", b1) - - name_conv1 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV1].format(bid=bid, xid=xid) - yield (name_conv1 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.weight")) - yield (name_conv1 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv1.conv.bias")) - - a2, b2 = snake_fold(get(f"decoder.decoder.{py}.block.{ridx}.act2.alpha"), get(f"decoder.decoder.{py}.block.{ridx}.act2.beta")) - name2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_ACT2].format(bid=bid, xid=xid) - yield (name2 + ".alpha", a2) - yield (name2 + ".beta", b2) - - name_conv2 = gguf.TENSOR_NAMES[T.A_GEN_WAV_DAC_RES_CONV2].format(bid=bid, xid=xid) - yield (name_conv2 + ".weight", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.weight")) - yield (name_conv2 + ".bias", get(f"decoder.decoder.{py}.block.{ridx}.conv2.conv.bias")) - - a5, b5 = snake_fold(get("decoder.decoder.5.alpha"), get("decoder.decoder.5.beta")) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".alpha"), a5) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_SNAKE, suffix=".beta"), b5) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".weight"), get("decoder.decoder.6.conv.weight")) - yield (self.format_tensor_name(T.A_GEN_WAV_DAC_POST_CONV, suffix=".bias"), get("decoder.decoder.6.conv.bias")) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 6e74c8764..d95311ffc 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -126,12 +126,8 @@ def parse_args() -> argparse.Namespace: help="Export only the multi-token prediction (MTP) head as a separate GGUF, suitable for use as a speculative draft. An 'mtp-' prefix will be added to the output file name.", ) parser.add_argument( - "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true", - help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.", - ) - parser.add_argument( - "--dspark", action="store_true", - help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.", + "--no-mtp", action="store_true", + help="Exclude the multi-token prediction (MTP) head from the converted GGUF. Pair with --mtp on a second run to publish trunk and MTP as two files. Note: the split form duplicates embeddings, but even though the bundled default is more space-efficient overall, this allows differing quantization which may be more performant.", ) parser.add_argument( "--mistral-format", action="store_true", @@ -262,20 +258,13 @@ def main() -> None: from conversion.mistral import MistralModel model_class = MistralModel - if sum((args.mtp, args.no_mtp, args.dspark)) > 1: - logger.error("--mtp, --no-nextn, and --dspark are mutually exclusive") + if args.mtp and args.no_mtp: + logger.error("--mtp and --no-mtp are mutually exclusive") sys.exit(1) - if args.dspark: - if is_mistral_format or model_architecture != "DeepseekV4ForCausalLM": - logger.error("--dspark is only supported for DeepseekV4ForCausalLM") - sys.exit(1) - from conversion.deepseek import DeepseekV4DSparkModel - model_class = DeepseekV4DSparkModel - if args.mtp or args.no_mtp: if not model_class.supports_mtp_export: - logger.error("--mtp / --no-nextn are not supported for %s", model_architecture) + logger.error("--mtp / --no-mtp are not supported for %s", model_architecture) sys.exit(1) if args.no_mtp: model_class.no_mtp = True diff --git a/docs/release.md b/docs/release.md deleted file mode 100644 index 4335ef9d4..000000000 --- a/docs/release.md +++ /dev/null @@ -1,49 +0,0 @@ -# Release process - -llama.cpp uses [semantic versioning](https://semver.org) (`MAJOR.MINOR.PATCH`). - -## Version bump guidelines - -| Change type | Version component | -|---|---| -| Breaking change to the public C API (`include/llama.h`) | `MAJOR` | -| Backward-compatible features, model support, or API addition | `MINOR` | -| Bug fix with no API change | `PATCH` | - -The version is set in the three variables at the top of the root `CMakeLists.txt`: - -```cmake -set(LLAMA_VERSION_MAJOR 0) -set(LLAMA_VERSION_MINOR 1) -set(LLAMA_VERSION_PATCH 0) -``` - -_A version bump should be included in the PR that introduces the change, or in a -dedicated bump commit merged before the release is cut._ - -_TODO: add PR labels (`semver: patch`, `semver: minor`, `semver: major`) to help -identify which PRs require a version bump before cutting a release._ - -## Making a release - -Releases are created by running the [make-release](.github/workflows/make-release.yml) -which is a manual workflow. - -The workflow creates an annotated git tag (e.g. `v0.1.0`) and pushes it to the -remote. No GitHub Release object is created, the tag is the release artifact. - -## Building a release - -By default, `LLAMA_BUILD_IS_DEV=ON` which appends a `-dev` suffix to `LLAMA_VERSION`, -marking the build as a nightly/development build. Distributors building from a -release tag must pass `-DLLAMA_BUILD_IS_DEV=OFF` to produce a clean version string -(e.g. `0.1.0` instead of `0.1.0-dev`). - -## How releases reach users -Currently releases are not published to github releases, only nightly/development -builds are available there. The way users can access releases are using the following -channels: - -- **llama-install.sh** โ€” downloads pre-built binaries built from the release tag. -- **Package managers** โ€” consume the git tag directly. -- **Build from source** โ€” users clone the repo and check out the tag. diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd index d6bc85f44..09f7a0400 100644 --- a/embd_res/kcpp_sdui.embd +++ b/embd_res/kcpp_sdui.embd @@ -5,18 +5,18 @@ Stable UI for KoboldCpp - - diff --git a/embd_res/klite.embd b/embd_res/klite.embd index a5d03f444..8da59c10d 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -12,7 +12,7 @@ Current version indicated by LITEVER below. --> {#each modalities as modality (modality)} diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte index 36895c8e7..e74bd8456 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsList.svelte @@ -28,18 +28,18 @@ } let { - activeModelId, - attachments = [], class: className = '', + style = '', + attachments = [], + readonly = false, + onFileRemove, + uploadedFiles = $bindable([]), // Default to small size for form previews imageClass = '', imageHeight = 'h-24', imageWidth = 'w-auto', limitToSingleRow = false, - onFileRemove, - readonly = false, - style = '', - uploadedFiles = $bindable([]) + activeModelId }: Props = $props(); let carouselRef: HorizontalScrollCarousel | undefined = $state(); @@ -48,7 +48,7 @@ let previewFocusIndex = $state(0); let viewAllDialogOpen = $state(false); - let displayItems = $derived(getAttachmentDisplayItems({ attachments, uploadedFiles })); + let displayItems = $derived(getAttachmentDisplayItems({ uploadedFiles, attachments })); function openPreview(item: ChatAttachmentDisplayItem, event?: MouseEvent) { event?.stopPropagation(); diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte index ba06e1815..143621cd9 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItem.svelte @@ -2,8 +2,8 @@ import { ChatAttachmentsListItemMcpPrompt, ChatAttachmentsListItemMcpResource, - ChatAttachmentsListItemThumbnailFile, - ChatAttachmentsListItemThumbnailImage + ChatAttachmentsListItemThumbnailImage, + ChatAttachmentsListItemThumbnailFile } from '$lib/components/app'; import { AttachmentType } from '$lib/enums'; import type { @@ -49,10 +49,10 @@ return { id, resource: { + uri: extra.uri, name: extra.name, - serverName: extra.serverName, title: extra.name, - uri: extra.uri + serverName: extra.serverName } }; } @@ -64,12 +64,12 @@ ? (item.attachment as DatabaseMessageExtraMcpPrompt) : item.uploadedFile?.mcpPrompt ? { - arguments: item.uploadedFile.mcpPrompt.arguments, - content: item.textContent ?? '', + type: AttachmentType.MCP_PROMPT as const, name: item.name, - promptName: item.uploadedFile.mcpPrompt.promptName, serverName: item.uploadedFile.mcpPrompt.serverName, - type: AttachmentType.MCP_PROMPT as const + promptName: item.uploadedFile.mcpPrompt.promptName, + content: item.textContent ?? '', + arguments: item.uploadedFile.mcpPrompt.arguments } : null} {#if mcpPrompt} diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte index f5452aade..636e93f22 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsList/ChatAttachmentsListItem/ChatAttachmentsListItemMcpPrompt.svelte @@ -1,8 +1,8 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte index 4be156edb..7c7cf5120 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewCurrentItem/ChatAttachmentsPreviewCurrentItemPdf.svelte @@ -1,13 +1,13 @@ {#if show} diff --git a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte index 366c8372b..8a85df7d0 100644 --- a/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatAttachments/ChatAttachmentsPreview/ChatAttachmentsPreviewThumbnailStrip.svelte @@ -1,7 +1,7 @@ @@ -36,7 +34,7 @@ {#each items as item, index (item.id)}
- - {#if toolsStore.hasEnabledCwdTools} - - {/if} + import { ICON_CLASS_DEFAULT } from '$lib/constants/css-classes'; import { Plus } from '@lucide/svelte'; import { Button } from '$lib/components/ui/button'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { ATTACHMENT_TOOLTIP_TEXT, ICON_CLASS_DEFAULT } from '$lib/constants'; + import { ATTACHMENT_TOOLTIP_TEXT } from '$lib/constants'; interface Props { disabled?: boolean; diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte index 02bfadb7e..f81dcf09c 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionAddDropdown.svelte @@ -1,30 +1,51 @@ {#if isMobile.current} - + {#snippet trigger({ disabled, onclick })} {/snippet} {:else} - + {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index 9f163c6d6..998e8dcb4 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -1,6 +1,15 @@
{#if showAddButton}
- + goto(ROUTES.MCP_SERVERS)} + />
{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte index 606fa6cd3..ff6d39fdd 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ChatFormContextGauge.svelte @@ -1,36 +1,32 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte index eaaba69de..fdec5aca5 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDetails.svelte @@ -1,9 +1,8 @@ - + Token usage details - + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte index 67d705ae4..6e2616d36 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugeDial.svelte @@ -1,6 +1,6 @@ {#if modelId !== null && !isLoading} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte index e6abb4a3e..af9ad010e 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormContextGauge/ContextGaugePopup.svelte @@ -1,10 +1,15 @@ - - - - - - - event.preventDefault()} - onCloseAutoFocus={(event) => event.preventDefault()} - class="w-[var(--bits-popover-anchor-width)] max-w-none rounded-xl border-border/50 p-0 shadow-xl" - > -
- - - {#if !fileSearchEnabled} -
{searchUnavailableMessage}
- {:else if query.trim() && (search.isSearching || queryResults.length > 0 || searchError)} - nav.setHover(index)} - /> - {/if} - - {#if pickerSupported && fileSearchEnabled} - - {/if} - - {#if homeBase && fileSearchEnabled} - - - - Searching in: - - {abbreviateHome(searchScope, homeBase)} - - {/if} -
- - - - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryChip.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryChip.svelte deleted file mode 100644 index 23661d223..000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryChip.svelte +++ /dev/null @@ -1,70 +0,0 @@ - - - -
- - - {#if showTooltip && displayLabelTitle} - - - {#snippet child({ props })} - {displayLabel} - {/snippet} - - -

{displayLabelTitle}

- - - {:else} - {displayLabel} - {/if} -
- - {#if directory} -
- -
- {/if} - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryResultsList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryResultsList.svelte deleted file mode 100644 index e8087d967..000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormCurrentWorkingDirectory/ChatFormCurrentWorkingDirectoryResultsList.svelte +++ /dev/null @@ -1,73 +0,0 @@ - - -
- {#if isSearching && results.length === 0} -
Searching...
- {:else if error} -
{error}
- {:else if results.length === 0} -
No matching folders
- {:else} - {#each results as path, index (path)} - - {/each} - {/if} -
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputFileInputInvisible.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormFileInputInvisible.svelte similarity index 100% rename from tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputFileInputInvisible.svelte rename to tools/ui/src/lib/components/app/chat/ChatForm/ChatFormFileInputInvisible.svelte diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte deleted file mode 100644 index b708ae047..000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInput.svelte +++ /dev/null @@ -1,78 +0,0 @@ - - -{#if useRichInput} - -{:else} - -{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte deleted file mode 100644 index d87817adb..000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputRich.svelte +++ /dev/null @@ -1,867 +0,0 @@ - - -
-
-
- - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte index 3f178da18..36c82224a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormMcpResourcesList.svelte @@ -1,9 +1,13 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte index 160c14ce8..6647928b2 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerList.svelte @@ -1,9 +1,8 @@ {#if showSearchInput}
- +
{/if} -
+
{#if isLoading} {#if skeleton} {@render skeleton()} - {:else} -
- {#each { length: skeletonCount } as _, rowIndex (rowIndex)} -
-
-
-
-
-
-
- {/each} -
- {/if} - {:else if items && items.length === 0} - {#if emptyMessage} -
{emptyMessage}
{/if} + {:else if items.length === 0} +
{emptyMessage}
{:else} {#each items as itemData, index (itemKey(itemData, index))} {@render item(itemData, index, index === selectedIndex)} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte index 045534f48..4d82c6b58 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItem.svelte @@ -1,37 +1,23 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte index cbf7b972e..5a2ab26fc 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerListItemSkeleton.svelte @@ -4,7 +4,7 @@ showBadge?: boolean; } - let { showBadge = false, titleWidth = 'w-48' }: Props = $props(); + let { titleWidth = 'w-48', showBadge = false }: Props = $props();
diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte index b09d346f1..c43a002e6 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPicker/ChatFormPickerPopover.svelte @@ -1,6 +1,6 @@ @@ -42,7 +42,6 @@ align="start" sideOffset={12} class="w-[var(--bits-popover-anchor-width)] max-w-none rounded-xl border-border/50 p-0 shadow-xl {className}" - preventScroll={false} onkeydown={onKeydown} onOpenAutoFocus={(event) => event.preventDefault()} > diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte deleted file mode 100644 index df654b25b..000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerCommand.svelte +++ /dev/null @@ -1,142 +0,0 @@ - - - - command.name} - scrollTrigger={nav.scrollTrigger} - > - {#snippet item(command, index, isSelected)} - {@const Icon = commandIcon[command.action]} - handleSelect(command)} - onmouseenter={() => { - if (!command.disabled) nav.setHover(index); - }} - > - -
- /{command.name} - - {command.description} - -
- - {/snippet} - - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte index 9b5a57b9b..f35d816de 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPickerMcpPrompts.svelte @@ -1,18 +1,19 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte index 074c69b84..638d10eef 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMcpPrompts/ChatFormPromptPickerArgumentInput.svelte @@ -1,8 +1,8 @@ @@ -66,7 +66,7 @@ {#if isAutocompleteActive && suggestions.length > 0}
{#each suggestions as suggestion, i (suggestion)} + {/if} + {/snippet} + + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte deleted file mode 100644 index 1c7c8f7d4..000000000 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickerMention.svelte +++ /dev/null @@ -1,278 +0,0 @@ - - - { - if (!open) onClose(); - }} -> - - - - event.preventDefault()} - onCloseAutoFocus={(event) => event.preventDefault()} - class={[ - 'w-[var(--bits-popover-anchor-width)] max-w-none rounded-xl border-border/50 p-0 shadow-xl', - className - ]} - > - entry.type + ':' + entry.path} - scrollTrigger={nav.scrollTrigger} - > - {#snippet item(entry, index, isSelected)} - handleSelect(entry)} - onmouseenter={() => nav.setHover(index)} - > - {@const Icon = entry.type === FileMentionEntryType.DIRECTORY ? Folder : File} - -
-
- {#if showTooltip} - - - {#snippet child({ props })} - {entry.name} - {/snippet} - - -

{entry.path}

- - - {:else} - {entry.name} - {/if} - - {entry.type} - -
- - - -
- - {/snippet} - - - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte index dbe03e2e0..7c5dc85b2 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormPickers/ChatFormPickers.svelte @@ -1,30 +1,16 @@ - {})} - onSelect={onCommandSelect ?? (() => {})} -/> - - {})} - onOpened={onMentionOpened} - onSelect={onMentionSelect ?? (() => {})} + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte similarity index 74% rename from tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte rename to tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte index e0c08c721..3e683389f 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormInput/ChatFormInputBasic.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte @@ -1,5 +1,5 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte index 78cb88721..b8068f790 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessage.svelte @@ -1,28 +1,26 @@ -
+
{#if message.role === MessageRole.SYSTEM} - + {:else if mcpPromptExtra} - - {:else if isSynthetic} - + {:else if message.role === MessageRole.USER} - + {:else} {/if}
@@ -410,17 +422,7 @@ * once known; 500px sizes messages that have never been rendered. */ .chat-message { - --chat-message-intrinsic-size: 500px; content-visibility: auto; - contain-intrinsic-size: auto var(--chat-message-intrinsic-size); - } - - /* - * Synthetic rows (e.g. the working-directory change) are small, so an - * accurate placeholder keeps the injected row from inflating the - * auto-scroll offset; the 500px default is for ordinary bubbles. - */ - .chat-message--synthetic { - --chat-message-intrinsic-size: 40px; + contain-intrinsic-size: auto 500px; } diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte index b92be9fbd..199d75fce 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistant.svelte @@ -1,55 +1,84 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte index d69337960..30ce16be9 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantRawOutput.svelte @@ -1,5 +1,5 @@ diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte index e6e18ae08..4cc4080c3 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageAssistant/ChatMessageAssistantStatistics.svelte @@ -2,7 +2,6 @@ import { ChatMessageStatistics } from '$lib/components/app'; import { ChatMessageStatisticsMode } from '$lib/enums'; import type { UseProcessingStateReturn } from '$lib/hooks/use-processing-state.svelte'; - import { agenticStore } from '$lib/stores'; interface Props { message: DatabaseMessage; @@ -11,27 +10,10 @@ showMessageStats: boolean; } - let { isLoading, message, processingState, showMessageStats }: Props = $props(); - - // A running agentic flow stamps per-turn timings on its root message at each - // turn boundary and the cumulative agentic totals only on exit; while it runs, - // show the session's live totals on the root message instead. - const liveLlm = $derived(agenticStore.getLiveLlmTotals(message.convId)); - const isLiveFlowRoot = $derived( - liveLlm !== null && agenticStore.getFlowRootMessageId(message.convId) === message.id - ); + let { message, isLoading, processingState, showMessageStats }: Props = $props(); -{#if showMessageStats && isLiveFlowRoot && liveLlm} - -{:else if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms} +{#if showMessageStats && message.timings && message.timings.predicted_n && message.timings.predicted_ms} {@const agentic = message.timings.agentic} - import { Folder, FolderX } from '@lucide/svelte'; - import type { DatabaseMessage } from '$lib/types'; - import { parseCwdMessage } from '$lib/utils'; - - interface Props { - class?: string; - message: DatabaseMessage; - } - - let { class: className = '', message }: Props = $props(); - - // Parse the synthetic message content in the UI so the row reuses the - // exact same text the model saw, including any guidance suffix. - let info = $derived(parseCwdMessage(message.content)); - - -{#if info} -
- {#if info.path === null} - - Working directory cleared - {:else} - - Set working directory to  - - {info.display} - - {/if} -
-{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte index 4563b1fa8..2dcb36baf 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPrompt.svelte @@ -4,20 +4,47 @@ ChatMessageEditForm, ChatMessageMcpPromptContent } from '$lib/components/app'; - import { getChatMessageEditContext } from '$lib/contexts'; - import { McpPromptVariant, MessageRole } from '$lib/enums'; + import { getMessageEditContext } from '$lib/contexts'; + import { MessageRole, McpPromptVariant } from '$lib/enums'; import type { DatabaseMessageExtraMcpPrompt } from '$lib/types'; interface Props { class?: string; message: DatabaseMessage; mcpPrompt: DatabaseMessageExtraMcpPrompt; + siblingInfo?: ChatMessageSiblingInfo | null; + showDeleteDialog: boolean; + deletionInfo: { + totalCount: number; + userMessages: number; + assistantMessages: number; + messageTypes: string[]; + } | null; + onCopy: () => void; + onEdit: () => void; + onDelete: () => void; + onConfirmDelete: () => void; + onNavigateToSibling?: (siblingId: string) => void; + onShowDeleteDialogChange: (show: boolean) => void; } - let { class: className = '', mcpPrompt, message }: Props = $props(); + let { + class: className = '', + message, + mcpPrompt, + siblingInfo = null, + showDeleteDialog, + deletionInfo, + onCopy, + onEdit, + onDelete, + onConfirmDelete, + onNavigateToSibling, + onShowDeleteDialogChange + }: Props = $props(); // Get edit context - const editCtx = getChatMessageEditContext(); + const editCtx = getMessageEditContext();
- +
{/if} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte index 9190c7e62..3d5dec3b6 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageMcpPrompt/ChatMessageMcpPromptContent.svelte @@ -1,11 +1,11 @@ - -{#if isCwdChange} - -{:else} - {message.content} -{/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte index c6222f568..24b3be4c5 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageSystem/ChatMessageSystem.svelte @@ -4,20 +4,47 @@ import { Button } from '$lib/components/ui/button'; import { Card } from '$lib/components/ui/card'; import { INPUT_CLASSES } from '$lib/constants'; - import { getChatMessageEditContext } from '$lib/contexts'; + import { getMessageEditContext } from '$lib/contexts'; import { KeyboardKey, MessageRole } from '$lib/enums'; - import { settingsStore } from '$lib/stores'; + import { config } from '$lib/stores/settings.svelte'; import { autoResizeTextarea, isIMEComposing } from '$lib/utils'; interface Props { class?: string; message: DatabaseMessage; + siblingInfo?: ChatMessageSiblingInfo | null; + showDeleteDialog: boolean; + deletionInfo: { + totalCount: number; + userMessages: number; + assistantMessages: number; + messageTypes: string[]; + } | null; + onCopy: () => void; + onEdit: () => void; + onDelete: () => void; + onConfirmDelete: () => void; + onNavigateToSibling?: (siblingId: string) => void; + onShowDeleteDialogChange: (show: boolean) => void; textareaElement?: HTMLTextAreaElement; } - let { class: className = '', message, textareaElement = $bindable() }: Props = $props(); + let { + class: className = '', + message, + siblingInfo = null, + showDeleteDialog, + deletionInfo, + onCopy, + onEdit, + onDelete, + onConfirmDelete, + onNavigateToSibling, + onShowDeleteDialogChange, + textareaElement = $bindable() + }: Props = $props(); - const editCtx = getChatMessageEditContext(); + const editCtx = getMessageEditContext(); function handleEditKeydown(event: KeyboardEvent) { if (event.key === KeyboardKey.ENTER && !event.shiftKey && !isIMEComposing(event)) { @@ -37,7 +64,7 @@ let contentHeight = $state(0); const MAX_HEIGHT = 200; // pixels - const currentConfig = settingsStore.config; + const currentConfig = config(); let showExpandButton = $derived(contentHeight > MAX_HEIGHT); @@ -191,7 +218,20 @@ {#if message.timestamp}
- +
{/if} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index a6fa2e250..b1daedfc8 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -1,19 +1,22 @@ @@ -103,26 +103,13 @@
{line.text}
- {#if line.media} - {#if line.media.type === AttachmentType.AUDIO} - {@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG} -
- -
- {:else} - {line.media.name} - {/if} + {#if line.image} + {line.image.name} {/if} {/each}
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte index 6545cc39f..b990c3898 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockEditFile.svelte @@ -1,11 +1,9 @@ {#snippet execShellTitle()} - {#if cwd} - {wdDisplay} - $ - {/if} - {#if highlightedCommandHtml} {@html highlightedCommandHtml} {:else} @@ -220,10 +200,10 @@ > {#each outputLines as line, i (i)}
{line.text}
- {#if line.media?.type === AttachmentType.IMAGE} + {#if line.image} {line.media.name} @@ -252,23 +232,6 @@ ${markup}` : ''; } diff --git a/tools/ui/src/lib/utils/svg-to-png.ts b/tools/ui/src/lib/utils/svg-to-png.ts index 07b84b3f8..d5a7f7d83 100644 --- a/tools/ui/src/lib/utils/svg-to-png.ts +++ b/tools/ui/src/lib/utils/svg-to-png.ts @@ -20,7 +20,6 @@ export function svgBase64UrlToPngDataURL( if (!ctx) { reject(new Error('Failed to get 2D canvas context.')); - return; } @@ -34,7 +33,6 @@ export function svgBase64UrlToPngDataURL( ctx.fillStyle = backgroundColor; ctx.fillRect(0, 0, canvas.width, canvas.height); } - ctx.drawImage(img, 0, 0, targetWidth, targetHeight); resolve(canvas.toDataURL(MimeTypeImage.PNG)); @@ -48,7 +46,6 @@ export function svgBase64UrlToPngDataURL( } catch (error) { const message = error instanceof Error ? error.message : String(error); const errorMessage = `Error converting SVG to PNG: ${message}`; - console.error(errorMessage, error); reject(new Error(errorMessage)); } diff --git a/tools/ui/src/lib/utils/text-files.ts b/tools/ui/src/lib/utils/text-files.ts index f77094047..3f7a55ebc 100644 --- a/tools/ui/src/lib/utils/text-files.ts +++ b/tools/ui/src/lib/utils/text-files.ts @@ -4,8 +4,8 @@ */ import { DEFAULT_BINARY_DETECTION_OPTIONS } from '$lib/constants'; -import { FileExtensionText } from '$lib/enums'; import type { BinaryDetectionOptions } from '$lib/types'; +import { FileExtensionText } from '$lib/enums'; /** * Check if a filename indicates a text file based on its extension diff --git a/tools/ui/src/lib/utils/text.ts b/tools/ui/src/lib/utils/text.ts index 32bf1f38f..18a36eb8a 100644 --- a/tools/ui/src/lib/utils/text.ts +++ b/tools/ui/src/lib/utils/text.ts @@ -15,7 +15,6 @@ export function getPreviewText(content: string, max = 150): string { export function generateConversationTitle(content: string, useFirstLine: boolean = false): string { if (useFirstLine) { const firstLine = content.split(NEWLINE).find((line) => line.trim().length > 0); - return firstLine ? firstLine.trim() : content.trim(); } diff --git a/tools/ui/src/lib/utils/tool-call-meta.ts b/tools/ui/src/lib/utils/tool-call-meta.ts index b64bca786..798ba7b25 100644 --- a/tools/ui/src/lib/utils/tool-call-meta.ts +++ b/tools/ui/src/lib/utils/tool-call-meta.ts @@ -15,14 +15,11 @@ export function tryParseToolResultObject( toolResultString: string | undefined ): Record | null { if (!toolResultString) return null; - try { const parsed: unknown = JSON.parse(toolResultString); - if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { return parsed as Record; } - return null; } catch { return null; diff --git a/tools/ui/src/lib/utils/uri-template.ts b/tools/ui/src/lib/utils/uri-template.ts index 4ba82719b..eb8dbfb36 100644 --- a/tools/ui/src/lib/utils/uri-template.ts +++ b/tools/ui/src/lib/utils/uri-template.ts @@ -1,10 +1,11 @@ import { - LEADING_SLASHES_REGEX, TEMPLATE_EXPRESSION_REGEX, URI_SCHEME_SEPARATOR, - URI_TEMPLATE_SYMBOLS, + URI_TEMPLATE_OPERATORS, + URI_TEMPLATE_SEPARATORS, VARIABLE_EXPLODE_MODIFIER_REGEX, - VARIABLE_PREFIX_MODIFIER_REGEX + VARIABLE_PREFIX_MODIFIER_REGEX, + LEADING_SLASHES_REGEX } from '../constants'; /** @@ -24,7 +25,6 @@ import { */ export function normalizeResourceUri(uri: string): string { const schemeEnd = uri.indexOf(URI_SCHEME_SEPARATOR); - if (schemeEnd === -1) return uri; const scheme = uri.substring(0, schemeEnd); @@ -65,7 +65,6 @@ export function extractTemplateVariables(template: string): UriTemplateVariable[ const seen = new Set(); let match; - TEMPLATE_EXPRESSION_REGEX.lastIndex = 0; while ((match = TEMPLATE_EXPRESSION_REGEX.exec(template)) !== null) { @@ -118,6 +117,7 @@ export function expandTemplate(template: string, values: Record) .replace(VARIABLE_PREFIX_MODIFIER_REGEX, '') .trim() ); + const expandedParts = varNames .map((name: string) => values[name] ?? '') .filter((v: string) => v !== ''); @@ -125,59 +125,60 @@ export function expandTemplate(template: string, values: Record) if (expandedParts.length === 0) return ''; switch (operator) { - case URI_TEMPLATE_SYMBOLS.RESERVED: + case URI_TEMPLATE_OPERATORS.RESERVED: // Reserved expansion: no encoding - return expandedParts.join(URI_TEMPLATE_SYMBOLS.COMMA); - case URI_TEMPLATE_SYMBOLS.FRAGMENT: + return expandedParts.join(URI_TEMPLATE_SEPARATORS.COMMA); + case URI_TEMPLATE_OPERATORS.FRAGMENT: // Fragment expansion - return URI_TEMPLATE_SYMBOLS.FRAGMENT + expandedParts.join(URI_TEMPLATE_SYMBOLS.COMMA); - case URI_TEMPLATE_SYMBOLS.PATH_SEGMENT: - // Path segments return ( - URI_TEMPLATE_SYMBOLS.PATH_SEGMENT + - expandedParts.join(URI_TEMPLATE_SYMBOLS.PATH_SEGMENT) + URI_TEMPLATE_OPERATORS.FRAGMENT + expandedParts.join(URI_TEMPLATE_SEPARATORS.COMMA) ); - case URI_TEMPLATE_SYMBOLS.LABEL: + case URI_TEMPLATE_OPERATORS.PATH_SEGMENT: + // Path segments + return URI_TEMPLATE_SEPARATORS.SLASH + expandedParts.join(URI_TEMPLATE_SEPARATORS.SLASH); + case URI_TEMPLATE_OPERATORS.LABEL: // Label expansion - return URI_TEMPLATE_SYMBOLS.LABEL + expandedParts.join(URI_TEMPLATE_SYMBOLS.LABEL); - case URI_TEMPLATE_SYMBOLS.PATH_PARAM: + return ( + URI_TEMPLATE_SEPARATORS.PERIOD + expandedParts.join(URI_TEMPLATE_SEPARATORS.PERIOD) + ); + case URI_TEMPLATE_OPERATORS.PATH_PARAM: // Path-style parameters return varNames .filter((_: string, i: number) => expandedParts[i]) .map( (name: string, i: number) => - `${URI_TEMPLATE_SYMBOLS.PATH_PARAM}${name}=${expandedParts[i]}` + `${URI_TEMPLATE_SEPARATORS.SEMICOLON}${name}=${expandedParts[i]}` ) .join(''); - case URI_TEMPLATE_SYMBOLS.FORM_QUERY: + case URI_TEMPLATE_OPERATORS.FORM_QUERY: // Form-style query return ( - URI_TEMPLATE_SYMBOLS.FORM_QUERY + + URI_TEMPLATE_SEPARATORS.QUERY_PREFIX + varNames .filter((_: string, i: number) => expandedParts[i]) .map( (name: string, i: number) => `${encodeURIComponent(name)}=${encodeURIComponent(expandedParts[i])}` ) - .join(URI_TEMPLATE_SYMBOLS.FORM_CONTINUATION) + .join(URI_TEMPLATE_SEPARATORS.QUERY_CONTINUATION) ); - case URI_TEMPLATE_SYMBOLS.FORM_CONTINUATION: + case URI_TEMPLATE_OPERATORS.FORM_CONTINUATION: // Form-style query continuation return ( - URI_TEMPLATE_SYMBOLS.FORM_CONTINUATION + + URI_TEMPLATE_SEPARATORS.QUERY_CONTINUATION + varNames .filter((_: string, i: number) => expandedParts[i]) .map( (name: string, i: number) => `${encodeURIComponent(name)}=${encodeURIComponent(expandedParts[i])}` ) - .join(URI_TEMPLATE_SYMBOLS.COMMA) + .join(URI_TEMPLATE_SEPARATORS.COMMA) ); default: // Simple string expansion (default operator) return expandedParts .map((v: string) => encodeURIComponent(v)) - .join(URI_TEMPLATE_SYMBOLS.COMMA); + .join(URI_TEMPLATE_SEPARATORS.COMMA); } } ); diff --git a/tools/ui/src/lib/utils/url.ts b/tools/ui/src/lib/utils/url.ts index 1d44720e1..f1bf9ecb8 100644 --- a/tools/ui/src/lib/utils/url.ts +++ b/tools/ui/src/lib/utils/url.ts @@ -28,7 +28,6 @@ function isIpAddress(hostname: string): boolean { */ export function extractRootDomain(url: URL): string | null { const hostname = url.hostname.toLowerCase(); - if (!hostname || isIpAddress(hostname)) return null; const parts = hostname.split('.'); @@ -96,6 +95,7 @@ export function canonicalizeServerUrl(raw: string): string { try { const parsed = new URL(trimmed); const pathname = parsed.pathname.replace(TRAILING_SLASHES_REGEX, ''); + // Aggressive: drop the port unconditionally. We only use this for // equality checks between user-typed URLs and a hard-coded list of // recommendations, where the port can never carry distinguishing diff --git a/tools/ui/src/lib/utils/webp-to-png.ts b/tools/ui/src/lib/utils/webp-to-png.ts index 8c61ecf85..ea5183802 100644 --- a/tools/ui/src/lib/utils/webp-to-png.ts +++ b/tools/ui/src/lib/utils/webp-to-png.ts @@ -20,7 +20,6 @@ export function webpBase64UrlToPngDataURL( if (!ctx) { reject(new Error('Failed to get 2D canvas context.')); - return; } @@ -34,7 +33,6 @@ export function webpBase64UrlToPngDataURL( ctx.fillStyle = backgroundColor; ctx.fillRect(0, 0, canvas.width, canvas.height); } - ctx.drawImage(img, 0, 0, targetWidth, targetHeight); resolve(canvas.toDataURL(MimeTypeImage.PNG)); @@ -48,7 +46,6 @@ export function webpBase64UrlToPngDataURL( } catch (error) { const message = error instanceof Error ? error.message : String(error); const errorMessage = `Error converting WebP to PNG: ${message}`; - console.error(errorMessage, error); reject(new Error(errorMessage)); } diff --git a/tools/ui/src/lib/utils/working-directory.ts b/tools/ui/src/lib/utils/working-directory.ts deleted file mode 100644 index 906142d1c..000000000 --- a/tools/ui/src/lib/utils/working-directory.ts +++ /dev/null @@ -1,167 +0,0 @@ -/** - * Pure helpers for the working-directory picker search, backed by the - * server's `file_glob_search` tool. Queries starting from a root (`/`, - * `C:\`, `\\host\share`) or `~` navigate the tree (search the parent for - * the last segment); anything else glob-matches home-relative entries. - */ - -import { lastPathSegment } from './path-display'; -import { - GLOB, - HOME_TILDE, - LEADING_SLASHES_REGEX, - PATH_SEPARATOR, - SEARCH, - TRAILING_SLASHES_REGEX -} from '$lib/constants'; -import type { GlobEntry, GlobSearchArgs } from '$lib/types/glob'; - -export interface PathQuery { - parent: string; - last: string; -} - -/** - * Rewrite `\` into `/` when the query carries a Windows root. Elsewhere the - * backslash is left alone: it is a legal filename character on POSIX. - */ -function toPosixSeparators(query: string): string { - if (!GLOB.DRIVE_PREFIX_REGEX.test(query) && !query.startsWith(GLOB.WINDOWS_SEPARATOR)) - return query; - - return query.split(GLOB.WINDOWS_SEPARATOR).join(PATH_SEPARATOR); -} - -export function rootPrefixLength(path: string): number { - const unc = path.match(GLOB.UNC_ROOT_REGEX); - - if (unc) return unc[0].length; - - const drive = path.match(GLOB.DRIVE_ROOT_REGEX); - - if (drive) return drive[0].length; - - return path.startsWith(PATH_SEPARATOR) ? PATH_SEPARATOR.length : 0; -} - -/** A query starting from a root or from `~` is path navigation, not a home-relative glob. */ -export function splitPathQuery(query: string): PathQuery | null { - const normalized = toPosixSeparators(query); - const rootLength = rootPrefixLength(normalized); - - if (rootLength === 0 && !normalized.startsWith(HOME_TILDE)) return null; - - // a root keeps its trailing separator so it stays absolute on its own - const root = - rootLength > 0 - ? normalized.slice(0, rootLength).replace(TRAILING_SLASHES_REGEX, '') + PATH_SEPARATOR - : HOME_TILDE; - const rest = normalized - .slice(rootLength > 0 ? rootLength : HOME_TILDE.length) - .replace(LEADING_SLASHES_REGEX, '') - .replace(TRAILING_SLASHES_REGEX, ''); - const parentOf = (dirs: string) => - rootLength > 0 ? root + dirs : HOME_TILDE + PATH_SEPARATOR + dirs; - - if (!rest) return { last: '', parent: root }; - - const idx = rest.lastIndexOf(PATH_SEPARATOR); - - if (idx === -1) return { last: rest, parent: root }; - - return { last: rest.slice(idx + 1), parent: parentOf(rest.slice(0, idx)) }; -} - -export function buildCaseInsensitiveGlob(query: string): string { - let out = GLOB.WILDCARD; - - for (const c of query) { - const lo = c.toLowerCase(); - const up = c.toUpperCase(); - - if (lo !== up) out += GLOB.RANGE_OPEN + lo + up + GLOB.RANGE_CLOSE; - // glob metacharacters are escaped into a literal character class so a - // query like "a*b" matches a literal '*' instead of becoming "ab" - else if (GLOB.SPECIAL_CHARS.includes(c)) out += GLOB.RANGE_OPEN + c + GLOB.RANGE_CLOSE; - else out += c; - } - - return out + GLOB.WILDCARD; -} - -export function buildGlobSearchArgs( - query: string, - scopePath: string, - searchDepth: number -): GlobSearchArgs { - const pathQuery = splitPathQuery(query); - const path = pathQuery ? pathQuery.parent : scopePath; - const include = pathQuery - ? pathQuery.last - ? buildCaseInsensitiveGlob(pathQuery.last) - : GLOB.WILDCARD - : buildCaseInsensitiveGlob(query); - const maxDepth = pathQuery ? SEARCH.PATH_NAV_MAX_DEPTH : searchDepth; - - return { include, last: pathQuery?.last, maxDepth, path, rankQuery: pathQuery?.last ?? query }; -} - -const RANK_EXACT = 0; -const RANK_PREFIX = 1; -const RANK_SUBSTRING = 2; -const RANK_OTHER = 3; - -function rankScore(path: string, query: string): number { - const name = lastPathSegment(path).toLowerCase(); - const q = query.toLowerCase(); - - if (name === q) return RANK_EXACT; - - if (name.startsWith(q)) return RANK_PREFIX; - - if (name.includes(q)) return RANK_SUBSTRING; - - return RANK_OTHER; -} - -export function rankEntries(entries: GlobEntry[], query: string): GlobEntry[] { - return [...entries].sort( - (a, b) => - rankScore(a.path, query) - rankScore(b.path, query) || - a.path.length - b.path.length || - a.path.localeCompare(b.path) - ); -} - -export function joinPath(base: string, rel: string): string { - if (!base) return rel; - - return base.replace(TRAILING_SLASHES_REGEX, '') + PATH_SEPARATOR + rel; -} - -export function highlightMatch(text: string, query: string): { text: string; match: boolean }[] { - if (!query) return [{ match: false, text }]; - - const segments: { text: string; match: boolean }[] = []; - const lowerText = text.toLowerCase(); - const lowerQuery = query.toLowerCase(); - - let i = 0; - - while (i < text.length) { - const idx = lowerText.indexOf(lowerQuery, i); - - if (idx < 0) { - segments.push({ match: false, text: text.slice(i) }); - - break; - } - - if (idx > i) segments.push({ match: false, text: text.slice(i, idx) }); - - segments.push({ match: true, text: text.slice(idx, idx + query.length) }); - i = idx + query.length; - } - - return segments; -} diff --git a/tools/ui/src/routes/(chat)/+page.svelte b/tools/ui/src/routes/(chat)/+page.svelte index c4958b7c1..9db1d445f 100644 --- a/tools/ui/src/routes/(chat)/+page.svelte +++ b/tools/ui/src/routes/(chat)/+page.svelte @@ -1,20 +1,21 @@ - {conversationsStore.activeConversation?.name || 'Chat'} - {APP_NAME} + {activeConversation()?.name || 'Chat'} - {APP_NAME} - import { goto } from '$app/navigation'; import { page } from '$app/stores'; + import { goto } from '$app/navigation'; import { ServerErrorSplash } from '$lib/components/app'; - import { APP_NAME, ROUTES } from '$lib/constants'; + import { ROUTES } from '$lib/constants/routes'; + import { APP_NAME } from '$lib/constants'; let error = $derived($page.error); let status = $derived($page.status); diff --git a/tools/ui/src/routes/+layout.svelte b/tools/ui/src/routes/+layout.svelte index f938c7edf..ad2dd1560 100644 --- a/tools/ui/src/routes/+layout.svelte +++ b/tools/ui/src/routes/+layout.svelte @@ -1,40 +1,37 @@ diff --git a/tools/ui/tests/client/components/ChatFormInputRichHarness.svelte b/tools/ui/tests/client/components/ChatFormInputRichHarness.svelte deleted file mode 100644 index 58768a1e8..000000000 --- a/tools/ui/tests/client/components/ChatFormInputRichHarness.svelte +++ /dev/null @@ -1,27 +0,0 @@ - - - diff --git a/tools/ui/tests/client/components/ChatFormPickersHarness.svelte b/tools/ui/tests/client/components/ChatFormPickersHarness.svelte deleted file mode 100644 index e8ef6465b..000000000 --- a/tools/ui/tests/client/components/ChatFormPickersHarness.svelte +++ /dev/null @@ -1,51 +0,0 @@ - diff --git a/tools/ui/tests/client/components/ChatFormTestWrapper.svelte b/tools/ui/tests/client/components/ChatFormTestWrapper.svelte deleted file mode 100644 index 7ec8bf7f8..000000000 --- a/tools/ui/tests/client/components/ChatFormTestWrapper.svelte +++ /dev/null @@ -1,12 +0,0 @@ - - - - - diff --git a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte index 504f68597..14d938978 100644 --- a/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte +++ b/tools/ui/tests/client/components/ChatMessagesPerfWrapper.svelte @@ -2,8 +2,8 @@ // Mounts the real ChatMessages list against the real conversations store, so // the harness exercises `displayMessages` (which rebuilds every message's // toolMessages array) rather than a single message subtree. - import ChatMessages from '$lib/components/app/chat/ChatMessages/ChatMessages.svelte'; import * as Tooltip from '$lib/components/ui/tooltip'; + import ChatMessages from '$lib/components/app/chat/ChatMessages/ChatMessages.svelte'; import { conversationsStore } from '$lib/stores/conversations.svelte'; diff --git a/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte index b11ea0f10..53d3b9cfd 100644 --- a/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte +++ b/tools/ui/tests/client/components/CollapsibleLazyBodyHarness.svelte @@ -7,7 +7,7 @@ open: boolean; } - let { open, variant }: Props = $props(); + let { variant, open }: Props = $props(); {#if variant === 'content'} diff --git a/tools/ui/tests/client/components/McpServerFormWrapper.svelte b/tools/ui/tests/client/components/McpServerFormWrapper.svelte index 7bbabbc82..fe2cc958b 100644 --- a/tools/ui/tests/client/components/McpServerFormWrapper.svelte +++ b/tools/ui/tests/client/components/McpServerFormWrapper.svelte @@ -1,6 +1,6 @@ - -
conversation
- -{#if open} -
- it.id} - > - {#snippet item(it, index, isSelected)} - {}}> - {it.label} - - {/snippet} - -
-{/if} diff --git a/tools/ui/tests/client/components/TestWrapper.svelte b/tools/ui/tests/client/components/TestWrapper.svelte index 3c874276a..1380ec851 100644 --- a/tools/ui/tests/client/components/TestWrapper.svelte +++ b/tools/ui/tests/client/components/TestWrapper.svelte @@ -1,6 +1,6 @@