diff --git a/.github/workflows/kcpp-build-release-macos.yaml b/.github/workflows/kcpp-build-release-macos.yaml index bfed75028..fa20518e6 100644 --- a/.github/workflows/kcpp-build-release-macos.yaml +++ b/.github/workflows/kcpp-build-release-macos.yaml @@ -34,16 +34,22 @@ jobs: - name: Build id: make_build run: | - make LLAMA_METAL=1 LLAMA_PORTABLE=1 + make LLAMA_METAL=1 koboldcpp_default + mkdir -p build_artifacts + mv koboldcpp_default.so build_artifacts/ + make clean + make koboldcpp_macos_failsafe + mv koboldcpp_macos_failsafe.so koboldcpp_failsafe.so + mv build_artifacts/koboldcpp_default.so . chmod +x './create_ver_file.sh' . create_ver_file.sh - pyinstaller --noconfirm --onefile --collect-all customtkinter --collect-all jinja2 --collect-all psutil --add-data './koboldcpp_default.so:.' --add-data './ggml-metal-merged.metal:.' --add-data './kcpp_adapters:./kcpp_adapters' --add-data './koboldcpp.py:.' --add-data './json_to_gbnf.py:.' --add-data './LICENSE.md:.' --add-data './MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md:.' --add-data './embd_res:./embd_res' --version-file './version.txt' --clean --console koboldcpp.py -n "koboldcpp-mac-arm64" + pyinstaller --noconfirm --onefile --collect-all customtkinter --collect-all jinja2 --collect-all psutil --add-data './koboldcpp_default.so:.' --add-data './koboldcpp_failsafe.so:.' --add-data './ggml-metal-merged.metal:.' --add-data './kcpp_adapters:./kcpp_adapters' --add-data './koboldcpp.py:.' --add-data './json_to_gbnf.py:.' --add-data './LICENSE.md:.' --add-data './MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md:.' --add-data './embd_res:./embd_res' --version-file './version.txt' --clean --console koboldcpp.py -n "koboldcpp-mac-arm64" - name: Test id: test run: | wget https://huggingface.co/concedo/koboldcpp/resolve/main/baby_llama.gguf - dist/koboldcpp-mac-arm64 --model baby_llama.gguf --gpulayers 99 --benchmark --prompt 'Hi, my name is' + dist/koboldcpp-mac-arm64 --model baby_llama.gguf --gpulayers 99 --benchmark --prompt 'Once upon a' - name: Save artifact uses: actions/upload-artifact@v6 diff --git a/CMakeLists.txt b/CMakeLists.txt index 469367299..b3f782aaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,9 @@ if (MSVC) add_compile_options("$<$:/utf-8>") add_compile_options("$<$:/bigobj>") add_compile_options("$<$:/bigobj>") +else() + add_compile_options("$<$:-Wno-unused-value>") + add_compile_options("$<$:-Wno-unused-value>") endif() file(GLOB GGML_SOURCES_CUDA "ggml/src/ggml-cuda/*.cu") @@ -375,6 +378,16 @@ if (MINGW) add_compile_definitions(_WIN32_WINNT=0x602) endif() +# Standalone libmtmd build without pulling in the rest of the tools/ tree. +# Useful when packaging just the mtmd library for language bindings (e.g. an +# Apple XCFramework, or a WASM build). When the full tools build is enabled, +# mtmd is already built by the tools/ subdirectory above; this hook only fires +# when LLAMA_BUILD_TOOLS is OFF to avoid double-adding the target. +option(LLAMA_BUILD_MTMD "llama: build tools/mtmd library standalone" OFF) +if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS)) + add_subdirectory(tools/mtmd) +endif() + # # Build libraries # @@ -495,15 +508,19 @@ add_library(sdtype_adapter otherarch/sdcpp/src/core/ggml_graph_cut.cpp otherarch/sdcpp/src/core/ggml_graph_cut.h otherarch/sdcpp/examples/cli/image_metadata.cpp - otherarch/sdcpp/src/core/layer_registry.cpp - otherarch/sdcpp/src/core/layer_registry.h - otherarch/sdcpp/src/model.cpp + otherarch/sdcpp/src/model_manager.cpp + otherarch/sdcpp/src/model_manager.h + otherarch/sdcpp/src/extensions/pulid_extension.cpp + otherarch/sdcpp/src/model_loader.cpp + otherarch/sdcpp/src/extensions/photomaker_extension.cpp otherarch/sdcpp/src/runtime/sample-cache.cpp otherarch/sdcpp/src/core/util.cpp otherarch/sdcpp/src/name_conversion.cpp otherarch/sdcpp/src/upscaler.cpp otherarch/sdcpp/src/runtime/guidance.cpp otherarch/sdcpp/src/runtime/guidance.h + otherarch/sdcpp/src/runtime/imatrix.cpp + otherarch/sdcpp/src/runtime/imatrix.h otherarch/sdcpp/src/stable-diffusion.cpp otherarch/sdcpp/thirdparty/zip.c otherarch/sdcpp/src/model_io/gguf_io.cpp @@ -520,6 +537,8 @@ add_library(sdtype_adapter otherarch/sdcpp/src/tokenizers/gpt_oss_tokenizer.cpp otherarch/sdcpp/src/tokenizers/tokenizer.cpp otherarch/sdcpp/src/tokenizers/tokenize_util.cpp + otherarch/sdcpp/src/core/backend_fit.cpp + otherarch/sdcpp/src/core/layer_split_partition.cpp otherarch/sdcpp/src/core/ggml_extend_backend.cpp) target_include_directories(sdtype_adapter PUBLIC . ./ggml/include ./ggml/src ./ggml/src/ggml-cpu ./include ./otherarch ./otherarch/tools ./vendor/stb ./vendor/nlohmann ./vendor ./otherarch/sdcpp ./otherarch/sdcpp/include ./otherarch/sdcpp/src ./otherarch/sdcpp/examples ./tools ./common) target_compile_features(sdtype_adapter PUBLIC cxx_std_17) # don't bump @@ -555,7 +574,10 @@ target_link_libraries(embeddings_adapter PRIVATE common2 ggml ${LLAMA_EXTRA_LIBS set_target_properties(embeddings_adapter PROPERTIES POSITION_INDEPENDENT_CODE ON) add_library(gpttype_adapter - gpttype_adapter.cpp) + gpttype_adapter.cpp + src/llama.cpp + common/chat.cpp + src/llama-model.cpp) target_include_directories(gpttype_adapter PUBLIC . ./src ./ggml/include ./ggml/src ./ggml/src/ggml-cpu ./include ./otherarch ./otherarch/tools ./vendor/stb ./vendor/nlohmann ./vendor ./otherarch/sdcpp ./otherarch/sdcpp/thirdparty ./tools ./common) target_compile_features(gpttype_adapter PUBLIC cxx_std_17) # don't bump target_link_libraries(gpttype_adapter PRIVATE common2 ggml ggml_v1 ggml_v2 ggml_v3 ${LLAMA_EXTRA_LIBS}) diff --git a/Makefile b/Makefile index f0b075975..717c81de9 100644 --- a/Makefile +++ b/Makefile @@ -71,8 +71,8 @@ CXXFLAGS += -DGGML_USE_LLAMAFILE endif #lets try enabling everything -CFLAGS += -pthread -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable -CXXFLAGS += -pthread -Wno-multichar -Wno-write-strings -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable +CFLAGS += -pthread -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable -Wno-unused-value +CXXFLAGS += -pthread -Wno-multichar -Wno-write-strings -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable -Wno-unused-value LDFLAGS = @@ -101,9 +101,9 @@ NONECFLAGS = LLAMA_USE_BUNDLED_GLSLC := 1 FAILSAFE_FLAGS = -DUSE_FAILSAFE -VULKAN_FLAGS = -DGGML_USE_VULKAN -DSD_USE_VULKAN +VULKAN_FLAGS = -DGGML_USE_VULKAN ifdef LLAMA_CUBLAS -CUBLAS_FLAGS = -DGGML_USE_CUDA -DSD_USE_CUDA +CUBLAS_FLAGS = -DGGML_USE_CUDA else CUBLAS_FLAGS = endif @@ -215,7 +215,7 @@ OBJS_CUDA_TEMP_INST += \ ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.o ifdef LLAMA_CUBLAS -CUBLAS_FLAGS = -DGGML_USE_CUDA -DSD_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include +CUBLAS_FLAGS = -DGGML_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include CUBLASLD_FLAGS = -lcuda -lcublas -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L$(CUDA_PATH)/targets/x86_64-linux/lib -L$(CUDA_PATH)/lib64/stubs -L/usr/local/cuda/targets/aarch64-linux/lib -L/usr/local/cuda/targets/sbsa-linux/lib -L/usr/lib/wsl/lib CUBLAS_OBJS = ggml-cuda.o ggml_v3-cuda.o ggml_v2-cuda.o ggml_v2-cuda-legacy.o CUBLAS_OBJS += $(patsubst %.cu,%.o,$(filter-out ggml/src/ggml-cuda/ggml-cuda.cu, $(wildcard ggml/src/ggml-cuda/*.cu))) @@ -315,7 +315,7 @@ HIPFLAGS += -DGGML_HIP_NO_ROCWMMA_FATTN endif endif -HIPFLAGS += -DGGML_USE_HIP -DGGML_HIP_NO_VMM -DGGML_USE_CUDA -DSD_USE_CUDA $(shell $(ROCM_PATH)/bin/hipconfig -C) +HIPFLAGS += -DGGML_USE_HIP -DGGML_HIP_NO_VMM -DGGML_USE_CUDA $(shell $(ROCM_PATH)/bin/hipconfig -C) HIPLDFLAGS += -L$(ROCM_PATH)/lib -Wl,-rpath=$(ROCM_PATH)/lib HIPLDFLAGS += -L$(ROCM_PATH)/lib64 -Wl,-rpath=$(ROCM_PATH)/lib64 HIPLDFLAGS += -lhipblas -lamdhip64 -lrocblas @@ -339,8 +339,8 @@ endif # LLAMA_HIPBLAS ifdef LLAMA_METAL -CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG -DSD_USE_METAL -CXXFLAGS += -DGGML_USE_METAL -DSD_USE_METAL +CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG +CXXFLAGS += -DGGML_USE_METAL LDFLAGS += -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders OBJS += ggml-metal.o ggml-metal-device.o ggml-metal-device-m.o ggml-metal-context-m.o ggml-metal-common.o ggml-metal-ops.o @@ -682,7 +682,9 @@ ggml-vulkan-shaders-noext.o: ggml/src/ggml-vulkan-shaders-noext.cpp ggml/include $(CXX) $(CXXFLAGS) $(VKGEN_NOEXT_FORCE) $(VULKAN_FLAGS) -c $< -o $@ # intermediate objects -llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h +llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h + $(CXX) $(CXXFLAGS) -c $< -o $@ +llama-model.o: src/llama-model.cpp src/llama-model.h src/models/models.h ggml/include/ggml.h include/llama.h $(CXX) $(CXXFLAGS) -c $< -o $@ common.o: common/common.cpp common/common.h common/log.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -698,8 +700,10 @@ llama-impl.o: src/llama-impl.cpp src/llama-impl.h $(CXX) $(CXXFLAGS) -c $< -o $@ budget.o: common/reasoning-budget.cpp common/reasoning-budget.h $(CXX) $(CXXFLAGS) -c $< -o $@ +chat.o: common/chat.cpp common/chat.h + $(CXX) $(CXXFLAGS) -c $< -o $@ -SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_registry.cpp src/core/layer_registry.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model.cpp src/model/diffusion/anima.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.inl src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h +SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/diffusion/anima.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp @@ -730,6 +734,11 @@ otherarch/sdcpp/thirdparty/zip.o: otherarch/sdcpp/thirdparty/zip.c OBJS_SDTYPE := otherarch/sdcpp/sdtype_adapter.o $(OBJS_SDCOMMON) +LLAMASERVER_SRCS := tools/server/main.cpp tools/server/server.cpp tools/server/server-schema.cpp tools/server/server-chat.cpp tools/server/server-common.cpp tools/server/server-context.cpp tools/server/server-http.cpp tools/server/server-models.cpp tools/server/server-queue.cpp tools/server/server-task.cpp tools/server/server-tools.cpp tools/server/ui.cpp +COMMON_DOWNLOAD_SRCS := common/download.cpp common/hf-cache.cpp vendor/cpp-httplib/httplib.cpp +LLAMASERVER_COMMON_SRCS := common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) +LLAMASERVER_CXXFLAGS := -I./tools/mtmd + #whisper objects whispercpp_default.o: otherarch/whispercpp/whisper_adapter.cpp @@ -750,7 +759,7 @@ music_default.o: otherarch/acestep/music_adapter.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ # idiotic "for easier compilation" -GPTTYPE_ADAPTER = gpttype_adapter.cpp otherarch/llama_v2.cpp otherarch/llama_v3.cpp src/llama.cpp src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model.cpp src/llama-quant.cpp src/llama-hparams.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h +GPTTYPE_ADAPTER = gpttype_adapter.cpp model_adapter.h otherarch/otherarch.h include/llama.h otherarch/llama_v2.cpp otherarch/llama_v3.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp gpttype_adapter_failsafe.o: $(GPTTYPE_ADAPTER) $(CXX) $(CXXFLAGS) $(FAILSAFE_FLAGS) -c $< -o $@ gpttype_adapter.o: $(GPTTYPE_ADAPTER) @@ -763,39 +772,43 @@ gpttype_adapter_vulkan_noavx2.o: $(GPTTYPE_ADAPTER) $(CXX) $(CXXFLAGS) $(FAILSAFE_FLAGS) $(VULKAN_FLAGS) -c $< -o $@ clean: - rm -vf *.o main ttsmain sdmain whispermain quantize_gguf quantize_gpt2 quantize_gptj quantize_neox quantize_mpt vulkan-shaders-gen vulkan-shaders-gen-noext gguf-split mtmd-cli mainvk fitparams embedding embeddingvk qwen3tts rpcserver rpcserver.exe qwen3tts.exe embeddingvk.exe embedding.exe fitparams.exe mainvk.exe mtmd-cli.exe gguf-split.exe vulkan-shaders-gen.exe vulkan-shaders-gen-noext.exe main.exe ttsmain.exe sdmain.exe whispermain.exe quantize_gguf.exe quantize_gptj.exe quantize_gpt2.exe quantize_neox.exe quantize_mpt.exe koboldcpp_default.dll koboldcpp_failsafe.dll koboldcpp_noavx2.dll koboldcpp_vulkan_failsafe.dll koboldcpp_cublas.dll koboldcpp_hipblas.dll koboldcpp_vulkan.dll koboldcpp_vulkan_noavx2.dll koboldcpp_default.so koboldcpp_failsafe.so koboldcpp_noavx2.so koboldcpp_vulkan_failsafe.so koboldcpp_cublas.so koboldcpp_hipblas.so koboldcpp_vulkan.so koboldcpp_vulkan_noavx2.so ggml/src/ggml-vulkan-shaders.cpp ggml/src/ggml-vulkan-shaders.hpp ggml/src/ggml-vulkan-shaders-noext.cpp ggml/src/ggml-vulkan-shaders-noext.hpp + rm -vf *.o main ttsmain sdmain whispermain quantize_gguf quantize_gpt2 quantize_gptj quantize_neox quantize_mpt vulkan-shaders-gen vulkan-shaders-gen-noext gguf-split mtmd-cli mainvk fitparams embedding embeddingvk qwen3tts rpcserver llamaserver llamaservervk rpcserver.exe llamaserver.exe llamaservervk.exe qwen3tts.exe embeddingvk.exe embedding.exe fitparams.exe mainvk.exe mtmd-cli.exe gguf-split.exe vulkan-shaders-gen.exe vulkan-shaders-gen-noext.exe main.exe ttsmain.exe sdmain.exe whispermain.exe quantize_gguf.exe quantize_gptj.exe quantize_gpt2.exe quantize_neox.exe quantize_mpt.exe koboldcpp_default.dll koboldcpp_failsafe.dll koboldcpp_noavx2.dll koboldcpp_vulkan_failsafe.dll koboldcpp_cublas.dll koboldcpp_hipblas.dll koboldcpp_vulkan.dll koboldcpp_vulkan_noavx2.dll koboldcpp_default.so koboldcpp_failsafe.so koboldcpp_macos_failsafe.so koboldcpp_noavx2.so koboldcpp_vulkan_failsafe.so koboldcpp_cublas.so koboldcpp_hipblas.so koboldcpp_vulkan.so koboldcpp_vulkan_noavx2.so ggml/src/ggml-vulkan-shaders.cpp ggml/src/ggml-vulkan-shaders.hpp ggml/src/ggml-vulkan-shaders-noext.cpp ggml/src/ggml-vulkan-shaders-noext.hpp rm -vrf ggml/src/ggml-cuda/*.o rm -vrf ggml/src/ggml-cuda/template-instances/*.o rm -vrf llguidance rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o # useful tools -main: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mainvk: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttsmain: tools/tts/tts.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embedding: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) + $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) +llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) ggml/src/ggml-vulkan-shaders.cpp: ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp ifdef VULKAN_BUILD @@ -895,11 +908,14 @@ else endif #generated libraries -koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) + $(DEFAULT_BUILD) + +koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(DEFAULT_BUILD) ifdef FAILSAFE_BUILD -koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) +koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) $(FAILSAFE_BUILD) else koboldcpp_failsafe: @@ -907,7 +923,7 @@ koboldcpp_failsafe: endif ifdef NOAVX2_BUILD -koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(NOAVX2_BUILD) else koboldcpp_noavx2: @@ -915,7 +931,7 @@ koboldcpp_noavx2: endif ifdef CUBLAS_BUILD -koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) $(CUBLAS_BUILD) else koboldcpp_cublas: @@ -923,7 +939,7 @@ koboldcpp_cublas: endif ifdef HIPBLAS_BUILD -koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) $(HIPBLAS_BUILD) else koboldcpp_hipblas: @@ -931,12 +947,12 @@ koboldcpp_hipblas: endif ifdef VULKAN_BUILD -koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o llama.o chat.o llama-model.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(VULKAN_BUILD) ifdef NOAVX2_BUILD -koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(VULKAN_BUILD) -koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) +koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) $(VULKAN_BUILD) else koboldcpp_vulkan_noavx2: @@ -954,17 +970,17 @@ koboldcpp_vulkan_failsafe: endif # tools -quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) diff --git a/app/download.cpp b/app/download.cpp new file mode 100644 index 000000000..7227baadc --- /dev/null +++ b/app/download.cpp @@ -0,0 +1,71 @@ +#include "arg.h" +#include "common.h" +#include "download.h" +#include "log.h" + +#include +#include + +static void print_usage(int /*argc*/, char ** argv) { + printf( + "\nexamples:\n" + " %s -hf ggml-org/gemma-3-4b-it-qat-GGUF\n" + " %s -hf ggml-org/gemma-3-4b-it-qat-GGUF:Q4_K_M\n" + " %s -hf ggml-org/models -hff model.gguf\n" + " %s -mu https://example.com/model.gguf -m model.gguf\n" + "\n", + argv[0], argv[0], argv[0], argv[0] + ); +} + +int llama_download(int argc, char ** argv); + +int llama_download(int argc, char ** argv) { + common_init(); + + common_params params; + params.verbosity = LOG_LEVEL_ERROR; + + if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_DOWNLOAD, print_usage)) { + return 1; + } + + const bool has_source = !params.model.hf_repo.empty() || !params.model.url.empty() || + !params.model.path.empty() || !params.model.docker_repo.empty(); + if (!has_source) { + fprintf(stderr, "error: no model source specified (use --hf-repo, --model-url, --model or --docker-repo)\n"); + return 1; + } + + try { + common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_DOWNLOAD); + common_models_handler_apply(handler, params); + } catch (const std::exception & e) { + fprintf(stderr, "error: %s\n", e.what()); + return 1; + } + + if (!params.models_preset.empty()) { + // -hf pointed at a preset repo: print the preset path and stop + printf("%s\n", params.models_preset.c_str()); + return 0; + } + if (params.model.path.empty()) { + fprintf(stderr, "error: model download failed\n"); + return 1; + } + if (!std::filesystem::exists(params.model.path)) { + fprintf(stderr, "error: model file does not exist: %s\n", params.model.path.c_str()); + return 1; + } + + printf("%s\n", params.model.path.c_str()); + if (!params.mmproj.path.empty()) { + printf("%s\n", params.mmproj.path.c_str()); + } + if (!params.speculative.draft.mparams.path.empty()) { + printf("%s\n", params.speculative.draft.mparams.path.c_str()); + } + + return 0; +} diff --git a/colab.ipynb b/colab.ipynb index e5194a430..b8778a411 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -62,7 +62,7 @@ "Model = \"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\" #@param [\"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n", "MdCommand = \"\" #@markdown
\n", "Layers = \"Auto\" #@param [\"Auto\",\"999\"]{allow-input: true}\n", - "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\"] {allow-input: true}\n", + "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\",\"24576\",\"32768\",\"40960\"] {allow-input: true}\n", "\n", "#@markdown
\n", "LoadVisionMMProjector = False #@param {type:\"boolean\"}\n", @@ -130,7 +130,7 @@ " if Template == \"Gemma4 E4B Uncensored (General)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\"\n", - " CustomCtxSize = \"16384\"\n", + " CustomCtxSize = \"40960\"\n", " CustomMmproj = \"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\"\n", " if Template == \"Tiefighter 13B (General)\":\n", " Customized = True\n", diff --git a/common/arg.cpp b/common/arg.cpp index fda1dd52e..bf2b62460 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -18,6 +18,7 @@ # define NOMINMAX #endif #include +#include #endif #define JSON_ASSERT GGML_ASSERT @@ -286,108 +287,17 @@ static std::string clean_file_name(const std::string & fname) { return clean_fname; } -static bool common_params_handle_remote_preset(common_params & params, llama_example ex) { - GGML_ASSERT(!params.model.hf_repo.empty()); - - // the returned hf_repo is without tag - auto [hf_repo, hf_tag] = common_download_split_repo_tag(params.model.hf_repo); - - // "latest" tag (default if not specified) is translated to "default" preset - if (hf_tag == "latest") { - hf_tag = "default"; - } - - std::string model_endpoint = common_get_model_endpoint(); - auto preset_url = model_endpoint + hf_repo + "/resolve/main/preset.ini"; - - // prepare local path for caching - auto preset_fname = clean_file_name(hf_repo + "_preset.ini"); - auto preset_path = fs_get_cache_file(preset_fname); - common_download_opts opts; - opts.bearer_token = params.hf_token; - opts.offline = params.offline; - - LOG_TRC("%s: looking for remote preset at %s\n", __func__, preset_url.c_str()); - const int status = common_download_file_single(preset_url, preset_path, opts); - const bool has_preset = status >= 200 && status < 400; - - // remote preset is optional, so we don't error out if not found - if (has_preset) { - LOG_TRC("%s: applying remote preset from %s\n", __func__, preset_url.c_str()); - common_preset_context ctx(ex, /* only_remote_allowed */ true); - common_preset global; - auto remote_presets = ctx.load_from_ini(preset_path, global); - remote_presets = ctx.cascade(global, remote_presets); - if (remote_presets.find(hf_tag) != remote_presets.end()) { - common_preset preset = remote_presets.at(hf_tag); - LOG_INF("\n%s", preset.to_ini().c_str()); // to_ini already added trailing newline - preset.apply_to_params(params); - } else { - throw std::runtime_error("Remote preset.ini does not contain [" + std::string(hf_tag) + "] section"); - } - } else { - LOG_TRC("%s: no remote preset found, skipping\n", __func__); - } - - return has_preset; -} - struct handle_model_result { bool found_mmproj = false; common_params_model mmproj; bool found_mtp = false; common_params_model mtp; + + bool found_preset = false; + std::string preset_path; }; -static handle_model_result common_params_handle_model(struct common_params_model & model, - const common_download_opts & opts) { - handle_model_result result; - - if (!model.docker_repo.empty()) { - model.path = common_docker_resolve_model(model.docker_repo); - model.name = model.docker_repo; - } else if (!model.hf_repo.empty()) { - // If -m was used with -hf, treat the model "path" as the hf_file to download - if (model.hf_file.empty() && !model.path.empty()) { - model.hf_file = model.path; - model.path = ""; - } - common_download_opts hf_opts = opts; - auto download_result = common_download_model(model, hf_opts); - - if (download_result.model_path.empty()) { - throw std::runtime_error("failed to download model from Hugging Face"); - } - - model.name = model.hf_repo; - model.path = download_result.model_path; - - if (!download_result.mmproj_path.empty()) { - result.found_mmproj = true; - result.mmproj.path = download_result.mmproj_path; - } - - if (!download_result.mtp_path.empty()) { - result.found_mtp = true; - result.mtp.path = download_result.mtp_path; - } - } else if (!model.url.empty()) { - if (model.path.empty()) { - auto f = string_split(model.url, '#').front(); - f = string_split(f, '?').front(); - model.path = fs_get_cache_file(string_split(f, '/').back()); - } - - auto download_result = common_download_model(model, opts); - if (download_result.model_path.empty()) { - throw std::runtime_error("failed to download model from " + model.url); - } - } - - return result; -} - const std::vector kv_cache_types = { GGML_TYPE_F32, GGML_TYPE_F16, @@ -432,61 +342,243 @@ static bool parse_bool_value(const std::string & value) { } // -// CLI argument parsing functions +// common_models_handler // -bool common_params_handle_models(common_params & params, llama_example curr_ex) { - const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), - params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); +static std::string get_default_local_path(const std::string & url) { + auto f = string_split(url, '#').front(); + f = string_split(f, '?').front(); + return fs_get_cache_file(string_split(f, '/').back()); +} +common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) { + common_download_hf_plan plan; + common_download_hf_plan plan_spec; + common_download_hf_plan plan_voc; common_download_opts opts; + + const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), + params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + + // only download mmproj if the current example is using it + bool use_mmproj = false; + for (const auto & ex : mmproj_examples) { + if (curr_ex == ex) { + use_mmproj = true; + break; + } + } + opts.bearer_token = params.hf_token; opts.offline = params.offline; - opts.skip_download = params.skip_download; opts.download_mtp = spec_type_draft_mtp; - opts.download_mmproj = !params.no_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty(); + opts.download_mmproj = use_mmproj && !params.no_mmproj + && params.mmproj.path.empty() && params.mmproj.url.empty(); - // sub-models (draft, mmproj, vocoder) are explicitly specified by the user, - // so we should not auto-discover mtp/mmproj siblings for them - common_download_opts sub_opts = opts; - sub_opts.download_mtp = false; - sub_opts.download_mmproj = false; + if (!params.model.hf_repo.empty()) { + plan = common_download_get_hf_plan(params.model, opts); + } - try { - auto res = common_params_handle_model(params.model, opts); - if (params.no_mmproj) { - params.mmproj = {}; - } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) { - // optionally, handle mmproj model when -hf is specified - params.mmproj = res.mmproj; + if (!params.speculative.draft.mparams.hf_repo.empty()) { + plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts); + } + + if (!params.vocoder.model.hf_repo.empty()) { + plan_voc = common_download_get_hf_plan(params.vocoder.model, opts); + } + + return common_models_handler{plan, plan_spec, plan_voc, opts}; +} + +bool common_models_handler_is_preset_repo(const common_models_handler & handler) { + return !handler.plan.preset.url.empty(); +} + +static std::vector build_url_tasks(const common_params_model & model, common_download_opts opts) { + auto parts = common_download_get_all_parts(model.url); + std::vector tasks; + + // single-part: download straight to model.path if the user gave one (-m), else the cache default + if (parts.size() == 1) { + common_download_task task; + task.url = parts[0]; + task.local_path = model.path.empty() ? get_default_local_path(parts[0]) : model.path; + task.opts = opts; + tasks.push_back(std::move(task)); + return tasks; + } + + // multi-part: place each part under the user's -m directory (if given), else the cache default + std::string base_dir; + if (!model.path.empty()) { + auto pos = model.path.rfind('/'); + base_dir = pos == std::string::npos ? std::string(".") : model.path.substr(0, pos); + } + + for (const auto & part : parts) { + common_download_task task; + task.url = part; + task.opts = opts; + + std::string local = get_default_local_path(part); + if (!base_dir.empty()) { + auto pos = local.rfind('/'); + std::string name = pos == std::string::npos ? local : local.substr(pos + 1); + local = base_dir + "/" + name; } - // only download mmproj if the current example is using it - for (const auto & ex : mmproj_examples) { - if (curr_ex == ex) { - common_params_handle_model(params.mmproj, sub_opts); - break; + task.local_path = local; + tasks.push_back(std::move(task)); + } + return tasks; +} + +void common_models_handler_apply(common_models_handler & handler, common_params & params, common_download_callback * callback) { + std::vector tasks; + + auto & plan = handler.plan; + auto & plan_spec = handler.plan_spec; + auto & plan_voc = handler.plan_voc; + + auto opts = handler.opts; // copy + opts.callback = callback; + + // handle plain "url" if needed + auto handle_url = [&](common_params_model & model) { + if (!model.url.empty()) { + if (model.path.empty()) { + model.path = get_default_local_path(model.url); } } + }; + handle_url(params.model); + handle_url(params.mmproj); + handle_url(params.vocoder.model); + handle_url(params.speculative.draft.mparams); - // when --spec-type mtp is set and no draft model was provided explicitly, - // fall back to the MTP head discovered alongside the -hf model - if (spec_type_draft_mtp && res.found_mtp && - params.speculative.draft.mparams.path.empty() && - params.speculative.draft.mparams.hf_repo.empty() && - params.speculative.draft.mparams.url.empty()) { - params.speculative.draft.mparams.path = res.mtp.path; + // optionally, if docker repo is set, resolve it + if (!params.model.docker_repo.empty()) { + params.model.url = common_docker_resolve_model(params.model.docker_repo); + params.model.path = get_default_local_path(params.model.url); + } + + // handle plain "url" tasks (non-hf) + if (!params.model.url.empty()) { + auto url_tasks = build_url_tasks(params.model, opts); + // the first part is what gets loaded, so point params.model.path at it + if (!url_tasks.empty()) { + std::string first_path = url_tasks.front().local_path; + url_tasks.front().on_done = [&, first_path]() { params.model.path = first_path; }; + } + for (auto & task : url_tasks) { + tasks.push_back(std::move(task)); + } + } + if (!params.mmproj.url.empty()) { + common_download_task task; + task.url = params.mmproj.url; + task.local_path = params.mmproj.path; + task.opts = opts; + tasks.push_back(task); + } + if (!params.vocoder.model.url.empty()) { + common_download_task task; + task.url = params.vocoder.model.url; + task.local_path = params.vocoder.model.path; + task.opts = opts; + tasks.push_back(task); + } + if (!params.speculative.draft.mparams.url.empty()) { + common_download_task task; + task.url = params.speculative.draft.mparams.url; + task.local_path = params.speculative.draft.mparams.path; + task.opts = opts; + tasks.push_back(task); + } + + // handle hf_plan tasks + auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files, + const hf_cache::hf_file & primary, + common_params_model & model) { + for (size_t i = 0; i < model_files.size(); ++i) { + auto & model_file = model_files[i]; + bool is_primary = (model_file.path == primary.path); + tasks.emplace_back(model_file, opts, [&, is_primary]() { + if (is_primary) { + // the primary file is the first split (00001-of), use it as model path + model.path = hf_cache::finalize_file(model_file); + } else { + hf_cache::finalize_file(model_file); + } + }); + } + }; + if (!plan.model_files.empty()) { + add_tasks(plan.model_files, plan.primary, params.model); + } + if (!plan.mmproj.local_path.empty()) { + tasks.emplace_back(plan.mmproj, opts, [&]() { + params.mmproj.path = hf_cache::finalize_file(plan.mmproj); + }); + } + if (!plan.mtp.local_path.empty()) { + tasks.emplace_back(plan.mtp, opts, [&]() { + // only fall back to the discovered MTP head when no draft was explicitly provided + if (params.speculative.draft.mparams.empty()) { + params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.mtp); + } else { + hf_cache::finalize_file(plan.mtp); + } + }); + } + if (!plan.preset.local_path.empty()) { + tasks.emplace_back(plan.preset, opts, [&]() { + // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file + params.models_preset_hf = params.model.hf_repo; // only for showing a warning + params.models_preset = hf_cache::finalize_file(plan.preset); + params.model = common_params_model{}; // make sure to clear model, so server starts in router mode + }); + } + + // handle plan_spec (e.g. --spec-draft-hf) + if (!plan_spec.model_files.empty()) { + add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams); + } + + // handle vocoder plan (e.g. --hf-repo-v) + if (!plan_voc.model_files.empty()) { + add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model); + } + + // run all tasks in parallel + if (!params.offline) { + // if duplicated files are found, only download once (but still call on_done for each task) + std::unordered_map unique_tasks; + for (auto & task : tasks) { + auto it = unique_tasks.find(task.local_path); + if (it == unique_tasks.end()) { + unique_tasks[task.local_path] = &task; + } + } + std::vector unique_tasks_vec; + for (auto & pair : unique_tasks) { + unique_tasks_vec.push_back(*pair.second); + } + common_download_run_tasks(unique_tasks_vec); + } + + // download successful, update params with the downloaded paths + for (const auto & task : tasks) { + if (task.on_done) { + task.on_done(); } - common_params_handle_model(params.speculative.draft.mparams, sub_opts); - common_params_handle_model(params.vocoder.model, sub_opts); - return true; - } catch (const common_skip_download_exception &) { - return false; - } catch (const std::exception &) { - throw; } } +// +// CLI argument parsing functions +// + static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) { common_params & params = ctx_arg.params; @@ -602,30 +694,6 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context // parse the first time to get -hf option (used for remote preset) parse_cli_args(); - // export_graph_ops loads only metadata - const bool skip_model_download = ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS; - - // maybe handle remote preset - if (!params.model.hf_repo.empty() && !skip_model_download) { - std::string cli_hf_repo = params.model.hf_repo; - bool has_preset = common_params_handle_remote_preset(params, ctx_arg.ex); - - // special case: if hf_repo explicitly set by preset, we need to preserve it (ignore CLI value) - // this is useful when we have one HF repo pointing to other HF repos (one model - multiple GGUFs) - std::string preset_hf_repo = params.model.hf_repo; - bool preset_has_hf_repo = preset_hf_repo != cli_hf_repo; - - if (has_preset) { - // re-parse CLI args to override preset values - parse_cli_args(); - } - - // preserve hf_repo from preset if needed - if (preset_has_hf_repo) { - params.model.hf_repo = preset_hf_repo; - } - } - postprocess_cpu_params(params.cpuparams, nullptr); postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams); @@ -636,15 +704,26 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); } - // handle model and download - if (!skip_model_download) { - common_params_handle_models(params, ctx_arg.ex); - } + const bool skip_model_download = + // server will call common_params_handle_models() later, so we skip it here + ctx_arg.ex == LLAMA_EXAMPLE_SERVER || + // download calls common_params_handle_models() itself and prints the paths + ctx_arg.ex == LLAMA_EXAMPLE_DOWNLOAD || + // export_graph_ops loads only metadata + ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS; - // model is required (except for server) - // TODO @ngxson : maybe show a list of available models in CLI in this case - if (params.model.path.empty() && ctx_arg.ex != LLAMA_EXAMPLE_SERVER && !skip_model_download && !params.usage && !params.completion) { - throw std::invalid_argument("error: --model is required\n"); + if (!skip_model_download) { + // handle model and download + common_models_handler handler = common_models_handler_init(params, ctx_arg.ex); + common_models_handler_apply(handler, params); + + // model is required (except for server) + // TODO @ngxson : maybe show a list of available models in CLI in this case + if (params.model.path.empty() + && !params.usage + && !params.completion) { + throw std::invalid_argument("error: --model is required\n"); + } } if (params.escape) { @@ -708,15 +787,19 @@ static void common_params_print_usage(common_params_context & ctx_arg) { common_options.push_back(&opt); } } - printf("----- common params -----\n\n"); - print_options(common_options); - printf("\n\n----- sampling params -----\n\n"); - print_options(sampling_options); - printf("\n\n----- speculative params -----\n\n"); - print_options(spec_options); - // TODO: maybe convert enum llama_example to string - printf("\n\n----- example-specific params -----\n\n"); - print_options(specific_options); + bool first = true; + auto print_section = [&](const char * header, std::vector & options) { + if (options.empty()) { + return; + } + printf("%s----- %s -----\n\n", first ? "" : "\n\n", header); + first = false; + print_options(options); + }; + print_section("common params", common_options); + print_section("sampling params", sampling_options); + print_section("speculative params", spec_options); + print_section("example-specific params", specific_options); } static void common_params_print_completion(common_params_context & ctx_arg) { @@ -938,7 +1021,44 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map buf; + std::vector ptrs; +}; + +static utf8_argv make_utf8_argv() { + utf8_argv out; + int wargc = 0; + LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc); + if (!wargv) return out; + + out.buf.reserve(wargc); + for (int i = 0; i < wargc; ++i) { + int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr); + if (n <= 0) { out.buf.emplace_back(); continue; } + auto& s = out.buf.emplace_back(); + s.resize(static_cast(n - 1)); + (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr); + } + LocalFree(wargv); + + out.ptrs.reserve(out.buf.size() + 1); + for (auto& s : out.buf) out.ptrs.push_back(s.data()); + out.ptrs.push_back(nullptr); + return out; +} +#endif + bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) { +#ifdef _WIN32 + auto utf8 = make_utf8_argv(); + // repair argv only when it matches the process command line + if (static_cast(utf8.buf.size()) == argc) { + argv = utf8.ptrs.data(); + } +#endif + auto ctx_arg = common_params_parser_init(params, ex, print_usage); const common_params params_org = ctx_arg.params; // the example can modify the default params @@ -1079,7 +1199,9 @@ common_params_context common_params_parser_init(common_params & params, llama_ex * - if both {LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_*,} are set, we will prioritize the LLAMA_EXAMPLE_* matching current example */ auto add_opt = [&](common_arg arg) { - if ((arg.in_example(ex) || arg.in_example(LLAMA_EXAMPLE_COMMON)) && !arg.is_exclude(ex)) { + // download only exposes the handful of args explicitly tagged for it + const bool inherit_common = ex != LLAMA_EXAMPLE_DOWNLOAD; + if ((arg.in_example(ex) || (inherit_common && arg.in_example(LLAMA_EXAMPLE_COMMON))) && !arg.is_exclude(ex)) { ctx_arg.options.push_back(std::move(arg)); } }; @@ -1090,7 +1212,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.usage = true; } - )); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD})); add_opt(common_arg( {"--version"}, "show version and build info", @@ -2212,7 +2334,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, bool value) { params.no_mmproj = !value; } - ).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_AUTO")); + ).set_examples({LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MMPROJ_AUTO")); add_opt(common_arg( {"--mmproj-offload"}, {"--no-mmproj-offload"}, @@ -2244,6 +2366,13 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.image_max_tokens = value; } ).set_examples(mmproj_examples).set_env("LLAMA_ARG_IMAGE_MAX_TOKENS")); + add_opt(common_arg( + {"--mtmd-batch-max-tokens"}, "N", + string_format("maximum number of image tokens per batch when encoding images (default: %d)", params.mtmd_batch_max_tokens), + [](common_params & params, int value) { + params.mtmd_batch_max_tokens = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_MTMD_BATCH_MAX_TOKENS")); if (llama_supports_rpc()) { add_opt(common_arg( {"--rpc"}, "SERVERS", @@ -2604,14 +2733,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.path = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA}).set_env("LLAMA_ARG_MODEL")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL")); add_opt(common_arg( {"-mu", "--model-url"}, "MODEL_URL", "model download url (default: unused)", [](common_params & params, const std::string & value) { params.model.url = value; } - ).set_env("LLAMA_ARG_MODEL_URL")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL_URL")); add_opt(common_arg( { "-dr", "--docker-repo" }, "[/][:quant]", "Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.\n" @@ -2620,7 +2749,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.docker_repo = value; } - ).set_env("LLAMA_ARG_DOCKER_REPO")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_DOCKER_REPO")); add_opt(common_arg( {"-hf", "-hfr", "--hf-repo"}, "/[:quant]", "Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n" @@ -2630,14 +2759,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.hf_repo = value; } - ).set_env("LLAMA_ARG_HF_REPO")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_REPO")); add_opt(common_arg( {"-hff", "--hf-file"}, "FILE", "Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)", [](common_params & params, const std::string & value) { params.model.hf_file = value; } - ).set_env("LLAMA_ARG_HF_FILE")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_FILE")); add_opt(common_arg( {"-hfv", "-hfrv", "--hf-repo-v"}, "/[:quant]", "Hugging Face model repository for the vocoder model (default: unused)", @@ -2658,7 +2787,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.hf_token = value; } - ).set_env("HF_TOKEN")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("HF_TOKEN")); + add_opt(common_arg( + {"--mtp"}, + "also download the multi-token prediction (MTP) head, if available (default: unused)", + [](common_params & params) { + params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_DRAFT_MTP); + } + ).set_examples({LLAMA_EXAMPLE_DOWNLOAD})); add_opt(common_arg( {"--context-file"}, "FNAME", "file to load context from (use comma-separated values to specify multiple files)", @@ -2868,62 +3004,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.api_prefix = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_API_PREFIX")); - // Deprecated: use --ui-config instead (kept for backward compat) add_opt(common_arg( - {"--webui-config"}, "JSON", - "[DEPRECATED: use --ui-config] JSON that provides default WebUI settings (overrides WebUI defaults)", - [](common_params & params, const std::string & value) { - params.ui_config_json = value; - params.webui_config_json = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG")); - - add_opt(common_arg( - {"--ui-config"}, "JSON", + {"--ui-config", "--webui-config"}, "JSON", "JSON that provides default UI settings (overrides UI defaults)", [](common_params & params, const std::string & value) { params.ui_config_json = value; - params.webui_config_json = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG")); - - // Deprecated: use --ui-config-file instead (kept for backward compat) add_opt(common_arg( - {"--webui-config-file"}, "PATH", - "[DEPRECATED: use --ui-config-file] JSON file that provides default WebUI settings (overrides WebUI defaults)", - [](common_params & params, const std::string & value) { - params.ui_config_json = read_file(value); - params.webui_config_json = params.ui_config_json; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG_FILE")); - - add_opt(common_arg( - {"--ui-config-file"}, "PATH", + {"--ui-config-file", "--webui-config-file"}, "PATH", "JSON file that provides default UI settings (overrides UI defaults)", [](common_params & params, const std::string & value) { params.ui_config_json = read_file(value); - params.webui_config_json = params.ui_config_json; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG_FILE")); - - // Deprecated: use --ui-mcp-proxy instead (kept for backward compat) add_opt(common_arg( - {"--webui-mcp-proxy"}, - {"--no-webui-mcp-proxy"}, - "[DEPRECATED: use --ui-mcp-proxy/--no-ui-mcp-proxy] experimental: whether to enable MCP CORS proxy", - [](common_params & params, bool value) { - params.ui_mcp_proxy = value; - params.webui_mcp_proxy = value; - } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_MCP_PROXY")); - - add_opt(common_arg( - {"--ui-mcp-proxy"}, - {"--no-ui-mcp-proxy"}, + {"--ui-mcp-proxy", "--webui-mcp-proxy"}, + {"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"}, "experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)", [](common_params & params, bool value) { params.ui_mcp_proxy = value; - params.webui_mcp_proxy = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY")); add_opt(common_arg( @@ -2935,24 +3035,26 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.server_tools = parse_csv_row(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS")); - // Deprecated: use --ui/--no-ui instead (kept for backward compat) add_opt(common_arg( - {"--webui"}, - {"--no-webui"}, - "[DEPRECATED: use --ui/--no-ui] whether to enable the Web UI", + {"-ag", "--agent"}, + {"-no-ag", "--no-agent"}, + "whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)", [](common_params & params, bool value) { - params.ui = value; - params.webui = value; + if (value) { + params.server_tools = {"all"}; + params.ui_mcp_proxy = true; + } else { + params.server_tools.clear(); + params.ui_mcp_proxy = false; + } } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI")); - + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_AGENT")); add_opt(common_arg( - {"--ui"}, - {"--no-ui"}, + {"--ui", "--webui"}, + {"--no-ui", "--no-webui"}, string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"), [](common_params & params, bool value) { params.ui = value; - params.webui = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI")); add_opt(common_arg( @@ -2983,7 +3085,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_API_KEY")); add_opt(common_arg( {"--api-key-file"}, "FNAME", - "path to file containing API keys (default: none)", + "path to file containing API keys, one per line; lines starting with a hash are treated as comments (default: none)", [](common_params & params, const std::string & value) { std::ifstream key_file(value); if (!key_file) { @@ -2991,7 +3093,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } std::string key; while (std::getline(key_file, key)) { - if (!key.empty()) { + if (!key.empty() && key[0] != '#') { params.api_keys.push_back(key); } } @@ -3197,6 +3299,20 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.reasoning_budget_message = value; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_THINK_BUDGET_MESSAGE")); + add_opt(common_arg( + {"--reasoning-preserve"}, + {"--no-reasoning-preserve"}, + "preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n" + "compatible with certain templates having 'supports_preserve_reasoning' capability\n" + "example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking", + [](common_params & params, bool value) { + if (value) { + params.default_template_kwargs["preserve_reasoning"] = "true"; + } else { + params.default_template_kwargs["preserve_reasoning"] = "false"; + } + } + ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE")); add_opt(common_arg( {"--chat-template"}, "JINJA_TEMPLATE", string_format( @@ -3372,7 +3488,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.offline = true; } - ).set_env("LLAMA_ARG_OFFLINE")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE")); add_opt(common_arg( {"-lv", "--verbosity", "--log-verbosity"}, "N", string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n" @@ -3649,6 +3765,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex "draft model for speculative decoding (default: unused)", [](common_params & params, const std::string & value) { params.speculative.draft.mparams.path = value; + params.speculative.draft.mparams.hf_file = value; // will be used if --spec-draft-hf is set } ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL")); add_opt(common_arg( diff --git a/common/arg.h b/common/arg.h index 0010f2a9a..54a38b9cc 100644 --- a/common/arg.h +++ b/common/arg.h @@ -1,12 +1,14 @@ #pragma once #include "common.h" +#include "download.h" #include #include #include #include #include +#include // pseudo-env variable to identify preset-only arguments #define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP" @@ -129,11 +131,21 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map & args); -// populate model paths (main model, mmproj, etc) from -hf if necessary -// return true if the model is ready to use -// throw an exception if there is an error that prevents the model from being used (e.g. network error, model not found, etc) -// if params.skip_download is true, no downloads will be attempted. return false if the model is invalid or missing (e.g. ETag check failed) -bool common_params_handle_models(common_params & params, llama_example curr_ex); +struct common_models_handler { + common_download_hf_plan plan; + common_download_hf_plan plan_spec; + common_download_hf_plan plan_voc; + common_download_opts opts; +}; + +// initialize downloading opts and hf_plan if needed, but does not download anything yet +common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex); + +// check if the model is a preset repo (i.e. has a preset file) +bool common_models_handler_is_preset_repo(const common_models_handler & handler); + +// download and update params with the downloaded model path +void common_models_handler_apply(common_models_handler & handler, common_params & params, common_download_callback * callback = nullptr); // initialize argument parser context - used by test-arg-parser and preset common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr); diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index db3a6cc6f..36aab7ecb 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -103,6 +103,10 @@ common_chat_params peg_generator::generate_parser(const common_chat_template & data.grammar_triggers = { { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, trigger_marker } }; + if (autoparser.tools.format.openai_wrapper_trigger) { + // model emits the OpenAI function wrapper, trigger on it + data.grammar_triggers.push_back({ COMMON_GRAMMAR_TRIGGER_TYPE_WORD, "{\"type\": \"function\"," }); + } } } @@ -134,7 +138,7 @@ common_peg_arena autoparser::build_parser(const generation_params & inputs, cons auto response_format = p.rule("response-format", p.content(p.schema(p.json(), "response-format-schema", inputs.json_schema))); parser = ctx.reasoning_parser + p.space() + p.choice({ p.literal("```json") + p.space() + response_format + p.space() + p.literal("```"), - response_format + p.space() + response_format + p.space() }) + p.end(); pure_content = false; } else if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE && jinja_caps.supports_tool_calls) { @@ -224,13 +228,13 @@ common_peg_parser analyze_tools::build_tool_parser_json_native(parser_build_cont auto single_tool_parser = p.standard_json_tools( format.per_call_start, format.per_call_end, inputs.tools, inputs.parallel_tool_calls, inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped, - format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order); + format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger); tools_parser = p.trigger_rule("tool-calls", p.one_or_more(single_tool_parser + p.space())); } else { tools_parser = p.standard_json_tools( format.section_start, format.section_end, inputs.tools, inputs.parallel_tool_calls, inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED, name_field, args_field, format.tools_array_wrapped, - format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order); + format.fun_name_is_key, format.id_field, format.gen_id_field, format.parameter_order, format.openai_wrapper_trigger); } // Handle content wrappers if present @@ -391,11 +395,11 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte arguments.name_suffix) + arguments.value_prefix + (schema_info.resolves_to_string(param_schema) ? - p.tool_arg_string_value(until_suffix) : - p.tool_arg_json_value(p.schema( + p.ac(p.tool_arg_string_value(until_suffix) + + p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) : + (p.tool_arg_json_value(p.schema( p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) + - p.space()) + - p.tool_arg_close(p.literal(arguments.value_suffix))); + p.tool_arg_close(p.literal(arguments.value_suffix))))); auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); if (is_required) { diff --git a/common/chat-auto-parser.h b/common/chat-auto-parser.h index 7858f6572..9e8113f24 100644 --- a/common/chat-auto-parser.h +++ b/common/chat-auto-parser.h @@ -181,6 +181,7 @@ struct tool_format_analysis { bool fun_name_is_key = false; // In JSON format function name is JSON key, i.e. { "": { ... arguments ... } } bool tools_array_wrapped = false; // Tool calls wrapped in JSON array [...] + bool openai_wrapper_trigger = false; // model emits the OpenAI function wrapper, trigger on it std::string function_field = "function"; std::string name_field = "name"; diff --git a/common/chat-diff-analyzer.cpp b/common/chat-diff-analyzer.cpp index 0875c5347..b166ee5a1 100644 --- a/common/chat-diff-analyzer.cpp +++ b/common/chat-diff-analyzer.cpp @@ -165,6 +165,14 @@ static std::vector void { + if (tmpl.src.find("Respond in the format {\"name\": function name") != std::string::npos && + tmpl.src.find("Do not use variables.") != std::string::npos) { + analysis.tools.format.openai_wrapper_trigger = true; + LOG_DBG(ANSI_ORANGE "[Patch: JSON name/parameters tool instruction]\n" ANSI_RESET); + } + }, }); @@ -1229,8 +1237,8 @@ void analyze_tools::extract_argument_name_markers() { left_result.tags["pre"] == right_result.tags["pre"] && left_result.tags["suffix"] == right_result.tags["suffix"]) { // Name is inside a structure (e.g., JSON key): prefix is the shared wrapper - arguments.name_prefix = trim_whitespace(left_result.tags["pre"]); - arguments.name_suffix = trim_leading_whitespace(left_result.tags["suffix"]); + arguments.name_prefix = left_result.tags["pre"]; + arguments.name_suffix = left_result.tags["suffix"]; } else if (diff.left.substr(0, ARG_FIRST.length()) == ARG_FIRST && diff.right.substr(0, ARG_SECOND.length()) == ARG_SECOND) { // Name is directly in the diff: prefix comes from last marker in diff.prefix auto pre_parser = build_tagged_peg_parser([&](common_peg_parser_builder & p) { @@ -1315,8 +1323,7 @@ void analyze_tools::extract_argument_value_markers() { value_suffix = value_suffix.substr(0, end_marker_pos); } } - value_suffix = trim_leading_whitespace(value_suffix); - if (!value_suffix.empty()) { + if (!trim_whitespace(value_suffix).empty()) { arguments.value_suffix = value_suffix; } } diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index 9bc5ac98b..a309f0276 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -363,7 +363,7 @@ void common_chat_peg_mapper::map(const common_peg_ast_node & node) { } if ((is_arg_value || is_arg_string_value) && current_tool) { - std::string value_content = std::string(trim_trailing_space(trim_leading_space(node.text, 1), 1)); + std::string value_content = std::string(node.text); std::string value_to_add; if (value_content.empty() && is_arg_string_value) { @@ -540,10 +540,11 @@ common_peg_parser common_chat_peg_builder::python_style_tool_calls( auto arg_name_parser = literal(prop_name); common_peg_parser arg_value_parser = eps(); - auto string_value_parser = choice({ - literal("\"") + tool_arg_string_value(string_content('"')) + literal("\""), - literal("'") + tool_arg_string_value(string_content('\'')) + literal("'") - }); + // Quoted literal as a value: normalize_quotes_to_json preserves escapes. + auto string_value_parser = tool_arg_value(choice({ + literal("\"") + string_content('"') + literal("\""), + literal("'") + string_content('\'') + literal("'") + })); if (is_string_type) { arg_value_parser = string_value_parser; @@ -745,7 +746,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys( const std::string & effective_args_key, const std::string & call_id_key, const std::string & gen_call_id_key, - const std::vector & parameters_order) { + const std::vector & parameters_order, + bool accept_openai_wrapper) { auto tool_choices = choice(); auto name_key_parser = literal("\"" + effective_name_key + "\""); @@ -807,7 +809,13 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys( return idx_a < idx_b; }); - auto ordered_body = tool_open(literal("{")) + space(); + // accept an optional leading "type": "function" field when the model emits the OpenAI wrapper + common_peg_parser type_field = eps(); + if (accept_openai_wrapper) { + type_field = optional(literal("\"type\"") + space() + literal(":") + space() + + literal("\"function\"") + space() + literal(",") + space()); + } + auto ordered_body = tool_open(literal("{")) + space() + type_field; for (size_t i = 0; i < parser_pairs.size(); i++) { ordered_body = ordered_body + parser_pairs[i].first; if (i < parser_pairs.size() - 1) { @@ -870,7 +878,8 @@ common_peg_parser common_chat_peg_builder::standard_json_tools( bool function_is_key, const std::string & call_id_key, const std::string & gen_call_id_key, - const std::vector & parameters_order) { + const std::vector & parameters_order, + bool accept_openai_wrapper) { if (!tools.is_array() || tools.empty()) { return eps(); } @@ -888,7 +897,7 @@ common_peg_parser common_chat_peg_builder::standard_json_tools( if (!name_spec.first.empty() || !args_spec.first.empty()) { tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key); } else { - tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order); + tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper); } } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index a4643fbea..b3ffd7de2 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -120,7 +120,8 @@ class common_chat_peg_builder : public common_peg_parser_builder { bool function_is_key = false, const std::string & call_id_key = "", const std::string & gen_call_id_key = "", - const std::vector & parameters_order = {}); + const std::vector & parameters_order = {}, + bool accept_openai_wrapper = false); // Legacy-compatible helper for building XML/tagged style tool calls // Used by tests and manual parsers @@ -157,7 +158,8 @@ class common_chat_peg_builder : public common_peg_parser_builder { const std::string & effective_args_key, const std::string & call_id_key, const std::string & gen_call_id_key, - const std::vector & parameters_order); + const std::vector & parameters_order, + bool accept_openai_wrapper); }; inline common_peg_arena build_chat_peg_parser( diff --git a/common/chat.cpp b/common/chat.cpp index ca8cd040f..28fba2ff7 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -7,8 +7,6 @@ #include "ggml.h" #include "json-schema-to-grammar.h" #include "log.h" -#include "json-partial.cpp" -#include "regex-partial.cpp" #include "reasoning-budget.h" #include "chat-auto-parser-generator.cpp" #include "chat-auto-parser-helpers.cpp" @@ -101,41 +99,93 @@ std::string common_chat_msg::render_content(const std::string & delimiter) const return text; } -std::vector common_chat_split_by_role(const std::string & prompt, const std::vector & delims) { - if (delims.empty() || prompt.empty()) { - return {}; +common_chat_role common_chat_role_from_string(const std::string & role) { + if (role == "system") { return COMMON_CHAT_ROLE_SYSTEM; } + if (role == "assistant") { return COMMON_CHAT_ROLE_ASSISTANT; } + if (role == "user") { return COMMON_CHAT_ROLE_USER; } + if (role == "tool") { return COMMON_CHAT_ROLE_TOOL; } + return COMMON_CHAT_ROLE_UNKNOWN; +} + +const char * common_chat_role_to_string(common_chat_role role) { + switch (role) { + case COMMON_CHAT_ROLE_SYSTEM: return "system"; + case COMMON_CHAT_ROLE_ASSISTANT: return "assistant"; + case COMMON_CHAT_ROLE_USER: return "user"; + case COMMON_CHAT_ROLE_TOOL: return "tool"; + case COMMON_CHAT_ROLE_UNKNOWN: return ""; + } + return ""; +} + +json common_chat_msg_delimiters::to_json() const { + json result = json::array(); + for (const auto & d : delimiters) { + result.push_back({ + { "role", common_chat_role_to_string(d.role) }, + { "delimiter", d.delimiter }, + }); + } + return result; +} + +common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) { + common_chat_msg_delimiters result; + + if (!delimiters.is_array()) { + return result; } - auto parser = build_peg_parser([&](common_peg_parser_builder & p) { - std::vector all_delims; - std::vector tagged_messages; - - all_delims.reserve(delims.size()); - tagged_messages.reserve(delims.size()); - for (const auto & d : delims) { - all_delims.push_back(d.delimiter); + result.delimiters.reserve(delimiters.size()); + for (const auto & d : delimiters) { + if (!d.is_object()) { + continue; } - - auto any_delim = p.until_one_of(all_delims); - for (const auto & d : delims) { - tagged_messages.push_back(p.tag(d.role, p.literal(d.delimiter) + any_delim)); - } - - return any_delim + p.zero_or_more(p.choice(tagged_messages)) + p.end(); - }); - - common_peg_parse_context ctx(prompt); - const auto result = parser.parse(ctx); - if (!result.success()) { - return {}; + result.delimiters.push_back({ + common_chat_role_from_string(d.value("role", std::string())), + d.value("delimiter", std::string()), + }); } - std::vector spans; - ctx.ast.visit(result, [&](const common_peg_ast_node & node) { - if (!node.tag.empty()) { - spans.push_back({ node.tag, node.start, node.end - node.start }); + return result; +} + +void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) { + for (auto & d : delimiters) { + d.tokens = common_tokenize(vocab, d.delimiter, false, true); + } +} + +common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tokens, const std::map & skips) const { + std::vector> matches; + + auto skip = skips.begin(); + for (size_t i = 0; i < tokens.size();) { + if (skip != skips.end() && i == skip->first) { + i += skip->second; + ++skip; + continue; } - }); + for (const auto & d : delimiters) { + if (i + d.tokens.size() > tokens.size()) { + continue; + } + if (std::equal(d.tokens.begin(), d.tokens.end(), tokens.begin() + i)) { + matches.emplace_back(d.role, i); + break; + } + } + i++; + } + + matches.emplace_back(COMMON_CHAT_ROLE_UNKNOWN, tokens.size()); + + common_chat_msg_spans spans; + for (size_t i = 0; i + 1 < matches.size(); i++) { + const auto & curr = matches[i]; + const auto & next = matches[i + 1]; + spans.add(curr.first, curr.second, next.second - curr.second); + } return spans; } @@ -875,6 +925,10 @@ static std::string common_chat_template_direct_apply_impl( if (inputs.add_generation_prompt) { inp["add_generation_prompt"] = true; } + if (inp.contains("preserve_reasoning") && inp["preserve_reasoning"].is_boolean()) { + bool enabled = inp["preserve_reasoning"].get(); + jinja::caps_apply_preserve_reasoning(ctx, enabled); + } jinja::global_from_json(ctx, inp, inputs.mark_input); @@ -1096,13 +1150,13 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp data.prompt = prompt; data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); - data.message_spans = common_chat_split_by_role(prompt, { - { "assistant", "<|start|>assistant" }, - { "user", "<|start|>user" }, - { "system", "<|start|>developer" }, - { "system", "<|start|>system" }, - { "tool", "<|start|>functions" }, - }); + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, + { COMMON_CHAT_ROLE_USER, "<|start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, + { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" }, + }; data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; @@ -1243,10 +1297,10 @@ static common_chat_params common_chat_params_init_gemma4(const common_chat_templ data.prompt += data.generation_prompt; } - data.message_spans = common_chat_split_by_role(data.prompt, { - { "user", "<|turn>user\n" }, - { "assistant", "<|turn>model\n" }, - }); + data.message_delimiters = { + { COMMON_CHAT_ROLE_USER, "<|turn>user" }, + { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" }, + }; data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; data.supports_thinking = true; @@ -1994,6 +2048,146 @@ static common_chat_params common_chat_params_init_deepseek_v3_2(const common_cha return data; } +// Cohere2 MoE (a.k.a. "North Code") parser. +// +// The assistant turn is fully marker-wrapped: +// <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> +// <|START_THINKING|>{reasoning}<|END_THINKING|> +// then EITHER content: <|START_TEXT|>{content}<|END_TEXT|> +// OR tool calls: <|START_ACTION|>[ +// {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... +// ]<|END_ACTION|> +// <|END_OF_TURN_TOKEN|> +// +// The generation prompt forces a leading <|START_THINKING|> (when reasoning is enabled, which is +// the template default), so the model's output continues from *inside* the thinking block. The +// parser literal therefore only covers the stable <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|> prefix +// and the reasoning rule consumes the <|START_THINKING|> ... <|END_THINKING|> markers itself, +// regardless of whether they came from the generation prompt or the generated text. +static common_chat_params common_chat_params_init_cohere2moe(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string TURN_START = "<|START_OF_TURN_TOKEN|>"; + const std::string TURN_END = "<|END_OF_TURN_TOKEN|>"; + const std::string CHATBOT = "<|CHATBOT_TOKEN|>"; + const std::string USER = "<|USER_TOKEN|>"; + const std::string SYSTEM = "<|SYSTEM_TOKEN|>"; + const std::string THINK_START = "<|START_THINKING|>"; + const std::string THINK_END = "<|END_THINKING|>"; + const std::string TEXT_START = "<|START_TEXT|>"; + const std::string TEXT_END = "<|END_TEXT|>"; + const std::string ACTION_START = "<|START_ACTION|>"; + const std::string ACTION_END = "<|END_ACTION|>"; + const std::string RESULT_START = "<|START_TOOL_RESULT|>"; + const std::string RESULT_END = "<|END_TOOL_RESULT|>"; + + // Stable prefix of the generation prompt that precedes the (forced) <|START_THINKING|> marker. + const std::string GEN_PREFIX = TURN_START + CHATBOT; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.thinking_start_tag = THINK_START; + data.thinking_end_tag = THINK_END; + data.preserved_tokens = { + TURN_START, TURN_END, CHATBOT, USER, SYSTEM, + THINK_START, THINK_END, + TEXT_START, TEXT_END, + ACTION_START, ACTION_END, + RESULT_START, RESULT_END, + }; + + // Declare per-role message delimiters. Tool results are rendered with the + // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before + // the plain "system" one (it is a strict superset, and the role split tries delimiters in order). + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX }, + { COMMON_CHAT_ROLE_USER, TURN_START + USER }, + { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START }, + { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE; + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX + THINK_START + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += THINK_END + TEXT_START + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + auto end = p.end(); + + // The thinking block is always present (the generation prompt forces <|START_THINKING|>). + // When extracting reasoning, capture its body; otherwise keep the whole block (markers + // included) inline as content, matching reasoning_format=NONE conventions. + common_peg_parser reasoning = p.eps(); + if (extract_reasoning) { + reasoning = p.optional(p.literal(THINK_START) + + p.reasoning(p.until_one_of({ THINK_END, TEXT_START, ACTION_START })) + + p.optional(p.literal(THINK_END))); + } else { + reasoning = p.optional(p.content(p.literal(THINK_START) + + p.until_one_of({ THINK_END, TEXT_START, ACTION_START }) + + p.optional(p.literal(THINK_END)))); + } + + auto text_content = p.literal(TEXT_START) + p.content(p.until(TEXT_END)) + p.optional(p.literal(TEXT_END)); + + if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) { + return generation_prompt + reasoning + text_content + p.optional(p.literal(TURN_END)) + end; + } + + auto require_tools = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED; + + // <|START_ACTION|>[ {"tool_call_id": "0", "tool_name": "f", "parameters": {...}}, ... ]<|END_ACTION|> + auto tool_calls = p.standard_json_tools(ACTION_START, ACTION_END, inputs.tools, inputs.parallel_tool_calls, + /* force_tool_calls = */ true, + /* name_key = */ "tool_name", + /* args_key = */ "parameters", + /* array_wrapped = */ true, + /* function_is_key = */ false, + /* call_id_key = */ "", + /* gen_call_id_key = */ "tool_call_id", + /* parameters_order = */ { "tool_call_id", "tool_name", "parameters" }); + + // Content and tool calls are mutually exclusive in this format. + common_peg_parser body = require_tools ? tool_calls : p.choice({ tool_calls, text_content }); + + return generation_prompt + reasoning + body + p.optional(p.literal(TURN_END)) + end; + }); + + data.parser = parser.save(); + + if (include_grammar) { + data.grammar_lazy = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_AUTO; + data.grammar = build_grammar([&](const common_grammar_builder & builder) { + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + auto schema = function.at("parameters"); + builder.resolve_refs(schema); + }); + parser.build_grammar(builder, data.grammar_lazy); + }); + + data.grammar_triggers = { + { COMMON_GRAMMAR_TRIGGER_TYPE_WORD, ACTION_START } + }; + } + + return data; +} + namespace workaround { static void map_developer_role_to_system(json & messages) { @@ -2197,6 +2391,166 @@ static void func_args_not_string(json & messages) { } } +// Trim leading/trailing whitespace from message contents before rendering. This +// has to run on the messages (not on the rendered JSON) because templates with +// string-only content caps concatenate typed content parts into a single string +// during rendering, after which the per-part whitespace can no longer be reached. +// Both the plain string content and the text of typed content parts are trimmed. +static void trim_all_content(std::vector & messages) { + for (auto & message : messages) { + message.content = trim_whitespace(message.content); + message.reasoning_content = trim_whitespace(message.reasoning_content); + for (auto & part : message.content_parts) { + if (part.type == "text") { + part.text = trim_whitespace(part.text); + } + } + } +} + +} + +// MiniCPM5 format: +// - Reasoning: {reasoning} (optional) +// - Tool calls: value +static common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + data.supports_thinking = true; + data.preserved_tokens = { + "", + "", + "", + "", + }; + + data.thinking_start_tag = ""; + data.thinking_end_tag = ""; + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = "<|im_start|>assistant\n\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n\n\n" + msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal("<|im_start|>assistant\n"); + + auto reasoning = p.eps(); + if (extract_reasoning) { + reasoning = ("" << p.reasoning(p.until("")) << "") + p.space(); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); + } + + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + // CDATA lets a value carry characters that would otherwise close the tag (e.g. + // ); capture the inner text only, excluding the CDATA markers. + auto string_value = p.choice({ + p.literal("")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("")), + p.negate(p.literal("")) + p.tool_arg_close(p.literal("")), "") + }); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + const std::string name = function.at("name"); + auto params = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto args = p.eps(); + if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { + auto schema_info = common_schema_info(); + schema_info.resolve_refs(params); + + auto arg_choice = p.choice(); + for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { + auto value_parser = p.eps(); + if (schema_info.resolves_to_string(prop_schema)) { + value_parser = string_value; + } else { + value_parser = p.tool_arg_json_value( + p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) + ) + p.tool_arg_close(p.literal("")); + } + + auto arg_rule = p.tool_arg( + p.tool_arg_open(p.literal("")) + + value_parser + ); + + arg_choice |= arg_rule; + } + args = p.zero_or_more(arg_choice + p.space()); + } + + auto tool_parser = p.tool( + p.tool_open(p.literal("")) + << p.tool_args(args) + << p.tool_close(p.literal(""))); + + tool_choice |= p.rule("tool-" + name, tool_parser); + }); + + auto max_calls = inputs.parallel_tool_calls ? -1 : 1; + auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); + + auto content = p.content(p.until(" common_chat_try_specialized_template( return common_chat_params_init_kimi_k2(tmpl, params); } + // Cohere2 MoE / North Code - marker-wrapped format with <|START_TEXT|> content and + // <|START_ACTION|> JSON tool calls. <|START_TEXT|> is unique to this template (the older + // Command-R templates use <|START_RESPONSE|>). + if (src.find("<|START_TEXT|>") != std::string::npos && + src.find("<|START_ACTION|>") != std::string::npos) { + LOG_DBG("Using specialized template: Cohere2 MoE\n"); + return common_chat_params_init_cohere2moe(tmpl, params); + } + if (is_lfm2_template(src)) { LOG_DBG("Using specialized template: LFM2\n"); return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true); @@ -2282,6 +2645,14 @@ std::optional common_chat_try_specialized_template( return common_chat_params_init_gemma4(tmpl, params); } + // MiniCPM5 - XML tool calls with ... + if (src.find("Tool usage guidelines:") != std::string::npos && + src.find("template_tool_use ? *tmpls->template_tool_use : *tmpls->template_default; const auto & src = tmpl.source(); const auto & caps = tmpl.original_caps(); - params.messages = render_message_to_json(inputs.messages, tmpl.original_caps()); + std::vector trimmed_messages; + const std::vector * messages_to_render = &inputs.messages; + if (src.find("You have access to the following functions in JSONSchema format") != std::string::npos) { + // StepFun: trim message contents (including typed content parts) before rendering, + // otherwise leftover whitespace drives the model into reasoning loops (issue #24181) + trimmed_messages = inputs.messages; + workaround::trim_all_content(trimmed_messages); + messages_to_render = &trimmed_messages; + } + params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps()); params.tool_choice = inputs.tool_choice; params.reasoning_format = inputs.reasoning_format; params.enable_thinking = inputs.enable_thinking; @@ -2392,17 +2772,15 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ autoparser.analyze_template(tmpl); auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser); - std::vector delimiters; + common_chat_msg_delimiters delimiters; if (!autoparser.assistant_start.empty()) { - delimiters.push_back({ "assistant", autoparser.assistant_start }); + delimiters.add(COMMON_CHAT_ROLE_ASSISTANT, autoparser.assistant_start); } if (!autoparser.user_start.empty()) { - delimiters.push_back({ "user", autoparser.user_start }); + delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start); } - if (!delimiters.empty()) { - auto_params.message_spans = common_chat_split_by_role(auto_params.prompt, delimiters); - } + auto_params.message_delimiters = std::move(delimiters); auto_params.supports_thinking = autoparser.reasoning.mode != autoparser::reasoning_mode::NONE; if (auto_params.supports_thinking) { @@ -2544,8 +2922,9 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars } return msg; } - throw std::runtime_error(std::string("Failed to parse input at pos ") + std::to_string(result.end) + ": " + - effective_input.substr(result.end)); + LOG_WRN("%s: unparsed %s output: %s\n", __func__, common_chat_format_name(params.format), effective_input.substr(result.end).c_str()); + LOG_DBG("%s: full %s output triggering error:\n=== BEGIN ===\n%s\n=== END ===\n", __func__, common_chat_format_name(params.format), effective_input.c_str()); + throw std::runtime_error(std::string("The model produced output that does not match the expected ") + common_chat_format_name(params.format) + " format"); } common_chat_msg msg; @@ -2573,5 +2952,9 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars std::map common_chat_templates_get_caps(const common_chat_templates * chat_templates) { GGML_ASSERT(chat_templates != nullptr); GGML_ASSERT(chat_templates->template_default != nullptr); + if (chat_templates->template_tool_use != nullptr) { + // take the more expressive template when available + return chat_templates->template_tool_use->caps.to_map(); + } return chat_templates->template_default->caps.to_map(); } diff --git a/common/chat.h b/common/chat.h index 5659cd42a..7898f1623 100644 --- a/common/chat.h +++ b/common/chat.h @@ -143,15 +143,75 @@ struct common_chat_msg_diff { } }; +enum common_chat_role { + COMMON_CHAT_ROLE_UNKNOWN, + COMMON_CHAT_ROLE_SYSTEM, + COMMON_CHAT_ROLE_ASSISTANT, + COMMON_CHAT_ROLE_USER, + COMMON_CHAT_ROLE_TOOL +}; + +common_chat_role common_chat_role_from_string(const std::string & role); +const char * common_chat_role_to_string(common_chat_role role); + struct common_chat_msg_span { - std::string role; + common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; std::size_t pos = 0; std::size_t len = 0; + + bool valid() const { + return role != COMMON_CHAT_ROLE_UNKNOWN; + } +}; + +struct common_chat_msg_spans { + std::vector spans; + + void add(common_chat_role role, size_t pos, size_t len) { + spans.push_back({ role, pos, len }); + } + + bool is_user_start(int32_t pos) const { + for (auto it = spans.begin(); it != spans.end(); ++it) { + if (it->role == COMMON_CHAT_ROLE_USER && pos == (int32_t) it->pos) { + return true; + } + } + return false; + } + + int32_t last_user_message_pos() const { + for (auto it = spans.rbegin(); it != spans.rend(); ++it) { + if (it->role == COMMON_CHAT_ROLE_USER) { + return (int32_t) it->pos; + } + } + return -1; + } }; struct common_chat_msg_delimiter { - std::string role; - std::string delimiter; + common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; + std::string delimiter; + llama_tokens tokens = {}; +}; + +struct common_chat_msg_delimiters { + std::vector delimiters; + + common_chat_msg_delimiters() = default; + common_chat_msg_delimiters(std::initializer_list delims) : delimiters(delims) {} + + void add(common_chat_role role, const std::string & delimiter) { + delimiters.push_back({ role, delimiter }); + } + + void tokenize(const llama_vocab * vocab); + + // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching + common_chat_msg_spans split(const llama_tokens & tokens, const std::map & skips = {}) const; + + nlohmann::ordered_json to_json() const; }; struct common_chat_tool { @@ -219,7 +279,7 @@ struct common_chat_params { std::vector preserved_tokens; std::vector additional_stops; std::string parser; - std::vector message_spans; + common_chat_msg_delimiters message_delimiters; }; // per-message parsing syntax @@ -325,5 +385,4 @@ struct common_chat_prompt_preset { common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates); -std::vector common_chat_split_by_role(const std::string & prompt, const std::vector & delims); - +common_chat_msg_delimiters common_chat_msg_delimiters_parse(const nlohmann::ordered_json & delimiters); diff --git a/common/common.cpp b/common/common.cpp index 9fd0e9729..14f1a7892 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -231,7 +231,7 @@ bool set_process_priority(enum ggml_sched_priority prio) { } if (!SetPriorityClass(GetCurrentProcess(), p)) { - LOG_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError()); + COM_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError()); return false; } @@ -257,7 +257,7 @@ bool set_process_priority(enum ggml_sched_priority prio) { } if (setpriority(PRIO_PROCESS, 0, p) != 0) { - LOG_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno); + COM_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno); return false; } return true; @@ -290,14 +290,14 @@ void postprocess_cpu_params(common_cpu_params & cpuparams, const common_cpu_para if (n_set && n_set < cpuparams.n_threads) { // Not enough set bits, may experience performance issues. - LOG_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads); + COM_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads); } } bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) { size_t dash_loc = range.find('-'); if (dash_loc == std::string::npos) { - LOG_ERR("Format of CPU range is invalid! Expected []-[].\n"); + COM_ERR("%s", "Format of CPU range is invalid! Expected []-[].\n"); return false; } @@ -309,7 +309,7 @@ bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THRE } else { start_i = std::stoull(range.substr(0, dash_loc)); if (start_i >= GGML_MAX_N_THREADS) { - LOG_ERR("Start index out of bounds!\n"); + COM_ERR("%s", "Start index out of bounds!\n"); return false; } } @@ -319,7 +319,7 @@ bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THRE } else { end_i = std::stoull(range.substr(dash_loc + 1)); if (end_i >= GGML_MAX_N_THREADS) { - LOG_ERR("End index out of bounds!\n"); + COM_ERR("%s", "End index out of bounds!\n"); return false; } } @@ -339,7 +339,7 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD } size_t num_digits = mask.length() - start_i; - if (num_digits > 128) num_digits = 128; + num_digits = std::min(num_digits, 128); size_t end_i = num_digits + start_i; @@ -354,7 +354,7 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD } else if (c >= 'A' && c <= 'F') { id -= 'A' - 10; } else { - LOG_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i)); + COM_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i)); return false; } @@ -385,21 +385,21 @@ void common_params_print_info(const common_params & params, bool print_devices) #else const char * build_type = " (debug)"; #endif - LOG_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type); + COM_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type); - LOG_INF("log_info: verbosity = %d (adjust with the `-lv N` CLI arg)\n", common_log_get_verbosity_thold()); + COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, common_log_get_verbosity_thold()); // device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device if (print_devices) { - LOG_INF("device_info:\n"); + COM_TRC("%s", "device_info:\n"); for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { auto * dev = ggml_backend_dev_get(i); size_t free, total; ggml_backend_dev_memory(dev, &free, &total); - LOG_INF(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); + COM_TRC(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); } } - LOG_INF("%s\n", common_params_get_system_info(params).c_str()); + COM_TRC("%s\n", common_params_get_system_info(params).c_str()); } std::string common_params_get_system_info(const common_params & params) { @@ -666,7 +666,7 @@ void string_process_escapes(std::string & input) { bool string_parse_kv_override(const char * data, std::vector & overrides) { const char * sep = strchr(data, '='); if (sep == nullptr || sep - data >= 128) { - LOG_ERR("%s: malformed KV override '%s'\n", __func__, data); + COM_ERR("%s: malformed KV override '%s'\n", __func__, data); return false; } llama_model_kv_override kvo; @@ -689,20 +689,20 @@ bool string_parse_kv_override(const char * data, std::vector 127) { - LOG_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); + COM_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); return false; } strncpy(kvo.val_str, sep, 127); kvo.val_str[127] = '\0'; } else { - LOG_ERR("%s: invalid type for KV override '%s'\n", __func__, data); + COM_ERR("%s: invalid type for KV override '%s'\n", __func__, data); return false; } overrides.emplace_back(std::move(kvo)); @@ -1080,6 +1080,18 @@ std::vector fs_list(const std::string & path, bool include_dir return files; } +std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) { +#ifdef _WIN32 + int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0); + if (!wlen) { return std::ifstream(); } + std::vector wfname(wlen); + (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen); + return std::ifstream(wfname.data(), mode); +#else + return std::ifstream(fname, mode); +#endif +} + // // TTY utils // @@ -1193,8 +1205,8 @@ common_init_result::common_init_result(common_params & params, bool model_only) auto cparams = common_context_params_to_llama(params); if (params.fit_params) { - LOG_INF("%s: fitting params to device memory ...\n", __func__); - LOG_INF("%s: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n", __func__); + COM_TRC("%s", "fitting params to device memory ...\n"); + COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n"); common_fit_params(params.model.path.c_str(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), @@ -1221,7 +1233,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) llama_adapter_lora_ptr lora; lora.reset(llama_adapter_lora_init(model, la.path.c_str())); if (lora == nullptr) { - LOG_ERR("%s: failed to load lora adapter '%s'\n", __func__, la.path.c_str()); + COM_ERR("failed to load lora adapter '%s'\n", la.path.c_str()); pimpl->model.reset(model); return; } @@ -1240,14 +1252,14 @@ common_init_result::common_init_result(common_params & params, bool model_only) common_init_sampler_from_model(model, params.sampling); if (params.sampling.ignore_eos && llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) { - LOG_WRN("%s: warning: vocab does not have an EOS token, ignoring --ignore-eos\n", __func__); + COM_WRN("%s", "vocab does not have an EOS token, ignoring --ignore-eos\n"); params.sampling.ignore_eos = false; } // initialize once for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) { if (llama_vocab_is_eog(vocab, i)) { - LOG_TRC("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(vocab, i).c_str(), -INFINITY); + COM_TRC("added %s logit bias = %f\n", common_token_to_piece(vocab, i).c_str(), -INFINITY); params.sampling.logit_bias_eog.push_back({i, -INFINITY}); } } @@ -1285,7 +1297,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) llama_context * lctx = llama_init_from_model(model, cparams); if (lctx == NULL) { - LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); + COM_ERR("failed to create context with model '%s'\n", params.model.path.c_str()); return; } @@ -1322,7 +1334,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode llama_model * model = res->model(); if (model == NULL) { - LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str()); + COM_ERR("failed to load model '%s'\n", params.model.path.c_str()); return res; } @@ -1332,14 +1344,14 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode llama_context * lctx = res->context(); if (lctx == NULL) { - LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); + COM_ERR("failed to create context with model '%s'\n", params.model.path.c_str()); return res; } const llama_vocab * vocab = llama_model_get_vocab(model); if (params.ctx_shift && !llama_memory_can_shift(llama_get_memory(lctx))) { - LOG_WRN("%s: KV cache shifting is not supported for this context, disabling KV cache shifting\n", __func__); + COM_WRN("%s", "KV cache shifting is not supported for this context, disabling KV cache shifting\n"); params.ctx_shift = false; } @@ -1368,7 +1380,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode bool ok = true; if (llama_vocab_bos(vocab) == LLAMA_TOKEN_NULL) { - LOG_WRN("%s: warning: vocab does not have a BOS token, reranking will not work\n", __func__); + COM_WRN("%s", "vocab does not have a BOS token, reranking will not work\n"); ok = false; } @@ -1377,10 +1389,10 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode bool has_rerank_prompt = llama_model_chat_template(model, "rerank") != NULL; if (!has_eos && !has_sep && !has_rerank_prompt) { - LOG_WRN("%s: warning: vocab does not have an EOS token, SEP token, or rerank prompt. Reranking will not work\n", __func__); + COM_WRN("%s", "vocab does not have an EOS token, SEP token, or rerank prompt. Reranking will not work\n"); ok = false; } else if (!has_eos) { - LOG_WRN("%s: warning: vocab does not have an EOS token, using SEP token as fallback\n", __func__); + COM_WRN("%s", "vocab does not have an EOS token, using SEP token as fallback\n"); } if (!ok) { @@ -1393,7 +1405,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode } if (params.warmup) { - LOG_INF("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__); + COM_TRC("%s", "warming up the model with an empty run - please wait ... (--no-warmup to disable)\n"); std::vector tmp; llama_token bos = llama_vocab_bos(vocab); @@ -1467,20 +1479,20 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) { int ret = llama_decode(ctx, llama_batch_get_one(tmp.data(), tmp.size())); if (ret != 0) { - LOG_ERR("%s: llama_decode() failed: %d\n", __func__, ret); + COM_ERR("llama_decode() failed: %d\n", ret); res = COMMON_CONTEXT_SEQ_RM_TYPE_NO; goto done; } if (llama_n_rs_seq(ctx) > 0) { - LOG_INF("%s: the context supports bounded partial sequence removal\n", __func__); + COM_TRC("%s", "the context supports bounded partial sequence removal\n"); res = COMMON_CONTEXT_SEQ_RM_TYPE_RS; goto done; } // try to remove the last tokens if (!llama_memory_seq_rm(mem, 0, 1, -1)) { - LOG_TRC("%s: the context does not support partial sequence removal\n", __func__); + COM_TRC("%s", "the context does not support partial sequence removal\n"); res = COMMON_CONTEXT_SEQ_RM_TYPE_FULL; goto done; } @@ -1797,13 +1809,13 @@ static common_control_vector_data common_control_vector_load_one(const common_co }; struct gguf_context * ctx_gguf = gguf_init_from_file(load_info.fname.c_str(), meta_gguf_params); if (!ctx_gguf) { - LOG_ERR("%s: failed to load control vector file from %s\n", __func__, load_info.fname.c_str()); + COM_ERR("failed to load control vector file from %s\n", load_info.fname.c_str()); return result; } int32_t n_tensors = gguf_get_n_tensors(ctx_gguf); if (n_tensors == 0) { - LOG_WRN("%s: no direction tensors found in %s\n", __func__, load_info.fname.c_str()); + COM_WRN("no direction tensors found in %s\n", load_info.fname.c_str()); } for (int i = 0; i < n_tensors; i++) { @@ -1821,23 +1833,23 @@ static common_control_vector_data common_control_vector_load_one(const common_co } } if (layer_idx < 0) { - LOG_ERR("%s: invalid/unparsable direction tensor layer index in %s\n", __func__, load_info.fname.c_str()); + COM_ERR("invalid/unparsable direction tensor layer index in %s\n", load_info.fname.c_str()); result.n_embd = -1; break; } else if (layer_idx == 0) { - LOG_ERR("%s: invalid (zero) direction tensor layer index in %s\n", __func__, load_info.fname.c_str()); + COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str()); result.n_embd = -1; break; } struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str()); if (tensor->type != GGML_TYPE_F32) { - LOG_ERR("%s: invalid (non-F32) direction tensor type in %s\n", __func__, load_info.fname.c_str()); + COM_ERR("invalid (non-F32) direction tensor type in %s\n", load_info.fname.c_str()); result.n_embd = -1; break; } if (ggml_n_dims(tensor) != 1) { - LOG_ERR("%s: invalid (non-1D) direction tensor shape in %s\n", __func__, load_info.fname.c_str()); + COM_ERR("invalid (non-1D) direction tensor shape in %s\n", load_info.fname.c_str()); result.n_embd = -1; break; } @@ -1845,7 +1857,7 @@ static common_control_vector_data common_control_vector_load_one(const common_co if (result.n_embd == -1) { result.n_embd = ggml_nelements(tensor); } else if (ggml_nelements(tensor) != result.n_embd) { - LOG_ERR("%s: direction tensor in %s does not match previous dimensions\n", __func__, load_info.fname.c_str()); + COM_ERR("direction tensor in %s does not match previous dimensions\n", load_info.fname.c_str()); result.n_embd = -1; break; } @@ -1862,7 +1874,7 @@ static common_control_vector_data common_control_vector_load_one(const common_co } if (result.n_embd == -1) { - LOG_WRN("%s: skipping %s due to invalid direction tensors\n", __func__, load_info.fname.c_str()); + COM_WRN("skipping %s due to invalid direction tensors\n", load_info.fname.c_str()); result.data.clear(); } @@ -1883,7 +1895,7 @@ common_control_vector_data common_control_vector_load(const std::vector(all_tokens.data() + offset), n_tokens_before_last))) { - LOG_ERR("%s : failed to eval\n", __func__); + COM_ERR("%s", "failed to eval\n"); return false; } n_past += n_tokens_before_last; llama_state_save_file(ctx, state_path.data(), all_tokens.data(), all_tokens.size()); - LOG_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size()); + COM_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size()); llama_token last_token = all_tokens.back(); llama_batch batch = llama_batch_get_one(&last_token, 1); @@ -2024,13 +2036,13 @@ bool common_prompt_batch_decode( batch.pos = &pos; if (llama_decode(ctx, batch)) { - LOG_ERR("%s : failed to eval last token\n", __func__); + COM_ERR("%s", "failed to eval last token\n"); return false; } n_past++; } else { if (llama_decode(ctx, llama_batch_get_one(const_cast(all_tokens.data() + offset), n_new))) { - LOG_ERR("%s : failed to eval\n", __func__); + COM_ERR("%s", "failed to eval\n"); return false; } n_past += n_new; @@ -2040,7 +2052,7 @@ bool common_prompt_batch_decode( } size_t common_prompt_checkpoint::size() const { - return data_tgt.size() + data_dft.size(); + return data_tgt.size() + data_dft.size() + data_spec.size(); } bool common_prompt_checkpoint::empty() const { @@ -2055,6 +2067,7 @@ void common_prompt_checkpoint::clear() { data_tgt.clear(); data_dft.clear(); + data_spec.clear(); } void common_prompt_checkpoint::update_pos( @@ -2144,4 +2157,5 @@ void common_prompt_checkpoint::clear_tgt() { void common_prompt_checkpoint::clear_dft() { data_dft.clear(); + data_spec.clear(); } diff --git a/common/common.h b/common/common.h index 86646ba82..7939d3b07 100644 --- a/common/common.h +++ b/common/common.h @@ -26,6 +26,13 @@ #define DIRECTORY_SEPARATOR '/' #endif // _WIN32 +#define COM_DBG(fmt, ...) LOG_DBG("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define COM_TRC(fmt, ...) LOG_TRC("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define COM_INF(fmt, ...) LOG_INF("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define COM_WRN(fmt, ...) LOG_WRN("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define COM_ERR(fmt, ...) LOG_ERR("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define COM_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__) + #define die(msg) do { fputs("error: " msg "\n", stderr); exit(1); } while (0) #define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0) @@ -97,6 +104,7 @@ enum llama_example { LLAMA_EXAMPLE_FIT_PARAMS, LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS, + LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_COUNT, }; @@ -162,6 +170,7 @@ enum common_speculative_type { COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction + COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values @@ -291,12 +300,25 @@ struct common_params_sampling { }; struct common_params_model { - std::string path = ""; // model local path // NOLINT - std::string url = ""; // model url to download // NOLINT - std::string hf_repo = ""; // HF repo // NOLINT - std::string hf_file = ""; // HF file // NOLINT - std::string docker_repo = ""; // Docker repo // NOLINT - std::string name = ""; // in format /[:] (tag is optional) // NOLINT + std::string path = ""; // model local path + std::string url = ""; // model url to download + std::string hf_repo = ""; // HF repo + std::string hf_file = ""; // HF file + std::string docker_repo = ""; // Docker repo + + std::string get_name() const { + if (!hf_repo.empty()) { + return hf_repo; + } + if (!docker_repo.empty()) { + return docker_repo; + } + return path; + } + + bool empty() const { + return get_name().empty(); + } }; // draft-model-based speculative decoding parameters @@ -359,12 +381,12 @@ struct common_params_speculative { common_params_speculative_ngram_cache ngram_cache; bool has_dft() const { - return !draft.mparams.path.empty() || !draft.mparams.hf_repo.empty(); + return !draft.mparams.empty(); } uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH; }); return needs_rs_seq ? draft.n_max : 0u; @@ -511,7 +533,6 @@ struct common_params { int32_t control_vector_layer_start = -1; // layer range for control vector int32_t control_vector_layer_end = -1; // layer range for control vector bool offline = false; - bool skip_download = false; // skip model file downloading int32_t ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used. int32_t ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line @@ -576,6 +597,7 @@ struct common_params { std::vector image; // path to image file(s) ; TODO: change the name to "media" int image_min_tokens = -1; int image_max_tokens = -1; + int mtmd_batch_max_tokens = 1024; // finetune struct lr_opt lr; @@ -600,7 +622,7 @@ struct common_params { bool cache_prompt = true; // whether to enable prompt caching bool cache_idle_slots = true; // save and clear idle slots upon starting a new task int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot - int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints + int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string hostname = "127.0.0.1"; @@ -624,12 +646,6 @@ struct common_params { // UI configs bool ui = true; - - // Deprecated: use ui, ui_mcp_proxy, ui_config_json instead - bool webui = ui; - bool webui_mcp_proxy = false; - std::string webui_config_json; - bool ui_mcp_proxy = false; std::string ui_config_json; @@ -642,10 +658,11 @@ struct common_params { std::vector server_tools; // router server configs - std::string models_dir = ""; // directory containing models for the router server - std::string models_preset = ""; // directory containing model presets for the router server - int models_max = 4; // maximum number of models to load simultaneously - bool models_autoload = true; // automatically load models when requested via the router server + std::string models_dir = ""; // directory containing models for the router server + std::string models_preset = ""; // directory containing model presets for the router server + int models_max = 4; // maximum number of models to load simultaneously + bool models_autoload = true; // automatically load models when requested via the router server + std::string models_preset_hf = ""; // show a warning about remote presets on router loaded (if not empty) bool log_json = false; @@ -847,6 +864,9 @@ struct common_file_info { }; std::vector fs_list(const std::string & path, bool include_directories); +// fs open, also handle UTF8 on Windows +std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode); + // // TTY utils // @@ -1064,6 +1084,10 @@ struct common_prompt_checkpoint { std::vector data_tgt; std::vector data_dft; + // (optional) speculative-decoding implementation state stashed with the checkpoint + // (e.g. eagle3's deferred-boundary g_embd row) + std::vector data_spec; + size_t size() const; bool empty() const; diff --git a/common/download.cpp b/common/download.cpp index 72d4e068d..6b69a4418 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -21,9 +21,7 @@ #include #include -#if defined(LLAMA_USE_HTTPLIB) #include "http.h" -#endif #ifndef __EMSCRIPTEN__ #ifdef __linux__ @@ -117,7 +115,6 @@ std::pair common_download_split_repo_tag(const std::st return {hf_repo, tag}; } -#if defined(LLAMA_USE_HTTPLIB) class ProgressBar : public common_download_callback { static inline std::mutex mutex; static inline std::map lines; @@ -295,10 +292,6 @@ static int common_download_file_single_online(const std::string & url, const bool file_exists = std::filesystem::exists(path); - if (!file_exists && opts.skip_download) { - return -2; // file is missing and download is disabled - } - if (file_exists && skip_etag) { LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str()); return 304; // 304 Not Modified - fake cached response @@ -365,9 +358,6 @@ static int common_download_file_single_online(const std::string & url, return 304; // 304 Not Modified - fake cached response } // pass this point, the file exists but is different from the server version, so we need to redownload it - if (opts.skip_download) { - return -2; // special code to indicate that the download was skipped due to etag mismatch - } if (remove(path.c_str()) != 0) { LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str()); return -1; @@ -694,18 +684,8 @@ static void list_available_gguf_files(const hf_cache::hf_files & files) { } } -struct hf_plan { - hf_cache::hf_file primary; - hf_cache::hf_files model_files; - hf_cache::hf_file mmproj; - hf_cache::hf_file mtp; -}; - -static hf_plan get_hf_plan(const common_params_model & model, - const common_download_opts & opts, - bool download_mmproj, - bool download_mtp) { - hf_plan plan; +common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts) { + common_download_hf_plan plan; hf_cache::hf_files all; auto [repo, tag] = common_download_split_repo_tag(model.hf_repo); @@ -720,6 +700,14 @@ static hf_plan get_hf_plan(const common_params_model & model, return plan; } + // if preset.ini exists in the repo root, download only that file + for (const auto & f : all) { + if (f.path == "preset.ini") { + plan.preset = f; + return plan; + } + } + hf_cache::hf_file primary; if (!model.hf_file.empty()) { @@ -746,115 +734,49 @@ static hf_plan get_hf_plan(const common_params_model & model, plan.primary = primary; plan.model_files = get_split_files(all, primary); - if (download_mmproj) { + if (opts.download_mmproj) { plan.mmproj = find_best_mmproj(all, primary.path); } - - if (download_mtp) { + if (opts.download_mtp) { plan.mtp = find_best_mtp(all, primary.path); } return plan; } -struct download_task { - std::string url; - std::string path; -}; - -static std::vector get_url_tasks(const common_params_model & model) { - auto split = get_gguf_split_info(model.url); - - if (split.count <= 1) { - return {{model.url, model.path}}; - } - - auto filename = split.prefix; - if (auto pos = split.prefix.rfind('/'); pos != std::string::npos) { - filename = split.prefix.substr(pos + 1); - } - - auto parent_path = std::filesystem::path(model.path).parent_path(); - auto prefix_path = (parent_path / filename).string(); - - std::vector tasks; - for (int i = 1; i <= split.count; i++) { - auto suffix = string_format("-%05d-of-%05d.gguf", i, split.count); - tasks.push_back({split.prefix + suffix, prefix_path + suffix}); - } - return tasks; -} - -common_download_model_result common_download_model(const common_params_model & model, - const common_download_opts & opts) { - common_download_model_result result; - std::vector tasks; - hf_plan hf; - - bool download_mmproj = opts.download_mmproj; - bool download_mtp = opts.download_mtp; - bool is_hf = !model.hf_repo.empty(); - - if (is_hf) { - hf = get_hf_plan(model, opts, download_mmproj, download_mtp); - for (const auto & f : hf.model_files) { - tasks.push_back({f.url, f.local_path}); - } - if (!hf.mmproj.path.empty()) { - tasks.push_back({hf.mmproj.url, hf.mmproj.local_path}); - } - if (!hf.mtp.path.empty()) { - tasks.push_back({hf.mtp.url, hf.mtp.local_path}); - } - } else if (!model.url.empty()) { - tasks = get_url_tasks(model); - } else { - result.model_path = model.path; - return result; - } - - if (tasks.empty()) { - return result; - } - +void common_download_run_tasks(const std::vector & tasks) { std::vector> futures; for (const auto & task : tasks) { futures.push_back(std::async(std::launch::async, - [&task, &opts, is_hf]() { - return common_download_file_single(task.url, task.path, opts, is_hf); + [&task]() { + return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf); } )); } - for (auto & f : futures) { - int status = f.get(); - if (status == -2 && opts.skip_download) { - throw common_skip_download_exception(); - } + for (size_t i = 0; i < futures.size(); ++i) { + std::string url = tasks[i].url; + int status = futures[i].get(); bool is_ok = is_http_status_ok(status); if (!is_ok) { - return {}; + throw std::runtime_error(string_format("Download '%s' failed with status code: %d", url.c_str(), status)); } } +} - if (is_hf) { - for (const auto & f : hf.model_files) { - hf_cache::finalize_file(f); - } - result.model_path = hf.primary.final_path; +std::vector common_download_get_all_parts(const std::string & url) { + auto split = get_gguf_split_info(url); - if (!hf.mmproj.path.empty()) { - result.mmproj_path = hf_cache::finalize_file(hf.mmproj); - } - - if (!hf.mtp.path.empty()) { - result.mtp_path = hf_cache::finalize_file(hf.mtp); - } - } else { - result.model_path = model.path; + if (split.count <= 1) { + return {url}; } - return result; + std::vector parts; + for (int i = 1; i <= split.count; i++) { + auto suffix = string_format("-%05d-of-%05d.gguf", i, split.count); + parts.push_back(split.prefix + suffix); + } + return parts; } // @@ -1001,68 +923,86 @@ std::vector common_list_cached_models() { return result; } +bool common_download_remove(const std::string & hf_repo_with_tag) { + namespace fs = std::filesystem; -#else + auto [repo_id, tag] = common_download_split_repo_tag(hf_repo_with_tag); -// common_hf_file_res common_get_hf_file(const std::string &, const std::string &, bool, const common_header_list &) { -// throw std::runtime_error("download functionality is not enabled in this build"); -// } - -common_download_model_result common_download_model(const common_params_model & model, - const common_download_opts & opts) { - throw std::runtime_error("download functionality is not enabled in this build"); -} - -std::string common_docker_resolve_model(const std::string &) { - throw std::runtime_error("download functionality is not enabled in this build"); -} - -int common_download_file_single(const std::string & url, - const std::string & path, - const common_download_opts & opts, - bool skip_etag) { - throw std::runtime_error("download functionality is not enabled in this build"); -} - -struct gguf_split_info { - std::string prefix; // tag included - std::string tag; - int index; - int count; -}; -static gguf_split_info get_gguf_split_info(const std::string & path) { - static const std::regex re_split("^(.+)-([0-9]{5})-of-([0-9]{5})$", std::regex::icase); - static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); - std::smatch m; - - std::string prefix = path; - string_remove_suffix(prefix, ".gguf"); - - int index = 1; - int count = 1; - - if (std::regex_match(prefix, m, re_split)) { - index = std::stoi(m[2].str()); - count = std::stoi(m[3].str()); - prefix = m[1].str(); + if (tag.empty()) { + return hf_cache::remove_cached_repo(repo_id); } - std::string tag; - if (std::regex_search(prefix, m, re_tag)) { - tag = m[1].str(); - for (char & c : tag) { - c = std::toupper((unsigned char)c); + std::string tag_upper = tag; + for (char & c : tag_upper) { + c = (char) std::toupper((unsigned char) c); + } + + auto files = hf_cache::get_cached_files(repo_id); + if (files.empty()) { + return false; + } + + // collect snapshot entries whose tag matches + std::vector to_remove; + for (const auto & f : files) { + auto split = get_gguf_split_info(f.path); + if (split.tag == tag_upper) { + to_remove.emplace_back(f.local_path); } } - return {std::move(prefix), std::move(tag), index, count}; + if (to_remove.empty()) { + return false; + } + + // resolve blob paths from symlinks before deleting snapshot entries + std::vector blobs_to_check; + for (const auto & p : to_remove) { + std::error_code ec; + if (fs::is_symlink(p, ec)) { + auto target = fs::read_symlink(p, ec); + if (!ec) { + blobs_to_check.push_back((p.parent_path() / target).lexically_normal()); + } + } + } + + // remove snapshot entries + for (const auto & p : to_remove) { + std::error_code ec; + fs::remove(p, ec); + if (ec) { + LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str()); + } + } + + if (blobs_to_check.empty()) { + return true; + } + + // collect blobs still referenced by remaining snapshot entries + std::unordered_set still_referenced; + for (const auto & f : hf_cache::get_cached_files(repo_id)) { + fs::path p(f.local_path); + std::error_code ec; + if (fs::is_symlink(p, ec)) { + auto target = fs::read_symlink(p, ec); + if (!ec) { + still_referenced.insert((p.parent_path() / target).lexically_normal().string()); + } + } + } + + // remove orphaned blobs + for (const auto & blob : blobs_to_check) { + if (still_referenced.find(blob.string()) == still_referenced.end()) { + std::error_code ec; + fs::remove(blob, ec); + if (ec) { + LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str()); + } + } + } + + return true; } - -std::vector common_list_cached_models() { - std::vector result; - return result; -} - - -#endif // defined(LLAMA_USE_HTTPLIB) - diff --git a/common/download.h b/common/download.h index ebeedd605..6560a91b8 100644 --- a/common/download.h +++ b/common/download.h @@ -1,7 +1,11 @@ #pragma once +#include "hf-cache.h" + #include #include +#include +#include struct common_params_model; @@ -47,65 +51,40 @@ struct common_cached_model_info { } }; -// Options for common_download_model and common_download_file_single +// Options for common_download_file_single struct common_download_opts { std::string bearer_token; common_header_list headers; bool offline = false; - bool skip_download = false; // if true, only validation is performed, common_skip_download_exception may be thrown if the file is missing or invalid bool download_mmproj = false; bool download_mtp = false; common_download_callback * callback = nullptr; }; -// Result of common_download_model -struct common_download_model_result { - std::string model_path; - std::string mmproj_path; - std::string mtp_path; +struct common_download_task { + common_download_opts opts; + std::string url; + std::string local_path; + std::function on_done; + bool is_hf = false; + + common_download_task() = default; + common_download_task(hf_cache::hf_file f, + const common_download_opts & opts, + std::function on_done = nullptr) + : opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_hf(true) {} }; -// throw if the file is missing or invalid (e.g. ETag check failed) -struct common_skip_download_exception : public std::runtime_error { - common_skip_download_exception() : std::runtime_error("skip download") {} -}; +void common_download_run_tasks(const std::vector & tasks); -// Download model from HuggingFace repo or URL -// -// input (via model struct): -// - model.hf_repo: HF repo with optional tag, see common_download_split_repo_tag -// - model.hf_file: specific file in the repo (requires hf_repo) -// - model.url: simple download (used if hf_repo is empty) -// - model.path: local file path -// -// tag matching (for HF repos without model.hf_file): -// - if tag is specified, searches for GGUF matching that quantization -// - if no tag, searches for Q4_K_M, then Q4_0, then first available GGUF -// -// split GGUF: multi-part files like "model-00001-of-00003.gguf" are automatically -// detected and all parts are downloaded -// -// caching: -// - HF repos: uses HuggingFace cache -// - URLs: uses ETag-based caching -// -// when opts.offline=true, no network requests are made -// when download_mmproj=true, searches for mmproj in same directory as model or any parent directory -// then with the closest quantization bits -// when download_mtp=true, applies the same sibling search for an MTP-head GGUF -// -// returns result with model_path, mmproj_path and mtp_path (empty when not found / on failure) -common_download_model_result common_download_model( - const common_params_model & model, - const common_download_opts & opts = {} -); +// if url is a multi-part GGUF file, returns all parts, otherwise returns the single file +std::vector common_download_get_all_parts(const std::string & url); // returns list of cached models std::vector common_list_cached_models(); // download single file from url to local path // returns status code or -1 on error -// returns -2 if the download was skipped due to ETag mismatch (file outdated, skip_download=true) // skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash) int common_download_file_single(const std::string & url, const std::string & path, @@ -115,3 +94,19 @@ int common_download_file_single(const std::string & url, // resolve and download model from Docker registry // return local path to downloaded model file std::string common_docker_resolve_model(const std::string & docker); + +// Remove a cached model from disk +// input format: "user/model" or "user/model:tag" +// - if tag is omitted, removes the entire repo cache directory +// - if tag is present, removes only files matching that tag (and orphaned blobs) +// returns true if anything was removed +bool common_download_remove(const std::string & hf_repo_with_tag); + +struct common_download_hf_plan { + hf_cache::hf_file primary; + hf_cache::hf_files model_files; + hf_cache::hf_file mmproj; + hf_cache::hf_file mtp; + hf_cache::hf_file preset; // if set, only this file is downloaded +}; +common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); diff --git a/common/fit.cpp b/common/fit.cpp index 668d892e9..afbf0b10f 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -26,7 +26,7 @@ class common_params_fit_exception : public std::runtime_error { using std::runtime_error::runtime_error; }; -std::vector common_get_device_memory_data( +static std::vector common_get_device_memory_data_impl( const char * path_model, const llama_model_params * mparams, const llama_context_params * cparams, @@ -150,6 +150,29 @@ std::vector common_get_device_memory_data( return ret; } +common_device_memory_data_vec common_get_device_memory_data( + const char * path_model, + const llama_model_params * mparams, + const llama_context_params * cparams, + std::vector & devs, + uint32_t & hp_ngl, + uint32_t & hp_n_ctx_train, + uint32_t & hp_n_expert, + ggml_log_level log_level) { + std::vector impl = common_get_device_memory_data_impl( + path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level); + + common_device_memory_data_vec ret(impl.size()); + for (size_t i = 0; i < impl.size(); i++) { + ret[i].total = impl[i].total; + ret[i].free = impl[i].free; + ret[i].model = impl[i].mb.model; + ret[i].context = impl[i].mb.context; + ret[i].compute = impl[i].mb.compute; + } + return ret; +} + static void common_params_fit_impl( const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams, float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides, @@ -169,7 +192,7 @@ static void common_params_fit_impl( // step 1: get data for default parameters and check whether any changes are necessary in the first place LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__); - const dmds_t dmds_full = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + const dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); const size_t nd = devs.size(); // number of devices std::vector margins; // this function uses int64_t rather than size_t for memory sizes to more conveniently handle deficits @@ -210,7 +233,7 @@ static void common_params_fit_impl( sum_projected_used = dmds_full.back().mb.total(); sum_free = dmds_full.back().total; sum_projected_free = sum_free - sum_projected_used; - LOG_INF("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n", + LOG_TRC("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n", __func__, sum_projected_used/MiB, sum_free/MiB); if (sum_projected_free >= margins[0]) { LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of system memory, no changes needed\n", @@ -304,7 +327,7 @@ static void common_params_fit_impl( int64_t sum_projected_used_min_ctx = 0; cparams->n_ctx = n_ctx_min; - const dmds_t dmds_min_ctx = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + const dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); if (nd == 0) { sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total(); } else { @@ -482,7 +505,7 @@ static void common_params_fit_impl( llama_model_params mparams_copy = *mparams; set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy); - const dmds_t dmd_nl = common_get_device_memory_data( + const dmds_t dmd_nl = common_get_device_memory_data_impl( path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); LOG_TRC("%s: memory for test allocation by device:\n", func_name); @@ -510,7 +533,7 @@ static void common_params_fit_impl( mparams->tensor_buft_overrides = tensor_buft_overrides; LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__); - const dmds_t dmds_cpu_moe = common_get_device_memory_data( + const dmds_t dmds_cpu_moe = common_get_device_memory_data_impl( path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); for (size_t id = 0; id < nd; id++) { @@ -940,7 +963,7 @@ void common_fit_print( uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert - auto dmd = common_get_device_memory_data(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); + auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); GGML_ASSERT(dmd.size() == devs.size() + 1); for (size_t id = 0; id < devs.size(); id++) { diff --git a/common/fit.h b/common/fit.h index 643d34200..208fc3069 100644 --- a/common/fit.h +++ b/common/fit.h @@ -1,9 +1,7 @@ #pragma once #include "ggml.h" -#include "ggml-backend.h" #include "llama.h" -#include "../src/llama-ext.h" #include @@ -18,31 +16,41 @@ enum common_params_fit_status { // - this function is NOT thread safe because it modifies the global llama logger state // - only parameters that have the same value as in llama_default_model_params are modified // with the exception of the context size which is modified if and only if equal to 0 -enum common_params_fit_status common_fit_params( - const char * path_model, - struct llama_model_params * mparams, - struct llama_context_params * cparams, - float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements - struct llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements - size_t * margins, // margins of memory to leave per device in bytes - uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use - enum ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log +common_params_fit_status common_fit_params( + const char * path_model, + llama_model_params * mparams, + llama_context_params * cparams, + float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements + llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements + size_t * margins, // margins of memory to leave per device in bytes + uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use + ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log // print estimated memory to stdout void common_fit_print( - const char * path_model, - struct llama_model_params * mparams, - struct llama_context_params * cparams); + const char * path_model, + llama_model_params * mparams, + llama_context_params * cparams); -void common_memory_breakdown_print(const struct llama_context * ctx); +void common_memory_breakdown_print(const llama_context * ctx); + +struct common_device_memory_data { + int64_t total; + int64_t free; + size_t model; + size_t context; + size_t compute; +}; + +using common_device_memory_data_vec = std::vector; // Load a model + context with no_alloc and return the per-device memory breakdown. -std::vector common_get_device_memory_data( - const char * path_model, - const struct llama_model_params * mparams, - const struct llama_context_params * cparams, - std::vector & devs, - uint32_t & hp_ngl, - uint32_t & hp_n_ctx_train, - uint32_t & hp_n_expert, - enum ggml_log_level log_level); +common_device_memory_data_vec common_get_device_memory_data( + const char * path_model, + const llama_model_params * mparams, + const llama_context_params * cparams, + std::vector & devs, + uint32_t & hp_ngl, + uint32_t & hp_n_ctx_train, + uint32_t & hp_n_expert, + ggml_log_level log_level); diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index ba7417a12..f1dacaa47 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -495,4 +495,19 @@ std::string finalize_file(const hf_file & file) { return file.final_path; } +bool remove_cached_repo(const std::string & repo_id) { + if (!is_valid_repo_id(repo_id)) { + LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str()); + return false; + } + fs::path repo_path = get_repo_path(repo_id); + std::error_code ec; + auto removed = fs::remove_all(repo_path, ec); + if (ec) { + LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str()); + return false; + } + return removed > 0; +} + } // namespace hf_cache diff --git a/common/hf-cache.h b/common/hf-cache.h index 23fa0adb7..42c9c6ce3 100644 --- a/common/hf-cache.h +++ b/common/hf-cache.h @@ -29,4 +29,7 @@ hf_files get_cached_files(const std::string & repo_id = {}); // Create snapshot path (link or move/copy) and return it std::string finalize_file(const hf_file & file); +// Remove the entire cached directory for a repo, returns true if removed +bool remove_cached_repo(const std::string & repo_id); + } // namespace hf_cache diff --git a/common/http.h b/common/http.h index d3daccd6b..e88bc6a5e 100644 --- a/common/http.h +++ b/common/http.h @@ -11,6 +11,11 @@ struct common_http_url { std::string path; }; +// bracket an IPv6 literal host for a URL authority (RFC 3986) +static std::string common_http_format_host(const std::string & host) { + return host.find(':') != std::string::npos ? "[" + host + "]" : host; +} + static common_http_url common_http_parse_url(const std::string & url) { common_http_url parts; auto scheme_end = url.find("://"); @@ -49,11 +54,28 @@ static common_http_url common_http_parse_url(const std::string & url) { parts.path = "/"; } - auto colon_pos = parts.host.find(':'); + // split the authority into host and optional port, a bracketed IPv6 literal keeps its inner colons (RFC 3986) + std::string port_str; + if (!parts.host.empty() && parts.host.front() == '[') { + auto close = parts.host.find(']'); + if (close == std::string::npos) { + throw std::runtime_error("invalid IPv6 URL authority: " + parts.host); + } + auto after = parts.host.substr(close + 1); + if (!after.empty() && after.front() == ':') { + port_str = after.substr(1); + } + parts.host = parts.host.substr(1, close - 1); + } else { + auto colon_pos = parts.host.find(':'); + if (colon_pos != std::string::npos) { + port_str = parts.host.substr(colon_pos + 1); + parts.host = parts.host.substr(0, colon_pos); + } + } - if (colon_pos != std::string::npos) { - parts.port = std::stoi(parts.host.substr(colon_pos + 1)); - parts.host = parts.host.substr(0, colon_pos); + if (!port_str.empty()) { + parts.port = std::stoi(port_str); } else if (parts.scheme == "http") { parts.port = 80; } else if (parts.scheme == "https") { @@ -83,7 +105,7 @@ static std::pair common_http_client(const std: } #endif - httplib::Client cli(parts.scheme + "://" + parts.host + ":" + std::to_string(parts.port)); + httplib::Client cli(parts.scheme + "://" + common_http_format_host(parts.host) + ":" + std::to_string(parts.port)); if (!parts.user.empty()) { cli.set_basic_auth(parts.user, parts.password); @@ -95,5 +117,5 @@ static std::pair common_http_client(const std: } static std::string common_http_show_masked_url(const common_http_url & parts) { - return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + parts.host + parts.path; + return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + common_http_format_host(parts.host) + parts.path; } diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index ead864763..f7de8b806 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -9,6 +9,9 @@ #include #include +#ifdef FILENAME +#undef FILENAME +#endif #define FILENAME "jinja-caps" using json = nlohmann::ordered_json; @@ -16,22 +19,34 @@ using json = nlohmann::ordered_json; namespace jinja { using caps_json_fn = std::function; -using caps_analyze_fn = std::function; +using caps_ctx_fn = std::function; +using caps_analyze_fn = std::function; + +void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { + ctx.set_val("preserve_thinking", mk_val(enabled)); + ctx.set_val("clear_thinking", mk_val(!enabled)); + ctx.set_val("truncate_history_thinking", mk_val(!enabled)); +} static void caps_try_execute(jinja::program & prog, const caps_json_fn & messages_fn, + const caps_ctx_fn & ctx_fn, const caps_json_fn & tools_fn, const caps_analyze_fn & analyze_fn) { context ctx; ctx.is_get_stats = true; jinja::global_from_json(ctx, json{ {"messages", messages_fn()}, - {"tools", tools_fn()}, + {"tools", tools_fn ? tools_fn() : json::array()}, {"bos_token", ""}, {"eos_token", ""}, {"add_generation_prompt", true} }, true); + if (ctx_fn) { + ctx_fn(ctx); + } + auto messages = ctx.get_val("messages"); auto tools = ctx.get_val("tools"); @@ -49,7 +64,7 @@ static void caps_try_execute(jinja::program & prog, // ignore exceptions during capability analysis } - analyze_fn(success, messages, tools); + analyze_fn(success, messages, tools, result); } // for debugging only @@ -109,11 +124,9 @@ caps caps_get(jinja::program & prog) { } }); }, - [&]() { - // tools - return json{nullptr}; - }, - [&](bool success, value & messages, value &) { + nullptr, // ctx_fn + nullptr, // tools_fn + [&](bool success, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (has_op(content, "selectattr") || has_op(content, "array_access")) { @@ -145,11 +158,9 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&]() { - // tools - return json::array(); - }, - [&](bool, value & messages, value &) { + nullptr, // ctx_fn + nullptr, // tools_fn + [&](bool, value & messages, value &, const std::string &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (!content->stats.used) { @@ -201,6 +212,7 @@ caps caps_get(jinja::program & prog) { }, }); }, + nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -224,7 +236,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools) { + [&](bool success, value & messages, value & tools, const std::string &) { if (!success) { return; // Nothing can be inferred } @@ -293,6 +305,7 @@ caps caps_get(jinja::program & prog) { }, }); }, + nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -316,7 +329,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools) { + [&](bool success, value & messages, value & tools, const std::string &) { if (!success) { result.supports_tool_calls = false; result.supports_tools = false; @@ -394,6 +407,7 @@ caps caps_get(jinja::program & prog) { }, }); }, + nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -417,7 +431,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & /*tools*/) { + [&](bool success, value & messages, value &, const std::string &) { if (!success) { result.supports_parallel_tool_calls = false; return; @@ -438,11 +452,22 @@ caps caps_get(jinja::program & prog) { JJ_DEBUG("%s\n", ">>> Running capability check: preserve reasoning"); // case: preserve reasoning content in chat history + const std::string reasoning_placeholder = ""; caps_try_execute( prog, [&]() { // messages return json::array({ + { + {"role", "user"}, + {"content", "User message"} + }, + { + {"role", "assistant"}, + {"content", "Assistant message"}, + // check of reasoning_content deeper in the history, not just the last assistant message + {"reasoning_content", reasoning_placeholder} + }, { {"role", "user"}, {"content", "User message"} @@ -458,14 +483,13 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&]() { - // tools - return json::array(); + [&](context & ctx) { + caps_apply_preserve_reasoning(ctx, true); }, - [&](bool, value & messages, value &) { - auto & content = messages->at(1)->at("reasoning_content"); - caps_print_stats(content, "messages[1].reasoning_content"); - if (content->stats.used) { + nullptr, // tools_fn + [&](bool, value &, value &, const std::string & output) { + // note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result + if (output.find(reasoning_placeholder) != std::string::npos) { result.supports_preserve_reasoning = true; } } diff --git a/common/jinja/caps.h b/common/jinja/caps.h index 93a7fe092..a290cd7da 100644 --- a/common/jinja/caps.h +++ b/common/jinja/caps.h @@ -12,7 +12,9 @@ struct caps { bool supports_tool_calls = true; bool supports_system_role = true; bool supports_parallel_tool_calls = true; - bool supports_preserve_reasoning = false; // support assistant message with reasoning_content + + // supports preserve reasoning trace in the full history, not just the last assistant message + bool supports_preserve_reasoning = false; // one of the 2 content capabilities must be true bool supports_string_content = true; @@ -29,4 +31,6 @@ struct caps { caps caps_get(jinja::program & prog); +void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled); + } // namespace jinja diff --git a/common/jinja/lexer.cpp b/common/jinja/lexer.cpp index 598982c2f..63740dff6 100644 --- a/common/jinja/lexer.cpp +++ b/common/jinja/lexer.cpp @@ -7,6 +7,9 @@ #include #include +#ifdef FILENAME +#undef FILENAME +#endif #define FILENAME "jinja-lexer" namespace jinja { diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp index 2b25654a7..e9e6dcb7f 100644 --- a/common/jinja/parser.cpp +++ b/common/jinja/parser.cpp @@ -8,6 +8,9 @@ #include #include +#ifdef FILENAME +#undef FILENAME +#endif #define FILENAME "jinja-parser" namespace jinja { diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index f81d98d95..afba0025f 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -8,6 +8,9 @@ #include #include +#ifdef FILENAME +#undef FILENAME +#endif #define FILENAME "jinja-runtime" bool g_jinja_debug = false; @@ -316,12 +319,22 @@ value filter_expression::execute_impl(context & ctx) { JJ_DEBUG("Applying filter to %s", input->type().c_str()); + auto set_filter_alias = [](auto & filter_id) { + if (filter_id == "count") { + filter_id = "length"; + } else if (filter_id == "d") { + filter_id = "default"; + } else if (filter_id == "e") { + filter_id = "escape"; + } else if (filter_id == "trim") { + filter_id = "strip"; + } + }; + if (is_stmt(filter)) { auto filter_id = cast_stmt(filter)->val; - if (filter_id == "trim") { - filter_id = "strip"; // alias - } + set_filter_alias(filter_id); JJ_DEBUG("Applying filter '%s' to %s", filter_id.c_str(), input->type().c_str()); // TODO: Refactor filters so this coercion can be done automatically if (!input->is_undefined() && !is_val(input) && ( @@ -345,9 +358,7 @@ value filter_expression::execute_impl(context & ctx) { } auto filter_id = cast_stmt(call->callee)->val; - if (filter_id == "trim") { - filter_id = "strip"; // alias - } + set_filter_alias(filter_id); JJ_DEBUG("Applying filter '%s' with arguments to %s", filter_id.c_str(), input->type().c_str()); func_args args(ctx); for (const auto & arg_expr : call->args) { @@ -678,59 +689,62 @@ value set_statement::execute_impl(context & ctx) { return mk_val(); } +static inline void bind_parameters(const std::string & name, const statements & this_args, const func_args & args, context & ctx) { + const size_t expected_count = this_args.size(); + const size_t input_count = args.count(); + + JJ_DEBUG("Invoking '%s' with %zu input arguments (expected %zu)", name.c_str(), input_count, expected_count); + for (size_t i = 0; i < expected_count; ++i) { + if (i < input_count) { + if (is_stmt(this_args[i])) { + // normal parameter + std::string param_name = cast_stmt(this_args[i])->val; + value param_value = args.get_kwarg_or_pos(param_name, i); + JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); + ctx.set_val(param_name, param_value); + } else if (is_stmt(this_args[i])) { + // default argument used as normal parameter + auto kwarg = cast_stmt(this_args[i]); + if (!is_stmt(kwarg->key)) { + throw std::runtime_error("Keyword argument key must be an identifier in '" + name + "'"); + } + std::string param_name = cast_stmt(kwarg->key)->val; + value param_value = args.get_kwarg_or_pos(param_name, i); + JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); + ctx.set_val(param_name, param_value); + } else { + throw std::runtime_error("Invalid parameter type in '" + name + "'"); + } + } else { + auto & default_arg = this_args[i]; + if (is_stmt(default_arg)) { + auto kwarg = cast_stmt(default_arg); + if (!is_stmt(kwarg->key)) { + throw std::runtime_error("Keyword argument key must be an identifier in '" + name + "'"); + } + std::string param_name = cast_stmt(kwarg->key)->val; + JJ_DEBUG(" Binding parameter '%s' to default argument of type %s", param_name.c_str(), kwarg->val->type().c_str()); + ctx.set_val(param_name, kwarg->val->execute(args.ctx)); + } else { + throw std::runtime_error("Not enough arguments provided to '" + name + "'"); + } + //std::string param_name = cast_stmt(default_args[i])->val; + //JJ_DEBUG(" Binding parameter '%s' to default", param_name.c_str()); + //ctx.var[param_name] = default_args[i]->execute(ctx); + } + } +} + value macro_statement::execute_impl(context & ctx) { if (!is_stmt(this->name)) { throw std::runtime_error("Macro name must be an identifier"); } std::string name = cast_stmt(this->name)->val; - const func_handler func = [this, name, &ctx](const func_args & args) -> value { - size_t expected_count = this->args.size(); - size_t input_count = args.count(); + const func_handler func = [this, name](const func_args & args) -> value { + context macro_ctx(args.ctx); // new scope for macro execution - JJ_DEBUG("Invoking macro '%s' with %zu input arguments (expected %zu)", name.c_str(), input_count, expected_count); - context macro_ctx(ctx); // new scope for macro execution - - // bind parameters - for (size_t i = 0; i < expected_count; ++i) { - if (i < input_count) { - if (is_stmt(this->args[i])) { - // normal parameter - std::string param_name = cast_stmt(this->args[i])->val; - value param_value = args.get_kwarg_or_pos(param_name, i); - JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); - macro_ctx.set_val(param_name, param_value); - } else if (is_stmt(this->args[i])) { - // default argument used as normal parameter - auto kwarg = cast_stmt(this->args[i]); - if (!is_stmt(kwarg->key)) { - throw std::runtime_error("Keyword argument key must be an identifier in macro '" + name + "'"); - } - std::string param_name = cast_stmt(kwarg->key)->val; - value param_value = args.get_kwarg_or_pos(param_name, i); - JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); - macro_ctx.set_val(param_name, param_value); - } else { - throw std::runtime_error("Invalid parameter type in macro '" + name + "'"); - } - } else { - auto & default_arg = this->args[i]; - if (is_stmt(default_arg)) { - auto kwarg = cast_stmt(default_arg); - if (!is_stmt(kwarg->key)) { - throw std::runtime_error("Keyword argument key must be an identifier in macro '" + name + "'"); - } - std::string param_name = cast_stmt(kwarg->key)->val; - JJ_DEBUG(" Binding parameter '%s' to default argument of type %s", param_name.c_str(), kwarg->val->type().c_str()); - macro_ctx.set_val(param_name, kwarg->val->execute(ctx)); - } else { - throw std::runtime_error("Not enough arguments provided to macro '" + name + "'"); - } - //std::string param_name = cast_stmt(default_args[i])->val; - //JJ_DEBUG(" Binding parameter '%s' to default", param_name.c_str()); - //macro_ctx.var[param_name] = default_args[i]->execute(ctx); - } - } + bind_parameters(name, this->args, args, macro_ctx); // execute macro body JJ_DEBUG("Executing macro '%s' body with %zu statements", name.c_str(), this->body.size()); @@ -744,6 +758,46 @@ value macro_statement::execute_impl(context & ctx) { return mk_val(); } +value call_statement::execute_impl(context & ctx) { + auto call_expr = cast_stmt(this->call); + if (!call_expr) { + throw std::runtime_error("Call statement requires a valid call expression"); + } + + value callee_val = call_expr->callee->execute(ctx); + if (!is_val(callee_val)) { + throw std::runtime_error("Callee is not a function: got " + callee_val->type()); + } + auto * callee_func = cast_val(callee_val); + + context caller_ctx(ctx); // new scope for caller execution + + const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value { + context block_ctx(caller_ctx); // new scope for block execution + + bind_parameters("caller", this->caller_args, args, block_ctx); + + JJ_DEBUG("Executing call body with %zu statements", this->body.size()); + auto res = exec_statements(this->body, block_ctx); + JJ_DEBUG("Call body execution complete, result: %s", res->val_str.str().c_str()); + return res; + }; + + context call_ctx(ctx); + call_ctx.set_val("caller", mk_val("caller", func)); + + func_args args(call_ctx); + + for (const auto & arg_expr : call_expr->args) { + auto arg_val = arg_expr->execute(ctx); + JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); + args.push_back(arg_val); + } + + JJ_DEBUG("Calling macro '%s' with %zu arguments", callee_func->name.c_str(), args.count()); + return callee_func->invoke(args); +} + value member_expression::execute_impl(context & ctx) { value object = this->object->execute(ctx); @@ -761,9 +815,9 @@ value member_expression::execute_impl(context & ctx) { if (is_stmt(this->property)) { auto s = cast_stmt(this->property); - value start_val = s->start_expr ? s->start_expr->execute(ctx) : mk_val(0); - value stop_val = s->stop_expr ? s->stop_expr->execute(ctx) : mk_val(arr_size); value step_val = s->step_expr ? s->step_expr->execute(ctx) : mk_val(1); + value start_val = s->start_expr ? s->start_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val(arr_size - 1) : mk_val(0)); + value stop_val = s->stop_expr ? s->stop_expr->execute(ctx) : (step_val->as_int() < 0 ? mk_val(-1) : mk_val(arr_size)); // translate to function call: obj.slice(start, stop, step) JJ_DEBUG("Member expression is a slice: start %s, stop %s, step %s", @@ -903,4 +957,50 @@ value keyword_argument_expression::execute_impl(context & ctx) { return mk_val(k, v); } +std::string runtime::debug_dump_program(const program & prog, const std::string & src) { + std::ostringstream oss; + size_t lvl = 0; + context ctx; + ctx.src.reset(new std::string(src)); + + auto indent = [](size_t lvl) -> std::string { + return std::string(lvl * 2, ' '); + }; + + ctx.visitor = [&](bool is_leaf, statement * node, std::vector children) { + oss << indent(lvl) << node->type() << ":\n"; + lvl++; + if (is_leaf) { + const auto & pos = node->pos; + oss << indent(lvl) << "(leaf) at " << get_line_col(src, pos) << " in source:\n"; + std::string snippet = peak_source(src, pos); + string_replace_all(snippet, "\n", "\n" + indent(lvl)); + oss << indent(lvl) << snippet << "\n"; + } else { + for (auto & [label, children_vec] : children) { + oss << indent(lvl) << label << ":\n"; + lvl++; + if (children_vec.empty()) { + oss << indent(lvl) << "\n\n"; + } else { + for (auto * child : children_vec) { + if (!child) { + continue; + } + child->visit(ctx); + } + } + lvl--; + } + } + lvl--; + }; + + for (const auto & stmt : prog.body) { + stmt->visit(ctx); + } + + return oss.str(); +} + } // namespace jinja diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index b6f4a6ab4..0884a1592 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -47,12 +47,19 @@ const T * cast_stmt(const statement_ptr & ptr) { // not thread-safe void enable_debug(bool enable); +// for visiting AST nodes +// function signature: void(bool is_leaf, statement * node, pair of ) +using visitor_pair = std::pair>; +using visitor_fn = std::function)>; + struct context { std::shared_ptr src; // for debugging; use shared_ptr to avoid copying on scope creation std::time_t current_time; // for functions that need current time bool is_get_stats = false; // whether to collect stats + visitor_fn visitor; + // src is optional, used for error reporting context(std::string src = "") : src(std::make_shared(std::move(src))) { env = mk_val(); @@ -99,6 +106,15 @@ private: value_object env; }; +// utils for visiting AST nodes +static std::vector stmts_to_ptr(const statements & stmts) { + std::vector children; + for (const auto & stmt : stmts) { + children.push_back(stmt.get()); + } + return children; +} + /** * Base class for all nodes in the AST. */ @@ -106,6 +122,7 @@ struct statement { size_t pos; // position in source, for debugging virtual ~statement() = default; virtual std::string type() const { return "Statement"; } + virtual void visit(context & ctx) { ctx.visitor(true, this, {}); } // execute_impl must be overridden by derived classes virtual value execute_impl(context &) { throw_exec_error(); } @@ -166,6 +183,13 @@ struct if_statement : public statement { std::string type() const override { return "If"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"test", {test.get()}}, + {"body", stmts_to_ptr(body)}, + {"alternate", stmts_to_ptr(alternate)} + }); + } }; struct identifier; @@ -190,6 +214,14 @@ struct for_statement : public statement { std::string type() const override { return "For"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"loopvar", {loopvar.get()}}, + {"iterable", {iterable.get()}}, + {"body", stmts_to_ptr(body)}, + {"default_block", stmts_to_ptr(default_block)} + }); + } }; struct break_statement : public statement { @@ -241,6 +273,13 @@ struct set_statement : public statement { std::string type() const override { return "Set"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"assignee", {assignee.get()}}, + {"value", {val.get()}}, + {"body", stmts_to_ptr(body)} + }); + } }; struct macro_statement : public statement { @@ -256,6 +295,13 @@ struct macro_statement : public statement { std::string type() const override { return "Macro"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"name", {name.get()}}, + {"args", stmts_to_ptr(args)}, + {"body", stmts_to_ptr(body)} + }); + } }; struct comment_statement : public statement { @@ -289,6 +335,12 @@ struct member_expression : public expression { } std::string type() const override { return "MemberExpression"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"object", {object.get()}}, + {"property", {property.get()}} + }); + } }; struct call_expression : public expression { @@ -302,6 +354,12 @@ struct call_expression : public expression { } std::string type() const override { return "CallExpression"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"callee", {callee.get()}}, + {"args", stmts_to_ptr(args)} + }); + } }; /** @@ -405,6 +463,12 @@ struct binary_expression : public expression { } std::string type() const override { return "BinaryExpression"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"left", {left.get()}}, + {"right", {right.get()}} + }); + } }; /** @@ -431,6 +495,12 @@ struct filter_expression : public expression { std::string type() const override { return "FilterExpression"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"operand", {operand.get()}}, + {"filter", {filter.get()}} + }); + } }; struct filter_statement : public statement { @@ -443,6 +513,12 @@ struct filter_statement : public statement { } std::string type() const override { return "FilterStatement"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"filter", {filter.get()}}, + {"body", stmts_to_ptr(body)} + }); + } }; /** @@ -468,6 +544,12 @@ struct select_expression : public expression { } return lhs->execute_impl(ctx); } + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"lhs", {lhs.get()}}, + {"test", {test.get()}} + }); + } }; /** @@ -486,6 +568,12 @@ struct test_expression : public expression { } std::string type() const override { return "TestExpression"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"operand", {operand.get()}}, + {"test", {test.get()}} + }); + } }; /** @@ -501,6 +589,11 @@ struct unary_expression : public expression { } std::string type() const override { return "UnaryExpression"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"argument", {argument.get()}} + }); + } }; struct slice_expression : public expression { @@ -518,6 +611,13 @@ struct slice_expression : public expression { [[noreturn]] value execute_impl(context &) override { throw std::runtime_error("must be handled by MemberExpression"); } + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"start_expr", {start_expr.get()}}, + {"stop_expr", {stop_expr.get()}}, + {"step_expr", {step_expr.get()}} + }); + } }; struct keyword_argument_expression : public expression { @@ -531,6 +631,12 @@ struct keyword_argument_expression : public expression { } std::string type() const override { return "KeywordArgumentExpression"; } value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"key", {key.get()}}, + {"val", {val.get()}} + }); + } }; struct spread_expression : public expression { @@ -539,6 +645,11 @@ struct spread_expression : public expression { chk_type(this->argument); } std::string type() const override { return "SpreadExpression"; } + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"argument", {argument.get()}} + }); + } }; struct call_statement : public statement { @@ -552,6 +663,14 @@ struct call_statement : public statement { for (const auto & arg : this->caller_args) chk_type(arg); } std::string type() const override { return "CallStatement"; } + value execute_impl(context & ctx) override; + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"call", {call.get()}}, + {"caller_args", stmts_to_ptr(caller_args)}, + {"body", stmts_to_ptr(body)} + }); + } }; struct ternary_expression : public expression { @@ -574,6 +693,13 @@ struct ternary_expression : public expression { return false_expr->execute(ctx); } } + void visit(context & ctx) override { + ctx.visitor(false, this, { + {"condition", {condition.get()}}, + {"true_expr", {true_expr.get()}}, + {"false_expr", {false_expr.get()}} + }); + } }; struct raised_exception : public std::exception { @@ -647,6 +773,8 @@ struct runtime { } return parts; } + + static std::string debug_dump_program(const program & prog, const std::string & src); }; } // namespace jinja diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index aa4bd3c8c..f36bbbc0c 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -12,6 +12,9 @@ #include #include +#ifdef FILENAME +#undef FILENAME +#endif #define FILENAME "jinja-value" namespace jinja { @@ -90,14 +93,14 @@ static T slice(const T & array, int64_t start, int64_t stop, int64_t step = 1) { stop_val = std::min(stop_val, len); } } else { - start_val = len - 1; + start_val = start; if (start_val < 0) { - start_val = std::max(len + start_val, (int64_t)-1); + start_val = std::max(len + start_val, (int64_t)0); } else { start_val = std::min(start_val, len - 1); } - stop_val = -1; + stop_val = stop; if (stop_val < -1) { stop_val = std::max(len + stop_val, (int64_t)-1); } else { @@ -673,6 +676,9 @@ const func_builtins & value_string_t::get_builtins() const { std::string str = val_input->as_string().str(); // FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace) std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " "; + if (delim.empty()) { + throw raised_exception("empty separator"); + } int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1; auto result = mk_val(); size_t pos = 0; @@ -697,6 +703,9 @@ const func_builtins & value_string_t::get_builtins() const { std::string str = val_input->as_string().str(); // FIXME: Support non-specified delimiter (split on consecutive (no leading or trailing) whitespace) std::string delim = (args.count() > 1) ? args.get_pos(1)->as_string().str() : " "; + if (delim.empty()) { + throw raised_exception("empty separator"); + } int64_t maxsplit = (args.count() > 2) ? args.get_pos(2)->as_int() : -1; auto result = mk_val(); size_t pos = 0; @@ -722,10 +731,23 @@ const func_builtins & value_string_t::get_builtins() const { if (count > 0) { throw not_implemented_exception("String replace with count argument not implemented"); } - size_t pos = 0; - while ((pos = str.find(old_str, pos)) != std::string::npos) { - str.replace(pos, old_str.length(), new_str); - pos += new_str.length(); + if (old_str != new_str) { + size_t pos = 0; + if (old_str.empty()) { + std::string new_res; + new_res.reserve(str.length() + new_str.length() * (str.length() + 1)); + new_res += new_str; + for (const char c : str) { + new_res.push_back(c); + new_res += new_str; + } + str = new_res; + } else { + while ((pos = str.find(old_str, pos)) != std::string::npos) { + str.replace(pos, old_str.length(), new_str); + pos += new_str.length(); + } + } } auto res = mk_val(str); res->val_str.mark_input_based_on(args.get_pos(0)->val_str); @@ -1089,6 +1111,50 @@ const func_builtins & value_array_t::get_builtins() const { std::reverse(arr.begin(), arr.end()); return is_val(val) ? mk_val(std::move(arr)) : mk_val(std::move(arr)); }}, + {"min", [](const func_args & args) -> value { + args.ensure_count(1, 4); + args.ensure_vals(); + value val_case = args.get_kwarg_or_pos("case_sensitive", 1); + value attribute = args.get_kwarg_or_pos("attribute", 2); + if (!attribute->is_undefined()) { + throw not_implemented_exception("min: attribute not implemented"); + } + // FIXME: min is currently always case sensitive + (void) val_case; + const auto & arr = args.get_pos(0)->as_array(); + if (arr.empty()) { + return mk_val(); + } + value result = arr[0]; + for (size_t i = 1; i < arr.size(); ++i) { + if (value_compare(arr[i], result, value_compare_op::lt)) { + result = arr[i]; + } + } + return result; + }}, + {"max", [](const func_args & args) -> value { + args.ensure_count(1, 4); + args.ensure_vals(); + value val_case = args.get_kwarg_or_pos("case_sensitive", 1); + value attribute = args.get_kwarg_or_pos("attribute", 2); + if (!attribute->is_undefined()) { + throw not_implemented_exception("max: attribute not implemented"); + } + // FIXME: max is currently always case sensitive + (void) val_case; + const auto & arr = args.get_pos(0)->as_array(); + if (arr.empty()) { + return mk_val(); + } + value result = arr[0]; + for (size_t i = 1; i < arr.size(); ++i) { + if (value_compare(arr[i], result, value_compare_op::gt)) { + result = arr[i]; + } + } + return result; + }}, {"unique", array_unique_not_implemented}, }; return builtins; diff --git a/common/json-partial.cpp b/common/json-partial.cpp deleted file mode 100644 index aaf11310a..000000000 --- a/common/json-partial.cpp +++ /dev/null @@ -1,324 +0,0 @@ -#include "json-partial.h" - -#include "log.h" - -#include - -#include -#include - -using json = nlohmann::ordered_json; - -enum common_json_stack_element_type { - COMMON_JSON_STACK_ELEMENT_OBJECT, - COMMON_JSON_STACK_ELEMENT_KEY, - COMMON_JSON_STACK_ELEMENT_ARRAY, -}; - -struct common_json_stack_element { - common_json_stack_element_type type; - std::string key; -}; - -bool common_json_parse( - const std::string & input, - const std::string & healing_marker, - common_json & out) -{ - std::string::const_iterator it = input.begin(); - const auto end = input.end(); - return common_json_parse(it, end, healing_marker, out); -} - -bool common_json_parse( - std::string::const_iterator & it, - const std::string::const_iterator & end, - const std::string & healing_marker, - common_json & out) -{ - // // https://json.nlohmann.me/features/parsing/sax_interface/ - struct json_error_locator : public nlohmann::json_sax { - std::size_t position; - bool found_error; - std::string last_token; - std::string exception_message; - std::vector stack; - - json_error_locator() : position(0), found_error(false) {} - - bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override { // NOLINT - this->position = position - 1; - this->found_error = true; - this->last_token = last_token; - this->exception_message = ex.what(); - return false; - } - void close_value() { - if (!stack.empty() && (stack.back().type == COMMON_JSON_STACK_ELEMENT_KEY)) { - stack.pop_back(); - } - } - bool null() override { // NOLINT - close_value(); - return true; - } - bool boolean(bool) override { // NOLINT - close_value(); - return true; - } - bool number_integer(number_integer_t) override { // NOLINT - close_value(); - return true; - } - bool number_unsigned(number_unsigned_t) override { // NOLINT - close_value(); - return true; - } - bool number_float(number_float_t, const string_t &) override { // NOLINT - close_value(); - return true; - } - bool string(string_t &) override { // NOLINT - close_value(); - return true; - } - bool binary(binary_t &) override { // NOLINT - close_value(); - return true; - } - bool start_object(std::size_t) override { // NOLINT - stack.push_back({COMMON_JSON_STACK_ELEMENT_OBJECT, ""}); - return true; - } - bool end_object() override { - GGML_ASSERT(!stack.empty() && stack.back().type == COMMON_JSON_STACK_ELEMENT_OBJECT); - stack.pop_back(); - close_value(); - return true; - } - bool key(string_t & key) override { // NOLINT - stack.push_back({COMMON_JSON_STACK_ELEMENT_KEY, key}); - return true; - } - bool start_array(std::size_t) override { // NOLINT - stack.push_back({COMMON_JSON_STACK_ELEMENT_ARRAY, ""}); - return true; - } - bool end_array() override { - GGML_ASSERT(!stack.empty() && stack.back().type == COMMON_JSON_STACK_ELEMENT_ARRAY); - stack.pop_back(); - close_value(); - return true; - } - }; - json_error_locator err_loc; - auto start = it; - json::sax_parse(it, end, &err_loc); - - if (err_loc.found_error) { - it = start; - auto temptative_end = it + err_loc.position; - // LOG_DBG("Error at position %zu (is_end = %s): %s\n", err_loc.position, temptative_end == end ? "true" : "false", err_loc.exception_message.c_str()); - - auto input = std::string(it, temptative_end); - try { - out.json = json::parse(input); - // out.json = json::parse(it, temptative_end); - it = temptative_end; - return true; - } catch (const std::exception & ex) { - // No, needs healing. - LOG_DBG("Failed to parse up to error: %s: <<<%s>>>\n", ex.what(), std::string(it, temptative_end).c_str()); - } - auto can_parse = [](const std::string & str) { - try { - auto _ = json::parse(str); // NOLINT - return true; - } catch (const std::exception &) { - return false; - } - }; - if (!healing_marker.empty() && !err_loc.stack.empty()) { - std::string str(it, temptative_end); - auto last_non_sp_pos = str.find_last_not_of(" \n\r\t"); - if (last_non_sp_pos == std::string::npos) { - throw std::runtime_error("Cannot heal a truncated JSON that stopped in an unknown location"); - } - auto last_non_sp_char = str[last_non_sp_pos]; - // Used to detect stops on a number, which may not be complete. - auto was_maybe_number = [&]() { - if (!str.empty() && std::isspace(str.back())) { - return false; - } - return std::isdigit(last_non_sp_char) || - last_non_sp_char == '.' || - last_non_sp_char == 'e' || - last_non_sp_char == 'E' || - last_non_sp_char == '-'; - }; - - std::string closing; - for (size_t i = err_loc.stack.size(); i > 0; i--) { - auto & el = err_loc.stack[i - 1]; - if (el.type == COMMON_JSON_STACK_ELEMENT_OBJECT) { - closing += "}"; - } else if (el.type == COMMON_JSON_STACK_ELEMENT_ARRAY) { - closing += "]"; - } else if (el.type != COMMON_JSON_STACK_ELEMENT_KEY) { - throw std::runtime_error("Unexpected stack element type"); - } - } - - // Matches a potentially partial unicode escape sequence, e.g. \u, \uX, \uXX, \uXXX, \uXXXX - static const std::regex partial_unicode_regex(R"(\\u(?:[0-9a-fA-F](?:[0-9a-fA-F](?:[0-9a-fA-F](?:[0-9a-fA-F])?)?)?)?$)"); - - auto is_high_surrogate = [&](const std::string & s) { - // Check if a partial of a high surrogate (U+D800-U+DBFF) - return s.length() >= 4 && - s[0] == '\\' && s[1] == 'u' && - std::tolower(s[2]) == 'd' && - (s[3] == '8' || s[3] == '9' || std::tolower(s[3]) == 'a' || std::tolower(s[3]) == 'b'); - }; - - // Initialize the unicode marker to a low surrogate to handle the edge case - // where a high surrogate (U+D800-U+DBFF) is immediately followed by a - // backslash (\) - std::string unicode_marker_padding = "udc00"; - std::smatch last_unicode_seq; - - if (std::regex_search(str, last_unicode_seq, partial_unicode_regex)) { - std::smatch second_last_seq; - std::string prelude = str.substr(0, last_unicode_seq.position()); - - // Pad the escape sequence with 0s until it forms a complete sequence of 6 characters - unicode_marker_padding = std::string(6 - last_unicode_seq.length(), '0'); - - if (is_high_surrogate(last_unicode_seq.str())) { - // If the sequence is a partial match for a high surrogate, add a low surrogate (U+DC00-U+UDFF) - unicode_marker_padding += "\\udc00"; - } else if (std::regex_search(prelude, second_last_seq, partial_unicode_regex)) { - if (is_high_surrogate(second_last_seq.str())) { - // If this follows a high surrogate, pad it to be a low surrogate - if (last_unicode_seq.length() == 2) { - unicode_marker_padding = "dc00"; - } else if (last_unicode_seq.length() == 3) { - unicode_marker_padding = "c00"; - } else { - // The original unicode_marker_padding is already padded with 0s - } - } - } - } - - const auto & magic_seed = out.healing_marker.marker = healing_marker;//"$llama.cpp.json$"; - - if (err_loc.stack.back().type == COMMON_JSON_STACK_ELEMENT_KEY) { - // We're inside an object value - if (last_non_sp_char == ':' && can_parse(str + "1" + closing)) { - // Was about to create an object value - str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; - } else if (can_parse(str + ": 1" + closing)) { - str += (out.healing_marker.json_dump_marker = ":\"" + magic_seed) + "\"" + closing; - } else if (last_non_sp_char == '{' && can_parse(str + closing)) { - // Was about to create an object - str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\": 1" + closing; - } else if (can_parse(str + "\"" + closing)) { - // Was inside an object value string - str += (out.healing_marker.json_dump_marker = magic_seed) + "\"" + closing; - } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\"" + closing)) { - // Was inside an object value string after an escape - str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\"" + closing; - } else if (can_parse(str + unicode_marker_padding + "\"" + closing)) { - // Was inside an object value string after a partial unicode escape - str += (out.healing_marker.json_dump_marker = unicode_marker_padding + magic_seed) + "\"" + closing; - } else { - // find last : - auto last_pos = str.find_last_of(':'); - if (last_pos == std::string::npos) { - throw std::runtime_error("Cannot heal a truncated JSON that stopped in an unknown location"); - } - // Cutting back to opening : for object value - str = str.substr(0, last_pos + 1) + (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; - } - } else if (err_loc.stack.back().type == COMMON_JSON_STACK_ELEMENT_ARRAY) { - if ((last_non_sp_char == ',' || last_non_sp_char == '[') && can_parse(str + "1" + closing)) { - // Was about to create an array value - str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; - } else if (can_parse(str + "\"" + closing)) { - // Was inside an array value string - str += (out.healing_marker.json_dump_marker = magic_seed) + "\"" + closing; - } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\"" + closing)) { - // Was inside an array value string after an escape - str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\"" + closing; - } else if (can_parse(str + unicode_marker_padding + "\"" + closing)) { - // Was inside an array value string after a partial unicode escape - str += (out.healing_marker.json_dump_marker = unicode_marker_padding + magic_seed) + "\"" + closing; - } else if (!was_maybe_number() && can_parse(str + ", 1" + closing)) { - // Had just finished a value - str += (out.healing_marker.json_dump_marker = ",\"" + magic_seed) + "\"" + closing; - } else { - auto last_pos = str.find_last_of("[,"); - if (last_pos == std::string::npos) { - throw std::runtime_error("Cannot heal a truncated JSON array stopped in an unknown location"); - } - // Cutting back to last [ or , for array value - str = str.substr(0, last_pos + 1) + (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; - } - } else if (err_loc.stack.back().type == COMMON_JSON_STACK_ELEMENT_OBJECT) { - if ((last_non_sp_char == '{' && can_parse(str + closing)) || - (last_non_sp_char == ',' && can_parse(str + "\"\": 1" + closing))) { - // Was about to create an object key+value - str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\": 1" + closing; - } else if (!was_maybe_number() && can_parse(str + ",\"\": 1" + closing)) { - // Was about to create an object key+value - str += (out.healing_marker.json_dump_marker = ",\"" + magic_seed) + "\": 1" + closing; - } else if (can_parse(str + "\": 1" + closing)) { - // Was inside an object key string - str += (out.healing_marker.json_dump_marker = magic_seed) + "\": 1" + closing; - } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\": 1" + closing)) { - // Was inside an object key string after an escape - str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\": 1" + closing; - } else if (can_parse(str + unicode_marker_padding + "\": 1" + closing)) { - // Was inside an object key string after a partial unicode escape - str += (out.healing_marker.json_dump_marker = unicode_marker_padding + magic_seed) + "\": 1" + closing; - } else { - auto last_pos = str.find_last_of(':'); - if (last_pos == std::string::npos) { - throw std::runtime_error("Cannot heal a truncated JSON object stopped in an unknown location"); - } - // fprintf(stderr, "Cutting back to last : for object key+value\n"); - str = str.substr(0, last_pos + 1) + (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; - } - } else { - throw std::runtime_error("Cannot heal a truncated JSON object stopped in an unknown location"); - } - // fprintf(stderr, "HEALED:\nSTRING <<<\n%s\n>>>\n\nmagic_cut: <<<\n%s\n>>>\n\n", str.c_str(), out.healing_marker.json_dump_marker.c_str()); - out.json = json::parse(str); - it = temptative_end; - return true; - } - // handle unclosed top-level primitive - if (err_loc.position != 0 && !healing_marker.empty() && err_loc.stack.empty()) { - std::string str(it, temptative_end); - const auto & magic_seed = out.healing_marker.marker = healing_marker; - if (can_parse(str + "\"")) { - // Was inside an string - str += (out.healing_marker.json_dump_marker = magic_seed) + "\""; - } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\"")) { - // Was inside an string after an escape - str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\""; - } else { - // TODO: handle more unclosed top-level primitive if the stack was empty but we got an error (e.g. "tru", "\"", etc...) - // fprintf(stderr, "Closing: TODO\n"); - return false; - } - out.json = json::parse(str); - it = temptative_end; - return true; - } - return false; - } - out.json = json::parse(it, end); - it = end; - return true; -} diff --git a/common/json-partial.h b/common/json-partial.h deleted file mode 100644 index be51aabfb..000000000 --- a/common/json-partial.h +++ /dev/null @@ -1,39 +0,0 @@ -#pragma once - -// TODO: use json_fwd.hpp when possible -#include - -// Healing marker (empty if the JSON was fully parsed / wasn't healed). -struct common_healing_marker { - // Raw marker. - std::string marker; - - // Cutting the `common_json.json.dump()` string at the (only) occurrence of this marker should yield the original partial JSON string (modulo spaces / if it had the same dump format). - std::string json_dump_marker; -}; - -// Represents a parsed JSON object, with its optional healing marker (a JSON dump fragment that can be used to find the position of healing in the JSON dump string) -struct common_json { - nlohmann::ordered_json json; - - common_healing_marker healing_marker; -}; - -// Parse the JSON string, healing (closing) any partial JSON if `healing_marker` is not empty. -// -// Healing completes partial JSON strings by adding a (possibly modified) healing marker, then whatever is needed to close the JSON. -// This allows to parse the resulting healed JSON string, yet be able to cut it again if needed at the healing marker. -// (this is used when parsing JSON outputs from the models, then crafting partial JSONs for the partial tool calls in OAI format). -// -// For instance, parsing `{` with a healing marker `foo` will produce a healed JSON `{"foo":1}`, w/ json_dump_marker = `"foo"` (which can be used to break the JSON again). -bool common_json_parse( - const std::string & input, - const std::string & healing_marker, - common_json & out); - -// Parse the JSON string (see overload above), but advancing an iterator to the end of the input when the (potentially partial) parsing succeeds. -bool common_json_parse( - std::string::const_iterator & it, - const std::string::const_iterator & end, - const std::string & healing_marker, - common_json & out); diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index e2c4d6ce2..b18607cd6 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -233,27 +233,27 @@ struct BuiltinRule { }; static std::unordered_map PRIMITIVE_RULES = { - {"boolean", {"(\"true\" | \"false\") space", {}}}, + {"boolean", {"(\"true\" | \"false\")", {}}}, {"decimal-part", {"[0-9]{1,16}", {}}}, {"integral-part", {"[0] | [1-9] [0-9]{0,15}", {}}}, - {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)? space", {"integral-part", "decimal-part"}}}, - {"integer", {"(\"-\"? integral-part) space", {"integral-part"}}}, + {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)?", {"integral-part", "decimal-part"}}}, + {"integer", {"(\"-\"? integral-part)", {"integral-part"}}}, {"value", {"object | array | string | number | boolean | null", {"object", "array", "string", "number", "boolean", "null"}}}, - {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space", {"string", "value"}}}, - {"array", {"\"[\" space ( value (\",\" space value)* )? \"]\" space", {"value"}}}, - {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\" space", {}}}, + {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? space \"}\"", {"string", "value"}}}, + {"array", {"\"[\" space ( value (\",\" space value)* )? space \"]\"", {"value"}}}, + {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\"", {}}}, {"char", {"[^\"\\\\\\x7F\\x00-\\x1F] | [\\\\] ([\"\\\\bfnrt] | \"u\" [0-9a-fA-F]{4})", {}}}, - {"string", {"\"\\\"\" char* \"\\\"\" space", {"char"}}}, - {"null", {"\"null\" space", {}}}, + {"string", {"\"\\\"\" char* \"\\\"\"", {"char"}}}, + {"null", {"\"null\"", {}}}, }; static std::unordered_map STRING_FORMAT_RULES = { {"date", {"[0-9]{4} \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )", {}}}, {"time", {"([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9]{3} )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )", {}}}, {"date-time", {"date \"T\" time", {"date", "time"}}}, - {"date-string", {"\"\\\"\" date \"\\\"\" space", {"date"}}}, - {"time-string", {"\"\\\"\" time \"\\\"\" space", {"time"}}}, - {"date-time-string", {"\"\\\"\" date-time \"\\\"\" space", {"date-time"}}} + {"date-string", {"\"\\\"\" date \"\\\"\"", {"date"}}}, + {"time-string", {"\"\\\"\" time \"\\\"\"", {"time"}}}, + {"date-time-string", {"\"\\\"\" date-time \"\\\"\"", {"date-time"}}} }; static bool is_reserved_name(const std::string & name) { @@ -551,16 +551,16 @@ private: } return join_seq(); }; - return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space"); + return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\""); } /* Returns a rule that matches a JSON string that is none of the provided strings not_strings({"a"}) - -> ["] ( [a] char+ | [^"a] char* )? ["] space + -> ["] ( [a] char+ | [^"a] char* )? ["] not_strings({"and", "also"}) - -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] space + -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] */ std::string _not_strings(const std::vector & strings) { @@ -619,7 +619,7 @@ private: if (!trie.is_end_of_string) { out << "?"; } - out << " [\"] space"; + out << " [\"]"; return out.str(); } @@ -725,7 +725,7 @@ private: rule += " )?"; } - rule += " \"}\" space"; + rule += " space \"}\""; return rule; } @@ -858,14 +858,14 @@ public: return _add_rule(rule_name, _generate_union_rule(name, schema_types)); } if (schema.contains("const")) { - return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space"); + return _add_rule(rule_name, _generate_constant_rule(schema["const"])); } if (schema.contains("enum")) { std::vector enum_values; for (const auto & v : schema["enum"]) { enum_values.push_back(_generate_constant_rule(v)); } - return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ") space"); + return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); } if ((schema_type.is_null() || schema_type == "object") && (schema.contains("properties") || @@ -933,7 +933,7 @@ public: } } if (!enum_intersection.empty()) { - return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ") space"); + return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); } } return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json())); @@ -948,7 +948,7 @@ public: } rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); } - rule += " \"]\" space"; + rule += " space \"]\""; return _add_rule(rule_name, rule); } std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); @@ -956,7 +956,7 @@ public: json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); int max_items = max_items_json.is_number_integer() ? max_items_json.get() : std::numeric_limits::max(); - return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space"); + return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\""); } if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { return _visit_pattern(schema["pattern"], rule_name); @@ -972,7 +972,7 @@ public: std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); int min_len = schema.contains("minLength") ? schema["minLength"].get() : 0; int max_len = schema.contains("maxLength") ? schema["maxLength"].get() : std::numeric_limits::max(); - return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space"); + return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\""); } if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) { int64_t min_value = std::numeric_limits::min(); @@ -990,7 +990,7 @@ public: std::stringstream out; out << "("; build_min_max_int(min_value, max_value, out); - out << ") space"; + out << ")"; return _add_rule(rule_name, out.str()); } if (schema.empty() || schema_type == "object") { diff --git a/common/log.cpp b/common/log.cpp index bd62616d8..2d1e74ad1 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -11,8 +11,13 @@ #include #include #include +#include #if defined(_WIN32) +# define WIN32_LEAN_AND_MEAN +# ifndef NOMINMAX +# define NOMINMAX +# endif # include # include # define isatty _isatty @@ -62,16 +67,15 @@ static const char* g_col[] = { }; struct common_log_entry { - enum ggml_log_level level; - - bool prefix; - - int64_t timestamp; + enum ggml_log_level level {GGML_LOG_LEVEL_INFO}; std::vector msg; - // signals the worker thread to stop - bool is_end; + int64_t timestamp { 0 }; + bool is_end { false }; // signals the worker thread to stop + bool prefix { false }; + + common_log_entry(size_t size = 256) : msg(size) { } void print(FILE * file = nullptr) const { FILE * fcur = file; @@ -122,22 +126,15 @@ struct common_log_entry { }; struct common_log { - // default capacity - will be expanded if needed - common_log() : common_log(256) {} - - common_log(size_t capacity) { - file = nullptr; - prefix = false; + // default capacity + common_log(size_t capacity = 512) { + file = nullptr; + prefix = false; timestamps = false; - running = false; - t_start = t_us(); - - // initial message size - will be expanded if longer messages arrive - entries.resize(capacity); - for (auto & entry : entries) { - entry.msg.resize(256); - } + running = false; + t_start = t_us(); + queue.resize(capacity, common_log_entry(256)); head = 0; tail = 0; @@ -152,9 +149,10 @@ struct common_log { } private: - std::mutex mtx; - std::thread thrd; - std::condition_variable cv; + std::mutex mtx; + std::thread thrd; + std::condition_variable cv_new; // new entry + std::condition_variable cv_full; // wait on full FILE * file; @@ -164,24 +162,53 @@ private: int64_t t_start; - // ring buffer of entries - std::vector entries; + // queue of entries + std::vector queue; size_t head; size_t tail; - // worker thread copies into this - common_log_entry cur; + bool print_entry(const common_log_entry & e) const { + if (e.is_end) return true; + + e.print(); + if (file) { + e.print(file); + } + return false; + } + + bool flush_queue(size_t start_head, size_t end_tail, size_t & out_head) const { + bool stop = false; + size_t h = start_head; + while (h != end_tail && !stop) { + stop = print_entry(queue[h]); + h = (h + 1) % queue.size(); + } + out_head = h; + return stop; + } public: + bool is_full() const { + return ((tail + 1) % queue.size()) == head; + } + + bool is_empty() const { + return head == tail; + } + void add(enum ggml_log_level level, const char * fmt, va_list args) { - std::lock_guard lock(mtx); + std::unique_lock lock(mtx); + + // block if the queue is full + cv_full.wait(lock, [this]() { return !running || !is_full(); }); if (!running) { // discard messages while the worker thread is paused return; } - auto & entry = entries[tail]; + auto & entry = queue[tail]; { // cannot use args twice, so make a copy in case we need to expand the buffer @@ -216,38 +243,16 @@ public: va_end(args_copy); } - entry.level = level; - entry.prefix = prefix; + entry.is_end = false; + entry.level = level; + entry.prefix = prefix; entry.timestamp = 0; if (timestamps) { entry.timestamp = t_us() - t_start; } - entry.is_end = false; - tail = (tail + 1) % entries.size(); - if (tail == head) { - // expand the buffer - std::vector new_entries(2*entries.size()); - - size_t new_tail = 0; - - do { - new_entries[new_tail] = std::move(entries[head]); - - head = (head + 1) % entries.size(); - new_tail = (new_tail + 1); - } while (head != tail); - - head = 0; - tail = new_tail; - - for (size_t i = tail; i < new_entries.size(); i++) { - new_entries[i].msg.resize(256); - } - - entries = std::move(new_entries); - } - cv.notify_one(); + tail = (tail + 1) % queue.size(); + cv_new.notify_one(); } void resume() { @@ -261,23 +266,24 @@ public: thrd = std::thread([this]() { while (true) { - { - std::unique_lock lock(mtx); - cv.wait(lock, [this]() { return head != tail; }); - cur = entries[head]; + std::unique_lock lock(mtx); + cv_new.wait(lock, [this]() { return !is_empty(); }); - head = (head + 1) % entries.size(); - } + size_t cached_head = head; + size_t cached_tail = tail; - if (cur.is_end) { + lock.unlock(); // drop the lock during flush + + size_t next_head; + bool stop = flush_queue(cached_head, cached_tail, next_head); + + lock.lock(); + head = next_head; + cv_full.notify_all(); + + if (stop) { break; } - - cur.print(); // stdout and stderr - - if (file) { - cur.print(file); - } } }); } @@ -293,13 +299,13 @@ public: running = false; // push an entry to signal the worker thread to stop - { - auto & entry = entries[tail]; - entry.is_end = true; + auto & entry = queue[tail]; + entry.is_end = true; + tail = (tail + 1) % queue.size(); - tail = (tail + 1) % entries.size(); - } - cv.notify_one(); + // wakeup everyone + cv_new.notify_one(); + cv_full.notify_all(); } thrd.join(); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index e37c1ce80..807e952d9 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -6,13 +6,14 @@ #include "unicode.h" #include +#include #include #include #include #include #include +#include #include -#include // Trick to catch missing branches template @@ -88,40 +89,7 @@ struct trie { return match_result{match_result::NO_MATCH}; } - struct prefix_and_next { - std::vector prefix; - std::vector next_chars; - }; - - std::vector collect_prefix_and_next() { - std::vector prefix; - std::vector result; - collect_prefix_and_next(0, prefix, result); - return result; - } - private: - void collect_prefix_and_next(size_t index, std::vector & prefix, std::vector & out) { - if (!nodes[index].is_word) { - if (!nodes[index].children.empty()) { - std::vector chars; - chars.reserve(nodes[index].children.size()); - for (const auto & p : nodes[index].children) { - chars.push_back(p.first); - } - out.emplace_back(prefix_and_next{prefix, chars}); - } - } - - for (const auto & p : nodes[index].children) { - uint32_t ch = p.first; - auto child = p.second; - prefix.push_back(ch); - collect_prefix_and_next(child, prefix, out); - prefix.pop_back(); - } - } - size_t create_node() { size_t index = nodes.size(); nodes.emplace_back(); @@ -153,6 +121,65 @@ struct trie { } }; +// Aho-Corasick automaton +struct aho_corasick { + trie t; + std::vector fail; // failure links + std::vector order; // states in BFS order + std::vector terminal; // match states (directly or via a suffix link) + std::set alphabet; // every character with a transition + + aho_corasick(const std::vector & strings) : t(strings) { + const auto & nodes = t.nodes; + const size_t n = nodes.size(); + + fail.assign(n, 0); + order.reserve(n); + + std::deque queue{ 0 }; + while (!queue.empty()) { + size_t u = queue.front(); + queue.pop_front(); + order.push_back(u); + for (const auto & [ch, v] : nodes[u].children) { + if (u != 0) { + size_t f = fail[u]; + while (f && nodes[f].children.find(ch) == nodes[f].children.end()) { + f = fail[f]; + } + auto it = nodes[f].children.find(ch); + fail[v] = (it != nodes[f].children.end() && it->second != v) ? it->second : 0; + } + queue.push_back(v); + } + } + + terminal.assign(n, false); + for (size_t u : order) { + terminal[u] = nodes[u].is_word || (u != 0 && terminal[fail[u]]); + } + + for (const auto & node : nodes) { + for (const auto & [ch, v] : node.children) { + alphabet.insert(ch); + } + } + } + + size_t num_states() const { return t.nodes.size(); } + bool is_terminal(size_t s) const { return terminal[s]; } + + // follow failure links until a transition on `ch` exists. + size_t next(size_t state, uint32_t ch) const { + const auto & nodes = t.nodes; + while (state && nodes[state].children.find(ch) == nodes[state].children.end()) { + state = fail[state]; + } + auto it = nodes[state].children.find(ch); + return it != nodes[state].children.end() ? it->second : 0; + } +}; + static std::pair parse_hex_escape(const std::string & str, size_t pos, int hex_count) { if (pos + hex_count > str.length()) { return {0, 0}; @@ -894,6 +921,10 @@ struct parser_executor { common_peg_parse_result operator()(const common_peg_gbnf_parser & p) { return arena.parse(p.child, ctx, start_pos); } + + common_peg_parse_result operator()(const common_peg_ac_parser & p) { + return arena.parse(p.child, ctx, start_pos); + } }; common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const { @@ -962,7 +993,8 @@ void common_peg_arena::resolve_refs() { std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v) { + std::is_same_v || + std::is_same_v) { p.child = resolve_ref(p.child); } else if constexpr (std::is_same_v) { p.child = resolve_ref(p.child); @@ -992,12 +1024,12 @@ void common_peg_arena::resolve_refs() { } std::string common_peg_arena::dump(common_peg_parser_id id) const { - std::unordered_set visited; + std::set visited; return dump_impl(id, visited); } std::string common_peg_arena::dump_impl(common_peg_parser_id id, - std::unordered_set & visited) const { + std::set & visited) const { // Check for cycles if (visited.count(id)) { return "[cycle]"; @@ -1043,6 +1075,8 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id return "Atomic(" + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Gbnf(" + p.grammar + ", " + dump_impl(p.child, visited) + ")"; + } else if constexpr (std::is_same_v) { + return "Ac(" + string_join(p.delimiters, " | ") + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Any"; } else if constexpr (std::is_same_v) { @@ -1272,13 +1306,13 @@ common_peg_parser common_peg_parser_builder::string_content(char delimiter) { common_peg_parser common_peg_parser_builder::double_quoted_string() { return rule("double-quoted-string", [this]() { - return sequence({literal("\""), string_content('"'), literal("\""), space()}); + return sequence({literal("\""), string_content('"'), literal("\"")}); }); } common_peg_parser common_peg_parser_builder::single_quoted_string() { return rule("single-quoted-string", [this]() { - return sequence({literal("'"), string_content('\''), literal("'"), space()}); + return sequence({literal("'"), string_content('\''), literal("'")}); }); } @@ -1301,25 +1335,25 @@ common_peg_parser common_peg_parser_builder::json_number() { // At EOF in partial mode, chars returns NEED_MORE → negate propagates NEED_MORE → number not committed. // This prevents premature commits of partial numbers (e.g. "3" when "3.14" is incoming). auto not_number_continuation = negate(chars("[0-9.eE+-]", 1, 1)); - return sequence({ optional(literal("-")), int_part, optional(frac), optional(exp), not_number_continuation, space() }); + return sequence({ optional(literal("-")), int_part, optional(frac), optional(exp), not_number_continuation }); }); } common_peg_parser common_peg_parser_builder::json_string() { return rule("json-string", [this]() { - return sequence({literal("\""), string_content('"'), literal("\""), space()}); + return sequence({literal("\""), string_content('"'), literal("\"")}); }); } common_peg_parser common_peg_parser_builder::json_bool() { return rule("json-bool", [this]() { - return sequence({choice({literal("true"), literal("false")}), space()}); + return choice({literal("true"), literal("false")}); }); } common_peg_parser common_peg_parser_builder::json_null() { return rule("json-null", [this]() { - return sequence({literal("null"), space()}); + return literal("null"); }); } @@ -1334,8 +1368,7 @@ common_peg_parser common_peg_parser_builder::json_object() { choice({ literal("}"), sequence({members, ws, literal("}")}) - }), - ws + }) }); }); } @@ -1343,15 +1376,14 @@ common_peg_parser common_peg_parser_builder::json_object() { common_peg_parser common_peg_parser_builder::json_array() { return rule("json-array", [this]() { auto ws = space(); - auto elements = sequence({json(), zero_or_more(sequence({literal(","), ws, json()}))}); + auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))}); return sequence({ literal("["), ws, choice({ literal("]"), sequence({elements, ws, literal("]")}) - }), - ws + }) }); }); } @@ -1381,16 +1413,13 @@ common_peg_parser common_peg_parser_builder::python_number() { common_peg_parser common_peg_parser_builder::python_bool() { return rule("python-bool", [this]() { - return sequence({ - choice({literal("True"), literal("False")}), - space() - }); + return choice({literal("True"), literal("False")}); }); } common_peg_parser common_peg_parser_builder::python_null() { return rule("python-none", [this]() { - return sequence({literal("None"), space()}); + return literal("None"); }); } @@ -1457,6 +1486,13 @@ common_peg_parser common_peg_parser_builder::json_member(const std::string & key }); } +common_peg_parser common_peg_parser_builder::ac(const common_peg_parser & p, const std::vector & delimiters) { + if (delimiters.empty()) { + throw std::runtime_error("ac parser requires at least one delimiter"); + } + return add(common_peg_ac_parser{p, delimiters}); +} + static std::string gbnf_escape_char_class(uint32_t c) { if (c == '-' || c == ']' || c == '[' || c == '\\') { return "\\" + std::string(1, (char) c); @@ -1507,41 +1543,118 @@ static std::string gbnf_escape_char_class(uint32_t c) { return std::string(buf); } -static std::string gbnf_excluding_pattern(const std::vector & strings) { - trie matcher(strings); - auto pieces = matcher.collect_prefix_and_next(); - - std::string pattern; - for (size_t i = 0; i < pieces.size(); ++i) { - if (i > 0) { - pattern += " | "; - } - - const auto & pre = pieces[i].prefix; - const auto & chars = pieces[i].next_chars; - - std::string cls; - cls.reserve(chars.size()); - for (uint32_t ch : chars) { - cls += gbnf_escape_char_class(ch); - } - - if (!pre.empty()) { - pattern += gbnf_format_literal(common_unicode_cpts_to_utf8(pre)) + " [^" + cls + "]"; - } else { - pattern += "[^" + cls + "]"; - } +static std::string gbnf_char_class(const std::vector & chars, bool negate) { + std::string s = negate ? "[^" : "["; + for (uint32_t ch : chars) { + s += gbnf_escape_char_class(ch); } - - return "(" + pattern + ")*"; + return s + "]"; } -static std::unordered_set collect_reachable_rules( +static std::string gbnf_ac_grammar( + const common_grammar_builder & builder, + const std::string & prefix, + const std::vector & strings, + const std::function &, + const std::map> &, + const std::vector &, + const std::function &)> & build_rule) { + aho_corasick ac(strings); + + auto state_name = [&](size_t s) -> std::string { + if (s == 0) { + return prefix; + } + std::string num = std::to_string(s); + num = num.size() == 1 ? ("0" + num) : num; + return prefix + "-" + num; + }; + + for (size_t q = 0; q < ac.num_states(); q++) { + if (ac.is_terminal(q)) { + continue; // match states + } + + std::map> buckets; + std::vector completing; // chars that complete a delimiter + std::vector specific; // chars with an explicit transition + for (uint32_t c : ac.alphabet) { + size_t d = ac.next(q, c); + if (ac.is_terminal(d)) { + completing.push_back(c); + specific.push_back(c); + } else if (d != 0) { + buckets[d].push_back(c); // specific non-root destination + specific.push_back(c); + } + } + + builder.add_rule(state_name(q), build_rule(completing, buckets, specific, state_name)); + } + + // An empty delimiter makes the start state terminal. Emit an entry rule + // that matches the empty string so the returned reference stays valid. + if (ac.is_terminal(0)) { + builder.add_rule(prefix, "|"); + } + + return state_name(0); +} + +// GBNF grammar matching strings that contain no string in `strings` as a +// substring. Emits the complement of an Aho-Corasick automaton DFA and returns +// the start state rule name. +// +// ref: https://github.com/ggml-org/llama.cpp/pull/24839 +static std::string gbnf_excluding_grammar(const common_grammar_builder & builder, + const std::string & prefix, + const std::vector & strings) { + return gbnf_ac_grammar(builder, prefix, strings, + [](const std::vector & /*completing*/, + const std::map> & buckets, + const std::vector & specific, + const std::function & state_name) { + // every state is accepting and completing chars get no + // alternative, so a forbidden string can never be matched + std::string rhs = "|"; + for (const auto & [d, chars] : buckets) { + rhs += " " + gbnf_char_class(chars, false) + " " + state_name(d) + " |"; + } + rhs += " " + gbnf_char_class(specific, true) + " " + state_name(0); + return rhs; + }); +} + +// GBNF grammar matching everything up to and including the first occurrence of +// any string in `strings`. Emits the Aho-Corasick automaton DFA and returns +// the start state rule name. +static std::string gbnf_including_grammar(const common_grammar_builder & builder, + const std::string & prefix, + const std::vector & strings) { + return gbnf_ac_grammar(builder, prefix, strings, + [](const std::vector & completing, + const std::map> & buckets, + const std::vector & specific, + const std::function & state_name) { + std::vector alts; + if (!completing.empty()) { + alts.push_back(gbnf_char_class(completing, false)); // terminate on match + } + for (const auto & [d, chars] : buckets) { + alts.push_back(gbnf_char_class(chars, false) + " " + state_name(d)); + } + // every other character keeps scanning from the start state + alts.push_back(gbnf_char_class(specific, true) + " " + state_name(0)); + return string_join(alts, " | "); + }); +} + +static std::set collect_reachable_rules( const common_peg_arena & arena, const common_peg_parser_id & rule ) { - std::unordered_set reachable; - std::unordered_set visited; + std::set reachable; + std::set visited; std::function visit = [&](common_peg_parser_id id) { const auto & parser = arena.get(id); @@ -1573,6 +1686,7 @@ static std::unordered_set collect_reachable_rules( std::is_same_v || std::is_same_v || std::is_same_v || + std::is_same_v || std::is_same_v) { visit(p.child); } else if constexpr (std::is_same_v) { @@ -1750,7 +1864,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo if (p.delimiters.empty()) { return ".*"; } - return gbnf_excluding_pattern(p.delimiters); + return gbnf_excluding_grammar(builder, "until-" + std::to_string(id), p.delimiters); } else if constexpr (std::is_same_v) { if (schema_delegates(p)) { return to_gbnf(p.child); @@ -1767,6 +1881,8 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return to_gbnf(p.child); } else if constexpr (std::is_same_v) { return p.grammar; + } else if constexpr (std::is_same_v) { + return gbnf_including_grammar(builder, "ac-" + std::to_string(id), p.delimiters); } else { static_assert(is_always_false_v); } @@ -1774,7 +1890,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo }; // Collect reachable rules - std::unordered_set reachable_rules; + std::set reachable_rules; if (lazy) { // Collect rules reachable from trigger rules @@ -1903,6 +2019,8 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & }; } else if constexpr (std::is_same_v) { return json{{"type", "gbnf"}, {"child", p.child}, {"grammar", p.grammar}}; + } else if constexpr (std::is_same_v) { + return json{{"type", "ac"}, {"child", p.child}, {"delimiters", p.delimiters}}; } }, variant); } @@ -2075,6 +2193,16 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json }; } + if (type == "ac") { + if (!j.contains("child") || !j.contains("delimiters") || !j["delimiters"].is_array() || j["delimiters"].empty()) { + throw std::runtime_error("ac parser requires 'child' and a non-empty 'delimiters' array"); + } + return common_peg_ac_parser{ + j["child"].get(), + j["delimiters"].get>(), + }; + } + throw std::runtime_error("Unknown parser type: " + type); } diff --git a/common/peg-parser.h b/common/peg-parser.h index b6bb05214..c198499dd 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -3,8 +3,8 @@ #include #include +#include #include -#include #include #include #include @@ -275,6 +275,11 @@ struct common_peg_gbnf_parser { std::string grammar; }; +struct common_peg_ac_parser { + common_peg_parser_id child; + std::vector delimiters; +}; + // Variant holding all parser types using common_peg_parser_variant = std::variant< common_peg_epsilon_parser, @@ -296,7 +301,8 @@ using common_peg_parser_variant = std::variant< common_peg_ref_parser, common_peg_atomic_parser, common_peg_tag_parser, - common_peg_gbnf_parser + common_peg_gbnf_parser, + common_peg_ac_parser >; class common_peg_arena { @@ -335,7 +341,7 @@ class common_peg_arena { friend class common_peg_parser_builder; private: - std::string dump_impl(common_peg_parser_id id, std::unordered_set & visited) const; + std::string dump_impl(common_peg_parser_id id, std::set & visited) const; common_peg_parser_id add_parser(common_peg_parser_variant parser); void add_rule(const std::string & name, common_peg_parser_id id); @@ -514,6 +520,13 @@ class common_peg_parser_builder { // the child's grammar. Parsing delegates entirely to the child. common_peg_parser gbnf(const common_peg_parser & p, const std::string & grammar) { return add(common_peg_gbnf_parser{p, grammar}); } + // Wraps a child parser but emits a GBNF grammar built from the Aho-Corasick + // automaton of `delimiters`, matching everything up to and including the + // first delimiter. Parsing delegates entirely to the child, which is + // responsible for consuming the delimiter (e.g. until(D) + literal(D)). + common_peg_parser ac(const common_peg_parser & p, const std::vector & delimiters); + common_peg_parser ac(const common_peg_parser & p, const std::string & delimiter) { return ac(p, std::vector{delimiter}); } + void set_root(const common_peg_parser & p); common_peg_arena build(); diff --git a/common/preset.cpp b/common/preset.cpp index 51ea984d8..4362c0621 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -7,6 +7,7 @@ #include #include #include +#include static std::string rm_leading_dashes(const std::string & str) { size_t pos = 0; @@ -16,46 +17,21 @@ static std::string rm_leading_dashes(const std::string & str) { return str.substr(pos); } -// only allow a subset of args for remote presets for security reasons -// do not add more args unless absolutely necessary -// args that output to files are strictly prohibited -static std::set get_remote_preset_whitelist(const std::map & key_to_opt) { - static const std::set allowed_options = { - "model-url", - "hf-repo", - "hf-repo-draft", - "hf-repo-v", // vocoder - "hf-file-v", // vocoder - "mmproj-url", - "pooling", - "jinja", - "batch-size", - "ubatch-size", - "cache-reuse", - "chat-template-kwargs", - "mmap", - // note: sampling params are automatically allowed by default - // negated args will be added automatically if the positive arg is specified above - }; - - std::set allowed_keys; - - for (const auto & it : key_to_opt) { - const std::string & key = it.first; - const common_arg & opt = it.second; - if (allowed_options.find(key) != allowed_options.end() || opt.is_sampling) { - allowed_keys.insert(key); - // also add variant keys (args without leading dashes and env vars) - for (const auto & arg : opt.get_args()) { - allowed_keys.insert(rm_leading_dashes(arg)); - } - for (const auto & env : opt.get_env()) { - allowed_keys.insert(env); - } +static std::string canonical_tag(const std::string & tag) { + static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); + std::smatch m; + if (std::regex_search(tag, m, re_tag)) { + std::string canon = m[1].str(); + for (char & c : canon) { + c = (char) std::toupper((unsigned char) c); } + return canon; } - - return allowed_keys; + std::string upper = tag; + for (char & c : upper) { + c = (char) std::toupper((unsigned char) c); + } + return upper; } std::vector common_preset::to_args(const std::string & bin_path) const { @@ -300,16 +276,10 @@ static std::string parse_bool_arg(const common_arg & arg, const std::string & ke return value; } -common_preset_context::common_preset_context(llama_example ex, bool only_remote_allowed) +common_preset_context::common_preset_context(llama_example ex) : ctx_params(common_params_parser_init(default_params, ex)) { common_params_add_preset_options(ctx_params.options); key_to_opt = get_map_key_opt(ctx_params); - - // setup allowed keys if only_remote_allowed is true - if (only_remote_allowed) { - filter_allowed_keys = true; - allowed_keys = get_remote_preset_whitelist(key_to_opt); - } } common_presets common_preset_context::load_from_ini(const std::string & path, common_preset & global) const { @@ -318,11 +288,18 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co for (auto section : ini_data) { common_preset preset; - if (section.first.empty()) { - preset.name = COMMON_PRESET_DEFAULT_NAME; - } else { - preset.name = section.first; + std::string section_name = section.first.empty() ? std::string(COMMON_PRESET_DEFAULT_NAME) : section.first; + if (section_name != "*" && section_name != COMMON_PRESET_DEFAULT_NAME) { + auto colon_idx = section_name.rfind(':'); + if (colon_idx != std::string::npos) { + std::string tag = section_name.substr(colon_idx + 1); + std::string canon_tag = canonical_tag(tag); + if (canon_tag != tag) { + section_name = section_name.substr(0, colon_idx + 1) + canon_tag; + } + } } + preset.name = section_name; LOG_DBG("loading preset: %s\n", preset.name.c_str()); for (const auto & [key, value] : section.second) { if (key == "version") { diff --git a/common/preset.h b/common/preset.h index 06f829c3e..52935ebde 100644 --- a/common/preset.h +++ b/common/preset.h @@ -60,7 +60,7 @@ struct common_preset_context { std::set allowed_keys; // if only_remote_allowed is true, only accept whitelisted keys - common_preset_context(llama_example ex, bool only_remote_allowed = false); + common_preset_context(llama_example ex); // load presets from INI file common_presets load_from_ini(const std::string & path, common_preset & global) const; diff --git a/common/reasoning-budget.cpp b/common/reasoning-budget.cpp index ce41d029b..7da0bb1c5 100644 --- a/common/reasoning-budget.cpp +++ b/common/reasoning-budget.cpp @@ -65,12 +65,12 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to if (ctx->start_matcher.advance(token)) { ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; - LOG_INF("reasoning-budget: activated, budget=%d tokens\n", ctx->budget); + COM_TRC("activated, budget=%d tokens\n", ctx->budget); if (ctx->remaining <= 0) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; - LOG_INF("reasoning-budget: budget=0, forcing immediately\n"); + COM_TRC("%s", "budget=0, forcing immediately\n"); } } break; @@ -80,7 +80,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to { if (ctx->end_matcher.advance(token)) { ctx->state = REASONING_BUDGET_DONE; - LOG_INF("reasoning-budget: deactivated (natural end)\n"); + COM_TRC("%s", "deactivated (natural end)\n"); break; } @@ -95,7 +95,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - LOG_INF("reasoning-budget: UTF-8 complete, now forcing end sequence\n"); + COM_TRC("%s", "UTF-8 complete, now forcing end sequence\n"); } } else if (ctx->state == REASONING_BUDGET_COUNTING) { ctx->remaining--; @@ -104,11 +104,11 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - LOG_INF("reasoning-budget: budget exhausted, forcing end sequence\n"); + COM_TRC("%s", "budget exhausted, forcing end sequence\n"); } else { ctx->state = REASONING_BUDGET_WAITING_UTF8; ctx->end_matcher.reset(); - LOG_INF("reasoning-budget: budget exhausted, waiting for UTF-8 completion\n"); + COM_TRC("%s", "budget exhausted, waiting for UTF-8 completion\n"); } } } @@ -118,7 +118,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->force_pos++; if (ctx->force_pos >= ctx->forced_tokens.size()) { ctx->state = REASONING_BUDGET_DONE; - LOG_INF("reasoning-budget: forced sequence complete, done\n"); + COM_TRC("%s", "forced sequence complete, done\n"); } break; case REASONING_BUDGET_DONE: @@ -128,12 +128,12 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; ctx->end_matcher.reset(); - LOG_INF("reasoning-budget: re-activated on new start tag, budget=%d tokens\n", ctx->budget); + COM_TRC("re-activated on new start tag, budget=%d tokens\n", ctx->budget); if (ctx->remaining <= 0) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; - LOG_INF("reasoning-budget: budget=0, forcing immediately\n"); + COM_TRC("%s", "budget=0, forcing immediately\n"); } } break; @@ -264,7 +264,7 @@ bool common_reasoning_budget_force(struct llama_sampler * smpl) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - LOG_INF("reasoning-budget: forced into forcing state (manual transition)\n"); + COM_TRC("%s", "forced into forcing state (manual transition)\n"); return true; } diff --git a/common/regex-partial.cpp b/common/regex-partial.cpp deleted file mode 100644 index bd9034e93..000000000 --- a/common/regex-partial.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include "regex-partial.h" -#include "common.h" -#include -#include - -common_regex::common_regex(const std::string & pattern) : - pattern(pattern), - rx(pattern), - rx_reversed_partial(regex_to_reversed_partial_regex(pattern)) {} - -common_regex_match common_regex::search(const std::string & input, size_t pos, bool as_match) const { - std::smatch match; - if (pos > input.size()) { - throw std::runtime_error("Position out of bounds"); - } - auto start = input.begin() + pos; - auto found = as_match - ? std::regex_match(start, input.end(), match, rx) - : std::regex_search(start, input.end(), match, rx); - if (found) { - common_regex_match res; - res.type = COMMON_REGEX_MATCH_TYPE_FULL; - for (size_t i = 0; i < match.size(); ++i) { - auto begin = pos + match.position(i); - res.groups.emplace_back(begin, begin + match.length(i)); - } - return res; - } - std::match_results srmatch; - if (std::regex_search(input.rbegin(), input.rend() - pos, srmatch, rx_reversed_partial, std::regex_constants::match_continuous)) { - auto group = srmatch[1].str(); - if (group.length() != 0) { - auto it = srmatch[1].second.base(); - // auto position = static_cast(std::distance(input.begin(), it)); - if ((!as_match) || it == input.begin()) { - common_regex_match res; - res.type = COMMON_REGEX_MATCH_TYPE_PARTIAL; - const size_t begin = std::distance(input.begin(), it); - const size_t end = input.size(); - if (begin == std::string::npos || end == std::string::npos || begin > end) { - throw std::runtime_error("Invalid range"); - } - res.groups.push_back({begin, end}); - return res; - } - } - } - return {}; -} - -/* - Transforms a regex pattern to a partial match pattern that operates on a reversed input string to find partial final matches of the original pattern. - - Ideally we'd like to use boost::match_partial (https://beta.boost.org/doc/libs/1_59_0/libs/regex/doc/html/boost_regex/partial_matches.html) - to see if a string ends with a partial regex match, but but it's not in std::regex yet. - Instead, we'll the regex into a partial match regex operating as a full match on the reverse iterators of the input. - - - /abcd/ -> ^(dcba|cba|ba|a) -> ^((?:(?:(?:(?:d)?c)?b)?a) - - /a|b/ -> ^(a|b) - - /a*?/ -> error, could match "" - - /a*b/ -> ^((?:b)?a*+) (final repetitions become eager) - - /.*?ab/ -> ^((?:b)?a) (omit .*) - - /a.*?b/ -> ^((?:b)?.*?a) (keep reluctant matches) - - /a(bc)d/ -> ^((?:(?:d)?(?:(?:c)?b))?a) - - /a(bc|de)/ -> ^((?:(?:(?:e)?d)?|(?:(?:c)?b)?)?a) - - /ab{2,4}c/ -> ^cbbb?b?a -> ^((?:(?:(?:(?:(?:c)?b)?b)?b?)?b?)?a) - - The regex will match a reversed string fully, and the end of the first (And only) capturing group will indicate the reversed start of the original partial pattern. - All other groups are turned into non-capturing groups, and reluctant quantifiers are ignored. -*/ -std::string regex_to_reversed_partial_regex(const std::string & pattern) { - auto it = pattern.begin(); - const auto end = pattern.end(); - - std::function process = [&]() { - std::vector> alternatives(1); - std::vector * sequence = &alternatives.back(); - - while (it != end) { - if (*it == '[') { - auto start = it; - ++it; - while (it != end) { - if ((*it == '\\') && (++it != end)) { - ++it; - } else if ((it != end) && (*it == ']')) { - break; - } else { - ++it; - } - } - if (it == end) { - throw std::runtime_error("Unmatched '[' in pattern"); - } - ++it; - sequence->push_back(std::string(start, it)); - } else if (*it == '*' || *it == '?' || *it == '+') { - if (sequence->empty()) { - throw std::runtime_error("Quantifier without preceding element"); - } - sequence->back() += *it; - auto is_star = *it == '*'; - ++it; - if (is_star) { - if (it != end && *it == '?') { - ++it; - } - } - } else if (*it == '{') { - if (sequence->empty()) { - throw std::runtime_error("Repetition without preceding element"); - } - ++it; - auto start = it; - while (it != end && *it != '}') { - ++it; - } - if (it == end) { - throw std::runtime_error("Unmatched '{' in pattern"); - } - auto parts = string_split(std::string(start, it), ","); - ++it; - if (parts.size() > 2) { - throw std::runtime_error("Invalid repetition range in pattern"); - } - - auto parseOptInt = [&](const std::string & s, const std::optional & def = std::nullopt) -> std::optional { - if (s.empty()) { - return def; - } - return std::stoi(s); - }; - auto min = parseOptInt(parts[0], 0); - auto max = parts.size() == 1 ? min : parseOptInt(parts[1]); - if (min && max && *max < *min) { - throw std::runtime_error("Invalid repetition range in pattern"); - } - // Brutal but... let's repeat at least min times, then ? for the delta between min & max (or * for unbounded) - auto part = sequence->back(); - sequence->pop_back(); - for (int i = 0; i < *min; i++) { - sequence->push_back(part); - } - if (max) { - for (int i = *min; i < *max; i++) { - sequence->push_back(part + "?"); - } - } else { - sequence->push_back(part + "*"); - } - } else if (*it == '(') { - ++it; - if (it != end && *it == '?' && (it + 1 != end) && *(it + 1) == ':') { - it += 2; - } - auto sub = process(); - if (*it != ')') { - throw std::runtime_error("Unmatched '(' in pattern"); - } - ++it; - auto & part = sequence->emplace_back("(?:"); - part += sub; - part += ")"; - } else if (*it == ')') { - break; - } else if (*it == '|') { - ++it; - alternatives.emplace_back(); - sequence = &alternatives.back(); - } else if (*it == '\\' && (++it != end)) { - auto str = std::string("\\") + *it; - sequence->push_back(str); - ++it; - } else if (it != end) { - sequence->push_back(std::string(1, *it)); - ++it; - } - } - - // /abcd/ -> ^(dcba|cba|ba|a) -> ^((?:(?:(?:d)?c)?b)?a) - // if n(=4) parts, opening n-1(=3) non-capturing groups after the 1 capturing group - // We'll do the outermost capturing group and final .* in the enclosing function. - std::vector res_alts; - for (const auto & parts : alternatives) { - auto & res = res_alts.emplace_back(); - for (size_t i = 0; i < parts.size() - 1; i++) { - res += "(?:"; - } - for (auto it = parts.rbegin(); it != parts.rend(); ++it) { - res += *it; - if (it != parts.rend() - 1) { - res += ")?"; - } - } - } - return string_join(res_alts, "|"); - }; - auto res = process(); - if (it != end) { - throw std::runtime_error("Unmatched '(' in pattern"); - } - - return "^(" + res + ")"; -} diff --git a/common/regex-partial.h b/common/regex-partial.h deleted file mode 100644 index 634cb4022..000000000 --- a/common/regex-partial.h +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#include -#include - -enum common_regex_match_type { - COMMON_REGEX_MATCH_TYPE_NONE, - COMMON_REGEX_MATCH_TYPE_PARTIAL, - COMMON_REGEX_MATCH_TYPE_FULL, -}; - -struct common_string_range { - size_t begin; - size_t end; - common_string_range(size_t begin, size_t end) : begin(begin), end(end) { - if (begin > end) { - throw std::runtime_error("Invalid range"); - } - } - // prevent default ctor - common_string_range() = delete; - bool empty() const { - return begin == end; - } - bool operator==(const common_string_range & other) const { - return begin == other.begin && end == other.end; - } -}; - -struct common_regex_match { - common_regex_match_type type = COMMON_REGEX_MATCH_TYPE_NONE; - std::vector groups; - - bool operator==(const common_regex_match & other) const { - return type == other.type && groups == other.groups; - } - bool operator!=(const common_regex_match & other) const { - return !(*this == other); - } -}; - -class common_regex { - std::string pattern; - std::regex rx; - std::regex rx_reversed_partial; - - public: - explicit common_regex(const std::string & pattern); - - common_regex_match search(const std::string & input, size_t pos, bool as_match = false) const; - - const std::string & str() const { return pattern; } -}; - -// For testing only (pretty print of failures). -std::string regex_to_reversed_partial_regex(const std::string & pattern); diff --git a/common/sampling.cpp b/common/sampling.cpp index c537f3350..75a299e23 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -259,6 +259,9 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st } } } + if (!grmr && !grammar_str.empty()) { + throw std::runtime_error("failed to parse grammar"); + } // Compute prefill tokens from the generation prompt std::vector prefill_tokens; diff --git a/common/speculative.cpp b/common/speculative.cpp index 5d1a191ae..4e0439216 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -18,6 +18,13 @@ #include #include +#define SPC_DBG(fmt, ...) LOG_DBG("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SPC_TRC(fmt, ...) LOG_TRC("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SPC_INF(fmt, ...) LOG_INF("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SPC_WRN(fmt, ...) LOG_WRN("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SPC_ERR(fmt, ...) LOG_ERR("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) +#define SPC_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__) + #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128 #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 @@ -26,6 +33,7 @@ const std::map common_speculative_type_fro {"draft-simple", COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE}, {"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}, {"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP}, + {"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}, {"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, {"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, {"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, @@ -60,21 +68,20 @@ static bool common_speculative_are_compatible( const llama_vocab * vocab_dft = llama_model_get_vocab(model_dft); const auto vocab_type_tgt = llama_vocab_type(vocab_tgt); - LOG_DBG("%s: vocab_type tgt: %d\n", __func__, vocab_type_tgt); + SPC_DBG("vocab_type tgt: %d\n", vocab_type_tgt); const auto vocab_type_dft = llama_vocab_type(vocab_dft); - LOG_DBG("%s: vocab_type dft: %d\n", __func__, vocab_type_dft); + SPC_DBG("vocab_type dft: %d\n", vocab_type_dft); if (vocab_type_tgt != vocab_type_dft) { - LOG_WRN("%s: draft model vocab type must match target model to use speculation but " - "vocab_type_dft = %d while vocab_type_tgt = %d\n", __func__, vocab_type_dft, vocab_type_tgt); + SPC_WRN("draft model vocab type must match target model to use speculation but " + "vocab_type_dft = %d while vocab_type_tgt = %d\n", vocab_type_dft, vocab_type_tgt); return false; } if (llama_vocab_get_add_bos(vocab_tgt) != llama_vocab_get_add_bos(vocab_dft) || (llama_vocab_get_add_bos(vocab_tgt) && llama_vocab_bos(vocab_tgt) != llama_vocab_bos(vocab_dft))) { - LOG_WRN("%s: draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", - __func__, + SPC_WRN("draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", llama_vocab_get_add_bos(vocab_tgt), llama_vocab_get_add_bos(vocab_dft), llama_vocab_bos(vocab_tgt), llama_vocab_bos(vocab_dft)); return false; @@ -82,8 +89,7 @@ static bool common_speculative_are_compatible( if (llama_vocab_get_add_eos(vocab_tgt) != llama_vocab_get_add_eos(vocab_dft) || (llama_vocab_get_add_eos(vocab_tgt) && llama_vocab_eos(vocab_tgt) != llama_vocab_eos(vocab_dft))) { - LOG_WRN("%s: draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", - __func__, + SPC_WRN("draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", llama_vocab_get_add_eos(vocab_tgt), llama_vocab_get_add_eos(vocab_dft), llama_vocab_eos(vocab_tgt), llama_vocab_eos(vocab_dft)); return false; @@ -97,8 +103,8 @@ static bool common_speculative_are_compatible( : n_vocab_dft - n_vocab_tgt; if (vocab_diff > SPEC_VOCAB_MAX_SIZE_DIFFERENCE) { - LOG_DBG("%s: draft model vocab must closely match target model to use speculation but ", __func__); - LOG_DBG("target vocab size %d does not match draft vocab size %d - difference %d, max allowed %d\n", + SPC_DBG("draft model vocab must closely match target model to use speculation but " + "target vocab size %d does not match draft vocab size %d - difference %d, max allowed %d\n", n_vocab_tgt, llama_vocab_n_tokens(vocab_dft), vocab_diff, SPEC_VOCAB_MAX_SIZE_DIFFERENCE); return false; } @@ -108,8 +114,8 @@ static bool common_speculative_are_compatible( const char * token_text_dft = llama_vocab_get_text(vocab_dft, i); if (std::strcmp(token_text_tgt, token_text_dft) != 0) { - LOG_DBG("%s: draft model vocab must match target model to use speculation but ", __func__); - LOG_DBG("token %d content differs - target '%s', draft '%s'\n", i, + SPC_DBG("draft model vocab must match target model to use speculation but " + "token %d content differs - target '%s', draft '%s'\n", i, common_token_to_piece(vocab_tgt, i).c_str(), common_token_to_piece(vocab_dft, i).c_str()); return false; @@ -140,6 +146,8 @@ struct common_speculative_impl { size_t n_gen_tokens = 0; // number of tokens generated by this implementation. size_t n_acc_tokens = 0; // number of tokens accepted by the target model. + std::vector n_acc_tokens_per_pos; // number of tokens accepted per draft position. + // TODO: track performance of most recent calls const bool gen_perf = true; // whether to generate performance stats. @@ -159,6 +167,10 @@ struct common_speculative_impl { virtual void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) = 0; + // (optional) serialize/restore per-seq internal state (e.g. eagle3's deferred boundary). + virtual bool get_state(llama_seq_id /*seq_id*/, std::vector & /*data*/) const { return false; } + virtual void set_state(llama_seq_id /*seq_id*/, const std::vector & /*data*/) {} + // true if this implementation requires the target context to extract post-norm embeddings virtual bool need_embd() const = 0; @@ -180,9 +192,9 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; auto * ctx_tgt = this->params.ctx_tgt; - LOG_INF("%s: adding speculative implementation 'draft-simple'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); - LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, + SPC_TRC("%s", "adding speculative implementation 'draft-simple'\n"); + SPC_TRC("- n_max=%d, n_min=%d, p_min=%f\n", this->params.n_max, this->params.n_min, this->params.p_min); + SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", this->params.n_gpu_layers, ggml_type_name(this->params.cache_type_k), ggml_type_name(this->params.cache_type_v), @@ -222,16 +234,16 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { } const bool vocab_cmpt = common_speculative_are_compatible(llama_get_model(ctx_tgt), llama_get_model(ctx_dft)); - LOG_DBG("%s: vocab_cmpt = %d\n", __func__, vocab_cmpt); + SPC_DBG("vocab_cmpt = %d\n", vocab_cmpt); if (!vocab_cmpt) { - LOG_ERR("%s: the target and draft vocabs are not compatible\n", __func__); + SPC_ERR("%s", "the target and draft vocabs are not compatible\n"); throw std::runtime_error("draft model vocab type must match target model to use speculation"); } if (n_seq != llama_n_seq_max(ctx_dft)) { - LOG_ERR("%s: n_seq mismatch: %d != %d\n", __func__, n_seq, llama_n_seq_max(ctx_dft)); + SPC_ERR("n_seq mismatch: %d != %d\n", n_seq, llama_n_seq_max(ctx_dft)); throw std::runtime_error("the draft model number of sequences is incompatible with the speculative n_seq"); } @@ -251,7 +263,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { const int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - LOG_ERR("%s: failed to decode draft batch, ret = %d\n", __func__, ret); + SPC_ERR("failed to decode draft batch, ret = %d\n", ret); return false; } @@ -284,7 +296,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); + SPC_ERR("llama_decode returned %d\n", ret); return; } @@ -308,7 +320,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -348,7 +360,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { // evaluate the drafted tokens on the draft model ret = llama_decode(ctx_dft, batch); if (ret != 0) { - LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); + SPC_ERR("llama_decode[%d] returned %d\n", i, ret); break; } @@ -416,6 +428,9 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { std::vector smpls; + // backend sampler chain per seq, attached to ctx_dft + std::vector backend_chains; + int32_t n_embd_dec = 0; // draft hidden size int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size int32_t n_embd_tgt = 0; // target model hidden size @@ -440,8 +455,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq) , params(params.draft) { - LOG_INF("%s: adding speculative implementation 'draft-eagle3'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, params.draft.n_max, params.draft.n_min, params.draft.p_min); + SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n"); + SPC_TRC("- n_max=%d, n_min=%d, p_min=%f, backend_sampling=%d\n", params.draft.n_max, params.draft.n_min, params.draft.p_min, (int) params.draft.backend_sampling); auto * ctx_tgt = this->params.ctx_tgt; auto * ctx_dft = this->params.ctx_dft; @@ -476,6 +491,22 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { s.reset(common_sampler_init(llama_get_model(ctx_dft), sparams)); } + // offload draft sampling to the backend + backend_chains.assign(n_seq, nullptr); + if (this->params.backend_sampling) { + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); + llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); + + if (!llama_set_sampler(ctx_dft, seq_id, chain)) { + SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); + llama_sampler_free(chain); + chain = nullptr; + } + backend_chains[seq_id] = chain; + } + } + // turn on extraction of the target layers' input embeddings for (uint32_t k = 0; k < target_layer_ids_n; ++k) { llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); @@ -494,6 +525,18 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { } ~common_speculative_impl_draft_eagle3() override { + auto * ctx_dft = this->params.ctx_dft; + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) backend_chains.size(); ++seq_id) { + if (backend_chains[seq_id] == nullptr) { + continue; + } + if (ctx_dft) { + llama_set_sampler(ctx_dft, seq_id, nullptr); + } + llama_sampler_free(backend_chains[seq_id]); + } + backend_chains.clear(); + if (batch.token != nullptr) { free(batch.token); batch.token = nullptr; @@ -511,9 +554,9 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); if (pos_max < N - 2) { - LOG_WRN("%s: ctx_dft pos_max=%d < N-2=%d — process() did not run on every prefill ubatch. " + SPC_WRN("ctx_dft pos_max=%d < N-2=%d — process() did not run on every prefill ubatch. " "Drafts may degrade.\n", - __func__, (int) pos_max, N - 2); + (int) pos_max, N - 2); } } @@ -584,8 +627,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { }; const int32_t rc = llama_encode(ctx_dft, enc_batch); if (rc != 0) { - LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", - __func__, rc, (int) n_chunk, (int) i); + SPC_ERR("llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", + rc, (int) n_chunk, (int) i); return false; } @@ -655,8 +698,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { if (batch.n_tokens > 0) { const int32_t rc = llama_decode(ctx_dft, batch); if (rc != 0) { - LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, ubatch_pos[0]=%d)\n", - __func__, rc, (int) batch.n_tokens, (int) batch_in.pos[0]); + SPC_ERR("llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, ubatch_pos[0]=%d)\n", + rc, (int) batch.n_tokens, (int) batch_in.pos[0]); return false; } } @@ -707,7 +750,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); + SPC_ERR("llama_decode returned %d\n", ret); return; } @@ -733,7 +776,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -772,7 +815,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { ret = llama_decode(ctx_dft, batch); if (ret != 0) { - LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); + SPC_ERR("llama_decode[%d] returned %d\n", i, ret); break; } @@ -808,6 +851,348 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { (size_t) n_embd_dec * sizeof(float)); } + // we only need to stash the deferred boundary's g_embd row for recurrent/hybrid targets: + // their single-position checkpoints drop it on restore + bool need_boundary_stash() const { + const llama_model * model_tgt = llama_get_model(params.ctx_tgt); + return llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt); + } + + bool get_state(llama_seq_id seq_id, std::vector & data) const override { + if (!need_boundary_stash()) { + return false; + } + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq || pending_pos_last[seq_id] < 0) { + return false; + } + + const llama_pos pos = pending_pos_last[seq_id]; + const std::vector & g = pending_g_last[seq_id]; + + data.resize(sizeof(llama_pos) + g.size() * sizeof(float)); + std::memcpy(data.data(), &pos, sizeof(llama_pos)); + std::memcpy(data.data() + sizeof(llama_pos), g.data(), g.size() * sizeof(float)); + return true; + } + + void set_state(llama_seq_id seq_id, const std::vector & data) override { + if (!need_boundary_stash()) { + return; + } + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + return; + } + if (data.size() != sizeof(llama_pos) + (size_t) n_embd_dec * sizeof(float)) { + return; + } + + llama_pos pos = -1; + std::memcpy(&pos, data.data(), sizeof(llama_pos)); + + pending_pos_last[seq_id] = pos; + pending_g_last[seq_id].resize(n_embd_dec); + std::memcpy(pending_g_last[seq_id].data(), data.data() + sizeof(llama_pos), (size_t) n_embd_dec * sizeof(float)); + } + + bool need_embd() const override { + return false; + } +}; + +// DFlash: block-diffusion drafting with a draft-side KV cache injection +struct common_speculative_impl_draft_dflash : public common_speculative_impl { + common_params_speculative_draft params; + + llama_batch batch; // noise tokens + llama_batch batch_inject; // target features for KV cache injection + + std::vector smpls; + + int32_t n_embd_dec = 0; // draft hidden size + int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size + int32_t n_embd_tgt = 0; // target model hidden size + + int32_t block_size = 0; + llama_token mask_token_id = 0; + + const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices + uint32_t target_layer_ids_n = 0; + + // scratch buffer for concatenated target features [n_tokens, n_embd_enc] + std::vector features_buf; + + common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq) + : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq) + , params(params.draft) + { + auto * ctx_tgt = this->params.ctx_tgt; + auto * ctx_dft = this->params.ctx_dft; + GGML_ASSERT(ctx_tgt && ctx_dft && "DFlash requires ctx_tgt and ctx_dft to be set"); + + const llama_model * model_dft = llama_get_model(ctx_dft); + const llama_model * model_tgt = llama_get_model(ctx_tgt); + + target_layer_ids = llama_model_target_layer_ids (model_dft); + target_layer_ids_n = llama_model_target_layer_ids_n(model_dft); + GGML_ASSERT(target_layer_ids_n > 0 && "DFlash model has no target_layer_ids"); + + n_embd_tgt = llama_model_n_embd(model_tgt); + n_embd_dec = llama_model_n_embd(model_dft); + n_embd_enc = (int32_t) target_layer_ids_n * n_embd_tgt; + + // read the trained block size from the dflash.block_size metadata key + block_size = 16; + { + char buf[32] = {}; + if (llama_model_meta_val_str(model_dft, "dflash.block_size", buf, sizeof(buf)) >= 0) { + block_size = std::atoi(buf); + } + } + mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft)); + + LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); + LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n); + + // DFlash input is [id_last, * (block_size-1)], so it can draft at most block_size-1 tokens per step + if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) { + LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n", + __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1); + this->params.n_max = std::min(this->params.n_max, block_size - 1); + this->params.n_min = std::min(this->params.n_min, block_size - 1); + } + + batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq); + batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq); + + smpls.resize(n_seq); + for (auto & s : smpls) { + common_params_sampling sparams; + sparams.no_perf = false; + sparams.top_k = 10; + sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K }; + s.reset(common_sampler_init(model_dft, sparams)); + } + + // turn on extraction of the target layers' input embeddings + for (uint32_t k = 0; k < target_layer_ids_n; ++k) { + llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); + } + + llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); + llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention + } + + ~common_speculative_impl_draft_dflash() override { + llama_batch_free(batch); + llama_batch_free(batch_inject); + } + + void begin(llama_seq_id seq_id, const llama_tokens & prompt) override { + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + return; + } + + const int32_t N = (int32_t) prompt.size(); + if (N <= 0) { + return; + } + + const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(params.ctx_dft), seq_id); + if (pos_max < N - 1) { + LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - process() did not run on every prefill ubatch. " + "Drafts may degrade.\n", + __func__, (int) pos_max, N - 1); + } + } + + bool process(const llama_batch & batch_in) override { + if (batch_in.n_tokens <= 0) { + return true; + } + + if (batch_in.token == nullptr || batch_in.embd != nullptr) { + return true; + } + + const int32_t n_tokens = batch_in.n_tokens; + + // per-seq inclusive batch range (assumes each seq's tokens are contiguous in the batch) + std::vector i_batch_beg(n_seq, -1); + std::vector i_batch_end(n_seq, -1); + for (int32_t k = 0; k < n_tokens; ++k) { + GGML_ASSERT(batch_in.n_seq_id[k] == 1); + const llama_seq_id seq_id = batch_in.seq_id[k][0]; + if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { + continue; + } + i_batch_end[seq_id] = k; + if (i_batch_beg[seq_id] < 0) { + i_batch_beg[seq_id] = k; + } + } + + auto * ctx_tgt = this->params.ctx_tgt; + auto * ctx_dft = this->params.ctx_dft; + + const int32_t n_ubatch = (int32_t) llama_n_ubatch(ctx_dft); + + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + if (i_batch_beg[seq_id] < 0) { + continue; + } + const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; + + for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) { + const int32_t n_chunk = std::min(n_ubatch, n_rows - offset); + + // gather this chunk's target features, interleaved by extract layer + features_buf.resize((size_t) n_chunk * n_embd_enc); + for (uint32_t k = 0; k < target_layer_ids_n; ++k) { + const float * layer = llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]); + if (!layer) { + GGML_ABORT("DFlash: target layer %d input not extracted.", target_layer_ids[k]); + } + for (int32_t i = 0; i < n_chunk; ++i) { + float * dst = features_buf.data() + (size_t) i * n_embd_enc + k * (size_t) n_embd_tgt; + const float * src = layer + (size_t) (i_batch_beg[seq_id] + offset + i) * n_embd_tgt; + std::memcpy(dst, src, (size_t) n_embd_tgt * sizeof(float)); + } + } + + // fuse extracted features through DFlash encoder + llama_batch enc_batch = { + /*.n_tokens =*/ n_chunk, + /*.token =*/ nullptr, + /*.embd =*/ features_buf.data(), + /*.pos =*/ nullptr, + /*.n_seq_id =*/ nullptr, + /*.seq_id =*/ nullptr, + /*.logits =*/ nullptr, + }; + + int32_t rc = llama_encode(ctx_dft, enc_batch); + if (rc != 0) { + LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", + __func__, rc, (int) n_chunk, (int) offset); + return false; + } + + const float * inp_g = llama_get_embeddings_nextn(ctx_dft); + GGML_ASSERT(inp_g && "DFlash encoder produced no output."); + + // inject the DFlash decoder K/V cache at the tokens' target positions + batch_inject.n_tokens = n_chunk; + std::memcpy(batch_inject.embd, inp_g, (size_t) n_chunk * n_embd_dec * sizeof(float)); + + for (int32_t i = 0; i < n_chunk; ++i) { + batch_inject.pos[i] = batch_in.pos[i_batch_beg[seq_id] + offset + i]; + batch_inject.n_seq_id[i] = 1; + batch_inject.seq_id[i][0] = seq_id; + batch_inject.logits[i] = false; + } + rc = llama_decode(ctx_dft, batch_inject); + if (rc != 0) { + LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", + __func__, rc, (int) n_chunk, (int) offset); + return false; + } + } + } + + return true; + } + + void draft(common_speculative_draft_params_vec & dparams) override { + auto & ctx_dft = params.ctx_dft; + + common_batch_clear(batch); + + // build one batch holding every drafting sequence's noise block into a single decode) + // record where each block starts and its size + std::vector i_block_beg(n_seq, -1); + std::vector n_block (n_seq, 0); + + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + auto & dp = dparams[seq_id]; + if (!dp.drafting) { + continue; + } + + common_sampler_reset(smpls[seq_id].get()); + + const int32_t n = (int32_t) dp.n_past; + + int32_t n_draft = params.n_max; + if (dp.n_max > 0) { + n_draft = std::min(n_draft, dp.n_max); + } + + const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * + i_block_beg[seq_id] = batch.n_tokens; + n_block [seq_id] = n_block_tokens; + for (int32_t i = 0; i < n_block_tokens; ++i) { + common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, true); + } + } + + if (batch.n_tokens == 0) { + return; + } + + // decode all sequence's noise block in a single batch + int ret = llama_decode(ctx_dft, batch); + if (ret != 0) { + LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); + return; + } + + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + if (i_block_beg[seq_id] < 0) { + continue; + } + auto & dp = dparams[seq_id]; + + const int32_t beg = i_block_beg[seq_id]; + const int32_t n_block_tokens = n_block[seq_id]; + + auto * smpl = smpls[seq_id].get(); + + auto & result = *dp.result; + + // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1 + for (int32_t i = 1; i < n_block_tokens; ++i) { + common_sampler_sample(smpl, ctx_dft, beg + i, true); + + const auto * cur_p = common_sampler_get_candidates(smpl, true); + + for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p, + common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); + } + + const llama_token id = cur_p->data[0].id; + + if (cur_p->data[0].p < params.p_min) { + break; + } + + common_sampler_accept(smpl, id, true); + + result.push_back(id); + } + + if (result.size() < (size_t) params.n_min) { + result.clear(); + } + } + } + + void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { + // noop + } + bool need_embd() const override { return false; } @@ -825,7 +1210,13 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { int32_t n_embd = 0; - bool is_mem_shared = false; + // One MTP draft driver, three modes (set once in the ctor): + // is_mem_shared (gemma4): shares the target KV, runs all heads in one graph. + // chain_heads (step35): n_mtp_layers trained heads, one per draft step. + // neither (qwen35 / qwen35moe): a single trained MTP head. + int32_t n_mtp_layers = 1; + bool is_mem_shared = false; // gemma4 + bool chain_heads = false; // derived in the ctor: n_mtp_layers > 1 && !is_mem_shared // Per-sequence cross-batch carryover: pair (h_p, x_{p+1}) at MTP pos p+1. // The last h-row of one process() call needs the first token of the NEXT @@ -840,10 +1231,8 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::vector> verify_h; std::vector verify_h_rows; - // Per-seq draft length from the last draft() call, used in accept() to - // roll back ctx_dft's recurrent state past the AR draft's redundant - // pre-advancement before process() mirrored the verify batch. - std::vector last_n_drafted; + std::vector i_last; + std::vector> chain_h; common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq) : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq) @@ -856,10 +1245,11 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft)); GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) && "MTP input row width must match the target h_nextn width"); + n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); - LOG_INF("%s: adding speculative implementation 'draft-mtp'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); - LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, + SPC_TRC("%s", "adding speculative implementation 'draft-mtp'\n"); + SPC_TRC("- n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); + SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", this->params.n_gpu_layers, ggml_type_name(this->params.cache_type_k), ggml_type_name(this->params.cache_type_v), @@ -890,7 +1280,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); if (!llama_set_sampler(ctx_dft, seq_id, chain)) { - LOG_WRN("%s: backend offload failed for seq_id=%d; using CPU sampler\n", __func__, (int) seq_id); + SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); llama_sampler_free(chain); chain = nullptr; } @@ -902,16 +1292,25 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; + chain_heads = n_mtp_layers > 1 && !is_mem_shared; + + if (chain_heads) { + this->params.n_max = std::min(this->params.n_max, n_mtp_layers); + + chain_h.assign(n_seq, {}); + for (auto & c : chain_h) { + c.reserve((size_t) (this->params.n_max + 1) * n_embd); + } + } pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); + i_last.assign(n_seq, -1); i_batch_beg.assign(n_seq, -1); i_batch_end.assign(n_seq, -1); verify_h.assign(n_seq, {}); verify_h_rows.assign(n_seq, 0); - - last_n_drafted.assign(n_seq, 0); } ~common_speculative_impl_draft_mtp() override { @@ -944,11 +1343,11 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); if (pos_max < N - 1 && !is_mem_shared) { - LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - " + SPC_WRN("ctx_dft pos_max=%d < N-1=%d - " "process() hook may not have run on every prefill ubatch " "(need_embd / logits=1 on every prompt position?). " "Drafts may degrade.\n", - __func__, (int) pos_max, N - 1); + (int) pos_max, N - 1); } } @@ -1017,9 +1416,34 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { set_h(i_batch_beg[seq_id], pending_h[seq_id].data()); } - const int32_t rc = llama_decode(ctx_dft, batch); - if (rc != 0) { - LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (pos=%d)\n", __func__, (int) rc, (int) batch_in.pos[0]); + auto * mem_dft = llama_get_memory(ctx_dft); + + bool ok = true; + for (int head = 0; head < n_mtp_layers; ++head) { + if (chain_heads) { + // ref: https://github.com/ggml-org/llama.cpp/pull/24340/changes#r3413498544 + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + if (i_batch_beg[seq_id] < 0) { + continue; + } + llama_memory_seq_rm(mem_dft, seq_id, batch_in.pos[i_batch_beg[seq_id]], -1); + } + llama_set_nextn_layer_offset(ctx_dft, head); + } + + const int32_t rc = llama_decode(ctx_dft, batch); + if (rc != 0) { + SPC_ERR("llama_decode(ctx_dft) head=%d failed rc=%d (pos=%d)\n", + head, (int) rc, (int) batch_in.pos[0]); + ok = false; + break; + } + } + + if (chain_heads) { + llama_set_nextn_layer_offset(ctx_dft, 0); // restore default for non-draft decodes + } + if (!ok) { return false; } } @@ -1054,7 +1478,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { int n_drafting = 0; std::vector drafting(n_seq); - const float * h_row = nullptr; const size_t row_bytes = (size_t) n_embd * sizeof(float); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { @@ -1069,22 +1492,43 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { common_sampler_reset(smpls[seq_id].get()); common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); + std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); - h_row = pending_h[seq_id].data(); - std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); - } + i_last[seq_id] = batch.n_tokens - 1; - int ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); - return; + if (chain_heads) { + chain_h[seq_id].assign(pending_h[seq_id].begin(), pending_h[seq_id].end()); + } } int i = 0; while (n_drafting > 0) { - int i_batch = 0; + // each step decodes under a different head, i.e. a different decoder layer, and + // KV is per layer. process() filled this layer's KV only for positions < n_past + // (prompt + accepted prefix) — nothing in the draft region yet. so reset the + // draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact) + // and select head i so it rebuilds its own layer's KV there; decoding just the + // latest token would leave its attention reading cells only another head wrote. + if (chain_heads) { + auto * mem_dft = llama_get_memory(ctx_dft); + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { + if (drafting[seq_id]) { + llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1); + } + } + llama_set_nextn_layer_offset(ctx_dft, i); + } + int ret = llama_decode(ctx_dft, batch); + if (ret != 0) { + SPC_ERR("llama_decode[%d] returned %d\n", i, ret); + break; + } + + // rebuild the batch for the next step: the growing-KV paths re-add only the + // new token (the KV already holds the prefix), while chained heads re-add the + // whole prefix at the next head. dropped sequences are simply not re-added. common_batch_clear(batch); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { @@ -1094,14 +1538,13 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { auto * smpl = smpls[seq_id].get(); - common_sampler_sample(smpl, ctx_dft, i_batch, true); - h_row = llama_get_embeddings_nextn_ith(ctx_dft, i_batch); - ++i_batch; + common_sampler_sample(smpl, ctx_dft, i_last[seq_id], true); + const float * h_row = llama_get_embeddings_nextn_ith(ctx_dft, i_last[seq_id]); const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -1130,30 +1573,41 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { continue; } - if (is_mem_shared) { + if (chain_heads) { + // ref: https://github.com/ggml-org/llama.cpp/pull/24340#discussion_r3448031546 + chain_h[seq_id].insert(chain_h[seq_id].end(), h_row, h_row + n_embd); + + const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far + for (int t = 0; t < n_rows; ++t) { + const llama_token tok = (t == 0) ? dp.id_last : result[t - 1]; + common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1); + std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, + chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes); + } + } else if (is_mem_shared) { // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37 common_batch_add(batch, id, dp.n_past, { seq_id }, true); + std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } else { common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); + std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } - std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); + + i_last[seq_id] = batch.n_tokens - 1; } if (batch.n_tokens == 0) { break; } - // evaluate the drafted tokens on the draft model - ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); - break; - } - ++i; } + if (chain_heads) { + llama_set_nextn_layer_offset(ctx_dft, 0); // restore default for non-draft decodes + } + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { auto & dp = dparams[seq_id]; if (!dp.drafting) { @@ -1163,8 +1617,6 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { if (dp.result->size() < (size_t) params.n_min) { dp.result->clear(); } - - last_n_drafted[seq_id] = (uint16_t) dp.result->size(); } } @@ -1206,8 +1658,8 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl { , params(params.ngram_simple) , config(config) { - LOG_INF("%s: adding speculative implementation 'ngram-simple'\n", __func__); - LOG_INF("%s: - size_n=%d, size_m=%d, min_hits=%d\n", __func__, + SPC_TRC("%s", "adding speculative implementation 'ngram-simple'\n"); + SPC_TRC("- size_n=%d, size_m=%d, min_hits=%d\n", this->params.size_n, this->params.size_m, this->params.min_hits); } @@ -1256,8 +1708,8 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl { this->config.push_back(config); } - LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(this->type).c_str()); - LOG_INF("%s: - size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", __func__, + SPC_TRC("adding speculative implementation '%s'\n", common_speculative_type_to_str(this->type).c_str()); + SPC_TRC("- size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", config.size_key, config.size_value, config.key_only, config.min_hits); } @@ -1331,15 +1783,15 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { , verbose(std::getenv("LLAMA_TRACE") != nullptr) { static_assert(sizeof(llama_token) == sizeof(common_ngram_mod::entry_t)); - LOG_INF("%s: adding speculative implementation 'ngram-mod'\n", __func__); - LOG_INF("%s: - n_match=%d, n_max=%d, n_min=%d\n", __func__, + SPC_TRC("%s", "adding speculative implementation 'ngram-mod'\n"); + SPC_TRC("- n_match=%d, n_max=%d, n_min=%d\n", this->params.n_match, this->params.n_max, this->params.n_min); - LOG_INF("%s: - mod size=%zu (%.3f MB)\n", __func__, + SPC_TRC("- mod size=%zu (%.3f MB)\n", mod.size(), (float)(mod.size_bytes())/1024/1024); if (this->params.n_match < 16) { - LOG_WRN("%s: ngram_mod n_match=%d is too small - poor quality is possible, " - "see: https://github.com/ggml-org/llama.cpp/pull/19164\n", __func__, this->params.n_match); + SPC_WRN("ngram_mod n_match=%d is too small - poor quality is possible, " + "see: https://github.com/ggml-org/llama.cpp/pull/19164\n", this->params.n_match); } sinfos.resize(n_seq); @@ -1363,11 +1815,11 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.i_last = prompt.size() - n; const double f = (double)mod.get_used() / (double)mod.size(); - LOG_INF("%s: ngram_mod occupancy = %zu/%zu (%.2f)\n", __func__, mod.get_used(), mod.size(), f); + SPC_TRC("ngram_mod occupancy = %zu/%zu (%.2f)\n", mod.get_used(), mod.size(), f); constexpr double f_thold = 0.25; if (f > f_thold) { - LOG_WRN("%s: ngram_mod occupancy %.2f exceeds threshold (%.2f) - resetting\n", __func__, f, f_thold); + SPC_WRN("ngram_mod occupancy %.2f exceeds threshold (%.2f) - resetting\n", f, f_thold); mod.reset(); } @@ -1461,7 +1913,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.n_low++; if (sinfo.n_low >= 5) { if (verbose) { - LOG_WRN("%s: low acceptance streak (%d) - resetting ngram_mod\n", __func__, sinfo.n_low); + SPC_TRC("low acceptance streak (%d) - resetting ngram_mod\n", sinfo.n_low); } mod.reset(); @@ -1511,8 +1963,8 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { , save_dynamic(save_dynamic) , save_static(save_static) { - LOG_INF("%s: adding speculative implementation 'ngram-cache'\n", __func__); - LOG_INF("%s: - n_draft=%d, cache_static=%s, cache_dynamic=%s\n", __func__, + SPC_TRC("%s", "adding speculative implementation 'ngram-cache'\n"); + SPC_TRC("- n_draft=%d, cache_static=%s, cache_dynamic=%s\n", n_draft, path_static.empty() ? "none" : path_static.c_str(), path_dynamic.empty() ? "none" : path_dynamic.c_str()); @@ -1527,7 +1979,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { sinfo.ngram_cache_static = ngram_cache_static; } } catch (...) { - LOG_ERR("failed to open static lookup cache: %s", path_static.c_str()); + SPC_ERR("failed to open static lookup cache: %s", path_static.c_str()); GGML_ABORT("Couldn't read static lookup cache"); } } @@ -1540,7 +1992,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { sinfo.ngram_cache_dynamic = ngram_cache_dynamic; } } catch (...) { - LOG_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str()); + SPC_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str()); GGML_ABORT("Couldn't read dynamic lookup cache"); } } @@ -1689,6 +2141,7 @@ std::string common_speculative_type_to_str(common_speculative_type type) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: return "draft-simple"; case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3"; case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp"; + case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash"; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v"; @@ -1741,6 +2194,7 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: + case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: n_max = std::max(n_max, std::max(0, spec->draft.n_max)); break; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: @@ -1777,7 +2231,8 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr; - bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; + bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; + bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr; @@ -1788,7 +2243,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); // when adding a new type - update here the logic above - static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 9); + static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10); // this list here defines the priority of the speculators // the one with highest priority are listed first @@ -1815,9 +2270,12 @@ common_speculative * common_speculative_init(common_params_speculative & params, if (has_draft_eagle3) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params)); } - if (has_mtp) { + if (has_draft_mtp) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); } + if (has_draft_dflash) { + configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params)); + } } std::vector> impls = {}; @@ -1838,6 +2296,10 @@ common_speculative * common_speculative_init(common_params_speculative & params, impls.push_back(std::make_unique(config.params, n_seq)); break; } + case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: { + impls.push_back(std::make_unique(config.params, n_seq)); + break; + } case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple); @@ -1887,7 +2349,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, } if (impls.empty()) { - LOG_WRN("%s: no implementations specified for speculative decoding\n", __func__); + SPC_TRC("%s", "no implementations specified for speculative decoding\n"); return nullptr; } @@ -2014,13 +2476,13 @@ void common_speculative_draft(common_speculative * spec) { if (dp.n_max > 0) { if (!result.empty() && (int) result.size() > dp.n_max) { - LOG_DBG("%s: truncating draft to %d tokens\n", __func__, dp.n_max); + SPC_DBG("truncating draft to %d tokens\n", dp.n_max); result.resize(dp.n_max); } } if (!result.empty()) { - LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__, + SPC_DBG("called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", common_speculative_type_to_str(impl.get()->type).c_str(), dp.prompt->size(), impl.get()->n_call_draft, result.size()); @@ -2059,6 +2521,15 @@ void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, u { common_time_meas tm(impl->t_accept_us, !impl->gen_perf); + + if (impl->n_acc_tokens_per_pos.size() < n_accepted) { + impl->n_acc_tokens_per_pos.resize(n_accepted, 0); + } + + for (size_t i = 0; i < n_accepted; ++i) { + impl->n_acc_tokens_per_pos[i]++; + } + if (n_accepted > 0) { impl->n_acc_drafts++; impl->n_acc_tokens += n_accepted; @@ -2076,6 +2547,31 @@ void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, u } } +// TODO: support the case of more than one speculative implementations having a state +bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector & data) { + if (spec == nullptr) { + return false; + } + + for (auto & impl : spec->impls) { + if (impl->get_state(seq_id, data)) { + return true; + } + } + + return false; +} + +void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector & data) { + if (spec == nullptr) { + return; + } + + for (auto & impl : spec->impls) { + impl->set_state(seq_id, data); + } +} + void common_speculative_print_stats(const common_speculative * spec) { if (spec == nullptr) { return; @@ -2093,13 +2589,31 @@ void common_speculative_print_stats(const common_speculative * spec) { str_perf = ""; } - LOG_INF("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s\n", + std::string str_stats; + if (impl->n_call_accept > 0) { + const double mean = + 1.0 + (double) impl->n_acc_tokens / (double) impl->n_call_accept; + std::ostringstream tmp; + tmp << std::fixed << std::setprecision(3); + for (size_t i = 0; i < impl->n_acc_tokens_per_pos.size(); ++i) { + if (i > 0) { + tmp << ", "; + } + tmp << (double) impl->n_acc_tokens_per_pos[i] / (double) impl->n_call_accept; + } + std::ostringstream oss; + oss << std::fixed << std::setprecision(2) << mean; + str_stats = ", #mean acc len = " + oss.str() + ", #acc rate/pos = (" + tmp.str() + ")"; + } + + SPC_TRC("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s%s\n", common_speculative_type_to_str(impl->type).c_str(), impl->n_call_begin, impl->n_call_draft, impl->n_call_accept, impl->n_gen_drafts, impl->n_acc_drafts, impl->n_gen_tokens, impl->n_acc_tokens, + str_stats.c_str(), str_perf.c_str()); } } diff --git a/common/speculative.h b/common/speculative.h index bf76ad709..c58fac3cc 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -68,6 +68,10 @@ void common_speculative_draft(common_speculative * spec); // informs the speculative context that n_accepted tokens were accepted by the target model void common_speculative_accept(common_speculative * spec, llama_seq_id, uint16_t n_accepted); +// (optional) get/set internal state +bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector & data); +void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector & data); + // print statistics about the speculative decoding void common_speculative_print_stats(const common_speculative * spec); diff --git a/conversion/__init__.py b/conversion/__init__.py index cd6f8e6b9..02ea63852 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -40,14 +40,18 @@ TEXT_MODEL_MAP: dict[str, str] = { "ChatGLMModel": "chatglm", "CodeShellForCausalLM": "codeshell", "CogVLMForCausalLM": "cogvlm", + "Cohere2MoeForCausalLM": "command_r", "Cohere2ForCausalLM": "command_r", "CohereForCausalLM": "command_r", "DbrxForCausalLM": "dbrx", "DeciLMForCausalLM": "deci", "DeepseekForCausalLM": "deepseek", + "DeepseekOCRForCausalLM": "deepseek", "DeepseekV2ForCausalLM": "deepseek", "DeepseekV3ForCausalLM": "deepseek", "DeepseekV32ForCausalLM": "deepseek", + "DFlashDraftModel": "qwen", + "DeepseekV4ForCausalLM": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", @@ -95,6 +99,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "GraniteMoeHybridForCausalLM": "granite", "GraniteMoeSharedForCausalLM": "granite", "GraniteSpeechForConditionalGeneration": "granite", + "GraniteSpeechPlusForConditionalGeneration": "granite", "Grok1ForCausalLM": "grok", "GrokForCausalLM": "grok", "GroveMoeForCausalLM": "grovemoe", @@ -122,6 +127,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "LLaDAModelLM": "llada", "LLaMAForCausalLM": "llama", "Lfm25AudioTokenizer": "lfm2", + "Lfm2BidirectionalModel": "lfm2", "Lfm2ForCausalLM": "lfm2", "Lfm2Model": "lfm2", "Lfm2MoeForCausalLM": "lfm2", @@ -132,6 +138,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "LlamaModel": "llama", "Eagle3DraftModel": "llama", "Eagle3Speculator": "llama", + "Eagle3LlamaForCausalLM": "llama", "LlamaForCausalLMEagle3": "llama", "LlavaForConditionalGeneration": "llama", "LlavaStableLMEpochForCausalLM": "stablelm", @@ -230,6 +237,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "UMT5ForConditionalGeneration": "t5", "UMT5Model": "t5", "UltravoxModel": "ultravox", + "UnlimitedOCRForCausalLM": "deepseek", "VLlama3ForCausalLM": "llama", "VoxtralForConditionalGeneration": "llama", "WavTokenizerDec": "wavtokenizer", @@ -260,6 +268,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "GlmasrModel": "ultravox", "Granite4VisionForConditionalGeneration": "granite", "GraniteSpeechForConditionalGeneration": "granite", + "GraniteSpeechPlusForConditionalGeneration": "granite", "HunYuanVLForConditionalGeneration": "hunyuan", "Idefics3ForConditionalGeneration": "smolvlm", "InternVisionModel": "internvl", @@ -295,6 +304,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "StepVLForConditionalGeneration": "step3", "Step3p7ForConditionalGeneration": "step3", "UltravoxModel": "ultravox", + "UnlimitedOCRForCausalLM": "deepseek", "VoxtralForConditionalGeneration": "ultravox", "YoutuVLForConditionalGeneration": "youtuvl", } diff --git a/conversion/bailingmoe.py b/conversion/bailingmoe.py index 319ff6dab..2c6425cb6 100644 --- a/conversion/bailingmoe.py +++ b/conversion/bailingmoe.py @@ -126,7 +126,7 @@ class BailingMoeV2Model(TextModel): if (rope_dim := hparams.get("head_dim")) is None: rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"] - self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5))) + self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.5))) self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"]) self.gguf_writer.add_vocab_size(hparams["vocab_size"]) self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) diff --git a/conversion/base.py b/conversion/base.py index 9d81c19b4..0421aa4bc 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1119,8 +1119,10 @@ class TextModel(ModelBase): rope_theta = self.find_hparam(["global_rope_theta", "rope_global_theta", "rope_theta_global", "rope_theta", "rotary_emb_base"], optional=True) local_rope_theta = self.find_hparam(["local_rope_theta", "rope_local_theta", "rope_theta_local", "swa_rope_theta", "rope_local_base_freq"], optional=True) + partial_rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct", "rope_percent"], optional=True) + original_max_position_embeddings = self.find_hparam(["original_max_position_embeddings"], optional=True) - # Ensure "rope_theta" and "rope_type" is mirrored in rope_parameters + # Ensure global params are mirrored in rope_parameters if "full_attention" not in self.rope_parameters and "sliding_attention" not in self.rope_parameters: if local_rope_theta is not None: self.rope_parameters["sliding_attention"] = {"rope_theta": local_rope_theta} @@ -1128,6 +1130,10 @@ class TextModel(ModelBase): self.rope_parameters["rope_theta"] = rope_theta if "rope_type" not in self.rope_parameters and (rope_type := self.rope_parameters.get("type")) is not None: self.rope_parameters["rope_type"] = rope_type + if "partial_rotary_factor" not in self.rope_parameters and partial_rotary_factor is not None: + self.rope_parameters["partial_rotary_factor"] = partial_rotary_factor + if "original_max_position_embeddings" not in self.rope_parameters and original_max_position_embeddings is not None: + self.rope_parameters["original_max_position_embeddings"] = original_max_position_embeddings @classmethod def __init_subclass__(cls): @@ -1195,7 +1201,7 @@ class TextModel(ModelBase): self.gguf_writer.add_embedding_length(n_embd) logger.info(f"gguf: embedding length = {n_embd}") - if (n_ff := self.find_hparam(["intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None: + if (n_ff := self.find_hparam(["prefix_dense_intermediate_size", "intermediate_size", "n_inner", "hidden_dim"], optional=True)) is not None: self.gguf_writer.add_feed_forward_length(n_ff) logger.info(f"gguf: feed forward length = {n_ff}") @@ -1267,7 +1273,7 @@ class TextModel(ModelBase): if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None: self.gguf_writer.add_layer_norm_eps(f_norm_eps) logger.info(f"gguf: layer norm epsilon = {f_norm_eps}") - if (n_experts := self.find_hparam(["num_local_experts", "num_experts"], optional=True)) is not None: + if (n_experts := self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True)) is not None: self.gguf_writer.add_expert_count(n_experts) logger.info(f"gguf: expert count = {n_experts}") if (n_experts_used := self.find_hparam(["num_experts_per_tok", "num_experts_per_token", "top_k_experts"], optional=True)) is not None: @@ -1280,11 +1286,13 @@ class TextModel(ModelBase): self.gguf_writer.add_expert_group_used_count(n_group_used) logger.info(f"gguf: expert groups used count = {n_group_used}") - if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func"], optional=True)) is not None: + if (score_func := self.find_hparam(["score_function", "scoring_func", "score_func", "moe_router_activation", "moe_router_activation_func", "expert_selection_fn"], optional=True)) is not None: if score_func == "sigmoid": self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) elif score_func == "softmax": self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX) + elif score_func == "sqrtsoftplus": + self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS) else: raise ValueError(f"Unsupported expert score gating function value: {score_func}") logger.info(f"gguf: expert score gating function = {score_func}") @@ -1495,6 +1503,9 @@ class TextModel(ModelBase): if chkhsh == "d772b220ace2baec124bed8cfafce0ead7d6c38a4b65ef11261cf9d5d62246d1": # ref: https://huggingface.co/CohereLabs/tiny-aya-base res = "tiny_aya" + if chkhsh == "52df12b4c8d4176e7481aab4b6e8454d1fd0a210a04a574f6d4e067d10e23c3e": + # ref: https://huggingface.co/CohereLabs/North-Mini-Code-1.0 + res = "cohere2moe" if chkhsh == "e636dc30a262dcc0d8c323492e32ae2b70728f4df7dfe9737d9f920a282b8aea": # ref: https://huggingface.co/Qwen/Qwen1.5-7B res = "qwen2" @@ -2591,6 +2602,17 @@ class LazyTorchTensor(gguf.LazyBase): return cls._wrap_fn(func)(*args, **kwargs) +if hasattr(torch, "float8_e8m0fnu"): + _torch_float8_e8m0 = torch.float8_e8m0fnu + LazyTorchTensor._dtype_map[_torch_float8_e8m0] = np.uint8 + LazyTorchTensor._dtype_byteswap_map[_torch_float8_e8m0] = np.uint8 + LazyTorchTensor._dtype_str_map["F8_E8M0"] = _torch_float8_e8m0 +else: + # Older torch builds do not expose F8_E8M0. Keep the raw bytes so callers + # that know the format can decode them explicitly. + LazyTorchTensor._dtype_str_map["F8_E8M0"] = torch.uint8 + + def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str: # TODO @ngxson : this won't work correctly if the model has both audio & vision encoders # maybe we should fallback to text model's arch in that case, since not many models have both diff --git a/conversion/chatglm.py b/conversion/chatglm.py index 7e323b890..801913075 100644 --- a/conversion/chatglm.py +++ b/conversion/chatglm.py @@ -148,7 +148,7 @@ class ChatGLMModel(TextModel): rope_dim = self.hparams["attention_dim"] else: rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] - self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5))) + self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.5))) self.gguf_writer.add_add_bos_token(False) rope_freq = 10000 if "rope_ratio" in self.hparams: diff --git a/conversion/command_r.py b/conversion/command_r.py index 603288d16..118565c66 100644 --- a/conversion/command_r.py +++ b/conversion/command_r.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re from typing import Iterable, TYPE_CHECKING import torch @@ -55,3 +56,122 @@ class Cohere2Model(TextModel): return yield from super().modify_tensors(data_torch, name, bid) + + +@ModelBase.register("Cohere2MoeForCausalLM") +class Cohere2MoeModel(TextModel): + model_arch = gguf.MODEL_ARCH.COHERE2MOE + _n_main_layers: int | None = None + _expert_tensor_re = re.compile( + r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.(down_proj|gate_proj|up_proj)\.weight" + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if (n_nextn := int(self.hparams.get("num_nextn_predict_layers", 0) or 0)) > 0 and not self.no_mtp: + self.block_count += n_nextn + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + self._experts: list[dict[str, Tensor]] = [{} for _ in range(self.block_count)] + + def _set_vocab_gpt2(self) -> None: + tokens, toktypes, tokpre = self.get_vocab_base() + self.gguf_writer.add_tokenizer_model("gpt2") + self.gguf_writer.add_tokenizer_pre(tokpre) + self.gguf_writer.add_token_list(tokens) + self.gguf_writer.add_token_types(toktypes) + + special_vocab = gguf.SpecialVocab(self.dir_model, load_merges=True) + special_vocab.add_to_gguf(self.gguf_writer) + + def set_gguf_parameters(self): + hparams = self.hparams + expert_intermediate_size = hparams["intermediate_size"] + mlp_layer_types = hparams.get("mlp_layer_types") + n_dense_lead = hparams.get("first_k_dense_replace", 0) + if mlp_layer_types is not None: + n_dense_lead = next((i for i, t in enumerate(mlp_layer_types) if t != "dense"), len(mlp_layer_types)) + + super().set_gguf_parameters() + + self.gguf_writer.add_logit_scale(hparams["logit_scale"]) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + self.gguf_writer.add_sliding_window_pattern([t == "sliding_attention" for t in hparams["layer_types"]]) + self.gguf_writer.add_vocab_size(hparams["vocab_size"]) + self.gguf_writer.add_expert_feed_forward_length(expert_intermediate_size) + self.gguf_writer.add_leading_dense_block_count(n_dense_lead) + self.gguf_writer.add_expert_weights_norm(hparams.get("norm_topk_prob", False)) + if (num_shared_experts := hparams.get("num_shared_experts", 0)) > 0: + if hparams.get("shared_expert_combination_strategy", "average") != "average": + raise ValueError("Cohere2 MoE only supports average shared expert combination") + self.gguf_writer.add_expert_shared_count(num_shared_experts) + self.gguf_writer.add_expert_shared_feed_forward_length(expert_intermediate_size * num_shared_experts) + if (n_nextn := hparams.get("num_nextn_predict_layers", 0)) > 0 and not self.no_mtp: + self.gguf_writer.add_nextn_predict_layers(n_nextn) + self.gguf_writer.add_rope_dimension_count(hparams["head_dim"]) + self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) + + def index_tensors(self, remote_hf_model_id: str | None = None): + hparams = {**self.hparams, **self.hparams.get("text_config", {})} + self._n_main_layers = hparams.get("num_hidden_layers") + type(self)._n_main_layers = self._n_main_layers + return super().index_tensors(remote_hf_model_id=remote_hf_model_id) + + @classmethod + def filter_tensors(cls, item): + if (titem := super().filter_tensors(item)) is None: + return None + name, gen = titem + + if cls._n_main_layers is not None: + is_mtp = (m := re.match(r"model\.layers\.(\d+)\.", name)) is not None and int(m.group(1)) >= cls._n_main_layers + if is_mtp and cls.no_mtp: + return None + if cls.mtp_only and not is_mtp and name not in ( + "model.embed_tokens.weight", "model.norm.weight", "lm_head.weight", + ): + return None + + return name, gen + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if name.endswith(".bias"): + if torch.any(data_torch != 0): + raise ValueError(f"Bias tensor {name!r} is not zero.") + logger.debug(f"Skipping bias tensor {name!r}.") + return + + if (m := self._expert_tensor_re.fullmatch(name)) is not None: + n_experts = self.hparams["num_experts"] + layer_idx = int(m.group(1)) + assert bid is None or bid == layer_idx + + self._experts[layer_idx][name] = data_torch + + expected = { + f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight" + for xid in range(n_experts) + for w_name in ("down_proj", "gate_proj", "up_proj") + } + if expected.issubset(self._experts[layer_idx]): + for w_name in ["down_proj", "gate_proj", "up_proj"]: + datas: list[Tensor] = [] + + for xid in range(n_experts): + ename = f"model.layers.{layer_idx}.mlp.experts.{xid}.{w_name}.weight" + datas.append(self._experts[layer_idx][ename]) + del self._experts[layer_idx][ename] + + data_torch = torch.stack(datas, dim=0) + merged_name = f"model.layers.{layer_idx}.mlp.experts.{w_name}.weight" + + yield from super().modify_tensors(data_torch, merged_name, layer_idx) + return + + yield from super().modify_tensors(data_torch, name, bid) + + def prepare_tensors(self): + super().prepare_tensors() + + experts = [k for d in self._experts for k in d.keys()] + if len(experts) > 0: + raise ValueError(f"Unprocessed experts: {experts}") diff --git a/conversion/deci.py b/conversion/deci.py index 46d8568c5..be446eefa 100644 --- a/conversion/deci.py +++ b/conversion/deci.py @@ -161,7 +161,7 @@ class DeciModel(TextModel): factor = rope_params.get("factor", 8.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = self.hparams.get("original_max_position_embeddings", 8192) + old_context_len = rope_params.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/conversion/deepseek.py b/conversion/deepseek.py index 72520cc9f..ea6ae23d5 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -1,20 +1,23 @@ from __future__ import annotations +import json import re +from pathlib import Path from typing import Any, Callable, Iterable, TYPE_CHECKING +import numpy as np import torch if TYPE_CHECKING: from torch import Tensor -from .base import MmprojModel, ModelBase, TextModel, gguf, logger +from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logger from .qwen import QwenModel -@ModelBase.register("DeepseekOCRForCausalLM") +@ModelBase.register("DeepseekOCRForCausalLM", "UnlimitedOCRForCausalLM") class DeepseekOCRVisionModel(MmprojModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -205,6 +208,8 @@ class DeepseekModel(TextModel): @ModelBase.register( "DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM", + "DeepseekOCRForCausalLM", + "UnlimitedOCRForCausalLM", "KimiVLForConditionalGeneration", "KimiK25ForConditionalGeneration", "YoutuForCausalLM", @@ -224,7 +229,7 @@ class DeepseekV2Model(TextModel): self.origin_hf_arch = hparams.get('architectures', [None])[0] # special handling for Deepseek OCR - if self.origin_hf_arch in ("DeepseekOCRForCausalLM", "DeepseekOCR2ForCausalLM"): + if self.origin_hf_arch in ("DeepseekOCRForCausalLM", "DeepseekOCR2ForCausalLM", "UnlimitedOCRForCausalLM"): self.model_arch = gguf.MODEL_ARCH.DEEPSEEK2OCR self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[self.model_arch] self.gguf_writer.add_architecture() @@ -350,6 +355,12 @@ class DeepseekV2Model(TextModel): self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"]) + # Unlimited-OCR sliding window; written for metadata, the decoder ignores it (full MHA) + if is_ocr: + sliding_window = hparams.get("sliding_window_size") or hparams.get("sliding_window") + if sliding_window: + self.gguf_writer.add_sliding_window(sliding_window) + if (rope_mscale_all := self.rope_parameters.get("mscale_all_dim")) is not None: # [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX] # note: for legacy reasons, this is not consistent with the other usages of self.gguf_writer.add_rope_scaling_yarn_log_mul @@ -459,3 +470,307 @@ class DeepseekV32Model(DeepseekV2Model): self.gguf_writer.add_indexer_head_count(self.hparams["index_n_heads"]) self.gguf_writer.add_indexer_key_length(self.hparams["index_head_dim"]) self.gguf_writer.add_indexer_top_k(self.hparams["index_topk"]) + + +@ModelBase.register("DeepseekV4ForCausalLM") +class DeepseekV4Model(TextModel): + model_arch = gguf.MODEL_ARCH.DEEPSEEK4 + _skipped_mtp_tensors = 0 + + def __init__(self, *args, **kwargs): + type(self)._skipped_mtp_tensors = 0 + super().__init__(*args, **kwargs) + + with open(self.dir_model / "config.json", "r", encoding="utf-8") as f: + raw_hparams = json.load(f) + for key, value in raw_hparams.items(): + self.hparams.setdefault(key, value) + + self.block_count = self.hparams["num_hidden_layers"] + self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) + + self._dsv4_fp8_dequantized: set[str] = set() + self._dsv4_bf16_tensors: set[str] = set() + self._dsv4_f32_tensors: set[str] = set() + self._dsv4_mxfp4_generated = False + self._collect_source_dtypes() + + if type(self)._skipped_mtp_tensors: + logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors) + + # add a default chat template; if the model has a built-in template, it will be overridden later + template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja" + if template_path.is_file(): + with open(template_path, "r", encoding="utf-8") as f: + self.gguf_writer.add_chat_template(f.read()) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, _ = item + if name.startswith("mtp."): + cls._skipped_mtp_tensors += 1 + return None + return super().filter_tensors(item) + + @staticmethod + def _float8_dtypes() -> tuple[torch.dtype, ...]: + return tuple( + dtype for dtype in ( + getattr(torch, "float8_e4m3fn", None), + getattr(torch, "float8_e5m2", None), + ) if dtype is not None + ) + + @staticmethod + def _e8m0_to_float(scale: Tensor) -> Tensor: + torch_float8_e8m0 = getattr(torch, "float8_e8m0fnu", None) + if torch_float8_e8m0 is not None and scale.dtype == torch_float8_e8m0: + return scale.float() + + bits = scale.view(torch.uint8).float() + return torch.exp2(bits - 127.0) + + def _collect_source_dtypes(self) -> None: + for name, gen in self.model_tensors.items(): + dtype = gen().dtype + if dtype == torch.bfloat16: + self._dsv4_bf16_tensors.add(name) + elif dtype == torch.float32: + self._dsv4_f32_tensors.add(name) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + hparams = self.hparams + + self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"]) + self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"]) + self.gguf_writer.add_sliding_window(hparams["sliding_window"]) + + self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) + self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"]) + self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"]) + self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"]) + self.gguf_writer.add_swiglu_clamp_exp([hparams["swiglu_limit"]] * self.block_count) + self.gguf_writer.add_swiglu_clamp_shexp([hparams["swiglu_limit"]] * self.block_count) + + self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"]) + self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"]) + self.gguf_writer.add_indexer_top_k(hparams["index_topk"]) + + self.gguf_writer.add_attention_output_group_count(hparams["o_groups"]) + self.gguf_writer.add_attention_output_lora_rank(hparams["o_lora_rank"]) + self.gguf_writer.add_attention_compress_ratios(hparams["compress_ratios"]) + self.gguf_writer.add_attention_compress_rope_freq_base(hparams["compress_rope_theta"]) + self.gguf_writer.add_hyper_connection_count(hparams["hc_mult"]) + self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) + self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) + self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) + + def dequant_model(self): + fp8_dtypes = self._float8_dtypes() + tensors_to_remove: list[str] = [] + + def dequant_fp8_weight(weight: Tensor, scale: Tensor) -> Tensor: + out_features, in_features = weight.shape + scale_f = self._e8m0_to_float(scale) + scale_f = scale_f.repeat_interleave(128, 0)[:out_features] + scale_f = scale_f.repeat_interleave(128, 1)[:, :in_features] + return weight.float() * scale_f + + for name in list(self.model_tensors.keys()): + if not name.endswith(".scale"): + continue + weight_name = name.removesuffix(".scale") + ".weight" + if weight_name not in self.model_tensors: + continue + + weight = self.model_tensors[weight_name] + scale = self.model_tensors[name] + if weight().dtype not in fp8_dtypes: + continue + + self.model_tensors[weight_name] = lambda w=weight, s=scale: dequant_fp8_weight(w(), s()) + self._dsv4_fp8_dequantized.add(weight_name) + tensors_to_remove.append(name) + + for name in tensors_to_remove: + del self.model_tensors[name] + + @staticmethod + def _pack_mxfp4_blocks(weight: Tensor, scale: Tensor) -> np.ndarray: + packed = weight.contiguous().view(torch.uint8) + scale_u8 = scale.contiguous().view(torch.uint8) + + out_features, packed_cols = packed.shape + logical_cols = packed_cols * 2 + if logical_cols % 32 != 0: + raise ValueError(f"MXFP4 source row has {logical_cols} values, expected a multiple of 32") + + n_blocks = logical_cols // 32 + if tuple(scale_u8.shape) != (out_features, n_blocks): + raise ValueError(f"MXFP4 scale shape {tuple(scale_u8.shape)} does not match {(out_features, n_blocks)}") + + src = packed.reshape(out_features, n_blocks, 16) + low = src & 0x0F + high = (src >> 4) & 0x0F + + # The safetensors bytes store adjacent values as low/high nibbles. + # ggml MXFP4 blocks store values 0..15 in low nibbles and 16..31 in high nibbles. + vals = torch.stack((low, high), dim=-1).reshape(out_features, n_blocks, 32) + qs = vals[:, :, :16] | (vals[:, :, 16:] << 4) + raw = torch.cat((scale_u8.unsqueeze(-1), qs.to(torch.uint8)), dim=-1) + return raw.reshape(out_features, n_blocks * 17).cpu().numpy() + + def _write_mxfp4_expert_tensor(self, bid: int, proj: str, tensor_key: gguf.MODEL_TENSOR) -> list[str]: + n_experts = self.hparams["n_routed_experts"] + data: np.ndarray | None = None + consumed: list[str] = [] + + for eid in range(n_experts): + weight_name = f"layers.{bid}.ffn.experts.{eid}.{proj}.weight" + scale_name = f"layers.{bid}.ffn.experts.{eid}.{proj}.scale" + if weight_name not in self.model_tensors or scale_name not in self.model_tensors: + raise KeyError(f"Missing routed expert tensors for {weight_name}") + + weight = LazyTorchTensor.to_eager(self.model_tensors[weight_name]()) + scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]()) + packed = self._pack_mxfp4_blocks(weight, scale) + if data is None: + data = np.empty((n_experts, *packed.shape), dtype=packed.dtype) + data[eid] = packed + consumed.extend((weight_name, scale_name)) + + assert data is not None + new_name = self.format_tensor_name(tensor_key, bid) + shape = gguf.quant_shape_from_byte_shape(data.shape, gguf.GGMLQuantizationType.MXFP4) + logger.info(f"{new_name}: repacked routed experts to MXFP4, shape = {{{', '.join(str(n) for n in reversed(shape))}}}") + self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP4) + + return consumed + + def _write_hash_routing_tensors(self) -> list[str]: + consumed: list[str] = [] + + for bid in range(self.hparams["num_hash_layers"]): + name = f"layers.{bid}.ffn.gate.tid2eid" + if name not in self.model_tensors: + raise KeyError(f"Missing hash routing tensor {name}") + + data_torch = LazyTorchTensor.to_eager(self.model_tensors[name]()) + data = data_torch.to(torch.int32).cpu().numpy() + new_name = self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_TID2EID, bid, ".weight") + logger.info(f"{new_name}: converted hash routing table to I32, shape = {{{', '.join(str(n) for n in reversed(data.shape))}}}") + self.gguf_writer.add_tensor(new_name, data) + consumed.append(name) + + return consumed + + def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: + if self._dsv4_mxfp4_generated: + return () + + consumed: list[str] = self._write_hash_routing_tensors() + for bid in range(self.block_count): + consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)) + consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)) + consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)) + + for name in consumed: + del self.model_tensors[name] + + self._dsv4_mxfp4_generated = True + return () + + def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str: + return self.format_tensor_name(key, bid, suffix) + + def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: + root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { + "embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"), + "norm.weight": (gguf.MODEL_TENSOR.OUTPUT_NORM, ".weight"), + "head.weight": (gguf.MODEL_TENSOR.OUTPUT, ".weight"), + "hc_head_fn": (gguf.MODEL_TENSOR.HC_HEAD_FN, ".weight"), + "hc_head_base": (gguf.MODEL_TENSOR.HC_HEAD_BASE, ".weight"), + "hc_head_scale": (gguf.MODEL_TENSOR.HC_HEAD_SCALE, ".weight"), + } + if name in root_map: + return root_map[name] + + match = re.match(r"layers\.(\d+)\.(.+)$", name) + if match is None: + raise ValueError(f"Unsupported DeepSeek-V4 tensor {name!r}") + + layer = int(match.group(1)) + if bid != layer: + raise ValueError(f"Tensor {name!r} parsed bid {bid} but layer name has {layer}") + + layer_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { + "hc_attn_fn": (gguf.MODEL_TENSOR.HC_ATTN_FN, ".weight"), + "hc_attn_base": (gguf.MODEL_TENSOR.HC_ATTN_BASE, ".weight"), + "hc_attn_scale": (gguf.MODEL_TENSOR.HC_ATTN_SCALE, ".weight"), + "hc_ffn_fn": (gguf.MODEL_TENSOR.HC_FFN_FN, ".weight"), + "hc_ffn_base": (gguf.MODEL_TENSOR.HC_FFN_BASE, ".weight"), + "hc_ffn_scale": (gguf.MODEL_TENSOR.HC_FFN_SCALE, ".weight"), + "attn.attn_sink": (gguf.MODEL_TENSOR.ATTN_SINKS, ".weight"), + "attn.wq_a.weight": (gguf.MODEL_TENSOR.ATTN_Q_A, ".weight"), + "attn.wq_b.weight": (gguf.MODEL_TENSOR.ATTN_Q_B, ".weight"), + "attn.q_norm.weight": (gguf.MODEL_TENSOR.ATTN_Q_A_NORM, ".weight"), + "attn.wkv.weight": (gguf.MODEL_TENSOR.ATTN_KV, ".weight"), + "attn.kv_norm.weight": (gguf.MODEL_TENSOR.ATTN_KV_NORM, ".weight"), + "attn.wo_a.weight": (gguf.MODEL_TENSOR.ATTN_OUT_A, ".weight"), + "attn.wo_b.weight": (gguf.MODEL_TENSOR.ATTN_OUT_B, ".weight"), + "attn.compressor.ape": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_APE, ".weight"), + "attn.compressor.wkv.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_WKV, ".weight"), + "attn.compressor.wgate.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_WGATE, ".weight"), + "attn.compressor.norm.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_NORM, ".weight"), + "attn.indexer.wq_b.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_Q_B, ".weight"), + "attn.indexer.weights_proj.weight": (gguf.MODEL_TENSOR.INDEXER_PROJ, ".weight"), + "attn.indexer.compressor.ape": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_APE, ".weight"), + "attn.indexer.compressor.wkv.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_WKV, ".weight"), + "attn.indexer.compressor.wgate.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE, ".weight"), + "attn.indexer.compressor.norm.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_NORM, ".weight"), + "attn_norm.weight": (gguf.MODEL_TENSOR.ATTN_NORM, ".weight"), + "ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"), + "ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"), + "ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"), + "ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"), + "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), + "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), + "ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"), + } + + tensor_name = match.group(2) + if tensor_name in layer_map: + return layer_map[tensor_name] + + if re.match(r"ffn\.experts\.\d+\.w[123]\.(weight|scale)$", tensor_name): + return gguf.MODEL_TENSOR.FFN_GATE_EXP, ".weight" + + raise ValueError(f"Unsupported DeepSeek-V4 tensor {name!r}") + + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: + if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name): + return [] + + tensor_key, suffix = self._map_dsv4_tensor_name(name, bid) + if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID: + return [] + + return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)] + + def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: + del new_name, bid # unused + + if name in self._dsv4_fp8_dequantized and n_dims >= 2: + return gguf.GGMLQuantizationType.Q8_0 + if name in self._dsv4_f32_tensors: + return gguf.GGMLQuantizationType.F32 + if name in self._dsv4_bf16_tensors and n_dims >= 2: + return gguf.GGMLQuantizationType.BF16 + + return False + + def prepare_tensors(self): + super().prepare_tensors() + self._is_mxfp4 = True + self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE diff --git a/conversion/exaone.py b/conversion/exaone.py index b21f02784..bc4fb3f1b 100644 --- a/conversion/exaone.py +++ b/conversion/exaone.py @@ -24,7 +24,7 @@ class ExaoneModel(TextModel): assert (hparams["activation_function"] == "silu") - rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"], optional=True) + rotary_factor = self.rope_parameters.get("partial_rotary_factor") rotary_factor = rotary_factor if rotary_factor is not None else 1.0 self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) @@ -39,7 +39,7 @@ class ExaoneModel(TextModel): factor = rope_params.get("factor", 8.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = self.hparams.get("original_max_position_embeddings", 8192) + old_context_len = rope_params.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor @@ -104,7 +104,7 @@ class Exaone4Model(TextModel): factor = rope_params.get("factor", 16.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = self.hparams.get("original_max_position_embeddings", 8192) + old_context_len = rope_params.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/conversion/gemma.py b/conversion/gemma.py index 5b4ca5c58..c552df732 100644 --- a/conversion/gemma.py +++ b/conversion/gemma.py @@ -693,7 +693,7 @@ class Gemma4Model(Gemma3Model): self.gguf_writer.add_head_count_kv(value_arr) # handle n_rot differently for global vs swa layers - partial_rotary_factor_swa = self.hparams.get("partial_rotary_factor", 1.0) + partial_rotary_factor_swa = self.rope_parameters.get("partial_rotary_factor", 1.0) n_rot_full = int(head_dim_full) # "proportional" is used, see generate_extra_tensors n_rot_swa = int(head_dim_swa * partial_rotary_factor_swa) self.gguf_writer.add_rope_dimension_count(n_rot_full) diff --git a/conversion/glm.py b/conversion/glm.py index 641937720..895cefc22 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -124,7 +124,7 @@ class Glm4MoeModel(TextModel): self.hparams["hidden_size"] // self.hparams["num_attention_heads"] ) self.gguf_writer.add_rope_dimension_count( - int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)) + int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.5)) ) # MoE parameters - Use only routed expert count (shared experts handled separately) @@ -226,7 +226,7 @@ class GlmMoeDsaModel(DeepseekV2Model): super().set_gguf_parameters() rope_dim = self.hparams["qk_rope_head_dim"] - partial_rotary_factor = self.hparams.get("partial_rotary_factor", 1.0) + partial_rotary_factor = self.rope_parameters.get("partial_rotary_factor", 1.0) self.gguf_writer.add_rope_dimension_count(int(rope_dim * partial_rotary_factor)) # NextN/MTP prediction layers diff --git a/conversion/granite.py b/conversion/granite.py index 53441fe57..8367ed225 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -348,6 +348,34 @@ class GraniteSpeechMmprojModel(MmprojModel): yield from super().modify_tensors(data_torch, name, bid) +@ModelBase.register("GraniteSpeechPlusForConditionalGeneration") +class GraniteSpeechPlusMmprojModel(GraniteSpeechMmprojModel): + """Conversion for GraniteSpeechPlus - extends GraniteSpeech with feature layer concatenation""" + has_vision_encoder = False + has_audio_encoder = True + + def set_gguf_parameters(self): + assert self.hparams_audio is not None + super().set_gguf_parameters() + + # Add feature_layer if present in encoder config + if feature_layers := self.hparams_audio.get("cat_hidden_layers"): + self.gguf_writer.add_audio_feature_layers(feature_layers) + logger.info(f"gguf: audio feature_layers = {feature_layers}") + + # Validate projector dimension matches concatenated encoder output + hidden_dim = self.hparams_audio["hidden_dim"] + expected_dim = hidden_dim * (len(feature_layers) + 1) + projector_dim = self.global_config["projector_config"]["encoder_hidden_size"] + + if projector_dim != expected_dim: + raise ValueError( + f"Projector encoder_hidden_size ({projector_dim}) does not match " + f"expected concatenated dimension ({expected_dim}). " + f"Expected: hidden_dim ({hidden_dim}) * (len(feature_layers) + 1) = {expected_dim}" + ) + + @ModelBase.register("Granite4VisionForConditionalGeneration") class Granite4VisionMmprojModel(MmprojModel): has_vision_encoder = True diff --git a/conversion/lfm2.py b/conversion/lfm2.py index f28fccf10..70ce45658 100644 --- a/conversion/lfm2.py +++ b/conversion/lfm2.py @@ -64,11 +64,17 @@ class LFM2Model(TextModel): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("Lfm2Model") +@ModelBase.register("Lfm2Model", "Lfm2BidirectionalModel") class LFM2ColBertModel(LFM2Model): model_arch = gguf.MODEL_ARCH.LFM2 dense_tensor_name = "dense_2" + def set_gguf_parameters(self): + super().set_gguf_parameters() + if self.hf_arch == "Lfm2BidirectionalModel": + self.gguf_writer.add_causal_attention(False) + self._try_set_pooling_type() + def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if not name.startswith(self.dense_tensor_name): name = "model." + name @@ -76,10 +82,11 @@ class LFM2ColBertModel(LFM2Model): yield from super().modify_tensors(data_torch, name, bid) def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - # dense tensor is stored in a separate safetensors file + # optional dense tensor is stored in a separate safetensors file from safetensors.torch import load_file tensors_file = self.dir_model / "1_Dense" / "model.safetensors" - assert tensors_file.is_file() + if not tensors_file.is_file(): + return tensor = load_file(tensors_file)["linear.weight"] self.gguf_writer.add_embedding_length_out(tensor.shape[0]) yield f"{self.dense_tensor_name}.weight", tensor.clone() diff --git a/conversion/llama.py b/conversion/llama.py index b87bf92d4..315a619c9 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -23,6 +23,7 @@ from .base import ModelBase, TextModel, gguf, logger "LlavaForConditionalGeneration", "VoxtralForConditionalGeneration", "LlamaForCausalLMEagle3", + "Eagle3LlamaForCausalLM", "Eagle3Speculator", "Eagle3DraftModel", "IQuestCoderForCausalLM", @@ -72,7 +73,7 @@ class LlamaModel(TextModel): target_num_layers = target_config["num_hidden_layers"] target_layers = [2, target_num_layers // 2, target_num_layers - 3] logger.info(f"EAGLE-3: target_layers = {target_layers} (target model has {target_num_layers} layers)") - self.gguf_writer.add_array(f"{self.gguf_writer.arch}.target_layers", target_layers) + self.gguf_writer.add_target_layers(target_layers) # target_hidden_size: prefer eagle3 config, fallback to target config if eagle3_raw_config.get("target_hidden_size") is not None: @@ -82,12 +83,12 @@ class LlamaModel(TextModel): target_hidden_size = target_config["hidden_size"] src = "target model config" logger.info(f"EAGLE-3: target_hidden_size = {target_hidden_size} (from {src})") - self.gguf_writer.add_uint32(f"{self.gguf_writer.arch}.target_hidden_size", target_hidden_size) + self.gguf_writer.add_target_hidden_size(target_hidden_size) # norm_before_residual (RedHat-style eagle3 specific) norm_before_residual = eagle3_raw_config.get("norm_before_residual", False) logger.info(f"EAGLE-3: norm_before_residual = {norm_before_residual}") - self.gguf_writer.add_bool(f"{self.gguf_writer.arch}.norm_before_residual", norm_before_residual) + self.gguf_writer.add_norm_before_residual(norm_before_residual) def set_vocab(self): # eagle3: use tokenizer from target model if provided @@ -289,7 +290,7 @@ class LlamaModel(TextModel): factor = rope_params.get("factor", 8.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = self.hparams.get("original_max_position_embeddings", 8192) + old_context_len = rope_params.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/conversion/mamba.py b/conversion/mamba.py index be0e36a29..43d559ffb 100644 --- a/conversion/mamba.py +++ b/conversion/mamba.py @@ -114,7 +114,8 @@ class Mamba2Model(TextModel): hparams["text_config"] = hparams["llm_config"] super().__init__(dir_model, *args, hparams=hparams, **kwargs) self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"]) - self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or 2 * self.d_model + self.expand = self.find_hparam(["mamba_expand", "expand"], optional=True) or 2 + self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or self.expand * self.d_model self.n_group = self.find_hparam(["n_groups"], optional=True) or 1 def set_vocab(self): @@ -144,11 +145,9 @@ class Mamba2Model(TextModel): rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5 - # Fail early for models which don't have a block expansion factor of 2 - # TODO: does this really matter? # skip the assertion for FalconH1 Model if self.model_arch != gguf.MODEL_ARCH.FALCON_H1: - assert self.d_inner == 2 * self.d_model + assert self.d_inner == self.expand * self.d_model assert self.d_inner % head_dim == 0 self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default diff --git a/conversion/mimo.py b/conversion/mimo.py index d4067aab4..11ec28679 100644 --- a/conversion/mimo.py +++ b/conversion/mimo.py @@ -154,7 +154,7 @@ class MimoV2Model(TextModel): self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"]) self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"]) - rope_dim = int(self.hparams["head_dim"] * self.hparams["partial_rotary_factor"]) + rope_dim = int(self.hparams["head_dim"] * self.rope_parameters["partial_rotary_factor"]) self.gguf_writer.add_rope_dimension_count(rope_dim) self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon", 1e-5)) diff --git a/conversion/minicpm.py b/conversion/minicpm.py index e9a4c4a74..e31b26a00 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -32,11 +32,9 @@ class MiniCPMModel(TextModel): def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: rope_dims = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] - rope_scaling = self.find_hparam(['rope_scaling'], True) - if rope_scaling is not None: - long_factors = rope_scaling.get('long_factor', None) - short_factors = rope_scaling.get('short_factor', None) - + long_factors = self.rope_parameters.get('long_factor') + short_factors = self.rope_parameters.get('short_factor') + if long_factors or short_factors: if long_factors is None or short_factors is None: raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') @@ -85,13 +83,11 @@ class MiniCPM3Model(TextModel): self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"]) def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - rope_scaling = self.find_hparam(['rope_scaling'], True) - if rope_scaling is not None: + long_factors = self.rope_parameters.get('long_factor') + short_factors = self.rope_parameters.get('short_factor') + if long_factors or short_factors: rope_dims = self.hparams["qk_rope_head_dim"] - long_factors = rope_scaling.get('long_factor', None) - short_factors = rope_scaling.get('short_factor', None) - if long_factors is None or short_factors is None: raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') diff --git a/conversion/nemotron.py b/conversion/nemotron.py index dfeeb9785..e44688a78 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -125,17 +125,18 @@ class NemotronModel(TextModel): self.gguf_writer.add_layer_norm_eps(f_norm_eps) # * Partial RoPE - rot_pct = self.find_hparam(["partial_rotary_factor", "rope_pct", "rope_percent"]) + rot_pct = self.rope_parameters["partial_rotary_factor"] n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head) # * RopeScaling for Nemotron - if "rope_scaling" not in self.hparams or self.hparams["rope_scaling"] is None: + factor = self.hparams.get("factor") or self.rope_parameters.get("factor") + if factor is None: self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) else: self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR) - self.gguf_writer.add_rope_scaling_factor(self.hparams["factor"]) + self.gguf_writer.add_rope_scaling_factor(factor) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: # * Adding +1 to LayerNorm's weights here to implement layernorm1p w/o changing anything on the GGML engine side diff --git a/conversion/phi.py b/conversion/phi.py index 5e0d72847..df4bfe809 100644 --- a/conversion/phi.py +++ b/conversion/phi.py @@ -18,7 +18,7 @@ class Phi2Model(TextModel): model_arch = gguf.MODEL_ARCH.PHI2 def set_gguf_parameters(self): - rot_pct = self.find_hparam(["partial_rotary_factor"]) + rot_pct = self.rope_parameters["partial_rotary_factor"] n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) @@ -149,8 +149,8 @@ class Phi3MiniModel(TextModel): n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"]) rms_eps = self.find_hparam(["rms_norm_eps"]) max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"]) - orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"]) - rot_pct = self.hparams.get("partial_rotary_factor", 1.0) + orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"] + rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0) rope_dims = int(rot_pct * n_embd) // n_head self.gguf_writer.add_context_length(max_pos_embds) @@ -174,18 +174,19 @@ class Phi3MiniModel(TextModel): n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"]) - orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"]) - rot_pct = self.hparams.get("partial_rotary_factor", 1.0) + orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"] + rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0) rope_dims = int(rot_pct * n_embd) // n_head # write rope scaling for long context (128k) model - rope_scaling = self.find_hparam(['rope_scaling'], True) - if rope_scaling is None: + long_factors = self.rope_parameters.get('long_factor') + short_factors = self.rope_parameters.get('short_factor') + if not long_factors: return scale = max_pos_embds / orig_max_pos_embds - rope_scaling_type = rope_scaling.get('rope_type', rope_scaling.get('type', '')).lower() + rope_scaling_type = self.rope_parameters.get('rope_type', '').lower() if len(rope_scaling_type) == 0: raise KeyError('Missing the required key rope_scaling.type') @@ -198,9 +199,6 @@ class Phi3MiniModel(TextModel): self.gguf_writer.add_rope_scaling_attn_factors(attn_factor) - long_factors = rope_scaling.get('long_factor', None) - short_factors = rope_scaling.get('short_factor', None) - if long_factors is None or short_factors is None: raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') diff --git a/conversion/qwen.py b/conversion/qwen.py index 7eb135c83..0356bd2da 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -280,7 +280,7 @@ class Qwen3NextModel(Qwen2MoeModel): self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4)) if (rope_dim := self.hparams.get("head_dim")) is None: rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] - self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.25))) + self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.25))) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: @@ -625,3 +625,51 @@ class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReor @ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM") class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35MOE + + +@ModelBase.register("DFlashDraftModel") +class DFlashModel(Qwen3Model): + model_arch = gguf.MODEL_ARCH.DFLASH + + def set_vocab(self): + if self.target_model_dir is None: + raise ValueError( + "DFlash draft model requires --target-model-dir to be specified. " + "Please provide the path to the target model directory containing the tokenizer." + ) + logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}") + original_dir = self.dir_model + self.dir_model = self.target_model_dir + super().set_vocab() + self.dir_model = original_dir + + mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id") + if mask_token_id is not None: + self.gguf_writer.add_mask_token_id(mask_token_id) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + + block_size = self.hparams.get("block_size", 16) + self.gguf_writer.add_block_size(block_size) + dflash_config = self.hparams.get("dflash_config", {}) + + target_layer_ids = dflash_config.get("target_layer_ids", []) + if target_layer_ids: + extract_layer_ids = [i + 1 for i in target_layer_ids] + self.gguf_writer.add_target_layers(extract_layer_ids) + + use_sliding_window = self.hparams.get("use_sliding_window", False) + sliding_window = self.hparams.get("sliding_window") + layer_types = self.hparams.get("layer_types") + if use_sliding_window and sliding_window and layer_types: + is_swa = [lt == "sliding_attention" for lt in layer_types] + self.gguf_writer.add_sliding_window(sliding_window) + self.gguf_writer.add_sliding_window_pattern(is_swa) + + @classmethod + def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: + name, gen = item + if not name.startswith("model."): + name = "model." + name + return super().filter_tensors((name, gen)) diff --git a/conversion/stablelm.py b/conversion/stablelm.py index ba5e9aa6c..6e16378a0 100644 --- a/conversion/stablelm.py +++ b/conversion/stablelm.py @@ -28,7 +28,7 @@ class StableLMModel(TextModel): self.gguf_writer.add_embedding_length(hparams["hidden_size"]) self.gguf_writer.add_block_count(self.block_count) self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) - rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"]) + rotary_factor = self.rope_parameters["partial_rotary_factor"] self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) self.gguf_writer.add_head_count(hparams["num_attention_heads"]) self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"]) diff --git a/conversion/step3.py b/conversion/step3.py index 8c45b61c9..49bb5244a 100644 --- a/conversion/step3.py +++ b/conversion/step3.py @@ -314,7 +314,7 @@ class Step35Model(TextModel): factor = float(rope_params.get("factor", 8.0)) low_freq_factor = float(rope_params.get("low_freq_factor", 1.0)) high_freq_factor = float(rope_params.get("high_freq_factor", 4.0)) - old_context_len = int(rope_params.get("original_max_position_embeddings", self.hparams.get("original_max_position_embeddings", 8192))) + old_context_len = int(rope_params.get("original_max_position_embeddings", 8192)) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/convert_hf_to_gguf_update.py b/convert_hf_to_gguf_update.py index b4c8a7cf0..91c006278 100755 --- a/convert_hf_to_gguf_update.py +++ b/convert_hf_to_gguf_update.py @@ -100,6 +100,7 @@ models = [ {"name": "refact", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/smallcloudai/Refact-1_6-base", }, {"name": "command-r", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereForAI/c4ai-command-r-v01", }, {"name": "tiny_aya", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/tiny-aya-base", }, + {"name": "cohere2moe", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/CohereLabs/North-Mini-Code-1.0", }, {"name": "qwen2", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/Qwen/Qwen1.5-7B", }, {"name": "olmo", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/allenai/OLMo-1.7-7B-hf", }, {"name": "dbrx", "tokt": TOKENIZER_TYPE.BPE, "repo": "https://huggingface.co/databricks/dbrx-base", }, diff --git a/convert_lora_to_gguf.py b/convert_lora_to_gguf.py index 45202b333..47c09af53 100755 --- a/convert_lora_to_gguf.py +++ b/convert_lora_to_gguf.py @@ -25,7 +25,7 @@ import gguf from gguf.constants import GGUFValueType # reuse model definitions from the conversion/ package -from conversion import LazyTorchTensor, ModelBase, get_model_class +from conversion import LazyTorchTensor, ModelBase, get_model_class, ModelType, get_model_architecture logger = logging.getLogger("lora-to-gguf") @@ -396,12 +396,12 @@ if __name__ == '__main__': hparams = ModelBase.load_hparams(dir_base_model, False) with torch.inference_mode(): + model_arch = get_model_architecture(hparams, ModelType.TEXT) try: - model_arch = hparams.get("text_config", {}).get("architectures", hparams["architectures"])[0] - logger.info("Using model architecture: %s", model_arch) model_class = get_model_class(model_arch) + logger.info("Using model architecture: %s", model_arch) except NotImplementedError: - logger.error(f"Model {hparams['architectures'][0]} is not supported") + logger.error(f"Model {model_arch} is not supported") sys.exit(1) class LoraModel(model_class): # ty: ignore[unsupported-base] diff --git a/embd_res/kcpp_docs.embd b/embd_res/kcpp_docs.embd index 92234c19d..ed44c09f9 100644 --- a/embd_res/kcpp_docs.embd +++ b/embd_res/kcpp_docs.embd @@ -2849,6 +2849,87 @@ "responses": {"default": {"description": ""}} } }, + "/v1/images/generations": { + "post": { + "summary": "Generates images from a text prompt. Please refer to OpenAI documentation", + "description": "Creates images from a text prompt.\n\n This is an OpenAI compatibility endpoint.\n\n Please refer to OpenAI documentation at [https://developers.openai.com/docs/api-reference/images/create](https://developers.openai.com/docs/api-reference/images/create).", + "requestBody": { + "content": { + "application/json": { + "example": {"model":"kcpp","prompt": "picture of a kobold, high quality HD render", "n": 1, "size": "512x512", "response_format": "b64_json"}, + "schema": { + "properties": { + "model": { + "type": "string", + "description": "Model identifier. Use kcpp for the currently loaded image model." + }, + "prompt": { + "type": "string", + "description": "Text prompt describing the image to generate." + }, + "n": { + "type": "integer", + "description": "Number of images to generate.", + "minimum": 1 + }, + "size": { + "type": "string", + "description": "Requested image size, such as 512x512 or 1024x1024." + }, + "response_format": { + "type": "string", + "description": "Response image format. b64_json returns base64 encoded image data." + } + }, + "required": [ + "prompt" + ], + "type": "object" + } + } + }, + "required": true + }, + "tags": [ + "v1" + ], + "responses": { + "200": { + "content": { + "application/json": { + "example": {"created": 1710000000, "data": [{"b64_json": "base64_image_data"}]}, + "schema": { + "properties": { + "created": { + "type": "integer", + "description": "Unix timestamp for the generation request." + }, + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "b64_json": { + "type": "string", + "description": "Base64 encoded image data." + }, + "url": { + "type": "string", + "description": "Image URL, if URL responses are supported." + } + } + } + } + }, + "type": "object" + } + } + }, + "description": "Successful request" + } + } + } + }, "/v1/models": { "get": { "summary": "List and describe the various models available in the API. Please refer to OpenAI documentation", diff --git a/embd_res/kcpp_musicui.embd b/embd_res/kcpp_musicui.embd index 85738e1b6..3c5d44990 100644 --- a/embd_res/kcpp_musicui.embd +++ b/embd_res/kcpp_musicui.embd @@ -307,6 +307,11 @@ select{ +
+ + +
+
@@ -445,7 +450,8 @@ async function generateTTS(){ const payload = { input: document.getElementById("tts_input").value, - voice: document.getElementById("tts_voice").value + voice: document.getElementById("tts_voice").value, + use_mp3: document.getElementById("tts_use_mp3").checked }; const instruction = document.getElementById("tts_instruction").value; @@ -495,6 +501,7 @@ async function generateTTS(){ function clearTTS(){ document.getElementById("tts_input").value=""; document.getElementById("tts_instruction").value=""; + document.getElementById("tts_use_mp3").checked=false; } //end of tts part @@ -935,4 +942,4 @@ fetchStats(); - \ No newline at end of file + diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd index 1fe8dcb96..745fec1d3 100644 --- a/embd_res/kcpp_sdui.embd +++ b/embd_res/kcpp_sdui.embd @@ -5,18 +5,18 @@ Stable UI for KoboldCpp - - diff --git a/embd_res/klite.embd b/embd_res/klite.embd index 6bb99b4d1..3b3f86b40 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -12,7 +12,7 @@ Current version indicated by LITEVER below. --> - - - - {#snippet child({ props })} - - {/snippet} - + onclick?.(e); + }} + class="h-6 w-6 p-0 {className} flex hover:bg-transparent data-[state=open]:bg-transparent!" + aria-label={ariaLabel || tooltip} + > + {#if icon} + {@const IconComponent = icon} - -

{tooltip}

- - + + {/if} + +{/snippet} + +{#if showTooltip} + + + + {#snippet child({ props })} + {@render button(props)} + {/snippet} + + + +

{tooltip}

+ + +{:else} + {@render button({ href })} +{/if} + + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index ed26f9ea5..9b2077b8d 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -494,7 +494,7 @@ />
{/each} - {#if getEnabledMcpServers().length === 0} + {#if visibleMcpServers.length === 0}
No MCP servers configured
@@ -270,14 +269,22 @@ {/if} - {#if hasMcpPromptsSupport} -
{:else if toolsStore.isToolsEndpointUnreachable}
- - - - - Run llama-server with {CLI_FLAGS.TOOLS} flag to enable - - Built-in Tools. - - - diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte index 6a91bf905..08d691c14 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte @@ -42,6 +42,7 @@ {hasMcpPromptsSupport} {hasMcpResourcesSupport} {onFileUpload} + {onSystemPromptClick} {onMcpPromptClick} {onMcpResourcesClick} > diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index 712326cba..998e8dcb4 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -63,8 +63,8 @@ modelsStore.selectedModelName = conversationModel; modelsStore.selectModelByName(conversationModel); } else { - modelsStore.selectedModelName = null; - modelsStore.clearSelection(); + modelsStore.selectedModelId = null; + modelsStore.selectedModelName = conversationModel; } lastSyncedConversationModel = conversationModel; } else if ( @@ -141,19 +141,7 @@ }); $effect(() => { - if (!isRouter) { - isSelectedModelInCache = true; - } else if (conversationModel) { - isSelectedModelInCache = modelOptions().some((option) => option.model === conversationModel); - } else { - const currentModelId = selectedModelId(); - - if (!currentModelId) { - isSelectedModelInCache = false; - } else { - isSelectedModelInCache = modelOptions().some((option) => option.id === currentModelId); - } - } + isSelectedModelInCache = !isRouter || !!conversationModel || !!selectedModelId(); }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte index 8774bf63a..eff0364fa 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte @@ -20,7 +20,7 @@ type="submit" disabled={isDisabled} class={[ - 'h-8 w-8 rounded-full p-0', + 'md:h-8 md:w-8 h-9 w-9 rounded-full p-0', showErrorState && 'bg-red-400/10 text-red-400 hover:bg-red-400/20 hover:text-red-400 disabled:opacity-100' ]} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte index 72e62f319..3e683389f 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte @@ -1,4 +1,5 @@ -
+
{#if message.role === MessageRole.SYSTEM} (null); + let isCurrentlyLoading = $derived(isLoading()); let isStreaming = $derived(isChatStreaming()); let hasNoContent = $derived(!message?.content?.trim()); let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming); + // during a router auto-load the message has no model yet, so target the selected one + let loadTargetModel = $derived(message.model ?? modelsStore.selectedModelName); + let modelLoadProgress = $derived( + isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null + ); + let modelLoadingText = $derived(modelLoadProgressText(modelLoadProgress)); + let showProcessingInfoTop = $derived( message?.role === MessageRole.ASSISTANT && isActivelyProcessing && @@ -200,6 +210,42 @@ isLastAssistantMessage ); + let assistantEl: HTMLDivElement | undefined = $state(); + let lastUserMessageHeight = $state(0); + let assistantMarginTop = $state(0); + + $effect(() => { + if (!assistantEl) return; + + assistantMarginTop = Math.round(parseFloat(getComputedStyle(assistantEl).marginTop)); + + const chatMessageEl = assistantEl.closest('.chat-message'); + const previousChatMessage = chatMessageEl?.previousElementSibling; + const userMessageEl = previousChatMessage?.querySelector( + '.chat-message-user' + ) as HTMLElement | null; + + if (!userMessageEl) { + lastUserMessageHeight = 0; + return; + } + + const updateHeight = () => { + const rect = userMessageEl.getBoundingClientRect(); + const marginTop = Math.round(parseFloat(getComputedStyle(userMessageEl).marginTop)); + lastUserMessageHeight = Math.round(rect.height + marginTop); + }; + + updateHeight(); + + const resizeObserver = new ResizeObserver(updateHeight); + resizeObserver.observe(userMessageEl); + + return () => { + resizeObserver.disconnect(); + }; + }); + function handleCopyModel() { void copyToClipboard(displayedModel ?? ''); } @@ -212,15 +258,21 @@
0 + ? `${lastUserMessageHeight}px` + : undefined} + style:--assistant-margin-top={assistantMarginTop > 0 ? `${assistantMarginTop}px` : undefined} role="group" aria-label="Assistant message with actions" > {#if showProcessingInfoTop} -
+
- {processingState.getPromptProgressText() ?? + {modelLoadingText ?? + processingState.getPromptProgressText() ?? processingState.getProcessingMessage() ?? 'Processing...'} @@ -249,10 +301,11 @@ {/if} {#if showProcessingInfoBottom} -
+
- {processingState.getPromptProgressText() ?? + {modelLoadingText ?? + processingState.getPromptProgressText() ?? processingState.getProcessingMessage() ?? 'Processing...'} @@ -268,13 +321,19 @@ > {#if isRouter} { const status = modelsStore.getModelStatus(modelId); if (status !== ServerModelStatus.LOADED) { - await modelsStore.loadModel(modelId); + pendingModel = modelId; + + try { + await modelsStore.loadModel(modelId); + } finally { + pendingModel = null; + } } onRegenerate(modelName); @@ -342,6 +401,23 @@
diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte index 349f7e7fb..ee99e6b5e 100644 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerAddNew.svelte @@ -4,7 +4,7 @@ import { McpServerForm } from '$lib/components/app/mcp'; import { mcpStore } from '$lib/stores/mcp.svelte'; import { conversationsStore } from '$lib/stores/conversations.svelte'; - import { uuid } from '$lib/utils'; + import { parseHeadersToArray, uuid } from '$lib/utils'; import { MCP_SERVER_ID_PREFIX } from '$lib/constants'; interface Props { @@ -26,6 +26,10 @@ return 'Invalid URL format'; } }); + let newServerHeaderPairsValid = $derived( + parseHeadersToArray(newServerHeaders).every((p) => p.key.trim() && p.value.trim()) + ); + let canSave = $derived(!newServerUrlError && newServerHeaderPairsValid); function handleOpenChange(value: boolean) { if (!value) { @@ -37,7 +41,7 @@ } function saveNewServer() { - if (newServerUrlError) return; + if (!canSave) return; const newServerId = uuid() ?? `${MCP_SERVER_ID_PREFIX}-${Date.now()}`; @@ -52,6 +56,11 @@ handleOpenChange(false); } + + function handleSubmit(event: SubmitEvent) { + event.preventDefault(); + saveNewServer(); + } @@ -60,29 +69,27 @@ Add New Server -
- (newServerUrl = v)} - onHeadersChange={(v) => (newServerHeaders = v)} - urlError={newServerUrl ? newServerUrlError : null} - id="new-server" - /> -
+
+
+ (newServerUrl = v)} + onHeadersChange={(v) => (newServerHeaders = v)} + urlError={newServerUrl ? newServerUrlError : null} + id="new-server" + /> +
- - + + - - + + +
diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte new file mode 100644 index 000000000..9b4489b82 --- /dev/null +++ b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte @@ -0,0 +1,180 @@ + + + + + + Do more with MCP + + Power-up your experience by adding tools, resources and more capabilities provided by MCP + servers. + + + +
+

Quickly get started with

+ + {#each RECOMMENDED_MCP_SERVERS as server (server.id)} + (selected[server.id] = enabled)} + /> + {/each} + + {#if addedServers.length > 0} + {#each addedServers as server (server.id)} + + {/each} + {/if} + + {#if showAddForm} + + (newServerUrl = v)} + onHeadersChange={(v) => (newServerHeaders = v)} + urlError={newServerUrl ? newServerUrlError : null} + id="recommendation-new-server" + /> + +
+ + + +
+ + {:else} + + + + {/if} +
+ + + + + + + + diff --git a/tools/ui/src/lib/components/app/dialogs/index.ts b/tools/ui/src/lib/components/app/dialogs/index.ts index 29136308c..73f22c565 100644 --- a/tools/ui/src/lib/components/app/dialogs/index.ts +++ b/tools/ui/src/lib/components/app/dialogs/index.ts @@ -18,6 +18,15 @@ */ export { default as DialogMcpServerAddNew } from './DialogMcpServerAddNew.svelte'; +/** + * **DialogMcpServerRecommendations** - Suggested MCP servers opt-in dialog + * + * Prompts the user to enable pre-defined recommended MCP servers on first launch. + * Shows one switch per suggested server and persists the choice as a per-chat + * override so the selected servers become available in conversations. + */ +export { default as DialogMcpServerRecommendations } from './DialogMcpServerRecommendations.svelte'; + /** * **DialogExportSettings** - Settings export dialog with sensitive data warning * diff --git a/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte b/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte index e0bd8d98e..fd6e59a5b 100644 --- a/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte +++ b/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte @@ -1,4 +1,5 @@
@@ -103,6 +123,7 @@ {#each pairs as pair, index (index)}
void; @@ -15,6 +16,7 @@ } let { + autofocus, value = $bindable(''), placeholder = 'Search...', onInput, @@ -39,7 +41,7 @@ if (value) { value = ''; onInput?.(''); - ref?.focus(); + ref?.focus({ preventScroll: true }); } else { onClose?.(); } @@ -52,6 +54,7 @@ /> -
+
{#if showSkeleton} {:else if protocolVersion} diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte new file mode 100644 index 000000000..6cb3e18b6 --- /dev/null +++ b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte @@ -0,0 +1,156 @@ + + + +
+
+ {#if showSkeleton} + + + + + {:else} + + {/if} +
+ + +
+ + {#if isError && errorMessage} +

{errorMessage}

+ {/if} + + {#if showSkeleton} +
+ +
+ +
+ + + + +
+ {:else} + {#if description} + {#if description.lines === 2} +

+ {description.text} +

+ {:else} +

+ {description.text} +

+ {/if} + {/if} + + {#if tools.length > 0} +
+ {#each visibleTools as tool (tool.name)} + + + + {tool.name} + + + + +

+ {tool.description ?? 'No description'} +

+ + + {/each} + + {#if hiddenToolCount > 0} + + + + + {hiddenToolCount} more tools + + + + +

+ {hiddenTools.map((tool) => tool.name).join(', ')} +

+ + + {/if} +
+ {/if} + {/if} + diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte index 6727a9000..8ed4ee8b8 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte @@ -1,6 +1,7 @@ -
-

Configure Server

+
+
+

Configure Server

- (editUrl = v)} - onHeadersChange={(v) => (editHeaders = v)} - onUseProxyChange={(v) => (editUseProxy = v)} - urlError={editUrl ? urlError : null} - id={serverId} - /> + (editUrl = v)} + onHeadersChange={(v) => (editHeaders = v)} + onUseProxyChange={(v) => (editUseProxy = v)} + urlError={editUrl ? urlError : null} + id={serverId} + /> -
- +
+ - + +
-
+
diff --git a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte index 79738e30d..7f05d5fef 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte @@ -38,14 +38,87 @@ let headerPairs = $derived(parseHeadersToArray(headers)); + const AUTHORIZATION_HEADER = 'Authorization'; + const BEARER_PREFIX = 'Bearer '; + + // Heuristic: this dedicated UI only owns Authorization headers that already + // carry a Bearer scheme. Anything else (e.g. Basic, raw tokens) stays in the + // KV section so the user can still edit those values verbatim. + const matchesAuthorizationKey = (key: string): boolean => + key.trim().toLowerCase() === AUTHORIZATION_HEADER.toLowerCase(); + + const isBearerScheme = (value: string): boolean => + value.trim().toLowerCase().startsWith(BEARER_PREFIX.toLowerCase()); + + const ownedByBearerUi = (p: KeyValuePair): boolean => + matchesAuthorizationKey(p.key) && isBearerScheme(p.value); + + let hasAuthorization = $derived(headerPairs.some(ownedByBearerUi)); + + let wantsAuthorization = $state(false); + + let showAuthorization = $derived(hasAuthorization || wantsAuthorization); + + let urlInput: HTMLInputElement | null = $state(null); + let bearerInput: HTMLInputElement | null = $state(null); + + $effect(() => { + urlInput?.focus(); + }); + + $effect(() => { + if (wantsAuthorization && bearerInput) { + bearerInput.focus(); + } + }); + + let bearerToken = $derived.by(() => { + const auth = headerPairs.find(ownedByBearerUi); + if (!auth) return ''; + return auth.value.trim().slice(BEARER_PREFIX.length).trim(); + }); + + $effect(() => { + if (!headers.trim()) { + wantsAuthorization = false; + } + }); + function updateHeaderPairs(newPairs: KeyValuePair[]) { headerPairs = newPairs; onHeadersChange(serializeHeaders(newPairs)); } + + // The dedicated UI owns the Authorization slot end-to-end when the user + // engages it: any prior Authorization row (Bearer or otherwise) is replaced + // by exactly one { Authorization: "Bearer " } entry. JSON's last-key + // behavior would otherwise pick one arbitrarily, so we strip first. + function updateBearerToken(token: string) { + const filtered = headerPairs.filter((p) => !matchesAuthorizationKey(p.key)); + + const trimmed = token.trim(); + + if (trimmed) { + filtered.push({ key: AUTHORIZATION_HEADER, value: `${BEARER_PREFIX}${trimmed}` }); + } + + updateHeaderPairs(filtered); + } + + function setUseAuthorization(checked: boolean) { + wantsAuthorization = checked; + + if (!checked) { + // Only drop the entry this UI owns; a non-Bearer Authorization row + // authored in the KV section must survive a toggle off untouched. + const filtered = headerPairs.filter((p) => !ownedByBearerUi(p)); + updateHeaderPairs(filtered); + } + } -
-
+
+
@@ -57,50 +130,52 @@ value={url} oninput={(e) => onUrlChange(e.currentTarget.value)} class={urlError ? 'border-destructive' : ''} + bind:ref={urlInput} /> {#if urlError}

{urlError}

{/if} - - {#if !isWebSocket && onUseProxyChange} - - {/if}
+ + + {#if showAuthorization} +
+ updateBearerToken(e.currentTarget.value)} + class="pl-16" + bind:ref={bearerInput} + /> + + + Bearer + +
+ {/if} + !ownedByBearerUi(p))} + onPairsChange={(pairs) => { + const auth = headerPairs.find(ownedByBearerUi); + updateHeaderPairs(auth ? [...pairs, auth] : pairs); + }} keyPlaceholder="Header name" valuePlaceholder="Value" addButtonLabel="Add" @@ -108,4 +183,37 @@ sectionLabel="Custom Headers" sectionLabelOptional /> + + {#if !isWebSocket && onUseProxyChange} + + {/if}
diff --git a/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte b/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte index feafc5d81..3f128e02c 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte @@ -1,6 +1,7 @@ + +
+ {@html logoMark} +
+ + diff --git a/tools/ui/src/lib/components/app/misc/index.ts b/tools/ui/src/lib/components/app/misc/index.ts index 64b76fb71..b550ae66a 100644 --- a/tools/ui/src/lib/components/app/misc/index.ts +++ b/tools/ui/src/lib/components/app/misc/index.ts @@ -51,3 +51,11 @@ export { default as KeyboardShortcutInfo } from './KeyboardShortcutInfo.svelte'; * Preview button is shown only for HTML code blocks. */ export { default as CodeBlockActions } from './CodeBlockActions.svelte'; + +/** + * **Logo** - Application brand mark + * + * Inline SVG of the application logo. Accepts styling via the standard + * `class` and `style` props and inherits color via `currentColor`. + */ +export { default as Logo } from './Logo.svelte'; diff --git a/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte b/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte new file mode 100644 index 000000000..fa9a02108 --- /dev/null +++ b/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte @@ -0,0 +1,11 @@ + + + +
+
+
diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte index 40006a4c9..720963a8d 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte @@ -2,8 +2,10 @@ import { ChevronDown, Loader2, Package } from '@lucide/svelte'; import * as DropdownMenu from '$lib/components/ui/dropdown-menu'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { KeyboardKey } from '$lib/enums'; + import { KeyboardKey, ServerModelStatus } from '$lib/enums'; import { useModelsSelector } from '$lib/hooks/use-models-selector.svelte'; + import { modelsStore, routerModels } from '$lib/stores/models.svelte'; + import { modelLoadFraction } from '$lib/utils'; import { DialogModelInformation, DropdownMenuSearchable, @@ -11,6 +13,7 @@ ModelsSelectorList, ModelsSelectorOption } from '$lib/components/app'; + import ModelLoadHighlight from './ModelLoadHighlight.svelte'; import type { ModelItem } from './utils'; interface Props { @@ -113,6 +116,17 @@ {/if} {:else} {@const selectedOption = ms.getDisplayOption()} + {@const triggerModel = selectedOption?.model} + {@const triggerStatus = triggerModel + ? routerModels().find((m) => m.id === triggerModel)?.status?.value + : undefined} + {@const triggerLoading = + !!triggerModel && + (triggerStatus === ServerModelStatus.LOADING || + modelsStore.isModelOperationInProgress(triggerModel))} + {@const triggerLoadPercent = triggerLoading + ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) + : 0} {#if ms.isRouter} @@ -123,7 +137,7 @@ {/if} + + {#if triggerLoading} + + {/if} {/snippet} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte index d103d4b67..981111e20 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte @@ -10,9 +10,11 @@ RotateCw } from '@lucide/svelte'; import { ActionIcon, ModelId } from '$lib/components/app'; + import ModelLoadHighlight from './ModelLoadHighlight.svelte'; import type { ModelOption } from '$lib/types/models'; import { ServerModelStatus } from '$lib/enums'; import { modelsStore, routerModels } from '$lib/stores/models.svelte'; + import { modelLoadFraction, modelLoadProgressText } from '$lib/utils'; interface Props { option: ModelOption; @@ -50,11 +52,15 @@ (serverStatus === ServerModelStatus.LOADED || isSleeping) && !isOperationInProgress ); let isLoading = $derived(serverStatus === ServerModelStatus.LOADING || isOperationInProgress); + + let loadProgress = $derived(isLoading ? modelsStore.getLoadProgress(option.model) : null); + let loadPercent = $derived(Math.round(modelLoadFraction(loadProgress) * 100)); + let loadTitle = $derived(modelLoadProgressText(loadProgress));
onSelect(option.id)} onmouseenter={onMouseEnter} @@ -79,7 +86,7 @@
e.stopPropagation()} > {#if isFav} @@ -113,12 +120,16 @@
{#if isLoading} - +
+ +
{:else if isFailed} -
- +
+ -