diff --git a/.github/workflows/kcpp-build-release-macos.yaml b/.github/workflows/kcpp-build-release-macos.yaml index fa20518e6..bfed75028 100644 --- a/.github/workflows/kcpp-build-release-macos.yaml +++ b/.github/workflows/kcpp-build-release-macos.yaml @@ -34,22 +34,16 @@ jobs: - name: Build id: make_build run: | - make LLAMA_METAL=1 koboldcpp_default - mkdir -p build_artifacts - mv koboldcpp_default.so build_artifacts/ - make clean - make koboldcpp_macos_failsafe - mv koboldcpp_macos_failsafe.so koboldcpp_failsafe.so - mv build_artifacts/koboldcpp_default.so . + make LLAMA_METAL=1 LLAMA_PORTABLE=1 chmod +x './create_ver_file.sh' . create_ver_file.sh - pyinstaller --noconfirm --onefile --collect-all customtkinter --collect-all jinja2 --collect-all psutil --add-data './koboldcpp_default.so:.' --add-data './koboldcpp_failsafe.so:.' --add-data './ggml-metal-merged.metal:.' --add-data './kcpp_adapters:./kcpp_adapters' --add-data './koboldcpp.py:.' --add-data './json_to_gbnf.py:.' --add-data './LICENSE.md:.' --add-data './MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md:.' --add-data './embd_res:./embd_res' --version-file './version.txt' --clean --console koboldcpp.py -n "koboldcpp-mac-arm64" + pyinstaller --noconfirm --onefile --collect-all customtkinter --collect-all jinja2 --collect-all psutil --add-data './koboldcpp_default.so:.' --add-data './ggml-metal-merged.metal:.' --add-data './kcpp_adapters:./kcpp_adapters' --add-data './koboldcpp.py:.' --add-data './json_to_gbnf.py:.' --add-data './LICENSE.md:.' --add-data './MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md:.' --add-data './embd_res:./embd_res' --version-file './version.txt' --clean --console koboldcpp.py -n "koboldcpp-mac-arm64" - name: Test id: test run: | wget https://huggingface.co/concedo/koboldcpp/resolve/main/baby_llama.gguf - dist/koboldcpp-mac-arm64 --model baby_llama.gguf --gpulayers 99 --benchmark --prompt 'Once upon a' + dist/koboldcpp-mac-arm64 --model baby_llama.gguf --gpulayers 99 --benchmark --prompt 'Hi, my name is' - name: Save artifact uses: actions/upload-artifact@v6 diff --git a/CMakeLists.txt b/CMakeLists.txt index b3f782aaf..c44618e7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,9 +72,6 @@ if (MSVC) add_compile_options("$<$:/utf-8>") add_compile_options("$<$:/bigobj>") add_compile_options("$<$:/bigobj>") -else() - add_compile_options("$<$:-Wno-unused-value>") - add_compile_options("$<$:-Wno-unused-value>") endif() file(GLOB GGML_SOURCES_CUDA "ggml/src/ggml-cuda/*.cu") @@ -378,16 +375,6 @@ if (MINGW) add_compile_definitions(_WIN32_WINNT=0x602) endif() -# Standalone libmtmd build without pulling in the rest of the tools/ tree. -# Useful when packaging just the mtmd library for language bindings (e.g. an -# Apple XCFramework, or a WASM build). When the full tools build is enabled, -# mtmd is already built by the tools/ subdirectory above; this hook only fires -# when LLAMA_BUILD_TOOLS is OFF to avoid double-adding the target. -option(LLAMA_BUILD_MTMD "llama: build tools/mtmd library standalone" OFF) -if (LLAMA_BUILD_MTMD AND NOT (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TOOLS)) - add_subdirectory(tools/mtmd) -endif() - # # Build libraries # @@ -508,9 +495,8 @@ add_library(sdtype_adapter otherarch/sdcpp/src/core/ggml_graph_cut.cpp otherarch/sdcpp/src/core/ggml_graph_cut.h otherarch/sdcpp/examples/cli/image_metadata.cpp - otherarch/sdcpp/src/model_manager.cpp - otherarch/sdcpp/src/model_manager.h - otherarch/sdcpp/src/extensions/pulid_extension.cpp + otherarch/sdcpp/src/core/layer_registry.cpp + otherarch/sdcpp/src/core/layer_registry.h otherarch/sdcpp/src/model_loader.cpp otherarch/sdcpp/src/extensions/photomaker_extension.cpp otherarch/sdcpp/src/runtime/sample-cache.cpp @@ -519,8 +505,6 @@ add_library(sdtype_adapter otherarch/sdcpp/src/upscaler.cpp otherarch/sdcpp/src/runtime/guidance.cpp otherarch/sdcpp/src/runtime/guidance.h - otherarch/sdcpp/src/runtime/imatrix.cpp - otherarch/sdcpp/src/runtime/imatrix.h otherarch/sdcpp/src/stable-diffusion.cpp otherarch/sdcpp/thirdparty/zip.c otherarch/sdcpp/src/model_io/gguf_io.cpp @@ -537,8 +521,6 @@ add_library(sdtype_adapter otherarch/sdcpp/src/tokenizers/gpt_oss_tokenizer.cpp otherarch/sdcpp/src/tokenizers/tokenizer.cpp otherarch/sdcpp/src/tokenizers/tokenize_util.cpp - otherarch/sdcpp/src/core/backend_fit.cpp - otherarch/sdcpp/src/core/layer_split_partition.cpp otherarch/sdcpp/src/core/ggml_extend_backend.cpp) target_include_directories(sdtype_adapter PUBLIC . ./ggml/include ./ggml/src ./ggml/src/ggml-cpu ./include ./otherarch ./otherarch/tools ./vendor/stb ./vendor/nlohmann ./vendor ./otherarch/sdcpp ./otherarch/sdcpp/include ./otherarch/sdcpp/src ./otherarch/sdcpp/examples ./tools ./common) target_compile_features(sdtype_adapter PUBLIC cxx_std_17) # don't bump @@ -574,10 +556,7 @@ target_link_libraries(embeddings_adapter PRIVATE common2 ggml ${LLAMA_EXTRA_LIBS set_target_properties(embeddings_adapter PROPERTIES POSITION_INDEPENDENT_CODE ON) add_library(gpttype_adapter - gpttype_adapter.cpp - src/llama.cpp - common/chat.cpp - src/llama-model.cpp) + gpttype_adapter.cpp) target_include_directories(gpttype_adapter PUBLIC . ./src ./ggml/include ./ggml/src ./ggml/src/ggml-cpu ./include ./otherarch ./otherarch/tools ./vendor/stb ./vendor/nlohmann ./vendor ./otherarch/sdcpp ./otherarch/sdcpp/thirdparty ./tools ./common) target_compile_features(gpttype_adapter PUBLIC cxx_std_17) # don't bump target_link_libraries(gpttype_adapter PRIVATE common2 ggml ggml_v1 ggml_v2 ggml_v3 ${LLAMA_EXTRA_LIBS}) diff --git a/Makefile b/Makefile index 717c81de9..2a80ef65f 100644 --- a/Makefile +++ b/Makefile @@ -71,8 +71,8 @@ CXXFLAGS += -DGGML_USE_LLAMAFILE endif #lets try enabling everything -CFLAGS += -pthread -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable -Wno-unused-value -CXXFLAGS += -pthread -Wno-multichar -Wno-write-strings -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable -Wno-unused-value +CFLAGS += -pthread -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable +CXXFLAGS += -pthread -Wno-multichar -Wno-write-strings -Wno-deprecated -Wno-deprecated-declarations -Wno-unused-variable LDFLAGS = @@ -101,9 +101,9 @@ NONECFLAGS = LLAMA_USE_BUNDLED_GLSLC := 1 FAILSAFE_FLAGS = -DUSE_FAILSAFE -VULKAN_FLAGS = -DGGML_USE_VULKAN +VULKAN_FLAGS = -DGGML_USE_VULKAN -DSD_USE_VULKAN ifdef LLAMA_CUBLAS -CUBLAS_FLAGS = -DGGML_USE_CUDA +CUBLAS_FLAGS = -DGGML_USE_CUDA -DSD_USE_CUDA else CUBLAS_FLAGS = endif @@ -215,7 +215,7 @@ OBJS_CUDA_TEMP_INST += \ ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.o ifdef LLAMA_CUBLAS -CUBLAS_FLAGS = -DGGML_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include +CUBLAS_FLAGS = -DGGML_USE_CUDA -DSD_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include CUBLASLD_FLAGS = -lcuda -lcublas -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L$(CUDA_PATH)/targets/x86_64-linux/lib -L$(CUDA_PATH)/lib64/stubs -L/usr/local/cuda/targets/aarch64-linux/lib -L/usr/local/cuda/targets/sbsa-linux/lib -L/usr/lib/wsl/lib CUBLAS_OBJS = ggml-cuda.o ggml_v3-cuda.o ggml_v2-cuda.o ggml_v2-cuda-legacy.o CUBLAS_OBJS += $(patsubst %.cu,%.o,$(filter-out ggml/src/ggml-cuda/ggml-cuda.cu, $(wildcard ggml/src/ggml-cuda/*.cu))) @@ -315,7 +315,7 @@ HIPFLAGS += -DGGML_HIP_NO_ROCWMMA_FATTN endif endif -HIPFLAGS += -DGGML_USE_HIP -DGGML_HIP_NO_VMM -DGGML_USE_CUDA $(shell $(ROCM_PATH)/bin/hipconfig -C) +HIPFLAGS += -DGGML_USE_HIP -DGGML_HIP_NO_VMM -DGGML_USE_CUDA -DSD_USE_CUDA $(shell $(ROCM_PATH)/bin/hipconfig -C) HIPLDFLAGS += -L$(ROCM_PATH)/lib -Wl,-rpath=$(ROCM_PATH)/lib HIPLDFLAGS += -L$(ROCM_PATH)/lib64 -Wl,-rpath=$(ROCM_PATH)/lib64 HIPLDFLAGS += -lhipblas -lamdhip64 -lrocblas @@ -339,8 +339,8 @@ endif # LLAMA_HIPBLAS ifdef LLAMA_METAL -CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG -CXXFLAGS += -DGGML_USE_METAL +CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG -DSD_USE_METAL +CXXFLAGS += -DGGML_USE_METAL -DSD_USE_METAL LDFLAGS += -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders OBJS += ggml-metal.o ggml-metal-device.o ggml-metal-device-m.o ggml-metal-context-m.o ggml-metal-common.o ggml-metal-ops.o @@ -682,9 +682,7 @@ ggml-vulkan-shaders-noext.o: ggml/src/ggml-vulkan-shaders-noext.cpp ggml/include $(CXX) $(CXXFLAGS) $(VKGEN_NOEXT_FORCE) $(VULKAN_FLAGS) -c $< -o $@ # intermediate objects -llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h - $(CXX) $(CXXFLAGS) -c $< -o $@ -llama-model.o: src/llama-model.cpp src/llama-model.h src/models/models.h ggml/include/ggml.h include/llama.h +llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h $(CXX) $(CXXFLAGS) -c $< -o $@ common.o: common/common.cpp common/common.h common/log.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -700,10 +698,8 @@ llama-impl.o: src/llama-impl.cpp src/llama-impl.h $(CXX) $(CXXFLAGS) -c $< -o $@ budget.o: common/reasoning-budget.cpp common/reasoning-budget.h $(CXX) $(CXXFLAGS) -c $< -o $@ -chat.o: common/chat.cpp common/chat.h - $(CXX) $(CXXFLAGS) -c $< -o $@ -SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/diffusion/anima.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h +SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_registry.cpp src/core/layer_registry.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/diffusion/anima.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp @@ -734,9 +730,8 @@ otherarch/sdcpp/thirdparty/zip.o: otherarch/sdcpp/thirdparty/zip.c OBJS_SDTYPE := otherarch/sdcpp/sdtype_adapter.o $(OBJS_SDCOMMON) -LLAMASERVER_SRCS := tools/server/main.cpp tools/server/server.cpp tools/server/server-schema.cpp tools/server/server-chat.cpp tools/server/server-common.cpp tools/server/server-context.cpp tools/server/server-http.cpp tools/server/server-models.cpp tools/server/server-queue.cpp tools/server/server-task.cpp tools/server/server-tools.cpp tools/server/ui.cpp -COMMON_DOWNLOAD_SRCS := common/download.cpp common/hf-cache.cpp vendor/cpp-httplib/httplib.cpp -LLAMASERVER_COMMON_SRCS := common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) +LLAMASERVER_SRCS := tools/server/main.cpp tools/server/server.cpp tools/server/server-chat.cpp tools/server/server-common.cpp tools/server/server-context.cpp tools/server/server-http.cpp tools/server/server-models.cpp tools/server/server-queue.cpp tools/server/server-task.cpp tools/server/server-tools.cpp tools/server/ui.cpp +LLAMASERVER_COMMON_SRCS := common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp vendor/cpp-httplib/httplib.cpp LLAMASERVER_CXXFLAGS := -I./tools/mtmd @@ -759,7 +754,7 @@ music_default.o: otherarch/acestep/music_adapter.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ # idiotic "for easier compilation" -GPTTYPE_ADAPTER = gpttype_adapter.cpp model_adapter.h otherarch/otherarch.h include/llama.h otherarch/llama_v2.cpp otherarch/llama_v3.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp +GPTTYPE_ADAPTER = gpttype_adapter.cpp otherarch/llama_v2.cpp otherarch/llama_v3.cpp src/llama.cpp src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model.cpp src/llama-quant.cpp src/llama-hparams.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h gpttype_adapter_failsafe.o: $(GPTTYPE_ADAPTER) $(CXX) $(CXXFLAGS) $(FAILSAFE_FLAGS) -c $< -o $@ gpttype_adapter.o: $(GPTTYPE_ADAPTER) @@ -772,43 +767,43 @@ gpttype_adapter_vulkan_noavx2.o: $(GPTTYPE_ADAPTER) $(CXX) $(CXXFLAGS) $(FAILSAFE_FLAGS) $(VULKAN_FLAGS) -c $< -o $@ clean: - rm -vf *.o main ttsmain sdmain whispermain quantize_gguf quantize_gpt2 quantize_gptj quantize_neox quantize_mpt vulkan-shaders-gen vulkan-shaders-gen-noext gguf-split mtmd-cli mainvk fitparams embedding embeddingvk qwen3tts rpcserver llamaserver llamaservervk rpcserver.exe llamaserver.exe llamaservervk.exe qwen3tts.exe embeddingvk.exe embedding.exe fitparams.exe mainvk.exe mtmd-cli.exe gguf-split.exe vulkan-shaders-gen.exe vulkan-shaders-gen-noext.exe main.exe ttsmain.exe sdmain.exe whispermain.exe quantize_gguf.exe quantize_gptj.exe quantize_gpt2.exe quantize_neox.exe quantize_mpt.exe koboldcpp_default.dll koboldcpp_failsafe.dll koboldcpp_noavx2.dll koboldcpp_vulkan_failsafe.dll koboldcpp_cublas.dll koboldcpp_hipblas.dll koboldcpp_vulkan.dll koboldcpp_vulkan_noavx2.dll koboldcpp_default.so koboldcpp_failsafe.so koboldcpp_macos_failsafe.so koboldcpp_noavx2.so koboldcpp_vulkan_failsafe.so koboldcpp_cublas.so koboldcpp_hipblas.so koboldcpp_vulkan.so koboldcpp_vulkan_noavx2.so ggml/src/ggml-vulkan-shaders.cpp ggml/src/ggml-vulkan-shaders.hpp ggml/src/ggml-vulkan-shaders-noext.cpp ggml/src/ggml-vulkan-shaders-noext.hpp + rm -vf *.o main ttsmain sdmain whispermain quantize_gguf quantize_gpt2 quantize_gptj quantize_neox quantize_mpt vulkan-shaders-gen vulkan-shaders-gen-noext gguf-split mtmd-cli mainvk fitparams embedding embeddingvk qwen3tts rpcserver llamaserver llamaservervk rpcserver.exe llamaserver.exe llamaservervk.exe qwen3tts.exe embeddingvk.exe embedding.exe fitparams.exe mainvk.exe mtmd-cli.exe gguf-split.exe vulkan-shaders-gen.exe vulkan-shaders-gen-noext.exe main.exe ttsmain.exe sdmain.exe whispermain.exe quantize_gguf.exe quantize_gptj.exe quantize_gpt2.exe quantize_neox.exe quantize_mpt.exe koboldcpp_default.dll koboldcpp_failsafe.dll koboldcpp_noavx2.dll koboldcpp_vulkan_failsafe.dll koboldcpp_cublas.dll koboldcpp_hipblas.dll koboldcpp_vulkan.dll koboldcpp_vulkan_noavx2.dll koboldcpp_default.so koboldcpp_failsafe.so koboldcpp_noavx2.so koboldcpp_vulkan_failsafe.so koboldcpp_cublas.so koboldcpp_hipblas.so koboldcpp_vulkan.so koboldcpp_vulkan_noavx2.so ggml/src/ggml-vulkan-shaders.cpp ggml/src/ggml-vulkan-shaders.hpp ggml/src/ggml-vulkan-shaders-noext.cpp ggml/src/ggml-vulkan-shaders-noext.hpp rm -vrf ggml/src/ggml-cuda/*.o rm -vrf ggml/src/ggml-cuda/template-instances/*.o rm -vrf llguidance rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o # useful tools -main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +main: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mainvk: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttsmain: tools/tts/tts.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embedding: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) ggml/src/ggml-vulkan-shaders.cpp: ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp ifdef VULKAN_BUILD @@ -908,14 +903,11 @@ else endif #generated libraries -koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) - $(DEFAULT_BUILD) - -koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(DEFAULT_BUILD) ifdef FAILSAFE_BUILD -koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) +koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) $(FAILSAFE_BUILD) else koboldcpp_failsafe: @@ -923,7 +915,7 @@ koboldcpp_failsafe: endif ifdef NOAVX2_BUILD -koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(NOAVX2_BUILD) else koboldcpp_noavx2: @@ -931,7 +923,7 @@ koboldcpp_noavx2: endif ifdef CUBLAS_BUILD -koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) $(CUBLAS_BUILD) else koboldcpp_cublas: @@ -939,7 +931,7 @@ koboldcpp_cublas: endif ifdef HIPBLAS_BUILD -koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) $(HIPBLAS_BUILD) else koboldcpp_hipblas: @@ -947,12 +939,12 @@ koboldcpp_hipblas: endif ifdef VULKAN_BUILD -koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o llama.o chat.o llama-model.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(VULKAN_BUILD) ifdef NOAVX2_BUILD -koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(VULKAN_BUILD) -koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) +koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) $(VULKAN_BUILD) else koboldcpp_vulkan_noavx2: @@ -970,17 +962,17 @@ koboldcpp_vulkan_failsafe: endif # tools -quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) diff --git a/app/download.cpp b/app/download.cpp deleted file mode 100644 index 7227baadc..000000000 --- a/app/download.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include "arg.h" -#include "common.h" -#include "download.h" -#include "log.h" - -#include -#include - -static void print_usage(int /*argc*/, char ** argv) { - printf( - "\nexamples:\n" - " %s -hf ggml-org/gemma-3-4b-it-qat-GGUF\n" - " %s -hf ggml-org/gemma-3-4b-it-qat-GGUF:Q4_K_M\n" - " %s -hf ggml-org/models -hff model.gguf\n" - " %s -mu https://example.com/model.gguf -m model.gguf\n" - "\n", - argv[0], argv[0], argv[0], argv[0] - ); -} - -int llama_download(int argc, char ** argv); - -int llama_download(int argc, char ** argv) { - common_init(); - - common_params params; - params.verbosity = LOG_LEVEL_ERROR; - - if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_DOWNLOAD, print_usage)) { - return 1; - } - - const bool has_source = !params.model.hf_repo.empty() || !params.model.url.empty() || - !params.model.path.empty() || !params.model.docker_repo.empty(); - if (!has_source) { - fprintf(stderr, "error: no model source specified (use --hf-repo, --model-url, --model or --docker-repo)\n"); - return 1; - } - - try { - common_models_handler handler = common_models_handler_init(params, LLAMA_EXAMPLE_DOWNLOAD); - common_models_handler_apply(handler, params); - } catch (const std::exception & e) { - fprintf(stderr, "error: %s\n", e.what()); - return 1; - } - - if (!params.models_preset.empty()) { - // -hf pointed at a preset repo: print the preset path and stop - printf("%s\n", params.models_preset.c_str()); - return 0; - } - if (params.model.path.empty()) { - fprintf(stderr, "error: model download failed\n"); - return 1; - } - if (!std::filesystem::exists(params.model.path)) { - fprintf(stderr, "error: model file does not exist: %s\n", params.model.path.c_str()); - return 1; - } - - printf("%s\n", params.model.path.c_str()); - if (!params.mmproj.path.empty()) { - printf("%s\n", params.mmproj.path.c_str()); - } - if (!params.speculative.draft.mparams.path.empty()) { - printf("%s\n", params.speculative.draft.mparams.path.c_str()); - } - - return 0; -} diff --git a/colab.ipynb b/colab.ipynb index b8778a411..e5194a430 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -62,7 +62,7 @@ "Model = \"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\" #@param [\"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n", "MdCommand = \"\" #@markdown
\n", "Layers = \"Auto\" #@param [\"Auto\",\"999\"]{allow-input: true}\n", - "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\",\"24576\",\"32768\",\"40960\"] {allow-input: true}\n", + "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\"] {allow-input: true}\n", "\n", "#@markdown
\n", "LoadVisionMMProjector = False #@param {type:\"boolean\"}\n", @@ -130,7 +130,7 @@ " if Template == \"Gemma4 E4B Uncensored (General)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\"\n", - " CustomCtxSize = \"40960\"\n", + " CustomCtxSize = \"16384\"\n", " CustomMmproj = \"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\"\n", " if Template == \"Tiefighter 13B (General)\":\n", " Customized = True\n", diff --git a/common/arg.cpp b/common/arg.cpp index bf2b62460..f433c5f6f 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -18,7 +18,6 @@ # define NOMINMAX #endif #include -#include #endif #define JSON_ASSERT GGML_ASSERT @@ -287,17 +286,108 @@ static std::string clean_file_name(const std::string & fname) { return clean_fname; } +static bool common_params_handle_remote_preset(common_params & params, llama_example ex) { + GGML_ASSERT(!params.model.hf_repo.empty()); + + // the returned hf_repo is without tag + auto [hf_repo, hf_tag] = common_download_split_repo_tag(params.model.hf_repo); + + // "latest" tag (default if not specified) is translated to "default" preset + if (hf_tag == "latest") { + hf_tag = "default"; + } + + std::string model_endpoint = common_get_model_endpoint(); + auto preset_url = model_endpoint + hf_repo + "/resolve/main/preset.ini"; + + // prepare local path for caching + auto preset_fname = clean_file_name(hf_repo + "_preset.ini"); + auto preset_path = fs_get_cache_file(preset_fname); + common_download_opts opts; + opts.bearer_token = params.hf_token; + opts.offline = params.offline; + + LOG_TRC("%s: looking for remote preset at %s\n", __func__, preset_url.c_str()); + const int status = common_download_file_single(preset_url, preset_path, opts); + const bool has_preset = status >= 200 && status < 400; + + // remote preset is optional, so we don't error out if not found + if (has_preset) { + LOG_TRC("%s: applying remote preset from %s\n", __func__, preset_url.c_str()); + common_preset_context ctx(ex, /* only_remote_allowed */ true); + common_preset global; + auto remote_presets = ctx.load_from_ini(preset_path, global); + remote_presets = ctx.cascade(global, remote_presets); + if (remote_presets.find(hf_tag) != remote_presets.end()) { + common_preset preset = remote_presets.at(hf_tag); + LOG_INF("\n%s", preset.to_ini().c_str()); // to_ini already added trailing newline + preset.apply_to_params(params); + } else { + throw std::runtime_error("Remote preset.ini does not contain [" + std::string(hf_tag) + "] section"); + } + } else { + LOG_TRC("%s: no remote preset found, skipping\n", __func__); + } + + return has_preset; +} + struct handle_model_result { bool found_mmproj = false; common_params_model mmproj; bool found_mtp = false; common_params_model mtp; - - bool found_preset = false; - std::string preset_path; }; +static handle_model_result common_params_handle_model(struct common_params_model & model, + const common_download_opts & opts) { + handle_model_result result; + + if (!model.docker_repo.empty()) { + model.path = common_docker_resolve_model(model.docker_repo); + model.name = model.docker_repo; + } else if (!model.hf_repo.empty()) { + // If -m was used with -hf, treat the model "path" as the hf_file to download + if (model.hf_file.empty() && !model.path.empty()) { + model.hf_file = model.path; + model.path = ""; + } + common_download_opts hf_opts = opts; + auto download_result = common_download_model(model, hf_opts); + + if (download_result.model_path.empty()) { + throw std::runtime_error("failed to download model from Hugging Face"); + } + + model.name = model.hf_repo; + model.path = download_result.model_path; + + if (!download_result.mmproj_path.empty()) { + result.found_mmproj = true; + result.mmproj.path = download_result.mmproj_path; + } + + if (!download_result.mtp_path.empty()) { + result.found_mtp = true; + result.mtp.path = download_result.mtp_path; + } + } else if (!model.url.empty()) { + if (model.path.empty()) { + auto f = string_split(model.url, '#').front(); + f = string_split(f, '?').front(); + model.path = fs_get_cache_file(string_split(f, '/').back()); + } + + auto download_result = common_download_model(model, opts); + if (download_result.model_path.empty()) { + throw std::runtime_error("failed to download model from " + model.url); + } + } + + return result; +} + const std::vector kv_cache_types = { GGML_TYPE_F32, GGML_TYPE_F16, @@ -341,244 +431,62 @@ static bool parse_bool_value(const std::string & value) { throw std::invalid_argument("the argument has been removed. " + msg); } -// -// common_models_handler -// - -static std::string get_default_local_path(const std::string & url) { - auto f = string_split(url, '#').front(); - f = string_split(f, '?').front(); - return fs_get_cache_file(string_split(f, '/').back()); -} - -common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) { - common_download_hf_plan plan; - common_download_hf_plan plan_spec; - common_download_hf_plan plan_voc; - common_download_opts opts; - - const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), - params.speculative.types.end(), - COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); - - // only download mmproj if the current example is using it - bool use_mmproj = false; - for (const auto & ex : mmproj_examples) { - if (curr_ex == ex) { - use_mmproj = true; - break; - } - } - - opts.bearer_token = params.hf_token; - opts.offline = params.offline; - opts.download_mtp = spec_type_draft_mtp; - opts.download_mmproj = use_mmproj && !params.no_mmproj - && params.mmproj.path.empty() && params.mmproj.url.empty(); - - if (!params.model.hf_repo.empty()) { - plan = common_download_get_hf_plan(params.model, opts); - } - - if (!params.speculative.draft.mparams.hf_repo.empty()) { - plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts); - } - - if (!params.vocoder.model.hf_repo.empty()) { - plan_voc = common_download_get_hf_plan(params.vocoder.model, opts); - } - - return common_models_handler{plan, plan_spec, plan_voc, opts}; -} - -bool common_models_handler_is_preset_repo(const common_models_handler & handler) { - return !handler.plan.preset.url.empty(); -} - -static std::vector build_url_tasks(const common_params_model & model, common_download_opts opts) { - auto parts = common_download_get_all_parts(model.url); - std::vector tasks; - - // single-part: download straight to model.path if the user gave one (-m), else the cache default - if (parts.size() == 1) { - common_download_task task; - task.url = parts[0]; - task.local_path = model.path.empty() ? get_default_local_path(parts[0]) : model.path; - task.opts = opts; - tasks.push_back(std::move(task)); - return tasks; - } - - // multi-part: place each part under the user's -m directory (if given), else the cache default - std::string base_dir; - if (!model.path.empty()) { - auto pos = model.path.rfind('/'); - base_dir = pos == std::string::npos ? std::string(".") : model.path.substr(0, pos); - } - - for (const auto & part : parts) { - common_download_task task; - task.url = part; - task.opts = opts; - - std::string local = get_default_local_path(part); - if (!base_dir.empty()) { - auto pos = local.rfind('/'); - std::string name = pos == std::string::npos ? local : local.substr(pos + 1); - local = base_dir + "/" + name; - } - task.local_path = local; - tasks.push_back(std::move(task)); - } - return tasks; -} - -void common_models_handler_apply(common_models_handler & handler, common_params & params, common_download_callback * callback) { - std::vector tasks; - - auto & plan = handler.plan; - auto & plan_spec = handler.plan_spec; - auto & plan_voc = handler.plan_voc; - - auto opts = handler.opts; // copy - opts.callback = callback; - - // handle plain "url" if needed - auto handle_url = [&](common_params_model & model) { - if (!model.url.empty()) { - if (model.path.empty()) { - model.path = get_default_local_path(model.url); - } - } - }; - handle_url(params.model); - handle_url(params.mmproj); - handle_url(params.vocoder.model); - handle_url(params.speculative.draft.mparams); - - // optionally, if docker repo is set, resolve it - if (!params.model.docker_repo.empty()) { - params.model.url = common_docker_resolve_model(params.model.docker_repo); - params.model.path = get_default_local_path(params.model.url); - } - - // handle plain "url" tasks (non-hf) - if (!params.model.url.empty()) { - auto url_tasks = build_url_tasks(params.model, opts); - // the first part is what gets loaded, so point params.model.path at it - if (!url_tasks.empty()) { - std::string first_path = url_tasks.front().local_path; - url_tasks.front().on_done = [&, first_path]() { params.model.path = first_path; }; - } - for (auto & task : url_tasks) { - tasks.push_back(std::move(task)); - } - } - if (!params.mmproj.url.empty()) { - common_download_task task; - task.url = params.mmproj.url; - task.local_path = params.mmproj.path; - task.opts = opts; - tasks.push_back(task); - } - if (!params.vocoder.model.url.empty()) { - common_download_task task; - task.url = params.vocoder.model.url; - task.local_path = params.vocoder.model.path; - task.opts = opts; - tasks.push_back(task); - } - if (!params.speculative.draft.mparams.url.empty()) { - common_download_task task; - task.url = params.speculative.draft.mparams.url; - task.local_path = params.speculative.draft.mparams.path; - task.opts = opts; - tasks.push_back(task); - } - - // handle hf_plan tasks - auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files, - const hf_cache::hf_file & primary, - common_params_model & model) { - for (size_t i = 0; i < model_files.size(); ++i) { - auto & model_file = model_files[i]; - bool is_primary = (model_file.path == primary.path); - tasks.emplace_back(model_file, opts, [&, is_primary]() { - if (is_primary) { - // the primary file is the first split (00001-of), use it as model path - model.path = hf_cache::finalize_file(model_file); - } else { - hf_cache::finalize_file(model_file); - } - }); - } - }; - if (!plan.model_files.empty()) { - add_tasks(plan.model_files, plan.primary, params.model); - } - if (!plan.mmproj.local_path.empty()) { - tasks.emplace_back(plan.mmproj, opts, [&]() { - params.mmproj.path = hf_cache::finalize_file(plan.mmproj); - }); - } - if (!plan.mtp.local_path.empty()) { - tasks.emplace_back(plan.mtp, opts, [&]() { - // only fall back to the discovered MTP head when no draft was explicitly provided - if (params.speculative.draft.mparams.empty()) { - params.speculative.draft.mparams.path = hf_cache::finalize_file(plan.mtp); - } else { - hf_cache::finalize_file(plan.mtp); - } - }); - } - if (!plan.preset.local_path.empty()) { - tasks.emplace_back(plan.preset, opts, [&]() { - // if HF repo is a preset repo, we simply run server in router mode with the preset.ini file - params.models_preset_hf = params.model.hf_repo; // only for showing a warning - params.models_preset = hf_cache::finalize_file(plan.preset); - params.model = common_params_model{}; // make sure to clear model, so server starts in router mode - }); - } - - // handle plan_spec (e.g. --spec-draft-hf) - if (!plan_spec.model_files.empty()) { - add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams); - } - - // handle vocoder plan (e.g. --hf-repo-v) - if (!plan_voc.model_files.empty()) { - add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model); - } - - // run all tasks in parallel - if (!params.offline) { - // if duplicated files are found, only download once (but still call on_done for each task) - std::unordered_map unique_tasks; - for (auto & task : tasks) { - auto it = unique_tasks.find(task.local_path); - if (it == unique_tasks.end()) { - unique_tasks[task.local_path] = &task; - } - } - std::vector unique_tasks_vec; - for (auto & pair : unique_tasks) { - unique_tasks_vec.push_back(*pair.second); - } - common_download_run_tasks(unique_tasks_vec); - } - - // download successful, update params with the downloaded paths - for (const auto & task : tasks) { - if (task.on_done) { - task.on_done(); - } - } -} - // // CLI argument parsing functions // +bool common_params_handle_models(common_params & params, llama_example curr_ex) { + const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), + params.speculative.types.end(), + COMMON_SPECULATIVE_TYPE_DRAFT_MTP) != params.speculative.types.end(); + + common_download_opts opts; + opts.bearer_token = params.hf_token; + opts.offline = params.offline; + opts.skip_download = params.skip_download; + opts.download_mtp = spec_type_draft_mtp; + opts.download_mmproj = !params.no_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty(); + + // sub-models (draft, mmproj, vocoder) are explicitly specified by the user, + // so we should not auto-discover mtp/mmproj siblings for them + common_download_opts sub_opts = opts; + sub_opts.download_mtp = false; + sub_opts.download_mmproj = false; + + try { + auto res = common_params_handle_model(params.model, opts); + if (params.no_mmproj) { + params.mmproj = {}; + } else if (res.found_mmproj && params.mmproj.path.empty() && params.mmproj.url.empty()) { + // optionally, handle mmproj model when -hf is specified + params.mmproj = res.mmproj; + } + // only download mmproj if the current example is using it + for (const auto & ex : mmproj_examples) { + if (curr_ex == ex) { + common_params_handle_model(params.mmproj, sub_opts); + break; + } + } + + // when --spec-type mtp is set and no draft model was provided explicitly, + // fall back to the MTP head discovered alongside the -hf model + if (spec_type_draft_mtp && res.found_mtp && + params.speculative.draft.mparams.path.empty() && + params.speculative.draft.mparams.hf_repo.empty() && + params.speculative.draft.mparams.url.empty()) { + params.speculative.draft.mparams.path = res.mtp.path; + } + common_params_handle_model(params.speculative.draft.mparams, sub_opts); + common_params_handle_model(params.vocoder.model, sub_opts); + return true; + } catch (const common_skip_download_exception &) { + return false; + } catch (const std::exception &) { + throw; + } +} + static bool common_params_parse_ex(int argc, char ** argv, common_params_context & ctx_arg) { common_params & params = ctx_arg.params; @@ -694,6 +602,30 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context // parse the first time to get -hf option (used for remote preset) parse_cli_args(); + // export_graph_ops loads only metadata + const bool skip_model_download = ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS; + + // maybe handle remote preset + if (!params.model.hf_repo.empty() && !skip_model_download) { + std::string cli_hf_repo = params.model.hf_repo; + bool has_preset = common_params_handle_remote_preset(params, ctx_arg.ex); + + // special case: if hf_repo explicitly set by preset, we need to preserve it (ignore CLI value) + // this is useful when we have one HF repo pointing to other HF repos (one model - multiple GGUFs) + std::string preset_hf_repo = params.model.hf_repo; + bool preset_has_hf_repo = preset_hf_repo != cli_hf_repo; + + if (has_preset) { + // re-parse CLI args to override preset values + parse_cli_args(); + } + + // preserve hf_repo from preset if needed + if (preset_has_hf_repo) { + params.model.hf_repo = preset_hf_repo; + } + } + postprocess_cpu_params(params.cpuparams, nullptr); postprocess_cpu_params(params.cpuparams_batch, ¶ms.cpuparams); @@ -704,26 +636,15 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context throw std::invalid_argument("error: --prompt-cache-all not supported in interactive mode yet\n"); } - const bool skip_model_download = - // server will call common_params_handle_models() later, so we skip it here - ctx_arg.ex == LLAMA_EXAMPLE_SERVER || - // download calls common_params_handle_models() itself and prints the paths - ctx_arg.ex == LLAMA_EXAMPLE_DOWNLOAD || - // export_graph_ops loads only metadata - ctx_arg.ex == LLAMA_EXAMPLE_EXPORT_GRAPH_OPS; - + // handle model and download if (!skip_model_download) { - // handle model and download - common_models_handler handler = common_models_handler_init(params, ctx_arg.ex); - common_models_handler_apply(handler, params); + common_params_handle_models(params, ctx_arg.ex); + } - // model is required (except for server) - // TODO @ngxson : maybe show a list of available models in CLI in this case - if (params.model.path.empty() - && !params.usage - && !params.completion) { - throw std::invalid_argument("error: --model is required\n"); - } + // model is required (except for server) + // TODO @ngxson : maybe show a list of available models in CLI in this case + if (params.model.path.empty() && ctx_arg.ex != LLAMA_EXAMPLE_SERVER && !skip_model_download && !params.usage && !params.completion) { + throw std::invalid_argument("error: --model is required\n"); } if (params.escape) { @@ -787,19 +708,15 @@ static void common_params_print_usage(common_params_context & ctx_arg) { common_options.push_back(&opt); } } - bool first = true; - auto print_section = [&](const char * header, std::vector & options) { - if (options.empty()) { - return; - } - printf("%s----- %s -----\n\n", first ? "" : "\n\n", header); - first = false; - print_options(options); - }; - print_section("common params", common_options); - print_section("sampling params", sampling_options); - print_section("speculative params", spec_options); - print_section("example-specific params", specific_options); + printf("----- common params -----\n\n"); + print_options(common_options); + printf("\n\n----- sampling params -----\n\n"); + print_options(sampling_options); + printf("\n\n----- speculative params -----\n\n"); + print_options(spec_options); + // TODO: maybe convert enum llama_example to string + printf("\n\n----- example-specific params -----\n\n"); + print_options(specific_options); } static void common_params_print_completion(common_params_context & ctx_arg) { @@ -1021,44 +938,7 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map buf; - std::vector ptrs; -}; - -static utf8_argv make_utf8_argv() { - utf8_argv out; - int wargc = 0; - LPWSTR* wargv = CommandLineToArgvW(GetCommandLineW(), &wargc); - if (!wargv) return out; - - out.buf.reserve(wargc); - for (int i = 0; i < wargc; ++i) { - int n = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, wargv[i], -1, nullptr, 0, nullptr, nullptr); - if (n <= 0) { out.buf.emplace_back(); continue; } - auto& s = out.buf.emplace_back(); - s.resize(static_cast(n - 1)); - (void)WideCharToMultiByte(CP_UTF8, 0, wargv[i], -1, s.data(), n, nullptr, nullptr); - } - LocalFree(wargv); - - out.ptrs.reserve(out.buf.size() + 1); - for (auto& s : out.buf) out.ptrs.push_back(s.data()); - out.ptrs.push_back(nullptr); - return out; -} -#endif - bool common_params_parse(int argc, char ** argv, common_params & params, llama_example ex, void(*print_usage)(int, char **)) { -#ifdef _WIN32 - auto utf8 = make_utf8_argv(); - // repair argv only when it matches the process command line - if (static_cast(utf8.buf.size()) == argc) { - argv = utf8.ptrs.data(); - } -#endif - auto ctx_arg = common_params_parser_init(params, ex, print_usage); const common_params params_org = ctx_arg.params; // the example can modify the default params @@ -1199,9 +1079,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex * - if both {LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_*,} are set, we will prioritize the LLAMA_EXAMPLE_* matching current example */ auto add_opt = [&](common_arg arg) { - // download only exposes the handful of args explicitly tagged for it - const bool inherit_common = ex != LLAMA_EXAMPLE_DOWNLOAD; - if ((arg.in_example(ex) || (inherit_common && arg.in_example(LLAMA_EXAMPLE_COMMON))) && !arg.is_exclude(ex)) { + if ((arg.in_example(ex) || arg.in_example(LLAMA_EXAMPLE_COMMON)) && !arg.is_exclude(ex)) { ctx_arg.options.push_back(std::move(arg)); } }; @@ -1212,7 +1090,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.usage = true; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD})); + )); add_opt(common_arg( {"--version"}, "show version and build info", @@ -2334,7 +2212,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, bool value) { params.no_mmproj = !value; } - ).set_examples({LLAMA_EXAMPLE_MTMD, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MMPROJ_AUTO")); + ).set_examples(mmproj_examples).set_env("LLAMA_ARG_MMPROJ_AUTO")); add_opt(common_arg( {"--mmproj-offload"}, {"--no-mmproj-offload"}, @@ -2733,14 +2611,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.path = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL")); + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA}).set_env("LLAMA_ARG_MODEL")); add_opt(common_arg( {"-mu", "--model-url"}, "MODEL_URL", "model download url (default: unused)", [](common_params & params, const std::string & value) { params.model.url = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MODEL_URL")); + ).set_env("LLAMA_ARG_MODEL_URL")); add_opt(common_arg( { "-dr", "--docker-repo" }, "[/][:quant]", "Docker Hub model repository. repo is optional, default to ai/. quant is optional, default to :latest.\n" @@ -2749,7 +2627,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.docker_repo = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_DOCKER_REPO")); + ).set_env("LLAMA_ARG_DOCKER_REPO")); add_opt(common_arg( {"-hf", "-hfr", "--hf-repo"}, "/[:quant]", "Hugging Face model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n" @@ -2759,14 +2637,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.model.hf_repo = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_REPO")); + ).set_env("LLAMA_ARG_HF_REPO")); add_opt(common_arg( {"-hff", "--hf-file"}, "FILE", "Hugging Face model file. If specified, it will override the quant in --hf-repo (default: unused)", [](common_params & params, const std::string & value) { params.model.hf_file = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_HF_FILE")); + ).set_env("LLAMA_ARG_HF_FILE")); add_opt(common_arg( {"-hfv", "-hfrv", "--hf-repo-v"}, "/[:quant]", "Hugging Face model repository for the vocoder model (default: unused)", @@ -2787,14 +2665,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params, const std::string & value) { params.hf_token = value; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("HF_TOKEN")); - add_opt(common_arg( - {"--mtp"}, - "also download the multi-token prediction (MTP) head, if available (default: unused)", - [](common_params & params) { - params.speculative.types.push_back(COMMON_SPECULATIVE_TYPE_DRAFT_MTP); - } - ).set_examples({LLAMA_EXAMPLE_DOWNLOAD})); + ).set_env("HF_TOKEN")); add_opt(common_arg( {"--context-file"}, "FNAME", "file to load context from (use comma-separated values to specify multiple files)", @@ -3004,26 +2875,62 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.api_prefix = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_API_PREFIX")); + // Deprecated: use --ui-config instead (kept for backward compat) add_opt(common_arg( - {"--ui-config", "--webui-config"}, "JSON", + {"--webui-config"}, "JSON", + "[DEPRECATED: use --ui-config] JSON that provides default WebUI settings (overrides WebUI defaults)", + [](common_params & params, const std::string & value) { + params.ui_config_json = value; + params.webui_config_json = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG")); + + add_opt(common_arg( + {"--ui-config"}, "JSON", "JSON that provides default UI settings (overrides UI defaults)", [](common_params & params, const std::string & value) { params.ui_config_json = value; + params.webui_config_json = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG")); + + // Deprecated: use --ui-config-file instead (kept for backward compat) add_opt(common_arg( - {"--ui-config-file", "--webui-config-file"}, "PATH", + {"--webui-config-file"}, "PATH", + "[DEPRECATED: use --ui-config-file] JSON file that provides default WebUI settings (overrides WebUI defaults)", + [](common_params & params, const std::string & value) { + params.ui_config_json = read_file(value); + params.webui_config_json = params.ui_config_json; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_CONFIG_FILE")); + + add_opt(common_arg( + {"--ui-config-file"}, "PATH", "JSON file that provides default UI settings (overrides UI defaults)", [](common_params & params, const std::string & value) { params.ui_config_json = read_file(value); + params.webui_config_json = params.ui_config_json; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_CONFIG_FILE")); + + // Deprecated: use --ui-mcp-proxy instead (kept for backward compat) add_opt(common_arg( - {"--ui-mcp-proxy", "--webui-mcp-proxy"}, - {"--no-ui-mcp-proxy", "--no-webui-mcp-proxy"}, + {"--webui-mcp-proxy"}, + {"--no-webui-mcp-proxy"}, + "[DEPRECATED: use --ui-mcp-proxy/--no-ui-mcp-proxy] experimental: whether to enable MCP CORS proxy", + [](common_params & params, bool value) { + params.ui_mcp_proxy = value; + params.webui_mcp_proxy = value; + } + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI_MCP_PROXY")); + + add_opt(common_arg( + {"--ui-mcp-proxy"}, + {"--no-ui-mcp-proxy"}, "experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)", [](common_params & params, bool value) { params.ui_mcp_proxy = value; + params.webui_mcp_proxy = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI_MCP_PROXY")); add_opt(common_arg( @@ -3035,26 +2942,24 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.server_tools = parse_csv_row(value); } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_TOOLS")); + // Deprecated: use --ui/--no-ui instead (kept for backward compat) add_opt(common_arg( - {"-ag", "--agent"}, - {"-no-ag", "--no-agent"}, - "whether to enable CORS proxy and all built-in tools - do not enable in untrusted environments (default: disabled)", + {"--webui"}, + {"--no-webui"}, + "[DEPRECATED: use --ui/--no-ui] whether to enable the Web UI", [](common_params & params, bool value) { - if (value) { - params.server_tools = {"all"}; - params.ui_mcp_proxy = true; - } else { - params.server_tools.clear(); - params.ui_mcp_proxy = false; - } + params.ui = value; + params.webui = value; } - ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_AGENT")); + ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_WEBUI")); + add_opt(common_arg( - {"--ui", "--webui"}, - {"--no-ui", "--no-webui"}, + {"--ui"}, + {"--no-ui"}, string_format("whether to enable the Web UI (default: %s)", params.ui ? "enabled" : "disabled"), [](common_params & params, bool value) { params.ui = value; + params.webui = value; } ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_UI")); add_opt(common_arg( @@ -3085,7 +2990,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_API_KEY")); add_opt(common_arg( {"--api-key-file"}, "FNAME", - "path to file containing API keys, one per line; lines starting with a hash are treated as comments (default: none)", + "path to file containing API keys (default: none)", [](common_params & params, const std::string & value) { std::ifstream key_file(value); if (!key_file) { @@ -3093,7 +2998,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex } std::string key; while (std::getline(key_file, key)) { - if (!key.empty() && key[0] != '#') { + if (!key.empty()) { params.api_keys.push_back(key); } } @@ -3299,20 +3204,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.reasoning_budget_message = value; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_THINK_BUDGET_MESSAGE")); - add_opt(common_arg( - {"--reasoning-preserve"}, - {"--no-reasoning-preserve"}, - "preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n" - "compatible with certain templates having 'supports_preserve_reasoning' capability\n" - "example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking", - [](common_params & params, bool value) { - if (value) { - params.default_template_kwargs["preserve_reasoning"] = "true"; - } else { - params.default_template_kwargs["preserve_reasoning"] = "false"; - } - } - ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE")); add_opt(common_arg( {"--chat-template"}, "JINJA_TEMPLATE", string_format( @@ -3488,7 +3379,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.offline = true; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE")); + ).set_env("LLAMA_ARG_OFFLINE")); add_opt(common_arg( {"-lv", "--verbosity", "--log-verbosity"}, "N", string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n" @@ -3765,7 +3656,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex "draft model for speculative decoding (default: unused)", [](common_params & params, const std::string & value) { params.speculative.draft.mparams.path = value; - params.speculative.draft.mparams.hf_file = value; // will be used if --spec-draft-hf is set } ).set_spec().set_examples({LLAMA_EXAMPLE_SPECULATIVE, LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_SPEC_DRAFT_MODEL")); add_opt(common_arg( diff --git a/common/arg.h b/common/arg.h index 54a38b9cc..0010f2a9a 100644 --- a/common/arg.h +++ b/common/arg.h @@ -1,14 +1,12 @@ #pragma once #include "common.h" -#include "download.h" #include #include #include #include #include -#include // pseudo-env variable to identify preset-only arguments #define COMMON_ARG_PRESET_LOAD_ON_STARTUP "__PRESET_LOAD_ON_STARTUP" @@ -131,21 +129,11 @@ bool common_params_to_map(int argc, char ** argv, llama_example ex, std::map & args); -struct common_models_handler { - common_download_hf_plan plan; - common_download_hf_plan plan_spec; - common_download_hf_plan plan_voc; - common_download_opts opts; -}; - -// initialize downloading opts and hf_plan if needed, but does not download anything yet -common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex); - -// check if the model is a preset repo (i.e. has a preset file) -bool common_models_handler_is_preset_repo(const common_models_handler & handler); - -// download and update params with the downloaded model path -void common_models_handler_apply(common_models_handler & handler, common_params & params, common_download_callback * callback = nullptr); +// populate model paths (main model, mmproj, etc) from -hf if necessary +// return true if the model is ready to use +// throw an exception if there is an error that prevents the model from being used (e.g. network error, model not found, etc) +// if params.skip_download is true, no downloads will be attempted. return false if the model is invalid or missing (e.g. ETag check failed) +bool common_params_handle_models(common_params & params, llama_example curr_ex); // initialize argument parser context - used by test-arg-parser and preset common_params_context common_params_parser_init(common_params & params, llama_example ex, void(*print_usage)(int, char **) = nullptr); diff --git a/common/chat-auto-parser-generator.cpp b/common/chat-auto-parser-generator.cpp index 36aab7ecb..37ca55c8d 100644 --- a/common/chat-auto-parser-generator.cpp +++ b/common/chat-auto-parser-generator.cpp @@ -395,11 +395,10 @@ common_peg_parser analyze_tools::build_tool_parser_tag_tagged(parser_build_conte arguments.name_suffix) + arguments.value_prefix + (schema_info.resolves_to_string(param_schema) ? - p.ac(p.tool_arg_string_value(until_suffix) + - p.tool_arg_close(p.literal(arguments.value_suffix)), arguments.value_suffix) : - (p.tool_arg_json_value(p.schema( - p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false)) + - p.tool_arg_close(p.literal(arguments.value_suffix))))); + p.tool_arg_string_value(until_suffix) : + p.tool_arg_json_value(p.schema( + p.json(), "tool-" + name + "-arg-" + param_name + "-schema", param_schema, false))) + + p.tool_arg_close(p.literal(arguments.value_suffix))); auto named_arg = p.rule("tool-" + name + "-arg-" + param_name, arg); if (is_required) { diff --git a/common/chat.cpp b/common/chat.cpp index 28fba2ff7..74cfe9302 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -7,6 +7,8 @@ #include "ggml.h" #include "json-schema-to-grammar.h" #include "log.h" +#include "json-partial.cpp" +#include "regex-partial.cpp" #include "reasoning-budget.h" #include "chat-auto-parser-generator.cpp" #include "chat-auto-parser-helpers.cpp" @@ -99,93 +101,41 @@ std::string common_chat_msg::render_content(const std::string & delimiter) const return text; } -common_chat_role common_chat_role_from_string(const std::string & role) { - if (role == "system") { return COMMON_CHAT_ROLE_SYSTEM; } - if (role == "assistant") { return COMMON_CHAT_ROLE_ASSISTANT; } - if (role == "user") { return COMMON_CHAT_ROLE_USER; } - if (role == "tool") { return COMMON_CHAT_ROLE_TOOL; } - return COMMON_CHAT_ROLE_UNKNOWN; -} - -const char * common_chat_role_to_string(common_chat_role role) { - switch (role) { - case COMMON_CHAT_ROLE_SYSTEM: return "system"; - case COMMON_CHAT_ROLE_ASSISTANT: return "assistant"; - case COMMON_CHAT_ROLE_USER: return "user"; - case COMMON_CHAT_ROLE_TOOL: return "tool"; - case COMMON_CHAT_ROLE_UNKNOWN: return ""; - } - return ""; -} - -json common_chat_msg_delimiters::to_json() const { - json result = json::array(); - for (const auto & d : delimiters) { - result.push_back({ - { "role", common_chat_role_to_string(d.role) }, - { "delimiter", d.delimiter }, - }); - } - return result; -} - -common_chat_msg_delimiters common_chat_msg_delimiters_parse(const json & delimiters) { - common_chat_msg_delimiters result; - - if (!delimiters.is_array()) { - return result; +std::vector common_chat_split_by_role(const std::string & prompt, const std::vector & delims) { + if (delims.empty() || prompt.empty()) { + return {}; } - result.delimiters.reserve(delimiters.size()); - for (const auto & d : delimiters) { - if (!d.is_object()) { - continue; + auto parser = build_peg_parser([&](common_peg_parser_builder & p) { + std::vector all_delims; + std::vector tagged_messages; + + all_delims.reserve(delims.size()); + tagged_messages.reserve(delims.size()); + for (const auto & d : delims) { + all_delims.push_back(d.delimiter); } - result.delimiters.push_back({ - common_chat_role_from_string(d.value("role", std::string())), - d.value("delimiter", std::string()), - }); - } - return result; -} - -void common_chat_msg_delimiters::tokenize(const llama_vocab * vocab) { - for (auto & d : delimiters) { - d.tokens = common_tokenize(vocab, d.delimiter, false, true); - } -} - -common_chat_msg_spans common_chat_msg_delimiters::split(const llama_tokens & tokens, const std::map & skips) const { - std::vector> matches; - - auto skip = skips.begin(); - for (size_t i = 0; i < tokens.size();) { - if (skip != skips.end() && i == skip->first) { - i += skip->second; - ++skip; - continue; + auto any_delim = p.until_one_of(all_delims); + for (const auto & d : delims) { + tagged_messages.push_back(p.tag(d.role, p.literal(d.delimiter) + any_delim)); } - for (const auto & d : delimiters) { - if (i + d.tokens.size() > tokens.size()) { - continue; - } - if (std::equal(d.tokens.begin(), d.tokens.end(), tokens.begin() + i)) { - matches.emplace_back(d.role, i); - break; - } + + return any_delim + p.zero_or_more(p.choice(tagged_messages)) + p.end(); + }); + + common_peg_parse_context ctx(prompt); + const auto result = parser.parse(ctx); + if (!result.success()) { + return {}; + } + + std::vector spans; + ctx.ast.visit(result, [&](const common_peg_ast_node & node) { + if (!node.tag.empty()) { + spans.push_back({ node.tag, node.start, node.end - node.start }); } - i++; - } - - matches.emplace_back(COMMON_CHAT_ROLE_UNKNOWN, tokens.size()); - - common_chat_msg_spans spans; - for (size_t i = 0; i + 1 < matches.size(); i++) { - const auto & curr = matches[i]; - const auto & next = matches[i + 1]; - spans.add(curr.first, curr.second, next.second - curr.second); - } + }); return spans; } @@ -925,10 +875,6 @@ static std::string common_chat_template_direct_apply_impl( if (inputs.add_generation_prompt) { inp["add_generation_prompt"] = true; } - if (inp.contains("preserve_reasoning") && inp["preserve_reasoning"].is_boolean()) { - bool enabled = inp["preserve_reasoning"].get(); - jinja::caps_apply_preserve_reasoning(ctx, enabled); - } jinja::global_from_json(ctx, inp, inputs.mark_input); @@ -1150,13 +1096,13 @@ static common_chat_params common_chat_params_init_gpt_oss(const common_chat_temp data.prompt = prompt; data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs, /* messages_override= */ adjusted_messages); - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|start|>assistant" }, - { COMMON_CHAT_ROLE_USER, "<|start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>developer" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|start|>system" }, - { COMMON_CHAT_ROLE_TOOL, "<|start|>functions" }, - }; + data.message_spans = common_chat_split_by_role(prompt, { + { "assistant", "<|start|>assistant" }, + { "user", "<|start|>user" }, + { "system", "<|start|>developer" }, + { "system", "<|start|>system" }, + { "tool", "<|start|>functions" }, + }); data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; data.supports_thinking = true; @@ -1297,10 +1243,10 @@ static common_chat_params common_chat_params_init_gemma4(const common_chat_templ data.prompt += data.generation_prompt; } - data.message_delimiters = { - { COMMON_CHAT_ROLE_USER, "<|turn>user" }, - { COMMON_CHAT_ROLE_ASSISTANT, "<|turn>model" }, - }; + data.message_spans = common_chat_split_by_role(data.prompt, { + { "user", "<|turn>user\n" }, + { "assistant", "<|turn>model\n" }, + }); data.format = COMMON_CHAT_FORMAT_PEG_GEMMA4; data.supports_thinking = true; @@ -2099,15 +2045,15 @@ static common_chat_params common_chat_params_init_cohere2moe(const common_chat_t RESULT_START, RESULT_END, }; - // Declare per-role message delimiters. Tool results are rendered with the + // Split the rendered prompt into per-role message spans. Tool results are rendered with the // system token followed by <|START_TOOL_RESULT|>, so the "tool" delimiter must be listed before // the plain "system" one (it is a strict superset, and the role split tries delimiters in order). - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, GEN_PREFIX }, - { COMMON_CHAT_ROLE_USER, TURN_START + USER }, - { COMMON_CHAT_ROLE_TOOL, TURN_START + SYSTEM + RESULT_START }, - { COMMON_CHAT_ROLE_SYSTEM, TURN_START + SYSTEM }, - }; + data.message_spans = common_chat_split_by_role(data.prompt, { + { "assistant", GEN_PREFIX }, + { "user", TURN_START + USER }, + { "tool", TURN_START + SYSTEM + RESULT_START }, + { "system", TURN_START + SYSTEM }, + }); auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; @@ -2391,166 +2337,6 @@ static void func_args_not_string(json & messages) { } } -// Trim leading/trailing whitespace from message contents before rendering. This -// has to run on the messages (not on the rendered JSON) because templates with -// string-only content caps concatenate typed content parts into a single string -// during rendering, after which the per-part whitespace can no longer be reached. -// Both the plain string content and the text of typed content parts are trimmed. -static void trim_all_content(std::vector & messages) { - for (auto & message : messages) { - message.content = trim_whitespace(message.content); - message.reasoning_content = trim_whitespace(message.reasoning_content); - for (auto & part : message.content_parts) { - if (part.type == "text") { - part.text = trim_whitespace(part.text); - } - } - } -} - -} - -// MiniCPM5 format: -// - Reasoning: {reasoning} (optional) -// - Tool calls: value -static common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { - "", - "", - "", - "", - }; - - data.thinking_start_tag = ""; - data.thinking_end_tag = ""; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, - { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|im_start|>assistant\n\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "\n\n\n" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal("<|im_start|>assistant\n"); - - auto reasoning = p.eps(); - if (extract_reasoning) { - reasoning = ("" << p.reasoning(p.until("")) << "") + p.space(); - } - - // Response format parser - if (has_response_format) { - return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // CDATA lets a value carry characters that would otherwise close the tag (e.g. - // ); capture the inner text only, excluding the CDATA markers. - auto string_value = p.choice({ - p.literal("")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("")), - p.negate(p.literal("")) + p.tool_arg_close(p.literal("")), "") - }); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) - ) + p.tool_arg_close(p.literal("")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("")) + - value_parser - ); - - arg_choice |= arg_rule; - } - args = p.zero_or_more(arg_choice + p.space()); - } - - auto tool_parser = p.tool( - p.tool_open(p.literal("")) - << p.tool_args(args) - << p.tool_close(p.literal(""))); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); - - auto content = p.content(p.until(" common_chat_try_specialized_template( return common_chat_params_init_gemma4(tmpl, params); } - // MiniCPM5 - XML tool calls with ... - if (src.find("Tool usage guidelines:") != std::string::npos && - src.find("template_tool_use ? *tmpls->template_tool_use : *tmpls->template_default; const auto & src = tmpl.source(); const auto & caps = tmpl.original_caps(); - std::vector trimmed_messages; - const std::vector * messages_to_render = &inputs.messages; - if (src.find("You have access to the following functions in JSONSchema format") != std::string::npos) { - // StepFun: trim message contents (including typed content parts) before rendering, - // otherwise leftover whitespace drives the model into reasoning loops (issue #24181) - trimmed_messages = inputs.messages; - workaround::trim_all_content(trimmed_messages); - messages_to_render = &trimmed_messages; - } - params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps()); + params.messages = render_message_to_json(inputs.messages, tmpl.original_caps()); params.tool_choice = inputs.tool_choice; params.reasoning_format = inputs.reasoning_format; params.enable_thinking = inputs.enable_thinking; @@ -2772,15 +2541,17 @@ static common_chat_params common_chat_templates_apply_jinja(const struct common_ autoparser.analyze_template(tmpl); auto auto_params = autoparser::peg_generator::generate_parser(tmpl, params, autoparser); - common_chat_msg_delimiters delimiters; + std::vector delimiters; if (!autoparser.assistant_start.empty()) { - delimiters.add(COMMON_CHAT_ROLE_ASSISTANT, autoparser.assistant_start); + delimiters.push_back({ "assistant", autoparser.assistant_start }); } if (!autoparser.user_start.empty()) { - delimiters.add(COMMON_CHAT_ROLE_USER, autoparser.user_start); + delimiters.push_back({ "user", autoparser.user_start }); } - auto_params.message_delimiters = std::move(delimiters); + if (!delimiters.empty()) { + auto_params.message_spans = common_chat_split_by_role(auto_params.prompt, delimiters); + } auto_params.supports_thinking = autoparser.reasoning.mode != autoparser::reasoning_mode::NONE; if (auto_params.supports_thinking) { @@ -2952,9 +2723,5 @@ common_chat_msg common_chat_peg_parse(const common_peg_arena & src_pars std::map common_chat_templates_get_caps(const common_chat_templates * chat_templates) { GGML_ASSERT(chat_templates != nullptr); GGML_ASSERT(chat_templates->template_default != nullptr); - if (chat_templates->template_tool_use != nullptr) { - // take the more expressive template when available - return chat_templates->template_tool_use->caps.to_map(); - } return chat_templates->template_default->caps.to_map(); } diff --git a/common/chat.h b/common/chat.h index 7898f1623..5659cd42a 100644 --- a/common/chat.h +++ b/common/chat.h @@ -143,75 +143,15 @@ struct common_chat_msg_diff { } }; -enum common_chat_role { - COMMON_CHAT_ROLE_UNKNOWN, - COMMON_CHAT_ROLE_SYSTEM, - COMMON_CHAT_ROLE_ASSISTANT, - COMMON_CHAT_ROLE_USER, - COMMON_CHAT_ROLE_TOOL -}; - -common_chat_role common_chat_role_from_string(const std::string & role); -const char * common_chat_role_to_string(common_chat_role role); - struct common_chat_msg_span { - common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; + std::string role; std::size_t pos = 0; std::size_t len = 0; - - bool valid() const { - return role != COMMON_CHAT_ROLE_UNKNOWN; - } -}; - -struct common_chat_msg_spans { - std::vector spans; - - void add(common_chat_role role, size_t pos, size_t len) { - spans.push_back({ role, pos, len }); - } - - bool is_user_start(int32_t pos) const { - for (auto it = spans.begin(); it != spans.end(); ++it) { - if (it->role == COMMON_CHAT_ROLE_USER && pos == (int32_t) it->pos) { - return true; - } - } - return false; - } - - int32_t last_user_message_pos() const { - for (auto it = spans.rbegin(); it != spans.rend(); ++it) { - if (it->role == COMMON_CHAT_ROLE_USER) { - return (int32_t) it->pos; - } - } - return -1; - } }; struct common_chat_msg_delimiter { - common_chat_role role = COMMON_CHAT_ROLE_UNKNOWN; - std::string delimiter; - llama_tokens tokens = {}; -}; - -struct common_chat_msg_delimiters { - std::vector delimiters; - - common_chat_msg_delimiters() = default; - common_chat_msg_delimiters(std::initializer_list delims) : delimiters(delims) {} - - void add(common_chat_role role, const std::string & delimiter) { - delimiters.push_back({ role, delimiter }); - } - - void tokenize(const llama_vocab * vocab); - - // split tokens into message spans. skips maps a start index to a length of a region to jump over without matching - common_chat_msg_spans split(const llama_tokens & tokens, const std::map & skips = {}) const; - - nlohmann::ordered_json to_json() const; + std::string role; + std::string delimiter; }; struct common_chat_tool { @@ -279,7 +219,7 @@ struct common_chat_params { std::vector preserved_tokens; std::vector additional_stops; std::string parser; - common_chat_msg_delimiters message_delimiters; + std::vector message_spans; }; // per-message parsing syntax @@ -385,4 +325,5 @@ struct common_chat_prompt_preset { common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates); -common_chat_msg_delimiters common_chat_msg_delimiters_parse(const nlohmann::ordered_json & delimiters); +std::vector common_chat_split_by_role(const std::string & prompt, const std::vector & delims); + diff --git a/common/common.cpp b/common/common.cpp index 14f1a7892..9fd0e9729 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -231,7 +231,7 @@ bool set_process_priority(enum ggml_sched_priority prio) { } if (!SetPriorityClass(GetCurrentProcess(), p)) { - COM_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError()); + LOG_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError()); return false; } @@ -257,7 +257,7 @@ bool set_process_priority(enum ggml_sched_priority prio) { } if (setpriority(PRIO_PROCESS, 0, p) != 0) { - COM_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno); + LOG_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno); return false; } return true; @@ -290,14 +290,14 @@ void postprocess_cpu_params(common_cpu_params & cpuparams, const common_cpu_para if (n_set && n_set < cpuparams.n_threads) { // Not enough set bits, may experience performance issues. - COM_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads); + LOG_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads); } } bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) { size_t dash_loc = range.find('-'); if (dash_loc == std::string::npos) { - COM_ERR("%s", "Format of CPU range is invalid! Expected []-[].\n"); + LOG_ERR("Format of CPU range is invalid! Expected []-[].\n"); return false; } @@ -309,7 +309,7 @@ bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THRE } else { start_i = std::stoull(range.substr(0, dash_loc)); if (start_i >= GGML_MAX_N_THREADS) { - COM_ERR("%s", "Start index out of bounds!\n"); + LOG_ERR("Start index out of bounds!\n"); return false; } } @@ -319,7 +319,7 @@ bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THRE } else { end_i = std::stoull(range.substr(dash_loc + 1)); if (end_i >= GGML_MAX_N_THREADS) { - COM_ERR("%s", "End index out of bounds!\n"); + LOG_ERR("End index out of bounds!\n"); return false; } } @@ -339,7 +339,7 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD } size_t num_digits = mask.length() - start_i; - num_digits = std::min(num_digits, 128); + if (num_digits > 128) num_digits = 128; size_t end_i = num_digits + start_i; @@ -354,7 +354,7 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD } else if (c >= 'A' && c <= 'F') { id -= 'A' - 10; } else { - COM_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i)); + LOG_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i)); return false; } @@ -385,21 +385,21 @@ void common_params_print_info(const common_params & params, bool print_devices) #else const char * build_type = " (debug)"; #endif - COM_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type); + LOG_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type); - COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, common_log_get_verbosity_thold()); + LOG_INF("log_info: verbosity = %d (adjust with the `-lv N` CLI arg)\n", common_log_get_verbosity_thold()); // device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device if (print_devices) { - COM_TRC("%s", "device_info:\n"); + LOG_INF("device_info:\n"); for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { auto * dev = ggml_backend_dev_get(i); size_t free, total; ggml_backend_dev_memory(dev, &free, &total); - COM_TRC(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); + LOG_INF(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); } } - COM_TRC("%s\n", common_params_get_system_info(params).c_str()); + LOG_INF("%s\n", common_params_get_system_info(params).c_str()); } std::string common_params_get_system_info(const common_params & params) { @@ -666,7 +666,7 @@ void string_process_escapes(std::string & input) { bool string_parse_kv_override(const char * data, std::vector & overrides) { const char * sep = strchr(data, '='); if (sep == nullptr || sep - data >= 128) { - COM_ERR("%s: malformed KV override '%s'\n", __func__, data); + LOG_ERR("%s: malformed KV override '%s'\n", __func__, data); return false; } llama_model_kv_override kvo; @@ -689,20 +689,20 @@ bool string_parse_kv_override(const char * data, std::vector 127) { - COM_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); + LOG_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); return false; } strncpy(kvo.val_str, sep, 127); kvo.val_str[127] = '\0'; } else { - COM_ERR("%s: invalid type for KV override '%s'\n", __func__, data); + LOG_ERR("%s: invalid type for KV override '%s'\n", __func__, data); return false; } overrides.emplace_back(std::move(kvo)); @@ -1080,18 +1080,6 @@ std::vector fs_list(const std::string & path, bool include_dir return files; } -std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode) { -#ifdef _WIN32 - int wlen = MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, NULL, 0); - if (!wlen) { return std::ifstream(); } - std::vector wfname(wlen); - (void)MultiByteToWideChar(CP_UTF8, 0, fname.c_str(), -1, wfname.data(), wlen); - return std::ifstream(wfname.data(), mode); -#else - return std::ifstream(fname, mode); -#endif -} - // // TTY utils // @@ -1205,8 +1193,8 @@ common_init_result::common_init_result(common_params & params, bool model_only) auto cparams = common_context_params_to_llama(params); if (params.fit_params) { - COM_TRC("%s", "fitting params to device memory ...\n"); - COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n"); + LOG_INF("%s: fitting params to device memory ...\n", __func__); + LOG_INF("%s: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n", __func__); common_fit_params(params.model.path.c_str(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), @@ -1233,7 +1221,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) llama_adapter_lora_ptr lora; lora.reset(llama_adapter_lora_init(model, la.path.c_str())); if (lora == nullptr) { - COM_ERR("failed to load lora adapter '%s'\n", la.path.c_str()); + LOG_ERR("%s: failed to load lora adapter '%s'\n", __func__, la.path.c_str()); pimpl->model.reset(model); return; } @@ -1252,14 +1240,14 @@ common_init_result::common_init_result(common_params & params, bool model_only) common_init_sampler_from_model(model, params.sampling); if (params.sampling.ignore_eos && llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) { - COM_WRN("%s", "vocab does not have an EOS token, ignoring --ignore-eos\n"); + LOG_WRN("%s: warning: vocab does not have an EOS token, ignoring --ignore-eos\n", __func__); params.sampling.ignore_eos = false; } // initialize once for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) { if (llama_vocab_is_eog(vocab, i)) { - COM_TRC("added %s logit bias = %f\n", common_token_to_piece(vocab, i).c_str(), -INFINITY); + LOG_TRC("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(vocab, i).c_str(), -INFINITY); params.sampling.logit_bias_eog.push_back({i, -INFINITY}); } } @@ -1297,7 +1285,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) llama_context * lctx = llama_init_from_model(model, cparams); if (lctx == NULL) { - COM_ERR("failed to create context with model '%s'\n", params.model.path.c_str()); + LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); return; } @@ -1334,7 +1322,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode llama_model * model = res->model(); if (model == NULL) { - COM_ERR("failed to load model '%s'\n", params.model.path.c_str()); + LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str()); return res; } @@ -1344,14 +1332,14 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode llama_context * lctx = res->context(); if (lctx == NULL) { - COM_ERR("failed to create context with model '%s'\n", params.model.path.c_str()); + LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); return res; } const llama_vocab * vocab = llama_model_get_vocab(model); if (params.ctx_shift && !llama_memory_can_shift(llama_get_memory(lctx))) { - COM_WRN("%s", "KV cache shifting is not supported for this context, disabling KV cache shifting\n"); + LOG_WRN("%s: KV cache shifting is not supported for this context, disabling KV cache shifting\n", __func__); params.ctx_shift = false; } @@ -1380,7 +1368,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode bool ok = true; if (llama_vocab_bos(vocab) == LLAMA_TOKEN_NULL) { - COM_WRN("%s", "vocab does not have a BOS token, reranking will not work\n"); + LOG_WRN("%s: warning: vocab does not have a BOS token, reranking will not work\n", __func__); ok = false; } @@ -1389,10 +1377,10 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode bool has_rerank_prompt = llama_model_chat_template(model, "rerank") != NULL; if (!has_eos && !has_sep && !has_rerank_prompt) { - COM_WRN("%s", "vocab does not have an EOS token, SEP token, or rerank prompt. Reranking will not work\n"); + LOG_WRN("%s: warning: vocab does not have an EOS token, SEP token, or rerank prompt. Reranking will not work\n", __func__); ok = false; } else if (!has_eos) { - COM_WRN("%s", "vocab does not have an EOS token, using SEP token as fallback\n"); + LOG_WRN("%s: warning: vocab does not have an EOS token, using SEP token as fallback\n", __func__); } if (!ok) { @@ -1405,7 +1393,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode } if (params.warmup) { - COM_TRC("%s", "warming up the model with an empty run - please wait ... (--no-warmup to disable)\n"); + LOG_INF("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__); std::vector tmp; llama_token bos = llama_vocab_bos(vocab); @@ -1479,20 +1467,20 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) { int ret = llama_decode(ctx, llama_batch_get_one(tmp.data(), tmp.size())); if (ret != 0) { - COM_ERR("llama_decode() failed: %d\n", ret); + LOG_ERR("%s: llama_decode() failed: %d\n", __func__, ret); res = COMMON_CONTEXT_SEQ_RM_TYPE_NO; goto done; } if (llama_n_rs_seq(ctx) > 0) { - COM_TRC("%s", "the context supports bounded partial sequence removal\n"); + LOG_INF("%s: the context supports bounded partial sequence removal\n", __func__); res = COMMON_CONTEXT_SEQ_RM_TYPE_RS; goto done; } // try to remove the last tokens if (!llama_memory_seq_rm(mem, 0, 1, -1)) { - COM_TRC("%s", "the context does not support partial sequence removal\n"); + LOG_TRC("%s: the context does not support partial sequence removal\n", __func__); res = COMMON_CONTEXT_SEQ_RM_TYPE_FULL; goto done; } @@ -1809,13 +1797,13 @@ static common_control_vector_data common_control_vector_load_one(const common_co }; struct gguf_context * ctx_gguf = gguf_init_from_file(load_info.fname.c_str(), meta_gguf_params); if (!ctx_gguf) { - COM_ERR("failed to load control vector file from %s\n", load_info.fname.c_str()); + LOG_ERR("%s: failed to load control vector file from %s\n", __func__, load_info.fname.c_str()); return result; } int32_t n_tensors = gguf_get_n_tensors(ctx_gguf); if (n_tensors == 0) { - COM_WRN("no direction tensors found in %s\n", load_info.fname.c_str()); + LOG_WRN("%s: no direction tensors found in %s\n", __func__, load_info.fname.c_str()); } for (int i = 0; i < n_tensors; i++) { @@ -1833,23 +1821,23 @@ static common_control_vector_data common_control_vector_load_one(const common_co } } if (layer_idx < 0) { - COM_ERR("invalid/unparsable direction tensor layer index in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid/unparsable direction tensor layer index in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } else if (layer_idx == 0) { - COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid (zero) direction tensor layer index in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str()); if (tensor->type != GGML_TYPE_F32) { - COM_ERR("invalid (non-F32) direction tensor type in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid (non-F32) direction tensor type in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } if (ggml_n_dims(tensor) != 1) { - COM_ERR("invalid (non-1D) direction tensor shape in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid (non-1D) direction tensor shape in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } @@ -1857,7 +1845,7 @@ static common_control_vector_data common_control_vector_load_one(const common_co if (result.n_embd == -1) { result.n_embd = ggml_nelements(tensor); } else if (ggml_nelements(tensor) != result.n_embd) { - COM_ERR("direction tensor in %s does not match previous dimensions\n", load_info.fname.c_str()); + LOG_ERR("%s: direction tensor in %s does not match previous dimensions\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } @@ -1874,7 +1862,7 @@ static common_control_vector_data common_control_vector_load_one(const common_co } if (result.n_embd == -1) { - COM_WRN("skipping %s due to invalid direction tensors\n", load_info.fname.c_str()); + LOG_WRN("%s: skipping %s due to invalid direction tensors\n", __func__, load_info.fname.c_str()); result.data.clear(); } @@ -1895,7 +1883,7 @@ common_control_vector_data common_control_vector_load(const std::vector(all_tokens.data() + offset), n_tokens_before_last))) { - COM_ERR("%s", "failed to eval\n"); + LOG_ERR("%s : failed to eval\n", __func__); return false; } n_past += n_tokens_before_last; llama_state_save_file(ctx, state_path.data(), all_tokens.data(), all_tokens.size()); - COM_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size()); + LOG_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size()); llama_token last_token = all_tokens.back(); llama_batch batch = llama_batch_get_one(&last_token, 1); @@ -2036,13 +2024,13 @@ bool common_prompt_batch_decode( batch.pos = &pos; if (llama_decode(ctx, batch)) { - COM_ERR("%s", "failed to eval last token\n"); + LOG_ERR("%s : failed to eval last token\n", __func__); return false; } n_past++; } else { if (llama_decode(ctx, llama_batch_get_one(const_cast(all_tokens.data() + offset), n_new))) { - COM_ERR("%s", "failed to eval\n"); + LOG_ERR("%s : failed to eval\n", __func__); return false; } n_past += n_new; @@ -2052,7 +2040,7 @@ bool common_prompt_batch_decode( } size_t common_prompt_checkpoint::size() const { - return data_tgt.size() + data_dft.size() + data_spec.size(); + return data_tgt.size() + data_dft.size(); } bool common_prompt_checkpoint::empty() const { @@ -2067,7 +2055,6 @@ void common_prompt_checkpoint::clear() { data_tgt.clear(); data_dft.clear(); - data_spec.clear(); } void common_prompt_checkpoint::update_pos( @@ -2157,5 +2144,4 @@ void common_prompt_checkpoint::clear_tgt() { void common_prompt_checkpoint::clear_dft() { data_dft.clear(); - data_spec.clear(); } diff --git a/common/common.h b/common/common.h index 7939d3b07..0b77a7ea9 100644 --- a/common/common.h +++ b/common/common.h @@ -26,13 +26,6 @@ #define DIRECTORY_SEPARATOR '/' #endif // _WIN32 -#define COM_DBG(fmt, ...) LOG_DBG("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_TRC(fmt, ...) LOG_TRC("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_INF(fmt, ...) LOG_INF("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_WRN(fmt, ...) LOG_WRN("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_ERR(fmt, ...) LOG_ERR("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__) - #define die(msg) do { fputs("error: " msg "\n", stderr); exit(1); } while (0) #define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0) @@ -104,7 +97,6 @@ enum llama_example { LLAMA_EXAMPLE_FIT_PARAMS, LLAMA_EXAMPLE_RESULTS, LLAMA_EXAMPLE_EXPORT_GRAPH_OPS, - LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_COUNT, }; @@ -170,7 +162,6 @@ enum common_speculative_type { COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction - COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values @@ -300,25 +291,12 @@ struct common_params_sampling { }; struct common_params_model { - std::string path = ""; // model local path - std::string url = ""; // model url to download - std::string hf_repo = ""; // HF repo - std::string hf_file = ""; // HF file - std::string docker_repo = ""; // Docker repo - - std::string get_name() const { - if (!hf_repo.empty()) { - return hf_repo; - } - if (!docker_repo.empty()) { - return docker_repo; - } - return path; - } - - bool empty() const { - return get_name().empty(); - } + std::string path = ""; // model local path // NOLINT + std::string url = ""; // model url to download // NOLINT + std::string hf_repo = ""; // HF repo // NOLINT + std::string hf_file = ""; // HF file // NOLINT + std::string docker_repo = ""; // Docker repo // NOLINT + std::string name = ""; // in format /[:] (tag is optional) // NOLINT }; // draft-model-based speculative decoding parameters @@ -381,12 +359,12 @@ struct common_params_speculative { common_params_speculative_ngram_cache ngram_cache; bool has_dft() const { - return !draft.mparams.empty(); + return !draft.mparams.path.empty() || !draft.mparams.hf_repo.empty(); } uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP; }); return needs_rs_seq ? draft.n_max : 0u; @@ -533,6 +511,7 @@ struct common_params { int32_t control_vector_layer_start = -1; // layer range for control vector int32_t control_vector_layer_end = -1; // layer range for control vector bool offline = false; + bool skip_download = false; // skip model file downloading int32_t ppl_stride = 0; // stride for perplexity calculations. If left at 0, the pre-existing approach will be used. int32_t ppl_output_type = 0; // = 0 -> ppl output is as usual, = 1 -> ppl output is num_tokens, ppl, one per line @@ -622,7 +601,7 @@ struct common_params { bool cache_prompt = true; // whether to enable prompt caching bool cache_idle_slots = true; // save and clear idle slots upon starting a new task int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot - int32_t checkpoint_min_step = 8192; // minimum spacing between context checkpoints + int32_t checkpoint_min_step = 256; // minimum spacing between context checkpoints int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string hostname = "127.0.0.1"; @@ -646,6 +625,12 @@ struct common_params { // UI configs bool ui = true; + + // Deprecated: use ui, ui_mcp_proxy, ui_config_json instead + bool webui = ui; + bool webui_mcp_proxy = false; + std::string webui_config_json; + bool ui_mcp_proxy = false; std::string ui_config_json; @@ -658,11 +643,10 @@ struct common_params { std::vector server_tools; // router server configs - std::string models_dir = ""; // directory containing models for the router server - std::string models_preset = ""; // directory containing model presets for the router server - int models_max = 4; // maximum number of models to load simultaneously - bool models_autoload = true; // automatically load models when requested via the router server - std::string models_preset_hf = ""; // show a warning about remote presets on router loaded (if not empty) + std::string models_dir = ""; // directory containing models for the router server + std::string models_preset = ""; // directory containing model presets for the router server + int models_max = 4; // maximum number of models to load simultaneously + bool models_autoload = true; // automatically load models when requested via the router server bool log_json = false; @@ -864,9 +848,6 @@ struct common_file_info { }; std::vector fs_list(const std::string & path, bool include_directories); -// fs open, also handle UTF8 on Windows -std::ifstream fs_open_ifstream(const std::string & fname, std::ios_base::openmode mode); - // // TTY utils // @@ -1084,10 +1065,6 @@ struct common_prompt_checkpoint { std::vector data_tgt; std::vector data_dft; - // (optional) speculative-decoding implementation state stashed with the checkpoint - // (e.g. eagle3's deferred-boundary g_embd row) - std::vector data_spec; - size_t size() const; bool empty() const; diff --git a/common/download.cpp b/common/download.cpp index 6b69a4418..61b2f25fa 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -21,7 +21,9 @@ #include #include +#if defined(LLAMA_USE_HTTPLIB) #include "http.h" +#endif #ifndef __EMSCRIPTEN__ #ifdef __linux__ @@ -115,6 +117,7 @@ std::pair common_download_split_repo_tag(const std::st return {hf_repo, tag}; } +#if defined(LLAMA_USE_HTTPLIB) class ProgressBar : public common_download_callback { static inline std::mutex mutex; static inline std::map lines; @@ -292,6 +295,10 @@ static int common_download_file_single_online(const std::string & url, const bool file_exists = std::filesystem::exists(path); + if (!file_exists && opts.skip_download) { + return -2; // file is missing and download is disabled + } + if (file_exists && skip_etag) { LOG_DBG("%s: using cached file: %s\n", __func__, path.c_str()); return 304; // 304 Not Modified - fake cached response @@ -358,6 +365,9 @@ static int common_download_file_single_online(const std::string & url, return 304; // 304 Not Modified - fake cached response } // pass this point, the file exists but is different from the server version, so we need to redownload it + if (opts.skip_download) { + return -2; // special code to indicate that the download was skipped due to etag mismatch + } if (remove(path.c_str()) != 0) { LOG_ERR("%s: unable to delete file: %s\n", __func__, path.c_str()); return -1; @@ -684,8 +694,18 @@ static void list_available_gguf_files(const hf_cache::hf_files & files) { } } -common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts) { - common_download_hf_plan plan; +struct hf_plan { + hf_cache::hf_file primary; + hf_cache::hf_files model_files; + hf_cache::hf_file mmproj; + hf_cache::hf_file mtp; +}; + +static hf_plan get_hf_plan(const common_params_model & model, + const common_download_opts & opts, + bool download_mmproj, + bool download_mtp) { + hf_plan plan; hf_cache::hf_files all; auto [repo, tag] = common_download_split_repo_tag(model.hf_repo); @@ -700,14 +720,6 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & return plan; } - // if preset.ini exists in the repo root, download only that file - for (const auto & f : all) { - if (f.path == "preset.ini") { - plan.preset = f; - return plan; - } - } - hf_cache::hf_file primary; if (!model.hf_file.empty()) { @@ -734,49 +746,115 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & plan.primary = primary; plan.model_files = get_split_files(all, primary); - if (opts.download_mmproj) { + if (download_mmproj) { plan.mmproj = find_best_mmproj(all, primary.path); } - if (opts.download_mtp) { + + if (download_mtp) { plan.mtp = find_best_mtp(all, primary.path); } return plan; } -void common_download_run_tasks(const std::vector & tasks) { +struct download_task { + std::string url; + std::string path; +}; + +static std::vector get_url_tasks(const common_params_model & model) { + auto split = get_gguf_split_info(model.url); + + if (split.count <= 1) { + return {{model.url, model.path}}; + } + + auto filename = split.prefix; + if (auto pos = split.prefix.rfind('/'); pos != std::string::npos) { + filename = split.prefix.substr(pos + 1); + } + + auto parent_path = std::filesystem::path(model.path).parent_path(); + auto prefix_path = (parent_path / filename).string(); + + std::vector tasks; + for (int i = 1; i <= split.count; i++) { + auto suffix = string_format("-%05d-of-%05d.gguf", i, split.count); + tasks.push_back({split.prefix + suffix, prefix_path + suffix}); + } + return tasks; +} + +common_download_model_result common_download_model(const common_params_model & model, + const common_download_opts & opts) { + common_download_model_result result; + std::vector tasks; + hf_plan hf; + + bool download_mmproj = opts.download_mmproj; + bool download_mtp = opts.download_mtp; + bool is_hf = !model.hf_repo.empty(); + + if (is_hf) { + hf = get_hf_plan(model, opts, download_mmproj, download_mtp); + for (const auto & f : hf.model_files) { + tasks.push_back({f.url, f.local_path}); + } + if (!hf.mmproj.path.empty()) { + tasks.push_back({hf.mmproj.url, hf.mmproj.local_path}); + } + if (!hf.mtp.path.empty()) { + tasks.push_back({hf.mtp.url, hf.mtp.local_path}); + } + } else if (!model.url.empty()) { + tasks = get_url_tasks(model); + } else { + result.model_path = model.path; + return result; + } + + if (tasks.empty()) { + return result; + } + std::vector> futures; for (const auto & task : tasks) { futures.push_back(std::async(std::launch::async, - [&task]() { - return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf); + [&task, &opts, is_hf]() { + return common_download_file_single(task.url, task.path, opts, is_hf); } )); } - for (size_t i = 0; i < futures.size(); ++i) { - std::string url = tasks[i].url; - int status = futures[i].get(); + for (auto & f : futures) { + int status = f.get(); + if (status == -2 && opts.skip_download) { + throw common_skip_download_exception(); + } bool is_ok = is_http_status_ok(status); if (!is_ok) { - throw std::runtime_error(string_format("Download '%s' failed with status code: %d", url.c_str(), status)); + return {}; } } -} -std::vector common_download_get_all_parts(const std::string & url) { - auto split = get_gguf_split_info(url); + if (is_hf) { + for (const auto & f : hf.model_files) { + hf_cache::finalize_file(f); + } + result.model_path = hf.primary.final_path; - if (split.count <= 1) { - return {url}; + if (!hf.mmproj.path.empty()) { + result.mmproj_path = hf_cache::finalize_file(hf.mmproj); + } + + if (!hf.mtp.path.empty()) { + result.mtp_path = hf_cache::finalize_file(hf.mtp); + } + } else { + result.model_path = model.path; } - std::vector parts; - for (int i = 1; i <= split.count; i++) { - auto suffix = string_format("-%05d-of-%05d.gguf", i, split.count); - parts.push_back(split.prefix + suffix); - } - return parts; + return result; } // @@ -923,86 +1001,73 @@ std::vector common_list_cached_models() { return result; } -bool common_download_remove(const std::string & hf_repo_with_tag) { - namespace fs = std::filesystem; - auto [repo_id, tag] = common_download_split_repo_tag(hf_repo_with_tag); +#else - if (tag.empty()) { - return hf_cache::remove_cached_repo(repo_id); - } +// common_hf_file_res common_get_hf_file(const std::string &, const std::string &, bool, const common_header_list &) { +// throw std::runtime_error("download functionality is not enabled in this build"); +// } - std::string tag_upper = tag; - for (char & c : tag_upper) { - c = (char) std::toupper((unsigned char) c); - } - - auto files = hf_cache::get_cached_files(repo_id); - if (files.empty()) { - return false; - } - - // collect snapshot entries whose tag matches - std::vector to_remove; - for (const auto & f : files) { - auto split = get_gguf_split_info(f.path); - if (split.tag == tag_upper) { - to_remove.emplace_back(f.local_path); - } - } - - if (to_remove.empty()) { - return false; - } - - // resolve blob paths from symlinks before deleting snapshot entries - std::vector blobs_to_check; - for (const auto & p : to_remove) { - std::error_code ec; - if (fs::is_symlink(p, ec)) { - auto target = fs::read_symlink(p, ec); - if (!ec) { - blobs_to_check.push_back((p.parent_path() / target).lexically_normal()); - } - } - } - - // remove snapshot entries - for (const auto & p : to_remove) { - std::error_code ec; - fs::remove(p, ec); - if (ec) { - LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str()); - } - } - - if (blobs_to_check.empty()) { - return true; - } - - // collect blobs still referenced by remaining snapshot entries - std::unordered_set still_referenced; - for (const auto & f : hf_cache::get_cached_files(repo_id)) { - fs::path p(f.local_path); - std::error_code ec; - if (fs::is_symlink(p, ec)) { - auto target = fs::read_symlink(p, ec); - if (!ec) { - still_referenced.insert((p.parent_path() / target).lexically_normal().string()); - } - } - } - - // remove orphaned blobs - for (const auto & blob : blobs_to_check) { - if (still_referenced.find(blob.string()) == still_referenced.end()) { - std::error_code ec; - fs::remove(blob, ec); - if (ec) { - LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str()); - } - } - } - - return true; +common_download_model_result common_download_model(const common_params_model & model, + const common_download_opts & opts) { + throw std::runtime_error("download functionality is not enabled in this build"); } + +std::string common_docker_resolve_model(const std::string &) { + throw std::runtime_error("download functionality is not enabled in this build"); +} + +int common_download_file_single(const std::string & url, + const std::string & path, + const common_download_opts & opts, + bool skip_etag) { + throw std::runtime_error("download functionality is not enabled in this build"); +} + +std::pair> common_remote_get_content(const std::string & url, + const common_remote_params & params) { + throw std::runtime_error("download functionality is not enabled in this build"); +} + +struct gguf_split_info { + std::string prefix; // tag included + std::string tag; + int index; + int count; +}; +static gguf_split_info get_gguf_split_info(const std::string & path) { + static const std::regex re_split("^(.+)-([0-9]{5})-of-([0-9]{5})$", std::regex::icase); + static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); + std::smatch m; + + std::string prefix = path; + string_remove_suffix(prefix, ".gguf"); + + int index = 1; + int count = 1; + + if (std::regex_match(prefix, m, re_split)) { + index = std::stoi(m[2].str()); + count = std::stoi(m[3].str()); + prefix = m[1].str(); + } + + std::string tag; + if (std::regex_search(prefix, m, re_tag)) { + tag = m[1].str(); + for (char & c : tag) { + c = std::toupper((unsigned char)c); + } + } + + return {std::move(prefix), std::move(tag), index, count}; +} + +std::vector common_list_cached_models() { + std::vector result; + return result; +} + + +#endif // defined(LLAMA_USE_HTTPLIB) + diff --git a/common/download.h b/common/download.h index 6560a91b8..4f67180ec 100644 --- a/common/download.h +++ b/common/download.h @@ -1,11 +1,8 @@ #pragma once -#include "hf-cache.h" - #include #include #include -#include struct common_params_model; @@ -51,40 +48,65 @@ struct common_cached_model_info { } }; -// Options for common_download_file_single +// Options for common_download_model and common_download_file_single struct common_download_opts { std::string bearer_token; common_header_list headers; bool offline = false; + bool skip_download = false; // if true, only validation is performed, common_skip_download_exception may be thrown if the file is missing or invalid bool download_mmproj = false; bool download_mtp = false; common_download_callback * callback = nullptr; }; -struct common_download_task { - common_download_opts opts; - std::string url; - std::string local_path; - std::function on_done; - bool is_hf = false; - - common_download_task() = default; - common_download_task(hf_cache::hf_file f, - const common_download_opts & opts, - std::function on_done = nullptr) - : opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_hf(true) {} +// Result of common_download_model +struct common_download_model_result { + std::string model_path; + std::string mmproj_path; + std::string mtp_path; }; -void common_download_run_tasks(const std::vector & tasks); +// throw if the file is missing or invalid (e.g. ETag check failed) +struct common_skip_download_exception : public std::runtime_error { + common_skip_download_exception() : std::runtime_error("skip download") {} +}; -// if url is a multi-part GGUF file, returns all parts, otherwise returns the single file -std::vector common_download_get_all_parts(const std::string & url); +// Download model from HuggingFace repo or URL +// +// input (via model struct): +// - model.hf_repo: HF repo with optional tag, see common_download_split_repo_tag +// - model.hf_file: specific file in the repo (requires hf_repo) +// - model.url: simple download (used if hf_repo is empty) +// - model.path: local file path +// +// tag matching (for HF repos without model.hf_file): +// - if tag is specified, searches for GGUF matching that quantization +// - if no tag, searches for Q4_K_M, then Q4_0, then first available GGUF +// +// split GGUF: multi-part files like "model-00001-of-00003.gguf" are automatically +// detected and all parts are downloaded +// +// caching: +// - HF repos: uses HuggingFace cache +// - URLs: uses ETag-based caching +// +// when opts.offline=true, no network requests are made +// when download_mmproj=true, searches for mmproj in same directory as model or any parent directory +// then with the closest quantization bits +// when download_mtp=true, applies the same sibling search for an MTP-head GGUF +// +// returns result with model_path, mmproj_path and mtp_path (empty when not found / on failure) +common_download_model_result common_download_model( + const common_params_model & model, + const common_download_opts & opts = {} +); // returns list of cached models std::vector common_list_cached_models(); // download single file from url to local path // returns status code or -1 on error +// returns -2 if the download was skipped due to ETag mismatch (file outdated, skip_download=true) // skip_etag: if true, don't read/write .etag files (for HF cache where filename is the hash) int common_download_file_single(const std::string & url, const std::string & path, @@ -94,19 +116,3 @@ int common_download_file_single(const std::string & url, // resolve and download model from Docker registry // return local path to downloaded model file std::string common_docker_resolve_model(const std::string & docker); - -// Remove a cached model from disk -// input format: "user/model" or "user/model:tag" -// - if tag is omitted, removes the entire repo cache directory -// - if tag is present, removes only files matching that tag (and orphaned blobs) -// returns true if anything was removed -bool common_download_remove(const std::string & hf_repo_with_tag); - -struct common_download_hf_plan { - hf_cache::hf_file primary; - hf_cache::hf_files model_files; - hf_cache::hf_file mmproj; - hf_cache::hf_file mtp; - hf_cache::hf_file preset; // if set, only this file is downloaded -}; -common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); diff --git a/common/fit.cpp b/common/fit.cpp index afbf0b10f..a8565bfc9 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -233,7 +233,7 @@ static void common_params_fit_impl( sum_projected_used = dmds_full.back().mb.total(); sum_free = dmds_full.back().total; sum_projected_free = sum_free - sum_projected_used; - LOG_TRC("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n", + LOG_INF("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n", __func__, sum_projected_used/MiB, sum_free/MiB); if (sum_projected_free >= margins[0]) { LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of system memory, no changes needed\n", diff --git a/common/hf-cache.cpp b/common/hf-cache.cpp index f1dacaa47..ba7417a12 100644 --- a/common/hf-cache.cpp +++ b/common/hf-cache.cpp @@ -495,19 +495,4 @@ std::string finalize_file(const hf_file & file) { return file.final_path; } -bool remove_cached_repo(const std::string & repo_id) { - if (!is_valid_repo_id(repo_id)) { - LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str()); - return false; - } - fs::path repo_path = get_repo_path(repo_id); - std::error_code ec; - auto removed = fs::remove_all(repo_path, ec); - if (ec) { - LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str()); - return false; - } - return removed > 0; -} - } // namespace hf_cache diff --git a/common/hf-cache.h b/common/hf-cache.h index 42c9c6ce3..23fa0adb7 100644 --- a/common/hf-cache.h +++ b/common/hf-cache.h @@ -29,7 +29,4 @@ hf_files get_cached_files(const std::string & repo_id = {}); // Create snapshot path (link or move/copy) and return it std::string finalize_file(const hf_file & file); -// Remove the entire cached directory for a repo, returns true if removed -bool remove_cached_repo(const std::string & repo_id); - } // namespace hf_cache diff --git a/common/http.h b/common/http.h index e88bc6a5e..d3daccd6b 100644 --- a/common/http.h +++ b/common/http.h @@ -11,11 +11,6 @@ struct common_http_url { std::string path; }; -// bracket an IPv6 literal host for a URL authority (RFC 3986) -static std::string common_http_format_host(const std::string & host) { - return host.find(':') != std::string::npos ? "[" + host + "]" : host; -} - static common_http_url common_http_parse_url(const std::string & url) { common_http_url parts; auto scheme_end = url.find("://"); @@ -54,28 +49,11 @@ static common_http_url common_http_parse_url(const std::string & url) { parts.path = "/"; } - // split the authority into host and optional port, a bracketed IPv6 literal keeps its inner colons (RFC 3986) - std::string port_str; - if (!parts.host.empty() && parts.host.front() == '[') { - auto close = parts.host.find(']'); - if (close == std::string::npos) { - throw std::runtime_error("invalid IPv6 URL authority: " + parts.host); - } - auto after = parts.host.substr(close + 1); - if (!after.empty() && after.front() == ':') { - port_str = after.substr(1); - } - parts.host = parts.host.substr(1, close - 1); - } else { - auto colon_pos = parts.host.find(':'); - if (colon_pos != std::string::npos) { - port_str = parts.host.substr(colon_pos + 1); - parts.host = parts.host.substr(0, colon_pos); - } - } + auto colon_pos = parts.host.find(':'); - if (!port_str.empty()) { - parts.port = std::stoi(port_str); + if (colon_pos != std::string::npos) { + parts.port = std::stoi(parts.host.substr(colon_pos + 1)); + parts.host = parts.host.substr(0, colon_pos); } else if (parts.scheme == "http") { parts.port = 80; } else if (parts.scheme == "https") { @@ -105,7 +83,7 @@ static std::pair common_http_client(const std: } #endif - httplib::Client cli(parts.scheme + "://" + common_http_format_host(parts.host) + ":" + std::to_string(parts.port)); + httplib::Client cli(parts.scheme + "://" + parts.host + ":" + std::to_string(parts.port)); if (!parts.user.empty()) { cli.set_basic_auth(parts.user, parts.password); @@ -117,5 +95,5 @@ static std::pair common_http_client(const std: } static std::string common_http_show_masked_url(const common_http_url & parts) { - return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + common_http_format_host(parts.host) + parts.path; + return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + parts.host + parts.path; } diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index f7de8b806..ead864763 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -9,9 +9,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-caps" using json = nlohmann::ordered_json; @@ -19,34 +16,22 @@ using json = nlohmann::ordered_json; namespace jinja { using caps_json_fn = std::function; -using caps_ctx_fn = std::function; -using caps_analyze_fn = std::function; - -void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { - ctx.set_val("preserve_thinking", mk_val(enabled)); - ctx.set_val("clear_thinking", mk_val(!enabled)); - ctx.set_val("truncate_history_thinking", mk_val(!enabled)); -} +using caps_analyze_fn = std::function; static void caps_try_execute(jinja::program & prog, const caps_json_fn & messages_fn, - const caps_ctx_fn & ctx_fn, const caps_json_fn & tools_fn, const caps_analyze_fn & analyze_fn) { context ctx; ctx.is_get_stats = true; jinja::global_from_json(ctx, json{ {"messages", messages_fn()}, - {"tools", tools_fn ? tools_fn() : json::array()}, + {"tools", tools_fn()}, {"bos_token", ""}, {"eos_token", ""}, {"add_generation_prompt", true} }, true); - if (ctx_fn) { - ctx_fn(ctx); - } - auto messages = ctx.get_val("messages"); auto tools = ctx.get_val("tools"); @@ -64,7 +49,7 @@ static void caps_try_execute(jinja::program & prog, // ignore exceptions during capability analysis } - analyze_fn(success, messages, tools, result); + analyze_fn(success, messages, tools); } // for debugging only @@ -124,9 +109,11 @@ caps caps_get(jinja::program & prog) { } }); }, - nullptr, // ctx_fn - nullptr, // tools_fn - [&](bool success, value & messages, value &, const std::string &) { + [&]() { + // tools + return json{nullptr}; + }, + [&](bool success, value & messages, value &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (has_op(content, "selectattr") || has_op(content, "array_access")) { @@ -158,9 +145,11 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn - nullptr, // tools_fn - [&](bool, value & messages, value &, const std::string &) { + [&]() { + // tools + return json::array(); + }, + [&](bool, value & messages, value &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (!content->stats.used) { @@ -212,7 +201,6 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -236,7 +224,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](bool success, value & messages, value & tools) { if (!success) { return; // Nothing can be inferred } @@ -305,7 +293,6 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -329,7 +316,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](bool success, value & messages, value & tools) { if (!success) { result.supports_tool_calls = false; result.supports_tools = false; @@ -407,7 +394,6 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -431,7 +417,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value &, const std::string &) { + [&](bool success, value & messages, value & /*tools*/) { if (!success) { result.supports_parallel_tool_calls = false; return; @@ -452,22 +438,11 @@ caps caps_get(jinja::program & prog) { JJ_DEBUG("%s\n", ">>> Running capability check: preserve reasoning"); // case: preserve reasoning content in chat history - const std::string reasoning_placeholder = ""; caps_try_execute( prog, [&]() { // messages return json::array({ - { - {"role", "user"}, - {"content", "User message"} - }, - { - {"role", "assistant"}, - {"content", "Assistant message"}, - // check of reasoning_content deeper in the history, not just the last assistant message - {"reasoning_content", reasoning_placeholder} - }, { {"role", "user"}, {"content", "User message"} @@ -483,13 +458,14 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](context & ctx) { - caps_apply_preserve_reasoning(ctx, true); + [&]() { + // tools + return json::array(); }, - nullptr, // tools_fn - [&](bool, value &, value &, const std::string & output) { - // note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result - if (output.find(reasoning_placeholder) != std::string::npos) { + [&](bool, value & messages, value &) { + auto & content = messages->at(1)->at("reasoning_content"); + caps_print_stats(content, "messages[1].reasoning_content"); + if (content->stats.used) { result.supports_preserve_reasoning = true; } } diff --git a/common/jinja/caps.h b/common/jinja/caps.h index a290cd7da..93a7fe092 100644 --- a/common/jinja/caps.h +++ b/common/jinja/caps.h @@ -12,9 +12,7 @@ struct caps { bool supports_tool_calls = true; bool supports_system_role = true; bool supports_parallel_tool_calls = true; - - // supports preserve reasoning trace in the full history, not just the last assistant message - bool supports_preserve_reasoning = false; + bool supports_preserve_reasoning = false; // support assistant message with reasoning_content // one of the 2 content capabilities must be true bool supports_string_content = true; @@ -31,6 +29,4 @@ struct caps { caps caps_get(jinja::program & prog); -void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled); - } // namespace jinja diff --git a/common/jinja/lexer.cpp b/common/jinja/lexer.cpp index 63740dff6..598982c2f 100644 --- a/common/jinja/lexer.cpp +++ b/common/jinja/lexer.cpp @@ -7,9 +7,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-lexer" namespace jinja { diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp index e9e6dcb7f..2b25654a7 100644 --- a/common/jinja/parser.cpp +++ b/common/jinja/parser.cpp @@ -8,9 +8,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-parser" namespace jinja { diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index afba0025f..1fae7884e 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -8,9 +8,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-runtime" bool g_jinja_debug = false; @@ -689,62 +686,59 @@ value set_statement::execute_impl(context & ctx) { return mk_val(); } -static inline void bind_parameters(const std::string & name, const statements & this_args, const func_args & args, context & ctx) { - const size_t expected_count = this_args.size(); - const size_t input_count = args.count(); - - JJ_DEBUG("Invoking '%s' with %zu input arguments (expected %zu)", name.c_str(), input_count, expected_count); - for (size_t i = 0; i < expected_count; ++i) { - if (i < input_count) { - if (is_stmt(this_args[i])) { - // normal parameter - std::string param_name = cast_stmt(this_args[i])->val; - value param_value = args.get_kwarg_or_pos(param_name, i); - JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); - ctx.set_val(param_name, param_value); - } else if (is_stmt(this_args[i])) { - // default argument used as normal parameter - auto kwarg = cast_stmt(this_args[i]); - if (!is_stmt(kwarg->key)) { - throw std::runtime_error("Keyword argument key must be an identifier in '" + name + "'"); - } - std::string param_name = cast_stmt(kwarg->key)->val; - value param_value = args.get_kwarg_or_pos(param_name, i); - JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); - ctx.set_val(param_name, param_value); - } else { - throw std::runtime_error("Invalid parameter type in '" + name + "'"); - } - } else { - auto & default_arg = this_args[i]; - if (is_stmt(default_arg)) { - auto kwarg = cast_stmt(default_arg); - if (!is_stmt(kwarg->key)) { - throw std::runtime_error("Keyword argument key must be an identifier in '" + name + "'"); - } - std::string param_name = cast_stmt(kwarg->key)->val; - JJ_DEBUG(" Binding parameter '%s' to default argument of type %s", param_name.c_str(), kwarg->val->type().c_str()); - ctx.set_val(param_name, kwarg->val->execute(args.ctx)); - } else { - throw std::runtime_error("Not enough arguments provided to '" + name + "'"); - } - //std::string param_name = cast_stmt(default_args[i])->val; - //JJ_DEBUG(" Binding parameter '%s' to default", param_name.c_str()); - //ctx.var[param_name] = default_args[i]->execute(ctx); - } - } -} - value macro_statement::execute_impl(context & ctx) { if (!is_stmt(this->name)) { throw std::runtime_error("Macro name must be an identifier"); } std::string name = cast_stmt(this->name)->val; - const func_handler func = [this, name](const func_args & args) -> value { - context macro_ctx(args.ctx); // new scope for macro execution + const func_handler func = [this, name, &ctx](const func_args & args) -> value { + size_t expected_count = this->args.size(); + size_t input_count = args.count(); - bind_parameters(name, this->args, args, macro_ctx); + JJ_DEBUG("Invoking macro '%s' with %zu input arguments (expected %zu)", name.c_str(), input_count, expected_count); + context macro_ctx(ctx); // new scope for macro execution + + // bind parameters + for (size_t i = 0; i < expected_count; ++i) { + if (i < input_count) { + if (is_stmt(this->args[i])) { + // normal parameter + std::string param_name = cast_stmt(this->args[i])->val; + value param_value = args.get_kwarg_or_pos(param_name, i); + JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); + macro_ctx.set_val(param_name, param_value); + } else if (is_stmt(this->args[i])) { + // default argument used as normal parameter + auto kwarg = cast_stmt(this->args[i]); + if (!is_stmt(kwarg->key)) { + throw std::runtime_error("Keyword argument key must be an identifier in macro '" + name + "'"); + } + std::string param_name = cast_stmt(kwarg->key)->val; + value param_value = args.get_kwarg_or_pos(param_name, i); + JJ_DEBUG(" Binding parameter '%s' to argument of type %s", param_name.c_str(), param_value->type().c_str()); + macro_ctx.set_val(param_name, param_value); + } else { + throw std::runtime_error("Invalid parameter type in macro '" + name + "'"); + } + } else { + auto & default_arg = this->args[i]; + if (is_stmt(default_arg)) { + auto kwarg = cast_stmt(default_arg); + if (!is_stmt(kwarg->key)) { + throw std::runtime_error("Keyword argument key must be an identifier in macro '" + name + "'"); + } + std::string param_name = cast_stmt(kwarg->key)->val; + JJ_DEBUG(" Binding parameter '%s' to default argument of type %s", param_name.c_str(), kwarg->val->type().c_str()); + macro_ctx.set_val(param_name, kwarg->val->execute(ctx)); + } else { + throw std::runtime_error("Not enough arguments provided to macro '" + name + "'"); + } + //std::string param_name = cast_stmt(default_args[i])->val; + //JJ_DEBUG(" Binding parameter '%s' to default", param_name.c_str()); + //macro_ctx.var[param_name] = default_args[i]->execute(ctx); + } + } // execute macro body JJ_DEBUG("Executing macro '%s' body with %zu statements", name.c_str(), this->body.size()); @@ -758,46 +752,6 @@ value macro_statement::execute_impl(context & ctx) { return mk_val(); } -value call_statement::execute_impl(context & ctx) { - auto call_expr = cast_stmt(this->call); - if (!call_expr) { - throw std::runtime_error("Call statement requires a valid call expression"); - } - - value callee_val = call_expr->callee->execute(ctx); - if (!is_val(callee_val)) { - throw std::runtime_error("Callee is not a function: got " + callee_val->type()); - } - auto * callee_func = cast_val(callee_val); - - context caller_ctx(ctx); // new scope for caller execution - - const func_handler func = [this, caller_ctx = std::move(caller_ctx)](const func_args & args) -> value { - context block_ctx(caller_ctx); // new scope for block execution - - bind_parameters("caller", this->caller_args, args, block_ctx); - - JJ_DEBUG("Executing call body with %zu statements", this->body.size()); - auto res = exec_statements(this->body, block_ctx); - JJ_DEBUG("Call body execution complete, result: %s", res->val_str.str().c_str()); - return res; - }; - - context call_ctx(ctx); - call_ctx.set_val("caller", mk_val("caller", func)); - - func_args args(call_ctx); - - for (const auto & arg_expr : call_expr->args) { - auto arg_val = arg_expr->execute(ctx); - JJ_DEBUG(" Argument type: %s", arg_val->type().c_str()); - args.push_back(arg_val); - } - - JJ_DEBUG("Calling macro '%s' with %zu arguments", callee_func->name.c_str(), args.count()); - return callee_func->invoke(args); -} - value member_expression::execute_impl(context & ctx) { value object = this->object->execute(ctx); @@ -957,50 +911,4 @@ value keyword_argument_expression::execute_impl(context & ctx) { return mk_val(k, v); } -std::string runtime::debug_dump_program(const program & prog, const std::string & src) { - std::ostringstream oss; - size_t lvl = 0; - context ctx; - ctx.src.reset(new std::string(src)); - - auto indent = [](size_t lvl) -> std::string { - return std::string(lvl * 2, ' '); - }; - - ctx.visitor = [&](bool is_leaf, statement * node, std::vector children) { - oss << indent(lvl) << node->type() << ":\n"; - lvl++; - if (is_leaf) { - const auto & pos = node->pos; - oss << indent(lvl) << "(leaf) at " << get_line_col(src, pos) << " in source:\n"; - std::string snippet = peak_source(src, pos); - string_replace_all(snippet, "\n", "\n" + indent(lvl)); - oss << indent(lvl) << snippet << "\n"; - } else { - for (auto & [label, children_vec] : children) { - oss << indent(lvl) << label << ":\n"; - lvl++; - if (children_vec.empty()) { - oss << indent(lvl) << "\n\n"; - } else { - for (auto * child : children_vec) { - if (!child) { - continue; - } - child->visit(ctx); - } - } - lvl--; - } - } - lvl--; - }; - - for (const auto & stmt : prog.body) { - stmt->visit(ctx); - } - - return oss.str(); -} - } // namespace jinja diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 0884a1592..b6f4a6ab4 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -47,19 +47,12 @@ const T * cast_stmt(const statement_ptr & ptr) { // not thread-safe void enable_debug(bool enable); -// for visiting AST nodes -// function signature: void(bool is_leaf, statement * node, pair of ) -using visitor_pair = std::pair>; -using visitor_fn = std::function)>; - struct context { std::shared_ptr src; // for debugging; use shared_ptr to avoid copying on scope creation std::time_t current_time; // for functions that need current time bool is_get_stats = false; // whether to collect stats - visitor_fn visitor; - // src is optional, used for error reporting context(std::string src = "") : src(std::make_shared(std::move(src))) { env = mk_val(); @@ -106,15 +99,6 @@ private: value_object env; }; -// utils for visiting AST nodes -static std::vector stmts_to_ptr(const statements & stmts) { - std::vector children; - for (const auto & stmt : stmts) { - children.push_back(stmt.get()); - } - return children; -} - /** * Base class for all nodes in the AST. */ @@ -122,7 +106,6 @@ struct statement { size_t pos; // position in source, for debugging virtual ~statement() = default; virtual std::string type() const { return "Statement"; } - virtual void visit(context & ctx) { ctx.visitor(true, this, {}); } // execute_impl must be overridden by derived classes virtual value execute_impl(context &) { throw_exec_error(); } @@ -183,13 +166,6 @@ struct if_statement : public statement { std::string type() const override { return "If"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"test", {test.get()}}, - {"body", stmts_to_ptr(body)}, - {"alternate", stmts_to_ptr(alternate)} - }); - } }; struct identifier; @@ -214,14 +190,6 @@ struct for_statement : public statement { std::string type() const override { return "For"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"loopvar", {loopvar.get()}}, - {"iterable", {iterable.get()}}, - {"body", stmts_to_ptr(body)}, - {"default_block", stmts_to_ptr(default_block)} - }); - } }; struct break_statement : public statement { @@ -273,13 +241,6 @@ struct set_statement : public statement { std::string type() const override { return "Set"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"assignee", {assignee.get()}}, - {"value", {val.get()}}, - {"body", stmts_to_ptr(body)} - }); - } }; struct macro_statement : public statement { @@ -295,13 +256,6 @@ struct macro_statement : public statement { std::string type() const override { return "Macro"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"name", {name.get()}}, - {"args", stmts_to_ptr(args)}, - {"body", stmts_to_ptr(body)} - }); - } }; struct comment_statement : public statement { @@ -335,12 +289,6 @@ struct member_expression : public expression { } std::string type() const override { return "MemberExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"object", {object.get()}}, - {"property", {property.get()}} - }); - } }; struct call_expression : public expression { @@ -354,12 +302,6 @@ struct call_expression : public expression { } std::string type() const override { return "CallExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"callee", {callee.get()}}, - {"args", stmts_to_ptr(args)} - }); - } }; /** @@ -463,12 +405,6 @@ struct binary_expression : public expression { } std::string type() const override { return "BinaryExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"left", {left.get()}}, - {"right", {right.get()}} - }); - } }; /** @@ -495,12 +431,6 @@ struct filter_expression : public expression { std::string type() const override { return "FilterExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"operand", {operand.get()}}, - {"filter", {filter.get()}} - }); - } }; struct filter_statement : public statement { @@ -513,12 +443,6 @@ struct filter_statement : public statement { } std::string type() const override { return "FilterStatement"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"filter", {filter.get()}}, - {"body", stmts_to_ptr(body)} - }); - } }; /** @@ -544,12 +468,6 @@ struct select_expression : public expression { } return lhs->execute_impl(ctx); } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"lhs", {lhs.get()}}, - {"test", {test.get()}} - }); - } }; /** @@ -568,12 +486,6 @@ struct test_expression : public expression { } std::string type() const override { return "TestExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"operand", {operand.get()}}, - {"test", {test.get()}} - }); - } }; /** @@ -589,11 +501,6 @@ struct unary_expression : public expression { } std::string type() const override { return "UnaryExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"argument", {argument.get()}} - }); - } }; struct slice_expression : public expression { @@ -611,13 +518,6 @@ struct slice_expression : public expression { [[noreturn]] value execute_impl(context &) override { throw std::runtime_error("must be handled by MemberExpression"); } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"start_expr", {start_expr.get()}}, - {"stop_expr", {stop_expr.get()}}, - {"step_expr", {step_expr.get()}} - }); - } }; struct keyword_argument_expression : public expression { @@ -631,12 +531,6 @@ struct keyword_argument_expression : public expression { } std::string type() const override { return "KeywordArgumentExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"key", {key.get()}}, - {"val", {val.get()}} - }); - } }; struct spread_expression : public expression { @@ -645,11 +539,6 @@ struct spread_expression : public expression { chk_type(this->argument); } std::string type() const override { return "SpreadExpression"; } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"argument", {argument.get()}} - }); - } }; struct call_statement : public statement { @@ -663,14 +552,6 @@ struct call_statement : public statement { for (const auto & arg : this->caller_args) chk_type(arg); } std::string type() const override { return "CallStatement"; } - value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"call", {call.get()}}, - {"caller_args", stmts_to_ptr(caller_args)}, - {"body", stmts_to_ptr(body)} - }); - } }; struct ternary_expression : public expression { @@ -693,13 +574,6 @@ struct ternary_expression : public expression { return false_expr->execute(ctx); } } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"condition", {condition.get()}}, - {"true_expr", {true_expr.get()}}, - {"false_expr", {false_expr.get()}} - }); - } }; struct raised_exception : public std::exception { @@ -773,8 +647,6 @@ struct runtime { } return parts; } - - static std::string debug_dump_program(const program & prog, const std::string & src); }; } // namespace jinja diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index f36bbbc0c..189451620 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -12,9 +12,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-value" namespace jinja { @@ -1111,50 +1108,6 @@ const func_builtins & value_array_t::get_builtins() const { std::reverse(arr.begin(), arr.end()); return is_val(val) ? mk_val(std::move(arr)) : mk_val(std::move(arr)); }}, - {"min", [](const func_args & args) -> value { - args.ensure_count(1, 4); - args.ensure_vals(); - value val_case = args.get_kwarg_or_pos("case_sensitive", 1); - value attribute = args.get_kwarg_or_pos("attribute", 2); - if (!attribute->is_undefined()) { - throw not_implemented_exception("min: attribute not implemented"); - } - // FIXME: min is currently always case sensitive - (void) val_case; - const auto & arr = args.get_pos(0)->as_array(); - if (arr.empty()) { - return mk_val(); - } - value result = arr[0]; - for (size_t i = 1; i < arr.size(); ++i) { - if (value_compare(arr[i], result, value_compare_op::lt)) { - result = arr[i]; - } - } - return result; - }}, - {"max", [](const func_args & args) -> value { - args.ensure_count(1, 4); - args.ensure_vals(); - value val_case = args.get_kwarg_or_pos("case_sensitive", 1); - value attribute = args.get_kwarg_or_pos("attribute", 2); - if (!attribute->is_undefined()) { - throw not_implemented_exception("max: attribute not implemented"); - } - // FIXME: max is currently always case sensitive - (void) val_case; - const auto & arr = args.get_pos(0)->as_array(); - if (arr.empty()) { - return mk_val(); - } - value result = arr[0]; - for (size_t i = 1; i < arr.size(); ++i) { - if (value_compare(arr[i], result, value_compare_op::gt)) { - result = arr[i]; - } - } - return result; - }}, {"unique", array_unique_not_implemented}, }; return builtins; diff --git a/common/json-partial.cpp b/common/json-partial.cpp new file mode 100644 index 000000000..aaf11310a --- /dev/null +++ b/common/json-partial.cpp @@ -0,0 +1,324 @@ +#include "json-partial.h" + +#include "log.h" + +#include + +#include +#include + +using json = nlohmann::ordered_json; + +enum common_json_stack_element_type { + COMMON_JSON_STACK_ELEMENT_OBJECT, + COMMON_JSON_STACK_ELEMENT_KEY, + COMMON_JSON_STACK_ELEMENT_ARRAY, +}; + +struct common_json_stack_element { + common_json_stack_element_type type; + std::string key; +}; + +bool common_json_parse( + const std::string & input, + const std::string & healing_marker, + common_json & out) +{ + std::string::const_iterator it = input.begin(); + const auto end = input.end(); + return common_json_parse(it, end, healing_marker, out); +} + +bool common_json_parse( + std::string::const_iterator & it, + const std::string::const_iterator & end, + const std::string & healing_marker, + common_json & out) +{ + // // https://json.nlohmann.me/features/parsing/sax_interface/ + struct json_error_locator : public nlohmann::json_sax { + std::size_t position; + bool found_error; + std::string last_token; + std::string exception_message; + std::vector stack; + + json_error_locator() : position(0), found_error(false) {} + + bool parse_error(std::size_t position, const std::string & last_token, const json::exception & ex) override { // NOLINT + this->position = position - 1; + this->found_error = true; + this->last_token = last_token; + this->exception_message = ex.what(); + return false; + } + void close_value() { + if (!stack.empty() && (stack.back().type == COMMON_JSON_STACK_ELEMENT_KEY)) { + stack.pop_back(); + } + } + bool null() override { // NOLINT + close_value(); + return true; + } + bool boolean(bool) override { // NOLINT + close_value(); + return true; + } + bool number_integer(number_integer_t) override { // NOLINT + close_value(); + return true; + } + bool number_unsigned(number_unsigned_t) override { // NOLINT + close_value(); + return true; + } + bool number_float(number_float_t, const string_t &) override { // NOLINT + close_value(); + return true; + } + bool string(string_t &) override { // NOLINT + close_value(); + return true; + } + bool binary(binary_t &) override { // NOLINT + close_value(); + return true; + } + bool start_object(std::size_t) override { // NOLINT + stack.push_back({COMMON_JSON_STACK_ELEMENT_OBJECT, ""}); + return true; + } + bool end_object() override { + GGML_ASSERT(!stack.empty() && stack.back().type == COMMON_JSON_STACK_ELEMENT_OBJECT); + stack.pop_back(); + close_value(); + return true; + } + bool key(string_t & key) override { // NOLINT + stack.push_back({COMMON_JSON_STACK_ELEMENT_KEY, key}); + return true; + } + bool start_array(std::size_t) override { // NOLINT + stack.push_back({COMMON_JSON_STACK_ELEMENT_ARRAY, ""}); + return true; + } + bool end_array() override { + GGML_ASSERT(!stack.empty() && stack.back().type == COMMON_JSON_STACK_ELEMENT_ARRAY); + stack.pop_back(); + close_value(); + return true; + } + }; + json_error_locator err_loc; + auto start = it; + json::sax_parse(it, end, &err_loc); + + if (err_loc.found_error) { + it = start; + auto temptative_end = it + err_loc.position; + // LOG_DBG("Error at position %zu (is_end = %s): %s\n", err_loc.position, temptative_end == end ? "true" : "false", err_loc.exception_message.c_str()); + + auto input = std::string(it, temptative_end); + try { + out.json = json::parse(input); + // out.json = json::parse(it, temptative_end); + it = temptative_end; + return true; + } catch (const std::exception & ex) { + // No, needs healing. + LOG_DBG("Failed to parse up to error: %s: <<<%s>>>\n", ex.what(), std::string(it, temptative_end).c_str()); + } + auto can_parse = [](const std::string & str) { + try { + auto _ = json::parse(str); // NOLINT + return true; + } catch (const std::exception &) { + return false; + } + }; + if (!healing_marker.empty() && !err_loc.stack.empty()) { + std::string str(it, temptative_end); + auto last_non_sp_pos = str.find_last_not_of(" \n\r\t"); + if (last_non_sp_pos == std::string::npos) { + throw std::runtime_error("Cannot heal a truncated JSON that stopped in an unknown location"); + } + auto last_non_sp_char = str[last_non_sp_pos]; + // Used to detect stops on a number, which may not be complete. + auto was_maybe_number = [&]() { + if (!str.empty() && std::isspace(str.back())) { + return false; + } + return std::isdigit(last_non_sp_char) || + last_non_sp_char == '.' || + last_non_sp_char == 'e' || + last_non_sp_char == 'E' || + last_non_sp_char == '-'; + }; + + std::string closing; + for (size_t i = err_loc.stack.size(); i > 0; i--) { + auto & el = err_loc.stack[i - 1]; + if (el.type == COMMON_JSON_STACK_ELEMENT_OBJECT) { + closing += "}"; + } else if (el.type == COMMON_JSON_STACK_ELEMENT_ARRAY) { + closing += "]"; + } else if (el.type != COMMON_JSON_STACK_ELEMENT_KEY) { + throw std::runtime_error("Unexpected stack element type"); + } + } + + // Matches a potentially partial unicode escape sequence, e.g. \u, \uX, \uXX, \uXXX, \uXXXX + static const std::regex partial_unicode_regex(R"(\\u(?:[0-9a-fA-F](?:[0-9a-fA-F](?:[0-9a-fA-F](?:[0-9a-fA-F])?)?)?)?$)"); + + auto is_high_surrogate = [&](const std::string & s) { + // Check if a partial of a high surrogate (U+D800-U+DBFF) + return s.length() >= 4 && + s[0] == '\\' && s[1] == 'u' && + std::tolower(s[2]) == 'd' && + (s[3] == '8' || s[3] == '9' || std::tolower(s[3]) == 'a' || std::tolower(s[3]) == 'b'); + }; + + // Initialize the unicode marker to a low surrogate to handle the edge case + // where a high surrogate (U+D800-U+DBFF) is immediately followed by a + // backslash (\) + std::string unicode_marker_padding = "udc00"; + std::smatch last_unicode_seq; + + if (std::regex_search(str, last_unicode_seq, partial_unicode_regex)) { + std::smatch second_last_seq; + std::string prelude = str.substr(0, last_unicode_seq.position()); + + // Pad the escape sequence with 0s until it forms a complete sequence of 6 characters + unicode_marker_padding = std::string(6 - last_unicode_seq.length(), '0'); + + if (is_high_surrogate(last_unicode_seq.str())) { + // If the sequence is a partial match for a high surrogate, add a low surrogate (U+DC00-U+UDFF) + unicode_marker_padding += "\\udc00"; + } else if (std::regex_search(prelude, second_last_seq, partial_unicode_regex)) { + if (is_high_surrogate(second_last_seq.str())) { + // If this follows a high surrogate, pad it to be a low surrogate + if (last_unicode_seq.length() == 2) { + unicode_marker_padding = "dc00"; + } else if (last_unicode_seq.length() == 3) { + unicode_marker_padding = "c00"; + } else { + // The original unicode_marker_padding is already padded with 0s + } + } + } + } + + const auto & magic_seed = out.healing_marker.marker = healing_marker;//"$llama.cpp.json$"; + + if (err_loc.stack.back().type == COMMON_JSON_STACK_ELEMENT_KEY) { + // We're inside an object value + if (last_non_sp_char == ':' && can_parse(str + "1" + closing)) { + // Was about to create an object value + str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; + } else if (can_parse(str + ": 1" + closing)) { + str += (out.healing_marker.json_dump_marker = ":\"" + magic_seed) + "\"" + closing; + } else if (last_non_sp_char == '{' && can_parse(str + closing)) { + // Was about to create an object + str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\": 1" + closing; + } else if (can_parse(str + "\"" + closing)) { + // Was inside an object value string + str += (out.healing_marker.json_dump_marker = magic_seed) + "\"" + closing; + } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\"" + closing)) { + // Was inside an object value string after an escape + str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\"" + closing; + } else if (can_parse(str + unicode_marker_padding + "\"" + closing)) { + // Was inside an object value string after a partial unicode escape + str += (out.healing_marker.json_dump_marker = unicode_marker_padding + magic_seed) + "\"" + closing; + } else { + // find last : + auto last_pos = str.find_last_of(':'); + if (last_pos == std::string::npos) { + throw std::runtime_error("Cannot heal a truncated JSON that stopped in an unknown location"); + } + // Cutting back to opening : for object value + str = str.substr(0, last_pos + 1) + (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; + } + } else if (err_loc.stack.back().type == COMMON_JSON_STACK_ELEMENT_ARRAY) { + if ((last_non_sp_char == ',' || last_non_sp_char == '[') && can_parse(str + "1" + closing)) { + // Was about to create an array value + str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; + } else if (can_parse(str + "\"" + closing)) { + // Was inside an array value string + str += (out.healing_marker.json_dump_marker = magic_seed) + "\"" + closing; + } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\"" + closing)) { + // Was inside an array value string after an escape + str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\"" + closing; + } else if (can_parse(str + unicode_marker_padding + "\"" + closing)) { + // Was inside an array value string after a partial unicode escape + str += (out.healing_marker.json_dump_marker = unicode_marker_padding + magic_seed) + "\"" + closing; + } else if (!was_maybe_number() && can_parse(str + ", 1" + closing)) { + // Had just finished a value + str += (out.healing_marker.json_dump_marker = ",\"" + magic_seed) + "\"" + closing; + } else { + auto last_pos = str.find_last_of("[,"); + if (last_pos == std::string::npos) { + throw std::runtime_error("Cannot heal a truncated JSON array stopped in an unknown location"); + } + // Cutting back to last [ or , for array value + str = str.substr(0, last_pos + 1) + (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; + } + } else if (err_loc.stack.back().type == COMMON_JSON_STACK_ELEMENT_OBJECT) { + if ((last_non_sp_char == '{' && can_parse(str + closing)) || + (last_non_sp_char == ',' && can_parse(str + "\"\": 1" + closing))) { + // Was about to create an object key+value + str += (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\": 1" + closing; + } else if (!was_maybe_number() && can_parse(str + ",\"\": 1" + closing)) { + // Was about to create an object key+value + str += (out.healing_marker.json_dump_marker = ",\"" + magic_seed) + "\": 1" + closing; + } else if (can_parse(str + "\": 1" + closing)) { + // Was inside an object key string + str += (out.healing_marker.json_dump_marker = magic_seed) + "\": 1" + closing; + } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\": 1" + closing)) { + // Was inside an object key string after an escape + str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\": 1" + closing; + } else if (can_parse(str + unicode_marker_padding + "\": 1" + closing)) { + // Was inside an object key string after a partial unicode escape + str += (out.healing_marker.json_dump_marker = unicode_marker_padding + magic_seed) + "\": 1" + closing; + } else { + auto last_pos = str.find_last_of(':'); + if (last_pos == std::string::npos) { + throw std::runtime_error("Cannot heal a truncated JSON object stopped in an unknown location"); + } + // fprintf(stderr, "Cutting back to last : for object key+value\n"); + str = str.substr(0, last_pos + 1) + (out.healing_marker.json_dump_marker = "\"" + magic_seed) + "\"" + closing; + } + } else { + throw std::runtime_error("Cannot heal a truncated JSON object stopped in an unknown location"); + } + // fprintf(stderr, "HEALED:\nSTRING <<<\n%s\n>>>\n\nmagic_cut: <<<\n%s\n>>>\n\n", str.c_str(), out.healing_marker.json_dump_marker.c_str()); + out.json = json::parse(str); + it = temptative_end; + return true; + } + // handle unclosed top-level primitive + if (err_loc.position != 0 && !healing_marker.empty() && err_loc.stack.empty()) { + std::string str(it, temptative_end); + const auto & magic_seed = out.healing_marker.marker = healing_marker; + if (can_parse(str + "\"")) { + // Was inside an string + str += (out.healing_marker.json_dump_marker = magic_seed) + "\""; + } else if (str[str.length() - 1] == '\\' && can_parse(str + "\\\"")) { + // Was inside an string after an escape + str += (out.healing_marker.json_dump_marker = "\\" + magic_seed) + "\""; + } else { + // TODO: handle more unclosed top-level primitive if the stack was empty but we got an error (e.g. "tru", "\"", etc...) + // fprintf(stderr, "Closing: TODO\n"); + return false; + } + out.json = json::parse(str); + it = temptative_end; + return true; + } + return false; + } + out.json = json::parse(it, end); + it = end; + return true; +} diff --git a/common/json-partial.h b/common/json-partial.h new file mode 100644 index 000000000..be51aabfb --- /dev/null +++ b/common/json-partial.h @@ -0,0 +1,39 @@ +#pragma once + +// TODO: use json_fwd.hpp when possible +#include + +// Healing marker (empty if the JSON was fully parsed / wasn't healed). +struct common_healing_marker { + // Raw marker. + std::string marker; + + // Cutting the `common_json.json.dump()` string at the (only) occurrence of this marker should yield the original partial JSON string (modulo spaces / if it had the same dump format). + std::string json_dump_marker; +}; + +// Represents a parsed JSON object, with its optional healing marker (a JSON dump fragment that can be used to find the position of healing in the JSON dump string) +struct common_json { + nlohmann::ordered_json json; + + common_healing_marker healing_marker; +}; + +// Parse the JSON string, healing (closing) any partial JSON if `healing_marker` is not empty. +// +// Healing completes partial JSON strings by adding a (possibly modified) healing marker, then whatever is needed to close the JSON. +// This allows to parse the resulting healed JSON string, yet be able to cut it again if needed at the healing marker. +// (this is used when parsing JSON outputs from the models, then crafting partial JSONs for the partial tool calls in OAI format). +// +// For instance, parsing `{` with a healing marker `foo` will produce a healed JSON `{"foo":1}`, w/ json_dump_marker = `"foo"` (which can be used to break the JSON again). +bool common_json_parse( + const std::string & input, + const std::string & healing_marker, + common_json & out); + +// Parse the JSON string (see overload above), but advancing an iterator to the end of the input when the (potentially partial) parsing succeeds. +bool common_json_parse( + std::string::const_iterator & it, + const std::string::const_iterator & end, + const std::string & healing_marker, + common_json & out); diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index b18607cd6..e2c4d6ce2 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -233,27 +233,27 @@ struct BuiltinRule { }; static std::unordered_map PRIMITIVE_RULES = { - {"boolean", {"(\"true\" | \"false\")", {}}}, + {"boolean", {"(\"true\" | \"false\") space", {}}}, {"decimal-part", {"[0-9]{1,16}", {}}}, {"integral-part", {"[0] | [1-9] [0-9]{0,15}", {}}}, - {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)?", {"integral-part", "decimal-part"}}}, - {"integer", {"(\"-\"? integral-part)", {"integral-part"}}}, + {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)? space", {"integral-part", "decimal-part"}}}, + {"integer", {"(\"-\"? integral-part) space", {"integral-part"}}}, {"value", {"object | array | string | number | boolean | null", {"object", "array", "string", "number", "boolean", "null"}}}, - {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? space \"}\"", {"string", "value"}}}, - {"array", {"\"[\" space ( value (\",\" space value)* )? space \"]\"", {"value"}}}, - {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\"", {}}}, + {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space", {"string", "value"}}}, + {"array", {"\"[\" space ( value (\",\" space value)* )? \"]\" space", {"value"}}}, + {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\" space", {}}}, {"char", {"[^\"\\\\\\x7F\\x00-\\x1F] | [\\\\] ([\"\\\\bfnrt] | \"u\" [0-9a-fA-F]{4})", {}}}, - {"string", {"\"\\\"\" char* \"\\\"\"", {"char"}}}, - {"null", {"\"null\"", {}}}, + {"string", {"\"\\\"\" char* \"\\\"\" space", {"char"}}}, + {"null", {"\"null\" space", {}}}, }; static std::unordered_map STRING_FORMAT_RULES = { {"date", {"[0-9]{4} \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )", {}}}, {"time", {"([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9]{3} )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )", {}}}, {"date-time", {"date \"T\" time", {"date", "time"}}}, - {"date-string", {"\"\\\"\" date \"\\\"\"", {"date"}}}, - {"time-string", {"\"\\\"\" time \"\\\"\"", {"time"}}}, - {"date-time-string", {"\"\\\"\" date-time \"\\\"\"", {"date-time"}}} + {"date-string", {"\"\\\"\" date \"\\\"\" space", {"date"}}}, + {"time-string", {"\"\\\"\" time \"\\\"\" space", {"time"}}}, + {"date-time-string", {"\"\\\"\" date-time \"\\\"\" space", {"date-time"}}} }; static bool is_reserved_name(const std::string & name) { @@ -551,16 +551,16 @@ private: } return join_seq(); }; - return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\""); + return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space"); } /* Returns a rule that matches a JSON string that is none of the provided strings not_strings({"a"}) - -> ["] ( [a] char+ | [^"a] char* )? ["] + -> ["] ( [a] char+ | [^"a] char* )? ["] space not_strings({"and", "also"}) - -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] + -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] space */ std::string _not_strings(const std::vector & strings) { @@ -619,7 +619,7 @@ private: if (!trie.is_end_of_string) { out << "?"; } - out << " [\"]"; + out << " [\"] space"; return out.str(); } @@ -725,7 +725,7 @@ private: rule += " )?"; } - rule += " space \"}\""; + rule += " \"}\" space"; return rule; } @@ -858,14 +858,14 @@ public: return _add_rule(rule_name, _generate_union_rule(name, schema_types)); } if (schema.contains("const")) { - return _add_rule(rule_name, _generate_constant_rule(schema["const"])); + return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space"); } if (schema.contains("enum")) { std::vector enum_values; for (const auto & v : schema["enum"]) { enum_values.push_back(_generate_constant_rule(v)); } - return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); + return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ") space"); } if ((schema_type.is_null() || schema_type == "object") && (schema.contains("properties") || @@ -933,7 +933,7 @@ public: } } if (!enum_intersection.empty()) { - return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); + return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ") space"); } } return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json())); @@ -948,7 +948,7 @@ public: } rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); } - rule += " space \"]\""; + rule += " \"]\" space"; return _add_rule(rule_name, rule); } std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); @@ -956,7 +956,7 @@ public: json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); int max_items = max_items_json.is_number_integer() ? max_items_json.get() : std::numeric_limits::max(); - return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\""); + return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space"); } if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { return _visit_pattern(schema["pattern"], rule_name); @@ -972,7 +972,7 @@ public: std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); int min_len = schema.contains("minLength") ? schema["minLength"].get() : 0; int max_len = schema.contains("maxLength") ? schema["maxLength"].get() : std::numeric_limits::max(); - return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\""); + return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space"); } if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) { int64_t min_value = std::numeric_limits::min(); @@ -990,7 +990,7 @@ public: std::stringstream out; out << "("; build_min_max_int(min_value, max_value, out); - out << ")"; + out << ") space"; return _add_rule(rule_name, out.str()); } if (schema.empty() || schema_type == "object") { diff --git a/common/log.cpp b/common/log.cpp index 2d1e74ad1..bd62616d8 100644 --- a/common/log.cpp +++ b/common/log.cpp @@ -11,13 +11,8 @@ #include #include #include -#include #if defined(_WIN32) -# define WIN32_LEAN_AND_MEAN -# ifndef NOMINMAX -# define NOMINMAX -# endif # include # include # define isatty _isatty @@ -67,15 +62,16 @@ static const char* g_col[] = { }; struct common_log_entry { - enum ggml_log_level level {GGML_LOG_LEVEL_INFO}; + enum ggml_log_level level; + + bool prefix; + + int64_t timestamp; std::vector msg; - int64_t timestamp { 0 }; - bool is_end { false }; // signals the worker thread to stop - bool prefix { false }; - - common_log_entry(size_t size = 256) : msg(size) { } + // signals the worker thread to stop + bool is_end; void print(FILE * file = nullptr) const { FILE * fcur = file; @@ -126,15 +122,22 @@ struct common_log_entry { }; struct common_log { - // default capacity - common_log(size_t capacity = 512) { - file = nullptr; - prefix = false; - timestamps = false; - running = false; - t_start = t_us(); + // default capacity - will be expanded if needed + common_log() : common_log(256) {} + + common_log(size_t capacity) { + file = nullptr; + prefix = false; + timestamps = false; + running = false; + t_start = t_us(); + + // initial message size - will be expanded if longer messages arrive + entries.resize(capacity); + for (auto & entry : entries) { + entry.msg.resize(256); + } - queue.resize(capacity, common_log_entry(256)); head = 0; tail = 0; @@ -149,10 +152,9 @@ struct common_log { } private: - std::mutex mtx; - std::thread thrd; - std::condition_variable cv_new; // new entry - std::condition_variable cv_full; // wait on full + std::mutex mtx; + std::thread thrd; + std::condition_variable cv; FILE * file; @@ -162,53 +164,24 @@ private: int64_t t_start; - // queue of entries - std::vector queue; + // ring buffer of entries + std::vector entries; size_t head; size_t tail; - bool print_entry(const common_log_entry & e) const { - if (e.is_end) return true; - - e.print(); - if (file) { - e.print(file); - } - return false; - } - - bool flush_queue(size_t start_head, size_t end_tail, size_t & out_head) const { - bool stop = false; - size_t h = start_head; - while (h != end_tail && !stop) { - stop = print_entry(queue[h]); - h = (h + 1) % queue.size(); - } - out_head = h; - return stop; - } + // worker thread copies into this + common_log_entry cur; public: - bool is_full() const { - return ((tail + 1) % queue.size()) == head; - } - - bool is_empty() const { - return head == tail; - } - void add(enum ggml_log_level level, const char * fmt, va_list args) { - std::unique_lock lock(mtx); - - // block if the queue is full - cv_full.wait(lock, [this]() { return !running || !is_full(); }); + std::lock_guard lock(mtx); if (!running) { // discard messages while the worker thread is paused return; } - auto & entry = queue[tail]; + auto & entry = entries[tail]; { // cannot use args twice, so make a copy in case we need to expand the buffer @@ -243,16 +216,38 @@ public: va_end(args_copy); } - entry.is_end = false; - entry.level = level; - entry.prefix = prefix; + entry.level = level; + entry.prefix = prefix; entry.timestamp = 0; if (timestamps) { entry.timestamp = t_us() - t_start; } + entry.is_end = false; - tail = (tail + 1) % queue.size(); - cv_new.notify_one(); + tail = (tail + 1) % entries.size(); + if (tail == head) { + // expand the buffer + std::vector new_entries(2*entries.size()); + + size_t new_tail = 0; + + do { + new_entries[new_tail] = std::move(entries[head]); + + head = (head + 1) % entries.size(); + new_tail = (new_tail + 1); + } while (head != tail); + + head = 0; + tail = new_tail; + + for (size_t i = tail; i < new_entries.size(); i++) { + new_entries[i].msg.resize(256); + } + + entries = std::move(new_entries); + } + cv.notify_one(); } void resume() { @@ -266,24 +261,23 @@ public: thrd = std::thread([this]() { while (true) { - std::unique_lock lock(mtx); - cv_new.wait(lock, [this]() { return !is_empty(); }); + { + std::unique_lock lock(mtx); + cv.wait(lock, [this]() { return head != tail; }); + cur = entries[head]; - size_t cached_head = head; - size_t cached_tail = tail; + head = (head + 1) % entries.size(); + } - lock.unlock(); // drop the lock during flush - - size_t next_head; - bool stop = flush_queue(cached_head, cached_tail, next_head); - - lock.lock(); - head = next_head; - cv_full.notify_all(); - - if (stop) { + if (cur.is_end) { break; } + + cur.print(); // stdout and stderr + + if (file) { + cur.print(file); + } } }); } @@ -299,13 +293,13 @@ public: running = false; // push an entry to signal the worker thread to stop - auto & entry = queue[tail]; - entry.is_end = true; - tail = (tail + 1) % queue.size(); + { + auto & entry = entries[tail]; + entry.is_end = true; - // wakeup everyone - cv_new.notify_one(); - cv_full.notify_all(); + tail = (tail + 1) % entries.size(); + } + cv.notify_one(); } thrd.join(); diff --git a/common/peg-parser.cpp b/common/peg-parser.cpp index 807e952d9..d4b491a80 100644 --- a/common/peg-parser.cpp +++ b/common/peg-parser.cpp @@ -6,14 +6,13 @@ #include "unicode.h" #include -#include #include #include #include #include #include -#include #include +#include // Trick to catch missing branches template @@ -89,7 +88,40 @@ struct trie { return match_result{match_result::NO_MATCH}; } + struct prefix_and_next { + std::vector prefix; + std::vector next_chars; + }; + + std::vector collect_prefix_and_next() { + std::vector prefix; + std::vector result; + collect_prefix_and_next(0, prefix, result); + return result; + } + private: + void collect_prefix_and_next(size_t index, std::vector & prefix, std::vector & out) { + if (!nodes[index].is_word) { + if (!nodes[index].children.empty()) { + std::vector chars; + chars.reserve(nodes[index].children.size()); + for (const auto & p : nodes[index].children) { + chars.push_back(p.first); + } + out.emplace_back(prefix_and_next{prefix, chars}); + } + } + + for (const auto & p : nodes[index].children) { + uint32_t ch = p.first; + auto child = p.second; + prefix.push_back(ch); + collect_prefix_and_next(child, prefix, out); + prefix.pop_back(); + } + } + size_t create_node() { size_t index = nodes.size(); nodes.emplace_back(); @@ -121,65 +153,6 @@ struct trie { } }; -// Aho-Corasick automaton -struct aho_corasick { - trie t; - std::vector fail; // failure links - std::vector order; // states in BFS order - std::vector terminal; // match states (directly or via a suffix link) - std::set alphabet; // every character with a transition - - aho_corasick(const std::vector & strings) : t(strings) { - const auto & nodes = t.nodes; - const size_t n = nodes.size(); - - fail.assign(n, 0); - order.reserve(n); - - std::deque queue{ 0 }; - while (!queue.empty()) { - size_t u = queue.front(); - queue.pop_front(); - order.push_back(u); - for (const auto & [ch, v] : nodes[u].children) { - if (u != 0) { - size_t f = fail[u]; - while (f && nodes[f].children.find(ch) == nodes[f].children.end()) { - f = fail[f]; - } - auto it = nodes[f].children.find(ch); - fail[v] = (it != nodes[f].children.end() && it->second != v) ? it->second : 0; - } - queue.push_back(v); - } - } - - terminal.assign(n, false); - for (size_t u : order) { - terminal[u] = nodes[u].is_word || (u != 0 && terminal[fail[u]]); - } - - for (const auto & node : nodes) { - for (const auto & [ch, v] : node.children) { - alphabet.insert(ch); - } - } - } - - size_t num_states() const { return t.nodes.size(); } - bool is_terminal(size_t s) const { return terminal[s]; } - - // follow failure links until a transition on `ch` exists. - size_t next(size_t state, uint32_t ch) const { - const auto & nodes = t.nodes; - while (state && nodes[state].children.find(ch) == nodes[state].children.end()) { - state = fail[state]; - } - auto it = nodes[state].children.find(ch); - return it != nodes[state].children.end() ? it->second : 0; - } -}; - static std::pair parse_hex_escape(const std::string & str, size_t pos, int hex_count) { if (pos + hex_count > str.length()) { return {0, 0}; @@ -921,10 +894,6 @@ struct parser_executor { common_peg_parse_result operator()(const common_peg_gbnf_parser & p) { return arena.parse(p.child, ctx, start_pos); } - - common_peg_parse_result operator()(const common_peg_ac_parser & p) { - return arena.parse(p.child, ctx, start_pos); - } }; common_peg_parse_result common_peg_arena::parse(common_peg_parse_context & ctx, size_t start) const { @@ -993,8 +962,7 @@ void common_peg_arena::resolve_refs() { std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || - std::is_same_v) { + std::is_same_v) { p.child = resolve_ref(p.child); } else if constexpr (std::is_same_v) { p.child = resolve_ref(p.child); @@ -1024,12 +992,12 @@ void common_peg_arena::resolve_refs() { } std::string common_peg_arena::dump(common_peg_parser_id id) const { - std::set visited; + std::unordered_set visited; return dump_impl(id, visited); } std::string common_peg_arena::dump_impl(common_peg_parser_id id, - std::set & visited) const { + std::unordered_set & visited) const { // Check for cycles if (visited.count(id)) { return "[cycle]"; @@ -1075,8 +1043,6 @@ std::string common_peg_arena::dump_impl(common_peg_parser_id return "Atomic(" + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Gbnf(" + p.grammar + ", " + dump_impl(p.child, visited) + ")"; - } else if constexpr (std::is_same_v) { - return "Ac(" + string_join(p.delimiters, " | ") + ", " + dump_impl(p.child, visited) + ")"; } else if constexpr (std::is_same_v) { return "Any"; } else if constexpr (std::is_same_v) { @@ -1376,7 +1342,7 @@ common_peg_parser common_peg_parser_builder::json_object() { common_peg_parser common_peg_parser_builder::json_array() { return rule("json-array", [this]() { auto ws = space(); - auto elements = sequence({json(), zero_or_more(sequence({ws, literal(","), ws, json()}))}); + auto elements = sequence({json(), zero_or_more(sequence({literal(","), ws, json()}))}); return sequence({ literal("["), ws, @@ -1486,13 +1452,6 @@ common_peg_parser common_peg_parser_builder::json_member(const std::string & key }); } -common_peg_parser common_peg_parser_builder::ac(const common_peg_parser & p, const std::vector & delimiters) { - if (delimiters.empty()) { - throw std::runtime_error("ac parser requires at least one delimiter"); - } - return add(common_peg_ac_parser{p, delimiters}); -} - static std::string gbnf_escape_char_class(uint32_t c) { if (c == '-' || c == ']' || c == '[' || c == '\\') { return "\\" + std::string(1, (char) c); @@ -1543,118 +1502,61 @@ static std::string gbnf_escape_char_class(uint32_t c) { return std::string(buf); } -static std::string gbnf_char_class(const std::vector & chars, bool negate) { - std::string s = negate ? "[^" : "["; - for (uint32_t ch : chars) { - s += gbnf_escape_char_class(ch); - } - return s + "]"; -} +static std::string gbnf_excluding_pattern(const std::vector & strings) { + trie matcher(strings); + auto pieces = matcher.collect_prefix_and_next(); -static std::string gbnf_ac_grammar( - const common_grammar_builder & builder, - const std::string & prefix, - const std::vector & strings, - const std::function &, - const std::map> &, - const std::vector &, - const std::function &)> & build_rule) { - aho_corasick ac(strings); - - auto state_name = [&](size_t s) -> std::string { - if (s == 0) { - return prefix; - } - std::string num = std::to_string(s); - num = num.size() == 1 ? ("0" + num) : num; - return prefix + "-" + num; - }; - - for (size_t q = 0; q < ac.num_states(); q++) { - if (ac.is_terminal(q)) { - continue; // match states + std::string pattern; + std::string trailing; // optional proper-prefix of a delimiter, allowed only at the very end + for (size_t i = 0; i < pieces.size(); ++i) { + if (i > 0) { + pattern += " | "; } - std::map> buckets; - std::vector completing; // chars that complete a delimiter - std::vector specific; // chars with an explicit transition - for (uint32_t c : ac.alphabet) { - size_t d = ac.next(q, c); - if (ac.is_terminal(d)) { - completing.push_back(c); - specific.push_back(c); - } else if (d != 0) { - buckets[d].push_back(c); // specific non-root destination - specific.push_back(c); - } + const auto & pre = pieces[i].prefix; + const auto & chars = pieces[i].next_chars; + + std::string cls; + cls.reserve(chars.size()); + for (uint32_t ch : chars) { + cls += gbnf_escape_char_class(ch); } - builder.add_rule(state_name(q), build_rule(completing, buckets, specific, state_name)); + if (!pre.empty()) { + std::string pre_literal = gbnf_format_literal(common_unicode_cpts_to_utf8(pre)); + pattern += pre_literal + " [^" + cls + "]"; + // Each interior alternative consumes a delimiter-prefix plus a disambiguating + // char, so the repetition alone cannot match a value that *ends* on a proper + // prefix of a delimiter (e.g. a trailing "\n" when the delimiter is + // "\n\n"). The runtime until() (greedy first-match) accepts such + // values, so without this the grammar would reject input the parser accepts. + // Allow the value to terminate on any proper prefix as an optional tail. + // This makes the grammar a slight superset of the runtime language (a value + // may end on the longest prefix, which greedy first-match would not itself + // produce); harmless for constrained generation, which only needs to admit + // every runtime-valid string. + if (!trailing.empty()) { + trailing += " | "; + } + trailing += pre_literal; + } else { + pattern += "[^" + cls + "]"; + } } - // An empty delimiter makes the start state terminal. Emit an entry rule - // that matches the empty string so the returned reference stays valid. - if (ac.is_terminal(0)) { - builder.add_rule(prefix, "|"); + std::string result = "(" + pattern + ")*"; + if (!trailing.empty()) { + result += " (" + trailing + ")?"; } - - return state_name(0); + return result; } -// GBNF grammar matching strings that contain no string in `strings` as a -// substring. Emits the complement of an Aho-Corasick automaton DFA and returns -// the start state rule name. -// -// ref: https://github.com/ggml-org/llama.cpp/pull/24839 -static std::string gbnf_excluding_grammar(const common_grammar_builder & builder, - const std::string & prefix, - const std::vector & strings) { - return gbnf_ac_grammar(builder, prefix, strings, - [](const std::vector & /*completing*/, - const std::map> & buckets, - const std::vector & specific, - const std::function & state_name) { - // every state is accepting and completing chars get no - // alternative, so a forbidden string can never be matched - std::string rhs = "|"; - for (const auto & [d, chars] : buckets) { - rhs += " " + gbnf_char_class(chars, false) + " " + state_name(d) + " |"; - } - rhs += " " + gbnf_char_class(specific, true) + " " + state_name(0); - return rhs; - }); -} - -// GBNF grammar matching everything up to and including the first occurrence of -// any string in `strings`. Emits the Aho-Corasick automaton DFA and returns -// the start state rule name. -static std::string gbnf_including_grammar(const common_grammar_builder & builder, - const std::string & prefix, - const std::vector & strings) { - return gbnf_ac_grammar(builder, prefix, strings, - [](const std::vector & completing, - const std::map> & buckets, - const std::vector & specific, - const std::function & state_name) { - std::vector alts; - if (!completing.empty()) { - alts.push_back(gbnf_char_class(completing, false)); // terminate on match - } - for (const auto & [d, chars] : buckets) { - alts.push_back(gbnf_char_class(chars, false) + " " + state_name(d)); - } - // every other character keeps scanning from the start state - alts.push_back(gbnf_char_class(specific, true) + " " + state_name(0)); - return string_join(alts, " | "); - }); -} - -static std::set collect_reachable_rules( +static std::unordered_set collect_reachable_rules( const common_peg_arena & arena, const common_peg_parser_id & rule ) { - std::set reachable; - std::set visited; + std::unordered_set reachable; + std::unordered_set visited; std::function visit = [&](common_peg_parser_id id) { const auto & parser = arena.get(id); @@ -1686,7 +1588,6 @@ static std::set collect_reachable_rules( std::is_same_v || std::is_same_v || std::is_same_v || - std::is_same_v || std::is_same_v) { visit(p.child); } else if constexpr (std::is_same_v) { @@ -1864,7 +1765,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo if (p.delimiters.empty()) { return ".*"; } - return gbnf_excluding_grammar(builder, "until-" + std::to_string(id), p.delimiters); + return gbnf_excluding_pattern(p.delimiters); } else if constexpr (std::is_same_v) { if (schema_delegates(p)) { return to_gbnf(p.child); @@ -1881,8 +1782,6 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo return to_gbnf(p.child); } else if constexpr (std::is_same_v) { return p.grammar; - } else if constexpr (std::is_same_v) { - return gbnf_including_grammar(builder, "ac-" + std::to_string(id), p.delimiters); } else { static_assert(is_always_false_v); } @@ -1890,7 +1789,7 @@ void common_peg_arena::build_grammar(const common_grammar_builder & builder, boo }; // Collect reachable rules - std::set reachable_rules; + std::unordered_set reachable_rules; if (lazy) { // Collect rules reachable from trigger rules @@ -2019,8 +1918,6 @@ static nlohmann::json serialize_parser_variant(const common_peg_parser_variant & }; } else if constexpr (std::is_same_v) { return json{{"type", "gbnf"}, {"child", p.child}, {"grammar", p.grammar}}; - } else if constexpr (std::is_same_v) { - return json{{"type", "ac"}, {"child", p.child}, {"delimiters", p.delimiters}}; } }, variant); } @@ -2193,16 +2090,6 @@ static common_peg_parser_variant deserialize_parser_variant(const nlohmann::json }; } - if (type == "ac") { - if (!j.contains("child") || !j.contains("delimiters") || !j["delimiters"].is_array() || j["delimiters"].empty()) { - throw std::runtime_error("ac parser requires 'child' and a non-empty 'delimiters' array"); - } - return common_peg_ac_parser{ - j["child"].get(), - j["delimiters"].get>(), - }; - } - throw std::runtime_error("Unknown parser type: " + type); } diff --git a/common/peg-parser.h b/common/peg-parser.h index c198499dd..b6bb05214 100644 --- a/common/peg-parser.h +++ b/common/peg-parser.h @@ -3,8 +3,8 @@ #include #include -#include #include +#include #include #include #include @@ -275,11 +275,6 @@ struct common_peg_gbnf_parser { std::string grammar; }; -struct common_peg_ac_parser { - common_peg_parser_id child; - std::vector delimiters; -}; - // Variant holding all parser types using common_peg_parser_variant = std::variant< common_peg_epsilon_parser, @@ -301,8 +296,7 @@ using common_peg_parser_variant = std::variant< common_peg_ref_parser, common_peg_atomic_parser, common_peg_tag_parser, - common_peg_gbnf_parser, - common_peg_ac_parser + common_peg_gbnf_parser >; class common_peg_arena { @@ -341,7 +335,7 @@ class common_peg_arena { friend class common_peg_parser_builder; private: - std::string dump_impl(common_peg_parser_id id, std::set & visited) const; + std::string dump_impl(common_peg_parser_id id, std::unordered_set & visited) const; common_peg_parser_id add_parser(common_peg_parser_variant parser); void add_rule(const std::string & name, common_peg_parser_id id); @@ -520,13 +514,6 @@ class common_peg_parser_builder { // the child's grammar. Parsing delegates entirely to the child. common_peg_parser gbnf(const common_peg_parser & p, const std::string & grammar) { return add(common_peg_gbnf_parser{p, grammar}); } - // Wraps a child parser but emits a GBNF grammar built from the Aho-Corasick - // automaton of `delimiters`, matching everything up to and including the - // first delimiter. Parsing delegates entirely to the child, which is - // responsible for consuming the delimiter (e.g. until(D) + literal(D)). - common_peg_parser ac(const common_peg_parser & p, const std::vector & delimiters); - common_peg_parser ac(const common_peg_parser & p, const std::string & delimiter) { return ac(p, std::vector{delimiter}); } - void set_root(const common_peg_parser & p); common_peg_arena build(); diff --git a/common/preset.cpp b/common/preset.cpp index 4362c0621..51ea984d8 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -7,7 +7,6 @@ #include #include #include -#include static std::string rm_leading_dashes(const std::string & str) { size_t pos = 0; @@ -17,21 +16,46 @@ static std::string rm_leading_dashes(const std::string & str) { return str.substr(pos); } -static std::string canonical_tag(const std::string & tag) { - static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); - std::smatch m; - if (std::regex_search(tag, m, re_tag)) { - std::string canon = m[1].str(); - for (char & c : canon) { - c = (char) std::toupper((unsigned char) c); +// only allow a subset of args for remote presets for security reasons +// do not add more args unless absolutely necessary +// args that output to files are strictly prohibited +static std::set get_remote_preset_whitelist(const std::map & key_to_opt) { + static const std::set allowed_options = { + "model-url", + "hf-repo", + "hf-repo-draft", + "hf-repo-v", // vocoder + "hf-file-v", // vocoder + "mmproj-url", + "pooling", + "jinja", + "batch-size", + "ubatch-size", + "cache-reuse", + "chat-template-kwargs", + "mmap", + // note: sampling params are automatically allowed by default + // negated args will be added automatically if the positive arg is specified above + }; + + std::set allowed_keys; + + for (const auto & it : key_to_opt) { + const std::string & key = it.first; + const common_arg & opt = it.second; + if (allowed_options.find(key) != allowed_options.end() || opt.is_sampling) { + allowed_keys.insert(key); + // also add variant keys (args without leading dashes and env vars) + for (const auto & arg : opt.get_args()) { + allowed_keys.insert(rm_leading_dashes(arg)); + } + for (const auto & env : opt.get_env()) { + allowed_keys.insert(env); + } } - return canon; } - std::string upper = tag; - for (char & c : upper) { - c = (char) std::toupper((unsigned char) c); - } - return upper; + + return allowed_keys; } std::vector common_preset::to_args(const std::string & bin_path) const { @@ -276,10 +300,16 @@ static std::string parse_bool_arg(const common_arg & arg, const std::string & ke return value; } -common_preset_context::common_preset_context(llama_example ex) +common_preset_context::common_preset_context(llama_example ex, bool only_remote_allowed) : ctx_params(common_params_parser_init(default_params, ex)) { common_params_add_preset_options(ctx_params.options); key_to_opt = get_map_key_opt(ctx_params); + + // setup allowed keys if only_remote_allowed is true + if (only_remote_allowed) { + filter_allowed_keys = true; + allowed_keys = get_remote_preset_whitelist(key_to_opt); + } } common_presets common_preset_context::load_from_ini(const std::string & path, common_preset & global) const { @@ -288,18 +318,11 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co for (auto section : ini_data) { common_preset preset; - std::string section_name = section.first.empty() ? std::string(COMMON_PRESET_DEFAULT_NAME) : section.first; - if (section_name != "*" && section_name != COMMON_PRESET_DEFAULT_NAME) { - auto colon_idx = section_name.rfind(':'); - if (colon_idx != std::string::npos) { - std::string tag = section_name.substr(colon_idx + 1); - std::string canon_tag = canonical_tag(tag); - if (canon_tag != tag) { - section_name = section_name.substr(0, colon_idx + 1) + canon_tag; - } - } + if (section.first.empty()) { + preset.name = COMMON_PRESET_DEFAULT_NAME; + } else { + preset.name = section.first; } - preset.name = section_name; LOG_DBG("loading preset: %s\n", preset.name.c_str()); for (const auto & [key, value] : section.second) { if (key == "version") { diff --git a/common/preset.h b/common/preset.h index 52935ebde..06f829c3e 100644 --- a/common/preset.h +++ b/common/preset.h @@ -60,7 +60,7 @@ struct common_preset_context { std::set allowed_keys; // if only_remote_allowed is true, only accept whitelisted keys - common_preset_context(llama_example ex); + common_preset_context(llama_example ex, bool only_remote_allowed = false); // load presets from INI file common_presets load_from_ini(const std::string & path, common_preset & global) const; diff --git a/common/reasoning-budget.cpp b/common/reasoning-budget.cpp index 7da0bb1c5..ce41d029b 100644 --- a/common/reasoning-budget.cpp +++ b/common/reasoning-budget.cpp @@ -65,12 +65,12 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to if (ctx->start_matcher.advance(token)) { ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; - COM_TRC("activated, budget=%d tokens\n", ctx->budget); + LOG_INF("reasoning-budget: activated, budget=%d tokens\n", ctx->budget); if (ctx->remaining <= 0) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; - COM_TRC("%s", "budget=0, forcing immediately\n"); + LOG_INF("reasoning-budget: budget=0, forcing immediately\n"); } } break; @@ -80,7 +80,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to { if (ctx->end_matcher.advance(token)) { ctx->state = REASONING_BUDGET_DONE; - COM_TRC("%s", "deactivated (natural end)\n"); + LOG_INF("reasoning-budget: deactivated (natural end)\n"); break; } @@ -95,7 +95,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - COM_TRC("%s", "UTF-8 complete, now forcing end sequence\n"); + LOG_INF("reasoning-budget: UTF-8 complete, now forcing end sequence\n"); } } else if (ctx->state == REASONING_BUDGET_COUNTING) { ctx->remaining--; @@ -104,11 +104,11 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - COM_TRC("%s", "budget exhausted, forcing end sequence\n"); + LOG_INF("reasoning-budget: budget exhausted, forcing end sequence\n"); } else { ctx->state = REASONING_BUDGET_WAITING_UTF8; ctx->end_matcher.reset(); - COM_TRC("%s", "budget exhausted, waiting for UTF-8 completion\n"); + LOG_INF("reasoning-budget: budget exhausted, waiting for UTF-8 completion\n"); } } } @@ -118,7 +118,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->force_pos++; if (ctx->force_pos >= ctx->forced_tokens.size()) { ctx->state = REASONING_BUDGET_DONE; - COM_TRC("%s", "forced sequence complete, done\n"); + LOG_INF("reasoning-budget: forced sequence complete, done\n"); } break; case REASONING_BUDGET_DONE: @@ -128,12 +128,12 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; ctx->end_matcher.reset(); - COM_TRC("re-activated on new start tag, budget=%d tokens\n", ctx->budget); + LOG_INF("reasoning-budget: re-activated on new start tag, budget=%d tokens\n", ctx->budget); if (ctx->remaining <= 0) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; - COM_TRC("%s", "budget=0, forcing immediately\n"); + LOG_INF("reasoning-budget: budget=0, forcing immediately\n"); } } break; @@ -264,7 +264,7 @@ bool common_reasoning_budget_force(struct llama_sampler * smpl) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - COM_TRC("%s", "forced into forcing state (manual transition)\n"); + LOG_INF("reasoning-budget: forced into forcing state (manual transition)\n"); return true; } diff --git a/common/regex-partial.cpp b/common/regex-partial.cpp new file mode 100644 index 000000000..bd9034e93 --- /dev/null +++ b/common/regex-partial.cpp @@ -0,0 +1,204 @@ +#include "regex-partial.h" +#include "common.h" +#include +#include + +common_regex::common_regex(const std::string & pattern) : + pattern(pattern), + rx(pattern), + rx_reversed_partial(regex_to_reversed_partial_regex(pattern)) {} + +common_regex_match common_regex::search(const std::string & input, size_t pos, bool as_match) const { + std::smatch match; + if (pos > input.size()) { + throw std::runtime_error("Position out of bounds"); + } + auto start = input.begin() + pos; + auto found = as_match + ? std::regex_match(start, input.end(), match, rx) + : std::regex_search(start, input.end(), match, rx); + if (found) { + common_regex_match res; + res.type = COMMON_REGEX_MATCH_TYPE_FULL; + for (size_t i = 0; i < match.size(); ++i) { + auto begin = pos + match.position(i); + res.groups.emplace_back(begin, begin + match.length(i)); + } + return res; + } + std::match_results srmatch; + if (std::regex_search(input.rbegin(), input.rend() - pos, srmatch, rx_reversed_partial, std::regex_constants::match_continuous)) { + auto group = srmatch[1].str(); + if (group.length() != 0) { + auto it = srmatch[1].second.base(); + // auto position = static_cast(std::distance(input.begin(), it)); + if ((!as_match) || it == input.begin()) { + common_regex_match res; + res.type = COMMON_REGEX_MATCH_TYPE_PARTIAL; + const size_t begin = std::distance(input.begin(), it); + const size_t end = input.size(); + if (begin == std::string::npos || end == std::string::npos || begin > end) { + throw std::runtime_error("Invalid range"); + } + res.groups.push_back({begin, end}); + return res; + } + } + } + return {}; +} + +/* + Transforms a regex pattern to a partial match pattern that operates on a reversed input string to find partial final matches of the original pattern. + + Ideally we'd like to use boost::match_partial (https://beta.boost.org/doc/libs/1_59_0/libs/regex/doc/html/boost_regex/partial_matches.html) + to see if a string ends with a partial regex match, but but it's not in std::regex yet. + Instead, we'll the regex into a partial match regex operating as a full match on the reverse iterators of the input. + + - /abcd/ -> ^(dcba|cba|ba|a) -> ^((?:(?:(?:(?:d)?c)?b)?a) + - /a|b/ -> ^(a|b) + - /a*?/ -> error, could match "" + - /a*b/ -> ^((?:b)?a*+) (final repetitions become eager) + - /.*?ab/ -> ^((?:b)?a) (omit .*) + - /a.*?b/ -> ^((?:b)?.*?a) (keep reluctant matches) + - /a(bc)d/ -> ^((?:(?:d)?(?:(?:c)?b))?a) + - /a(bc|de)/ -> ^((?:(?:(?:e)?d)?|(?:(?:c)?b)?)?a) + - /ab{2,4}c/ -> ^cbbb?b?a -> ^((?:(?:(?:(?:(?:c)?b)?b)?b?)?b?)?a) + + The regex will match a reversed string fully, and the end of the first (And only) capturing group will indicate the reversed start of the original partial pattern. + All other groups are turned into non-capturing groups, and reluctant quantifiers are ignored. +*/ +std::string regex_to_reversed_partial_regex(const std::string & pattern) { + auto it = pattern.begin(); + const auto end = pattern.end(); + + std::function process = [&]() { + std::vector> alternatives(1); + std::vector * sequence = &alternatives.back(); + + while (it != end) { + if (*it == '[') { + auto start = it; + ++it; + while (it != end) { + if ((*it == '\\') && (++it != end)) { + ++it; + } else if ((it != end) && (*it == ']')) { + break; + } else { + ++it; + } + } + if (it == end) { + throw std::runtime_error("Unmatched '[' in pattern"); + } + ++it; + sequence->push_back(std::string(start, it)); + } else if (*it == '*' || *it == '?' || *it == '+') { + if (sequence->empty()) { + throw std::runtime_error("Quantifier without preceding element"); + } + sequence->back() += *it; + auto is_star = *it == '*'; + ++it; + if (is_star) { + if (it != end && *it == '?') { + ++it; + } + } + } else if (*it == '{') { + if (sequence->empty()) { + throw std::runtime_error("Repetition without preceding element"); + } + ++it; + auto start = it; + while (it != end && *it != '}') { + ++it; + } + if (it == end) { + throw std::runtime_error("Unmatched '{' in pattern"); + } + auto parts = string_split(std::string(start, it), ","); + ++it; + if (parts.size() > 2) { + throw std::runtime_error("Invalid repetition range in pattern"); + } + + auto parseOptInt = [&](const std::string & s, const std::optional & def = std::nullopt) -> std::optional { + if (s.empty()) { + return def; + } + return std::stoi(s); + }; + auto min = parseOptInt(parts[0], 0); + auto max = parts.size() == 1 ? min : parseOptInt(parts[1]); + if (min && max && *max < *min) { + throw std::runtime_error("Invalid repetition range in pattern"); + } + // Brutal but... let's repeat at least min times, then ? for the delta between min & max (or * for unbounded) + auto part = sequence->back(); + sequence->pop_back(); + for (int i = 0; i < *min; i++) { + sequence->push_back(part); + } + if (max) { + for (int i = *min; i < *max; i++) { + sequence->push_back(part + "?"); + } + } else { + sequence->push_back(part + "*"); + } + } else if (*it == '(') { + ++it; + if (it != end && *it == '?' && (it + 1 != end) && *(it + 1) == ':') { + it += 2; + } + auto sub = process(); + if (*it != ')') { + throw std::runtime_error("Unmatched '(' in pattern"); + } + ++it; + auto & part = sequence->emplace_back("(?:"); + part += sub; + part += ")"; + } else if (*it == ')') { + break; + } else if (*it == '|') { + ++it; + alternatives.emplace_back(); + sequence = &alternatives.back(); + } else if (*it == '\\' && (++it != end)) { + auto str = std::string("\\") + *it; + sequence->push_back(str); + ++it; + } else if (it != end) { + sequence->push_back(std::string(1, *it)); + ++it; + } + } + + // /abcd/ -> ^(dcba|cba|ba|a) -> ^((?:(?:(?:d)?c)?b)?a) + // if n(=4) parts, opening n-1(=3) non-capturing groups after the 1 capturing group + // We'll do the outermost capturing group and final .* in the enclosing function. + std::vector res_alts; + for (const auto & parts : alternatives) { + auto & res = res_alts.emplace_back(); + for (size_t i = 0; i < parts.size() - 1; i++) { + res += "(?:"; + } + for (auto it = parts.rbegin(); it != parts.rend(); ++it) { + res += *it; + if (it != parts.rend() - 1) { + res += ")?"; + } + } + } + return string_join(res_alts, "|"); + }; + auto res = process(); + if (it != end) { + throw std::runtime_error("Unmatched '(' in pattern"); + } + + return "^(" + res + ")"; +} diff --git a/common/regex-partial.h b/common/regex-partial.h new file mode 100644 index 000000000..634cb4022 --- /dev/null +++ b/common/regex-partial.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +enum common_regex_match_type { + COMMON_REGEX_MATCH_TYPE_NONE, + COMMON_REGEX_MATCH_TYPE_PARTIAL, + COMMON_REGEX_MATCH_TYPE_FULL, +}; + +struct common_string_range { + size_t begin; + size_t end; + common_string_range(size_t begin, size_t end) : begin(begin), end(end) { + if (begin > end) { + throw std::runtime_error("Invalid range"); + } + } + // prevent default ctor + common_string_range() = delete; + bool empty() const { + return begin == end; + } + bool operator==(const common_string_range & other) const { + return begin == other.begin && end == other.end; + } +}; + +struct common_regex_match { + common_regex_match_type type = COMMON_REGEX_MATCH_TYPE_NONE; + std::vector groups; + + bool operator==(const common_regex_match & other) const { + return type == other.type && groups == other.groups; + } + bool operator!=(const common_regex_match & other) const { + return !(*this == other); + } +}; + +class common_regex { + std::string pattern; + std::regex rx; + std::regex rx_reversed_partial; + + public: + explicit common_regex(const std::string & pattern); + + common_regex_match search(const std::string & input, size_t pos, bool as_match = false) const; + + const std::string & str() const { return pattern; } +}; + +// For testing only (pretty print of failures). +std::string regex_to_reversed_partial_regex(const std::string & pattern); diff --git a/common/sampling.cpp b/common/sampling.cpp index 75a299e23..c537f3350 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -259,9 +259,6 @@ struct common_sampler * common_sampler_init(const struct llama_model * model, st } } } - if (!grmr && !grammar_str.empty()) { - throw std::runtime_error("failed to parse grammar"); - } // Compute prefill tokens from the generation prompt std::vector prefill_tokens; diff --git a/common/speculative.cpp b/common/speculative.cpp index 4e0439216..cc70894cb 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -18,13 +18,6 @@ #include #include -#define SPC_DBG(fmt, ...) LOG_DBG("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_TRC(fmt, ...) LOG_TRC("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_INF(fmt, ...) LOG_INF("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_WRN(fmt, ...) LOG_WRN("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_ERR(fmt, ...) LOG_ERR("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__) - #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128 #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 @@ -33,7 +26,6 @@ const std::map common_speculative_type_fro {"draft-simple", COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE}, {"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}, {"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP}, - {"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}, {"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, {"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, {"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, @@ -68,20 +60,21 @@ static bool common_speculative_are_compatible( const llama_vocab * vocab_dft = llama_model_get_vocab(model_dft); const auto vocab_type_tgt = llama_vocab_type(vocab_tgt); - SPC_DBG("vocab_type tgt: %d\n", vocab_type_tgt); + LOG_DBG("%s: vocab_type tgt: %d\n", __func__, vocab_type_tgt); const auto vocab_type_dft = llama_vocab_type(vocab_dft); - SPC_DBG("vocab_type dft: %d\n", vocab_type_dft); + LOG_DBG("%s: vocab_type dft: %d\n", __func__, vocab_type_dft); if (vocab_type_tgt != vocab_type_dft) { - SPC_WRN("draft model vocab type must match target model to use speculation but " - "vocab_type_dft = %d while vocab_type_tgt = %d\n", vocab_type_dft, vocab_type_tgt); + LOG_WRN("%s: draft model vocab type must match target model to use speculation but " + "vocab_type_dft = %d while vocab_type_tgt = %d\n", __func__, vocab_type_dft, vocab_type_tgt); return false; } if (llama_vocab_get_add_bos(vocab_tgt) != llama_vocab_get_add_bos(vocab_dft) || (llama_vocab_get_add_bos(vocab_tgt) && llama_vocab_bos(vocab_tgt) != llama_vocab_bos(vocab_dft))) { - SPC_WRN("draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + LOG_WRN("%s: draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + __func__, llama_vocab_get_add_bos(vocab_tgt), llama_vocab_get_add_bos(vocab_dft), llama_vocab_bos(vocab_tgt), llama_vocab_bos(vocab_dft)); return false; @@ -89,7 +82,8 @@ static bool common_speculative_are_compatible( if (llama_vocab_get_add_eos(vocab_tgt) != llama_vocab_get_add_eos(vocab_dft) || (llama_vocab_get_add_eos(vocab_tgt) && llama_vocab_eos(vocab_tgt) != llama_vocab_eos(vocab_dft))) { - SPC_WRN("draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + LOG_WRN("%s: draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + __func__, llama_vocab_get_add_eos(vocab_tgt), llama_vocab_get_add_eos(vocab_dft), llama_vocab_eos(vocab_tgt), llama_vocab_eos(vocab_dft)); return false; @@ -103,8 +97,8 @@ static bool common_speculative_are_compatible( : n_vocab_dft - n_vocab_tgt; if (vocab_diff > SPEC_VOCAB_MAX_SIZE_DIFFERENCE) { - SPC_DBG("draft model vocab must closely match target model to use speculation but " - "target vocab size %d does not match draft vocab size %d - difference %d, max allowed %d\n", + LOG_DBG("%s: draft model vocab must closely match target model to use speculation but ", __func__); + LOG_DBG("target vocab size %d does not match draft vocab size %d - difference %d, max allowed %d\n", n_vocab_tgt, llama_vocab_n_tokens(vocab_dft), vocab_diff, SPEC_VOCAB_MAX_SIZE_DIFFERENCE); return false; } @@ -114,8 +108,8 @@ static bool common_speculative_are_compatible( const char * token_text_dft = llama_vocab_get_text(vocab_dft, i); if (std::strcmp(token_text_tgt, token_text_dft) != 0) { - SPC_DBG("draft model vocab must match target model to use speculation but " - "token %d content differs - target '%s', draft '%s'\n", i, + LOG_DBG("%s: draft model vocab must match target model to use speculation but ", __func__); + LOG_DBG("token %d content differs - target '%s', draft '%s'\n", i, common_token_to_piece(vocab_tgt, i).c_str(), common_token_to_piece(vocab_dft, i).c_str()); return false; @@ -167,10 +161,6 @@ struct common_speculative_impl { virtual void accept(llama_seq_id seq_id, uint16_t n_accepted, bool is_other) = 0; - // (optional) serialize/restore per-seq internal state (e.g. eagle3's deferred boundary). - virtual bool get_state(llama_seq_id /*seq_id*/, std::vector & /*data*/) const { return false; } - virtual void set_state(llama_seq_id /*seq_id*/, const std::vector & /*data*/) {} - // true if this implementation requires the target context to extract post-norm embeddings virtual bool need_embd() const = 0; @@ -192,9 +182,9 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; auto * ctx_tgt = this->params.ctx_tgt; - SPC_TRC("%s", "adding speculative implementation 'draft-simple'\n"); - SPC_TRC("- n_max=%d, n_min=%d, p_min=%f\n", this->params.n_max, this->params.n_min, this->params.p_min); - SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", + LOG_INF("%s: adding speculative implementation 'draft-simple'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); + LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, this->params.n_gpu_layers, ggml_type_name(this->params.cache_type_k), ggml_type_name(this->params.cache_type_v), @@ -234,16 +224,16 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { } const bool vocab_cmpt = common_speculative_are_compatible(llama_get_model(ctx_tgt), llama_get_model(ctx_dft)); - SPC_DBG("vocab_cmpt = %d\n", vocab_cmpt); + LOG_DBG("%s: vocab_cmpt = %d\n", __func__, vocab_cmpt); if (!vocab_cmpt) { - SPC_ERR("%s", "the target and draft vocabs are not compatible\n"); + LOG_ERR("%s: the target and draft vocabs are not compatible\n", __func__); throw std::runtime_error("draft model vocab type must match target model to use speculation"); } if (n_seq != llama_n_seq_max(ctx_dft)) { - SPC_ERR("n_seq mismatch: %d != %d\n", n_seq, llama_n_seq_max(ctx_dft)); + LOG_ERR("%s: n_seq mismatch: %d != %d\n", __func__, n_seq, llama_n_seq_max(ctx_dft)); throw std::runtime_error("the draft model number of sequences is incompatible with the speculative n_seq"); } @@ -263,7 +253,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { const int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("failed to decode draft batch, ret = %d\n", ret); + LOG_ERR("%s: failed to decode draft batch, ret = %d\n", __func__, ret); return false; } @@ -296,7 +286,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode returned %d\n", ret); + LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); return; } @@ -320,7 +310,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -360,7 +350,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { // evaluate the drafted tokens on the draft model ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode[%d] returned %d\n", i, ret); + LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); break; } @@ -455,8 +445,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq) , params(params.draft) { - SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n"); - SPC_TRC("- n_max=%d, n_min=%d, p_min=%f, backend_sampling=%d\n", params.draft.n_max, params.draft.n_min, params.draft.p_min, (int) params.draft.backend_sampling); + LOG_INF("%s: adding speculative implementation 'draft-eagle3'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f, backend_sampling=%d\n", __func__, params.draft.n_max, params.draft.n_min, params.draft.p_min, (int) params.draft.backend_sampling); auto * ctx_tgt = this->params.ctx_tgt; auto * ctx_dft = this->params.ctx_dft; @@ -499,7 +489,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); if (!llama_set_sampler(ctx_dft, seq_id, chain)) { - SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); + LOG_WRN("%s: backend offload failed for seq_id=%d; using CPU sampler\n", __func__, (int) seq_id); llama_sampler_free(chain); chain = nullptr; } @@ -554,9 +544,9 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); if (pos_max < N - 2) { - SPC_WRN("ctx_dft pos_max=%d < N-2=%d — process() did not run on every prefill ubatch. " + LOG_WRN("%s: ctx_dft pos_max=%d < N-2=%d — process() did not run on every prefill ubatch. " "Drafts may degrade.\n", - (int) pos_max, N - 2); + __func__, (int) pos_max, N - 2); } } @@ -627,8 +617,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { }; const int32_t rc = llama_encode(ctx_dft, enc_batch); if (rc != 0) { - SPC_ERR("llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", - rc, (int) n_chunk, (int) i); + LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", + __func__, rc, (int) n_chunk, (int) i); return false; } @@ -698,8 +688,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { if (batch.n_tokens > 0) { const int32_t rc = llama_decode(ctx_dft, batch); if (rc != 0) { - SPC_ERR("llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, ubatch_pos[0]=%d)\n", - rc, (int) batch.n_tokens, (int) batch_in.pos[0]); + LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, ubatch_pos[0]=%d)\n", + __func__, rc, (int) batch.n_tokens, (int) batch_in.pos[0]); return false; } } @@ -750,7 +740,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode returned %d\n", ret); + LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); return; } @@ -776,7 +766,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -815,7 +805,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode[%d] returned %d\n", i, ret); + LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); break; } @@ -851,348 +841,6 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { (size_t) n_embd_dec * sizeof(float)); } - // we only need to stash the deferred boundary's g_embd row for recurrent/hybrid targets: - // their single-position checkpoints drop it on restore - bool need_boundary_stash() const { - const llama_model * model_tgt = llama_get_model(params.ctx_tgt); - return llama_model_is_recurrent(model_tgt) || llama_model_is_hybrid(model_tgt); - } - - bool get_state(llama_seq_id seq_id, std::vector & data) const override { - if (!need_boundary_stash()) { - return false; - } - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq || pending_pos_last[seq_id] < 0) { - return false; - } - - const llama_pos pos = pending_pos_last[seq_id]; - const std::vector & g = pending_g_last[seq_id]; - - data.resize(sizeof(llama_pos) + g.size() * sizeof(float)); - std::memcpy(data.data(), &pos, sizeof(llama_pos)); - std::memcpy(data.data() + sizeof(llama_pos), g.data(), g.size() * sizeof(float)); - return true; - } - - void set_state(llama_seq_id seq_id, const std::vector & data) override { - if (!need_boundary_stash()) { - return; - } - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { - return; - } - if (data.size() != sizeof(llama_pos) + (size_t) n_embd_dec * sizeof(float)) { - return; - } - - llama_pos pos = -1; - std::memcpy(&pos, data.data(), sizeof(llama_pos)); - - pending_pos_last[seq_id] = pos; - pending_g_last[seq_id].resize(n_embd_dec); - std::memcpy(pending_g_last[seq_id].data(), data.data() + sizeof(llama_pos), (size_t) n_embd_dec * sizeof(float)); - } - - bool need_embd() const override { - return false; - } -}; - -// DFlash: block-diffusion drafting with a draft-side KV cache injection -struct common_speculative_impl_draft_dflash : public common_speculative_impl { - common_params_speculative_draft params; - - llama_batch batch; // noise tokens - llama_batch batch_inject; // target features for KV cache injection - - std::vector smpls; - - int32_t n_embd_dec = 0; // draft hidden size - int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size - int32_t n_embd_tgt = 0; // target model hidden size - - int32_t block_size = 0; - llama_token mask_token_id = 0; - - const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices - uint32_t target_layer_ids_n = 0; - - // scratch buffer for concatenated target features [n_tokens, n_embd_enc] - std::vector features_buf; - - common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq) - , params(params.draft) - { - auto * ctx_tgt = this->params.ctx_tgt; - auto * ctx_dft = this->params.ctx_dft; - GGML_ASSERT(ctx_tgt && ctx_dft && "DFlash requires ctx_tgt and ctx_dft to be set"); - - const llama_model * model_dft = llama_get_model(ctx_dft); - const llama_model * model_tgt = llama_get_model(ctx_tgt); - - target_layer_ids = llama_model_target_layer_ids (model_dft); - target_layer_ids_n = llama_model_target_layer_ids_n(model_dft); - GGML_ASSERT(target_layer_ids_n > 0 && "DFlash model has no target_layer_ids"); - - n_embd_tgt = llama_model_n_embd(model_tgt); - n_embd_dec = llama_model_n_embd(model_dft); - n_embd_enc = (int32_t) target_layer_ids_n * n_embd_tgt; - - // read the trained block size from the dflash.block_size metadata key - block_size = 16; - { - char buf[32] = {}; - if (llama_model_meta_val_str(model_dft, "dflash.block_size", buf, sizeof(buf)) >= 0) { - block_size = std::atoi(buf); - } - } - mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft)); - - LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); - LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n); - - // DFlash input is [id_last, * (block_size-1)], so it can draft at most block_size-1 tokens per step - if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) { - LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n", - __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1); - this->params.n_max = std::min(this->params.n_max, block_size - 1); - this->params.n_min = std::min(this->params.n_min, block_size - 1); - } - - batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq); - batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq); - - smpls.resize(n_seq); - for (auto & s : smpls) { - common_params_sampling sparams; - sparams.no_perf = false; - sparams.top_k = 10; - sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K }; - s.reset(common_sampler_init(model_dft, sparams)); - } - - // turn on extraction of the target layers' input embeddings - for (uint32_t k = 0; k < target_layer_ids_n; ++k) { - llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); - } - - llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); - llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention - } - - ~common_speculative_impl_draft_dflash() override { - llama_batch_free(batch); - llama_batch_free(batch_inject); - } - - void begin(llama_seq_id seq_id, const llama_tokens & prompt) override { - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { - return; - } - - const int32_t N = (int32_t) prompt.size(); - if (N <= 0) { - return; - } - - const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(params.ctx_dft), seq_id); - if (pos_max < N - 1) { - LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - process() did not run on every prefill ubatch. " - "Drafts may degrade.\n", - __func__, (int) pos_max, N - 1); - } - } - - bool process(const llama_batch & batch_in) override { - if (batch_in.n_tokens <= 0) { - return true; - } - - if (batch_in.token == nullptr || batch_in.embd != nullptr) { - return true; - } - - const int32_t n_tokens = batch_in.n_tokens; - - // per-seq inclusive batch range (assumes each seq's tokens are contiguous in the batch) - std::vector i_batch_beg(n_seq, -1); - std::vector i_batch_end(n_seq, -1); - for (int32_t k = 0; k < n_tokens; ++k) { - GGML_ASSERT(batch_in.n_seq_id[k] == 1); - const llama_seq_id seq_id = batch_in.seq_id[k][0]; - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { - continue; - } - i_batch_end[seq_id] = k; - if (i_batch_beg[seq_id] < 0) { - i_batch_beg[seq_id] = k; - } - } - - auto * ctx_tgt = this->params.ctx_tgt; - auto * ctx_dft = this->params.ctx_dft; - - const int32_t n_ubatch = (int32_t) llama_n_ubatch(ctx_dft); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (i_batch_beg[seq_id] < 0) { - continue; - } - const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; - - for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) { - const int32_t n_chunk = std::min(n_ubatch, n_rows - offset); - - // gather this chunk's target features, interleaved by extract layer - features_buf.resize((size_t) n_chunk * n_embd_enc); - for (uint32_t k = 0; k < target_layer_ids_n; ++k) { - const float * layer = llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]); - if (!layer) { - GGML_ABORT("DFlash: target layer %d input not extracted.", target_layer_ids[k]); - } - for (int32_t i = 0; i < n_chunk; ++i) { - float * dst = features_buf.data() + (size_t) i * n_embd_enc + k * (size_t) n_embd_tgt; - const float * src = layer + (size_t) (i_batch_beg[seq_id] + offset + i) * n_embd_tgt; - std::memcpy(dst, src, (size_t) n_embd_tgt * sizeof(float)); - } - } - - // fuse extracted features through DFlash encoder - llama_batch enc_batch = { - /*.n_tokens =*/ n_chunk, - /*.token =*/ nullptr, - /*.embd =*/ features_buf.data(), - /*.pos =*/ nullptr, - /*.n_seq_id =*/ nullptr, - /*.seq_id =*/ nullptr, - /*.logits =*/ nullptr, - }; - - int32_t rc = llama_encode(ctx_dft, enc_batch); - if (rc != 0) { - LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", - __func__, rc, (int) n_chunk, (int) offset); - return false; - } - - const float * inp_g = llama_get_embeddings_nextn(ctx_dft); - GGML_ASSERT(inp_g && "DFlash encoder produced no output."); - - // inject the DFlash decoder K/V cache at the tokens' target positions - batch_inject.n_tokens = n_chunk; - std::memcpy(batch_inject.embd, inp_g, (size_t) n_chunk * n_embd_dec * sizeof(float)); - - for (int32_t i = 0; i < n_chunk; ++i) { - batch_inject.pos[i] = batch_in.pos[i_batch_beg[seq_id] + offset + i]; - batch_inject.n_seq_id[i] = 1; - batch_inject.seq_id[i][0] = seq_id; - batch_inject.logits[i] = false; - } - rc = llama_decode(ctx_dft, batch_inject); - if (rc != 0) { - LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", - __func__, rc, (int) n_chunk, (int) offset); - return false; - } - } - } - - return true; - } - - void draft(common_speculative_draft_params_vec & dparams) override { - auto & ctx_dft = params.ctx_dft; - - common_batch_clear(batch); - - // build one batch holding every drafting sequence's noise block into a single decode) - // record where each block starts and its size - std::vector i_block_beg(n_seq, -1); - std::vector n_block (n_seq, 0); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } - - common_sampler_reset(smpls[seq_id].get()); - - const int32_t n = (int32_t) dp.n_past; - - int32_t n_draft = params.n_max; - if (dp.n_max > 0) { - n_draft = std::min(n_draft, dp.n_max); - } - - const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * - i_block_beg[seq_id] = batch.n_tokens; - n_block [seq_id] = n_block_tokens; - for (int32_t i = 0; i < n_block_tokens; ++i) { - common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, true); - } - } - - if (batch.n_tokens == 0) { - return; - } - - // decode all sequence's noise block in a single batch - int ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); - return; - } - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (i_block_beg[seq_id] < 0) { - continue; - } - auto & dp = dparams[seq_id]; - - const int32_t beg = i_block_beg[seq_id]; - const int32_t n_block_tokens = n_block[seq_id]; - - auto * smpl = smpls[seq_id].get(); - - auto & result = *dp.result; - - // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1 - for (int32_t i = 1; i < n_block_tokens; ++i) { - common_sampler_sample(smpl, ctx_dft, beg + i, true); - - const auto * cur_p = common_sampler_get_candidates(smpl, true); - - for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", - seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p, - common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); - } - - const llama_token id = cur_p->data[0].id; - - if (cur_p->data[0].p < params.p_min) { - break; - } - - common_sampler_accept(smpl, id, true); - - result.push_back(id); - } - - if (result.size() < (size_t) params.n_min) { - result.clear(); - } - } - } - - void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { - // noop - } - bool need_embd() const override { return false; } @@ -1210,13 +858,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { int32_t n_embd = 0; - // One MTP draft driver, three modes (set once in the ctor): - // is_mem_shared (gemma4): shares the target KV, runs all heads in one graph. - // chain_heads (step35): n_mtp_layers trained heads, one per draft step. - // neither (qwen35 / qwen35moe): a single trained MTP head. - int32_t n_mtp_layers = 1; - bool is_mem_shared = false; // gemma4 - bool chain_heads = false; // derived in the ctor: n_mtp_layers > 1 && !is_mem_shared + bool is_mem_shared = false; // Per-sequence cross-batch carryover: pair (h_p, x_{p+1}) at MTP pos p+1. // The last h-row of one process() call needs the first token of the NEXT @@ -1231,8 +873,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::vector> verify_h; std::vector verify_h_rows; - std::vector i_last; - std::vector> chain_h; + // Per-seq draft length from the last draft() call, used in accept() to + // roll back ctx_dft's recurrent state past the AR draft's redundant + // pre-advancement before process() mirrored the verify batch. + std::vector last_n_drafted; common_speculative_impl_draft_mtp(const common_params_speculative & params, uint32_t n_seq) : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, n_seq) @@ -1245,11 +889,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { n_embd = llama_model_n_embd_out(llama_get_model(ctx_dft)); GGML_ASSERT(n_embd == llama_model_n_embd(llama_get_model(ctx_tgt)) && "MTP input row width must match the target h_nextn width"); - n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); - SPC_TRC("%s", "adding speculative implementation 'draft-mtp'\n"); - SPC_TRC("- n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); - SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", + LOG_INF("%s: adding speculative implementation 'draft-mtp'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); + LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, this->params.n_gpu_layers, ggml_type_name(this->params.cache_type_k), ggml_type_name(this->params.cache_type_v), @@ -1280,7 +923,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); if (!llama_set_sampler(ctx_dft, seq_id, chain)) { - SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); + LOG_WRN("%s: backend offload failed for seq_id=%d; using CPU sampler\n", __func__, (int) seq_id); llama_sampler_free(chain); chain = nullptr; } @@ -1292,25 +935,16 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); is_mem_shared = llama_get_ctx_other(ctx_dft) == ctx_tgt; - chain_heads = n_mtp_layers > 1 && !is_mem_shared; - - if (chain_heads) { - this->params.n_max = std::min(this->params.n_max, n_mtp_layers); - - chain_h.assign(n_seq, {}); - for (auto & c : chain_h) { - c.reserve((size_t) (this->params.n_max + 1) * n_embd); - } - } pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); - i_last.assign(n_seq, -1); i_batch_beg.assign(n_seq, -1); i_batch_end.assign(n_seq, -1); verify_h.assign(n_seq, {}); verify_h_rows.assign(n_seq, 0); + + last_n_drafted.assign(n_seq, 0); } ~common_speculative_impl_draft_mtp() override { @@ -1343,11 +977,11 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); if (pos_max < N - 1 && !is_mem_shared) { - SPC_WRN("ctx_dft pos_max=%d < N-1=%d - " + LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - " "process() hook may not have run on every prefill ubatch " "(need_embd / logits=1 on every prompt position?). " "Drafts may degrade.\n", - (int) pos_max, N - 1); + __func__, (int) pos_max, N - 1); } } @@ -1416,34 +1050,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { set_h(i_batch_beg[seq_id], pending_h[seq_id].data()); } - auto * mem_dft = llama_get_memory(ctx_dft); - - bool ok = true; - for (int head = 0; head < n_mtp_layers; ++head) { - if (chain_heads) { - // ref: https://github.com/ggml-org/llama.cpp/pull/24340/changes#r3413498544 - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (i_batch_beg[seq_id] < 0) { - continue; - } - llama_memory_seq_rm(mem_dft, seq_id, batch_in.pos[i_batch_beg[seq_id]], -1); - } - llama_set_nextn_layer_offset(ctx_dft, head); - } - - const int32_t rc = llama_decode(ctx_dft, batch); - if (rc != 0) { - SPC_ERR("llama_decode(ctx_dft) head=%d failed rc=%d (pos=%d)\n", - head, (int) rc, (int) batch_in.pos[0]); - ok = false; - break; - } - } - - if (chain_heads) { - llama_set_nextn_layer_offset(ctx_dft, 0); // restore default for non-draft decodes - } - if (!ok) { + const int32_t rc = llama_decode(ctx_dft, batch); + if (rc != 0) { + LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (pos=%d)\n", __func__, (int) rc, (int) batch_in.pos[0]); return false; } } @@ -1478,6 +1087,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { int n_drafting = 0; std::vector drafting(n_seq); + const float * h_row = nullptr; const size_t row_bytes = (size_t) n_embd * sizeof(float); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { @@ -1492,43 +1102,22 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { common_sampler_reset(smpls[seq_id].get()); common_batch_add(batch, dp.id_last, dp.n_past, { seq_id }, true); - std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[seq_id].data(), row_bytes); - i_last[seq_id] = batch.n_tokens - 1; + h_row = pending_h[seq_id].data(); + std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); + } - if (chain_heads) { - chain_h[seq_id].assign(pending_h[seq_id].begin(), pending_h[seq_id].end()); - } + int ret = llama_decode(ctx_dft, batch); + if (ret != 0) { + LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); + return; } int i = 0; while (n_drafting > 0) { - // each step decodes under a different head, i.e. a different decoder layer, and - // KV is per layer. process() filled this layer's KV only for positions < n_past - // (prompt + accepted prefix) — nothing in the draft region yet. so reset the - // draft region (the seq_rm lower bound is n_past, leaving the prompt KV intact) - // and select head i so it rebuilds its own layer's KV there; decoding just the - // latest token would leave its attention reading cells only another head wrote. - if (chain_heads) { - auto * mem_dft = llama_get_memory(ctx_dft); - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (drafting[seq_id]) { - llama_memory_seq_rm(mem_dft, seq_id, dparams[seq_id].n_past, -1); - } - } - llama_set_nextn_layer_offset(ctx_dft, i); - } + int i_batch = 0; - int ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - SPC_ERR("llama_decode[%d] returned %d\n", i, ret); - break; - } - - // rebuild the batch for the next step: the growing-KV paths re-add only the - // new token (the KV already holds the prefix), while chained heads re-add the - // whole prefix at the next head. dropped sequences are simply not re-added. common_batch_clear(batch); for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { @@ -1538,13 +1127,14 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { auto * smpl = smpls[seq_id].get(); - common_sampler_sample(smpl, ctx_dft, i_last[seq_id], true); - const float * h_row = llama_get_embeddings_nextn_ith(ctx_dft, i_last[seq_id]); + common_sampler_sample(smpl, ctx_dft, i_batch, true); + h_row = llama_get_embeddings_nextn_ith(ctx_dft, i_batch); + ++i_batch; const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -1573,39 +1163,28 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { continue; } - if (chain_heads) { - // ref: https://github.com/ggml-org/llama.cpp/pull/24340#discussion_r3448031546 - chain_h[seq_id].insert(chain_h[seq_id].end(), h_row, h_row + n_embd); - - const int n_rows = (int) result.size() + 1; // id_last + tokens drafted so far - for (int t = 0; t < n_rows; ++t) { - const llama_token tok = (t == 0) ? dp.id_last : result[t - 1]; - common_batch_add(batch, tok, dp.n_past + t, { seq_id }, t == n_rows - 1); - std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, - chain_h[seq_id].data() + (size_t) t * n_embd, row_bytes); - } - } else if (is_mem_shared) { + if (is_mem_shared) { // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37 common_batch_add(batch, id, dp.n_past, { seq_id }, true); - std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } else { common_batch_add(batch, id, dp.n_past + i + 1, { seq_id }, true); - std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, h_row, row_bytes); } - - i_last[seq_id] = batch.n_tokens - 1; + std::memcpy(batch.embd + n_embd*(batch.n_tokens - 1), h_row, row_bytes); } if (batch.n_tokens == 0) { break; } - ++i; - } + // evaluate the drafted tokens on the draft model + ret = llama_decode(ctx_dft, batch); + if (ret != 0) { + LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); + break; + } - if (chain_heads) { - llama_set_nextn_layer_offset(ctx_dft, 0); // restore default for non-draft decodes + ++i; } for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { @@ -1617,6 +1196,8 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { if (dp.result->size() < (size_t) params.n_min) { dp.result->clear(); } + + last_n_drafted[seq_id] = (uint16_t) dp.result->size(); } } @@ -1658,8 +1239,8 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl { , params(params.ngram_simple) , config(config) { - SPC_TRC("%s", "adding speculative implementation 'ngram-simple'\n"); - SPC_TRC("- size_n=%d, size_m=%d, min_hits=%d\n", + LOG_INF("%s: adding speculative implementation 'ngram-simple'\n", __func__); + LOG_INF("%s: - size_n=%d, size_m=%d, min_hits=%d\n", __func__, this->params.size_n, this->params.size_m, this->params.min_hits); } @@ -1708,8 +1289,8 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl { this->config.push_back(config); } - SPC_TRC("adding speculative implementation '%s'\n", common_speculative_type_to_str(this->type).c_str()); - SPC_TRC("- size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", + LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(this->type).c_str()); + LOG_INF("%s: - size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", __func__, config.size_key, config.size_value, config.key_only, config.min_hits); } @@ -1783,15 +1364,15 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { , verbose(std::getenv("LLAMA_TRACE") != nullptr) { static_assert(sizeof(llama_token) == sizeof(common_ngram_mod::entry_t)); - SPC_TRC("%s", "adding speculative implementation 'ngram-mod'\n"); - SPC_TRC("- n_match=%d, n_max=%d, n_min=%d\n", + LOG_INF("%s: adding speculative implementation 'ngram-mod'\n", __func__); + LOG_INF("%s: - n_match=%d, n_max=%d, n_min=%d\n", __func__, this->params.n_match, this->params.n_max, this->params.n_min); - SPC_TRC("- mod size=%zu (%.3f MB)\n", + LOG_INF("%s: - mod size=%zu (%.3f MB)\n", __func__, mod.size(), (float)(mod.size_bytes())/1024/1024); if (this->params.n_match < 16) { - SPC_WRN("ngram_mod n_match=%d is too small - poor quality is possible, " - "see: https://github.com/ggml-org/llama.cpp/pull/19164\n", this->params.n_match); + LOG_WRN("%s: ngram_mod n_match=%d is too small - poor quality is possible, " + "see: https://github.com/ggml-org/llama.cpp/pull/19164\n", __func__, this->params.n_match); } sinfos.resize(n_seq); @@ -1815,11 +1396,11 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.i_last = prompt.size() - n; const double f = (double)mod.get_used() / (double)mod.size(); - SPC_TRC("ngram_mod occupancy = %zu/%zu (%.2f)\n", mod.get_used(), mod.size(), f); + LOG_INF("%s: ngram_mod occupancy = %zu/%zu (%.2f)\n", __func__, mod.get_used(), mod.size(), f); constexpr double f_thold = 0.25; if (f > f_thold) { - SPC_WRN("ngram_mod occupancy %.2f exceeds threshold (%.2f) - resetting\n", f, f_thold); + LOG_WRN("%s: ngram_mod occupancy %.2f exceeds threshold (%.2f) - resetting\n", __func__, f, f_thold); mod.reset(); } @@ -1913,7 +1494,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.n_low++; if (sinfo.n_low >= 5) { if (verbose) { - SPC_TRC("low acceptance streak (%d) - resetting ngram_mod\n", sinfo.n_low); + LOG_WRN("%s: low acceptance streak (%d) - resetting ngram_mod\n", __func__, sinfo.n_low); } mod.reset(); @@ -1963,8 +1544,8 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { , save_dynamic(save_dynamic) , save_static(save_static) { - SPC_TRC("%s", "adding speculative implementation 'ngram-cache'\n"); - SPC_TRC("- n_draft=%d, cache_static=%s, cache_dynamic=%s\n", + LOG_INF("%s: adding speculative implementation 'ngram-cache'\n", __func__); + LOG_INF("%s: - n_draft=%d, cache_static=%s, cache_dynamic=%s\n", __func__, n_draft, path_static.empty() ? "none" : path_static.c_str(), path_dynamic.empty() ? "none" : path_dynamic.c_str()); @@ -1979,7 +1560,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { sinfo.ngram_cache_static = ngram_cache_static; } } catch (...) { - SPC_ERR("failed to open static lookup cache: %s", path_static.c_str()); + LOG_ERR("failed to open static lookup cache: %s", path_static.c_str()); GGML_ABORT("Couldn't read static lookup cache"); } } @@ -1992,7 +1573,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { sinfo.ngram_cache_dynamic = ngram_cache_dynamic; } } catch (...) { - SPC_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str()); + LOG_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str()); GGML_ABORT("Couldn't read dynamic lookup cache"); } } @@ -2141,7 +1722,6 @@ std::string common_speculative_type_to_str(common_speculative_type type) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: return "draft-simple"; case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3"; case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp"; - case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash"; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v"; @@ -2194,7 +1774,6 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: - case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: n_max = std::max(n_max, std::max(0, spec->draft.n_max)); break; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: @@ -2231,8 +1810,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr; - bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; - bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr; + bool has_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; @@ -2243,7 +1821,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); // when adding a new type - update here the logic above - static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10); + static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 9); // this list here defines the priority of the speculators // the one with highest priority are listed first @@ -2270,12 +1848,9 @@ common_speculative * common_speculative_init(common_params_speculative & params, if (has_draft_eagle3) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, params)); } - if (has_draft_mtp) { + if (has_mtp) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); } - if (has_draft_dflash) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params)); - } } std::vector> impls = {}; @@ -2296,10 +1871,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, impls.push_back(std::make_unique(config.params, n_seq)); break; } - case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: { - impls.push_back(std::make_unique(config.params, n_seq)); - break; - } case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple); @@ -2349,7 +1920,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, } if (impls.empty()) { - SPC_TRC("%s", "no implementations specified for speculative decoding\n"); + LOG_WRN("%s: no implementations specified for speculative decoding\n", __func__); return nullptr; } @@ -2476,13 +2047,13 @@ void common_speculative_draft(common_speculative * spec) { if (dp.n_max > 0) { if (!result.empty() && (int) result.size() > dp.n_max) { - SPC_DBG("truncating draft to %d tokens\n", dp.n_max); + LOG_DBG("%s: truncating draft to %d tokens\n", __func__, dp.n_max); result.resize(dp.n_max); } } if (!result.empty()) { - SPC_DBG("called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", + LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__, common_speculative_type_to_str(impl.get()->type).c_str(), dp.prompt->size(), impl.get()->n_call_draft, result.size()); @@ -2547,31 +2118,6 @@ void common_speculative_accept(common_speculative * spec, llama_seq_id seq_id, u } } -// TODO: support the case of more than one speculative implementations having a state -bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector & data) { - if (spec == nullptr) { - return false; - } - - for (auto & impl : spec->impls) { - if (impl->get_state(seq_id, data)) { - return true; - } - } - - return false; -} - -void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector & data) { - if (spec == nullptr) { - return; - } - - for (auto & impl : spec->impls) { - impl->set_state(seq_id, data); - } -} - void common_speculative_print_stats(const common_speculative * spec) { if (spec == nullptr) { return; @@ -2606,7 +2152,7 @@ void common_speculative_print_stats(const common_speculative * spec) { str_stats = ", #mean acc len = " + oss.str() + ", #acc rate/pos = (" + tmp.str() + ")"; } - SPC_TRC("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s%s\n", + LOG_INF("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s%s\n", common_speculative_type_to_str(impl->type).c_str(), impl->n_call_begin, impl->n_call_draft, impl->n_call_accept, impl->n_gen_drafts, diff --git a/common/speculative.h b/common/speculative.h index c58fac3cc..bf76ad709 100644 --- a/common/speculative.h +++ b/common/speculative.h @@ -68,10 +68,6 @@ void common_speculative_draft(common_speculative * spec); // informs the speculative context that n_accepted tokens were accepted by the target model void common_speculative_accept(common_speculative * spec, llama_seq_id, uint16_t n_accepted); -// (optional) get/set internal state -bool common_speculative_get_state(common_speculative * spec, llama_seq_id seq_id, std::vector & data); -void common_speculative_set_state(common_speculative * spec, llama_seq_id seq_id, const std::vector & data); - // print statistics about the speculative decoding void common_speculative_print_stats(const common_speculative * spec); diff --git a/conversion/__init__.py b/conversion/__init__.py index 02ea63852..00192cf33 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -46,12 +46,9 @@ TEXT_MODEL_MAP: dict[str, str] = { "DbrxForCausalLM": "dbrx", "DeciLMForCausalLM": "deci", "DeepseekForCausalLM": "deepseek", - "DeepseekOCRForCausalLM": "deepseek", "DeepseekV2ForCausalLM": "deepseek", "DeepseekV3ForCausalLM": "deepseek", "DeepseekV32ForCausalLM": "deepseek", - "DFlashDraftModel": "qwen", - "DeepseekV4ForCausalLM": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", @@ -99,7 +96,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "GraniteMoeHybridForCausalLM": "granite", "GraniteMoeSharedForCausalLM": "granite", "GraniteSpeechForConditionalGeneration": "granite", - "GraniteSpeechPlusForConditionalGeneration": "granite", "Grok1ForCausalLM": "grok", "GrokForCausalLM": "grok", "GroveMoeForCausalLM": "grovemoe", @@ -127,7 +123,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "LLaDAModelLM": "llada", "LLaMAForCausalLM": "llama", "Lfm25AudioTokenizer": "lfm2", - "Lfm2BidirectionalModel": "lfm2", "Lfm2ForCausalLM": "lfm2", "Lfm2Model": "lfm2", "Lfm2MoeForCausalLM": "lfm2", @@ -138,7 +133,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "LlamaModel": "llama", "Eagle3DraftModel": "llama", "Eagle3Speculator": "llama", - "Eagle3LlamaForCausalLM": "llama", "LlamaForCausalLMEagle3": "llama", "LlavaForConditionalGeneration": "llama", "LlavaStableLMEpochForCausalLM": "stablelm", @@ -237,7 +231,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "UMT5ForConditionalGeneration": "t5", "UMT5Model": "t5", "UltravoxModel": "ultravox", - "UnlimitedOCRForCausalLM": "deepseek", "VLlama3ForCausalLM": "llama", "VoxtralForConditionalGeneration": "llama", "WavTokenizerDec": "wavtokenizer", @@ -268,7 +261,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "GlmasrModel": "ultravox", "Granite4VisionForConditionalGeneration": "granite", "GraniteSpeechForConditionalGeneration": "granite", - "GraniteSpeechPlusForConditionalGeneration": "granite", "HunYuanVLForConditionalGeneration": "hunyuan", "Idefics3ForConditionalGeneration": "smolvlm", "InternVisionModel": "internvl", @@ -304,7 +296,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "StepVLForConditionalGeneration": "step3", "Step3p7ForConditionalGeneration": "step3", "UltravoxModel": "ultravox", - "UnlimitedOCRForCausalLM": "deepseek", "VoxtralForConditionalGeneration": "ultravox", "YoutuVLForConditionalGeneration": "youtuvl", } diff --git a/conversion/bailingmoe.py b/conversion/bailingmoe.py index 2c6425cb6..319ff6dab 100644 --- a/conversion/bailingmoe.py +++ b/conversion/bailingmoe.py @@ -126,7 +126,7 @@ class BailingMoeV2Model(TextModel): if (rope_dim := hparams.get("head_dim")) is None: rope_dim = hparams["hidden_size"] // hparams["num_attention_heads"] - self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.5))) + self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5))) self.gguf_writer.add_leading_dense_block_count(hparams["first_k_dense_replace"]) self.gguf_writer.add_vocab_size(hparams["vocab_size"]) self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) diff --git a/conversion/base.py b/conversion/base.py index 0421aa4bc..c872bcbb3 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1119,10 +1119,8 @@ class TextModel(ModelBase): rope_theta = self.find_hparam(["global_rope_theta", "rope_global_theta", "rope_theta_global", "rope_theta", "rotary_emb_base"], optional=True) local_rope_theta = self.find_hparam(["local_rope_theta", "rope_local_theta", "rope_theta_local", "swa_rope_theta", "rope_local_base_freq"], optional=True) - partial_rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct", "rope_percent"], optional=True) - original_max_position_embeddings = self.find_hparam(["original_max_position_embeddings"], optional=True) - # Ensure global params are mirrored in rope_parameters + # Ensure "rope_theta" and "rope_type" is mirrored in rope_parameters if "full_attention" not in self.rope_parameters and "sliding_attention" not in self.rope_parameters: if local_rope_theta is not None: self.rope_parameters["sliding_attention"] = {"rope_theta": local_rope_theta} @@ -1130,10 +1128,6 @@ class TextModel(ModelBase): self.rope_parameters["rope_theta"] = rope_theta if "rope_type" not in self.rope_parameters and (rope_type := self.rope_parameters.get("type")) is not None: self.rope_parameters["rope_type"] = rope_type - if "partial_rotary_factor" not in self.rope_parameters and partial_rotary_factor is not None: - self.rope_parameters["partial_rotary_factor"] = partial_rotary_factor - if "original_max_position_embeddings" not in self.rope_parameters and original_max_position_embeddings is not None: - self.rope_parameters["original_max_position_embeddings"] = original_max_position_embeddings @classmethod def __init_subclass__(cls): @@ -1273,7 +1267,7 @@ class TextModel(ModelBase): if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None: self.gguf_writer.add_layer_norm_eps(f_norm_eps) logger.info(f"gguf: layer norm epsilon = {f_norm_eps}") - if (n_experts := self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True)) is not None: + if (n_experts := self.find_hparam(["num_local_experts", "num_experts"], optional=True)) is not None: self.gguf_writer.add_expert_count(n_experts) logger.info(f"gguf: expert count = {n_experts}") if (n_experts_used := self.find_hparam(["num_experts_per_tok", "num_experts_per_token", "top_k_experts"], optional=True)) is not None: @@ -1291,8 +1285,6 @@ class TextModel(ModelBase): self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) elif score_func == "softmax": self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX) - elif score_func == "sqrtsoftplus": - self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS) else: raise ValueError(f"Unsupported expert score gating function value: {score_func}") logger.info(f"gguf: expert score gating function = {score_func}") @@ -2602,17 +2594,6 @@ class LazyTorchTensor(gguf.LazyBase): return cls._wrap_fn(func)(*args, **kwargs) -if hasattr(torch, "float8_e8m0fnu"): - _torch_float8_e8m0 = torch.float8_e8m0fnu - LazyTorchTensor._dtype_map[_torch_float8_e8m0] = np.uint8 - LazyTorchTensor._dtype_byteswap_map[_torch_float8_e8m0] = np.uint8 - LazyTorchTensor._dtype_str_map["F8_E8M0"] = _torch_float8_e8m0 -else: - # Older torch builds do not expose F8_E8M0. Keep the raw bytes so callers - # that know the format can decode them explicitly. - LazyTorchTensor._dtype_str_map["F8_E8M0"] = torch.uint8 - - def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str: # TODO @ngxson : this won't work correctly if the model has both audio & vision encoders # maybe we should fallback to text model's arch in that case, since not many models have both diff --git a/conversion/chatglm.py b/conversion/chatglm.py index 801913075..7e323b890 100644 --- a/conversion/chatglm.py +++ b/conversion/chatglm.py @@ -148,7 +148,7 @@ class ChatGLMModel(TextModel): rope_dim = self.hparams["attention_dim"] else: rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] - self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.5))) + self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5))) self.gguf_writer.add_add_bos_token(False) rope_freq = 10000 if "rope_ratio" in self.hparams: diff --git a/conversion/deci.py b/conversion/deci.py index be446eefa..46d8568c5 100644 --- a/conversion/deci.py +++ b/conversion/deci.py @@ -161,7 +161,7 @@ class DeciModel(TextModel): factor = rope_params.get("factor", 8.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = rope_params.get("original_max_position_embeddings", 8192) + old_context_len = self.hparams.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/conversion/deepseek.py b/conversion/deepseek.py index ea6ae23d5..72520cc9f 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -1,23 +1,20 @@ from __future__ import annotations -import json import re -from pathlib import Path from typing import Any, Callable, Iterable, TYPE_CHECKING -import numpy as np import torch if TYPE_CHECKING: from torch import Tensor -from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logger +from .base import MmprojModel, ModelBase, TextModel, gguf, logger from .qwen import QwenModel -@ModelBase.register("DeepseekOCRForCausalLM", "UnlimitedOCRForCausalLM") +@ModelBase.register("DeepseekOCRForCausalLM") class DeepseekOCRVisionModel(MmprojModel): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -208,8 +205,6 @@ class DeepseekModel(TextModel): @ModelBase.register( "DeepseekV2ForCausalLM", "DeepseekV3ForCausalLM", - "DeepseekOCRForCausalLM", - "UnlimitedOCRForCausalLM", "KimiVLForConditionalGeneration", "KimiK25ForConditionalGeneration", "YoutuForCausalLM", @@ -229,7 +224,7 @@ class DeepseekV2Model(TextModel): self.origin_hf_arch = hparams.get('architectures', [None])[0] # special handling for Deepseek OCR - if self.origin_hf_arch in ("DeepseekOCRForCausalLM", "DeepseekOCR2ForCausalLM", "UnlimitedOCRForCausalLM"): + if self.origin_hf_arch in ("DeepseekOCRForCausalLM", "DeepseekOCR2ForCausalLM"): self.model_arch = gguf.MODEL_ARCH.DEEPSEEK2OCR self.gguf_writer.arch = gguf.MODEL_ARCH_NAMES[self.model_arch] self.gguf_writer.add_architecture() @@ -355,12 +350,6 @@ class DeepseekV2Model(TextModel): self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"]) - # Unlimited-OCR sliding window; written for metadata, the decoder ignores it (full MHA) - if is_ocr: - sliding_window = hparams.get("sliding_window_size") or hparams.get("sliding_window") - if sliding_window: - self.gguf_writer.add_sliding_window(sliding_window) - if (rope_mscale_all := self.rope_parameters.get("mscale_all_dim")) is not None: # [TAG_DEEPSEEK2_YARN_LOG_MUL_FIX] # note: for legacy reasons, this is not consistent with the other usages of self.gguf_writer.add_rope_scaling_yarn_log_mul @@ -470,307 +459,3 @@ class DeepseekV32Model(DeepseekV2Model): self.gguf_writer.add_indexer_head_count(self.hparams["index_n_heads"]) self.gguf_writer.add_indexer_key_length(self.hparams["index_head_dim"]) self.gguf_writer.add_indexer_top_k(self.hparams["index_topk"]) - - -@ModelBase.register("DeepseekV4ForCausalLM") -class DeepseekV4Model(TextModel): - model_arch = gguf.MODEL_ARCH.DEEPSEEK4 - _skipped_mtp_tensors = 0 - - def __init__(self, *args, **kwargs): - type(self)._skipped_mtp_tensors = 0 - super().__init__(*args, **kwargs) - - with open(self.dir_model / "config.json", "r", encoding="utf-8") as f: - raw_hparams = json.load(f) - for key, value in raw_hparams.items(): - self.hparams.setdefault(key, value) - - self.block_count = self.hparams["num_hidden_layers"] - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - self._dsv4_fp8_dequantized: set[str] = set() - self._dsv4_bf16_tensors: set[str] = set() - self._dsv4_f32_tensors: set[str] = set() - self._dsv4_mxfp4_generated = False - self._collect_source_dtypes() - - if type(self)._skipped_mtp_tensors: - logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors) - - # add a default chat template; if the model has a built-in template, it will be overridden later - template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja" - if template_path.is_file(): - with open(template_path, "r", encoding="utf-8") as f: - self.gguf_writer.add_chat_template(f.read()) - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, _ = item - if name.startswith("mtp."): - cls._skipped_mtp_tensors += 1 - return None - return super().filter_tensors(item) - - @staticmethod - def _float8_dtypes() -> tuple[torch.dtype, ...]: - return tuple( - dtype for dtype in ( - getattr(torch, "float8_e4m3fn", None), - getattr(torch, "float8_e5m2", None), - ) if dtype is not None - ) - - @staticmethod - def _e8m0_to_float(scale: Tensor) -> Tensor: - torch_float8_e8m0 = getattr(torch, "float8_e8m0fnu", None) - if torch_float8_e8m0 is not None and scale.dtype == torch_float8_e8m0: - return scale.float() - - bits = scale.view(torch.uint8).float() - return torch.exp2(bits - 127.0) - - def _collect_source_dtypes(self) -> None: - for name, gen in self.model_tensors.items(): - dtype = gen().dtype - if dtype == torch.bfloat16: - self._dsv4_bf16_tensors.add(name) - elif dtype == torch.float32: - self._dsv4_f32_tensors.add(name) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - hparams = self.hparams - - self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"]) - self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"]) - self.gguf_writer.add_sliding_window(hparams["sliding_window"]) - - self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) - self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"]) - self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"]) - self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"]) - self.gguf_writer.add_swiglu_clamp_exp([hparams["swiglu_limit"]] * self.block_count) - self.gguf_writer.add_swiglu_clamp_shexp([hparams["swiglu_limit"]] * self.block_count) - - self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"]) - self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"]) - self.gguf_writer.add_indexer_top_k(hparams["index_topk"]) - - self.gguf_writer.add_attention_output_group_count(hparams["o_groups"]) - self.gguf_writer.add_attention_output_lora_rank(hparams["o_lora_rank"]) - self.gguf_writer.add_attention_compress_ratios(hparams["compress_ratios"]) - self.gguf_writer.add_attention_compress_rope_freq_base(hparams["compress_rope_theta"]) - self.gguf_writer.add_hyper_connection_count(hparams["hc_mult"]) - self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) - self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) - self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) - - def dequant_model(self): - fp8_dtypes = self._float8_dtypes() - tensors_to_remove: list[str] = [] - - def dequant_fp8_weight(weight: Tensor, scale: Tensor) -> Tensor: - out_features, in_features = weight.shape - scale_f = self._e8m0_to_float(scale) - scale_f = scale_f.repeat_interleave(128, 0)[:out_features] - scale_f = scale_f.repeat_interleave(128, 1)[:, :in_features] - return weight.float() * scale_f - - for name in list(self.model_tensors.keys()): - if not name.endswith(".scale"): - continue - weight_name = name.removesuffix(".scale") + ".weight" - if weight_name not in self.model_tensors: - continue - - weight = self.model_tensors[weight_name] - scale = self.model_tensors[name] - if weight().dtype not in fp8_dtypes: - continue - - self.model_tensors[weight_name] = lambda w=weight, s=scale: dequant_fp8_weight(w(), s()) - self._dsv4_fp8_dequantized.add(weight_name) - tensors_to_remove.append(name) - - for name in tensors_to_remove: - del self.model_tensors[name] - - @staticmethod - def _pack_mxfp4_blocks(weight: Tensor, scale: Tensor) -> np.ndarray: - packed = weight.contiguous().view(torch.uint8) - scale_u8 = scale.contiguous().view(torch.uint8) - - out_features, packed_cols = packed.shape - logical_cols = packed_cols * 2 - if logical_cols % 32 != 0: - raise ValueError(f"MXFP4 source row has {logical_cols} values, expected a multiple of 32") - - n_blocks = logical_cols // 32 - if tuple(scale_u8.shape) != (out_features, n_blocks): - raise ValueError(f"MXFP4 scale shape {tuple(scale_u8.shape)} does not match {(out_features, n_blocks)}") - - src = packed.reshape(out_features, n_blocks, 16) - low = src & 0x0F - high = (src >> 4) & 0x0F - - # The safetensors bytes store adjacent values as low/high nibbles. - # ggml MXFP4 blocks store values 0..15 in low nibbles and 16..31 in high nibbles. - vals = torch.stack((low, high), dim=-1).reshape(out_features, n_blocks, 32) - qs = vals[:, :, :16] | (vals[:, :, 16:] << 4) - raw = torch.cat((scale_u8.unsqueeze(-1), qs.to(torch.uint8)), dim=-1) - return raw.reshape(out_features, n_blocks * 17).cpu().numpy() - - def _write_mxfp4_expert_tensor(self, bid: int, proj: str, tensor_key: gguf.MODEL_TENSOR) -> list[str]: - n_experts = self.hparams["n_routed_experts"] - data: np.ndarray | None = None - consumed: list[str] = [] - - for eid in range(n_experts): - weight_name = f"layers.{bid}.ffn.experts.{eid}.{proj}.weight" - scale_name = f"layers.{bid}.ffn.experts.{eid}.{proj}.scale" - if weight_name not in self.model_tensors or scale_name not in self.model_tensors: - raise KeyError(f"Missing routed expert tensors for {weight_name}") - - weight = LazyTorchTensor.to_eager(self.model_tensors[weight_name]()) - scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]()) - packed = self._pack_mxfp4_blocks(weight, scale) - if data is None: - data = np.empty((n_experts, *packed.shape), dtype=packed.dtype) - data[eid] = packed - consumed.extend((weight_name, scale_name)) - - assert data is not None - new_name = self.format_tensor_name(tensor_key, bid) - shape = gguf.quant_shape_from_byte_shape(data.shape, gguf.GGMLQuantizationType.MXFP4) - logger.info(f"{new_name}: repacked routed experts to MXFP4, shape = {{{', '.join(str(n) for n in reversed(shape))}}}") - self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP4) - - return consumed - - def _write_hash_routing_tensors(self) -> list[str]: - consumed: list[str] = [] - - for bid in range(self.hparams["num_hash_layers"]): - name = f"layers.{bid}.ffn.gate.tid2eid" - if name not in self.model_tensors: - raise KeyError(f"Missing hash routing tensor {name}") - - data_torch = LazyTorchTensor.to_eager(self.model_tensors[name]()) - data = data_torch.to(torch.int32).cpu().numpy() - new_name = self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_TID2EID, bid, ".weight") - logger.info(f"{new_name}: converted hash routing table to I32, shape = {{{', '.join(str(n) for n in reversed(data.shape))}}}") - self.gguf_writer.add_tensor(new_name, data) - consumed.append(name) - - return consumed - - def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - if self._dsv4_mxfp4_generated: - return () - - consumed: list[str] = self._write_hash_routing_tensors() - for bid in range(self.block_count): - consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)) - consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)) - consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)) - - for name in consumed: - del self.model_tensors[name] - - self._dsv4_mxfp4_generated = True - return () - - def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str: - return self.format_tensor_name(key, bid, suffix) - - def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: - root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { - "embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"), - "norm.weight": (gguf.MODEL_TENSOR.OUTPUT_NORM, ".weight"), - "head.weight": (gguf.MODEL_TENSOR.OUTPUT, ".weight"), - "hc_head_fn": (gguf.MODEL_TENSOR.HC_HEAD_FN, ".weight"), - "hc_head_base": (gguf.MODEL_TENSOR.HC_HEAD_BASE, ".weight"), - "hc_head_scale": (gguf.MODEL_TENSOR.HC_HEAD_SCALE, ".weight"), - } - if name in root_map: - return root_map[name] - - match = re.match(r"layers\.(\d+)\.(.+)$", name) - if match is None: - raise ValueError(f"Unsupported DeepSeek-V4 tensor {name!r}") - - layer = int(match.group(1)) - if bid != layer: - raise ValueError(f"Tensor {name!r} parsed bid {bid} but layer name has {layer}") - - layer_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { - "hc_attn_fn": (gguf.MODEL_TENSOR.HC_ATTN_FN, ".weight"), - "hc_attn_base": (gguf.MODEL_TENSOR.HC_ATTN_BASE, ".weight"), - "hc_attn_scale": (gguf.MODEL_TENSOR.HC_ATTN_SCALE, ".weight"), - "hc_ffn_fn": (gguf.MODEL_TENSOR.HC_FFN_FN, ".weight"), - "hc_ffn_base": (gguf.MODEL_TENSOR.HC_FFN_BASE, ".weight"), - "hc_ffn_scale": (gguf.MODEL_TENSOR.HC_FFN_SCALE, ".weight"), - "attn.attn_sink": (gguf.MODEL_TENSOR.ATTN_SINKS, ".weight"), - "attn.wq_a.weight": (gguf.MODEL_TENSOR.ATTN_Q_A, ".weight"), - "attn.wq_b.weight": (gguf.MODEL_TENSOR.ATTN_Q_B, ".weight"), - "attn.q_norm.weight": (gguf.MODEL_TENSOR.ATTN_Q_A_NORM, ".weight"), - "attn.wkv.weight": (gguf.MODEL_TENSOR.ATTN_KV, ".weight"), - "attn.kv_norm.weight": (gguf.MODEL_TENSOR.ATTN_KV_NORM, ".weight"), - "attn.wo_a.weight": (gguf.MODEL_TENSOR.ATTN_OUT_A, ".weight"), - "attn.wo_b.weight": (gguf.MODEL_TENSOR.ATTN_OUT_B, ".weight"), - "attn.compressor.ape": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_APE, ".weight"), - "attn.compressor.wkv.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_WKV, ".weight"), - "attn.compressor.wgate.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_WGATE, ".weight"), - "attn.compressor.norm.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_NORM, ".weight"), - "attn.indexer.wq_b.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_Q_B, ".weight"), - "attn.indexer.weights_proj.weight": (gguf.MODEL_TENSOR.INDEXER_PROJ, ".weight"), - "attn.indexer.compressor.ape": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_APE, ".weight"), - "attn.indexer.compressor.wkv.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_WKV, ".weight"), - "attn.indexer.compressor.wgate.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE, ".weight"), - "attn.indexer.compressor.norm.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_NORM, ".weight"), - "attn_norm.weight": (gguf.MODEL_TENSOR.ATTN_NORM, ".weight"), - "ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"), - "ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"), - "ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"), - "ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"), - "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), - "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), - "ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"), - } - - tensor_name = match.group(2) - if tensor_name in layer_map: - return layer_map[tensor_name] - - if re.match(r"ffn\.experts\.\d+\.w[123]\.(weight|scale)$", tensor_name): - return gguf.MODEL_TENSOR.FFN_GATE_EXP, ".weight" - - raise ValueError(f"Unsupported DeepSeek-V4 tensor {name!r}") - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name): - return [] - - tensor_key, suffix = self._map_dsv4_tensor_name(name, bid) - if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID: - return [] - - return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)] - - def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: - del new_name, bid # unused - - if name in self._dsv4_fp8_dequantized and n_dims >= 2: - return gguf.GGMLQuantizationType.Q8_0 - if name in self._dsv4_f32_tensors: - return gguf.GGMLQuantizationType.F32 - if name in self._dsv4_bf16_tensors and n_dims >= 2: - return gguf.GGMLQuantizationType.BF16 - - return False - - def prepare_tensors(self): - super().prepare_tensors() - self._is_mxfp4 = True - self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE diff --git a/conversion/exaone.py b/conversion/exaone.py index bc4fb3f1b..b21f02784 100644 --- a/conversion/exaone.py +++ b/conversion/exaone.py @@ -24,7 +24,7 @@ class ExaoneModel(TextModel): assert (hparams["activation_function"] == "silu") - rotary_factor = self.rope_parameters.get("partial_rotary_factor") + rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"], optional=True) rotary_factor = rotary_factor if rotary_factor is not None else 1.0 self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) @@ -39,7 +39,7 @@ class ExaoneModel(TextModel): factor = rope_params.get("factor", 8.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = rope_params.get("original_max_position_embeddings", 8192) + old_context_len = self.hparams.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor @@ -104,7 +104,7 @@ class Exaone4Model(TextModel): factor = rope_params.get("factor", 16.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = rope_params.get("original_max_position_embeddings", 8192) + old_context_len = self.hparams.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/conversion/gemma.py b/conversion/gemma.py index c552df732..5b4ca5c58 100644 --- a/conversion/gemma.py +++ b/conversion/gemma.py @@ -693,7 +693,7 @@ class Gemma4Model(Gemma3Model): self.gguf_writer.add_head_count_kv(value_arr) # handle n_rot differently for global vs swa layers - partial_rotary_factor_swa = self.rope_parameters.get("partial_rotary_factor", 1.0) + partial_rotary_factor_swa = self.hparams.get("partial_rotary_factor", 1.0) n_rot_full = int(head_dim_full) # "proportional" is used, see generate_extra_tensors n_rot_swa = int(head_dim_swa * partial_rotary_factor_swa) self.gguf_writer.add_rope_dimension_count(n_rot_full) diff --git a/conversion/glm.py b/conversion/glm.py index 895cefc22..641937720 100644 --- a/conversion/glm.py +++ b/conversion/glm.py @@ -124,7 +124,7 @@ class Glm4MoeModel(TextModel): self.hparams["hidden_size"] // self.hparams["num_attention_heads"] ) self.gguf_writer.add_rope_dimension_count( - int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.5)) + int(rope_dim * self.hparams.get("partial_rotary_factor", 0.5)) ) # MoE parameters - Use only routed expert count (shared experts handled separately) @@ -226,7 +226,7 @@ class GlmMoeDsaModel(DeepseekV2Model): super().set_gguf_parameters() rope_dim = self.hparams["qk_rope_head_dim"] - partial_rotary_factor = self.rope_parameters.get("partial_rotary_factor", 1.0) + partial_rotary_factor = self.hparams.get("partial_rotary_factor", 1.0) self.gguf_writer.add_rope_dimension_count(int(rope_dim * partial_rotary_factor)) # NextN/MTP prediction layers diff --git a/conversion/granite.py b/conversion/granite.py index 8367ed225..53441fe57 100644 --- a/conversion/granite.py +++ b/conversion/granite.py @@ -348,34 +348,6 @@ class GraniteSpeechMmprojModel(MmprojModel): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("GraniteSpeechPlusForConditionalGeneration") -class GraniteSpeechPlusMmprojModel(GraniteSpeechMmprojModel): - """Conversion for GraniteSpeechPlus - extends GraniteSpeech with feature layer concatenation""" - has_vision_encoder = False - has_audio_encoder = True - - def set_gguf_parameters(self): - assert self.hparams_audio is not None - super().set_gguf_parameters() - - # Add feature_layer if present in encoder config - if feature_layers := self.hparams_audio.get("cat_hidden_layers"): - self.gguf_writer.add_audio_feature_layers(feature_layers) - logger.info(f"gguf: audio feature_layers = {feature_layers}") - - # Validate projector dimension matches concatenated encoder output - hidden_dim = self.hparams_audio["hidden_dim"] - expected_dim = hidden_dim * (len(feature_layers) + 1) - projector_dim = self.global_config["projector_config"]["encoder_hidden_size"] - - if projector_dim != expected_dim: - raise ValueError( - f"Projector encoder_hidden_size ({projector_dim}) does not match " - f"expected concatenated dimension ({expected_dim}). " - f"Expected: hidden_dim ({hidden_dim}) * (len(feature_layers) + 1) = {expected_dim}" - ) - - @ModelBase.register("Granite4VisionForConditionalGeneration") class Granite4VisionMmprojModel(MmprojModel): has_vision_encoder = True diff --git a/conversion/lfm2.py b/conversion/lfm2.py index 70ce45658..f28fccf10 100644 --- a/conversion/lfm2.py +++ b/conversion/lfm2.py @@ -64,17 +64,11 @@ class LFM2Model(TextModel): yield from super().modify_tensors(data_torch, name, bid) -@ModelBase.register("Lfm2Model", "Lfm2BidirectionalModel") +@ModelBase.register("Lfm2Model") class LFM2ColBertModel(LFM2Model): model_arch = gguf.MODEL_ARCH.LFM2 dense_tensor_name = "dense_2" - def set_gguf_parameters(self): - super().set_gguf_parameters() - if self.hf_arch == "Lfm2BidirectionalModel": - self.gguf_writer.add_causal_attention(False) - self._try_set_pooling_type() - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: if not name.startswith(self.dense_tensor_name): name = "model." + name @@ -82,11 +76,10 @@ class LFM2ColBertModel(LFM2Model): yield from super().modify_tensors(data_torch, name, bid) def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - # optional dense tensor is stored in a separate safetensors file + # dense tensor is stored in a separate safetensors file from safetensors.torch import load_file tensors_file = self.dir_model / "1_Dense" / "model.safetensors" - if not tensors_file.is_file(): - return + assert tensors_file.is_file() tensor = load_file(tensors_file)["linear.weight"] self.gguf_writer.add_embedding_length_out(tensor.shape[0]) yield f"{self.dense_tensor_name}.weight", tensor.clone() diff --git a/conversion/llama.py b/conversion/llama.py index 315a619c9..b87bf92d4 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -23,7 +23,6 @@ from .base import ModelBase, TextModel, gguf, logger "LlavaForConditionalGeneration", "VoxtralForConditionalGeneration", "LlamaForCausalLMEagle3", - "Eagle3LlamaForCausalLM", "Eagle3Speculator", "Eagle3DraftModel", "IQuestCoderForCausalLM", @@ -73,7 +72,7 @@ class LlamaModel(TextModel): target_num_layers = target_config["num_hidden_layers"] target_layers = [2, target_num_layers // 2, target_num_layers - 3] logger.info(f"EAGLE-3: target_layers = {target_layers} (target model has {target_num_layers} layers)") - self.gguf_writer.add_target_layers(target_layers) + self.gguf_writer.add_array(f"{self.gguf_writer.arch}.target_layers", target_layers) # target_hidden_size: prefer eagle3 config, fallback to target config if eagle3_raw_config.get("target_hidden_size") is not None: @@ -83,12 +82,12 @@ class LlamaModel(TextModel): target_hidden_size = target_config["hidden_size"] src = "target model config" logger.info(f"EAGLE-3: target_hidden_size = {target_hidden_size} (from {src})") - self.gguf_writer.add_target_hidden_size(target_hidden_size) + self.gguf_writer.add_uint32(f"{self.gguf_writer.arch}.target_hidden_size", target_hidden_size) # norm_before_residual (RedHat-style eagle3 specific) norm_before_residual = eagle3_raw_config.get("norm_before_residual", False) logger.info(f"EAGLE-3: norm_before_residual = {norm_before_residual}") - self.gguf_writer.add_norm_before_residual(norm_before_residual) + self.gguf_writer.add_bool(f"{self.gguf_writer.arch}.norm_before_residual", norm_before_residual) def set_vocab(self): # eagle3: use tokenizer from target model if provided @@ -290,7 +289,7 @@ class LlamaModel(TextModel): factor = rope_params.get("factor", 8.0) low_freq_factor = rope_params.get("low_freq_factor", 1.0) high_freq_factor = rope_params.get("high_freq_factor", 4.0) - old_context_len = rope_params.get("original_max_position_embeddings", 8192) + old_context_len = self.hparams.get("original_max_position_embeddings", 8192) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/conversion/mamba.py b/conversion/mamba.py index 43d559ffb..be0e36a29 100644 --- a/conversion/mamba.py +++ b/conversion/mamba.py @@ -114,8 +114,7 @@ class Mamba2Model(TextModel): hparams["text_config"] = hparams["llm_config"] super().__init__(dir_model, *args, hparams=hparams, **kwargs) self.d_model = self.find_hparam(["hidden_size", "d_model", "dim"]) - self.expand = self.find_hparam(["mamba_expand", "expand"], optional=True) or 2 - self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or self.expand * self.d_model + self.d_inner = self.find_hparam(["mamba_d_ssm", "intermediate_size", "d_inner"], optional=True) or 2 * self.d_model self.n_group = self.find_hparam(["n_groups"], optional=True) or 1 def set_vocab(self): @@ -145,9 +144,11 @@ class Mamba2Model(TextModel): rms_norm_eps = self.find_hparam(["layer_norm_epsilon", "rms_norm_eps"], optional=True) or 1e-5 + # Fail early for models which don't have a block expansion factor of 2 + # TODO: does this really matter? # skip the assertion for FalconH1 Model if self.model_arch != gguf.MODEL_ARCH.FALCON_H1: - assert self.d_inner == self.expand * self.d_model + assert self.d_inner == 2 * self.d_model assert self.d_inner % head_dim == 0 self.gguf_writer.add_context_length(2**20) # arbitrary value; for those who use the default diff --git a/conversion/mimo.py b/conversion/mimo.py index 11ec28679..d4067aab4 100644 --- a/conversion/mimo.py +++ b/conversion/mimo.py @@ -154,7 +154,7 @@ class MimoV2Model(TextModel): self.gguf_writer.add_expert_count(self.hparams["n_routed_experts"]) self.gguf_writer.add_expert_feed_forward_length(self.hparams["moe_intermediate_size"]) - rope_dim = int(self.hparams["head_dim"] * self.rope_parameters["partial_rotary_factor"]) + rope_dim = int(self.hparams["head_dim"] * self.hparams["partial_rotary_factor"]) self.gguf_writer.add_rope_dimension_count(rope_dim) self.gguf_writer.add_layer_norm_rms_eps(self.hparams.get("layernorm_epsilon", 1e-5)) diff --git a/conversion/minicpm.py b/conversion/minicpm.py index e31b26a00..e9a4c4a74 100644 --- a/conversion/minicpm.py +++ b/conversion/minicpm.py @@ -32,9 +32,11 @@ class MiniCPMModel(TextModel): def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: rope_dims = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] - long_factors = self.rope_parameters.get('long_factor') - short_factors = self.rope_parameters.get('short_factor') - if long_factors or short_factors: + rope_scaling = self.find_hparam(['rope_scaling'], True) + if rope_scaling is not None: + long_factors = rope_scaling.get('long_factor', None) + short_factors = rope_scaling.get('short_factor', None) + if long_factors is None or short_factors is None: raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') @@ -83,11 +85,13 @@ class MiniCPM3Model(TextModel): self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"]) def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - long_factors = self.rope_parameters.get('long_factor') - short_factors = self.rope_parameters.get('short_factor') - if long_factors or short_factors: + rope_scaling = self.find_hparam(['rope_scaling'], True) + if rope_scaling is not None: rope_dims = self.hparams["qk_rope_head_dim"] + long_factors = rope_scaling.get('long_factor', None) + short_factors = rope_scaling.get('short_factor', None) + if long_factors is None or short_factors is None: raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') diff --git a/conversion/nemotron.py b/conversion/nemotron.py index e44688a78..dfeeb9785 100644 --- a/conversion/nemotron.py +++ b/conversion/nemotron.py @@ -125,18 +125,17 @@ class NemotronModel(TextModel): self.gguf_writer.add_layer_norm_eps(f_norm_eps) # * Partial RoPE - rot_pct = self.rope_parameters["partial_rotary_factor"] + rot_pct = self.find_hparam(["partial_rotary_factor", "rope_pct", "rope_percent"]) n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) self.gguf_writer.add_rope_dimension_count(int(rot_pct * n_embd) // n_head) # * RopeScaling for Nemotron - factor = self.hparams.get("factor") or self.rope_parameters.get("factor") - if factor is None: + if "rope_scaling" not in self.hparams or self.hparams["rope_scaling"] is None: self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.NONE) else: self.gguf_writer.add_rope_scaling_type(gguf.RopeScalingType.LINEAR) - self.gguf_writer.add_rope_scaling_factor(factor) + self.gguf_writer.add_rope_scaling_factor(self.hparams["factor"]) def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: # * Adding +1 to LayerNorm's weights here to implement layernorm1p w/o changing anything on the GGML engine side diff --git a/conversion/phi.py b/conversion/phi.py index df4bfe809..5e0d72847 100644 --- a/conversion/phi.py +++ b/conversion/phi.py @@ -18,7 +18,7 @@ class Phi2Model(TextModel): model_arch = gguf.MODEL_ARCH.PHI2 def set_gguf_parameters(self): - rot_pct = self.rope_parameters["partial_rotary_factor"] + rot_pct = self.find_hparam(["partial_rotary_factor"]) n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) @@ -149,8 +149,8 @@ class Phi3MiniModel(TextModel): n_head_kv = self.find_hparam(["num_key_value_heads", "n_head_kv"]) rms_eps = self.find_hparam(["rms_norm_eps"]) max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"]) - orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"] - rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0) + orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"]) + rot_pct = self.hparams.get("partial_rotary_factor", 1.0) rope_dims = int(rot_pct * n_embd) // n_head self.gguf_writer.add_context_length(max_pos_embds) @@ -174,19 +174,18 @@ class Phi3MiniModel(TextModel): n_embd = self.find_hparam(["hidden_size", "n_embd"]) n_head = self.find_hparam(["num_attention_heads", "n_head"]) max_pos_embds = self.find_hparam(["n_positions", "max_position_embeddings"]) - orig_max_pos_embds = self.rope_parameters["original_max_position_embeddings"] - rot_pct = self.rope_parameters.get("partial_rotary_factor", 1.0) + orig_max_pos_embds = self.find_hparam(["original_max_position_embeddings"]) + rot_pct = self.hparams.get("partial_rotary_factor", 1.0) rope_dims = int(rot_pct * n_embd) // n_head # write rope scaling for long context (128k) model - long_factors = self.rope_parameters.get('long_factor') - short_factors = self.rope_parameters.get('short_factor') - if not long_factors: + rope_scaling = self.find_hparam(['rope_scaling'], True) + if rope_scaling is None: return scale = max_pos_embds / orig_max_pos_embds - rope_scaling_type = self.rope_parameters.get('rope_type', '').lower() + rope_scaling_type = rope_scaling.get('rope_type', rope_scaling.get('type', '')).lower() if len(rope_scaling_type) == 0: raise KeyError('Missing the required key rope_scaling.type') @@ -199,6 +198,9 @@ class Phi3MiniModel(TextModel): self.gguf_writer.add_rope_scaling_attn_factors(attn_factor) + long_factors = rope_scaling.get('long_factor', None) + short_factors = rope_scaling.get('short_factor', None) + if long_factors is None or short_factors is None: raise KeyError('Missing the required key rope_scaling.long_factor or rope_scaling_short_factor') diff --git a/conversion/qwen.py b/conversion/qwen.py index 0356bd2da..7eb135c83 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -280,7 +280,7 @@ class Qwen3NextModel(Qwen2MoeModel): self.gguf_writer.add_full_attention_interval(self.hparams.get("full_attention_interval", 4)) if (rope_dim := self.hparams.get("head_dim")) is None: rope_dim = self.hparams["hidden_size"] // self.hparams["num_attention_heads"] - self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.rope_parameters.get("partial_rotary_factor", 0.25))) + self.gguf_writer.add_rope_dimension_count(int(rope_dim * self.hparams.get("partial_rotary_factor", 0.25))) @classmethod def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: @@ -625,51 +625,3 @@ class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReor @ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM") class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35MOE - - -@ModelBase.register("DFlashDraftModel") -class DFlashModel(Qwen3Model): - model_arch = gguf.MODEL_ARCH.DFLASH - - def set_vocab(self): - if self.target_model_dir is None: - raise ValueError( - "DFlash draft model requires --target-model-dir to be specified. " - "Please provide the path to the target model directory containing the tokenizer." - ) - logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}") - original_dir = self.dir_model - self.dir_model = self.target_model_dir - super().set_vocab() - self.dir_model = original_dir - - mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id") - if mask_token_id is not None: - self.gguf_writer.add_mask_token_id(mask_token_id) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - - block_size = self.hparams.get("block_size", 16) - self.gguf_writer.add_block_size(block_size) - dflash_config = self.hparams.get("dflash_config", {}) - - target_layer_ids = dflash_config.get("target_layer_ids", []) - if target_layer_ids: - extract_layer_ids = [i + 1 for i in target_layer_ids] - self.gguf_writer.add_target_layers(extract_layer_ids) - - use_sliding_window = self.hparams.get("use_sliding_window", False) - sliding_window = self.hparams.get("sliding_window") - layer_types = self.hparams.get("layer_types") - if use_sliding_window and sliding_window and layer_types: - is_swa = [lt == "sliding_attention" for lt in layer_types] - self.gguf_writer.add_sliding_window(sliding_window) - self.gguf_writer.add_sliding_window_pattern(is_swa) - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - if not name.startswith("model."): - name = "model." + name - return super().filter_tensors((name, gen)) diff --git a/conversion/stablelm.py b/conversion/stablelm.py index 6e16378a0..ba5e9aa6c 100644 --- a/conversion/stablelm.py +++ b/conversion/stablelm.py @@ -28,7 +28,7 @@ class StableLMModel(TextModel): self.gguf_writer.add_embedding_length(hparams["hidden_size"]) self.gguf_writer.add_block_count(self.block_count) self.gguf_writer.add_feed_forward_length(hparams["intermediate_size"]) - rotary_factor = self.rope_parameters["partial_rotary_factor"] + rotary_factor = self.find_hparam(["partial_rotary_factor", "rope_pct"]) self.gguf_writer.add_rope_dimension_count(int(rotary_factor * (hparams["hidden_size"] // hparams["num_attention_heads"]))) self.gguf_writer.add_head_count(hparams["num_attention_heads"]) self.gguf_writer.add_head_count_kv(hparams["num_key_value_heads"]) diff --git a/conversion/step3.py b/conversion/step3.py index 49bb5244a..8c45b61c9 100644 --- a/conversion/step3.py +++ b/conversion/step3.py @@ -314,7 +314,7 @@ class Step35Model(TextModel): factor = float(rope_params.get("factor", 8.0)) low_freq_factor = float(rope_params.get("low_freq_factor", 1.0)) high_freq_factor = float(rope_params.get("high_freq_factor", 4.0)) - old_context_len = int(rope_params.get("original_max_position_embeddings", 8192)) + old_context_len = int(rope_params.get("original_max_position_embeddings", self.hparams.get("original_max_position_embeddings", 8192))) low_freq_wavelen = old_context_len / low_freq_factor high_freq_wavelen = old_context_len / high_freq_factor diff --git a/embd_res/kcpp_docs.embd b/embd_res/kcpp_docs.embd index ed44c09f9..92234c19d 100644 --- a/embd_res/kcpp_docs.embd +++ b/embd_res/kcpp_docs.embd @@ -2849,87 +2849,6 @@ "responses": {"default": {"description": ""}} } }, - "/v1/images/generations": { - "post": { - "summary": "Generates images from a text prompt. Please refer to OpenAI documentation", - "description": "Creates images from a text prompt.\n\n This is an OpenAI compatibility endpoint.\n\n Please refer to OpenAI documentation at [https://developers.openai.com/docs/api-reference/images/create](https://developers.openai.com/docs/api-reference/images/create).", - "requestBody": { - "content": { - "application/json": { - "example": {"model":"kcpp","prompt": "picture of a kobold, high quality HD render", "n": 1, "size": "512x512", "response_format": "b64_json"}, - "schema": { - "properties": { - "model": { - "type": "string", - "description": "Model identifier. Use kcpp for the currently loaded image model." - }, - "prompt": { - "type": "string", - "description": "Text prompt describing the image to generate." - }, - "n": { - "type": "integer", - "description": "Number of images to generate.", - "minimum": 1 - }, - "size": { - "type": "string", - "description": "Requested image size, such as 512x512 or 1024x1024." - }, - "response_format": { - "type": "string", - "description": "Response image format. b64_json returns base64 encoded image data." - } - }, - "required": [ - "prompt" - ], - "type": "object" - } - } - }, - "required": true - }, - "tags": [ - "v1" - ], - "responses": { - "200": { - "content": { - "application/json": { - "example": {"created": 1710000000, "data": [{"b64_json": "base64_image_data"}]}, - "schema": { - "properties": { - "created": { - "type": "integer", - "description": "Unix timestamp for the generation request." - }, - "data": { - "type": "array", - "items": { - "type": "object", - "properties": { - "b64_json": { - "type": "string", - "description": "Base64 encoded image data." - }, - "url": { - "type": "string", - "description": "Image URL, if URL responses are supported." - } - } - } - } - }, - "type": "object" - } - } - }, - "description": "Successful request" - } - } - } - }, "/v1/models": { "get": { "summary": "List and describe the various models available in the API. Please refer to OpenAI documentation", diff --git a/embd_res/kcpp_musicui.embd b/embd_res/kcpp_musicui.embd index 3c5d44990..85738e1b6 100644 --- a/embd_res/kcpp_musicui.embd +++ b/embd_res/kcpp_musicui.embd @@ -307,11 +307,6 @@ select{ -
- - -
-
@@ -450,8 +445,7 @@ async function generateTTS(){ const payload = { input: document.getElementById("tts_input").value, - voice: document.getElementById("tts_voice").value, - use_mp3: document.getElementById("tts_use_mp3").checked + voice: document.getElementById("tts_voice").value }; const instruction = document.getElementById("tts_instruction").value; @@ -501,7 +495,6 @@ async function generateTTS(){ function clearTTS(){ document.getElementById("tts_input").value=""; document.getElementById("tts_instruction").value=""; - document.getElementById("tts_use_mp3").checked=false; } //end of tts part @@ -942,4 +935,4 @@ fetchStats(); - + \ No newline at end of file diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd index 745fec1d3..1fe8dcb96 100644 --- a/embd_res/kcpp_sdui.embd +++ b/embd_res/kcpp_sdui.embd @@ -5,18 +5,18 @@ Stable UI for KoboldCpp - - diff --git a/embd_res/klite.embd b/embd_res/klite.embd index 3b3f86b40..81d70a8fc 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -12,7 +12,7 @@ Current version indicated by LITEVER below. --> -{#snippet button(props = {})} - + {/snippet} + - - {/if} - -{/snippet} - -{#if showTooltip} - - - - {#snippet child({ props })} - {@render button(props)} - {/snippet} - - - -

{tooltip}

- - -{:else} - {@render button({ href })} -{/if} - - + +

{tooltip}

+ + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte index 9b2077b8d..ed26f9ea5 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatForm.svelte @@ -494,7 +494,7 @@ />
{/each} - {#if visibleMcpServers.length === 0} + {#if getEnabledMcpServers().length === 0}
No MCP servers configured
@@ -269,22 +270,14 @@ {/if} - {#if hasMcpPromptsSupport} -
{:else if toolsStore.isToolsEndpointUnreachable}
+ + + + + Run llama-server with {CLI_FLAGS.TOOLS} flag to enable + + Built-in Tools. + + + diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte index 08d691c14..6a91bf905 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionAdd/ChatFormActionsAdd.svelte @@ -42,7 +42,6 @@ {hasMcpPromptsSupport} {hasMcpResourcesSupport} {onFileUpload} - {onSystemPromptClick} {onMcpPromptClick} {onMcpResourcesClick} > diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte index 998e8dcb4..712326cba 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionModels.svelte @@ -63,8 +63,8 @@ modelsStore.selectedModelName = conversationModel; modelsStore.selectModelByName(conversationModel); } else { - modelsStore.selectedModelId = null; - modelsStore.selectedModelName = conversationModel; + modelsStore.selectedModelName = null; + modelsStore.clearSelection(); } lastSyncedConversationModel = conversationModel; } else if ( @@ -141,7 +141,19 @@ }); $effect(() => { - isSelectedModelInCache = !isRouter || !!conversationModel || !!selectedModelId(); + if (!isRouter) { + isSelectedModelInCache = true; + } else if (conversationModel) { + isSelectedModelInCache = modelOptions().some((option) => option.model === conversationModel); + } else { + const currentModelId = selectedModelId(); + + if (!currentModelId) { + isSelectedModelInCache = false; + } else { + isSelectedModelInCache = modelOptions().some((option) => option.id === currentModelId); + } + } }); $effect(() => { diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte index eff0364fa..8774bf63a 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormActions/ChatFormActionSubmit.svelte @@ -20,7 +20,7 @@ type="submit" disabled={isDisabled} class={[ - 'md:h-8 md:w-8 h-9 w-9 rounded-full p-0', + 'h-8 w-8 rounded-full p-0', showErrorState && 'bg-red-400/10 text-red-400 hover:bg-red-400/20 hover:text-red-400 disabled:opacity-100' ]} diff --git a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte index 3e683389f..72e62f319 100644 --- a/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatForm/ChatFormTextarea.svelte @@ -1,5 +1,4 @@ -
+
{#if message.role === MessageRole.SYSTEM} (null); - let isCurrentlyLoading = $derived(isLoading()); let isStreaming = $derived(isChatStreaming()); let hasNoContent = $derived(!message?.content?.trim()); let isActivelyProcessing = $derived(isCurrentlyLoading || isStreaming); - // during a router auto-load the message has no model yet, so target the selected one - let loadTargetModel = $derived(message.model ?? modelsStore.selectedModelName); - let modelLoadProgress = $derived( - isRouter && loadTargetModel ? modelsStore.getLoadProgress(loadTargetModel) : null - ); - let modelLoadingText = $derived(modelLoadProgressText(modelLoadProgress)); - let showProcessingInfoTop = $derived( message?.role === MessageRole.ASSISTANT && isActivelyProcessing && @@ -210,42 +200,6 @@ isLastAssistantMessage ); - let assistantEl: HTMLDivElement | undefined = $state(); - let lastUserMessageHeight = $state(0); - let assistantMarginTop = $state(0); - - $effect(() => { - if (!assistantEl) return; - - assistantMarginTop = Math.round(parseFloat(getComputedStyle(assistantEl).marginTop)); - - const chatMessageEl = assistantEl.closest('.chat-message'); - const previousChatMessage = chatMessageEl?.previousElementSibling; - const userMessageEl = previousChatMessage?.querySelector( - '.chat-message-user' - ) as HTMLElement | null; - - if (!userMessageEl) { - lastUserMessageHeight = 0; - return; - } - - const updateHeight = () => { - const rect = userMessageEl.getBoundingClientRect(); - const marginTop = Math.round(parseFloat(getComputedStyle(userMessageEl).marginTop)); - lastUserMessageHeight = Math.round(rect.height + marginTop); - }; - - updateHeight(); - - const resizeObserver = new ResizeObserver(updateHeight); - resizeObserver.observe(userMessageEl); - - return () => { - resizeObserver.disconnect(); - }; - }); - function handleCopyModel() { void copyToClipboard(displayedModel ?? ''); } @@ -258,21 +212,15 @@
0 - ? `${lastUserMessageHeight}px` - : undefined} - style:--assistant-margin-top={assistantMarginTop > 0 ? `${assistantMarginTop}px` : undefined} + class="text-md group w-full leading-7.5 {className}" role="group" aria-label="Assistant message with actions" > {#if showProcessingInfoTop} -
+
- {modelLoadingText ?? - processingState.getPromptProgressText() ?? + {processingState.getPromptProgressText() ?? processingState.getProcessingMessage() ?? 'Processing...'} @@ -301,11 +249,10 @@ {/if} {#if showProcessingInfoBottom} -
+
- {modelLoadingText ?? - processingState.getPromptProgressText() ?? + {processingState.getPromptProgressText() ?? processingState.getProcessingMessage() ?? 'Processing...'} @@ -321,19 +268,13 @@ > {#if isRouter} { const status = modelsStore.getModelStatus(modelId); if (status !== ServerModelStatus.LOADED) { - pendingModel = modelId; - - try { - await modelsStore.loadModel(modelId); - } finally { - pendingModel = null; - } + await modelsStore.loadModel(modelId); } onRegenerate(modelName); @@ -401,23 +342,6 @@
diff --git a/tools/ui/src/lib/components/app/misc/index.ts b/tools/ui/src/lib/components/app/misc/index.ts index b550ae66a..64b76fb71 100644 --- a/tools/ui/src/lib/components/app/misc/index.ts +++ b/tools/ui/src/lib/components/app/misc/index.ts @@ -51,11 +51,3 @@ export { default as KeyboardShortcutInfo } from './KeyboardShortcutInfo.svelte'; * Preview button is shown only for HTML code blocks. */ export { default as CodeBlockActions } from './CodeBlockActions.svelte'; - -/** - * **Logo** - Application brand mark - * - * Inline SVG of the application logo. Accepts styling via the standard - * `class` and `style` props and inherits color via `currentColor`. - */ -export { default as Logo } from './Logo.svelte'; diff --git a/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte b/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte deleted file mode 100644 index fa9a02108..000000000 --- a/tools/ui/src/lib/components/app/models/ModelLoadHighlight.svelte +++ /dev/null @@ -1,11 +0,0 @@ - - - -
-
-
diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte index 720963a8d..40006a4c9 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorDropdown.svelte @@ -2,10 +2,8 @@ import { ChevronDown, Loader2, Package } from '@lucide/svelte'; import * as DropdownMenu from '$lib/components/ui/dropdown-menu'; import * as Tooltip from '$lib/components/ui/tooltip'; - import { KeyboardKey, ServerModelStatus } from '$lib/enums'; + import { KeyboardKey } from '$lib/enums'; import { useModelsSelector } from '$lib/hooks/use-models-selector.svelte'; - import { modelsStore, routerModels } from '$lib/stores/models.svelte'; - import { modelLoadFraction } from '$lib/utils'; import { DialogModelInformation, DropdownMenuSearchable, @@ -13,7 +11,6 @@ ModelsSelectorList, ModelsSelectorOption } from '$lib/components/app'; - import ModelLoadHighlight from './ModelLoadHighlight.svelte'; import type { ModelItem } from './utils'; interface Props { @@ -116,17 +113,6 @@ {/if} {:else} {@const selectedOption = ms.getDisplayOption()} - {@const triggerModel = selectedOption?.model} - {@const triggerStatus = triggerModel - ? routerModels().find((m) => m.id === triggerModel)?.status?.value - : undefined} - {@const triggerLoading = - !!triggerModel && - (triggerStatus === ServerModelStatus.LOADING || - modelsStore.isModelOperationInProgress(triggerModel))} - {@const triggerLoadPercent = triggerLoading - ? Math.round(modelLoadFraction(modelsStore.getLoadProgress(triggerModel)) * 100) - : 0} {#if ms.isRouter} @@ -137,7 +123,7 @@ {/if} - - {#if triggerLoading} - - {/if} {/snippet} diff --git a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte index 981111e20..d103d4b67 100644 --- a/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte +++ b/tools/ui/src/lib/components/app/models/ModelsSelectorOption.svelte @@ -10,11 +10,9 @@ RotateCw } from '@lucide/svelte'; import { ActionIcon, ModelId } from '$lib/components/app'; - import ModelLoadHighlight from './ModelLoadHighlight.svelte'; import type { ModelOption } from '$lib/types/models'; import { ServerModelStatus } from '$lib/enums'; import { modelsStore, routerModels } from '$lib/stores/models.svelte'; - import { modelLoadFraction, modelLoadProgressText } from '$lib/utils'; interface Props { option: ModelOption; @@ -52,15 +50,11 @@ (serverStatus === ServerModelStatus.LOADED || isSleeping) && !isOperationInProgress ); let isLoading = $derived(serverStatus === ServerModelStatus.LOADING || isOperationInProgress); - - let loadProgress = $derived(isLoading ? modelsStore.getLoadProgress(option.model) : null); - let loadPercent = $derived(Math.round(modelLoadFraction(loadProgress) * 100)); - let loadTitle = $derived(modelLoadProgressText(loadProgress));
onSelect(option.id)} onmouseenter={onMouseEnter} @@ -86,7 +79,7 @@
e.stopPropagation()} > {#if isFav} @@ -120,16 +113,12 @@
{#if isLoading} -
- -
+ {:else if isFailed} -
- +
+ -