diff --git a/CMakeLists.txt b/CMakeLists.txt index b3f782aaf..6102364f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -519,8 +519,6 @@ add_library(sdtype_adapter otherarch/sdcpp/src/upscaler.cpp otherarch/sdcpp/src/runtime/guidance.cpp otherarch/sdcpp/src/runtime/guidance.h - otherarch/sdcpp/src/runtime/imatrix.cpp - otherarch/sdcpp/src/runtime/imatrix.h otherarch/sdcpp/src/stable-diffusion.cpp otherarch/sdcpp/thirdparty/zip.c otherarch/sdcpp/src/model_io/gguf_io.cpp @@ -537,8 +535,6 @@ add_library(sdtype_adapter otherarch/sdcpp/src/tokenizers/gpt_oss_tokenizer.cpp otherarch/sdcpp/src/tokenizers/tokenizer.cpp otherarch/sdcpp/src/tokenizers/tokenize_util.cpp - otherarch/sdcpp/src/core/backend_fit.cpp - otherarch/sdcpp/src/core/layer_split_partition.cpp otherarch/sdcpp/src/core/ggml_extend_backend.cpp) target_include_directories(sdtype_adapter PUBLIC . ./ggml/include ./ggml/src ./ggml/src/ggml-cpu ./include ./otherarch ./otherarch/tools ./vendor/stb ./vendor/nlohmann ./vendor ./otherarch/sdcpp ./otherarch/sdcpp/include ./otherarch/sdcpp/src ./otherarch/sdcpp/examples ./tools ./common) target_compile_features(sdtype_adapter PUBLIC cxx_std_17) # don't bump @@ -574,10 +570,7 @@ target_link_libraries(embeddings_adapter PRIVATE common2 ggml ${LLAMA_EXTRA_LIBS set_target_properties(embeddings_adapter PROPERTIES POSITION_INDEPENDENT_CODE ON) add_library(gpttype_adapter - gpttype_adapter.cpp - src/llama.cpp - common/chat.cpp - src/llama-model.cpp) + gpttype_adapter.cpp) target_include_directories(gpttype_adapter PUBLIC . ./src ./ggml/include ./ggml/src ./ggml/src/ggml-cpu ./include ./otherarch ./otherarch/tools ./vendor/stb ./vendor/nlohmann ./vendor ./otherarch/sdcpp ./otherarch/sdcpp/thirdparty ./tools ./common) target_compile_features(gpttype_adapter PUBLIC cxx_std_17) # don't bump target_link_libraries(gpttype_adapter PRIVATE common2 ggml ggml_v1 ggml_v2 ggml_v3 ${LLAMA_EXTRA_LIBS}) diff --git a/Makefile b/Makefile index 717c81de9..d0d2f04a7 100644 --- a/Makefile +++ b/Makefile @@ -101,9 +101,9 @@ NONECFLAGS = LLAMA_USE_BUNDLED_GLSLC := 1 FAILSAFE_FLAGS = -DUSE_FAILSAFE -VULKAN_FLAGS = -DGGML_USE_VULKAN +VULKAN_FLAGS = -DGGML_USE_VULKAN -DSD_USE_VULKAN ifdef LLAMA_CUBLAS -CUBLAS_FLAGS = -DGGML_USE_CUDA +CUBLAS_FLAGS = -DGGML_USE_CUDA -DSD_USE_CUDA else CUBLAS_FLAGS = endif @@ -215,7 +215,7 @@ OBJS_CUDA_TEMP_INST += \ ggml/src/ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.o ifdef LLAMA_CUBLAS -CUBLAS_FLAGS = -DGGML_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include +CUBLAS_FLAGS = -DGGML_USE_CUDA -DSD_USE_CUDA -I/usr/local/cuda/include -I/opt/cuda/include -I$(CUDA_PATH)/targets/x86_64-linux/include CUBLASLD_FLAGS = -lcuda -lcublas -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/cuda/lib64 -L/opt/cuda/lib64 -L$(CUDA_PATH)/targets/x86_64-linux/lib -L$(CUDA_PATH)/lib64/stubs -L/usr/local/cuda/targets/aarch64-linux/lib -L/usr/local/cuda/targets/sbsa-linux/lib -L/usr/lib/wsl/lib CUBLAS_OBJS = ggml-cuda.o ggml_v3-cuda.o ggml_v2-cuda.o ggml_v2-cuda-legacy.o CUBLAS_OBJS += $(patsubst %.cu,%.o,$(filter-out ggml/src/ggml-cuda/ggml-cuda.cu, $(wildcard ggml/src/ggml-cuda/*.cu))) @@ -315,7 +315,7 @@ HIPFLAGS += -DGGML_HIP_NO_ROCWMMA_FATTN endif endif -HIPFLAGS += -DGGML_USE_HIP -DGGML_HIP_NO_VMM -DGGML_USE_CUDA $(shell $(ROCM_PATH)/bin/hipconfig -C) +HIPFLAGS += -DGGML_USE_HIP -DGGML_HIP_NO_VMM -DGGML_USE_CUDA -DSD_USE_CUDA $(shell $(ROCM_PATH)/bin/hipconfig -C) HIPLDFLAGS += -L$(ROCM_PATH)/lib -Wl,-rpath=$(ROCM_PATH)/lib HIPLDFLAGS += -L$(ROCM_PATH)/lib64 -Wl,-rpath=$(ROCM_PATH)/lib64 HIPLDFLAGS += -lhipblas -lamdhip64 -lrocblas @@ -339,8 +339,8 @@ endif # LLAMA_HIPBLAS ifdef LLAMA_METAL -CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG -CXXFLAGS += -DGGML_USE_METAL +CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG -DSD_USE_METAL +CXXFLAGS += -DGGML_USE_METAL -DSD_USE_METAL LDFLAGS += -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders OBJS += ggml-metal.o ggml-metal-device.o ggml-metal-device-m.o ggml-metal-context-m.o ggml-metal-common.o ggml-metal-ops.o @@ -682,9 +682,7 @@ ggml-vulkan-shaders-noext.o: ggml/src/ggml-vulkan-shaders-noext.cpp ggml/include $(CXX) $(CXXFLAGS) $(VKGEN_NOEXT_FORCE) $(VULKAN_FLAGS) -c $< -o $@ # intermediate objects -llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-dsa.cpp src/llama-kv-cache-dsv4.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model-saver.cpp src/llama-quant.cpp src/llama-hparams.cpp src/llama-graph.cpp src/llama-io.cpp src/llama-memory.cpp common/fit.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h - $(CXX) $(CXXFLAGS) -c $< -o $@ -llama-model.o: src/llama-model.cpp src/llama-model.h src/models/models.h ggml/include/ggml.h include/llama.h +llama.o: src/llama.cpp ggml/include/ggml.h ggml/include/ggml-alloc.h ggml/include/ggml-backend.h ggml/include/ggml-cuda.h ggml/include/ggml-metal.h include/llama.h otherarch/llama-util.h $(CXX) $(CXXFLAGS) -c $< -o $@ common.o: common/common.cpp common/common.h common/log.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -700,10 +698,8 @@ llama-impl.o: src/llama-impl.cpp src/llama-impl.h $(CXX) $(CXXFLAGS) -c $< -o $@ budget.o: common/reasoning-budget.cpp common/reasoning-budget.h $(CXX) $(CXXFLAGS) -c $< -o $@ -chat.o: common/chat.cpp common/chat.h - $(CXX) $(CXXFLAGS) -c $< -o $@ -SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/diffusion/anima.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h +SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/diffusion/anima.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp @@ -736,7 +732,7 @@ OBJS_SDTYPE := otherarch/sdcpp/sdtype_adapter.o $(OBJS_SDCOMMON) LLAMASERVER_SRCS := tools/server/main.cpp tools/server/server.cpp tools/server/server-schema.cpp tools/server/server-chat.cpp tools/server/server-common.cpp tools/server/server-context.cpp tools/server/server-http.cpp tools/server/server-models.cpp tools/server/server-queue.cpp tools/server/server-task.cpp tools/server/server-tools.cpp tools/server/ui.cpp COMMON_DOWNLOAD_SRCS := common/download.cpp common/hf-cache.cpp vendor/cpp-httplib/httplib.cpp -LLAMASERVER_COMMON_SRCS := common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) +LLAMASERVER_COMMON_SRCS := common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) LLAMASERVER_CXXFLAGS := -I./tools/mtmd @@ -759,7 +755,7 @@ music_default.o: otherarch/acestep/music_adapter.cpp $(CXX) $(CXXFLAGS) -c $< -o $@ # idiotic "for easier compilation" -GPTTYPE_ADAPTER = gpttype_adapter.cpp model_adapter.h otherarch/otherarch.h include/llama.h otherarch/llama_v2.cpp otherarch/llama_v3.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp +GPTTYPE_ADAPTER = gpttype_adapter.cpp common/chat.cpp otherarch/llama_v2.cpp otherarch/llama_v3.cpp src/llama.cpp src/llama-chat.cpp src/llama-mmap.cpp src/llama-context.cpp src/llama-adapter.cpp src/llama-arch.cpp src/llama-batch.cpp src/llama-vocab.cpp src/llama-grammar.cpp src/llama-sampler.cpp src/llama-kv-cache.cpp src/llama-kv-cache-iswa.cpp src/llama-memory-hybrid.cpp src/llama-memory-hybrid-iswa.cpp src/llama-memory-recurrent.cpp src/llama-model-loader.cpp src/llama-model.cpp src/llama-quant.cpp src/llama-hparams.cpp otherarch/gptj_v1.cpp otherarch/gptj_v2.cpp otherarch/gptj_v3.cpp otherarch/gpt2_v1.cpp otherarch/gpt2_v2.cpp otherarch/gpt2_v3.cpp otherarch/rwkv_v2.cpp otherarch/rwkv_v3.cpp otherarch/neox_v2.cpp otherarch/neox_v3.cpp otherarch/mpt_v3.cpp ggml/include/ggml.h ggml/include/ggml-cpu.h ggml/include/ggml-cuda.h include/llama.h otherarch/llama-util.h gpttype_adapter_failsafe.o: $(GPTTYPE_ADAPTER) $(CXX) $(CXXFLAGS) $(FAILSAFE_FLAGS) -c $< -o $@ gpttype_adapter.o: $(GPTTYPE_ADAPTER) @@ -779,36 +775,36 @@ clean: rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o # useful tools -main: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +main: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mainvk: tools/completion/main.cpp tools/completion/completion.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mainvk: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttsmain: tools/tts/tts.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttsmain: tools/tts/tts.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h mtmd.o mtmd-helper.o mtmd-image.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embedding: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embedding: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/chat.cpp common/preset.cpp $(COMMON_DOWNLOAD_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +llamaserver: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib - $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) +llamaservervk: $(LLAMASERVER_SRCS) $(LLAMASERVER_COMMON_SRCS) build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib + $(CXX) $(CXXFLAGS) $(LLAMASERVER_CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) ggml/src/ggml-vulkan-shaders.cpp: ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp ifdef VULKAN_BUILD @@ -908,14 +904,14 @@ else endif #generated libraries -koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(DEFAULT_BUILD) -koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_macos_failsafe: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(DEFAULT_BUILD) ifdef FAILSAFE_BUILD -koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) +koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) $(FAILSAFE_BUILD) else koboldcpp_failsafe: @@ -923,7 +919,7 @@ koboldcpp_failsafe: endif ifdef NOAVX2_BUILD -koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(NOAVX2_BUILD) else koboldcpp_noavx2: @@ -931,7 +927,7 @@ koboldcpp_noavx2: endif ifdef CUBLAS_BUILD -koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) $(CUBLAS_BUILD) else koboldcpp_cublas: @@ -939,7 +935,7 @@ koboldcpp_cublas: endif ifdef HIPBLAS_BUILD -koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o llama.o chat.o llama-model.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) $(HIPBLAS_BUILD) else koboldcpp_hipblas: @@ -947,12 +943,12 @@ koboldcpp_hipblas: endif ifdef VULKAN_BUILD -koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o llama.o chat.o llama-model.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(VULKAN_BUILD) ifdef NOAVX2_BUILD -koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(VULKAN_BUILD) -koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o llama.o chat.o llama-model.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) +koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) $(VULKAN_BUILD) else koboldcpp_vulkan_noavx2: @@ -970,17 +966,17 @@ koboldcpp_vulkan_failsafe: endif # tools -quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o mtmd-image.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o chat.o llama-model.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_ace: otherarch/acestep/quantize-acestep.cpp tools/mtmd/clip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) diff --git a/colab.ipynb b/colab.ipynb index b8778a411..e5194a430 100644 --- a/colab.ipynb +++ b/colab.ipynb @@ -62,7 +62,7 @@ "Model = \"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\" #@param [\"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Tiefighter-GGUF/resolve/main/LLaMA2-13B-Tiefighter.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/LLaMA2-13B-Estopia-GGUF/resolve/main/LLaMA2-13B-Estopia.Q4_K_S.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v4.2.0-GGUF/resolve/main/TheDrummer_Cydonia-24B-v4.2.0-Q4_K_S.gguf\",\"https://huggingface.co/mradermacher/Broken-Tutu-24B-GGUF/resolve/main/Broken-Tutu-24B.Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.3.0-24b-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Harbinger-24B-GGUF/resolve/main/Harbinger-24B-Q4_K_S.gguf\",\"https://huggingface.co/LatitudeGames/Muse-12B-GGUF/resolve/main/Muse-12B-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/Qwen3-VL-8B-Instruct-GGUF/resolve/main/Qwen3-VL-8B-Instruct-Q6_K.gguf\",\"https://huggingface.co/unsloth/Mistral-Small-3.2-24B-Instruct-2506-GGUF/resolve/main/Mistral-Small-3.2-24B-Instruct-2506-Q4_K_S.gguf\",\"https://huggingface.co/ggml-org/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-mxfp4.gguf\",\"https://huggingface.co/KoboldAI/Llama-3.1-8B-BookAdventures-GGUF/resolve/main/Llama-3.1-8B-BookAdventures.Q6_K.gguf\",\"https://huggingface.co/bartowski/google_gemma-3-12b-it-GGUF/resolve/main/google_gemma-3-12b-it-Q4_K_S.gguf\",\"https://huggingface.co/unsloth/gemma-3n-E4B-it-GGUF/resolve/main/gemma-3n-E4B-it-Q6_K.gguf\",\"https://huggingface.co/unsloth/GLM-4-9B-0414-GGUF/resolve/main/GLM-4-9B-0414-Q6_K.gguf\",\"https://huggingface.co/mradermacher/Fimbulvetr-11B-v2-GGUF/resolve/main/Fimbulvetr-11B-v2.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-13B-GGUF/resolve/main/mythomax-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/ReMM-SLERP-L2-13B-GGUF/resolve/main/remm-slerp-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/Xwin-LM-13B-v0.2-GGUF/resolve/main/xwin-lm-13b-v0.2.Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/mini-magnum-12b-v1.1-GGUF/resolve/main/mini-magnum-12b-v1.1.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/Stheno-L2-13B-GGUF/resolve/main/stheno-l2-13b.Q4_K_M.gguf\",\"https://huggingface.co/TheBloke/MythoMax-L2-Kimiko-v2-13B-GGUF/resolve/main/mythomax-l2-kimiko-v2-13b.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/Rocinante-12B-v1.1-GGUF/resolve/main/Rocinante-12B-v1.1-Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/MistRP-Airoboros-7B-GGUF/resolve/main/mistrp-airoboros-7b.Q4_K_S.gguf\",\"https://huggingface.co/TheBloke/airoboros-mistral2.2-7B-GGUF/resolve/main/airoboros-mistral2.2-7b.Q4_K_S.gguf\",\"https://huggingface.co/concedo/KobbleTinyV2-1.1B-GGUF/resolve/main/KobbleTiny-Q4_K.gguf\",\"https://huggingface.co/grimjim/kukulemon-7B-GGUF/resolve/main/kukulemon-7B.Q8_0.gguf\",\"https://huggingface.co/mradermacher/LemonKunoichiWizardV3-GGUF/resolve/main/LemonKunoichiWizardV3.Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Kunoichi-DPO-v2-7B-GGUF-Imatrix/resolve/main/Kunoichi-DPO-v2-7B-Q4_K_M-imatrix.gguf\",\"https://huggingface.co/mradermacher/L3-8B-Stheno-v3.2-i1-GGUF/resolve/main/L3-8B-Stheno-v3.2.i1-Q4_K_M.gguf\",\"https://huggingface.co/Lewdiculous/Llama-3-Lumimaid-8B-v0.1-OAS-GGUF-IQ-Imatrix/resolve/main/v2-Llama-3-Lumimaid-8B-v0.1-OAS-Q4_K_M-imat.gguf\",\"https://huggingface.co/bartowski/NeuralDaredevil-8B-abliterated-GGUF/resolve/main/NeuralDaredevil-8B-abliterated-Q4_K_M.gguf\",\"https://huggingface.co/bartowski/L3-8B-Lunaris-v1-GGUF/resolve/main/L3-8B-Lunaris-v1-Q4_K_M.gguf\",\"https://huggingface.co/mradermacher/L3-Umbral-Mind-RP-v2.0-8B-GGUF/resolve/main/L3-Umbral-Mind-RP-v2.0-8B.Q4_K_M.gguf\",\"https://huggingface.co/bartowski/TheDrummer_Cydonia-24B-v2-GGUF/resolve/main/TheDrummer_Cydonia-24B-v2-Q4_K_S.gguf\",\"https://huggingface.co/bartowski/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-GGUF/resolve/main/PocketDoc_Dans-PersonalityEngine-V1.2.0-24b-IQ4_XS.gguf\",\"https://huggingface.co/mradermacher/Tlacuilo-12B-GGUF/resolve/main/Tlacuilo-12B.Q4_K_S.gguf\"] {\"allow-input\":true}\n", "MdCommand = \"\" #@markdown
\n", "Layers = \"Auto\" #@param [\"Auto\",\"999\"]{allow-input: true}\n", - "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\",\"24576\",\"32768\",\"40960\"] {allow-input: true}\n", + "ContextSize = \"4096\" #@param [\"4096\",\"8192\",\"12288\",\"16384\"] {allow-input: true}\n", "\n", "#@markdown
\n", "LoadVisionMMProjector = False #@param {type:\"boolean\"}\n", @@ -130,7 +130,7 @@ " if Template == \"Gemma4 E4B Uncensored (General)\":\n", " Customized = True\n", " Model = \"https://huggingface.co/HauhauCS/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive/resolve/main/Gemma-4-E4B-Uncensored-HauhauCS-Aggressive-Q5_K_M.gguf\"\n", - " CustomCtxSize = \"40960\"\n", + " CustomCtxSize = \"16384\"\n", " CustomMmproj = \"https://huggingface.co/unsloth/gemma-4-E4B-it-GGUF/resolve/main/mmproj-BF16.gguf\"\n", " if Template == \"Tiefighter 13B (General)\":\n", " Customized = True\n", diff --git a/common/arg.cpp b/common/arg.cpp index bf2b62460..53c207bbc 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -468,7 +468,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params // the first part is what gets loaded, so point params.model.path at it if (!url_tasks.empty()) { std::string first_path = url_tasks.front().local_path; - url_tasks.front().on_done = [&, first_path]() { params.model.path = first_path; }; + url_tasks.front().on_done = [&]() { params.model.path = first_path; }; } for (auto & task : url_tasks) { tasks.push_back(std::move(task)); @@ -497,15 +497,13 @@ void common_models_handler_apply(common_models_handler & handler, common_params } // handle hf_plan tasks - auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files, - const hf_cache::hf_file & primary, - common_params_model & model) { + auto add_tasks = [&opts, &tasks](const hf_cache::hf_files & model_files, common_params_model & model) { for (size_t i = 0; i < model_files.size(); ++i) { auto & model_file = model_files[i]; - bool is_primary = (model_file.path == primary.path); - tasks.emplace_back(model_file, opts, [&, is_primary]() { - if (is_primary) { - // the primary file is the first split (00001-of), use it as model path + bool is_first = (i == 0); + tasks.emplace_back(model_file, opts, [&, is_first]() { + if (is_first) { + // only use first part as model path model.path = hf_cache::finalize_file(model_file); } else { hf_cache::finalize_file(model_file); @@ -514,7 +512,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params } }; if (!plan.model_files.empty()) { - add_tasks(plan.model_files, plan.primary, params.model); + add_tasks(plan.model_files, params.model); } if (!plan.mmproj.local_path.empty()) { tasks.emplace_back(plan.mmproj, opts, [&]() { @@ -542,12 +540,12 @@ void common_models_handler_apply(common_models_handler & handler, common_params // handle plan_spec (e.g. --spec-draft-hf) if (!plan_spec.model_files.empty()) { - add_tasks(plan_spec.model_files, plan_spec.primary, params.speculative.draft.mparams); + add_tasks(plan_spec.model_files, params.speculative.draft.mparams); } // handle vocoder plan (e.g. --hf-repo-v) if (!plan_voc.model_files.empty()) { - add_tasks(plan_voc.model_files, plan_voc.primary, params.vocoder.model); + add_tasks(plan_voc.model_files, params.vocoder.model); } // run all tasks in parallel @@ -3299,20 +3297,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.sampling.reasoning_budget_message = value; } ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_THINK_BUDGET_MESSAGE")); - add_opt(common_arg( - {"--reasoning-preserve"}, - {"--no-reasoning-preserve"}, - "preserve reasoning trace in the full history, not just the last assistant message (default: template default)\n" - "compatible with certain templates having 'supports_preserve_reasoning' capability\n" - "example: https://docs.z.ai/guides/capabilities/thinking-mode#preserved-thinking", - [](common_params & params, bool value) { - if (value) { - params.default_template_kwargs["preserve_reasoning"] = "true"; - } else { - params.default_template_kwargs["preserve_reasoning"] = "false"; - } - } - ).set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_COMPLETION, LLAMA_EXAMPLE_CLI}).set_env("LLAMA_ARG_REASONING_PRESERVE")); add_opt(common_arg( {"--chat-template"}, "JINJA_TEMPLATE", string_format( @@ -3488,7 +3472,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex [](common_params & params) { params.offline = true; } - ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_OFFLINE")); + ).set_env("LLAMA_ARG_OFFLINE")); add_opt(common_arg( {"-lv", "--verbosity", "--log-verbosity"}, "N", string_format("Set the verbosity threshold. Messages with a higher verbosity will be ignored. Values:\n" diff --git a/common/chat.cpp b/common/chat.cpp index 28fba2ff7..6db04e1ff 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -7,6 +7,7 @@ #include "ggml.h" #include "json-schema-to-grammar.h" #include "log.h" +#include "regex-partial.cpp" #include "reasoning-budget.h" #include "chat-auto-parser-generator.cpp" #include "chat-auto-parser-helpers.cpp" @@ -925,10 +926,6 @@ static std::string common_chat_template_direct_apply_impl( if (inputs.add_generation_prompt) { inp["add_generation_prompt"] = true; } - if (inp.contains("preserve_reasoning") && inp["preserve_reasoning"].is_boolean()) { - bool enabled = inp["preserve_reasoning"].get(); - jinja::caps_apply_preserve_reasoning(ctx, enabled); - } jinja::global_from_json(ctx, inp, inputs.mark_input); @@ -2391,166 +2388,6 @@ static void func_args_not_string(json & messages) { } } -// Trim leading/trailing whitespace from message contents before rendering. This -// has to run on the messages (not on the rendered JSON) because templates with -// string-only content caps concatenate typed content parts into a single string -// during rendering, after which the per-part whitespace can no longer be reached. -// Both the plain string content and the text of typed content parts are trimmed. -static void trim_all_content(std::vector & messages) { - for (auto & message : messages) { - message.content = trim_whitespace(message.content); - message.reasoning_content = trim_whitespace(message.reasoning_content); - for (auto & part : message.content_parts) { - if (part.type == "text") { - part.text = trim_whitespace(part.text); - } - } - } -} - -} - -// MiniCPM5 format: -// - Reasoning: {reasoning} (optional) -// - Tool calls: value -static common_chat_params common_chat_params_init_minicpm5(const common_chat_template & tmpl, - const autoparser::generation_params & inputs) { - common_chat_params data; - - data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); - data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); - data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; - data.supports_thinking = true; - data.preserved_tokens = { - "", - "", - "", - "", - }; - - data.thinking_start_tag = ""; - data.thinking_end_tag = ""; - - data.message_delimiters = { - { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, - { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, - { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, - { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, - }; - - auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); - auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); - auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; - auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); - - if (inputs.has_continuation()) { - const auto & msg = inputs.continue_msg; - - data.generation_prompt = "<|im_start|>assistant\n\n" + msg.reasoning_content; - if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { - data.generation_prompt += "\n\n\n" + msg.render_content(); - } - - data.prompt += data.generation_prompt; - } - - auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { - auto generation_prompt = p.literal("<|im_start|>assistant\n"); - - auto reasoning = p.eps(); - if (extract_reasoning) { - reasoning = ("" << p.reasoning(p.until("")) << "") + p.space(); - } - - // Response format parser - if (has_response_format) { - return generation_prompt + reasoning + p.content(p.schema(p.json(), "response-format", inputs.json_schema)); - } - - if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { - // CDATA lets a value carry characters that would otherwise close the tag (e.g. - // ); capture the inner text only, excluding the CDATA markers. - auto string_value = p.choice({ - p.literal("")) + p.literal("]]>"), "]]>") + p.tool_arg_close(p.literal("")), - p.negate(p.literal("")) + p.tool_arg_close(p.literal("")), "") - }); - - auto tool_choice = p.choice(); - foreach_function(inputs.tools, [&](const json & tool) { - const auto & function = tool.at("function"); - const std::string name = function.at("name"); - auto params = function.contains("parameters") ? function.at("parameters") : json::object(); - - auto args = p.eps(); - if (params.contains("properties") && params.at("properties").is_object() && !params.at("properties").empty()) { - auto schema_info = common_schema_info(); - schema_info.resolve_refs(params); - - auto arg_choice = p.choice(); - for (const auto & [prop_name, prop_schema] : params.at("properties").items()) { - auto value_parser = p.eps(); - if (schema_info.resolves_to_string(prop_schema)) { - value_parser = string_value; - } else { - value_parser = p.tool_arg_json_value( - p.schema(p.json(), "tool-" + name + "-arg-" + prop_name + "-schema", prop_schema, false) - ) + p.tool_arg_close(p.literal("")); - } - - auto arg_rule = p.tool_arg( - p.tool_arg_open(p.literal("")) + - value_parser - ); - - arg_choice |= arg_rule; - } - args = p.zero_or_more(arg_choice + p.space()); - } - - auto tool_parser = p.tool( - p.tool_open(p.literal("")) - << p.tool_args(args) - << p.tool_close(p.literal(""))); - - tool_choice |= p.rule("tool-" + name, tool_parser); - }); - - auto max_calls = inputs.parallel_tool_calls ? -1 : 1; - auto tool_calls = p.trigger_rule("tool-call", p.repeat(tool_choice + p.space(), 1, max_calls)); - - auto content = p.content(p.until(" common_chat_try_specialized_template( return common_chat_params_init_gemma4(tmpl, params); } - // MiniCPM5 - XML tool calls with ... - if (src.find("Tool usage guidelines:") != std::string::npos && - src.find("template_tool_use ? *tmpls->template_tool_use : *tmpls->template_default; const auto & src = tmpl.source(); const auto & caps = tmpl.original_caps(); - std::vector trimmed_messages; - const std::vector * messages_to_render = &inputs.messages; - if (src.find("You have access to the following functions in JSONSchema format") != std::string::npos) { - // StepFun: trim message contents (including typed content parts) before rendering, - // otherwise leftover whitespace drives the model into reasoning loops (issue #24181) - trimmed_messages = inputs.messages; - workaround::trim_all_content(trimmed_messages); - messages_to_render = &trimmed_messages; - } - params.messages = render_message_to_json(*messages_to_render, tmpl.original_caps()); + params.messages = render_message_to_json(inputs.messages, tmpl.original_caps()); params.tool_choice = inputs.tool_choice; params.reasoning_format = inputs.reasoning_format; params.enable_thinking = inputs.enable_thinking; diff --git a/common/common.cpp b/common/common.cpp index 14f1a7892..44c6f8ab1 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -231,7 +231,7 @@ bool set_process_priority(enum ggml_sched_priority prio) { } if (!SetPriorityClass(GetCurrentProcess(), p)) { - COM_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError()); + LOG_WRN("failed to set process priority class %d : (%d)\n", prio, (int) GetLastError()); return false; } @@ -257,7 +257,7 @@ bool set_process_priority(enum ggml_sched_priority prio) { } if (setpriority(PRIO_PROCESS, 0, p) != 0) { - COM_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno); + LOG_WRN("failed to set process priority %d : %s (%d)\n", prio, strerror(errno), errno); return false; } return true; @@ -290,14 +290,14 @@ void postprocess_cpu_params(common_cpu_params & cpuparams, const common_cpu_para if (n_set && n_set < cpuparams.n_threads) { // Not enough set bits, may experience performance issues. - COM_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads); + LOG_WRN("Not enough set bits in CPU mask (%d) to satisfy requested thread count: %d\n", n_set, cpuparams.n_threads); } } bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THREADS]) { size_t dash_loc = range.find('-'); if (dash_loc == std::string::npos) { - COM_ERR("%s", "Format of CPU range is invalid! Expected []-[].\n"); + LOG_ERR("Format of CPU range is invalid! Expected []-[].\n"); return false; } @@ -309,7 +309,7 @@ bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THRE } else { start_i = std::stoull(range.substr(0, dash_loc)); if (start_i >= GGML_MAX_N_THREADS) { - COM_ERR("%s", "Start index out of bounds!\n"); + LOG_ERR("Start index out of bounds!\n"); return false; } } @@ -319,7 +319,7 @@ bool parse_cpu_range(const std::string & range, bool (&boolmask)[GGML_MAX_N_THRE } else { end_i = std::stoull(range.substr(dash_loc + 1)); if (end_i >= GGML_MAX_N_THREADS) { - COM_ERR("%s", "End index out of bounds!\n"); + LOG_ERR("End index out of bounds!\n"); return false; } } @@ -339,7 +339,7 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD } size_t num_digits = mask.length() - start_i; - num_digits = std::min(num_digits, 128); + if (num_digits > 128) num_digits = 128; size_t end_i = num_digits + start_i; @@ -354,7 +354,7 @@ bool parse_cpu_mask(const std::string & mask, bool (&boolmask)[GGML_MAX_N_THREAD } else if (c >= 'A' && c <= 'F') { id -= 'A' - 10; } else { - COM_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i)); + LOG_ERR("Invalid hex character '%c' at position %d\n", c, int32_t(i)); return false; } @@ -385,21 +385,21 @@ void common_params_print_info(const common_params & params, bool print_devices) #else const char * build_type = " (debug)"; #endif - COM_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type); + LOG_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type); - COM_INF("%s: verbosity = %d (adjust with the `-lv N` CLI arg)\n", __func__, common_log_get_verbosity_thold()); + LOG_INF("log_info: verbosity = %d (adjust with the `-lv N` CLI arg)\n", common_log_get_verbosity_thold()); // device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device if (print_devices) { - COM_TRC("%s", "device_info:\n"); + LOG_INF("device_info:\n"); for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { auto * dev = ggml_backend_dev_get(i); size_t free, total; ggml_backend_dev_memory(dev, &free, &total); - COM_TRC(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); + LOG_INF(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024); } } - COM_TRC("%s\n", common_params_get_system_info(params).c_str()); + LOG_INF("%s\n", common_params_get_system_info(params).c_str()); } std::string common_params_get_system_info(const common_params & params) { @@ -666,7 +666,7 @@ void string_process_escapes(std::string & input) { bool string_parse_kv_override(const char * data, std::vector & overrides) { const char * sep = strchr(data, '='); if (sep == nullptr || sep - data >= 128) { - COM_ERR("%s: malformed KV override '%s'\n", __func__, data); + LOG_ERR("%s: malformed KV override '%s'\n", __func__, data); return false; } llama_model_kv_override kvo; @@ -689,20 +689,20 @@ bool string_parse_kv_override(const char * data, std::vector 127) { - COM_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); + LOG_ERR("%s: malformed KV override '%s', value cannot exceed 127 chars\n", __func__, data); return false; } strncpy(kvo.val_str, sep, 127); kvo.val_str[127] = '\0'; } else { - COM_ERR("%s: invalid type for KV override '%s'\n", __func__, data); + LOG_ERR("%s: invalid type for KV override '%s'\n", __func__, data); return false; } overrides.emplace_back(std::move(kvo)); @@ -1205,8 +1205,8 @@ common_init_result::common_init_result(common_params & params, bool model_only) auto cparams = common_context_params_to_llama(params); if (params.fit_params) { - COM_TRC("%s", "fitting params to device memory ...\n"); - COM_TRC("%s", "(for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n"); + LOG_INF("%s: fitting params to device memory ...\n", __func__); + LOG_INF("%s: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on)\n", __func__); common_fit_params(params.model.path.c_str(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), @@ -1233,7 +1233,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) llama_adapter_lora_ptr lora; lora.reset(llama_adapter_lora_init(model, la.path.c_str())); if (lora == nullptr) { - COM_ERR("failed to load lora adapter '%s'\n", la.path.c_str()); + LOG_ERR("%s: failed to load lora adapter '%s'\n", __func__, la.path.c_str()); pimpl->model.reset(model); return; } @@ -1252,14 +1252,14 @@ common_init_result::common_init_result(common_params & params, bool model_only) common_init_sampler_from_model(model, params.sampling); if (params.sampling.ignore_eos && llama_vocab_eos(vocab) == LLAMA_TOKEN_NULL) { - COM_WRN("%s", "vocab does not have an EOS token, ignoring --ignore-eos\n"); + LOG_WRN("%s: warning: vocab does not have an EOS token, ignoring --ignore-eos\n", __func__); params.sampling.ignore_eos = false; } // initialize once for (llama_token i = 0; i < llama_vocab_n_tokens(vocab); i++) { if (llama_vocab_is_eog(vocab, i)) { - COM_TRC("added %s logit bias = %f\n", common_token_to_piece(vocab, i).c_str(), -INFINITY); + LOG_TRC("%s: added %s logit bias = %f\n", __func__, common_token_to_piece(vocab, i).c_str(), -INFINITY); params.sampling.logit_bias_eog.push_back({i, -INFINITY}); } } @@ -1297,7 +1297,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) llama_context * lctx = llama_init_from_model(model, cparams); if (lctx == NULL) { - COM_ERR("failed to create context with model '%s'\n", params.model.path.c_str()); + LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); return; } @@ -1334,7 +1334,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode llama_model * model = res->model(); if (model == NULL) { - COM_ERR("failed to load model '%s'\n", params.model.path.c_str()); + LOG_ERR("%s: failed to load model '%s'\n", __func__, params.model.path.c_str()); return res; } @@ -1344,14 +1344,14 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode llama_context * lctx = res->context(); if (lctx == NULL) { - COM_ERR("failed to create context with model '%s'\n", params.model.path.c_str()); + LOG_ERR("%s: failed to create context with model '%s'\n", __func__, params.model.path.c_str()); return res; } const llama_vocab * vocab = llama_model_get_vocab(model); if (params.ctx_shift && !llama_memory_can_shift(llama_get_memory(lctx))) { - COM_WRN("%s", "KV cache shifting is not supported for this context, disabling KV cache shifting\n"); + LOG_WRN("%s: KV cache shifting is not supported for this context, disabling KV cache shifting\n", __func__); params.ctx_shift = false; } @@ -1380,7 +1380,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode bool ok = true; if (llama_vocab_bos(vocab) == LLAMA_TOKEN_NULL) { - COM_WRN("%s", "vocab does not have a BOS token, reranking will not work\n"); + LOG_WRN("%s: warning: vocab does not have a BOS token, reranking will not work\n", __func__); ok = false; } @@ -1389,10 +1389,10 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode bool has_rerank_prompt = llama_model_chat_template(model, "rerank") != NULL; if (!has_eos && !has_sep && !has_rerank_prompt) { - COM_WRN("%s", "vocab does not have an EOS token, SEP token, or rerank prompt. Reranking will not work\n"); + LOG_WRN("%s: warning: vocab does not have an EOS token, SEP token, or rerank prompt. Reranking will not work\n", __func__); ok = false; } else if (!has_eos) { - COM_WRN("%s", "vocab does not have an EOS token, using SEP token as fallback\n"); + LOG_WRN("%s: warning: vocab does not have an EOS token, using SEP token as fallback\n", __func__); } if (!ok) { @@ -1405,7 +1405,7 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode } if (params.warmup) { - COM_TRC("%s", "warming up the model with an empty run - please wait ... (--no-warmup to disable)\n"); + LOG_INF("%s: warming up the model with an empty run - please wait ... (--no-warmup to disable)\n", __func__); std::vector tmp; llama_token bos = llama_vocab_bos(vocab); @@ -1479,20 +1479,20 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) { int ret = llama_decode(ctx, llama_batch_get_one(tmp.data(), tmp.size())); if (ret != 0) { - COM_ERR("llama_decode() failed: %d\n", ret); + LOG_ERR("%s: llama_decode() failed: %d\n", __func__, ret); res = COMMON_CONTEXT_SEQ_RM_TYPE_NO; goto done; } if (llama_n_rs_seq(ctx) > 0) { - COM_TRC("%s", "the context supports bounded partial sequence removal\n"); + LOG_INF("%s: the context supports bounded partial sequence removal\n", __func__); res = COMMON_CONTEXT_SEQ_RM_TYPE_RS; goto done; } // try to remove the last tokens if (!llama_memory_seq_rm(mem, 0, 1, -1)) { - COM_TRC("%s", "the context does not support partial sequence removal\n"); + LOG_TRC("%s: the context does not support partial sequence removal\n", __func__); res = COMMON_CONTEXT_SEQ_RM_TYPE_FULL; goto done; } @@ -1809,13 +1809,13 @@ static common_control_vector_data common_control_vector_load_one(const common_co }; struct gguf_context * ctx_gguf = gguf_init_from_file(load_info.fname.c_str(), meta_gguf_params); if (!ctx_gguf) { - COM_ERR("failed to load control vector file from %s\n", load_info.fname.c_str()); + LOG_ERR("%s: failed to load control vector file from %s\n", __func__, load_info.fname.c_str()); return result; } int32_t n_tensors = gguf_get_n_tensors(ctx_gguf); if (n_tensors == 0) { - COM_WRN("no direction tensors found in %s\n", load_info.fname.c_str()); + LOG_WRN("%s: no direction tensors found in %s\n", __func__, load_info.fname.c_str()); } for (int i = 0; i < n_tensors; i++) { @@ -1833,23 +1833,23 @@ static common_control_vector_data common_control_vector_load_one(const common_co } } if (layer_idx < 0) { - COM_ERR("invalid/unparsable direction tensor layer index in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid/unparsable direction tensor layer index in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } else if (layer_idx == 0) { - COM_ERR("invalid (zero) direction tensor layer index in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid (zero) direction tensor layer index in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } struct ggml_tensor * tensor = ggml_get_tensor(ctx, name.c_str()); if (tensor->type != GGML_TYPE_F32) { - COM_ERR("invalid (non-F32) direction tensor type in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid (non-F32) direction tensor type in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } if (ggml_n_dims(tensor) != 1) { - COM_ERR("invalid (non-1D) direction tensor shape in %s\n", load_info.fname.c_str()); + LOG_ERR("%s: invalid (non-1D) direction tensor shape in %s\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } @@ -1857,7 +1857,7 @@ static common_control_vector_data common_control_vector_load_one(const common_co if (result.n_embd == -1) { result.n_embd = ggml_nelements(tensor); } else if (ggml_nelements(tensor) != result.n_embd) { - COM_ERR("direction tensor in %s does not match previous dimensions\n", load_info.fname.c_str()); + LOG_ERR("%s: direction tensor in %s does not match previous dimensions\n", __func__, load_info.fname.c_str()); result.n_embd = -1; break; } @@ -1874,7 +1874,7 @@ static common_control_vector_data common_control_vector_load_one(const common_co } if (result.n_embd == -1) { - COM_WRN("skipping %s due to invalid direction tensors\n", load_info.fname.c_str()); + LOG_WRN("%s: skipping %s due to invalid direction tensors\n", __func__, load_info.fname.c_str()); result.data.clear(); } @@ -1895,7 +1895,7 @@ common_control_vector_data common_control_vector_load(const std::vector(all_tokens.data() + offset), n_tokens_before_last))) { - COM_ERR("%s", "failed to eval\n"); + LOG_ERR("%s : failed to eval\n", __func__); return false; } n_past += n_tokens_before_last; llama_state_save_file(ctx, state_path.data(), all_tokens.data(), all_tokens.size()); - COM_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size()); + LOG_INF("saved session before last token to %s, n_new = %zu\n", state_path.data(), all_tokens.size()); llama_token last_token = all_tokens.back(); llama_batch batch = llama_batch_get_one(&last_token, 1); @@ -2036,13 +2036,13 @@ bool common_prompt_batch_decode( batch.pos = &pos; if (llama_decode(ctx, batch)) { - COM_ERR("%s", "failed to eval last token\n"); + LOG_ERR("%s : failed to eval last token\n", __func__); return false; } n_past++; } else { if (llama_decode(ctx, llama_batch_get_one(const_cast(all_tokens.data() + offset), n_new))) { - COM_ERR("%s", "failed to eval\n"); + LOG_ERR("%s : failed to eval\n", __func__); return false; } n_past += n_new; diff --git a/common/common.h b/common/common.h index 7939d3b07..819548103 100644 --- a/common/common.h +++ b/common/common.h @@ -26,13 +26,6 @@ #define DIRECTORY_SEPARATOR '/' #endif // _WIN32 -#define COM_DBG(fmt, ...) LOG_DBG("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_TRC(fmt, ...) LOG_TRC("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_INF(fmt, ...) LOG_INF("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_WRN(fmt, ...) LOG_WRN("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_ERR(fmt, ...) LOG_ERR("cmn %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define COM_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__) - #define die(msg) do { fputs("error: " msg "\n", stderr); exit(1); } while (0) #define die_fmt(fmt, ...) do { fprintf(stderr, "error: " fmt "\n", __VA_ARGS__); exit(1); } while (0) @@ -170,7 +163,6 @@ enum common_speculative_type { COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE, // standalone draft model speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, // Eagle3 speculative decoding COMMON_SPECULATIVE_TYPE_DRAFT_MTP, // Multi-token prediction - COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, // DFlash speculative decoding COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE, // simple self-speculative decoding based on n-grams COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K, // self-speculative decoding with n-gram keys only COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V, // self-speculative decoding with n-gram keys and 4 m-gram values @@ -386,7 +378,7 @@ struct common_params_speculative { uint32_t need_n_rs_seq() const { bool needs_rs_seq = std::any_of(types.begin(), types.end(), [&](auto t) { - return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3 || t == COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH; + return t == COMMON_SPECULATIVE_TYPE_DRAFT_MTP || t == COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3; }); return needs_rs_seq ? draft.n_max : 0u; diff --git a/common/fit.cpp b/common/fit.cpp index afbf0b10f..a8565bfc9 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -233,7 +233,7 @@ static void common_params_fit_impl( sum_projected_used = dmds_full.back().mb.total(); sum_free = dmds_full.back().total; sum_projected_free = sum_free - sum_projected_used; - LOG_TRC("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n", + LOG_INF("%s: projected to use %" PRId64 " MiB of host memory vs. %" PRId64 " MiB of total host memory\n", __func__, sum_projected_used/MiB, sum_free/MiB); if (sum_projected_free >= margins[0]) { LOG_TRC("%s: will leave %" PRId64 " >= %" PRId64 " MiB of system memory, no changes needed\n", diff --git a/common/http.h b/common/http.h index e88bc6a5e..d3daccd6b 100644 --- a/common/http.h +++ b/common/http.h @@ -11,11 +11,6 @@ struct common_http_url { std::string path; }; -// bracket an IPv6 literal host for a URL authority (RFC 3986) -static std::string common_http_format_host(const std::string & host) { - return host.find(':') != std::string::npos ? "[" + host + "]" : host; -} - static common_http_url common_http_parse_url(const std::string & url) { common_http_url parts; auto scheme_end = url.find("://"); @@ -54,28 +49,11 @@ static common_http_url common_http_parse_url(const std::string & url) { parts.path = "/"; } - // split the authority into host and optional port, a bracketed IPv6 literal keeps its inner colons (RFC 3986) - std::string port_str; - if (!parts.host.empty() && parts.host.front() == '[') { - auto close = parts.host.find(']'); - if (close == std::string::npos) { - throw std::runtime_error("invalid IPv6 URL authority: " + parts.host); - } - auto after = parts.host.substr(close + 1); - if (!after.empty() && after.front() == ':') { - port_str = after.substr(1); - } - parts.host = parts.host.substr(1, close - 1); - } else { - auto colon_pos = parts.host.find(':'); - if (colon_pos != std::string::npos) { - port_str = parts.host.substr(colon_pos + 1); - parts.host = parts.host.substr(0, colon_pos); - } - } + auto colon_pos = parts.host.find(':'); - if (!port_str.empty()) { - parts.port = std::stoi(port_str); + if (colon_pos != std::string::npos) { + parts.port = std::stoi(parts.host.substr(colon_pos + 1)); + parts.host = parts.host.substr(0, colon_pos); } else if (parts.scheme == "http") { parts.port = 80; } else if (parts.scheme == "https") { @@ -105,7 +83,7 @@ static std::pair common_http_client(const std: } #endif - httplib::Client cli(parts.scheme + "://" + common_http_format_host(parts.host) + ":" + std::to_string(parts.port)); + httplib::Client cli(parts.scheme + "://" + parts.host + ":" + std::to_string(parts.port)); if (!parts.user.empty()) { cli.set_basic_auth(parts.user, parts.password); @@ -117,5 +95,5 @@ static std::pair common_http_client(const std: } static std::string common_http_show_masked_url(const common_http_url & parts) { - return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + common_http_format_host(parts.host) + parts.path; + return parts.scheme + "://" + (parts.user.empty() ? "" : "****:****@") + parts.host + parts.path; } diff --git a/common/jinja/caps.cpp b/common/jinja/caps.cpp index f7de8b806..ead864763 100644 --- a/common/jinja/caps.cpp +++ b/common/jinja/caps.cpp @@ -9,9 +9,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-caps" using json = nlohmann::ordered_json; @@ -19,34 +16,22 @@ using json = nlohmann::ordered_json; namespace jinja { using caps_json_fn = std::function; -using caps_ctx_fn = std::function; -using caps_analyze_fn = std::function; - -void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled) { - ctx.set_val("preserve_thinking", mk_val(enabled)); - ctx.set_val("clear_thinking", mk_val(!enabled)); - ctx.set_val("truncate_history_thinking", mk_val(!enabled)); -} +using caps_analyze_fn = std::function; static void caps_try_execute(jinja::program & prog, const caps_json_fn & messages_fn, - const caps_ctx_fn & ctx_fn, const caps_json_fn & tools_fn, const caps_analyze_fn & analyze_fn) { context ctx; ctx.is_get_stats = true; jinja::global_from_json(ctx, json{ {"messages", messages_fn()}, - {"tools", tools_fn ? tools_fn() : json::array()}, + {"tools", tools_fn()}, {"bos_token", ""}, {"eos_token", ""}, {"add_generation_prompt", true} }, true); - if (ctx_fn) { - ctx_fn(ctx); - } - auto messages = ctx.get_val("messages"); auto tools = ctx.get_val("tools"); @@ -64,7 +49,7 @@ static void caps_try_execute(jinja::program & prog, // ignore exceptions during capability analysis } - analyze_fn(success, messages, tools, result); + analyze_fn(success, messages, tools); } // for debugging only @@ -124,9 +109,11 @@ caps caps_get(jinja::program & prog) { } }); }, - nullptr, // ctx_fn - nullptr, // tools_fn - [&](bool success, value & messages, value &, const std::string &) { + [&]() { + // tools + return json{nullptr}; + }, + [&](bool success, value & messages, value &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (has_op(content, "selectattr") || has_op(content, "array_access")) { @@ -158,9 +145,11 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn - nullptr, // tools_fn - [&](bool, value & messages, value &, const std::string &) { + [&]() { + // tools + return json::array(); + }, + [&](bool, value & messages, value &) { auto & content = messages->at(0)->at("content"); caps_print_stats(content, "messages[0].content"); if (!content->stats.used) { @@ -212,7 +201,6 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -236,7 +224,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](bool success, value & messages, value & tools) { if (!success) { return; // Nothing can be inferred } @@ -305,7 +293,6 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -329,7 +316,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value & tools, const std::string &) { + [&](bool success, value & messages, value & tools) { if (!success) { result.supports_tool_calls = false; result.supports_tools = false; @@ -407,7 +394,6 @@ caps caps_get(jinja::program & prog) { }, }); }, - nullptr, // ctx_fn [&]() { // tools return json::array({ @@ -431,7 +417,7 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](bool success, value & messages, value &, const std::string &) { + [&](bool success, value & messages, value & /*tools*/) { if (!success) { result.supports_parallel_tool_calls = false; return; @@ -452,22 +438,11 @@ caps caps_get(jinja::program & prog) { JJ_DEBUG("%s\n", ">>> Running capability check: preserve reasoning"); // case: preserve reasoning content in chat history - const std::string reasoning_placeholder = ""; caps_try_execute( prog, [&]() { // messages return json::array({ - { - {"role", "user"}, - {"content", "User message"} - }, - { - {"role", "assistant"}, - {"content", "Assistant message"}, - // check of reasoning_content deeper in the history, not just the last assistant message - {"reasoning_content", reasoning_placeholder} - }, { {"role", "user"}, {"content", "User message"} @@ -483,13 +458,14 @@ caps caps_get(jinja::program & prog) { }, }); }, - [&](context & ctx) { - caps_apply_preserve_reasoning(ctx, true); + [&]() { + // tools + return json::array(); }, - nullptr, // tools_fn - [&](bool, value &, value &, const std::string & output) { - // note: we cannot use stats here because the reasoning_content may be used for "if" condition test, but not actually outputted in the final result - if (output.find(reasoning_placeholder) != std::string::npos) { + [&](bool, value & messages, value &) { + auto & content = messages->at(1)->at("reasoning_content"); + caps_print_stats(content, "messages[1].reasoning_content"); + if (content->stats.used) { result.supports_preserve_reasoning = true; } } diff --git a/common/jinja/caps.h b/common/jinja/caps.h index a290cd7da..93a7fe092 100644 --- a/common/jinja/caps.h +++ b/common/jinja/caps.h @@ -12,9 +12,7 @@ struct caps { bool supports_tool_calls = true; bool supports_system_role = true; bool supports_parallel_tool_calls = true; - - // supports preserve reasoning trace in the full history, not just the last assistant message - bool supports_preserve_reasoning = false; + bool supports_preserve_reasoning = false; // support assistant message with reasoning_content // one of the 2 content capabilities must be true bool supports_string_content = true; @@ -31,6 +29,4 @@ struct caps { caps caps_get(jinja::program & prog); -void caps_apply_preserve_reasoning(jinja::context & ctx, bool enabled); - } // namespace jinja diff --git a/common/jinja/lexer.cpp b/common/jinja/lexer.cpp index 63740dff6..598982c2f 100644 --- a/common/jinja/lexer.cpp +++ b/common/jinja/lexer.cpp @@ -7,9 +7,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-lexer" namespace jinja { diff --git a/common/jinja/parser.cpp b/common/jinja/parser.cpp index e9e6dcb7f..2b25654a7 100644 --- a/common/jinja/parser.cpp +++ b/common/jinja/parser.cpp @@ -8,9 +8,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-parser" namespace jinja { diff --git a/common/jinja/runtime.cpp b/common/jinja/runtime.cpp index afba0025f..f98cb0876 100644 --- a/common/jinja/runtime.cpp +++ b/common/jinja/runtime.cpp @@ -8,9 +8,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-runtime" bool g_jinja_debug = false; @@ -957,50 +954,4 @@ value keyword_argument_expression::execute_impl(context & ctx) { return mk_val(k, v); } -std::string runtime::debug_dump_program(const program & prog, const std::string & src) { - std::ostringstream oss; - size_t lvl = 0; - context ctx; - ctx.src.reset(new std::string(src)); - - auto indent = [](size_t lvl) -> std::string { - return std::string(lvl * 2, ' '); - }; - - ctx.visitor = [&](bool is_leaf, statement * node, std::vector children) { - oss << indent(lvl) << node->type() << ":\n"; - lvl++; - if (is_leaf) { - const auto & pos = node->pos; - oss << indent(lvl) << "(leaf) at " << get_line_col(src, pos) << " in source:\n"; - std::string snippet = peak_source(src, pos); - string_replace_all(snippet, "\n", "\n" + indent(lvl)); - oss << indent(lvl) << snippet << "\n"; - } else { - for (auto & [label, children_vec] : children) { - oss << indent(lvl) << label << ":\n"; - lvl++; - if (children_vec.empty()) { - oss << indent(lvl) << "\n\n"; - } else { - for (auto * child : children_vec) { - if (!child) { - continue; - } - child->visit(ctx); - } - } - lvl--; - } - } - lvl--; - }; - - for (const auto & stmt : prog.body) { - stmt->visit(ctx); - } - - return oss.str(); -} - } // namespace jinja diff --git a/common/jinja/runtime.h b/common/jinja/runtime.h index 0884a1592..37b4c35ca 100644 --- a/common/jinja/runtime.h +++ b/common/jinja/runtime.h @@ -47,19 +47,12 @@ const T * cast_stmt(const statement_ptr & ptr) { // not thread-safe void enable_debug(bool enable); -// for visiting AST nodes -// function signature: void(bool is_leaf, statement * node, pair of ) -using visitor_pair = std::pair>; -using visitor_fn = std::function)>; - struct context { std::shared_ptr src; // for debugging; use shared_ptr to avoid copying on scope creation std::time_t current_time; // for functions that need current time bool is_get_stats = false; // whether to collect stats - visitor_fn visitor; - // src is optional, used for error reporting context(std::string src = "") : src(std::make_shared(std::move(src))) { env = mk_val(); @@ -106,15 +99,6 @@ private: value_object env; }; -// utils for visiting AST nodes -static std::vector stmts_to_ptr(const statements & stmts) { - std::vector children; - for (const auto & stmt : stmts) { - children.push_back(stmt.get()); - } - return children; -} - /** * Base class for all nodes in the AST. */ @@ -122,7 +106,6 @@ struct statement { size_t pos; // position in source, for debugging virtual ~statement() = default; virtual std::string type() const { return "Statement"; } - virtual void visit(context & ctx) { ctx.visitor(true, this, {}); } // execute_impl must be overridden by derived classes virtual value execute_impl(context &) { throw_exec_error(); } @@ -183,13 +166,6 @@ struct if_statement : public statement { std::string type() const override { return "If"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"test", {test.get()}}, - {"body", stmts_to_ptr(body)}, - {"alternate", stmts_to_ptr(alternate)} - }); - } }; struct identifier; @@ -214,14 +190,6 @@ struct for_statement : public statement { std::string type() const override { return "For"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"loopvar", {loopvar.get()}}, - {"iterable", {iterable.get()}}, - {"body", stmts_to_ptr(body)}, - {"default_block", stmts_to_ptr(default_block)} - }); - } }; struct break_statement : public statement { @@ -273,13 +241,6 @@ struct set_statement : public statement { std::string type() const override { return "Set"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"assignee", {assignee.get()}}, - {"value", {val.get()}}, - {"body", stmts_to_ptr(body)} - }); - } }; struct macro_statement : public statement { @@ -295,13 +256,6 @@ struct macro_statement : public statement { std::string type() const override { return "Macro"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"name", {name.get()}}, - {"args", stmts_to_ptr(args)}, - {"body", stmts_to_ptr(body)} - }); - } }; struct comment_statement : public statement { @@ -335,12 +289,6 @@ struct member_expression : public expression { } std::string type() const override { return "MemberExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"object", {object.get()}}, - {"property", {property.get()}} - }); - } }; struct call_expression : public expression { @@ -354,12 +302,6 @@ struct call_expression : public expression { } std::string type() const override { return "CallExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"callee", {callee.get()}}, - {"args", stmts_to_ptr(args)} - }); - } }; /** @@ -463,12 +405,6 @@ struct binary_expression : public expression { } std::string type() const override { return "BinaryExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"left", {left.get()}}, - {"right", {right.get()}} - }); - } }; /** @@ -495,12 +431,6 @@ struct filter_expression : public expression { std::string type() const override { return "FilterExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"operand", {operand.get()}}, - {"filter", {filter.get()}} - }); - } }; struct filter_statement : public statement { @@ -513,12 +443,6 @@ struct filter_statement : public statement { } std::string type() const override { return "FilterStatement"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"filter", {filter.get()}}, - {"body", stmts_to_ptr(body)} - }); - } }; /** @@ -544,12 +468,6 @@ struct select_expression : public expression { } return lhs->execute_impl(ctx); } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"lhs", {lhs.get()}}, - {"test", {test.get()}} - }); - } }; /** @@ -568,12 +486,6 @@ struct test_expression : public expression { } std::string type() const override { return "TestExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"operand", {operand.get()}}, - {"test", {test.get()}} - }); - } }; /** @@ -589,11 +501,6 @@ struct unary_expression : public expression { } std::string type() const override { return "UnaryExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"argument", {argument.get()}} - }); - } }; struct slice_expression : public expression { @@ -611,13 +518,6 @@ struct slice_expression : public expression { [[noreturn]] value execute_impl(context &) override { throw std::runtime_error("must be handled by MemberExpression"); } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"start_expr", {start_expr.get()}}, - {"stop_expr", {stop_expr.get()}}, - {"step_expr", {step_expr.get()}} - }); - } }; struct keyword_argument_expression : public expression { @@ -631,12 +531,6 @@ struct keyword_argument_expression : public expression { } std::string type() const override { return "KeywordArgumentExpression"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"key", {key.get()}}, - {"val", {val.get()}} - }); - } }; struct spread_expression : public expression { @@ -645,11 +539,6 @@ struct spread_expression : public expression { chk_type(this->argument); } std::string type() const override { return "SpreadExpression"; } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"argument", {argument.get()}} - }); - } }; struct call_statement : public statement { @@ -664,13 +553,6 @@ struct call_statement : public statement { } std::string type() const override { return "CallStatement"; } value execute_impl(context & ctx) override; - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"call", {call.get()}}, - {"caller_args", stmts_to_ptr(caller_args)}, - {"body", stmts_to_ptr(body)} - }); - } }; struct ternary_expression : public expression { @@ -693,13 +575,6 @@ struct ternary_expression : public expression { return false_expr->execute(ctx); } } - void visit(context & ctx) override { - ctx.visitor(false, this, { - {"condition", {condition.get()}}, - {"true_expr", {true_expr.get()}}, - {"false_expr", {false_expr.get()}} - }); - } }; struct raised_exception : public std::exception { @@ -773,8 +648,6 @@ struct runtime { } return parts; } - - static std::string debug_dump_program(const program & prog, const std::string & src); }; } // namespace jinja diff --git a/common/jinja/value.cpp b/common/jinja/value.cpp index f36bbbc0c..189451620 100644 --- a/common/jinja/value.cpp +++ b/common/jinja/value.cpp @@ -12,9 +12,6 @@ #include #include -#ifdef FILENAME -#undef FILENAME -#endif #define FILENAME "jinja-value" namespace jinja { @@ -1111,50 +1108,6 @@ const func_builtins & value_array_t::get_builtins() const { std::reverse(arr.begin(), arr.end()); return is_val(val) ? mk_val(std::move(arr)) : mk_val(std::move(arr)); }}, - {"min", [](const func_args & args) -> value { - args.ensure_count(1, 4); - args.ensure_vals(); - value val_case = args.get_kwarg_or_pos("case_sensitive", 1); - value attribute = args.get_kwarg_or_pos("attribute", 2); - if (!attribute->is_undefined()) { - throw not_implemented_exception("min: attribute not implemented"); - } - // FIXME: min is currently always case sensitive - (void) val_case; - const auto & arr = args.get_pos(0)->as_array(); - if (arr.empty()) { - return mk_val(); - } - value result = arr[0]; - for (size_t i = 1; i < arr.size(); ++i) { - if (value_compare(arr[i], result, value_compare_op::lt)) { - result = arr[i]; - } - } - return result; - }}, - {"max", [](const func_args & args) -> value { - args.ensure_count(1, 4); - args.ensure_vals(); - value val_case = args.get_kwarg_or_pos("case_sensitive", 1); - value attribute = args.get_kwarg_or_pos("attribute", 2); - if (!attribute->is_undefined()) { - throw not_implemented_exception("max: attribute not implemented"); - } - // FIXME: max is currently always case sensitive - (void) val_case; - const auto & arr = args.get_pos(0)->as_array(); - if (arr.empty()) { - return mk_val(); - } - value result = arr[0]; - for (size_t i = 1; i < arr.size(); ++i) { - if (value_compare(arr[i], result, value_compare_op::gt)) { - result = arr[i]; - } - } - return result; - }}, {"unique", array_unique_not_implemented}, }; return builtins; diff --git a/common/preset.cpp b/common/preset.cpp index 4362c0621..f0cc1fa1a 100644 --- a/common/preset.cpp +++ b/common/preset.cpp @@ -7,7 +7,6 @@ #include #include #include -#include static std::string rm_leading_dashes(const std::string & str) { size_t pos = 0; @@ -17,23 +16,6 @@ static std::string rm_leading_dashes(const std::string & str) { return str.substr(pos); } -static std::string canonical_tag(const std::string & tag) { - static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); - std::smatch m; - if (std::regex_search(tag, m, re_tag)) { - std::string canon = m[1].str(); - for (char & c : canon) { - c = (char) std::toupper((unsigned char) c); - } - return canon; - } - std::string upper = tag; - for (char & c : upper) { - c = (char) std::toupper((unsigned char) c); - } - return upper; -} - std::vector common_preset::to_args(const std::string & bin_path) const { std::vector args; @@ -288,18 +270,11 @@ common_presets common_preset_context::load_from_ini(const std::string & path, co for (auto section : ini_data) { common_preset preset; - std::string section_name = section.first.empty() ? std::string(COMMON_PRESET_DEFAULT_NAME) : section.first; - if (section_name != "*" && section_name != COMMON_PRESET_DEFAULT_NAME) { - auto colon_idx = section_name.rfind(':'); - if (colon_idx != std::string::npos) { - std::string tag = section_name.substr(colon_idx + 1); - std::string canon_tag = canonical_tag(tag); - if (canon_tag != tag) { - section_name = section_name.substr(0, colon_idx + 1) + canon_tag; - } - } + if (section.first.empty()) { + preset.name = COMMON_PRESET_DEFAULT_NAME; + } else { + preset.name = section.first; } - preset.name = section_name; LOG_DBG("loading preset: %s\n", preset.name.c_str()); for (const auto & [key, value] : section.second) { if (key == "version") { diff --git a/common/reasoning-budget.cpp b/common/reasoning-budget.cpp index 7da0bb1c5..ce41d029b 100644 --- a/common/reasoning-budget.cpp +++ b/common/reasoning-budget.cpp @@ -65,12 +65,12 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to if (ctx->start_matcher.advance(token)) { ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; - COM_TRC("activated, budget=%d tokens\n", ctx->budget); + LOG_INF("reasoning-budget: activated, budget=%d tokens\n", ctx->budget); if (ctx->remaining <= 0) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; - COM_TRC("%s", "budget=0, forcing immediately\n"); + LOG_INF("reasoning-budget: budget=0, forcing immediately\n"); } } break; @@ -80,7 +80,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to { if (ctx->end_matcher.advance(token)) { ctx->state = REASONING_BUDGET_DONE; - COM_TRC("%s", "deactivated (natural end)\n"); + LOG_INF("reasoning-budget: deactivated (natural end)\n"); break; } @@ -95,7 +95,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - COM_TRC("%s", "UTF-8 complete, now forcing end sequence\n"); + LOG_INF("reasoning-budget: UTF-8 complete, now forcing end sequence\n"); } } else if (ctx->state == REASONING_BUDGET_COUNTING) { ctx->remaining--; @@ -104,11 +104,11 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - COM_TRC("%s", "budget exhausted, forcing end sequence\n"); + LOG_INF("reasoning-budget: budget exhausted, forcing end sequence\n"); } else { ctx->state = REASONING_BUDGET_WAITING_UTF8; ctx->end_matcher.reset(); - COM_TRC("%s", "budget exhausted, waiting for UTF-8 completion\n"); + LOG_INF("reasoning-budget: budget exhausted, waiting for UTF-8 completion\n"); } } } @@ -118,7 +118,7 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->force_pos++; if (ctx->force_pos >= ctx->forced_tokens.size()) { ctx->state = REASONING_BUDGET_DONE; - COM_TRC("%s", "forced sequence complete, done\n"); + LOG_INF("reasoning-budget: forced sequence complete, done\n"); } break; case REASONING_BUDGET_DONE: @@ -128,12 +128,12 @@ static void common_reasoning_budget_accept(struct llama_sampler * smpl, llama_to ctx->state = REASONING_BUDGET_COUNTING; ctx->remaining = ctx->budget; ctx->end_matcher.reset(); - COM_TRC("re-activated on new start tag, budget=%d tokens\n", ctx->budget); + LOG_INF("reasoning-budget: re-activated on new start tag, budget=%d tokens\n", ctx->budget); if (ctx->remaining <= 0) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; - COM_TRC("%s", "budget=0, forcing immediately\n"); + LOG_INF("reasoning-budget: budget=0, forcing immediately\n"); } } break; @@ -264,7 +264,7 @@ bool common_reasoning_budget_force(struct llama_sampler * smpl) { ctx->state = REASONING_BUDGET_FORCING; ctx->force_pos = 0; ctx->end_matcher.reset(); - COM_TRC("%s", "forced into forcing state (manual transition)\n"); + LOG_INF("reasoning-budget: forced into forcing state (manual transition)\n"); return true; } diff --git a/common/regex-partial.cpp b/common/regex-partial.cpp new file mode 100644 index 000000000..bd9034e93 --- /dev/null +++ b/common/regex-partial.cpp @@ -0,0 +1,204 @@ +#include "regex-partial.h" +#include "common.h" +#include +#include + +common_regex::common_regex(const std::string & pattern) : + pattern(pattern), + rx(pattern), + rx_reversed_partial(regex_to_reversed_partial_regex(pattern)) {} + +common_regex_match common_regex::search(const std::string & input, size_t pos, bool as_match) const { + std::smatch match; + if (pos > input.size()) { + throw std::runtime_error("Position out of bounds"); + } + auto start = input.begin() + pos; + auto found = as_match + ? std::regex_match(start, input.end(), match, rx) + : std::regex_search(start, input.end(), match, rx); + if (found) { + common_regex_match res; + res.type = COMMON_REGEX_MATCH_TYPE_FULL; + for (size_t i = 0; i < match.size(); ++i) { + auto begin = pos + match.position(i); + res.groups.emplace_back(begin, begin + match.length(i)); + } + return res; + } + std::match_results srmatch; + if (std::regex_search(input.rbegin(), input.rend() - pos, srmatch, rx_reversed_partial, std::regex_constants::match_continuous)) { + auto group = srmatch[1].str(); + if (group.length() != 0) { + auto it = srmatch[1].second.base(); + // auto position = static_cast(std::distance(input.begin(), it)); + if ((!as_match) || it == input.begin()) { + common_regex_match res; + res.type = COMMON_REGEX_MATCH_TYPE_PARTIAL; + const size_t begin = std::distance(input.begin(), it); + const size_t end = input.size(); + if (begin == std::string::npos || end == std::string::npos || begin > end) { + throw std::runtime_error("Invalid range"); + } + res.groups.push_back({begin, end}); + return res; + } + } + } + return {}; +} + +/* + Transforms a regex pattern to a partial match pattern that operates on a reversed input string to find partial final matches of the original pattern. + + Ideally we'd like to use boost::match_partial (https://beta.boost.org/doc/libs/1_59_0/libs/regex/doc/html/boost_regex/partial_matches.html) + to see if a string ends with a partial regex match, but but it's not in std::regex yet. + Instead, we'll the regex into a partial match regex operating as a full match on the reverse iterators of the input. + + - /abcd/ -> ^(dcba|cba|ba|a) -> ^((?:(?:(?:(?:d)?c)?b)?a) + - /a|b/ -> ^(a|b) + - /a*?/ -> error, could match "" + - /a*b/ -> ^((?:b)?a*+) (final repetitions become eager) + - /.*?ab/ -> ^((?:b)?a) (omit .*) + - /a.*?b/ -> ^((?:b)?.*?a) (keep reluctant matches) + - /a(bc)d/ -> ^((?:(?:d)?(?:(?:c)?b))?a) + - /a(bc|de)/ -> ^((?:(?:(?:e)?d)?|(?:(?:c)?b)?)?a) + - /ab{2,4}c/ -> ^cbbb?b?a -> ^((?:(?:(?:(?:(?:c)?b)?b)?b?)?b?)?a) + + The regex will match a reversed string fully, and the end of the first (And only) capturing group will indicate the reversed start of the original partial pattern. + All other groups are turned into non-capturing groups, and reluctant quantifiers are ignored. +*/ +std::string regex_to_reversed_partial_regex(const std::string & pattern) { + auto it = pattern.begin(); + const auto end = pattern.end(); + + std::function process = [&]() { + std::vector> alternatives(1); + std::vector * sequence = &alternatives.back(); + + while (it != end) { + if (*it == '[') { + auto start = it; + ++it; + while (it != end) { + if ((*it == '\\') && (++it != end)) { + ++it; + } else if ((it != end) && (*it == ']')) { + break; + } else { + ++it; + } + } + if (it == end) { + throw std::runtime_error("Unmatched '[' in pattern"); + } + ++it; + sequence->push_back(std::string(start, it)); + } else if (*it == '*' || *it == '?' || *it == '+') { + if (sequence->empty()) { + throw std::runtime_error("Quantifier without preceding element"); + } + sequence->back() += *it; + auto is_star = *it == '*'; + ++it; + if (is_star) { + if (it != end && *it == '?') { + ++it; + } + } + } else if (*it == '{') { + if (sequence->empty()) { + throw std::runtime_error("Repetition without preceding element"); + } + ++it; + auto start = it; + while (it != end && *it != '}') { + ++it; + } + if (it == end) { + throw std::runtime_error("Unmatched '{' in pattern"); + } + auto parts = string_split(std::string(start, it), ","); + ++it; + if (parts.size() > 2) { + throw std::runtime_error("Invalid repetition range in pattern"); + } + + auto parseOptInt = [&](const std::string & s, const std::optional & def = std::nullopt) -> std::optional { + if (s.empty()) { + return def; + } + return std::stoi(s); + }; + auto min = parseOptInt(parts[0], 0); + auto max = parts.size() == 1 ? min : parseOptInt(parts[1]); + if (min && max && *max < *min) { + throw std::runtime_error("Invalid repetition range in pattern"); + } + // Brutal but... let's repeat at least min times, then ? for the delta between min & max (or * for unbounded) + auto part = sequence->back(); + sequence->pop_back(); + for (int i = 0; i < *min; i++) { + sequence->push_back(part); + } + if (max) { + for (int i = *min; i < *max; i++) { + sequence->push_back(part + "?"); + } + } else { + sequence->push_back(part + "*"); + } + } else if (*it == '(') { + ++it; + if (it != end && *it == '?' && (it + 1 != end) && *(it + 1) == ':') { + it += 2; + } + auto sub = process(); + if (*it != ')') { + throw std::runtime_error("Unmatched '(' in pattern"); + } + ++it; + auto & part = sequence->emplace_back("(?:"); + part += sub; + part += ")"; + } else if (*it == ')') { + break; + } else if (*it == '|') { + ++it; + alternatives.emplace_back(); + sequence = &alternatives.back(); + } else if (*it == '\\' && (++it != end)) { + auto str = std::string("\\") + *it; + sequence->push_back(str); + ++it; + } else if (it != end) { + sequence->push_back(std::string(1, *it)); + ++it; + } + } + + // /abcd/ -> ^(dcba|cba|ba|a) -> ^((?:(?:(?:d)?c)?b)?a) + // if n(=4) parts, opening n-1(=3) non-capturing groups after the 1 capturing group + // We'll do the outermost capturing group and final .* in the enclosing function. + std::vector res_alts; + for (const auto & parts : alternatives) { + auto & res = res_alts.emplace_back(); + for (size_t i = 0; i < parts.size() - 1; i++) { + res += "(?:"; + } + for (auto it = parts.rbegin(); it != parts.rend(); ++it) { + res += *it; + if (it != parts.rend() - 1) { + res += ")?"; + } + } + } + return string_join(res_alts, "|"); + }; + auto res = process(); + if (it != end) { + throw std::runtime_error("Unmatched '(' in pattern"); + } + + return "^(" + res + ")"; +} diff --git a/common/regex-partial.h b/common/regex-partial.h new file mode 100644 index 000000000..634cb4022 --- /dev/null +++ b/common/regex-partial.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +enum common_regex_match_type { + COMMON_REGEX_MATCH_TYPE_NONE, + COMMON_REGEX_MATCH_TYPE_PARTIAL, + COMMON_REGEX_MATCH_TYPE_FULL, +}; + +struct common_string_range { + size_t begin; + size_t end; + common_string_range(size_t begin, size_t end) : begin(begin), end(end) { + if (begin > end) { + throw std::runtime_error("Invalid range"); + } + } + // prevent default ctor + common_string_range() = delete; + bool empty() const { + return begin == end; + } + bool operator==(const common_string_range & other) const { + return begin == other.begin && end == other.end; + } +}; + +struct common_regex_match { + common_regex_match_type type = COMMON_REGEX_MATCH_TYPE_NONE; + std::vector groups; + + bool operator==(const common_regex_match & other) const { + return type == other.type && groups == other.groups; + } + bool operator!=(const common_regex_match & other) const { + return !(*this == other); + } +}; + +class common_regex { + std::string pattern; + std::regex rx; + std::regex rx_reversed_partial; + + public: + explicit common_regex(const std::string & pattern); + + common_regex_match search(const std::string & input, size_t pos, bool as_match = false) const; + + const std::string & str() const { return pattern; } +}; + +// For testing only (pretty print of failures). +std::string regex_to_reversed_partial_regex(const std::string & pattern); diff --git a/common/speculative.cpp b/common/speculative.cpp index 4e0439216..0a293edeb 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -18,13 +18,6 @@ #include #include -#define SPC_DBG(fmt, ...) LOG_DBG("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_TRC(fmt, ...) LOG_TRC("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_INF(fmt, ...) LOG_INF("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_WRN(fmt, ...) LOG_WRN("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_ERR(fmt, ...) LOG_ERR("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) -#define SPC_CNT(fmt, ...) LOG_CNT("" fmt, __VA_ARGS__) - #define SPEC_VOCAB_MAX_SIZE_DIFFERENCE 128 #define SPEC_VOCAB_CHECK_START_TOKEN_ID 5 @@ -33,7 +26,6 @@ const std::map common_speculative_type_fro {"draft-simple", COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE}, {"draft-eagle3", COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}, {"draft-mtp", COMMON_SPECULATIVE_TYPE_DRAFT_MTP}, - {"draft-dflash", COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}, {"ngram-simple", COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE}, {"ngram-map-k", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K}, {"ngram-map-k4v", COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V}, @@ -68,20 +60,21 @@ static bool common_speculative_are_compatible( const llama_vocab * vocab_dft = llama_model_get_vocab(model_dft); const auto vocab_type_tgt = llama_vocab_type(vocab_tgt); - SPC_DBG("vocab_type tgt: %d\n", vocab_type_tgt); + LOG_DBG("%s: vocab_type tgt: %d\n", __func__, vocab_type_tgt); const auto vocab_type_dft = llama_vocab_type(vocab_dft); - SPC_DBG("vocab_type dft: %d\n", vocab_type_dft); + LOG_DBG("%s: vocab_type dft: %d\n", __func__, vocab_type_dft); if (vocab_type_tgt != vocab_type_dft) { - SPC_WRN("draft model vocab type must match target model to use speculation but " - "vocab_type_dft = %d while vocab_type_tgt = %d\n", vocab_type_dft, vocab_type_tgt); + LOG_WRN("%s: draft model vocab type must match target model to use speculation but " + "vocab_type_dft = %d while vocab_type_tgt = %d\n", __func__, vocab_type_dft, vocab_type_tgt); return false; } if (llama_vocab_get_add_bos(vocab_tgt) != llama_vocab_get_add_bos(vocab_dft) || (llama_vocab_get_add_bos(vocab_tgt) && llama_vocab_bos(vocab_tgt) != llama_vocab_bos(vocab_dft))) { - SPC_WRN("draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + LOG_WRN("%s: draft model bos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + __func__, llama_vocab_get_add_bos(vocab_tgt), llama_vocab_get_add_bos(vocab_dft), llama_vocab_bos(vocab_tgt), llama_vocab_bos(vocab_dft)); return false; @@ -89,7 +82,8 @@ static bool common_speculative_are_compatible( if (llama_vocab_get_add_eos(vocab_tgt) != llama_vocab_get_add_eos(vocab_dft) || (llama_vocab_get_add_eos(vocab_tgt) && llama_vocab_eos(vocab_tgt) != llama_vocab_eos(vocab_dft))) { - SPC_WRN("draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + LOG_WRN("%s: draft model eos tokens must match target model to use speculation. add: %d - %d, id: %d - %d)\n", + __func__, llama_vocab_get_add_eos(vocab_tgt), llama_vocab_get_add_eos(vocab_dft), llama_vocab_eos(vocab_tgt), llama_vocab_eos(vocab_dft)); return false; @@ -103,8 +97,8 @@ static bool common_speculative_are_compatible( : n_vocab_dft - n_vocab_tgt; if (vocab_diff > SPEC_VOCAB_MAX_SIZE_DIFFERENCE) { - SPC_DBG("draft model vocab must closely match target model to use speculation but " - "target vocab size %d does not match draft vocab size %d - difference %d, max allowed %d\n", + LOG_DBG("%s: draft model vocab must closely match target model to use speculation but ", __func__); + LOG_DBG("target vocab size %d does not match draft vocab size %d - difference %d, max allowed %d\n", n_vocab_tgt, llama_vocab_n_tokens(vocab_dft), vocab_diff, SPEC_VOCAB_MAX_SIZE_DIFFERENCE); return false; } @@ -114,8 +108,8 @@ static bool common_speculative_are_compatible( const char * token_text_dft = llama_vocab_get_text(vocab_dft, i); if (std::strcmp(token_text_tgt, token_text_dft) != 0) { - SPC_DBG("draft model vocab must match target model to use speculation but " - "token %d content differs - target '%s', draft '%s'\n", i, + LOG_DBG("%s: draft model vocab must match target model to use speculation but ", __func__); + LOG_DBG("token %d content differs - target '%s', draft '%s'\n", i, common_token_to_piece(vocab_tgt, i).c_str(), common_token_to_piece(vocab_dft, i).c_str()); return false; @@ -192,9 +186,9 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; auto * ctx_tgt = this->params.ctx_tgt; - SPC_TRC("%s", "adding speculative implementation 'draft-simple'\n"); - SPC_TRC("- n_max=%d, n_min=%d, p_min=%f\n", this->params.n_max, this->params.n_min, this->params.p_min); - SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", + LOG_INF("%s: adding speculative implementation 'draft-simple'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); + LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, this->params.n_gpu_layers, ggml_type_name(this->params.cache_type_k), ggml_type_name(this->params.cache_type_v), @@ -234,16 +228,16 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { } const bool vocab_cmpt = common_speculative_are_compatible(llama_get_model(ctx_tgt), llama_get_model(ctx_dft)); - SPC_DBG("vocab_cmpt = %d\n", vocab_cmpt); + LOG_DBG("%s: vocab_cmpt = %d\n", __func__, vocab_cmpt); if (!vocab_cmpt) { - SPC_ERR("%s", "the target and draft vocabs are not compatible\n"); + LOG_ERR("%s: the target and draft vocabs are not compatible\n", __func__); throw std::runtime_error("draft model vocab type must match target model to use speculation"); } if (n_seq != llama_n_seq_max(ctx_dft)) { - SPC_ERR("n_seq mismatch: %d != %d\n", n_seq, llama_n_seq_max(ctx_dft)); + LOG_ERR("%s: n_seq mismatch: %d != %d\n", __func__, n_seq, llama_n_seq_max(ctx_dft)); throw std::runtime_error("the draft model number of sequences is incompatible with the speculative n_seq"); } @@ -263,7 +257,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { const int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("failed to decode draft batch, ret = %d\n", ret); + LOG_ERR("%s: failed to decode draft batch, ret = %d\n", __func__, ret); return false; } @@ -296,7 +290,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode returned %d\n", ret); + LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); return; } @@ -320,7 +314,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -360,7 +354,7 @@ struct common_speculative_impl_draft_simple : public common_speculative_impl { // evaluate the drafted tokens on the draft model ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode[%d] returned %d\n", i, ret); + LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); break; } @@ -455,8 +449,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3, n_seq) , params(params.draft) { - SPC_TRC("%s", "adding speculative implementation 'draft-eagle3'\n"); - SPC_TRC("- n_max=%d, n_min=%d, p_min=%f, backend_sampling=%d\n", params.draft.n_max, params.draft.n_min, params.draft.p_min, (int) params.draft.backend_sampling); + LOG_INF("%s: adding speculative implementation 'draft-eagle3'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%f, backend_sampling=%d\n", __func__, params.draft.n_max, params.draft.n_min, params.draft.p_min, (int) params.draft.backend_sampling); auto * ctx_tgt = this->params.ctx_tgt; auto * ctx_dft = this->params.ctx_dft; @@ -499,7 +493,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); if (!llama_set_sampler(ctx_dft, seq_id, chain)) { - SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); + LOG_WRN("%s: backend offload failed for seq_id=%d; using CPU sampler\n", __func__, (int) seq_id); llama_sampler_free(chain); chain = nullptr; } @@ -554,9 +548,9 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { auto * ctx_dft = this->params.ctx_dft; const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); if (pos_max < N - 2) { - SPC_WRN("ctx_dft pos_max=%d < N-2=%d — process() did not run on every prefill ubatch. " + LOG_WRN("%s: ctx_dft pos_max=%d < N-2=%d — process() did not run on every prefill ubatch. " "Drafts may degrade.\n", - (int) pos_max, N - 2); + __func__, (int) pos_max, N - 2); } } @@ -627,8 +621,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { }; const int32_t rc = llama_encode(ctx_dft, enc_batch); if (rc != 0) { - SPC_ERR("llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", - rc, (int) n_chunk, (int) i); + LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", + __func__, rc, (int) n_chunk, (int) i); return false; } @@ -698,8 +692,8 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { if (batch.n_tokens > 0) { const int32_t rc = llama_decode(ctx_dft, batch); if (rc != 0) { - SPC_ERR("llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, ubatch_pos[0]=%d)\n", - rc, (int) batch.n_tokens, (int) batch_in.pos[0]); + LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, ubatch_pos[0]=%d)\n", + __func__, rc, (int) batch.n_tokens, (int) batch_in.pos[0]); return false; } } @@ -750,7 +744,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode returned %d\n", ret); + LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); return; } @@ -776,7 +770,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -815,7 +809,7 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode[%d] returned %d\n", i, ret); + LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); break; } @@ -899,305 +893,6 @@ struct common_speculative_impl_draft_eagle3 : public common_speculative_impl { } }; -// DFlash: block-diffusion drafting with a draft-side KV cache injection -struct common_speculative_impl_draft_dflash : public common_speculative_impl { - common_params_speculative_draft params; - - llama_batch batch; // noise tokens - llama_batch batch_inject; // target features for KV cache injection - - std::vector smpls; - - int32_t n_embd_dec = 0; // draft hidden size - int32_t n_embd_enc = 0; // target_layer_ids_n * target_hidden_size - int32_t n_embd_tgt = 0; // target model hidden size - - int32_t block_size = 0; - llama_token mask_token_id = 0; - - const int32_t * target_layer_ids = nullptr; // model_dft's extract layer indices - uint32_t target_layer_ids_n = 0; - - // scratch buffer for concatenated target features [n_tokens, n_embd_enc] - std::vector features_buf; - - common_speculative_impl_draft_dflash(const common_params_speculative & params, uint32_t n_seq) - : common_speculative_impl(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, n_seq) - , params(params.draft) - { - auto * ctx_tgt = this->params.ctx_tgt; - auto * ctx_dft = this->params.ctx_dft; - GGML_ASSERT(ctx_tgt && ctx_dft && "DFlash requires ctx_tgt and ctx_dft to be set"); - - const llama_model * model_dft = llama_get_model(ctx_dft); - const llama_model * model_tgt = llama_get_model(ctx_tgt); - - target_layer_ids = llama_model_target_layer_ids (model_dft); - target_layer_ids_n = llama_model_target_layer_ids_n(model_dft); - GGML_ASSERT(target_layer_ids_n > 0 && "DFlash model has no target_layer_ids"); - - n_embd_tgt = llama_model_n_embd(model_tgt); - n_embd_dec = llama_model_n_embd(model_dft); - n_embd_enc = (int32_t) target_layer_ids_n * n_embd_tgt; - - // read the trained block size from the dflash.block_size metadata key - block_size = 16; - { - char buf[32] = {}; - if (llama_model_meta_val_str(model_dft, "dflash.block_size", buf, sizeof(buf)) >= 0) { - block_size = std::atoi(buf); - } - } - mask_token_id = llama_vocab_mask(llama_model_get_vocab(model_dft)); - - LOG_INF("%s: adding speculative implementation 'draft-dflash'\n", __func__); - LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min); - LOG_INF("%s: - block_size=%d, mask_token_id=%d, n_extract=%u\n", __func__, block_size, mask_token_id, target_layer_ids_n); - - // DFlash input is [id_last, * (block_size-1)], so it can draft at most block_size-1 tokens per step - if (this->params.n_max > block_size - 1 || this->params.n_min > block_size - 1) { - LOG_WRN("%s: requested draft size (n_max=%d, n_min=%d) exceeds the trained DFlash block size %d -- clamping to %d\n", - __func__, this->params.n_max, this->params.n_min, block_size, block_size - 1); - this->params.n_max = std::min(this->params.n_max, block_size - 1); - this->params.n_min = std::min(this->params.n_min, block_size - 1); - } - - batch = llama_batch_init(llama_n_batch(ctx_dft), 0, n_seq); - batch_inject = llama_batch_init(llama_n_batch(ctx_dft), n_embd_dec, n_seq); - - smpls.resize(n_seq); - for (auto & s : smpls) { - common_params_sampling sparams; - sparams.no_perf = false; - sparams.top_k = 10; - sparams.samplers = { COMMON_SAMPLER_TYPE_TOP_K }; - s.reset(common_sampler_init(model_dft, sparams)); - } - - // turn on extraction of the target layers' input embeddings - for (uint32_t k = 0; k < target_layer_ids_n; ++k) { - llama_set_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k], true); - } - - llama_set_embeddings_nextn(ctx_dft, true, /*masked*/ true); - llama_set_causal_attn(ctx_dft, false); // DFlash needs non-causal attention - } - - ~common_speculative_impl_draft_dflash() override { - llama_batch_free(batch); - llama_batch_free(batch_inject); - } - - void begin(llama_seq_id seq_id, const llama_tokens & prompt) override { - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { - return; - } - - const int32_t N = (int32_t) prompt.size(); - if (N <= 0) { - return; - } - - const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(params.ctx_dft), seq_id); - if (pos_max < N - 1) { - LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - process() did not run on every prefill ubatch. " - "Drafts may degrade.\n", - __func__, (int) pos_max, N - 1); - } - } - - bool process(const llama_batch & batch_in) override { - if (batch_in.n_tokens <= 0) { - return true; - } - - if (batch_in.token == nullptr || batch_in.embd != nullptr) { - return true; - } - - const int32_t n_tokens = batch_in.n_tokens; - - // per-seq inclusive batch range (assumes each seq's tokens are contiguous in the batch) - std::vector i_batch_beg(n_seq, -1); - std::vector i_batch_end(n_seq, -1); - for (int32_t k = 0; k < n_tokens; ++k) { - GGML_ASSERT(batch_in.n_seq_id[k] == 1); - const llama_seq_id seq_id = batch_in.seq_id[k][0]; - if (seq_id < 0 || seq_id >= (llama_seq_id) n_seq) { - continue; - } - i_batch_end[seq_id] = k; - if (i_batch_beg[seq_id] < 0) { - i_batch_beg[seq_id] = k; - } - } - - auto * ctx_tgt = this->params.ctx_tgt; - auto * ctx_dft = this->params.ctx_dft; - - const int32_t n_ubatch = (int32_t) llama_n_ubatch(ctx_dft); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (i_batch_beg[seq_id] < 0) { - continue; - } - const int32_t n_rows = i_batch_end[seq_id] - i_batch_beg[seq_id] + 1; - - for (int32_t offset = 0; offset < n_rows; offset += n_ubatch) { - const int32_t n_chunk = std::min(n_ubatch, n_rows - offset); - - // gather this chunk's target features, interleaved by extract layer - features_buf.resize((size_t) n_chunk * n_embd_enc); - for (uint32_t k = 0; k < target_layer_ids_n; ++k) { - const float * layer = llama_get_embeddings_layer_inp(ctx_tgt, (uint32_t) target_layer_ids[k]); - if (!layer) { - GGML_ABORT("DFlash: target layer %d input not extracted.", target_layer_ids[k]); - } - for (int32_t i = 0; i < n_chunk; ++i) { - float * dst = features_buf.data() + (size_t) i * n_embd_enc + k * (size_t) n_embd_tgt; - const float * src = layer + (size_t) (i_batch_beg[seq_id] + offset + i) * n_embd_tgt; - std::memcpy(dst, src, (size_t) n_embd_tgt * sizeof(float)); - } - } - - // fuse extracted features through DFlash encoder - llama_batch enc_batch = { - /*.n_tokens =*/ n_chunk, - /*.token =*/ nullptr, - /*.embd =*/ features_buf.data(), - /*.pos =*/ nullptr, - /*.n_seq_id =*/ nullptr, - /*.seq_id =*/ nullptr, - /*.logits =*/ nullptr, - }; - - int32_t rc = llama_encode(ctx_dft, enc_batch); - if (rc != 0) { - LOG_ERR("%s: llama_encode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", - __func__, rc, (int) n_chunk, (int) offset); - return false; - } - - const float * inp_g = llama_get_embeddings_nextn(ctx_dft); - GGML_ASSERT(inp_g && "DFlash encoder produced no output."); - - // inject the DFlash decoder K/V cache at the tokens' target positions - batch_inject.n_tokens = n_chunk; - std::memcpy(batch_inject.embd, inp_g, (size_t) n_chunk * n_embd_dec * sizeof(float)); - - for (int32_t i = 0; i < n_chunk; ++i) { - batch_inject.pos[i] = batch_in.pos[i_batch_beg[seq_id] + offset + i]; - batch_inject.n_seq_id[i] = 1; - batch_inject.seq_id[i][0] = seq_id; - batch_inject.logits[i] = false; - } - rc = llama_decode(ctx_dft, batch_inject); - if (rc != 0) { - LOG_ERR("%s: llama_decode(ctx_dft) failed rc=%d (n_tokens=%d, offset=%d)\n", - __func__, rc, (int) n_chunk, (int) offset); - return false; - } - } - } - - return true; - } - - void draft(common_speculative_draft_params_vec & dparams) override { - auto & ctx_dft = params.ctx_dft; - - common_batch_clear(batch); - - // build one batch holding every drafting sequence's noise block into a single decode) - // record where each block starts and its size - std::vector i_block_beg(n_seq, -1); - std::vector n_block (n_seq, 0); - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - auto & dp = dparams[seq_id]; - if (!dp.drafting) { - continue; - } - - common_sampler_reset(smpls[seq_id].get()); - - const int32_t n = (int32_t) dp.n_past; - - int32_t n_draft = params.n_max; - if (dp.n_max > 0) { - n_draft = std::min(n_draft, dp.n_max); - } - - const int32_t n_block_tokens = n_draft + 1; // id_last + n_draft * - i_block_beg[seq_id] = batch.n_tokens; - n_block [seq_id] = n_block_tokens; - for (int32_t i = 0; i < n_block_tokens; ++i) { - common_batch_add(batch, i == 0 ? dp.id_last : mask_token_id, n + i, { seq_id }, true); - } - } - - if (batch.n_tokens == 0) { - return; - } - - // decode all sequence's noise block in a single batch - int ret = llama_decode(ctx_dft, batch); - if (ret != 0) { - LOG_WRN("%s: llama_decode returned %d\n", __func__, ret); - return; - } - - for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) n_seq; ++seq_id) { - if (i_block_beg[seq_id] < 0) { - continue; - } - auto & dp = dparams[seq_id]; - - const int32_t beg = i_block_beg[seq_id]; - const int32_t n_block_tokens = n_block[seq_id]; - - auto * smpl = smpls[seq_id].get(); - - auto & result = *dp.result; - - // greedily read the predicted block at this sequence's noise positions 1..n_block_tokens-1 - for (int32_t i = 1; i < n_block_tokens; ++i) { - common_sampler_sample(smpl, ctx_dft, beg + i, true); - - const auto * cur_p = common_sampler_get_candidates(smpl, true); - - for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", - seq_id, k, i - 1, cur_p->data[k].id, cur_p->data[k].p, - common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); - } - - const llama_token id = cur_p->data[0].id; - - if (cur_p->data[0].p < params.p_min) { - break; - } - - common_sampler_accept(smpl, id, true); - - result.push_back(id); - } - - if (result.size() < (size_t) params.n_min) { - result.clear(); - } - } - } - - void accept(llama_seq_id /*seq_id*/, uint16_t /*n_accepted*/, bool /*is_other*/) override { - // noop - } - - bool need_embd() const override { - return false; - } -}; - struct common_speculative_impl_draft_mtp : public common_speculative_impl { common_params_speculative_draft params; // reuses the draft-model params slot (ctx_tgt/ctx_dft) @@ -1247,9 +942,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { "MTP input row width must match the target h_nextn width"); n_mtp_layers = std::max(1, (int) llama_model_n_layer_nextn(llama_get_model(ctx_dft))); - SPC_TRC("%s", "adding speculative implementation 'draft-mtp'\n"); - SPC_TRC("- n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); - SPC_TRC("- gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", + LOG_INF("%s: adding speculative implementation 'draft-mtp'\n", __func__); + LOG_INF("%s: - n_max=%d, n_min=%d, p_min=%.2f, n_embd=%d, backend_sampling=%d\n", __func__, this->params.n_max, this->params.n_min, this->params.p_min, n_embd, (int) this->params.backend_sampling); + LOG_INF("%s: - gpu_layers=%d, cache_k=%s, cache_v=%s, ctx_tgt=%s, ctx_dft=%s, devices=[%s]\n", __func__, this->params.n_gpu_layers, ggml_type_name(this->params.cache_type_k), ggml_type_name(this->params.cache_type_v), @@ -1280,7 +975,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); if (!llama_set_sampler(ctx_dft, seq_id, chain)) { - SPC_WRN("backend offload failed for seq_id=%d; using CPU sampler\n", (int) seq_id); + LOG_WRN("%s: backend offload failed for seq_id=%d; using CPU sampler\n", __func__, (int) seq_id); llama_sampler_free(chain); chain = nullptr; } @@ -1343,11 +1038,11 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const llama_pos pos_max = llama_memory_seq_pos_max(llama_get_memory(ctx_dft), seq_id); if (pos_max < N - 1 && !is_mem_shared) { - SPC_WRN("ctx_dft pos_max=%d < N-1=%d - " + LOG_WRN("%s: ctx_dft pos_max=%d < N-1=%d - " "process() hook may not have run on every prefill ubatch " "(need_embd / logits=1 on every prompt position?). " "Drafts may degrade.\n", - (int) pos_max, N - 1); + __func__, (int) pos_max, N - 1); } } @@ -1433,8 +1128,8 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const int32_t rc = llama_decode(ctx_dft, batch); if (rc != 0) { - SPC_ERR("llama_decode(ctx_dft) head=%d failed rc=%d (pos=%d)\n", - head, (int) rc, (int) batch_in.pos[0]); + LOG_ERR("%s: llama_decode(ctx_dft) head=%d failed rc=%d (pos=%d)\n", + __func__, head, (int) rc, (int) batch_in.pos[0]); ok = false; break; } @@ -1522,7 +1217,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { int ret = llama_decode(ctx_dft, batch); if (ret != 0) { - SPC_ERR("llama_decode[%d] returned %d\n", i, ret); + LOG_WRN("%s: llama_decode[%d] returned %d\n", __func__, i, ret); break; } @@ -1544,7 +1239,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const auto * cur_p = common_sampler_get_candidates(smpl, true); for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { - SPC_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", + LOG_DBG(" - seq_id %d, draft candidate %3d, pos %3d: %6d (%8.3f) '%s'\n", seq_id, k, i, cur_p->data[k].id, cur_p->data[k].p, common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } @@ -1658,8 +1353,8 @@ struct common_speculative_impl_ngram_simple : public common_speculative_impl { , params(params.ngram_simple) , config(config) { - SPC_TRC("%s", "adding speculative implementation 'ngram-simple'\n"); - SPC_TRC("- size_n=%d, size_m=%d, min_hits=%d\n", + LOG_INF("%s: adding speculative implementation 'ngram-simple'\n", __func__); + LOG_INF("%s: - size_n=%d, size_m=%d, min_hits=%d\n", __func__, this->params.size_n, this->params.size_m, this->params.min_hits); } @@ -1708,8 +1403,8 @@ struct common_speculative_impl_ngram_map_k : public common_speculative_impl { this->config.push_back(config); } - SPC_TRC("adding speculative implementation '%s'\n", common_speculative_type_to_str(this->type).c_str()); - SPC_TRC("- size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", + LOG_INF("%s: adding speculative implementation '%s'\n", __func__, common_speculative_type_to_str(this->type).c_str()); + LOG_INF("%s: - size_key=%d, size_value=%d, key_only=%d, min_hits=%d\n", __func__, config.size_key, config.size_value, config.key_only, config.min_hits); } @@ -1783,15 +1478,15 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { , verbose(std::getenv("LLAMA_TRACE") != nullptr) { static_assert(sizeof(llama_token) == sizeof(common_ngram_mod::entry_t)); - SPC_TRC("%s", "adding speculative implementation 'ngram-mod'\n"); - SPC_TRC("- n_match=%d, n_max=%d, n_min=%d\n", + LOG_INF("%s: adding speculative implementation 'ngram-mod'\n", __func__); + LOG_INF("%s: - n_match=%d, n_max=%d, n_min=%d\n", __func__, this->params.n_match, this->params.n_max, this->params.n_min); - SPC_TRC("- mod size=%zu (%.3f MB)\n", + LOG_INF("%s: - mod size=%zu (%.3f MB)\n", __func__, mod.size(), (float)(mod.size_bytes())/1024/1024); if (this->params.n_match < 16) { - SPC_WRN("ngram_mod n_match=%d is too small - poor quality is possible, " - "see: https://github.com/ggml-org/llama.cpp/pull/19164\n", this->params.n_match); + LOG_WRN("%s: ngram_mod n_match=%d is too small - poor quality is possible, " + "see: https://github.com/ggml-org/llama.cpp/pull/19164\n", __func__, this->params.n_match); } sinfos.resize(n_seq); @@ -1815,11 +1510,11 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.i_last = prompt.size() - n; const double f = (double)mod.get_used() / (double)mod.size(); - SPC_TRC("ngram_mod occupancy = %zu/%zu (%.2f)\n", mod.get_used(), mod.size(), f); + LOG_INF("%s: ngram_mod occupancy = %zu/%zu (%.2f)\n", __func__, mod.get_used(), mod.size(), f); constexpr double f_thold = 0.25; if (f > f_thold) { - SPC_WRN("ngram_mod occupancy %.2f exceeds threshold (%.2f) - resetting\n", f, f_thold); + LOG_WRN("%s: ngram_mod occupancy %.2f exceeds threshold (%.2f) - resetting\n", __func__, f, f_thold); mod.reset(); } @@ -1913,7 +1608,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.n_low++; if (sinfo.n_low >= 5) { if (verbose) { - SPC_TRC("low acceptance streak (%d) - resetting ngram_mod\n", sinfo.n_low); + LOG_WRN("%s: low acceptance streak (%d) - resetting ngram_mod\n", __func__, sinfo.n_low); } mod.reset(); @@ -1963,8 +1658,8 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { , save_dynamic(save_dynamic) , save_static(save_static) { - SPC_TRC("%s", "adding speculative implementation 'ngram-cache'\n"); - SPC_TRC("- n_draft=%d, cache_static=%s, cache_dynamic=%s\n", + LOG_INF("%s: adding speculative implementation 'ngram-cache'\n", __func__); + LOG_INF("%s: - n_draft=%d, cache_static=%s, cache_dynamic=%s\n", __func__, n_draft, path_static.empty() ? "none" : path_static.c_str(), path_dynamic.empty() ? "none" : path_dynamic.c_str()); @@ -1979,7 +1674,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { sinfo.ngram_cache_static = ngram_cache_static; } } catch (...) { - SPC_ERR("failed to open static lookup cache: %s", path_static.c_str()); + LOG_ERR("failed to open static lookup cache: %s", path_static.c_str()); GGML_ABORT("Couldn't read static lookup cache"); } } @@ -1992,7 +1687,7 @@ struct common_speculative_impl_ngram_cache : public common_speculative_impl { sinfo.ngram_cache_dynamic = ngram_cache_dynamic; } } catch (...) { - SPC_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str()); + LOG_ERR("failed to open dynamic lookup cache: %s", path_dynamic.c_str()); GGML_ABORT("Couldn't read dynamic lookup cache"); } } @@ -2141,7 +1836,6 @@ std::string common_speculative_type_to_str(common_speculative_type type) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: return "draft-simple"; case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: return "draft-eagle3"; case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: return "draft-mtp"; - case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: return "draft-dflash"; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: return "ngram-simple"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K: return "ngram-map-k"; case COMMON_SPECULATIVE_TYPE_NGRAM_MAP_K4V: return "ngram-map-k4v"; @@ -2194,7 +1888,6 @@ int32_t common_speculative_n_max(const common_params_speculative * spec) { case COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE: case COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3: case COMMON_SPECULATIVE_TYPE_DRAFT_MTP: - case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: n_max = std::max(n_max, std::max(0, spec->draft.n_max)); break; case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: @@ -2232,7 +1925,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_draft_simple = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_SIMPLE)); bool has_draft_eagle3 = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3)) && params.draft.ctx_dft != nullptr; bool has_draft_mtp = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_MTP)) && params.draft.ctx_dft != nullptr; - bool has_draft_dflash = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH)) && params.draft.ctx_dft != nullptr; @@ -2243,7 +1935,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, bool has_ngram_mod = (enabled_configs & (1u << COMMON_SPECULATIVE_TYPE_NGRAM_MOD)); // when adding a new type - update here the logic above - static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 10); + static_assert(COMMON_SPECULATIVE_TYPE_COUNT == 9); // this list here defines the priority of the speculators // the one with highest priority are listed first @@ -2273,9 +1965,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, if (has_draft_mtp) { configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_MTP, params)); } - if (has_draft_dflash) { - configs.push_back(common_speculative_config(COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH, params)); - } } std::vector> impls = {}; @@ -2296,10 +1985,6 @@ common_speculative * common_speculative_init(common_params_speculative & params, impls.push_back(std::make_unique(config.params, n_seq)); break; } - case COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH: { - impls.push_back(std::make_unique(config.params, n_seq)); - break; - } case COMMON_SPECULATIVE_TYPE_NGRAM_SIMPLE: { common_ngram_map ngram_map = get_common_ngram_map(config.type, config.params.ngram_simple); @@ -2349,7 +2034,7 @@ common_speculative * common_speculative_init(common_params_speculative & params, } if (impls.empty()) { - SPC_TRC("%s", "no implementations specified for speculative decoding\n"); + LOG_WRN("%s: no implementations specified for speculative decoding\n", __func__); return nullptr; } @@ -2476,13 +2161,13 @@ void common_speculative_draft(common_speculative * spec) { if (dp.n_max > 0) { if (!result.empty() && (int) result.size() > dp.n_max) { - SPC_DBG("truncating draft to %d tokens\n", dp.n_max); + LOG_DBG("%s: truncating draft to %d tokens\n", __func__, dp.n_max); result.resize(dp.n_max); } } if (!result.empty()) { - SPC_DBG("called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", + LOG_DBG("%s: called impl %s, hist size = %zu, call_count = %zu, gen = %zu\n", __func__, common_speculative_type_to_str(impl.get()->type).c_str(), dp.prompt->size(), impl.get()->n_call_draft, result.size()); @@ -2606,7 +2291,7 @@ void common_speculative_print_stats(const common_speculative * spec) { str_stats = ", #mean acc len = " + oss.str() + ", #acc rate/pos = (" + tmp.str() + ")"; } - SPC_TRC("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s%s\n", + LOG_INF("statistics %16s: #calls(b,g,a) = %4zu %6zu %6zu, #gen drafts = %6zu, #acc drafts = %5zu, #gen tokens = %6zu, #acc tokens = %5zu%s%s\n", common_speculative_type_to_str(impl->type).c_str(), impl->n_call_begin, impl->n_call_draft, impl->n_call_accept, impl->n_gen_drafts, diff --git a/conversion/__init__.py b/conversion/__init__.py index 02ea63852..5aad203e5 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -50,8 +50,6 @@ TEXT_MODEL_MAP: dict[str, str] = { "DeepseekV2ForCausalLM": "deepseek", "DeepseekV3ForCausalLM": "deepseek", "DeepseekV32ForCausalLM": "deepseek", - "DFlashDraftModel": "qwen", - "DeepseekV4ForCausalLM": "deepseek", "DistilBertForMaskedLM": "bert", "DistilBertForSequenceClassification": "bert", "DistilBertModel": "bert", diff --git a/conversion/base.py b/conversion/base.py index 0421aa4bc..08fd3747c 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -1273,7 +1273,7 @@ class TextModel(ModelBase): if (f_norm_eps := self.find_hparam(["layer_norm_eps", "layer_norm_epsilon", "norm_epsilon"], optional=True)) is not None: self.gguf_writer.add_layer_norm_eps(f_norm_eps) logger.info(f"gguf: layer norm epsilon = {f_norm_eps}") - if (n_experts := self.find_hparam(["num_local_experts", "num_experts", "n_routed_experts"], optional=True)) is not None: + if (n_experts := self.find_hparam(["num_local_experts", "num_experts"], optional=True)) is not None: self.gguf_writer.add_expert_count(n_experts) logger.info(f"gguf: expert count = {n_experts}") if (n_experts_used := self.find_hparam(["num_experts_per_tok", "num_experts_per_token", "top_k_experts"], optional=True)) is not None: @@ -1291,8 +1291,6 @@ class TextModel(ModelBase): self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SIGMOID) elif score_func == "softmax": self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SOFTMAX) - elif score_func == "sqrtsoftplus": - self.gguf_writer.add_expert_gating_func(gguf.ExpertGatingFuncType.SQRTSOFTPLUS) else: raise ValueError(f"Unsupported expert score gating function value: {score_func}") logger.info(f"gguf: expert score gating function = {score_func}") @@ -2602,17 +2600,6 @@ class LazyTorchTensor(gguf.LazyBase): return cls._wrap_fn(func)(*args, **kwargs) -if hasattr(torch, "float8_e8m0fnu"): - _torch_float8_e8m0 = torch.float8_e8m0fnu - LazyTorchTensor._dtype_map[_torch_float8_e8m0] = np.uint8 - LazyTorchTensor._dtype_byteswap_map[_torch_float8_e8m0] = np.uint8 - LazyTorchTensor._dtype_str_map["F8_E8M0"] = _torch_float8_e8m0 -else: - # Older torch builds do not expose F8_E8M0. Keep the raw bytes so callers - # that know the format can decode them explicitly. - LazyTorchTensor._dtype_str_map["F8_E8M0"] = torch.uint8 - - def get_model_architecture(hparams: dict[str, Any], model_type: ModelType) -> str: # TODO @ngxson : this won't work correctly if the model has both audio & vision encoders # maybe we should fallback to text model's arch in that case, since not many models have both diff --git a/conversion/deepseek.py b/conversion/deepseek.py index ea6ae23d5..4c93fb66d 100644 --- a/conversion/deepseek.py +++ b/conversion/deepseek.py @@ -1,18 +1,15 @@ from __future__ import annotations -import json import re -from pathlib import Path from typing import Any, Callable, Iterable, TYPE_CHECKING -import numpy as np import torch if TYPE_CHECKING: from torch import Tensor -from .base import LazyTorchTensor, MmprojModel, ModelBase, TextModel, gguf, logger +from .base import MmprojModel, ModelBase, TextModel, gguf, logger from .qwen import QwenModel @@ -470,307 +467,3 @@ class DeepseekV32Model(DeepseekV2Model): self.gguf_writer.add_indexer_head_count(self.hparams["index_n_heads"]) self.gguf_writer.add_indexer_key_length(self.hparams["index_head_dim"]) self.gguf_writer.add_indexer_top_k(self.hparams["index_topk"]) - - -@ModelBase.register("DeepseekV4ForCausalLM") -class DeepseekV4Model(TextModel): - model_arch = gguf.MODEL_ARCH.DEEPSEEK4 - _skipped_mtp_tensors = 0 - - def __init__(self, *args, **kwargs): - type(self)._skipped_mtp_tensors = 0 - super().__init__(*args, **kwargs) - - with open(self.dir_model / "config.json", "r", encoding="utf-8") as f: - raw_hparams = json.load(f) - for key, value in raw_hparams.items(): - self.hparams.setdefault(key, value) - - self.block_count = self.hparams["num_hidden_layers"] - self.tensor_map = gguf.get_tensor_name_map(self.model_arch, self.block_count) - - self._dsv4_fp8_dequantized: set[str] = set() - self._dsv4_bf16_tensors: set[str] = set() - self._dsv4_f32_tensors: set[str] = set() - self._dsv4_mxfp4_generated = False - self._collect_source_dtypes() - - if type(self)._skipped_mtp_tensors: - logger.info("Skipping %d DeepSeek-V4 MTP tensor(s) for conversion v0", type(self)._skipped_mtp_tensors) - - # add a default chat template; if the model has a built-in template, it will be overridden later - template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.jinja" - if template_path.is_file(): - with open(template_path, "r", encoding="utf-8") as f: - self.gguf_writer.add_chat_template(f.read()) - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, _ = item - if name.startswith("mtp."): - cls._skipped_mtp_tensors += 1 - return None - return super().filter_tensors(item) - - @staticmethod - def _float8_dtypes() -> tuple[torch.dtype, ...]: - return tuple( - dtype for dtype in ( - getattr(torch, "float8_e4m3fn", None), - getattr(torch, "float8_e5m2", None), - ) if dtype is not None - ) - - @staticmethod - def _e8m0_to_float(scale: Tensor) -> Tensor: - torch_float8_e8m0 = getattr(torch, "float8_e8m0fnu", None) - if torch_float8_e8m0 is not None and scale.dtype == torch_float8_e8m0: - return scale.float() - - bits = scale.view(torch.uint8).float() - return torch.exp2(bits - 127.0) - - def _collect_source_dtypes(self) -> None: - for name, gen in self.model_tensors.items(): - dtype = gen().dtype - if dtype == torch.bfloat16: - self._dsv4_bf16_tensors.add(name) - elif dtype == torch.float32: - self._dsv4_f32_tensors.add(name) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - hparams = self.hparams - - self.gguf_writer.add_rope_dimension_count(hparams["qk_rope_head_dim"]) - self.gguf_writer.add_q_lora_rank(hparams["q_lora_rank"]) - self.gguf_writer.add_sliding_window(hparams["sliding_window"]) - - self.gguf_writer.add_expert_feed_forward_length(hparams["moe_intermediate_size"]) - self.gguf_writer.add_expert_shared_count(hparams["n_shared_experts"]) - self.gguf_writer.add_expert_weights_scale(hparams["routed_scaling_factor"]) - self.gguf_writer.add_expert_weights_norm(hparams["norm_topk_prob"]) - self.gguf_writer.add_swiglu_clamp_exp([hparams["swiglu_limit"]] * self.block_count) - self.gguf_writer.add_swiglu_clamp_shexp([hparams["swiglu_limit"]] * self.block_count) - - self.gguf_writer.add_indexer_head_count(hparams["index_n_heads"]) - self.gguf_writer.add_indexer_key_length(hparams["index_head_dim"]) - self.gguf_writer.add_indexer_top_k(hparams["index_topk"]) - - self.gguf_writer.add_attention_output_group_count(hparams["o_groups"]) - self.gguf_writer.add_attention_output_lora_rank(hparams["o_lora_rank"]) - self.gguf_writer.add_attention_compress_ratios(hparams["compress_ratios"]) - self.gguf_writer.add_attention_compress_rope_freq_base(hparams["compress_rope_theta"]) - self.gguf_writer.add_hyper_connection_count(hparams["hc_mult"]) - self.gguf_writer.add_hyper_connection_sinkhorn_iterations(hparams["hc_sinkhorn_iters"]) - self.gguf_writer.add_hyper_connection_epsilon(hparams["hc_eps"]) - self.gguf_writer.add_hash_layer_count(hparams["num_hash_layers"]) - - def dequant_model(self): - fp8_dtypes = self._float8_dtypes() - tensors_to_remove: list[str] = [] - - def dequant_fp8_weight(weight: Tensor, scale: Tensor) -> Tensor: - out_features, in_features = weight.shape - scale_f = self._e8m0_to_float(scale) - scale_f = scale_f.repeat_interleave(128, 0)[:out_features] - scale_f = scale_f.repeat_interleave(128, 1)[:, :in_features] - return weight.float() * scale_f - - for name in list(self.model_tensors.keys()): - if not name.endswith(".scale"): - continue - weight_name = name.removesuffix(".scale") + ".weight" - if weight_name not in self.model_tensors: - continue - - weight = self.model_tensors[weight_name] - scale = self.model_tensors[name] - if weight().dtype not in fp8_dtypes: - continue - - self.model_tensors[weight_name] = lambda w=weight, s=scale: dequant_fp8_weight(w(), s()) - self._dsv4_fp8_dequantized.add(weight_name) - tensors_to_remove.append(name) - - for name in tensors_to_remove: - del self.model_tensors[name] - - @staticmethod - def _pack_mxfp4_blocks(weight: Tensor, scale: Tensor) -> np.ndarray: - packed = weight.contiguous().view(torch.uint8) - scale_u8 = scale.contiguous().view(torch.uint8) - - out_features, packed_cols = packed.shape - logical_cols = packed_cols * 2 - if logical_cols % 32 != 0: - raise ValueError(f"MXFP4 source row has {logical_cols} values, expected a multiple of 32") - - n_blocks = logical_cols // 32 - if tuple(scale_u8.shape) != (out_features, n_blocks): - raise ValueError(f"MXFP4 scale shape {tuple(scale_u8.shape)} does not match {(out_features, n_blocks)}") - - src = packed.reshape(out_features, n_blocks, 16) - low = src & 0x0F - high = (src >> 4) & 0x0F - - # The safetensors bytes store adjacent values as low/high nibbles. - # ggml MXFP4 blocks store values 0..15 in low nibbles and 16..31 in high nibbles. - vals = torch.stack((low, high), dim=-1).reshape(out_features, n_blocks, 32) - qs = vals[:, :, :16] | (vals[:, :, 16:] << 4) - raw = torch.cat((scale_u8.unsqueeze(-1), qs.to(torch.uint8)), dim=-1) - return raw.reshape(out_features, n_blocks * 17).cpu().numpy() - - def _write_mxfp4_expert_tensor(self, bid: int, proj: str, tensor_key: gguf.MODEL_TENSOR) -> list[str]: - n_experts = self.hparams["n_routed_experts"] - data: np.ndarray | None = None - consumed: list[str] = [] - - for eid in range(n_experts): - weight_name = f"layers.{bid}.ffn.experts.{eid}.{proj}.weight" - scale_name = f"layers.{bid}.ffn.experts.{eid}.{proj}.scale" - if weight_name not in self.model_tensors or scale_name not in self.model_tensors: - raise KeyError(f"Missing routed expert tensors for {weight_name}") - - weight = LazyTorchTensor.to_eager(self.model_tensors[weight_name]()) - scale = LazyTorchTensor.to_eager(self.model_tensors[scale_name]()) - packed = self._pack_mxfp4_blocks(weight, scale) - if data is None: - data = np.empty((n_experts, *packed.shape), dtype=packed.dtype) - data[eid] = packed - consumed.extend((weight_name, scale_name)) - - assert data is not None - new_name = self.format_tensor_name(tensor_key, bid) - shape = gguf.quant_shape_from_byte_shape(data.shape, gguf.GGMLQuantizationType.MXFP4) - logger.info(f"{new_name}: repacked routed experts to MXFP4, shape = {{{', '.join(str(n) for n in reversed(shape))}}}") - self.gguf_writer.add_tensor(new_name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP4) - - return consumed - - def _write_hash_routing_tensors(self) -> list[str]: - consumed: list[str] = [] - - for bid in range(self.hparams["num_hash_layers"]): - name = f"layers.{bid}.ffn.gate.tid2eid" - if name not in self.model_tensors: - raise KeyError(f"Missing hash routing tensor {name}") - - data_torch = LazyTorchTensor.to_eager(self.model_tensors[name]()) - data = data_torch.to(torch.int32).cpu().numpy() - new_name = self.format_tensor_name(gguf.MODEL_TENSOR.FFN_GATE_TID2EID, bid, ".weight") - logger.info(f"{new_name}: converted hash routing table to I32, shape = {{{', '.join(str(n) for n in reversed(data.shape))}}}") - self.gguf_writer.add_tensor(new_name, data) - consumed.append(name) - - return consumed - - def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - if self._dsv4_mxfp4_generated: - return () - - consumed: list[str] = self._write_hash_routing_tensors() - for bid in range(self.block_count): - consumed.extend(self._write_mxfp4_expert_tensor(bid, "w1", gguf.MODEL_TENSOR.FFN_GATE_EXP)) - consumed.extend(self._write_mxfp4_expert_tensor(bid, "w2", gguf.MODEL_TENSOR.FFN_DOWN_EXP)) - consumed.extend(self._write_mxfp4_expert_tensor(bid, "w3", gguf.MODEL_TENSOR.FFN_UP_EXP)) - - for name in consumed: - del self.model_tensors[name] - - self._dsv4_mxfp4_generated = True - return () - - def _format_dsv4_tensor_name(self, key: gguf.MODEL_TENSOR, bid: int | None, suffix: str = ".weight") -> str: - return self.format_tensor_name(key, bid, suffix) - - def _map_dsv4_tensor_name(self, name: str, bid: int | None) -> tuple[gguf.MODEL_TENSOR, str]: - root_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { - "embed.weight": (gguf.MODEL_TENSOR.TOKEN_EMBD, ".weight"), - "norm.weight": (gguf.MODEL_TENSOR.OUTPUT_NORM, ".weight"), - "head.weight": (gguf.MODEL_TENSOR.OUTPUT, ".weight"), - "hc_head_fn": (gguf.MODEL_TENSOR.HC_HEAD_FN, ".weight"), - "hc_head_base": (gguf.MODEL_TENSOR.HC_HEAD_BASE, ".weight"), - "hc_head_scale": (gguf.MODEL_TENSOR.HC_HEAD_SCALE, ".weight"), - } - if name in root_map: - return root_map[name] - - match = re.match(r"layers\.(\d+)\.(.+)$", name) - if match is None: - raise ValueError(f"Unsupported DeepSeek-V4 tensor {name!r}") - - layer = int(match.group(1)) - if bid != layer: - raise ValueError(f"Tensor {name!r} parsed bid {bid} but layer name has {layer}") - - layer_map: dict[str, tuple[gguf.MODEL_TENSOR, str]] = { - "hc_attn_fn": (gguf.MODEL_TENSOR.HC_ATTN_FN, ".weight"), - "hc_attn_base": (gguf.MODEL_TENSOR.HC_ATTN_BASE, ".weight"), - "hc_attn_scale": (gguf.MODEL_TENSOR.HC_ATTN_SCALE, ".weight"), - "hc_ffn_fn": (gguf.MODEL_TENSOR.HC_FFN_FN, ".weight"), - "hc_ffn_base": (gguf.MODEL_TENSOR.HC_FFN_BASE, ".weight"), - "hc_ffn_scale": (gguf.MODEL_TENSOR.HC_FFN_SCALE, ".weight"), - "attn.attn_sink": (gguf.MODEL_TENSOR.ATTN_SINKS, ".weight"), - "attn.wq_a.weight": (gguf.MODEL_TENSOR.ATTN_Q_A, ".weight"), - "attn.wq_b.weight": (gguf.MODEL_TENSOR.ATTN_Q_B, ".weight"), - "attn.q_norm.weight": (gguf.MODEL_TENSOR.ATTN_Q_A_NORM, ".weight"), - "attn.wkv.weight": (gguf.MODEL_TENSOR.ATTN_KV, ".weight"), - "attn.kv_norm.weight": (gguf.MODEL_TENSOR.ATTN_KV_NORM, ".weight"), - "attn.wo_a.weight": (gguf.MODEL_TENSOR.ATTN_OUT_A, ".weight"), - "attn.wo_b.weight": (gguf.MODEL_TENSOR.ATTN_OUT_B, ".weight"), - "attn.compressor.ape": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_APE, ".weight"), - "attn.compressor.wkv.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_WKV, ".weight"), - "attn.compressor.wgate.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_WGATE, ".weight"), - "attn.compressor.norm.weight": (gguf.MODEL_TENSOR.ATTN_COMPRESSOR_NORM, ".weight"), - "attn.indexer.wq_b.weight": (gguf.MODEL_TENSOR.INDEXER_ATTN_Q_B, ".weight"), - "attn.indexer.weights_proj.weight": (gguf.MODEL_TENSOR.INDEXER_PROJ, ".weight"), - "attn.indexer.compressor.ape": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_APE, ".weight"), - "attn.indexer.compressor.wkv.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_WKV, ".weight"), - "attn.indexer.compressor.wgate.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_WGATE, ".weight"), - "attn.indexer.compressor.norm.weight": (gguf.MODEL_TENSOR.INDEXER_COMPRESSOR_NORM, ".weight"), - "attn_norm.weight": (gguf.MODEL_TENSOR.ATTN_NORM, ".weight"), - "ffn_norm.weight": (gguf.MODEL_TENSOR.FFN_NORM, ".weight"), - "ffn.gate.weight": (gguf.MODEL_TENSOR.FFN_GATE_INP, ".weight"), - "ffn.gate.bias": (gguf.MODEL_TENSOR.FFN_EXP_PROBS_B, ".bias"), - "ffn.gate.tid2eid": (gguf.MODEL_TENSOR.FFN_GATE_TID2EID, ".weight"), - "ffn.shared_experts.w1.weight": (gguf.MODEL_TENSOR.FFN_GATE_SHEXP, ".weight"), - "ffn.shared_experts.w2.weight": (gguf.MODEL_TENSOR.FFN_DOWN_SHEXP, ".weight"), - "ffn.shared_experts.w3.weight": (gguf.MODEL_TENSOR.FFN_UP_SHEXP, ".weight"), - } - - tensor_name = match.group(2) - if tensor_name in layer_map: - return layer_map[tensor_name] - - if re.match(r"ffn\.experts\.\d+\.w[123]\.(weight|scale)$", tensor_name): - return gguf.MODEL_TENSOR.FFN_GATE_EXP, ".weight" - - raise ValueError(f"Unsupported DeepSeek-V4 tensor {name!r}") - - def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]: - if re.match(r"layers\.\d+\.ffn\.experts\.\d+\.w[123]\.(weight|scale)$", name): - return [] - - tensor_key, suffix = self._map_dsv4_tensor_name(name, bid) - if tensor_key == gguf.MODEL_TENSOR.FFN_GATE_TID2EID: - return [] - - return [(self._format_dsv4_tensor_name(tensor_key, bid, suffix), data_torch)] - - def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool: - del new_name, bid # unused - - if name in self._dsv4_fp8_dequantized and n_dims >= 2: - return gguf.GGMLQuantizationType.Q8_0 - if name in self._dsv4_f32_tensors: - return gguf.GGMLQuantizationType.F32 - if name in self._dsv4_bf16_tensors and n_dims >= 2: - return gguf.GGMLQuantizationType.BF16 - - return False - - def prepare_tensors(self): - super().prepare_tensors() - self._is_mxfp4 = True - self.ftype = gguf.LlamaFileType.MOSTLY_MXFP4_MOE diff --git a/conversion/llama.py b/conversion/llama.py index 315a619c9..b43cc994a 100644 --- a/conversion/llama.py +++ b/conversion/llama.py @@ -73,7 +73,7 @@ class LlamaModel(TextModel): target_num_layers = target_config["num_hidden_layers"] target_layers = [2, target_num_layers // 2, target_num_layers - 3] logger.info(f"EAGLE-3: target_layers = {target_layers} (target model has {target_num_layers} layers)") - self.gguf_writer.add_target_layers(target_layers) + self.gguf_writer.add_array(f"{self.gguf_writer.arch}.target_layers", target_layers) # target_hidden_size: prefer eagle3 config, fallback to target config if eagle3_raw_config.get("target_hidden_size") is not None: @@ -83,12 +83,12 @@ class LlamaModel(TextModel): target_hidden_size = target_config["hidden_size"] src = "target model config" logger.info(f"EAGLE-3: target_hidden_size = {target_hidden_size} (from {src})") - self.gguf_writer.add_target_hidden_size(target_hidden_size) + self.gguf_writer.add_uint32(f"{self.gguf_writer.arch}.target_hidden_size", target_hidden_size) # norm_before_residual (RedHat-style eagle3 specific) norm_before_residual = eagle3_raw_config.get("norm_before_residual", False) logger.info(f"EAGLE-3: norm_before_residual = {norm_before_residual}") - self.gguf_writer.add_norm_before_residual(norm_before_residual) + self.gguf_writer.add_bool(f"{self.gguf_writer.arch}.norm_before_residual", norm_before_residual) def set_vocab(self): # eagle3: use tokenizer from target model if provided diff --git a/conversion/qwen.py b/conversion/qwen.py index 0356bd2da..6b85eb9aa 100644 --- a/conversion/qwen.py +++ b/conversion/qwen.py @@ -625,51 +625,3 @@ class Qwen3_5TextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReor @ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM") class Qwen3_5MoeTextModel(_Qwen35MtpMixin, _Qwen35MRopeMixin, _LinearAttentionVReorderBase): model_arch = gguf.MODEL_ARCH.QWEN35MOE - - -@ModelBase.register("DFlashDraftModel") -class DFlashModel(Qwen3Model): - model_arch = gguf.MODEL_ARCH.DFLASH - - def set_vocab(self): - if self.target_model_dir is None: - raise ValueError( - "DFlash draft model requires --target-model-dir to be specified. " - "Please provide the path to the target model directory containing the tokenizer." - ) - logger.info(f"DFlash: Using tokenizer from target model: {self.target_model_dir}") - original_dir = self.dir_model - self.dir_model = self.target_model_dir - super().set_vocab() - self.dir_model = original_dir - - mask_token_id = self.hparams.get("dflash_config", {}).get("mask_token_id") - if mask_token_id is not None: - self.gguf_writer.add_mask_token_id(mask_token_id) - - def set_gguf_parameters(self): - super().set_gguf_parameters() - - block_size = self.hparams.get("block_size", 16) - self.gguf_writer.add_block_size(block_size) - dflash_config = self.hparams.get("dflash_config", {}) - - target_layer_ids = dflash_config.get("target_layer_ids", []) - if target_layer_ids: - extract_layer_ids = [i + 1 for i in target_layer_ids] - self.gguf_writer.add_target_layers(extract_layer_ids) - - use_sliding_window = self.hparams.get("use_sliding_window", False) - sliding_window = self.hparams.get("sliding_window") - layer_types = self.hparams.get("layer_types") - if use_sliding_window and sliding_window and layer_types: - is_swa = [lt == "sliding_attention" for lt in layer_types] - self.gguf_writer.add_sliding_window(sliding_window) - self.gguf_writer.add_sliding_window_pattern(is_swa) - - @classmethod - def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None: - name, gen = item - if not name.startswith("model."): - name = "model." + name - return super().filter_tensors((name, gen)) diff --git a/embd_res/kcpp_sdui.embd b/embd_res/kcpp_sdui.embd index 745fec1d3..50807c23b 100644 --- a/embd_res/kcpp_sdui.embd +++ b/embd_res/kcpp_sdui.embd @@ -5,18 +5,18 @@ Stable UI for KoboldCpp - - diff --git a/embd_res/klite.embd b/embd_res/klite.embd index 3b3f86b40..81d70a8fc 100644 --- a/embd_res/klite.embd +++ b/embd_res/klite.embd @@ -12,7 +12,7 @@ Current version indicated by LITEVER below. -->
@@ -151,13 +153,13 @@ MCP Servers - {visibleMcpServers.length} server{visibleMcpServers.length !== 1 ? 's' : ''} + {getEnabledMcpServers().length} server{getEnabledMcpServers().length !== 1 ? 's' : ''}
- {#each visibleMcpServers as server (server.id)} + {#each getEnabledMcpServers() as server (server.id)} {@const healthState = mcpStore.getHealthCheckState(server.id)} {@const hasError = healthState.status === HealthCheckStatus.ERROR} {@const displayName = mcpStore.getServerLabel(server)} @@ -200,7 +202,7 @@ {/each} - {#if visibleMcpServers.length === 0} + {#if getEnabledMcpServers().length === 0}
No MCP servers configured
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte index 4be582b39..5c2913202 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageUser/ChatMessageUserPending.svelte @@ -56,7 +56,7 @@
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte index 4337bb6a1..e466c84ee 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessageActions/ChatMessageActionCard/ChatMessageActionCardPermissionRequest.svelte @@ -1,9 +1,8 @@ @@ -69,27 +60,29 @@ Add New Server -
-
- (newServerUrl = v)} - onHeadersChange={(v) => (newServerHeaders = v)} - urlError={newServerUrl ? newServerUrlError : null} - id="new-server" - /> -
+
+ (newServerUrl = v)} + onHeadersChange={(v) => (newServerHeaders = v)} + urlError={newServerUrl ? newServerUrlError : null} + id="new-server" + /> +
- - + + - - - + +
diff --git a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte b/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte deleted file mode 100644 index 9b4489b82..000000000 --- a/tools/ui/src/lib/components/app/dialogs/DialogMcpServerRecommendations.svelte +++ /dev/null @@ -1,180 +0,0 @@ - - - - - - Do more with MCP - - Power-up your experience by adding tools, resources and more capabilities provided by MCP - servers. - - - -
-

Quickly get started with

- - {#each RECOMMENDED_MCP_SERVERS as server (server.id)} - (selected[server.id] = enabled)} - /> - {/each} - - {#if addedServers.length > 0} - {#each addedServers as server (server.id)} - - {/each} - {/if} - - {#if showAddForm} - - (newServerUrl = v)} - onHeadersChange={(v) => (newServerHeaders = v)} - urlError={newServerUrl ? newServerUrlError : null} - id="recommendation-new-server" - /> - -
- - - -
- - {:else} - - - - {/if} -
- - - - - - - - diff --git a/tools/ui/src/lib/components/app/dialogs/index.ts b/tools/ui/src/lib/components/app/dialogs/index.ts index 73f22c565..29136308c 100644 --- a/tools/ui/src/lib/components/app/dialogs/index.ts +++ b/tools/ui/src/lib/components/app/dialogs/index.ts @@ -18,15 +18,6 @@ */ export { default as DialogMcpServerAddNew } from './DialogMcpServerAddNew.svelte'; -/** - * **DialogMcpServerRecommendations** - Suggested MCP servers opt-in dialog - * - * Prompts the user to enable pre-defined recommended MCP servers on first launch. - * Shows one switch per suggested server and persists the choice as a per-chat - * override so the selected servers become available in conversations. - */ -export { default as DialogMcpServerRecommendations } from './DialogMcpServerRecommendations.svelte'; - /** * **DialogExportSettings** - Settings export dialog with sensitive data warning * diff --git a/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte b/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte index fd6e59a5b..e0bd8d98e 100644 --- a/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte +++ b/tools/ui/src/lib/components/app/forms/KeyValuePairs.svelte @@ -1,5 +1,4 @@
@@ -123,7 +103,6 @@ {#each pairs as pair, index (index)}
-
+
{#if showSkeleton} {:else if protocolVersion} diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte deleted file mode 100644 index 6cb3e18b6..000000000 --- a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardCompact.svelte +++ /dev/null @@ -1,156 +0,0 @@ - - - -
-
- {#if showSkeleton} - - - - - {:else} - - {/if} -
- - -
- - {#if isError && errorMessage} -

{errorMessage}

- {/if} - - {#if showSkeleton} -
- -
- -
- - - - -
- {:else} - {#if description} - {#if description.lines === 2} -

- {description.text} -

- {:else} -

- {description.text} -

- {/if} - {/if} - - {#if tools.length > 0} -
- {#each visibleTools as tool (tool.name)} - - - - {tool.name} - - - - -

- {tool.description ?? 'No description'} -

- - - {/each} - - {#if hiddenToolCount > 0} - - - - + {hiddenToolCount} more tools - - - - -

- {hiddenTools.map((tool) => tool.name).join(', ')} -

- - - {/if} -
- {/if} - {/if} - diff --git a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte index 8ed4ee8b8..6727a9000 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerCard/McpServerCardEditForm.svelte @@ -1,7 +1,6 @@ -
-
-

Configure Server

+
+

Configure Server

- (editUrl = v)} - onHeadersChange={(v) => (editHeaders = v)} - onUseProxyChange={(v) => (editUseProxy = v)} - urlError={editUrl ? urlError : null} - id={serverId} - /> + (editUrl = v)} + onHeadersChange={(v) => (editHeaders = v)} + onUseProxyChange={(v) => (editUseProxy = v)} + urlError={editUrl ? urlError : null} + id={serverId} + /> -
- +
+ - -
+
- +
diff --git a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte index 7f05d5fef..79738e30d 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerForm.svelte @@ -38,87 +38,14 @@ let headerPairs = $derived(parseHeadersToArray(headers)); - const AUTHORIZATION_HEADER = 'Authorization'; - const BEARER_PREFIX = 'Bearer '; - - // Heuristic: this dedicated UI only owns Authorization headers that already - // carry a Bearer scheme. Anything else (e.g. Basic, raw tokens) stays in the - // KV section so the user can still edit those values verbatim. - const matchesAuthorizationKey = (key: string): boolean => - key.trim().toLowerCase() === AUTHORIZATION_HEADER.toLowerCase(); - - const isBearerScheme = (value: string): boolean => - value.trim().toLowerCase().startsWith(BEARER_PREFIX.toLowerCase()); - - const ownedByBearerUi = (p: KeyValuePair): boolean => - matchesAuthorizationKey(p.key) && isBearerScheme(p.value); - - let hasAuthorization = $derived(headerPairs.some(ownedByBearerUi)); - - let wantsAuthorization = $state(false); - - let showAuthorization = $derived(hasAuthorization || wantsAuthorization); - - let urlInput: HTMLInputElement | null = $state(null); - let bearerInput: HTMLInputElement | null = $state(null); - - $effect(() => { - urlInput?.focus(); - }); - - $effect(() => { - if (wantsAuthorization && bearerInput) { - bearerInput.focus(); - } - }); - - let bearerToken = $derived.by(() => { - const auth = headerPairs.find(ownedByBearerUi); - if (!auth) return ''; - return auth.value.trim().slice(BEARER_PREFIX.length).trim(); - }); - - $effect(() => { - if (!headers.trim()) { - wantsAuthorization = false; - } - }); - function updateHeaderPairs(newPairs: KeyValuePair[]) { headerPairs = newPairs; onHeadersChange(serializeHeaders(newPairs)); } - - // The dedicated UI owns the Authorization slot end-to-end when the user - // engages it: any prior Authorization row (Bearer or otherwise) is replaced - // by exactly one { Authorization: "Bearer " } entry. JSON's last-key - // behavior would otherwise pick one arbitrarily, so we strip first. - function updateBearerToken(token: string) { - const filtered = headerPairs.filter((p) => !matchesAuthorizationKey(p.key)); - - const trimmed = token.trim(); - - if (trimmed) { - filtered.push({ key: AUTHORIZATION_HEADER, value: `${BEARER_PREFIX}${trimmed}` }); - } - - updateHeaderPairs(filtered); - } - - function setUseAuthorization(checked: boolean) { - wantsAuthorization = checked; - - if (!checked) { - // Only drop the entry this UI owns; a non-Bearer Authorization row - // authored in the KV section must survive a toggle off untouched. - const filtered = headerPairs.filter((p) => !ownedByBearerUi(p)); - updateHeaderPairs(filtered); - } - } -
-
+
+
@@ -130,52 +57,50 @@ value={url} oninput={(e) => onUrlChange(e.currentTarget.value)} class={urlError ? 'border-destructive' : ''} - bind:ref={urlInput} /> {#if urlError}

{urlError}

{/if} + + {#if !isWebSocket && onUseProxyChange} + + {/if}
- - - {#if showAuthorization} -
- updateBearerToken(e.currentTarget.value)} - class="pl-16" - bind:ref={bearerInput} - /> - - - Bearer - -
- {/if} - !ownedByBearerUi(p))} - onPairsChange={(pairs) => { - const auth = headerPairs.find(ownedByBearerUi); - updateHeaderPairs(auth ? [...pairs, auth] : pairs); - }} + class="mt-2" + pairs={headerPairs} + onPairsChange={updateHeaderPairs} keyPlaceholder="Header name" valuePlaceholder="Value" addButtonLabel="Add" @@ -183,37 +108,4 @@ sectionLabel="Custom Headers" sectionLabelOptional /> - - {#if !isWebSocket && onUseProxyChange} - - {/if}
diff --git a/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte b/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte index 3f128e02c..feafc5d81 100644 --- a/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte +++ b/tools/ui/src/lib/components/app/mcp/McpServerIdentity.svelte @@ -1,7 +1,6 @@ - -
{#if depth > 0} @@ -130,7 +109,7 @@ @@ -146,18 +125,15 @@ {#if isLoading} -
e.key === 'Enter' && handleStop(e)} - role="button" - tabindex="0" aria-label="Stop generation" >
+ @@ -169,52 +145,50 @@
- {#if renderActionsDropdown} -
- { - e.stopPropagation(); - handleTogglePin(); - } - }, - { - icon: Pencil, - label: 'Edit', - onclick: handleEdit, - shortcut: ['shift', 'cmd', 'e'] - }, - { - icon: Download, - label: 'Export', - onclick: (e: Event) => { - e.stopPropagation(); - conversationsStore.downloadConversation(conversation.id); - }, - shortcut: ['shift', 'cmd', 's'] - }, - { - icon: Trash2, - label: 'Delete', - onclick: handleDelete, - variant: 'destructive', - shortcut: ['shift', 'cmd', 'd'], - separator: true +
+ { + e.stopPropagation(); + handleTogglePin(); } - ]} - /> -
- {/if} - + }, + { + icon: Pencil, + label: 'Edit', + onclick: handleEdit, + shortcut: ['shift', 'cmd', 'e'] + }, + { + icon: Download, + label: 'Export', + onclick: (e: Event) => { + e.stopPropagation(); + conversationsStore.downloadConversation(conversation.id); + }, + shortcut: ['shift', 'cmd', 's'] + }, + { + icon: Trash2, + label: 'Delete', + onclick: handleDelete, + variant: 'destructive', + shortcut: ['shift', 'cmd', 'd'], + separator: true + } + ]} + /> +
+