From d7da1eb35c0c5ecdd3efed0e691e18ae8f570afe Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Sun, 19 Oct 2025 23:07:37 -0300 Subject: [PATCH] invert KCPP_BAKE_SD_VOCAB logic, move define to sdtype_adapter.cpp (#1803) Using KCPP_BAKE_SD_VOCAB to turn off the change to not embed the vocabulary files makes testing new upstream merges harder, because we then need to set that macro on the sd.cpp original build. So, revert the tests, making the define turn the change on. Also, since model.cpp is always built by Koboldcpp as part of the sdtype_adapter.cpp, it's enough to set the macro on that file. --- Makefile | 2 +- otherarch/sdcpp/model.cpp | 12 ++++++------ otherarch/sdcpp/sdtype_adapter.cpp | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index ca1271863..cc799d77f 100644 --- a/Makefile +++ b/Makefile @@ -763,7 +763,7 @@ main: tools/main/main.cpp common/arg.cpp build-info.h ggml.o ggml-cpu.o ggml-ops mainvk: tools/main/main.cpp common/arg.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_vulkan.o llava.o ggml-backend_vulkan.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) sdmain: otherarch/sdcpp/util.cpp otherarch/sdcpp/main.cpp otherarch/sdcpp/stable-diffusion.cpp otherarch/sdcpp/upscaler.cpp otherarch/sdcpp/model.cpp otherarch/sdcpp/tokenize_util.cpp otherarch/sdcpp/thirdparty/zip.c build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend_default.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) - $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -DKCPP_BAKE_SD_VOCAB -o $@ $(LDFLAGS) + $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend_default.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) ttsmain: tools/tts/tts.cpp common/arg.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) diff --git a/otherarch/sdcpp/model.cpp b/otherarch/sdcpp/model.cpp index a00b7340a..3d04cf7f1 100644 --- a/otherarch/sdcpp/model.cpp +++ b/otherarch/sdcpp/model.cpp @@ -17,7 +17,7 @@ #include "model.h" #include "stable-diffusion.h" #include "util.h" -#ifdef KCPP_BAKE_SD_VOCAB +#ifndef KCPP_BAKE_SD_VOCAB #include "vocab.hpp" #include "vocab_qwen.hpp" #include "vocab_umt5.hpp" @@ -2056,7 +2056,7 @@ void ModelLoader::set_wtype_override(ggml_type wtype, std::string prefix) { } std::string ModelLoader::load_merges() { -#ifdef KCPP_BAKE_SD_VOCAB +#ifndef KCPP_BAKE_SD_VOCAB std::string merges_utf8_str(reinterpret_cast(merges_utf8_c_str), sizeof(merges_utf8_c_str)); return merges_utf8_str; #else @@ -2065,7 +2065,7 @@ std::string ModelLoader::load_merges() { } std::string ModelLoader::load_qwen2_merges() { -#ifdef KCPP_BAKE_SD_VOCAB +#ifndef KCPP_BAKE_SD_VOCAB std::string merges_utf8_str(reinterpret_cast(qwen2_merges_utf8_c_str), sizeof(qwen2_merges_utf8_c_str)); return merges_utf8_str; #else @@ -2074,7 +2074,7 @@ std::string ModelLoader::load_qwen2_merges() { } std::string ModelLoader::load_t5_tokenizer_json() { -#ifdef KCPP_BAKE_SD_VOCAB +#ifndef KCPP_BAKE_SD_VOCAB std::string json_str(reinterpret_cast(t5_tokenizer_json_str), sizeof(t5_tokenizer_json_str)); return json_str; #else @@ -2083,7 +2083,7 @@ std::string ModelLoader::load_t5_tokenizer_json() { } std::string ModelLoader::load_umt5_tokenizer_json() { -#ifdef KCPP_BAKE_SD_VOCAB +#ifndef KCPP_BAKE_SD_VOCAB std::string json_str(reinterpret_cast(umt5_tokenizer_json_str), sizeof(umt5_tokenizer_json_str)); return json_str; #else @@ -2681,4 +2681,4 @@ bool convert(const char* input_path, const char* vae_path, const char* output_pa } bool success = model_loader.save_to_gguf_file(output_path, (ggml_type)output_type, tensor_type_rules); return success; -} \ No newline at end of file +} diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp index 77504d26d..a4a38b485 100644 --- a/otherarch/sdcpp/sdtype_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -13,6 +13,8 @@ #include #include +#define KCPP_BAKE_SD_VOCAB + #include "model_adapter.h" std::string sd_load_merges();