diff --git a/.gitignore b/.gitignore index e0a52e60c..f7164a968 100644 --- a/.gitignore +++ b/.gitignore @@ -51,7 +51,6 @@ models-mnt /infill /libllama.so /llama-bench -/llava-cli /lookahead /lookup /main diff --git a/CMakeLists.txt b/CMakeLists.txt index df5cbccac..0619150a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -470,10 +470,10 @@ add_library(common2 common/common.h common/sampling.cpp common/sampling.h - tools/llava/llava.cpp - tools/llava/llava.h - tools/llava/clip.cpp - tools/llava/clip.h + tools/mtmd/llava.cpp + tools/mtmd/llava.h + tools/mtmd/clip.cpp + tools/mtmd/clip.h src/unicode.h src/unicode.cpp src/unicode-data.cpp diff --git a/Makefile b/Makefile index 84d838199..115ac3fd7 100644 --- a/Makefile +++ b/Makefile @@ -511,7 +511,7 @@ sgemm_failsafe.o: ggml/src/ggml-cpu/llamafile/sgemm.cpp ggml/src/ggml-cpu/llamaf #there's no intrinsics or special gpu ops used here, so we can have a universal object ggml-alloc.o: ggml/src/ggml-alloc.c ggml/include/ggml.h ggml/include/ggml-alloc.h $(CC) $(CFLAGS) -c $< -o $@ -llava.o: tools/llava/llava.cpp tools/llava/llava.h +llava.o: tools/mtmd/llava.cpp tools/mtmd/llava.h $(CXX) $(CXXFLAGS) -c $< -o $@ unicode.o: src/unicode.cpp src/unicode.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -541,11 +541,11 @@ ggml-backend-reg_vulkan.o: ggml/src/ggml-backend-reg.cpp ggml/src/ggml-backend-i $(CXX) $(CXXFLAGS) $(VULKAN_FLAGS) -c $< -o $@ ggml-backend-reg_cublas.o: ggml/src/ggml-backend-reg.cpp ggml/src/ggml-backend-impl.h ggml/include/ggml.h ggml/include/ggml-backend.h ggml/include/ggml-cpu.h $(CXX) $(CXXFLAGS) $(CUBLAS_FLAGS) $(HIPFLAGS) -c $< -o $@ -llavaclip_default.o: tools/llava/clip.cpp tools/llava/clip.h +llavaclip_default.o: tools/mtmd/clip.cpp tools/mtmd/clip.h $(CXX) $(CXXFLAGS) -c $< -o $@ -llavaclip_cublas.o: tools/llava/clip.cpp tools/llava/clip.h +llavaclip_cublas.o: tools/mtmd/clip.cpp tools/mtmd/clip.h $(CXX) $(CXXFLAGS) $(CUBLAS_FLAGS) $(HIPFLAGS) -c $< -o $@ -llavaclip_vulkan.o: tools/llava/clip.cpp tools/llava/clip.h +llavaclip_vulkan.o: tools/mtmd/clip.cpp tools/mtmd/clip.h $(CXX) $(CXXFLAGS) $(VULKAN_FLAGS) -c $< -o $@ #this is only used for accelerate @@ -673,7 +673,7 @@ ttsmain: tools/tts/tts.cpp common/arg.cpp build-info.h ggml.o ggml-cpu.o ggml-op $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mtmd-cli: tools/llava/mtmd-cli.cpp tools/llava/mtmd.cpp common/arg.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o $(OBJS_FULL) $(OBJS) +mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/mtmd.cpp common/arg.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) ggml/src/ggml-vulkan-shaders.cpp: @@ -827,7 +827,7 @@ quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o llavaclip_default.o llava.o ggml-backend_default.o ggml-backend-reg_default.o $(OBJS_FULL) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_clip: tools/llava/clip.cpp tools/llava/clip.h tools/quantclip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o ggml-backend_default.o ggml-backend-reg_default.o $(OBJS_FULL) +quantize_clip: tools/mtmd/clip.cpp tools/mtmd/clip.h tools/quantclip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o ggml-backend_default.o ggml-backend-reg_default.o $(OBJS_FULL) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) #window simple clinfo diff --git a/common/arg.cpp b/common/arg.cpp index 5dce9b63f..1e49964a2 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -2212,14 +2212,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_NO_CONT_BATCHING")); add_opt(common_arg( {"--mmproj"}, "FILE", - "path to a multimodal projector file. see tools/llava/README.md", + "path to a multimodal projector file. see tools/mtmd/README.md", [](common_params & params, const std::string & value) { params.mmproj.path = value; } ).set_examples(mmproj_examples)); add_opt(common_arg( {"--mmproj-url"}, "URL", - "URL to a multimodal projector file. see tools/llava/README.md", + "URL to a multimodal projector file. see tools/mtmd/README.md", [](common_params & params, const std::string & value) { params.mmproj.url = value; } diff --git a/common/common.h b/common/common.h index c835be13f..155d4ecbe 100644 --- a/common/common.h +++ b/common/common.h @@ -336,7 +336,7 @@ struct common_params { common_conversation_mode conversation_mode = COMMON_CONVERSATION_MODE_AUTO; - // multimodal models (see tools/llava) + // multimodal models (see tools/mtmd) struct common_params_model mmproj; bool mmproj_use_gpu = true; // use GPU for multimodal model bool no_mmproj = false; // explicitly disable multimodal model diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 52d619b3a..9c04c7727 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -40,8 +40,8 @@ #include "neox_v2.cpp" #include "neox_v3.cpp" #include "mpt_v3.cpp" -#include "tools/llava/clip.h" -#include "tools/llava/llava.h" +#include "tools/mtmd/clip.h" +#include "tools/mtmd/llava.h" #include "common/common.h" //const diff --git a/koboldcpp.py b/koboldcpp.py index 3b17462f5..e2347139a 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -52,7 +52,7 @@ logit_bias_max = 512 dry_seq_break_max = 128 # global vars -KcppVersion = "1.90.2" +KcppVersion = "1.91" showdebug = True kcpp_instance = None #global running instance global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False} diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 06a96af55..d64956b84 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -27,7 +27,7 @@ else() add_subdirectory(run) add_subdirectory(tokenize) add_subdirectory(tts) - add_subdirectory(llava) + add_subdirectory(mtmd) if (GGML_RPC) add_subdirectory(rpc) endif() diff --git a/tools/llava/README-quantize.md b/tools/mtmd/README-quantize.md similarity index 100% rename from tools/llava/README-quantize.md rename to tools/mtmd/README-quantize.md diff --git a/tools/llava/clip-impl.h b/tools/mtmd/clip-impl.h similarity index 100% rename from tools/llava/clip-impl.h rename to tools/mtmd/clip-impl.h diff --git a/tools/llava/clip.cpp b/tools/mtmd/clip.cpp similarity index 100% rename from tools/llava/clip.cpp rename to tools/mtmd/clip.cpp diff --git a/tools/llava/clip.h b/tools/mtmd/clip.h similarity index 100% rename from tools/llava/clip.h rename to tools/mtmd/clip.h diff --git a/tools/llava/convert_image_encoder_to_gguf.py b/tools/mtmd/convert_image_encoder_to_gguf.py similarity index 100% rename from tools/llava/convert_image_encoder_to_gguf.py rename to tools/mtmd/convert_image_encoder_to_gguf.py diff --git a/tools/llava/deprecation-warning.cpp b/tools/mtmd/deprecation-warning.cpp similarity index 100% rename from tools/llava/deprecation-warning.cpp rename to tools/mtmd/deprecation-warning.cpp diff --git a/tools/llava/glmedge-convert-image-encoder-to-gguf.py b/tools/mtmd/glmedge-convert-image-encoder-to-gguf.py similarity index 100% rename from tools/llava/glmedge-convert-image-encoder-to-gguf.py rename to tools/mtmd/glmedge-convert-image-encoder-to-gguf.py diff --git a/tools/llava/glmedge-surgery.py b/tools/mtmd/glmedge-surgery.py similarity index 100% rename from tools/llava/glmedge-surgery.py rename to tools/mtmd/glmedge-surgery.py diff --git a/tools/llava/llava.cpp b/tools/mtmd/llava.cpp similarity index 100% rename from tools/llava/llava.cpp rename to tools/mtmd/llava.cpp diff --git a/tools/llava/llava.h b/tools/mtmd/llava.h similarity index 100% rename from tools/llava/llava.h rename to tools/mtmd/llava.h diff --git a/tools/llava/llava_surgery.py b/tools/mtmd/llava_surgery.py similarity index 100% rename from tools/llava/llava_surgery.py rename to tools/mtmd/llava_surgery.py diff --git a/tools/llava/llava_surgery_v2.py b/tools/mtmd/llava_surgery_v2.py similarity index 100% rename from tools/llava/llava_surgery_v2.py rename to tools/mtmd/llava_surgery_v2.py diff --git a/tools/llava/minicpmv-convert-image-encoder-to-gguf.py b/tools/mtmd/minicpmv-convert-image-encoder-to-gguf.py similarity index 100% rename from tools/llava/minicpmv-convert-image-encoder-to-gguf.py rename to tools/mtmd/minicpmv-convert-image-encoder-to-gguf.py diff --git a/tools/llava/minicpmv-surgery.py b/tools/mtmd/minicpmv-surgery.py similarity index 100% rename from tools/llava/minicpmv-surgery.py rename to tools/mtmd/minicpmv-surgery.py diff --git a/tools/llava/mtmd-cli.cpp b/tools/mtmd/mtmd-cli.cpp similarity index 100% rename from tools/llava/mtmd-cli.cpp rename to tools/mtmd/mtmd-cli.cpp diff --git a/tools/llava/mtmd.cpp b/tools/mtmd/mtmd.cpp similarity index 100% rename from tools/llava/mtmd.cpp rename to tools/mtmd/mtmd.cpp diff --git a/tools/llava/mtmd.h b/tools/mtmd/mtmd.h similarity index 100% rename from tools/llava/mtmd.h rename to tools/mtmd/mtmd.h diff --git a/tools/llava/qwen2vl-test.cpp b/tools/mtmd/qwen2vl-test.cpp similarity index 100% rename from tools/llava/qwen2vl-test.cpp rename to tools/mtmd/qwen2vl-test.cpp diff --git a/tools/llava/requirements.txt b/tools/mtmd/requirements.txt similarity index 100% rename from tools/llava/requirements.txt rename to tools/mtmd/requirements.txt diff --git a/tools/llava/test-1.jpeg b/tools/mtmd/test-1.jpeg similarity index 100% rename from tools/llava/test-1.jpeg rename to tools/mtmd/test-1.jpeg diff --git a/tools/llava/tests.sh b/tools/mtmd/tests.sh similarity index 100% rename from tools/llava/tests.sh rename to tools/mtmd/tests.sh