From 794a271cfac7e577b6f71eb27e670480c32afafb Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:06:30 +0800 Subject: [PATCH] upgrade to mtmd checkpoint 1 --- CMakeLists.txt | 6 +- Makefile | 62 +++--- gpttype_adapter.cpp | 476 +++++++++++------------------------------- otherarch/otherarch.h | 5 +- 4 files changed, 164 insertions(+), 385 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2a5018447..14960f9d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -463,8 +463,10 @@ add_library(common2 common/speculative.h common/sampling.cpp common/sampling.h - tools/mtmd/llava.cpp - tools/mtmd/llava.h + tools/mtmd/mtmd.cpp + tools/mtmd/mtmd.h + tools/mtmd/mtmd-helper.cpp + tools/mtmd/mtmd-helper.h tools/mtmd/clip.cpp tools/mtmd/clip.h src/unicode.h diff --git a/Makefile b/Makefile index 396c7c7ab..dbea676cf 100644 --- a/Makefile +++ b/Makefile @@ -590,7 +590,9 @@ sgemm_failsafe.o: ggml/src/ggml-cpu/llamafile/sgemm.cpp ggml/src/ggml-cpu/llamaf #there's no intrinsics or special gpu ops used here, so we can have a universal object ggml-alloc.o: ggml/src/ggml-alloc.c ggml/include/ggml.h ggml/include/ggml-alloc.h $(CC) $(CFLAGS) -c $< -o $@ -llava.o: tools/mtmd/llava.cpp tools/mtmd/llava.h +mtmd.o: tools/mtmd/mtmd.cpp tools/mtmd/mtmd.h + $(CXX) $(CXXFLAGS) -c $< -o $@ +mtmd-helper.o: tools/mtmd/mtmd-helper.cpp tools/mtmd/mtmd-helper.h $(CXX) $(CXXFLAGS) -c $< -o $@ unicode-common.o: common/unicode.cpp common/unicode.h $(CXX) $(CXXFLAGS) -c $< -o $@ @@ -624,11 +626,11 @@ ggml-backend-reg_vulkan.o: ggml/src/ggml-backend-reg.cpp ggml/src/ggml-backend-i $(CXX) $(CXXFLAGS) $(VULKAN_FLAGS) -c $< -o $@ ggml-backend-reg_cublas.o: ggml/src/ggml-backend-reg.cpp ggml/src/ggml-backend-impl.h ggml/include/ggml.h ggml/include/ggml-backend.h ggml/include/ggml-cpu.h $(CXX) $(CXXFLAGS) $(CUBLAS_FLAGS) $(HIPFLAGS) -c $< -o $@ -llavaclip_default.o: tools/mtmd/clip.cpp tools/mtmd/clip.h +clip_default.o: tools/mtmd/clip.cpp tools/mtmd/clip.h $(CXX) $(CXXFLAGS) -c $< -o $@ -llavaclip_cublas.o: tools/mtmd/clip.cpp tools/mtmd/clip.h +clip_cublas.o: tools/mtmd/clip.cpp tools/mtmd/clip.h $(CXX) $(CXXFLAGS) $(CUBLAS_FLAGS) $(HIPFLAGS) -c $< -o $@ -llavaclip_vulkan.o: tools/mtmd/clip.cpp tools/mtmd/clip.h +clip_vulkan.o: tools/mtmd/clip.cpp tools/mtmd/clip.h $(CXX) $(CXXFLAGS) $(VULKAN_FLAGS) -c $< -o $@ #rpc @@ -766,31 +768,31 @@ clean: rm -vf otherarch/sdcpp/*.o otherarch/sdcpp/*/*.o otherarch/sdcpp/*/*/*.o otherarch/sdcpp/*/*/*/*.o # useful tools -main: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +main: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mainvk: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_vulkan.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +mainvk: tools/completion/completion.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_vulkan.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +fitparams: tools/fit-params/main.cpp tools/fit-params/fit-params.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +sdmain: $(OBJS_SDCOMMON) $(OBJS_SDMAIN) build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +whispermain: otherarch/whispercpp/main.cpp otherarch/whispercpp/whisper.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttsmain: tools/tts/tts.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttsmain: tools/tts/tts.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +gguf-split: tools/gguf-split/gguf-split.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o build-info.h clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/mtmd.cpp tools/mtmd/mtmd-helper.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +mtmd-cli: tools/mtmd/mtmd-cli.cpp tools/mtmd/clip.cpp common/debug.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h mtmd.o mtmd-helper.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embedding: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +embedding: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_vulkan.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +embeddingvk: examples/embedding/embedding.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp src/llama-cparams.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) -ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +ttscppmain: otherarch/ttscpp/cli/cli.cpp otherarch/ttscpp/cli/playback.cpp otherarch/ttscpp/cli/playback.h otherarch/ttscpp/cli/write_file.cpp otherarch/ttscpp/cli/write_file.h otherarch/ttscpp/cli/vad.cpp otherarch/ttscpp/cli/vad.h otherarch/ttscpp/src/ttscpp.cpp otherarch/ttscpp/src/ttstokenizer.cpp otherarch/ttscpp/src/ttssampler.cpp otherarch/ttscpp/src/parler_model.cpp otherarch/ttscpp/src/dac_model.cpp otherarch/ttscpp/src/ttsutil.cpp otherarch/ttscpp/src/ttsargs.cpp otherarch/ttscpp/src/ttst5_encoder_model.cpp otherarch/ttscpp/src/phonemizer.cpp otherarch/ttscpp/src/tts_model.cpp otherarch/ttscpp/src/kokoro_model.cpp otherarch/ttscpp/src/dia_model.cpp otherarch/ttscpp/src/orpheus_model.cpp otherarch/ttscpp/src/snac_model.cpp otherarch/ttscpp/src/general_neural_audio_codec.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +qwen3tts: otherarch/qwen3tts/q3ttsmain.cpp otherarch/qwen3tts/qwen3_tts.cpp otherarch/qwen3tts/text_tokenizer.cpp otherarch/qwen3tts/gguf_loader.cpp otherarch/qwen3tts/tts_transformer.cpp otherarch/qwen3tts/audio_tokenizer_decoder.cpp otherarch/qwen3tts/audio_tokenizer_encoder.cpp otherarch/qwen3tts/coreml_code_predictor_stub.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $(filter-out %.h,$^) -o $@ $(LDFLAGS) -rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o llavaclip_vulkan.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib +rpcserver: tools/rpc/rpc-server.cpp common/arg.cpp common/chat.cpp common/preset.cpp common/download.cpp build-info.h ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o console.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o ggml-repack.o $(OBJS_FULL) $(OBJS) lib/vulkan-1.lib $(CXX) $(CXXFLAGS) -DGGML_USE_VULKAN -DSD_USE_VULKAN $(filter-out %.h,$^) -o $@ $(LDFLAGS) ggml/src/ggml-vulkan-shaders.cpp: @@ -891,11 +893,11 @@ else endif #generated libraries -koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_default: ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(DEFAULT_BUILD) ifdef FAILSAFE_BUILD -koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) +koboldcpp_failsafe: ggml_v4_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FAILSAFE) $(OBJS) $(FAILSAFE_BUILD) else koboldcpp_failsafe: @@ -903,7 +905,7 @@ koboldcpp_failsafe: endif ifdef NOAVX2_BUILD -koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_noavx2: ggml_v4_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_failsafe.o $(OBJS_SDTYPE) whispercpp_default.o tts_default.o music_default.o embeddings_default.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(NOAVX2_BUILD) else koboldcpp_noavx2: @@ -911,7 +913,7 @@ koboldcpp_noavx2: endif ifdef CUBLAS_BUILD -koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o llavaclip_cublas.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_cublas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(CUBLAS_OBJS) $(OBJS_FULL) $(OBJS) $(CUBLAS_BUILD) else koboldcpp_cublas: @@ -919,7 +921,7 @@ koboldcpp_cublas: endif ifdef HIPBLAS_BUILD -koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o llavaclip_cublas.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) +koboldcpp_hipblas: ggml_v4_cublas.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3_cublas.o ggml_v2_cublas.o ggml_v1.o expose.o gpttype_adapter_cublas.o $(OBJS_SDTYPE) whispercpp_cublas.o tts_default.o music_default.o embeddings_default.o clip_cublas.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_cublas.o ggml-repack.o $(HIP_OBJS) $(OBJS_FULL) $(OBJS) $(HIPBLAS_BUILD) else koboldcpp_hipblas: @@ -927,12 +929,12 @@ koboldcpp_hipblas: endif ifdef VULKAN_BUILD -koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o llavaclip_vulkan.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) +koboldcpp_vulkan: ggml_v4_vulkan.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o ggml_v3.o ggml_v2.o ggml_v1.o expose.o gpttype_adapter_vulkan.o ggml-vulkan.o ggml-vulkan-shaders.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(VULKAN_BUILD) ifdef NOAVX2_BUILD -koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o llavaclip_vulkan.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) +koboldcpp_vulkan_noavx2: ggml_v4_vulkan_noavx2.o ggml-cpu_v4_noavx2.o ggml-ops-noavx2.o ggml-vec-noavx2.o ggml-binops.o ggml-unops.o ggml_v3_noavx2.o ggml_v2_noavx2.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLE) $(OBJS) $(VULKAN_BUILD) -koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o llavaclip_vulkan.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) +koboldcpp_vulkan_failsafe: ggml_v4_vulkan_failsafe.o ggml-cpu_v4_failsafe.o ggml-ops-failsafe.o ggml-vec-failsafe.o ggml-binops.o ggml-unops.o ggml_v3_failsafe.o ggml_v2_failsafe.o ggml_v1_failsafe.o expose.o gpttype_adapter_vulkan_noavx2.o ggml-vulkan-noext.o ggml-vulkan-shaders-noext.o $(OBJS_SDTYPE) whispercpp_vulkan.o tts_default.o music_default.o embeddings_default.o clip_vulkan.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_vulkan.o ggml-repack.o $(OBJS_SIMPLER) $(OBJS) $(VULKAN_BUILD) else koboldcpp_vulkan_noavx2: @@ -950,15 +952,15 @@ koboldcpp_vulkan_failsafe: endif # tools -quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gguf: tools/quantize/main.cpp tools/quantize/quantize.cpp common/imatrix-loader.cpp ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gptj: otherarch/tools/gptj_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_gpt2: otherarch/tools/gpt2_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_neox: otherarch/tools/neox_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) -quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o llavaclip_default.o llava.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) +quantize_mpt: otherarch/tools/mpt_quantize.cpp otherarch/tools/common-ggml.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o clip_default.o mtmd.o mtmd-helper.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) quantize_clip: tools/mtmd/clip.cpp tools/quantclip.cpp ggml_v3.o ggml.o ggml-cpu.o ggml-ops.o ggml-vec.o ggml-binops.o ggml-unops.o llama.o ggml-backend.o ggml-backend-meta.o ggml-backend-reg_default.o ggml-repack.o $(OBJS_FULL) $(OBJS) $(CXX) $(CXXFLAGS) $^ -o $@ $(LDFLAGS) diff --git a/gpttype_adapter.cpp b/gpttype_adapter.cpp index 314de955b..62f3f986b 100644 --- a/gpttype_adapter.cpp +++ b/gpttype_adapter.cpp @@ -47,9 +47,8 @@ #include "neox_v2.cpp" #include "neox_v3.cpp" #include "mpt_v3.cpp" -#include "tools/mtmd/clip.h" -#include "tools/mtmd/llava.h" -#include "tools/mtmd/mtmd-audio.h" +#include "tools/mtmd/mtmd.h" +#include "tools/mtmd/mtmd-helper.h" #include "common/common.h" #include "ggml-rpc.h" @@ -117,17 +116,12 @@ static llama_context * llama_ctx_v4 = nullptr; static llama_context * draft_ctx = nullptr; //will remain null if speculative is unused static llama_context * guidance_ctx = nullptr; //for classifier free guidance, will be null if unused -static clip_ctx * clp_ctx_v = nullptr; //for llava -static clip_image_u8 * clp_img_data = nullptr; //most recent image -static clip_ctx * clp_ctx_a = nullptr; //for audio multimodal -static std::unique_ptr audio_preproc; //for audio processing +static mtmd_context * mtmd_ctx = nullptr; //for multimodal media static std::vector media_objects; -static std::vector last_media_mem; //for storing dummy tokens that will be consumed by llava -static std::string media_composite_image_signature = ""; //for identifying when the llava images change, we need to invalidate the cache +static std::vector last_media_mem; //for storing dummy tokens that will be consumed by mtmd +static std::string media_composite_image_signature = ""; //for identifying when the media changes, we need to invalidate the cache static int current_media_identifier = MEDIA_TOKEN_IDENTIFIER_A; -static int vision_max_res = 2048; static bool use_mrope = false; -static bool vision_use_non_casual = false; static kcpp_params * kcpp_data = nullptr; static int max_context_limit_at_load = 0; @@ -2000,6 +1994,25 @@ static void load_grammar(const std::string & gammarstr) } static bool kcpp_eval_media(llama_context * ctx_llama, const media_chunk & mediachunk, int n_batch, int * n_past, bool is2d) { + if (mtmd_ctx && mediachunk.mtmd_chunk) { + llama_pos new_n_past = *n_past; + int32_t result = mtmd_helper_eval_chunk_single( + mtmd_ctx, + ctx_llama, + static_cast(mediachunk.mtmd_chunk), + *n_past, + 0, + n_batch, + true, + &new_n_past); + if (result != 0) { + fprintf(stderr, "\n%s : failed to eval mtmd media chunk, status %d\n", __func__, result); + return false; + } + *n_past = new_n_past; + return true; + } + float * img_embd = mediachunk.clp_img_embd; int num_img_tokens = mediachunk.clp_image_tokens; int img_nx = mediachunk.nx; @@ -2008,27 +2021,14 @@ static bool kcpp_eval_media(llama_context * ctx_llama, const media_chunk & media const int image_n_past = *n_past; kcpp_embd_batch media_batch = kcpp_embd_batch(img_embd, num_img_tokens, image_n_past, use_mrope, is2d, img_nx, img_ny); - const bool non_casual = (vision_use_non_casual && !mediachunk.is_audio); - if(non_casual) - { - llama_set_causal_attn(llama_ctx_v4, false); - } for (int i = 0; i < num_img_tokens; i += n_batch) { const int n_eval = std::min(n_batch, num_img_tokens - i); llama_batch batch_embd_view = media_batch.get_view(i, n_eval, n_embd_mmproj); if (llama_decode(ctx_llama, batch_embd_view)) { fprintf(stderr, "\n%s : failed to eval image\n", __func__); - if(non_casual) - { - llama_set_causal_attn(llama_ctx_v4, true); - } return false; } } - if(non_casual) - { - llama_set_causal_attn(llama_ctx_v4, true); - } *n_past += num_img_tokens; return true; } @@ -2265,45 +2265,7 @@ static void connect_rpc_servers(const std::string & servers) { } } -void kcpp_init_audio_proj(clip_ctx * ctx_a) -{ - projector_type proj = clip_get_projector_type(ctx_a); - - // set preprocessor - switch (proj) { - case PROJECTOR_TYPE_QWEN2A: - case PROJECTOR_TYPE_QWEN25O: - case PROJECTOR_TYPE_ULTRAVOX: - case PROJECTOR_TYPE_VOXTRAL: - case PROJECTOR_TYPE_GLMA: - case PROJECTOR_TYPE_MUSIC_FLAMINGO: - case PROJECTOR_TYPE_MERALION: - audio_preproc = std::make_unique(ctx_a); - break; - case PROJECTOR_TYPE_QWEN3A: - audio_preproc = std::make_unique(ctx_a); - break; - case PROJECTOR_TYPE_LFM2A: - audio_preproc = std::make_unique(ctx_a); - break; - case PROJECTOR_TYPE_GRANITE_SPEECH: - audio_preproc = std::make_unique(ctx_a); - break; - case PROJECTOR_TYPE_GEMMA4A: - audio_preproc = std::make_unique(ctx_a); - break; - case PROJECTOR_TYPE_GEMMA4UA: - audio_preproc = std::make_unique(ctx_a); - break; - default: - GGML_ABORT("unsupported audio projector type"); - } - - // initialize audio preprocessor - audio_preproc->initialize(); -} - -clip_context_params init_clip_ctx_params(bool mmproj_cpu, bool dryrun) +mtmd_context_params init_mtmd_ctx_params(bool mmproj_cpu, bool dryrun) { #if defined(GGML_USE_METAL) if(file_format_meta.model_architecture == llm_arch::LLM_ARCH_QWEN2VL || file_format_meta.model_architecture == llm_arch::LLM_ARCH_GEMMA3) @@ -2311,36 +2273,33 @@ clip_context_params init_clip_ctx_params(bool mmproj_cpu, bool dryrun) mmproj_cpu = true; if(!dryrun) { - set_clip_uses_gpu(false); - printf("Clip will use CPU for this model!\n"); + printf("MTMD will use CPU for this model!\n"); } } #endif - clip_flash_attn_type clip_fa = (kcpp_data->flash_attn?CLIP_FLASH_ATTN_TYPE_ENABLED:CLIP_FLASH_ATTN_TYPE_DISABLED); //kcpp: disabled in 1.102.2 as some headsizes break on turing + llama_flash_attn_type mtmd_fa = (kcpp_data->flash_attn?LLAMA_FLASH_ATTN_TYPE_ENABLED:LLAMA_FLASH_ATTN_TYPE_DISABLED); #if defined(GGML_USE_CUDA) - clip_fa = CLIP_FLASH_ATTN_TYPE_DISABLED; //kcpp: disabled in 1.102.2 as some headsizes break on turing + mtmd_fa = LLAMA_FLASH_ATTN_TYPE_DISABLED; //kcpp: disabled in 1.102.2 as some headsizes break on turing #endif if(mmproj_cpu) { if(!dryrun) { - set_clip_uses_gpu(false); - printf("Clip forced to use CPU!\n"); + printf("MTMD forced to use CPU!\n"); } - clip_fa = (kcpp_data->flash_attn?CLIP_FLASH_ATTN_TYPE_ENABLED:CLIP_FLASH_ATTN_TYPE_DISABLED); //however if using CPU, fa is fine + mtmd_fa = (kcpp_data->flash_attn?LLAMA_FLASH_ATTN_TYPE_ENABLED:LLAMA_FLASH_ATTN_TYPE_DISABLED); //however if using CPU, fa is fine } - clip_context_params ctx_clip_params { - /* use_gpu */ true, - /* flash_attn_type */ clip_fa, - /* image_min_tokens */ kcpp_data->vision_min_tokens, - /* image_max_tokens */ kcpp_data->vision_max_tokens, - /* warmup */ false, - /* cb_eval */ nullptr, - /* cb_eval_user_data */ nullptr, - /* no_alloc */ dryrun?true:false, - }; + mtmd_context_params ctx_mtmd_params = mtmd_context_params_default(); + ctx_mtmd_params.use_gpu = !mmproj_cpu; + ctx_mtmd_params.print_timings = false; + ctx_mtmd_params.n_threads = kcpp_data->n_threads; + ctx_mtmd_params.media_marker = mtmd_default_marker(); + ctx_mtmd_params.flash_attn_type = mtmd_fa; + ctx_mtmd_params.warmup = false; + ctx_mtmd_params.image_min_tokens = kcpp_data->vision_min_tokens; + ctx_mtmd_params.image_max_tokens = kcpp_data->vision_max_tokens; - return ctx_clip_params; + return ctx_mtmd_params; } ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in_file_format, FileFormatExtraMeta in_file_format_meta) @@ -2381,6 +2340,11 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in debugmode = inputs.debugmode; draft_ctx = nullptr; guidance_ctx = nullptr; + if(mtmd_ctx) + { + mtmd_free(mtmd_ctx); + mtmd_ctx = nullptr; + } audio_multimodal_supported = false; vision_multimodal_supported = false; use_mrope = false; @@ -2412,7 +2376,6 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in = gpt2_ctx_v1.hparams.n_ctx = gpt2_ctx_v2.hparams.n_ctx = gpt2_ctx_v3.hparams.n_ctx = mpt_ctx_v3.hparams.n_ctx = kcpp_data->n_ctx; - vision_max_res = inputs.visionmaxres; //determine rope scaling params float rope_freq_scale = 1.0f; @@ -2787,30 +2750,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in if(mmproj_filename != "" && file_format==FileFormat::GGUF_GENERIC && !inputs.mmproj_cpu) { printf("\nEstimating MMProj GPU usage..."); - auto saved_log_callback = g_logger_state.log_callback; - auto saved_log_user_data = g_logger_state.log_callback_user_data; - g_logger_state.log_callback = log_callback_off; - g_logger_state.log_callback_user_data = nullptr; - clip_context_params ctx_clip_params = init_clip_ctx_params(inputs.mmproj_cpu,true); - clip_init_result cres = clip_init(mmproj_filename.c_str(), ctx_clip_params); - g_logger_state.log_callback = saved_log_callback; - g_logger_state.log_callback_user_data = saved_log_user_data; - clip_ctx * test1 = cres.ctx_v; - clip_ctx * test2 = cres.ctx_a; - - if(test1) - { - for (auto & [dev, size] : clip_get_mem_usage(test1)) { - totalmmprojtax += size; - } - clip_free(test1); - } - if(test2) - { - for (auto & [dev, size] : clip_get_mem_usage(test2)) { - totalmmprojtax += size; - } - clip_free(test2); + mtmd_context_params ctx_mtmd_params = init_mtmd_ctx_params(inputs.mmproj_cpu,true); + auto mtmd_mem = mtmd_get_memory_usage(mmproj_filename.c_str(), ctx_mtmd_params); + for (auto & [dev, size] : mtmd_mem) { + totalmmprojtax += size; } totalmmprojtax = totalmmprojtax / (1024*1024); printf("MMProj Autofit Usage: %zu MB", totalmmprojtax); @@ -3000,58 +2943,14 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in if(mmproj_filename != "" && file_format==FileFormat::GGUF_GENERIC) { printf("\nAttempting to apply Multimodal Projector: %s\n", mmproj_filename.c_str()); - clip_context_params ctx_clip_params = init_clip_ctx_params(inputs.mmproj_cpu,false); - clip_init_result cres = clip_init(mmproj_filename.c_str(), ctx_clip_params); - clp_ctx_v = cres.ctx_v; - clp_ctx_a = cres.ctx_a; - if(clp_ctx_v == nullptr && clp_ctx_a == nullptr) { + mtmd_context_params ctx_mtmd_params = init_mtmd_ctx_params(inputs.mmproj_cpu,false); + mtmd_ctx = mtmd_init_from_file(mmproj_filename.c_str(), llamamodel, ctx_mtmd_params); + if(mtmd_ctx == nullptr) { fprintf(stderr, "%s: error: failed to load mmproj model!\n", __func__); return ModelLoadResult::FAIL; } - const int n_embd_llm = llama_model_n_embd_inp(llamamodel); - int n_embd_clip_a = -1; - int n_embd_clip_v = -1; - if (clp_ctx_v) - { - n_embd_clip_v = clip_n_mmproj_embd(clp_ctx_v); - if (n_embd_clip_v != n_embd_llm) { - fprintf(stderr, "%s: mmproj vision embedding mismatch (%d and %d)! Make sure you use the correct mmproj file!\n", __func__,n_embd_clip_v, n_embd_llm); - return ModelLoadResult::FAIL; - } - } - if (clp_ctx_a) - { - n_embd_clip_a = clip_n_mmproj_embd(clp_ctx_a); - if (n_embd_clip_a != n_embd_llm) { - fprintf(stderr, "%s: mmproj audio embedding mismatch (%d and %d)! Make sure you use the correct mmproj file!\n", __func__,n_embd_clip_a, n_embd_llm); - return ModelLoadResult::FAIL; - } - } - if (clp_ctx_v && clp_ctx_a && n_embd_clip_v != n_embd_clip_a) { - fprintf(stderr, "%s: mmproj embedding mismatch between Audio and Vision (%d and %d)! Make sure you use the correct mmproj file!\n", __func__,n_embd_clip_v, n_embd_clip_a); - return ModelLoadResult::FAIL; - } - - if(clp_ctx_v) - { - vision_multimodal_supported = true; - int proj_type = clip_get_projector_type_ext(clp_ctx_v); - switch (proj_type) { - case PROJECTOR_TYPE_GEMMA3: - case PROJECTOR_TYPE_GEMMA4V: - case PROJECTOR_TYPE_GEMMA4UV: - vision_use_non_casual = true; - break; - default: - break; - } - } - clp_img_data = clip_image_u8_init(); - if(clp_ctx_a) //init audio - { - kcpp_init_audio_proj(clp_ctx_a); - audio_multimodal_supported = true; - } + vision_multimodal_supported = mtmd_support_vision(mtmd_ctx); + audio_multimodal_supported = mtmd_support_audio(mtmd_ctx); } const llama_vocab * tmpvocab = llama_model_get_vocab(llamamodel); @@ -3063,7 +2962,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in { printf("Error: Speculative decoding cannot be used with Recurrent models!\n"); } - else if(clp_ctx_v!=nullptr || clp_ctx_a!=nullptr) + else if(mtmd_ctx!=nullptr) { printf("Error: Speculative decoding cannot be used with multimodal projectors!\n"); } @@ -4489,141 +4388,71 @@ int GetThreadsToUse(bool blasmode) return kcpp_data->n_threads; } -//this function prepares the clip embds for llava. it's only needed when images change +//this function prepares the mtmd chunks for media. it's only needed when media changes static void PrepareMediaEmbds(const int nctx, const std::vector & media_intro, const std::vector & media_outro) { - bool vision_on = (clp_ctx_v != nullptr && clp_img_data != nullptr); - bool audio_on = (clp_ctx_a != nullptr); - if (vision_on || audio_on) + if (mtmd_ctx) { int introsize = media_intro.size(); int outrosize = media_outro.size(); last_media_mem.clear(); - bool clip_is_mrope = false; - if(file_format == FileFormat::GGUF_GENERIC) - { - //added after https://github.com/ggml-org/llama.cpp/pull/22161, replacing clip_is_mrope function - auto decoder_rope_type = llama_model_rope_type(llama_get_model(llama_ctx_v4)); - switch (decoder_rope_type) { - case LLAMA_ROPE_TYPE_NONE: - case LLAMA_ROPE_TYPE_NORM: - case LLAMA_ROPE_TYPE_NEOX: - { - clip_is_mrope = false; - } break; - case LLAMA_ROPE_TYPE_MROPE: - case LLAMA_ROPE_TYPE_IMROPE: - { - clip_is_mrope = true; - } - break; - default: - printf("\nWARNING: clip unsupported decoder rope type: %d\n", decoder_rope_type); - } - } - for(int i=0;i media_data_buffer = kcpp_base64_decode(media_obj); - if(!media_objects[i].is_audio && vision_on) + mtmd::bitmap bitmap(mtmd_helper_bitmap_init_from_buf(mtmd_ctx, media_data_buffer.data(), media_data_buffer.size())); + if(!bitmap.ptr) { - //images - if (!clip_image_load_from_bytes(media_data_buffer.data(), media_data_buffer.size(), clp_img_data, vision_max_res)) - { - //failed to load image - printf("\nError: Clip image %d failed to load!",i); - } - else - { - if(debugmode==1 && !is_quiet) - { - printf("\nCreating clip image embed..."); - } - media_chunk chunk; - chunk.is_audio = media_objects[i].is_audio; - if (!llava_image_embed_make_with_clip_img(clp_ctx_v, kcpp_data->n_threads, clp_img_data, &chunk.clp_img_embd, &chunk.clp_image_tokens, &chunk.nx, &chunk.ny, clip_is_mrope)) { - printf("\nError: Clip image %d failed to create embd!",i); - } - if(debugmode==1 && !is_quiet) - { - printf("\nVision Clip Embed %i used Tokens: %d",i,chunk.clp_image_tokens); - } - int cliptokensneeded = chunk.clp_image_tokens; - if(cliptokensneeded>0 && cliptokensneeded < nctx) - { - int tokcnt = (chunk.clp_image_tokens + media_objects[i].chunk_start_seq.size() + media_objects[i].chunk_end_seq.size()); - if(i==0) - { - tokcnt += introsize + outrosize; - } - for(int n=0;n pcmf32; - int samplerate = clip_get_hparams(clp_ctx_a)->audio_sample_rate; - bool ok = kcpp_decode_audio_from_buf(media_data_buffer.data(), media_data_buffer.size(), samplerate, pcmf32); - if (!ok) { - printf("\nError: Clip audio %d failed to convert!",i); - continue; - } - std::vector mel_spec_chunks; - ok = audio_preproc->preprocess(pcmf32.data(), pcmf32.size(), mel_spec_chunks); - if (!ok) { - printf("\nError: Clip audio %d failed to load!",i); - continue; - } + printf("\nError: MTMD media %d failed to load!",i); + continue; + } + mtmd_input_text inp_txt = { + mtmd_default_marker(), + /* add_special */ false, + /* parse_special */ true, + }; + mtmd::input_chunks chunks(mtmd_input_chunks_init()); + std::vector bitmaps = { bitmap.ptr.get() }; + int32_t tokenized = mtmd_tokenize(mtmd_ctx, chunks.ptr.get(), &inp_txt, bitmaps.data(), bitmaps.size()); + if(tokenized != 0) + { + media_composite_image_signature = ""; //force invalidate + printf("\nError: MTMD media %d failed to tokenize! (status %d)",i, tokenized); + continue; + } - // consider each mel_spec as a separate audio chunk - int total_chunk_tokens = 0; - for (auto & mel_spec : mel_spec_chunks) { - media_chunk chunk; - chunk.is_audio = media_objects[i].is_audio; - bool ok = audio_embd_make_with_clip_img(clp_ctx_a, kcpp_data->n_threads, mel_spec, &chunk.clp_img_embd, &chunk.clp_image_tokens); - if (!ok) { - printf("\nError: Clip audio chunk in %d failed to make embd!",i); - } else { - if(debugmode==1 && !is_quiet) - { - printf("\nAudio Clip %i Embed Chunk used Tokens: %d",i,chunk.clp_image_tokens); - } - total_chunk_tokens += chunk.clp_image_tokens; - media_objects[i].mediachunks.push_back(chunk); - } - } - int cliptokensneeded = total_chunk_tokens; - if(cliptokensneeded>0 && cliptokensneeded < nctx) + int mediatokensneeded = 0; + for(size_t j=0;j0 && mediatokensneeded < nctx) + { + int tokcnt = mediatokensneeded; + if(i==0) { - int tokcnt = (cliptokensneeded + media_objects[i].chunk_start_seq.size() + media_objects[i].chunk_end_seq.size()); - if(i==0) - { - tokcnt += introsize + outrosize; - } - for(int n=0;n media_intro; //added before media list std::vector media_outro; //added before media list std::string intro = "\nAttached Media:\n"; - if(clp_ctx_v) //ugly fix for gemma4uv vision coherency - { - int ptype = clip_get_projector_type_ext(clp_ctx_v); - if(ptype==PROJECTOR_TYPE_GEMMA4UV) - { - intro = "\n<|channel>" + intro; - } - } TokenizeString(intro, media_intro, file_format, true); - //clear previous run llava embd memory, just-in-time free + //clear previous run media memory, just-in-time free for(int i=0;i(media_objects[i].mediachunks[j].mtmd_chunk)); + media_objects[i].mediachunks[j].mtmd_chunk = nullptr; + } } media_objects[i].mediachunks.clear(); } @@ -4852,29 +4678,6 @@ generation_outputs gpttype_generate(const generation_inputs inputs) media_object lv; lv.b64data = item; lv.is_audio = false; - std::string img_start = ""; - std::string img_end = "\n\n"; - if(clp_ctx_v) - { - int ptype = clip_get_projector_type_ext(clp_ctx_v); - if(ptype == PROJECTOR_TYPE_QWEN2VL || ptype == PROJECTOR_TYPE_QWEN25VL || ptype == PROJECTOR_TYPE_QWEN3VL) //qwen - { - img_start = "<|vision_start|>"; - img_end = "<|vision_end|>\n\n"; - } - else if(ptype==PROJECTOR_TYPE_GLM4V) - { - img_start = "<|begin_of_image|>"; - img_end = "<|end_of_image|>\n\n"; - } - else if(ptype==PROJECTOR_TYPE_GEMMA4V || ptype==PROJECTOR_TYPE_GEMMA4UV) - { - img_start = "<|image>"; - img_end = "\n\n"; - } - } - TokenizeString(img_start, lv.chunk_start_seq, file_format, false); - TokenizeString(img_end, lv.chunk_end_seq, file_format, false); media_objects.push_back(lv); new_media_composite += item; } @@ -4887,35 +4690,6 @@ generation_outputs gpttype_generate(const generation_inputs inputs) media_object lv; lv.b64data = item; lv.is_audio = true; - std::string aud_start = "\n\n"; - if(clp_ctx_a) - { - int ptype = clip_get_projector_type_ext(clp_ctx_a); - if(ptype==PROJECTOR_TYPE_QWEN2A || ptype==PROJECTOR_TYPE_QWEN25O) //qwen omni - { - aud_start = "<|audio_bos|>"; - aud_end = "<|audio_eos|>\n"; - } - else if(ptype==PROJECTOR_TYPE_QWEN3A) - { - aud_start = "<|audio_start|>"; - aud_end = "<|audio_end|>"; - } - else if(ptype==PROJECTOR_TYPE_VOXTRAL) //voxtral - { - aud_start = "[INST][BEGIN_AUDIO]"; - aud_end = "[/INST]\n"; - } - else if(ptype==PROJECTOR_TYPE_GEMMA4A || ptype==PROJECTOR_TYPE_GEMMA4UA) - { - aud_start = "<|audio>"; - aud_end = "\n"; - } - } - - TokenizeString(aud_start, lv.chunk_start_seq, file_format, false); - TokenizeString(aud_end, lv.chunk_end_seq, file_format, false); media_objects.push_back(lv); new_media_composite += item; } @@ -5140,7 +4914,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) } } - if(last_media_mem.size()>0) //stick the llava mem before the added mem + if(last_media_mem.size()>0) //stick the media placeholders before the added mem { if(last_media_mem.size() + kcpp_data->n_predict + 4 > nctx) { @@ -5157,7 +4931,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) } } - //append llava dummy tokens + //append media dummy tokens embd_inp_mem.insert(embd_inp_mem.begin(), last_media_mem.begin(), last_media_mem.end()); if (bos.size() > 0 && embd_inp_mem.size() > 0) { @@ -6216,7 +5990,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) while ((int)embd_inp.size() > input_consumed) { int currtoken = embd_inp[input_consumed]; - if(currtoken==MEDIA_TOKEN_IDENTIFIER_A || currtoken==MEDIA_TOKEN_IDENTIFIER_B) //special llava token hit + if(currtoken==MEDIA_TOKEN_IDENTIFIER_A || currtoken==MEDIA_TOKEN_IDENTIFIER_B) //special media token hit { if(!media_embds_built) //this should never happen! however, handle it anyway { @@ -6233,8 +6007,8 @@ generation_outputs gpttype_generate(const generation_inputs inputs) else { //batch is empty, do image processing - int llavatokenscounted = 0; - int llavatokensevaled = 0; + int mediatokenscounted = 0; + int mediatokensevaled = 0; int introsize = media_intro.size(); int outrosize = media_outro.size(); while(input_consumed < embd_inp.size() && (embd_inp[input_consumed]==MEDIA_TOKEN_IDENTIFIER_A || embd_inp[input_consumed]==MEDIA_TOKEN_IDENTIFIER_B)) @@ -6246,7 +6020,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) last_n_tokens.push_back(currtoken); current_context_tokens.push_back(currtoken); ++input_consumed; - ++llavatokenscounted; + ++mediatokenscounted; } for(int i=0;in_batch,&n_past,is2d); - llavatokensevaled += chunk.clp_image_tokens; + mediatokensevaled += chunk.clp_image_tokens; if(!err) { media_composite_image_signature = ""; //force invalidate @@ -6323,7 +6097,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs) printf("\rProcessing Media End Separator (%d tokens)",end_size); } n_past += end_size; - llavatokensevaled += end_size; + mediatokensevaled += end_size; } } if(media_objects.size()>0 && outrosize>0) @@ -6340,12 +6114,12 @@ generation_outputs gpttype_generate(const generation_inputs inputs) printf("\rProcessing Media Outro (%d tokens)",outrosize); } n_past += outrosize; - llavatokensevaled += outrosize; + mediatokensevaled += outrosize; } - if(llavatokenscounted!=llavatokensevaled) + if(mediatokenscounted!=mediatokensevaled) { media_composite_image_signature = ""; //force invalidate - fprintf(stderr, "\nMedia tokens mismatch at %d! (%d vs %d tokens)\n",n_past,llavatokenscounted,llavatokensevaled); + fprintf(stderr, "\nMedia tokens mismatch at %d! (%d vs %d tokens)\n",n_past,mediatokenscounted,mediatokensevaled); output.text = nullptr; output.status = 0; output.prompt_tokens = output.completion_tokens = 0; diff --git a/otherarch/otherarch.h b/otherarch/otherarch.h index 1a903fde9..10771dca3 100644 --- a/otherarch/otherarch.h +++ b/otherarch/otherarch.h @@ -512,8 +512,9 @@ struct mpt_model { struct media_chunk { bool is_audio = false; //if true its audio, otherwise its vision - int32_t clp_image_tokens = 0; //holds number of tokens llava used in this chunk - float * clp_img_embd = nullptr; //this holds dynamic memory and must be freed each use! + void * mtmd_chunk = nullptr; // mtmd_input_chunk, owned by this chunk + int32_t clp_image_tokens = 0; //holds number of tokens used in this chunk + float * clp_img_embd = nullptr; //legacy embedding memory, must be freed each use int32_t nx = 0; //only used for 2d roped images int32_t ny = 0; };