From 39be55c97e70bb17bd7e8547c512a39e3d584598 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 17 Aug 2026 18:21:02 +0200 Subject: [PATCH 1/6] vendor: move hash to vendor (#27262) * vendor: move hash to vendor * group hashes into one single static lib --- CMakeLists.txt | 1 + examples/gguf-hash/CMakeLists.txt | 18 +------ .../gguf-hash/deps/rotate-bits/package.json | 13 ----- examples/gguf-hash/deps/sha1/package.json | 9 ---- examples/gguf-hash/deps/sha256/package.json | 15 ------ examples/gguf-hash/deps/xxhash/clib.json | 12 ----- scripts/sync_vendor.py | 51 +++++++++++++++++++ vendor/hash/CMakeLists.txt | 21 ++++++++ vendor/hash/rotate-bits/LICENSE.md | 21 ++++++++ .../hash}/rotate-bits/rotate-bits.h | 0 .../deps => vendor/hash}/sha1/sha1.c | 0 .../deps => vendor/hash}/sha1/sha1.h | 0 vendor/hash/sha256/LICENSE | 1 + .../deps => vendor/hash}/sha256/sha256.c | 0 .../deps => vendor/hash}/sha256/sha256.h | 0 vendor/hash/xxhash/LICENSE | 26 ++++++++++ .../deps => vendor/hash}/xxhash/xxhash.c | 0 .../deps => vendor/hash}/xxhash/xxhash.h | 0 18 files changed, 122 insertions(+), 66 deletions(-) delete mode 100644 examples/gguf-hash/deps/rotate-bits/package.json delete mode 100644 examples/gguf-hash/deps/sha1/package.json delete mode 100644 examples/gguf-hash/deps/sha256/package.json delete mode 100644 examples/gguf-hash/deps/xxhash/clib.json create mode 100644 vendor/hash/CMakeLists.txt create mode 100644 vendor/hash/rotate-bits/LICENSE.md rename {examples/gguf-hash/deps => vendor/hash}/rotate-bits/rotate-bits.h (100%) rename {examples/gguf-hash/deps => vendor/hash}/sha1/sha1.c (100%) rename {examples/gguf-hash/deps => vendor/hash}/sha1/sha1.h (100%) create mode 100644 vendor/hash/sha256/LICENSE rename {examples/gguf-hash/deps => vendor/hash}/sha256/sha256.c (100%) rename {examples/gguf-hash/deps => vendor/hash}/sha256/sha256.h (100%) create mode 100644 vendor/hash/xxhash/LICENSE rename {examples/gguf-hash/deps => vendor/hash}/xxhash/xxhash.c (100%) rename {examples/gguf-hash/deps => vendor/hash}/xxhash/xxhash.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index e0ebb3ccb..4e1fd45dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -227,6 +227,7 @@ add_subdirectory(src) if (LLAMA_BUILD_COMMON) add_subdirectory(common) add_subdirectory(vendor/cpp-httplib) + add_subdirectory(vendor/hash) endif() if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION) diff --git a/examples/gguf-hash/CMakeLists.txt b/examples/gguf-hash/CMakeLists.txt index 15c5c68c6..2542074fb 100644 --- a/examples/gguf-hash/CMakeLists.txt +++ b/examples/gguf-hash/CMakeLists.txt @@ -2,21 +2,5 @@ set(TARGET llama-gguf-hash) add_executable(${TARGET} gguf-hash.cpp) install(TARGETS ${TARGET} RUNTIME) -# clibs dependencies -include_directories(deps/) - -add_library(xxhash OBJECT deps/xxhash/xxhash.c deps/xxhash/xxhash.h) -target_link_libraries(${TARGET} PRIVATE xxhash) - -add_library(sha1 OBJECT deps/sha1/sha1.c deps/sha1/sha1.h) -target_link_libraries(${TARGET} PRIVATE sha1) -if (NOT MSVC) - # disable warnings in 3rd party code - target_compile_options(sha1 PRIVATE -w) -endif() - -add_library(sha256 OBJECT deps/sha256/sha256.c deps/sha256/sha256.h) -target_link_libraries(${TARGET} PRIVATE sha256) - -target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) +target_link_libraries(${TARGET} PRIVATE vendor-hash ggml ${CMAKE_THREAD_LIBS_INIT}) target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/examples/gguf-hash/deps/rotate-bits/package.json b/examples/gguf-hash/deps/rotate-bits/package.json deleted file mode 100644 index 74c0bef68..000000000 --- a/examples/gguf-hash/deps/rotate-bits/package.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name": "rotate-bits", - "version": "0.1.1", - "repo": "jb55/rotate-bits.h", - "description": "rotate bits", - "keywords": ["rotl", "rotr"], - "src": ["rotate-bits.h"], - "license": "Public Domain", - "development": { - "thlorenz/tap.c": "*" - } -} - diff --git a/examples/gguf-hash/deps/sha1/package.json b/examples/gguf-hash/deps/sha1/package.json deleted file mode 100644 index 6a5843dd1..000000000 --- a/examples/gguf-hash/deps/sha1/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "sha1", - "version": "0.0.1", - "repo": "clibs/sha1", - "description": "sha1 hash algorithm", - "keywords": ["sha1", "hash"], - "license": "public domain", - "src": ["sha1.c", "sha1.h"] -} diff --git a/examples/gguf-hash/deps/sha256/package.json b/examples/gguf-hash/deps/sha256/package.json deleted file mode 100644 index b92a04127..000000000 --- a/examples/gguf-hash/deps/sha256/package.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "name": "sha256", - "version": "0.0.2", - "repo": "jb55/sha256.c", - "description": "sha256 in c", - "keywords": ["sha256", "sha2"], - "src": ["sha256.c", "sha256.h"], - "dependencies": { - "jb55/rotate-bits.h": "0.1.1" - }, - "development": { - "thlorenz/tap.c": "*" - } -} - diff --git a/examples/gguf-hash/deps/xxhash/clib.json b/examples/gguf-hash/deps/xxhash/clib.json deleted file mode 100644 index 242343c5d..000000000 --- a/examples/gguf-hash/deps/xxhash/clib.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "xxhash", - "version": "0.8.2", - "repo": "Cyan4973/xxhash", - "description": "Extremely fast non-cryptographic hash algorithm", - "keywords": ["xxhash", "hashing"], - "license": "BSD-2-Clause", - "src": [ - "xxhash.c", - "xxhash.h" - ] -} diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 4ecf50f90..3ab62f34c 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -7,6 +7,12 @@ import subprocess HTTPLIB_VERSION = "refs/tags/v0.53.1" +# used by examples/gguf-hash, these repos have no release tag, so we pin a commit +XXHASH_COMMIT = "9f465f1ea932d6ad9a26cd77496311ffa544cd68" +SHA1_COMMIT = "e1e2536fcf6a8f9703be8c85d58724b408552287" +SHA256_COMMIT = "5e637272c13f200872d55ff579f7e2ab6c3f252f" +ROTATE_BITS_COMMIT = "27e784942f67db44abf2115c6638e735b579acd1" + vendor = { "https://github.com/nlohmann/json/releases/latest/download/json.hpp": "vendor/nlohmann/json.hpp", "https://github.com/nlohmann/json/releases/latest/download/json_fwd.hpp": "vendor/nlohmann/json_fwd.hpp", @@ -22,12 +28,57 @@ vendor = { f"https://raw.githubusercontent.com/yhirose/cpp-httplib/{HTTPLIB_VERSION}/LICENSE": "vendor/cpp-httplib/LICENSE", "https://raw.githubusercontent.com/sheredom/subprocess.h/9ce0d701b6fb10f8f8c4445edd31e7c60a1237e3/subprocess.h": "vendor/sheredom/subprocess.h", + + f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.c": "vendor/hash/xxhash/xxhash.c", + f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/xxhash.h": "vendor/hash/xxhash/xxhash.h", + f"https://raw.githubusercontent.com/Cyan4973/xxHash/{XXHASH_COMMIT}/LICENSE": "vendor/hash/xxhash/LICENSE", + + # clibs/sha1 ships no license file, the source header says public domain + f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.c": "vendor/hash/sha1/sha1.c", + f"https://raw.githubusercontent.com/clibs/sha1/{SHA1_COMMIT}/sha1.h": "vendor/hash/sha1/sha1.h", + + f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.c": "vendor/hash/sha256/sha256.c", + f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/sha256.h": "vendor/hash/sha256/sha256.h", + f"https://raw.githubusercontent.com/jb55/sha256.c/{SHA256_COMMIT}/LICENSE": "vendor/hash/sha256/LICENSE", + + f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/rotate-bits.h": "vendor/hash/rotate-bits/rotate-bits.h", + f"https://raw.githubusercontent.com/jb55/rotate-bits.h/{ROTATE_BITS_COMMIT}/LICENSE.md": "vendor/hash/rotate-bits/LICENSE.md", +} + +# local changes kept on top of the upstream sources +patches = { + "vendor/hash/xxhash/xxhash.h": [( + '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */\n', + '/* Windows SDK under 10.0.22000 is missing stdalign.h so we add a check\n' + ' before allowing the windows compiler to use the C11 form.\n' + ' Reference: https://github.com/Cyan4973/xxHash/issues/955 */\n' + '#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) \\\n' + ' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n' + )], + + # silence a maybe-uninitialized warning + "vendor/hash/sha256/sha256.c": [( + " uint32_t W[16];\n", + " uint32_t W[16] = {0};\n" + )], } for url, filename in vendor.items(): print(f"downloading {url} to {filename}") # noqa: NP100 urllib.request.urlretrieve(url, filename) +for filename, replacements in patches.items(): + print(f"patching {filename}") # noqa: NP100 + with open(filename, "r", encoding="utf-8", newline="") as f: + content = f.read() + for old, new in replacements: + if content.count(old) != 1: + print(f"Error: cannot apply patch on {filename}, upstream code has changed") # noqa: NP100 + sys.exit(1) + content = content.replace(old, new) + with open(filename, "w", encoding="utf-8", newline="") as f: + f.write(content) + print("Splitting httplib.h...") # noqa: NP100 try: subprocess.check_call([ diff --git a/vendor/hash/CMakeLists.txt b/vendor/hash/CMakeLists.txt new file mode 100644 index 000000000..122a6419f --- /dev/null +++ b/vendor/hash/CMakeLists.txt @@ -0,0 +1,21 @@ +set(TARGET vendor-hash) + +llama_add_compile_flags() + +set(CMAKE_POSITION_INDEPENDENT_CODE ON) + +add_library(${TARGET} STATIC + xxhash/xxhash.c + sha1/sha1.c + sha256/sha256.c +) + +# disable warnings in 3rd party code +if (CMAKE_C_COMPILER_ID STREQUAL "MSVC") + target_compile_options(${TARGET} PRIVATE /w) +else() + target_compile_options(${TARGET} PRIVATE -w) +endif() + +# sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too +target_include_directories(${TARGET} PUBLIC .) diff --git a/vendor/hash/rotate-bits/LICENSE.md b/vendor/hash/rotate-bits/LICENSE.md new file mode 100644 index 000000000..4a07c57f2 --- /dev/null +++ b/vendor/hash/rotate-bits/LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 William Casarin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/examples/gguf-hash/deps/rotate-bits/rotate-bits.h b/vendor/hash/rotate-bits/rotate-bits.h similarity index 100% rename from examples/gguf-hash/deps/rotate-bits/rotate-bits.h rename to vendor/hash/rotate-bits/rotate-bits.h diff --git a/examples/gguf-hash/deps/sha1/sha1.c b/vendor/hash/sha1/sha1.c similarity index 100% rename from examples/gguf-hash/deps/sha1/sha1.c rename to vendor/hash/sha1/sha1.c diff --git a/examples/gguf-hash/deps/sha1/sha1.h b/vendor/hash/sha1/sha1.h similarity index 100% rename from examples/gguf-hash/deps/sha1/sha1.h rename to vendor/hash/sha1/sha1.h diff --git a/vendor/hash/sha256/LICENSE b/vendor/hash/sha256/LICENSE new file mode 100644 index 000000000..950b19483 --- /dev/null +++ b/vendor/hash/sha256/LICENSE @@ -0,0 +1 @@ +2010-06-11 : Igor Pavlov : Public domain diff --git a/examples/gguf-hash/deps/sha256/sha256.c b/vendor/hash/sha256/sha256.c similarity index 100% rename from examples/gguf-hash/deps/sha256/sha256.c rename to vendor/hash/sha256/sha256.c diff --git a/examples/gguf-hash/deps/sha256/sha256.h b/vendor/hash/sha256/sha256.h similarity index 100% rename from examples/gguf-hash/deps/sha256/sha256.h rename to vendor/hash/sha256/sha256.h diff --git a/vendor/hash/xxhash/LICENSE b/vendor/hash/xxhash/LICENSE new file mode 100644 index 000000000..e4c5da723 --- /dev/null +++ b/vendor/hash/xxhash/LICENSE @@ -0,0 +1,26 @@ +xxHash Library +Copyright (c) 2012-2021 Yann Collet +All rights reserved. + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/examples/gguf-hash/deps/xxhash/xxhash.c b/vendor/hash/xxhash/xxhash.c similarity index 100% rename from examples/gguf-hash/deps/xxhash/xxhash.c rename to vendor/hash/xxhash/xxhash.c diff --git a/examples/gguf-hash/deps/xxhash/xxhash.h b/vendor/hash/xxhash/xxhash.h similarity index 100% rename from examples/gguf-hash/deps/xxhash/xxhash.h rename to vendor/hash/xxhash/xxhash.h From 60eeeb6082c1126bb8bc72902c83123cd056811b Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Mon, 17 Aug 2026 11:35:36 -0500 Subject: [PATCH 2/6] cuda : skip UMA override for HIP builds (#27083) AMD APUs report accurate memory via hipMemGetInfo. Using MemAvailable over-promises on small-carveout systems. fixes #18159 --- ggml/src/ggml-cuda/ggml-cuda.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 598f3228c..f2e381ee0 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -4770,7 +4770,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * } // ref: https://github.com/ggml-org/llama.cpp/pull/17368 -#if defined(__linux__) +#if defined(__linux__) && !defined(GGML_USE_HIP) // Check if this is a UMA (Unified Memory Architecture) system cudaDeviceProp prop; CUDA_CHECK(cudaGetDeviceProperties(&prop, ggml_cuda_get_physical_device(ctx->device))); @@ -4790,7 +4790,7 @@ static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * GGML_LOG_ERROR("%s: /proc/meminfo reading failed, using cudaMemGetInfo\n", __func__); } } -#endif // defined(__linux__) +#endif // defined(__linux__) && !defined(GGML_USE_HIP) // virtual devices sharing one physical GPU share its memory pool; split it between them const int share_count = ggml_cuda_physical_device_share_count(ctx->device); From b75ecd1971bf2d3f29d5d334520868a01942cbc6 Mon Sep 17 00:00:00 2001 From: Tarek Dakhran Date: Mon, 17 Aug 2026 19:41:32 +0200 Subject: [PATCH 3/6] mtmd : skip thumbnail for non-tiled LFM2 images (#27246) * mtmd : skip thumbnail for non-tiled LFM2 images Matches HF behavior. * Update tools/mtmd/mtmd-image.cpp --------- Co-authored-by: Xuan-Son Nguyen --- tools/mtmd/mtmd-image.cpp | 14 ++++++++++++++ tools/mtmd/mtmd-image.h | 1 + 2 files changed, 15 insertions(+) diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index d4b5ec0d9..769b6efe6 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -999,6 +999,20 @@ mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_mi // mtmd_image_preprocessor_lfm2 // +mtmd_image_preproc_out mtmd_image_preprocessor_lfm2::preprocess(const clip_image_u8 & img) { + auto const inst = get_slice_instructions(img.get_size()); + if (!inst.slices.empty()) { + return mtmd_image_preprocessor_llava_uhd::preprocess(img); + } + + // single tile: no thumbnail + // note: not using output.overview here because it will emit <|img_thumbnail|> token, which we don't want in this case + auto sliced = slice_image(img, inst); + mtmd_image_preproc_out output; + output.append(hparams, sliced.overview, true); + return output; +} + mtmd_image_preprocessor_llava_uhd::slice_instructions mtmd_image_preprocessor_lfm2::get_slice_instructions(const clip_image_size & original_size) { mtmd_image_preprocessor_llava_uhd::slice_instructions inst; const int align_size = hparams.patch_size * hparams.n_merge; diff --git a/tools/mtmd/mtmd-image.h b/tools/mtmd/mtmd-image.h index 238332934..40dfea7eb 100644 --- a/tools/mtmd/mtmd-image.h +++ b/tools/mtmd/mtmd-image.h @@ -145,6 +145,7 @@ struct mtmd_image_preprocessor_lfm2 : mtmd_image_preprocessor_llava_uhd { static constexpr int tile_size = 512; using mtmd_image_preprocessor_llava_uhd::mtmd_image_preprocessor_llava_uhd; + mtmd_image_preproc_out preprocess(const clip_image_u8 & img) override; slice_instructions get_slice_instructions(const clip_image_size & original_size) override; private: From d8df12ebc4592b55dcecf97a32953623d031efdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sigbj=C3=B8rn=20Skj=C3=A6ret?= Date: Mon, 17 Aug 2026 20:13:15 +0200 Subject: [PATCH 4/6] vocab : support integer tokenizer scores (#27260) --- src/llama-vocab.cpp | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp index c1b447f3f..ff926ceec 100644 --- a/src/llama-vocab.cpp +++ b/src/llama-vocab.cpp @@ -2428,17 +2428,24 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { const uint32_t n_tokens = gguf_get_arr_n(ctx, token_idx); const float * scores = nullptr; + const int * iscores = nullptr; const int score_idx = gguf_find_key(ctx, kv(LLM_KV_TOKENIZER_SCORES).c_str()); if (score_idx != -1) { - if (gguf_get_kv_type(ctx, score_idx) != GGUF_TYPE_ARRAY || - gguf_get_arr_type(ctx, score_idx) != GGUF_TYPE_FLOAT32) { + const gguf_type kv_type = gguf_get_kv_type(ctx, score_idx); + const gguf_type arr_type = kv_type == GGUF_TYPE_ARRAY ? gguf_get_arr_type(ctx, score_idx) : GGUF_TYPE_COUNT; + if (arr_type != GGUF_TYPE_INT32 && + arr_type != GGUF_TYPE_FLOAT32) { throw std::runtime_error(format("invalid gguf type for %s", kv(LLM_KV_TOKENIZER_SCORES).c_str())); } const uint32_t n_scores = gguf_get_arr_n(ctx, score_idx); if (n_scores < n_tokens) { throw std::runtime_error("Index out of array bounds for scores (" + std::to_string(n_scores) + " < " + std::to_string(n_tokens) + ")\n"); } - scores = (const float * ) gguf_get_arr_data(ctx, score_idx); + if (arr_type == GGUF_TYPE_INT32) { + iscores = (const int *) gguf_get_arr_data(ctx, score_idx); + } else { + scores = (const float * ) gguf_get_arr_data(ctx, score_idx); + } } const int * toktypes = nullptr; @@ -2469,7 +2476,13 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { auto & token_data = id_to_token[i]; token_data.text = std::move(word); - token_data.score = scores ? scores[i] : 0.0f; + if (scores) { + token_data.score = scores[i]; + } else if (iscores) { + token_data.score = static_cast(iscores[i]); + } else { + token_data.score = 0.0f; + } token_data.attr = LLAMA_TOKEN_ATTR_NORMAL; if (toktypes) { //TODO: remove, required until per token attributes are available from GGUF file From ed1c3a20f5e3f0892ac7135d43542815bd7c4aba Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 17 Aug 2026 21:20:12 +0200 Subject: [PATCH 5/6] mtmd: use sha256 for input hashing (#27274) * mtmd: use sha256 for input hashing * void conflict with boringssl --- CMakeLists.txt | 4 +++- examples/gguf-hash/gguf-hash.cpp | 5 ++++- scripts/sync_vendor.py | 38 ++++++++++++++++++++++++++++++++ tools/mtmd/CMakeLists.txt | 2 +- tools/mtmd/mtmd-helper.cpp | 17 ++++---------- tools/mtmd/mtmd-helper.h | 2 +- tools/ui/embed.cpp | 2 ++ vendor/hash/CMakeLists.txt | 20 +++++++++++++---- vendor/hash/hash.cpp | 23 +++++++++++++++++++ vendor/hash/hash.h | 9 ++++++++ vendor/hash/sha1/sha1.c | 4 ++++ vendor/hash/sha1/sha1.h | 8 ++----- 12 files changed, 107 insertions(+), 27 deletions(-) create mode 100644 vendor/hash/hash.cpp create mode 100644 vendor/hash/hash.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e1fd45dd..19f14e0d1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,10 +224,12 @@ add_subdirectory(src) # utils, programs, examples and tests # +# mtmd needs this even when common is not built +add_subdirectory(vendor/hash) + if (LLAMA_BUILD_COMMON) add_subdirectory(common) add_subdirectory(vendor/cpp-httplib) - add_subdirectory(vendor/hash) endif() if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION) diff --git a/examples/gguf-hash/gguf-hash.cpp b/examples/gguf-hash/gguf-hash.cpp index 331de301f..43de6300d 100644 --- a/examples/gguf-hash/gguf-hash.cpp +++ b/examples/gguf-hash/gguf-hash.cpp @@ -18,13 +18,16 @@ extern "C" { #endif #include "xxhash/xxhash.h" -#include "sha1/sha1.h" #include "sha256/sha256.h" #ifdef __cplusplus } #endif +// sha1 is compiled as C++ and lives in a namespace, see scripts/sync_vendor.py +#include "sha1/sha1.h" +using namespace vendor_hash; + // uuid.uuid5(uuid.NAMESPACE_URL, 'en.wikipedia.org/wiki/Llama.cpp') #define UUID_NAMESPACE_LLAMA_CPP "ef001206-dadc-5f6d-a15f-3359e577d4e5" diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 3ab62f34c..18a94e1c6 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -56,6 +56,44 @@ patches = { ' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n' )], + # sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time. + # we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace. + "vendor/hash/sha1/sha1.h": [ + ( + '#if defined(__cplusplus)\n' + 'extern "C" {\n' + '#endif\n', + + 'namespace vendor_hash {\n' + ), + ( + '#if defined(__cplusplus)\n' + '}\n' + '#endif\n', + + '} // namespace vendor_hash\n' + ), + ], + + "vendor/hash/sha1/sha1.c": [ + ( + '#include "sha1.h"\n', + + '#include "sha1.h"\n' + '\n' + 'namespace vendor_hash {\n' + ), + ( + ' SHA1Final((unsigned char *)hash_out, &ctx);\n' + '}\n', + + ' SHA1Final((unsigned char *)hash_out, &ctx);\n' + '}\n' + '\n' + '} // namespace vendor_hash\n' + ), + ], + # silence a maybe-uninitialized warning "vendor/hash/sha256/sha256.c": [( " uint32_t W[16];\n", diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 769a44e0b..db758395f 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -78,7 +78,7 @@ set_target_properties(mtmd PROPERTIES ) target_link_libraries (mtmd PUBLIC ggml llama) -target_link_libraries (mtmd PRIVATE Threads::Threads) +target_link_libraries (mtmd PRIVATE Threads::Threads vendor-hash) target_include_directories(mtmd PUBLIC .) target_include_directories(mtmd PRIVATE ../..) target_include_directories(mtmd PRIVATE ../../vendor) diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index d77c93966..bce8e38cc 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -12,6 +12,8 @@ #include "mtmd-helper-common.h" #include "llama.h" +#include "hash.h" + #include #include #include @@ -356,25 +358,14 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int } // namespace audio_helpers -// Computes FNV-1a hash of the data -static std::string fnv_hash(const uint8_t * data, size_t len) { - const uint64_t fnv_prime = 0x100000001b3ULL; - uint64_t hash = 0xcbf29ce484222325ULL; - - for (size_t i = 0; i < len; ++i) { - hash ^= data[i]; - hash *= fnv_prime; - } - return std::to_string(hash); -} - mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) { // calculate the hash if needed std::string id; mtmd_bitmap * result = nullptr; if (!placeholder) { - id = fnv_hash(buf, len); + // use sha256 to prevent cache poisoning + id = hash_sha256_hex(buf, len); } if (audio_helpers::is_audio_file((const char *)buf, len)) { diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 832f7171a..5c6b92419 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -49,7 +49,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm // note: // - for now, video input is only supported via C++ helper functions // - audio files will be auto-detected based on magic bytes -// - output bitmap will have FNV hash as the ID +// - output bitmap will have SHA-256 hash (hex string) as the ID // returns nullptr on failure // this function is thread-safe MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); diff --git a/tools/ui/embed.cpp b/tools/ui/embed.cpp index 914d51fa1..b76c9047f 100644 --- a/tools/ui/embed.cpp +++ b/tools/ui/embed.cpp @@ -259,6 +259,8 @@ int main(int argc, char ** argv) { } cpp += fmt("static const unsigned char asset_%d_data[] = {", i); append_bytes_hex(cpp, bytes); + + // note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough here const auto hash = fnv_hash(bytes.data(), bytes.size()); cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n", diff --git a/vendor/hash/CMakeLists.txt b/vendor/hash/CMakeLists.txt index 122a6419f..efdf58e63 100644 --- a/vendor/hash/CMakeLists.txt +++ b/vendor/hash/CMakeLists.txt @@ -4,18 +4,30 @@ llama_add_compile_flags() set(CMAKE_POSITION_INDEPENDENT_CODE ON) -add_library(${TARGET} STATIC +set(VENDOR_SRCS xxhash/xxhash.c sha1/sha1.c sha256/sha256.c ) -# disable warnings in 3rd party code +add_library(${TARGET} STATIC + hash.cpp + hash.h + ${VENDOR_SRCS} +) + +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +# disable warnings in 3rd party code, but keep them for hash.cpp if (CMAKE_C_COMPILER_ID STREQUAL "MSVC") - target_compile_options(${TARGET} PRIVATE /w) + set(NO_WARN_FLAG /w) else() - target_compile_options(${TARGET} PRIVATE -w) + set(NO_WARN_FLAG -w) endif() +set_source_files_properties(${VENDOR_SRCS} PROPERTIES COMPILE_OPTIONS ${NO_WARN_FLAG}) + +# sha1 lives in a namespace to avoid a clash with boringssl, see scripts/sync_vendor.py +set_source_files_properties(sha1/sha1.c PROPERTIES LANGUAGE CXX) # sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too target_include_directories(${TARGET} PUBLIC .) diff --git a/vendor/hash/hash.cpp b/vendor/hash/hash.cpp new file mode 100644 index 000000000..6493716bf --- /dev/null +++ b/vendor/hash/hash.cpp @@ -0,0 +1,23 @@ +#include "hash.h" + +extern "C" { +#include "sha256/sha256.h" +} + +static std::string to_hex(const unsigned char * digest, size_t len) { + static const char hex[] = "0123456789abcdef"; + + std::string out; + out.reserve(2*len); + for (size_t i = 0; i < len; ++i) { + out += hex[digest[i] >> 4]; + out += hex[digest[i] & 0xf]; + } + return out; +} + +std::string hash_sha256_hex(const void * data, size_t len) { + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_hash(digest, (const unsigned char *) data, len); + return to_hex(digest, SHA256_DIGEST_SIZE); +} diff --git a/vendor/hash/hash.h b/vendor/hash/hash.h new file mode 100644 index 000000000..1298b4196 --- /dev/null +++ b/vendor/hash/hash.h @@ -0,0 +1,9 @@ +#pragma once + +// C++ wrapper for the vendored hash functions + +#include +#include + +// returns the SHA-256 digest as a lowercase hex string +std::string hash_sha256_hex(const void * data, size_t len); diff --git a/vendor/hash/sha1/sha1.c b/vendor/hash/sha1/sha1.c index 76cd6ca33..4d84340d4 100644 --- a/vendor/hash/sha1/sha1.c +++ b/vendor/hash/sha1/sha1.c @@ -25,6 +25,8 @@ A million repetitions of "a" #include "sha1.h" +namespace vendor_hash { + #define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) @@ -293,3 +295,5 @@ void SHA1( SHA1Final((unsigned char *)hash_out, &ctx); } +} // namespace vendor_hash + diff --git a/vendor/hash/sha1/sha1.h b/vendor/hash/sha1/sha1.h index f492009c9..4ec5df0f4 100644 --- a/vendor/hash/sha1/sha1.h +++ b/vendor/hash/sha1/sha1.h @@ -9,9 +9,7 @@ #include "stdint.h" -#if defined(__cplusplus) -extern "C" { -#endif +namespace vendor_hash { typedef struct { @@ -45,8 +43,6 @@ void SHA1( const char *str, uint32_t len); -#if defined(__cplusplus) -} -#endif +} // namespace vendor_hash #endif /* SHA1_H */ From 533b18257b7da879a5de39f5d6437041e0e42c39 Mon Sep 17 00:00:00 2001 From: Xuan-Son Nguyen Date: Mon, 17 Aug 2026 21:29:50 +0200 Subject: [PATCH 6/6] server: save processed mtmd chunks as placeholder (#27278) --- tools/mtmd/mtmd.cpp | 46 +++++++++++++++++++++++---------- tools/mtmd/mtmd.h | 3 +++ tools/server/server-common.cpp | 17 ++++++++++++ tools/server/server-common.h | 4 +++ tools/server/server-context.cpp | 3 ++- 5 files changed, 58 insertions(+), 15 deletions(-) diff --git a/tools/mtmd/mtmd.cpp b/tools/mtmd/mtmd.cpp index 6596b2785..4063d28e0 100644 --- a/tools/mtmd/mtmd.cpp +++ b/tools/mtmd/mtmd.cpp @@ -2322,23 +2322,12 @@ void mtmd_input_chunk_free(mtmd_input_chunk * chunk) { } } -int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len) { +// returns 0 on success +static int32_t mtmd_input_chunk_save_impl(const mtmd_input_chunk * chunk, std::vector & out_buf) { try { mtmd_serialization ser(MTMD_SERIALIZATION_VERSION); chunk->serialize(ser); - - if (expected_out_len) { - *expected_out_len = ser.data.size(); - } - if (!out_buf) { - // caller is only querying the required size - return 0; - } - if (out_len < ser.data.size()) { - LOG_ERR("%s: out_buf is too small, need %zu bytes, got %zu\n", __func__, ser.data.size(), out_len); - return -1; - } - std::memcpy(out_buf, ser.data.data(), ser.data.size()); + out_buf = std::move(ser.data); return 0; } catch (const std::exception & e) { LOG_ERR("%s: %s\n", __func__, e.what()); @@ -2346,6 +2335,35 @@ int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, si } } +mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk) { + // this is hacky, but still faster than copy the whole batch data + std::vector buf; + if (mtmd_input_chunk_save_impl(chunk, buf) != 0) { + return nullptr; + } + return mtmd_input_chunk_load(buf.data(), buf.size()); +} + +int32_t mtmd_input_chunk_save(const mtmd_input_chunk * chunk, char * out_buf, size_t out_len, size_t * expected_out_len) { + std::vector buf; + if (mtmd_input_chunk_save_impl(chunk, buf) != 0) { + return -1; + } + if (expected_out_len) { + *expected_out_len = buf.size(); + } + if (!out_buf) { + // caller is only querying the required size + return 0; + } + if (out_len < buf.size()) { + LOG_ERR("%s: out_buf is too small, need %zu bytes, got %zu\n", __func__, buf.size(), out_len); + return -1; + } + std::memcpy(out_buf, buf.data(), buf.size()); + return 0; +} + mtmd_input_chunk * mtmd_input_chunk_load(const char * buf, size_t len) { try { mtmd_serialization ser(MTMD_SERIALIZATION_VERSION, buf, len); diff --git a/tools/mtmd/mtmd.h b/tools/mtmd/mtmd.h index c1a5921db..78587f3fe 100644 --- a/tools/mtmd/mtmd.h +++ b/tools/mtmd/mtmd.h @@ -233,6 +233,9 @@ MTMD_API llama_pos mtmd_input_chunk_get_n_pos (const mtmd MTMD_API mtmd_input_chunk * mtmd_input_chunk_copy(const mtmd_input_chunk * chunk); MTMD_API void mtmd_input_chunk_free(mtmd_input_chunk * chunk); +// similar to mtmd_input_chunk_copy, but returns a placeholder chunk +MTMD_API mtmd_input_chunk * mtmd_input_chunk_get_placeholder(const mtmd_input_chunk * chunk); + // save/load an input chunk to/from a buffer (useful for KV save/load) // important: only chunk's metadata will be saved, the actual image/audio data will not be saved // the loaded chunk will always be a placeholder, cannot be used for mtmd_encode() or mtmd_batch_encode() diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 7ed486528..585f65e83 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -507,6 +507,23 @@ void server_tokens::push_back(const mtmd_input_chunk * chunk) { } } +void server_tokens::push_back_placeholder(const mtmd_input_chunk * chunk) { + auto type = mtmd_input_chunk_get_type(chunk); + if (type == MTMD_INPUT_CHUNK_TYPE_IMAGE || type == MTMD_INPUT_CHUNK_TYPE_AUDIO) { + GGML_ASSERT(has_mtmd); + mtmd::input_chunk_ptr new_chunk(mtmd_input_chunk_get_placeholder(chunk)); + GGML_ASSERT(new_chunk != nullptr && "failed to create placeholder chunk"); + const size_t n_tokens = mtmd_input_chunk_get_n_tokens(chunk); + size_t start_idx = tokens.size(); + for (size_t i = 0; i < n_tokens; ++i) { + tokens.emplace_back(LLAMA_TOKEN_NULL); + } + map_idx_to_media[start_idx] = std::move(new_chunk); + } else { + push_back(chunk); + } +} + void server_tokens::push_back(server_tokens & tokens) { size_t start_idx = size(); for (size_t i = 0; i < tokens.size(); i++) { diff --git a/tools/server/server-common.h b/tools/server/server-common.h index 7082abdd9..6488be344 100644 --- a/tools/server/server-common.h +++ b/tools/server/server-common.h @@ -195,6 +195,10 @@ public: // will create a copy of the chunk if it contains non-text data void push_back(const mtmd_input_chunk * chunk); + // same as push_back, but media chunks are stored as placeholders (no image/audio data) + // only use this if the chunk will never be encoded again (e.g. it is already in the KV cache) + void push_back_placeholder(const mtmd_input_chunk * chunk); + // appends server tokens, updates the media map. copies media chunks. void push_back(server_tokens & tokens); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index fdc8f2b80..842e4203c 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3416,7 +3416,8 @@ private: // add the mtmd chunk to cache { const auto & chunk = input_tokens.find_chunk(cur_token_idx); - slot.prompt.tokens.push_back(chunk.get()); // copy + // the chunk is already in the KV cache at this point, so we don't need to keep its data around + slot.prompt.tokens.push_back_placeholder(chunk.get()); } has_mtmd = true;