vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-07-10 01:18:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: OOB reads in UGM tokenizer (precompiled_charsmap handling) (#18750)

Browse source 
* fix: OOB reads in UGM tokenizer (precompiled_charsmap handling)

- Validate minimum size (4 bytes) before reading xcda_blob_size
- Use strnlen with bounds check instead of unsafe strlen

Both issues allow heap-buffer-overflow from malicious T5/UGM GGUF files.

* Replace unsafe strnlen() with a bounds-checked loop that scans for \0 within the remaining array size.

* move bounds checks to load

* typo merge fix

---------

Co-authored-by: hourhl <hourhl8200@gmail.com>
Co-authored-by: Sigbjørn Skjæret <1629204+CISC@users.noreply.github.com>
This commit is contained in:
hourhl 2026-07-08 13:02:09 +08:00 • committed by GitHub
parent 57b50e1f6b
commit 4a7ee3126d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 19 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

26
src/llama-vocab.cpp
View file

@ -887,9 +887,6 @@ struct llm_tokenizer_ugm : llm_tokenizer {
// blob containing XOR-compressed compact double array (XCDA) entries
uint32_t xcda_blob_size = *(const uint32_t *) &precompiled_charsmap[0];
charsmap_offset += sizeof(xcda_blob_size);
if (xcda_blob_size + charsmap_offset >= precompiled_charsmap.size()) {
throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
}
// Next xcda_blob_size bytes contain entries of XOR-compressed compact
// double array (XCDA). Each entry is bit-packed into a 32-bit integer.
@ -1205,7 +1202,15 @@ private:
throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
}
const char * prefix_replacement = &(tokenizer.prefix_replacements)[longest_prefix_offset];
return { prefix_replacement, strlen(prefix_replacement), longest_prefix_length };
size_t max_len = tokenizer.prefix_replacements_size - longest_prefix_offset;
size_t repl_len = 0;
while (repl_len < max_len && prefix_replacement[repl_len] != '\0') {
repl_len++;
}
if (repl_len == max_len) {
throw std::runtime_error("Unterminated string in precompiled charsmap!");
}
return { prefix_replacement, repl_len, longest_prefix_length };
}
// check if the input prefix contains a valid sequence of UTF-8 code units
@ -2018,11 +2023,18 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
const size_t n_precompiled_charsmap = gguf_get_arr_n(ctx, precompiled_charsmap_keyidx);
const char * pc = (const char *) gguf_get_arr_data(ctx, precompiled_charsmap_keyidx);
precompiled_charsmap.assign(pc, pc + n_precompiled_charsmap);
if (precompiled_charsmap.size() < sizeof(uint32_t)) {
throw std::runtime_error("precompiled_charsmap too small for xcda_blob_size header!");
}
uint32_t * xcda_blob_size = (uint32_t *) &precompiled_charsmap[0];
#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
*xcda_blob_size = __builtin_bswap32(*xcda_blob_size);
#endif
if (*xcda_blob_size + sizeof(uint32_t) >= precompiled_charsmap.size()) {
throw std::runtime_error("Index out of array bounds in precompiled charsmap!");
}
#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
// correct endianness of data in precompiled_charsmap binary blob
uint32_t * xcda_blob_size = (uint32_t *) &precompiled_charsmap[0];
*xcda_blob_size = __builtin_bswap32(*xcda_blob_size);
assert(*xcda_blob_size + sizeof(uint32_t) < n_precompiled_charsmap);
size_t xcda_array_size = *xcda_blob_size / sizeof(uint32_t);
uint32_t * xcda_array = (uint32_t *) &precompiled_charsmap[sizeof(uint32_t)];
for (size_t i = 0; i < xcda_array_size; ++i) {