vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-11 09:25:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

vocab : validate default special token ids (#26506)

Browse source
This commit is contained in:
Sigbjørn Skjæret 2026-08-03 17:40:53 +02:00 • committed by GitHub
parent dbadb68eec
commit e8e06f78e2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 6 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/llama-vocab.cpp
View file

@ -2532,6 +2532,12 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) {
const std::string & key = kv(std::get<0>(it));
int32_t & id = std::get<1>(it);
if (id >= 0 && static_cast<size_t>(id) >= id_to_token.size()) {
LLAMA_LOG_WARN("%s: default special token '%s' = %d out of vocab range, disabling\n",
__func__, key.c_str(), id);
id = LLAMA_TOKEN_NULL;
}
uint32_t new_id;
if (!ml.get_key(std::get<0>(it), new_id, false)) {
continue;