vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-09 08:34:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

server : add openai-style logit_bias support (#14946)

Browse source 
Signed-off-by: Lukas Straub <lukasstraub2@web.de>
This commit is contained in:
Lukas Straub 2025-07-31 14:08:23 +02:00 • committed by GitHub
parent 8a4a856277
commit a9f77a8be3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 90 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

27
tools/server/server.cpp
View file

@ -473,6 +473,33 @@ struct server_task {
}
}
}
} else if (logit_bias != data.end() && logit_bias->is_object()) {
const int n_vocab = llama_vocab_n_tokens(vocab);
for (const auto & el : logit_bias->items()) {
float bias;
const auto & key = el.key();
const auto & value = el.value();
if (value.is_number()) {
bias = value.get<float>();
} else if (value.is_boolean() && !value.get<bool>()) {
bias = -INFINITY;
} else {
continue;
}
char *end;
llama_token tok = strtol(key.c_str(), &end, 10);
if (*end == 0) {
if (tok >= 0 && tok < n_vocab) {
params.sampling.logit_bias.push_back({tok, bias});
}
} else {
auto toks = common_tokenize(vocab, key, false);
for (auto tok : toks) {
params.sampling.logit_bias.push_back({tok, bias});
}
}
}
}
params.sampling.ignore_eos = json_value(data, "ignore_eos", params_base.sampling.ignore_eos);