vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 09:04:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add logit_bias to the OpenAI api (#577)

Browse source 
* Add logit_bias to the OpenAI api

* Cleanup and refactor, test in swagger.

---------

Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
This commit is contained in:
DebuggingLife46 2023-12-26 21:56:19 +05:30 • committed by GitHub
parent 5006b23099
commit e733a9e425
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 70 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

18
gpttype_adapter.cpp
View file

@ -101,6 +101,7 @@ static int stopper_unused_tokens = 0;
static std::mutex concat_output_mtx;
static std::string concat_output = "";
static std::string concat_output_reader_copy = "";
static std::vector<logit_bias> logit_biases;
const int extra_context_handle_fragmentation = 80;
@ -489,6 +490,12 @@ int mirostat, float mirostat_tau, float mirostat_eta, const std::vector<samplers
candidates.emplace_back(llama_token_data{token_id, logits[token_id], 0.0f});
}
for(int i=0;i<logit_biases.size();++i)
{
auto & itm = logit_biases[i];
candidates[itm.token_id].logit += itm.bias;
}
llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
if (grammar != nullptr) {
@ -1437,6 +1444,17 @@ generation_outputs gpttype_generate(const generation_inputs inputs, generation_o
}
}
logit_biases.clear();
for(int x=0;x<logit_bias_max;++x)
{
int32_t t_id = inputs.logit_biases[x].token_id;
float bias = inputs.logit_biases[x].bias;
if(t_id >= 0 && t_id < n_vocab && bias!=0)
{
logit_biases.push_back(inputs.logit_biases[x]);
}
}
std::string addedmemory = inputs.memory;
params.prompt = inputs.prompt;
params.seed = inputs.seed;