vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 09:04:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Dynamic sizes for sequences (#1157)

Browse source 
* Dynamic sizes for sequences

* cleanup PR - move all dynamic fields to end of payload, ensure correct null handling to match existing behavior, add anti abuse limit of max 512 for dynamic fields

* adjust anti abuse limits

---------

Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
This commit is contained in:
Maya 2024-10-16 18:55:11 +03:00 • committed by GitHub
parent 7f76425450
commit 8bb220329c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 74 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

8
gpttype_adapter.cpp
View file

@ -2488,7 +2488,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
double time0 = 0, time1 = 0, time2 = 0;
timer_start();
for(int x=0;x<stop_token_max;++x)
for(int x=0;x<inputs.stop_sequence_len;++x)
{
std::string stopper = inputs.stop_sequence[x];
if(stopper!="")
@ -2516,7 +2516,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
delayed_generated_tokens_limit = 0;
antislop_banned_token_ids.clear();
banned_tokens.clear();
for(int x=0;x<ban_token_max;++x)
for(int x=0;x<inputs.banned_tokens_len;++x)
{
std::string word = inputs.banned_tokens[x];
word = toLowerCase(word);
@ -2574,7 +2574,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
}
logit_biases.clear();
for(int x=0;x<logit_bias_max;++x)
for(int x=0;x<inputs.logit_biases_len;++x)
{
int32_t t_id = inputs.logit_biases[x].token_id;
float bias = inputs.logit_biases[x].bias;
@ -2652,7 +2652,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
if (kcpp_data->dry_multiplier > 0)
{
for (int x = 0; x < dry_seq_break_max; ++x)
for (int x = 0; x < inputs.dry_sequence_breakers_len; ++x)
{
std::string word = inputs.dry_sequence_breakers[x];
if (word != "")