vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

try to optimize context shifting

Browse source
This commit is contained in:
Concedo 2024-08-26 23:07:31 +08:00
parent c61fa9155d
commit b78a637da5
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
gpttype_adapter.cpp
View file

@ -1036,8 +1036,8 @@ void PurgeMissingTokens(llama_context * ctx, std::vector<int> &current_context_t
//if passed, save beginning of LCQ from old ctx as p1
//remove all tokens from old ctx between p0 and p1, updating both arrays and kv, then continue as normal
const int ShortfallThreshold = 200 + (nctx/30); //dont trigger shifting if the distance between trimstart and currhead < this
const int SlackAllowance = 60 + (nctx/50); //in case the end text is slightly modified, be forgiving
const int ShortfallThreshold = 200 + std::min((nctx/30),140); //dont trigger shifting if the distance between trimstart and currhead < this
const int SlackAllowance = 60 + std::min((nctx/60),70); //in case the end text is slightly modified, be forgiving
int trimstart = 0;
int new_tokens_len = new_context_tokens.size();