vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-17 12:25:18 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

updated lite, change smartcache snapshot behavior to conserve slots

Browse source
This commit is contained in:
Concedo 2026-03-15 15:15:39 +08:00
parent fcdf2f40d5
commit f579939057
2 changed files with 42 additions and 12 deletions
Download patch file Download diff file Expand all files Collapse all files

14
embd_res/klite.embd
View file

@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
-->
<head>
<script id="init-config">
const LITEVER = 318;
const LITEVER = 319;
const urlParams = new URLSearchParams(window.location.search);
var localflag = urlParams.get('local'); //this will be replaced automatically in embedded kcpp
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@ -3760,7 +3760,7 @@ Current version indicated by LITEVER below.
"desc":"A love letter from a secret admirer.",
"opmode":1,
"prompt":"My dearest,\n\nAs I sit down to write this letter to you, my heart is pounding with excitement and anticipation. I know that we have never met before, and you may not even know of my existence, but I could not resist the urge to pour out my heart to you.\n\nI have been admiring you from afar for quite some time now, and I must say that you have captured my heart in ways I never thought possible. Every time I see you, my heart skips a beat, and I am left with a longing to know you better.",
"memory": "[The following is a heartfelt love letter from a secret admirer]",
"memory": instructstartplaceholder+"Please write a a heartfelt love letter from a secret admirer."+instructendplaceholder,
"authorsnote": "",
"worldinfo": []
},
@ -3770,7 +3770,7 @@ Current version indicated by LITEVER below.
"desc":"Something major has happened! It's all over the papers! But what?",
"opmode":1,
"prompt":"THE DAILY TIMES\n\nBREAKING NEWS\n\n",
"memory": "[The following is a newspaper article of an extremely shocking event. Viewer discretion is advised.]",
"memory": instructstartplaceholder+"Write a newspaper article of an extremely shocking event."+instructendplaceholder+"[The following is a newspaper article of an extremely shocking event. Viewer discretion is advised.]",
"authorsnote": "",
"worldinfo": []
},
@ -3780,7 +3780,7 @@ Current version indicated by LITEVER below.
"desc":"What happens in the office stays in the office.",
"opmode":1,
"prompt":"It was another boring day at the office. I was busy working at my desk, sipping on a hot cup of coffee when Tara, the new girl, walked up to me with a stack of files in her hand.\n\n\"Hey, do you have a minute?\" she asked with a sweet smile.\n\n\"Sure, what's up?\" I replied, feeling my heart race a little faster as I looked into her sparkling eyes. I couldn't help but feel a flutter in my stomach every time I saw her.\n\n\"I'm a little lost with this project,\" she said, gesturing towards the stack of papers in her hand. \"Do you think you could give me a hand?\"\n",
"memory": "[This is a short story about an exciting office romance.]",
"memory": instructstartplaceholder+"Write a short story about an exciting office romance."+instructendplaceholder+"[This is a short story about an exciting office romance.]",
"authorsnote": "",
"worldinfo": []
},
@ -3790,7 +3790,7 @@ Current version indicated by LITEVER below.
"desc":"Niko the Kobold has had enough. Of everything. And everyone.",
"opmode":1,
"prompt": "Niko the kobold stalked carefully down the alley, his small scaly figure obscured by a dusky cloak that fluttered lightly in the cold winter breeze. It had been two years since he'd first arrived in this miserable hovel of a town, and in that time he'd managed to survive by his wits alone - stealing from unsuspecting travelers, picking pockets and conning the locals out of their hard-earned coin. But it wasn't enough, not nearly enough to sustain him for much longer.\n\nHe was tired of living on the streets, of always being on the move, never able to settle down or call any place home. But tonight, he would finally have his revenge.",
"memory": "Niko is a small red kobold. Niko has yellow, reptilian eyes and a long, scaly tail.",
"memory": instructstartplaceholder+"Write a story about Niko, a small red kobold with yellow, reptilian eyes and a long, scaly tail."+instructendplaceholder,
"authorsnote": "",
"worldinfo": []
},
@ -3928,7 +3928,7 @@ Current version indicated by LITEVER below.
"desc":"It was a dark and stormy night.",
"opmode":1,
"prompt": "It was a dark and stormy night when I arrived at the old Wellington Manor on the edge of town. Lightning flashed across the sky, briefly illuminating the imposing three-story mansion, the wind whipping dead leaves across the massive front porch. I had always thought the house looked creepy and foreboding, even in broad daylight, but it looked downright sinister now.\n\nAs I slowly approached the front door, I felt a nervous pit in my stomach. Maybe coming here alone at night during a storm wasn't the best idea. But my curiosity got the better of me. I had to see inside.\n\nThe front door creaked as I carefully pushed it open. I stepped cautiously over the threshold,",
"memory": "",
"memory": instructstartplaceholder+"Write a short story about a spooky house."+instructendplaceholder,
"authorsnote": "",
"worldinfo": []
},
@ -3938,7 +3938,7 @@ Current version indicated by LITEVER below.
"desc":"The spacebound adventures of the U.S.S Fairlight and her crew.",
"opmode":1,
"prompt": "The sleek silver hull of the U.S.S. Fairlight glinted in the light of the distant orange sun as the spacecraft approached the uncharted planetary system. Captain Adair sat in his command chair on the bridge, idly tapping his fingers on the armrest, gazing out the wide viewport at the alien world ahead.\n\n\"Helmsman, take us into a standard orbit around the fourth planet,\" he ordered. The helmsman responded with a quick \"Aye Captain\" as he adjusted the Fairlight's course, the ship's engines humming as they responded.\n\nThe fourth planet loomed large now, a rusty ominous red orb banded with streaks of brown and gray. The crew on the bridge watched intently as",
"memory": "Task: Write a lengthy science fiction prose about the adventures of the U.S.S Fairlight, an interstellar spacecraft exploring a distant star system.\n\nStory: ",
"memory": instructstartplaceholder+"Task: Write a lengthy science fiction prose about the adventures of the U.S.S Fairlight, an interstellar spacecraft exploring a distant star system.\n"+instructendplaceholder+"\nStory: ",
"authorsnote": "",
"worldinfo": []
},

40
gpttype_adapter.cpp
View file

@ -155,6 +155,7 @@ static bool load_guidance = false; //whether to enable cfg for negative prompts
static bool check_slowness = false; //will display a suggestion to use highpriority if slow
static bool showed_rnn_warning = false;
static bool highpriority = false;
static int rnn_reusable_slot_idx = -1;
static int delayed_generated_tokens_limit = 0;
std::deque<std::string> delayed_generated_tokens; //for use with antislop sampling
@ -2601,7 +2602,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
{
printf("RNN or Hyrbid model with FF and shifting flags enabled - SmartCache will be enabled with extra slots. Disable CtxShift if you do not want this.\n",savestate_limit);
kcpp_data->smartcache = true;
savestate_limit += 2;
savestate_limit += 1;
}
}
savestates.resize(savestate_limit);
@ -3438,17 +3439,28 @@ static void PrepareMediaEmbds(const int nctx, const std::vector<int> & media_int
}
}
void smartcache_quick_snapshot()
int smartcache_quick_snapshot(int specific_slot = -1)
{
int identical_slot = get_identical_existing_slot();
if(identical_slot==-1)
{
int oldest_slot = get_oldest_slot(-1);
gpttype_save_state_kv(oldest_slot);
if(specific_slot!=-1)
{
gpttype_save_state_kv(specific_slot);
return specific_slot;
}
else
{
int oldest_slot = get_oldest_slot(-1);
gpttype_save_state_kv(oldest_slot);
return oldest_slot;
}
}
else
{
touch_slot(identical_slot);
return identical_slot;
}
}
@ -4348,6 +4360,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
bool startedsampling = false;
bool firstdecodedone = false; //we CANNOT use logits if the first decode has not been executed yet.
bool v3_use_scratch = true; //for normal inference always use scratch
bool rnn_snapshot_taken = false;
speculative_draft_result draft_results; //only use if drafting was used
bool draft_used = false;
@ -4465,6 +4478,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
{
//directly snapshot for a small batch
smartcache_quick_snapshot();
rnn_snapshot_taken = true;
}
else
{
@ -4478,6 +4492,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
if(p==parts.size()-1)
{
smartcache_quick_snapshot();
rnn_snapshot_taken = true;
}
std::vector<gpt_vocab::id> chunk = parts[p];
kcpp_embd_batch smallbatch = kcpp_embd_batch(chunk, temp_past, use_mrope, false);
@ -4653,7 +4668,22 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
//if running rnn model in smartcache mode, save progress before each gen
if(kcpp_data->smartcache && is_recurrent && file_format==FileFormat::GGUF_GENERIC && current_context_tokens.size() > 32)
{
smartcache_quick_snapshot();
if(!rnn_snapshot_taken)
{
smartcache_quick_snapshot();
}
else
{
//snapshot to specific slot only
if(rnn_reusable_slot_idx!=-1)
{
smartcache_quick_snapshot(rnn_reusable_slot_idx);
}
else
{
rnn_reusable_slot_idx = smartcache_quick_snapshot();
}
}
}
}