vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-08-11 09:25:51 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

server : respect min-step when splitting prompt batches (#25420)

Browse source
This commit is contained in:
Aldehir Rojas 2026-07-09 01:23:30 -05:00 • committed by GitHub
parent f2d1c2f398
commit 64c8b7db72
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 8 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

11
tools/server/server-context.cpp
View file

@ -3435,9 +3435,14 @@ private:
slot.n_prompt_tokens_processed++;
// stop the prompt batch exactly before a user message
if (spans.is_user_start(slot.prompt.n_tokens())) {
break;
// break at the last user message, or at user messages at least min step past the last checkpoint
if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
const auto pos = slot.prompt.n_tokens();
const auto & checkpoints = slot.prompt.checkpoints;
if (pos == last_user_pos || checkpoints.empty() || pos > checkpoints.back().n_tokens + params_base.checkpoint_min_step) {
break;
}
}
// process the last few tokens of the prompt separately in order to allow for a checkpoint to be created.