vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-05-11 21:32:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

speculative-simple : add checkpoint support (#22227)

Browse source 
* speculative-simple : add checkpoint support

* cont : fix build
This commit is contained in:
Georgi Gerganov 2026-04-22 15:44:45 +03:00 • committed by GitHub
parent 225088ea76
commit bcb5eeb645
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 112 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

10
tools/server/server-context.cpp
View file

@ -2961,7 +2961,13 @@ private:
// verify and try to accept the draft
{
common_sampler_ptr smpl_save(common_sampler_clone(slot.smpl.get()));
const bool use_ckpt = slot.ctx_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL;
// only save the sampler sampler state if we use checkpoints
common_sampler_ptr smpl_save;
if (use_ckpt) {
smpl_save.reset(common_sampler_clone(slot.smpl.get()));
}
GGML_ASSERT(slot.spec_i_batch.size() == n_draft + 1);
auto accepted = common_sampler_sample_and_accept_n(slot.smpl.get(), slot.ctx, slot.spec_i_batch, slot.spec_draft);
@ -2973,7 +2979,7 @@ private:
// check for partial draft acceptance
if (accepted.size() < slot.spec_draft.size() + 1) {
if (slot.ctx_seq_rm_type == COMMON_CONTEXT_SEQ_RM_TYPE_FULL) {
if (use_ckpt) {
// partial acceptance is not supported by the context -> truncate the draft and restore the state
slot.spec_draft = std::move(accepted);