vrr/prima.cpp
2
0
Fork 0
mirror of https://github.com/Lizonghang/prima.cpp.git synced 2025-09-17 04:59:39 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

server: fix bugs when running speculative decoding

Browse source
This commit is contained in:
Li, Zonghang 2025-07-13 21:52:59 +08:00
parent b019a707b8
commit 86ca21e49c
2 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
examples/server/server.cpp
View file

@ -2542,7 +2542,7 @@ struct server_context {
llama_batch_add(slot.batch_spec, draft[i], slot.n_past + 1 + i, { slot.id + 1 }, true);
}
llama_decode(ctx, slot.batch_spec);
llama_decode(ctx, slot.batch_spec, true);
// the accepted tokens from the speculation
const auto ids = gpt_sampler_sample_and_accept_n(slot.smpl, ctx, draft);