vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 09:04:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix-loss-of-destroyed-tokens-in-grammar-pre-pass (#1600)

Browse source
This commit is contained in:
Reithan 2025-06-13 03:46:38 -07:00 • committed by GitHub
parent 5bac0fb3d5
commit f1c9db4174
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
gpttype_adapter.cpp
View file

@ -1653,7 +1653,7 @@ const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dyna
//prefilter to top 3k tokens for improved speed
bool use_grammar = grammar != nullptr;
size_t n_pre_cull = candidates_p.size;
std::vector<llama_token_data> precache = (use_grammar ? std::vector<llama_token_data>(candidates) : std::vector<llama_token_data>(0));
sample_top_k(&candidates_p, 3000);
@ -1661,7 +1661,7 @@ const std::vector<samplers> & sampler_order, llama_grammar * grammar, float dyna
sample_grammar(file_format, n_vocab, &candidates_p, grammar);
// if top_k 3000 doesn't contain a valid candidate for this grammar, try again pre-cull
if (candidates_p.size <= 0) {
candidates_p.size = n_pre_cull;
candidates_p = { precache.data(), precache.size(), false };
sample_grammar(file_format, n_vocab, &candidates_p, grammar);
sample_top_k(&candidates_p, 3000);
}