vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 17:44:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

always prefilter to 5k logits

Browse source
This commit is contained in:
Concedo 2024-08-12 22:27:06 +08:00
parent 6e0ebc90ad
commit 689a17d756
1 changed files with 3 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

3
gpttype_adapter.cpp
View file

@ -839,6 +839,9 @@ int mirostat, float mirostat_tau, float mirostat_eta, float dry_multiplier, floa
sample_grammar(file_format, n_vocab, &candidates_p, grammar); sample_grammar(file_format, n_vocab, &candidates_p, grammar);
} }
//prefilter to top 5k tokens for improved speed
llama_sample_top_k(nullptr, &candidates_p, 5000, 1);
if (mirostat == 1 || mirostat == 2) if (mirostat == 1 || mirostat == 2)
{ {
static float mirostat_mu = 2.0f * mirostat_tau; static float mirostat_mu = 2.0f * mirostat_tau;