vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 00:54:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

batch size improvements

Browse source
This commit is contained in:
Concedo 2023-12-22 15:27:40 +08:00
parent 230a638512
commit 77463e0e9c
3 changed files with 7 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

9
gpttype_adapter.cpp
View file

@ -82,6 +82,9 @@ static int n_batch = 8;
static bool useSmartContext = false;
static bool useContextShift = false;
static int blasbatchsize = 512;
static int dontblasbatchsize = 16;
static int normalbatchsize = 32;
static int smallbatchsize = 8;
static int debugmode = 0; //-1 = hide all, 0 = normal, 1 = showall
static std::string modelname;
static std::vector<gpt_vocab::id> last_n_tokens;
@ -671,7 +674,9 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
file_format = in_file_format;
n_threads = params.n_threads = inputs.threads;
n_blasthreads = params.n_threads_batch = inputs.blasthreads;
n_batch = params.n_batch = inputs.batch_size;
bool isGguf = (file_format == FileFormat::GGUF_LLAMA || file_format==FileFormat::GGUF_FALCON);
n_batch = params.n_batch = (isGguf?normalbatchsize:smallbatchsize);
modelname = params.model = inputs.model_filename;
useSmartContext = inputs.use_smartcontext;
useContextShift = inputs.use_contextshift;
@ -679,7 +684,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
blasbatchsize = inputs.blasbatchsize;
if(blasbatchsize<=0)
{
blasbatchsize = 8;
blasbatchsize = (isGguf?dontblasbatchsize:smallbatchsize);
}
auto clamped_max_context_length = inputs.max_context_length;