vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix for physical batch size

Browse source
This commit is contained in:
Concedo 2024-03-15 16:45:20 +08:00
parent 93d3871056
commit 2ef03c9de6
1 changed files with 4 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

4
gpttype_adapter.cpp
View file

@ -748,6 +748,10 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
kcpp_params->n_threads_batch = inputs.blasthreads; kcpp_params->n_threads_batch = inputs.blasthreads;
bool isGguf = (file_format == FileFormat::GGUF_GENERIC); bool isGguf = (file_format == FileFormat::GGUF_GENERIC);
kcpp_params->n_batch = GetBatchSize(inputs.blasbatchsize, in_file_format); kcpp_params->n_batch = GetBatchSize(inputs.blasbatchsize, in_file_format);
if(kcpp_params->n_batch>512)
{
kcpp_params->n_ubatch = (kcpp_params->n_batch>1024?1024:kcpp_params->n_batch);
}
modelname = kcpp_params->model = inputs.model_filename; modelname = kcpp_params->model = inputs.model_filename;
useSmartContext = inputs.use_smartcontext; useSmartContext = inputs.use_smartcontext;
useContextShift = inputs.use_contextshift; useContextShift = inputs.use_contextshift;