vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

batch size improvements

Browse source
This commit is contained in:
Concedo 2023-12-22 15:27:40 +08:00
parent 230a638512
commit 77463e0e9c
3 changed files with 7 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
koboldcpp.py
View file

@ -23,7 +23,6 @@ class load_model_inputs(ctypes.Structure):
_fields_ = [("threads", ctypes.c_int),
("blasthreads", ctypes.c_int),
("max_context_length", ctypes.c_int),
("batch_size", ctypes.c_int),
("low_vram", ctypes.c_bool),
("use_mmq", ctypes.c_bool),
("executable_path", ctypes.c_char_p),
@ -229,7 +228,6 @@ def load_model(model_filename):
global args
inputs = load_model_inputs()
inputs.model_filename = model_filename.encode("UTF-8")
inputs.batch_size = 8
inputs.max_context_length = maxctx #initial value to use for ctx, can be overwritten
inputs.threads = args.threads
inputs.low_vram = (True if (args.usecublas and "lowvram" in args.usecublas) else False)