vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into concedo

Browse source 
# Conflicts:
#	.github/workflows/build.yml
#	CMakeLists.txt
#	Makefile
#	README.md
This commit is contained in:
Concedo 2023-03-26 14:52:08 +08:00
commit 57474944d6
34 changed files with 970 additions and 1116 deletions
Download patch file Download diff file Expand all files Collapse all files

4
llama_for_kobold.py
View file

@ -44,9 +44,9 @@ def load_model(model_filename,batch_size=8,max_context_length=512,n_parts_overwr
inputs.model_filename = model_filename.encode("UTF-8")
inputs.batch_size = batch_size
inputs.max_context_length = max_context_length #initial value to use for ctx, can be overwritten
inputs.threads = 4 #seems to outperform os.cpu_count(), it's memory bottlenecked
inputs.threads = min(6,os.cpu_count()) #seems to outperform os.cpu_count(), it's memory bottlenecked
inputs.n_parts_overwrite = n_parts_overwrite
inputs.f16_kv = False
inputs.f16_kv = True
ret = handle.load_model(inputs)
return ret