vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

improve model estimation

Browse source
This commit is contained in:
Concedo 2025-03-16 16:14:13 +08:00
parent 5d7c5e9e33
commit 0954e9e476
1 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

4
koboldcpp.py
View file

@ -884,9 +884,9 @@ def autoset_gpu_layers(ctxsize,sdquanted,bbs): #shitty algo to determine how man
elif modelfile_extracted_meta[2] > 1024*1024*512: #normal sd tax
mem -= 1024*1024*1024*(3.25 if sdquanted else 4.25)
if modelfile_extracted_meta[3] > 1024*1024*10: #whisper tax
mem -= 350*1024*1024
mem -= max(350*1024*1024,modelfile_extracted_meta[3]*1.5)
if modelfile_extracted_meta[4] > 1024*1024*10: #mmproj tax
mem -= 350*1024*1024
mem -= max(350*1024*1024,modelfile_extracted_meta[4]*1.5)
if modelfile_extracted_meta[5] > 1024*1024*10: #draft model tax
mem -= (modelfile_extracted_meta[5] * 1.5)
if modelfile_extracted_meta[6] > 1024*1024*10: #tts model tax