vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

tweak ratios

Browse source
This commit is contained in:
Concedo 2024-11-02 12:35:04 +08:00
parent 4ae06b4a64
commit 6ac8b2bdb3
1 changed files with 1 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
koboldcpp.py
View file

@ -717,7 +717,7 @@ def autoset_gpu_layers(ctxsize,sdquanted,bbs): #shitty algo to determine how man
layers = ggufmeta[0] layers = ggufmeta[0]
headcount = ggufmeta[1] headcount = ggufmeta[1]
headkvlen = (ggufmeta[2] if ggufmeta[2] > 0 else 128) headkvlen = (ggufmeta[2] if ggufmeta[2] > 0 else 128)
ratio = (mem-usedmem)/(fsize*csmul*1.55) ratio = (mem-usedmem)/(fsize*csmul*1.6*(1.0 if bbs <= 512 else 1.2))
computemem = layers*(4 if bbs <= 512 else (bbs/128))*headkvlen*cs*4*1.5 # apply blasbatchsize calculations if over 512 computemem = layers*(4 if bbs <= 512 else (bbs/128))*headkvlen*cs*4*1.5 # apply blasbatchsize calculations if over 512
contextmem = layers*headcount*headkvlen*cs*4*1.1 contextmem = layers*headcount*headkvlen*cs*4*1.1
if headcount > 0: if headcount > 0: