vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add more warmup (+1 squashed commits)

Browse source 
Squashed commits:

[9578d5352] updated lite
This commit is contained in:
Concedo 2025-04-26 09:47:57 +08:00
parent 4decd6bea1
commit cb1c182673
3 changed files with 14 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
koboldcpp.py
View file

@ -994,7 +994,7 @@ def autoset_gpu_layers(ctxsize, sdquanted, bbs, qkv_level): #shitty algo to dete
fattn_discount = 1.0/(3.2 if qkv_level==2 else (1.6 if qkv_level==1 else 1.0))
mem1 = layers*(4 if bbs <= 512 else (bbs/128))*headkvlen*cs*fattn_discount*4*1.45
mem2 = layers*headcount*headkvlen*cs*fattn_discount*4*1.15
ratio = (mem - reservedmem - mem1) / (fsize + mem2)
ratio = max(ratio,(mem - reservedmem - mem1) / (fsize + mem2))
layerlimit = min(int(ratio*layers), (layers + 3))
layerlimit = (0 if layerlimit<=2 else layerlimit)
return layerlimit
@ -4325,7 +4325,7 @@ def show_gui():
quick_gpuname_label = ctk.CTkLabel(quick_tab, text="")
quick_gpuname_label.grid(row=3, column=1, padx=75, sticky="W")
quick_gpuname_label.configure(text_color="#ffff00")
quick_gpu_layers_entry,quick_gpu_layers_label = makelabelentry(quick_tab,"GPU Layers:", gpulayers_var, 6, 50,tooltip="How many layers to offload onto the GPU.\nVRAM intensive, usage increases with model and context size.\nRequires some trial and error to find the best fit value.\n\nCommon values for total layers, accuracy not guaranteed.\n\nLlama/Mistral 7b/8b: 33\nSolar 10.7b/11b: 49\nLlama 13b: 41\nLlama 20b(stack): 63\nLlama/Yi 34b: 61\nMixtral 8x7b: 33\nLlama 70b: 81")
quick_gpu_layers_entry,quick_gpu_layers_label = makelabelentry(quick_tab,"GPU Layers:", gpulayers_var, 6, 50,tooltip="How many layers to offload onto the GPU.\nUsage varies based on model type and increases with model and context size.\nRequires some trial and error to find the best fit value.\n\nNote: The auto estimation is often inaccurate! Please set layers yourself for best results!")
quick_layercounter_label = ctk.CTkLabel(quick_tab, text="")
quick_layercounter_label.grid(row=6, column=1, padx=75, sticky="W")
quick_layercounter_label.configure(text_color="#ffff00")