vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix ubatch, autoselect vulkan dgpu if possible

Browse source
This commit is contained in:
Concedo 2024-06-22 00:23:46 +08:00
parent c9c050f323
commit 13398477a1
2 changed files with 15 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

1
gpttype_adapter.cpp
View file

@ -1053,6 +1053,7 @@ ModelLoadResult gpttype_load_model(const load_model_inputs inputs, FileFormat in
#endif #endif
llama_ctx_params.n_batch = kcpp_params->n_batch; llama_ctx_params.n_batch = kcpp_params->n_batch;
llama_ctx_params.n_ubatch = kcpp_params->n_ubatch;
llama_ctx_params.n_threads = kcpp_params->n_threads; llama_ctx_params.n_threads = kcpp_params->n_threads;
llama_ctx_params.n_threads_batch = kcpp_params->n_threads_batch; llama_ctx_params.n_threads_batch = kcpp_params->n_threads_batch;

14
koboldcpp.py
View file

@ -1767,6 +1767,7 @@ def show_new_gui():
CLDevicesNames = ["","","",""] CLDevicesNames = ["","","",""]
CUDevicesNames = ["","","","",""] CUDevicesNames = ["","","","",""]
VKDevicesNames = ["","","",""] VKDevicesNames = ["","","",""]
VKIsDGPU = [0,0,0,0]
MaxMemory = [0] MaxMemory = [0]
tabcontent = {} tabcontent = {}
@ -2005,11 +2006,18 @@ def show_new_gui():
try: # Get Vulkan names try: # Get Vulkan names
output = subprocess.run(['vulkaninfo','--summary'], capture_output=True, text=True, check=True, encoding='utf-8').stdout output = subprocess.run(['vulkaninfo','--summary'], capture_output=True, text=True, check=True, encoding='utf-8').stdout
devicelist = [line.split("=")[1].strip() for line in output.splitlines() if "deviceName" in line] devicelist = [line.split("=")[1].strip() for line in output.splitlines() if "deviceName" in line]
devicetypes = [line.split("=")[1].strip() for line in output.splitlines() if "deviceType" in line]
idx = 0 idx = 0
for dname in devicelist: for dname in devicelist:
if idx<len(VKDevicesNames): if idx<len(VKDevicesNames):
VKDevicesNames[idx] = dname VKDevicesNames[idx] = dname
idx += 1 idx += 1
if len(devicetypes) == len(devicelist):
idx = 0
for dvtype in devicetypes:
if idx<len(VKIsDGPU):
VKIsDGPU[idx] = (1 if dvtype=="PHYSICAL_DEVICE_TYPE_DISCRETE_GPU" else 0)
idx += 1
except Exception as e: except Exception as e:
pass pass
@ -2029,6 +2037,12 @@ def show_new_gui():
runopts_var.set("Use CuBLAS") runopts_var.set("Use CuBLAS")
elif "Use hipBLAS (ROCm)" in runopts: elif "Use hipBLAS (ROCm)" in runopts:
runopts_var.set("Use hipBLAS (ROCm)") runopts_var.set("Use hipBLAS (ROCm)")
elif exitcounter < 100 and (1 in VKIsDGPU) and runmode_untouched and "Use Vulkan" in runopts:
for i in range(0,len(VKIsDGPU)):
if VKIsDGPU[i]==1:
runopts_var.set("Use Vulkan")
gpu_choice_var.set(str(i+1))
break
changed_gpu_choice_var() changed_gpu_choice_var()
return return