From 52dcfe42d63c7b575019e2dbb5b5db2d33ac27ba Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Sun, 22 Jun 2025 18:16:02 +0800 Subject: [PATCH] try auto selecting correct backend while checking intrinsics --- .../kcpp-build-release-win-oldpc.yaml | 2 +- .github/workflows/kcpp-build-release-win.yaml | 2 +- environment-nocuda.yaml | 1 + environment.yaml | 1 + koboldcpp.py | 81 ++++++++++++++++--- make_pyinstaller_cuda_oldpc.bat | 2 +- requirements.txt | 3 +- 7 files changed, 75 insertions(+), 17 deletions(-) diff --git a/.github/workflows/kcpp-build-release-win-oldpc.yaml b/.github/workflows/kcpp-build-release-win-oldpc.yaml index b29acfbd1..e6e59e76b 100644 --- a/.github/workflows/kcpp-build-release-win-oldpc.yaml +++ b/.github/workflows/kcpp-build-release-win-oldpc.yaml @@ -33,7 +33,7 @@ jobs: - name: Install python dependencies run: | python -m pip install --upgrade pip - pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5 + pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5 py-cpuinfo==9.0.0 - name: Display full Visual Studio info Before run: | diff --git a/.github/workflows/kcpp-build-release-win.yaml b/.github/workflows/kcpp-build-release-win.yaml index 54fa15b20..bac82ed53 100644 --- a/.github/workflows/kcpp-build-release-win.yaml +++ b/.github/workflows/kcpp-build-release-win.yaml @@ -33,7 +33,7 @@ jobs: - name: Install python dependencies run: | python -m pip install --upgrade pip - pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5 + pip install customtkinter==5.2.0 pyinstaller==5.11.0 psutil==5.9.5 py-cpuinfo==9.0.0 - name: Display full Visual Studio info Before run: | diff --git a/environment-nocuda.yaml b/environment-nocuda.yaml index 2449e4bbc..b95744513 100644 --- a/environment-nocuda.yaml +++ b/environment-nocuda.yaml @@ -19,3 +19,4 @@ dependencies: - tk=*=xft_* - pip: - customtkinter + - py-cpuinfo diff --git a/environment.yaml b/environment.yaml index c68fc3c96..1171c261a 100644 --- a/environment.yaml +++ b/environment.yaml @@ -22,3 +22,4 @@ dependencies: - tk=*=xft_* - pip: - customtkinter + - py-cpuinfo diff --git a/koboldcpp.py b/koboldcpp.py index 2b8954e7a..11f8a4536 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -471,12 +471,12 @@ def init_library(): libname = lib_default - if args.noavx2: + if args.noavx2: #failsafe implies noavx2 always if args.useclblast and (os.name!='nt' or file_exists("clblast.dll")): - if (args.failsafe) and file_exists(lib_clblast_failsafe): - libname = lib_clblast_failsafe - elif file_exists(lib_clblast_noavx2): + if file_exists(lib_clblast_noavx2) and not (args.failsafe): libname = lib_clblast_noavx2 + elif file_exists(lib_clblast_failsafe): + libname = lib_clblast_failsafe elif (args.usevulkan is not None) and file_exists(lib_vulkan_noavx2): libname = lib_vulkan_noavx2 elif (args.failsafe) and file_exists(lib_failsafe): @@ -489,10 +489,20 @@ def init_library(): libname = lib_cublas elif file_exists(lib_hipblas): libname = lib_hipblas - elif (args.usevulkan is not None) and file_exists(lib_vulkan): - libname = lib_vulkan - elif args.useclblast and file_exists(lib_clblast) and (os.name!='nt' or file_exists("clblast.dll")): - libname = lib_clblast + elif (args.usevulkan is not None): + if file_exists(lib_vulkan): + libname = lib_vulkan + elif file_exists(lib_vulkan_noavx2): + libname = lib_vulkan_noavx2 + elif args.useclblast (os.name!='nt' or file_exists("clblast.dll")): + if file_exists(lib_clblast): + libname = lib_clblast + elif file_exists(lib_clblast_noavx2): + libname = lib_clblast_noavx2 + elif file_exists(lib_clblast_failsafe): + libname = lib_clblast_failsafe + elif libname == lib_default and not file_exists(lib_default) and file_exists(lib_noavx2): + libname = lib_noavx2 print("Initializing dynamic library: " + libname) dir_path = getdirpath() @@ -668,6 +678,25 @@ def strip_base64_prefix(encoded_data): encoded_data = encoded_data.split(',', 1)[-1] return encoded_data +def old_cpu_check(): #return 0 if has avx2, 1 if has avx, 2 if has nothing + shouldcheck = ((sys.platform == "linux" and platform.machine().lower() in ("x86_64", "amd64")) or + (os.name == 'nt' and platform.machine().lower() in ("amd64", "x86_64"))) + if not shouldcheck: + return 0 #doesnt deal with avx at all. + try: + import cpuinfo + info = cpuinfo.get_cpu_info() + flags = info.get('flags', []) + if 'avx2' in flags: + return 0 + elif 'avx' in flags: + return 1 + else: + return 2 + except Exception: + return 0 #cannot determine + + def unpack_to_dir(destpath = ""): srcpath = os.path.abspath(os.path.dirname(__file__)) cliunpack = False if destpath == "" else True @@ -1253,12 +1282,24 @@ def fetch_gpu_properties(testCL,testCU,testVK): def auto_set_backend_cli(): fetch_gpu_properties(False,True,True) found_new_backend = False - if exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and any(CUDevicesNames): + + # check for avx2 and avx support + is_oldpc_ver = "Use CPU" not in runopts #on oldcpu ver, default lib does not exist + cpusupport = old_cpu_check() # 0 if has avx2, 1 if has avx, 2 if has nothing + eligible_cuda = (cpusupport==0 and not is_oldpc_ver) or (cpusupport==1 and is_oldpc_ver) + if not eligible_cuda: + if cpusupport==1: + args.noavx2 = True + elif cpusupport==2: + args.noavx2 = True + args.failsafe = True + + if eligible_cuda and exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and any(CUDevicesNames): if "Use CuBLAS" in runopts or "Use hipBLAS (ROCm)" in runopts: args.usecublas = ["normal","mmq"] print("Auto Selected CUDA Backend...\n") found_new_backend = True - elif exitcounter < 100 and (1 in VKIsDGPU) and "Use Vulkan" in runopts: + elif exitcounter < 100 and (1 in VKIsDGPU) and ("Use Vulkan" in runopts or "Use Vulkan (Old CPU)" in runopts): for i in range(0,len(VKIsDGPU)): if VKIsDGPU[i]==1: args.usevulkan = [] @@ -4623,9 +4664,15 @@ def show_gui(): else: fetch_gpu_properties(True,True,True) found_new_backend = False + + # check for avx2 and avx support + is_oldpc_ver = "Use CPU" not in runopts #on oldcpu ver, default lib does not exist + cpusupport = old_cpu_check() # 0 if has avx2, 1 if has avx, 2 if has nothing + eligible_cuda = (cpusupport==0 and not is_oldpc_ver) or (cpusupport==1 and is_oldpc_ver) + #autopick cublas if suitable, requires at least 3.5GB VRAM to auto pick #we do not want to autoselect hip/cublas if the user has already changed their desired backend! - if exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and (any(CUDevicesNames) or any(CLDevicesNames)) and runmode_untouched: + if eligible_cuda and exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and (any(CUDevicesNames) or any(CLDevicesNames)) and runmode_untouched: if "Use CuBLAS" in runopts: runopts_var.set("Use CuBLAS") gpu_choice_var.set("1") @@ -4636,14 +4683,22 @@ def show_gui(): gpu_choice_var.set("1") print("Auto Selected HIP Backend...\n") found_new_backend = True - elif exitcounter < 100 and (1 in VKIsDGPU) and runmode_untouched and "Use Vulkan" in runopts: + elif exitcounter < 100 and (1 in VKIsDGPU) and runmode_untouched and ("Use Vulkan" in runopts or "Use Vulkan (Old CPU)" in runopts): for i in range(0,len(VKIsDGPU)): if VKIsDGPU[i]==1: - runopts_var.set("Use Vulkan") + if cpusupport==0 and "Use Vulkan" in runopts: + runopts_var.set("Use Vulkan") + else: + runopts_var.set("Use Vulkan (Old CPU)") gpu_choice_var.set(str(i+1)) print("Auto Selected Vulkan Backend...\n") found_new_backend = True break + else: + if runopts_var.get()=="Use CPU" and cpusupport==1 and "Use CPU (Old CPU)" in runopts: + runopts_var.set("Use CPU (Old CPU)") + elif runopts_var.get()=="Use CPU" and cpusupport==2 and "Failsafe Mode (Older CPU)" in runopts: + runopts_var.set("Failsafe Mode (Older CPU)") if not found_new_backend: print("Auto Selected Default Backend...\n") changed_gpu_choice_var() diff --git a/make_pyinstaller_cuda_oldpc.bat b/make_pyinstaller_cuda_oldpc.bat index 0cfeed11a..40af74885 100644 --- a/make_pyinstaller_cuda_oldpc.bat +++ b/make_pyinstaller_cuda_oldpc.bat @@ -1,3 +1,3 @@ call create_ver_file.bat PyInstaller --onedir --noconfirm --clean --console --runtime-hook "./tools/kcpplauncherhook.py" --icon "./niko.ico" --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp-launcher" -PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --collect-all psutil --icon "./nikogreen.ico" --add-data "./dist/koboldcpp-launcher/koboldcpp-launcher.exe;." --add-data "./simpleclinfo.exe;." --add-data "./aria2c-win.exe;." --add-data "./OpenCL.dll;." --add-data "./kcpp_adapters;./kcpp_adapters" --add-data "./koboldcpp.py;." --add-data "./json_to_gbnf.py;." --add-data "./LICENSE.md;." --add-data "./MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md;." --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./kcpp_sdui.embd;." --add-data "./taesd.embd;." --add-data "./taesd_xl.embd;." --add-data "./taesd_f.embd;." --add-data "./taesd_3.embd;." --add-data "./koboldcpp_default.dll;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./koboldcpp_clblast.dll;." --add-data "./koboldcpp_clblast_noavx2.dll;." --add-data "./koboldcpp_clblast_failsafe.dll;." --add-data "./koboldcpp_vulkan_noavx2.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_vulkan.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cublas64_11.dll;." --add-data "./cublasLt64_11.dll;." --add-data "./cudart64_110.dll;." --add-data "C:/Windows/System32/msvcp140.dll;." --add-data "C:/Windows/System32/msvcp140_codecvt_ids.dll;." --add-data "C:/Windows/System32/vcruntime140.dll;." --add-data "C:/Windows/System32/vcruntime140_1.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp-oldpc.exe" \ No newline at end of file +PyInstaller --noconfirm --onefile --clean --console --collect-all customtkinter --collect-all psutil --icon "./nikogreen.ico" --add-data "./dist/koboldcpp-launcher/koboldcpp-launcher.exe;." --add-data "./simpleclinfo.exe;." --add-data "./aria2c-win.exe;." --add-data "./OpenCL.dll;." --add-data "./kcpp_adapters;./kcpp_adapters" --add-data "./koboldcpp.py;." --add-data "./json_to_gbnf.py;." --add-data "./LICENSE.md;." --add-data "./MIT_LICENSE_GGML_SDCPP_LLAMACPP_ONLY.md;." --add-data "./klite.embd;." --add-data "./kcpp_docs.embd;." --add-data "./kcpp_sdui.embd;." --add-data "./taesd.embd;." --add-data "./taesd_xl.embd;." --add-data "./taesd_f.embd;." --add-data "./taesd_3.embd;." --add-data "./koboldcpp_failsafe.dll;." --add-data "./koboldcpp_noavx2.dll;." --add-data "./koboldcpp_clblast_noavx2.dll;." --add-data "./koboldcpp_clblast_failsafe.dll;." --add-data "./koboldcpp_vulkan_noavx2.dll;." --add-data "./clblast.dll;." --add-data "./koboldcpp_cublas.dll;." --add-data "./cublas64_11.dll;." --add-data "./cublasLt64_11.dll;." --add-data "./cudart64_110.dll;." --add-data "C:/Windows/System32/msvcp140.dll;." --add-data "C:/Windows/System32/msvcp140_codecvt_ids.dll;." --add-data "C:/Windows/System32/vcruntime140.dll;." --add-data "C:/Windows/System32/vcruntime140_1.dll;." --add-data "./rwkv_vocab.embd;." --add-data "./rwkv_world_vocab.embd;." --version-file "./version.txt" "./koboldcpp.py" -n "koboldcpp-oldpc.exe" \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 86c701ce8..df8c25ee5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,5 @@ gguf>=0.1.0 customtkinter>=5.1.0 protobuf>=4.21.0 psutil>=5.9.4 -darkdetect>=0.8.0 \ No newline at end of file +darkdetect>=0.8.0 +py-cpuinfo>=9.0.0 \ No newline at end of file