From ad86b1aeb83569205bc48f3a8ef9bf405abd46fd Mon Sep 17 00:00:00 2001 From: Concedo <39025047+LostRuins@users.noreply.github.com> Date: Wed, 17 Jul 2024 23:09:54 +0800 Subject: [PATCH] Implemented Kcpp Launch Templates (+1 squashed commits) Squashed commits: [5ea4c1de] wip integrating skcpps templates (+1 squashed commits) Squashed commits: [737daa7f] skcpps wip --- koboldcpp.py | 148 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 100 insertions(+), 48 deletions(-) diff --git a/koboldcpp.py b/koboldcpp.py index 496082f1f..0521da89e 100644 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -221,6 +221,16 @@ def getabspath(): def file_exists(filename): return os.path.exists(os.path.join(getdirpath(), filename)) +def get_default_threads(): + physical_core_limit = 1 + if os.cpu_count()!=None and os.cpu_count()>1: + physical_core_limit = os.cpu_count() // 2 + default_threads = (physical_core_limit if physical_core_limit<=3 else max(3,physical_core_limit-1)) + processor = platform.processor() + if 'Intel' in processor: + default_threads = (8 if default_threads > 8 else default_threads) #this helps avoid e-cores. + return default_threads + def pick_existant_file(ntoption,nonntoption): precompiled_prefix = "precompiled_" ntexist = file_exists(ntoption) @@ -251,6 +261,19 @@ lib_hipblas = pick_existant_file("koboldcpp_hipblas.dll","koboldcpp_hipblas.so") lib_vulkan = pick_existant_file("koboldcpp_vulkan.dll","koboldcpp_vulkan.so") lib_vulkan_noavx2 = pick_existant_file("koboldcpp_vulkan_noavx2.dll","koboldcpp_vulkan_noavx2.so") libname = "" +lib_option_pairs = [ + (lib_openblas, "Use OpenBLAS"), + (lib_default, "Use No BLAS"), + (lib_clblast, "Use CLBlast"), + (lib_cublas, "Use CuBLAS"), + (lib_hipblas, "Use hipBLAS (ROCm)"), + (lib_vulkan, "Use Vulkan"), + (lib_noavx2, "NoAVX2 Mode (Old CPU)"), + (lib_clblast_noavx2, "CLBlast NoAVX2 (Old CPU)"), + (lib_vulkan_noavx2, "Vulkan NoAVX2 (Old CPU)"), + (lib_failsafe, "Failsafe Mode (Old CPU)")] +openblas_option, default_option, clblast_option, cublas_option, hipblas_option, vulkan_option, noavx2_option, clblast_noavx2_option, vulkan_noavx2_option, failsafe_option = (opt if file_exists(lib) or (os.name == 'nt' and file_exists(opt + ".dll")) else None for lib, opt in lib_option_pairs) +runopts = [opt for lib, opt in lib_option_pairs if file_exists(lib)] def init_library(): global handle, args, libname @@ -672,6 +695,18 @@ def fetch_gpu_properties(testCL,testCU,testVK): pass return +def auto_set_backend_cli(): + print("\nA .kcppt template was selected - automatically selecting your backend...\n") + fetch_gpu_properties(False,True,True) + if exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and any(CUDevicesNames): + if "Use CuBLAS" in runopts or "Use hipBLAS (ROCm)" in runopts: + args.usecublas = ["normal","mmq"] + elif exitcounter < 100 and (1 in VKIsDGPU) and "Use Vulkan" in runopts: + for i in range(0,len(VKIsDGPU)): + if VKIsDGPU[i]==1: + args.usevulkan = [] + break + def load_model(model_filename): global args inputs = load_model_inputs() @@ -2004,8 +2039,8 @@ def show_gui(): args.model_param = askopenfilename(title="Select ggml model .bin or .gguf file or .kcpps config") root.withdraw() root.quit() - if args.model_param and args.model_param!="" and (args.model_param.lower().endswith('.kcpps') or args.model_param.lower().endswith('.skcpps')): - loadconfigfile(args.model_param) + if args.model_param and args.model_param!="" and (args.model_param.lower().endswith('.kcpps') or args.model_param.lower().endswith('.kcppt')): + load_config_cli(args.model_param) if not args.model_param and not args.sdmodel and not args.whispermodel: global exitcounter exitcounter = 999 @@ -2072,6 +2107,7 @@ def show_gui(): root.bind("", on_resize) global using_gui_launcher using_gui_launcher = True + kcpp_exporting_template = False # trigger empty tooltip then remove it def show_tooltip(event, tooltip_text=None): @@ -2097,6 +2133,8 @@ def show_gui(): show_tooltip(None,"") #initialize tooltip objects hide_tooltip(None) + default_threads = get_default_threads() + tabs = ctk.CTkFrame(root, corner_radius = 0, width=windowwidth, height=windowheight-50) tabs.grid(row=0, stick="nsew") tabnames= ["Quick Launch", "Hardware", "Tokens", "Model Files", "Network", "Horde Worker","Image Gen","Audio","Extra"] @@ -2110,23 +2148,10 @@ def show_gui(): tabcontentframe.grid_propagate(False) tabcontent = {} - lib_option_pairs = [ - (lib_openblas, "Use OpenBLAS"), - (lib_default, "Use No BLAS"), - (lib_clblast, "Use CLBlast"), - (lib_cublas, "Use CuBLAS"), - (lib_hipblas, "Use hipBLAS (ROCm)"), - (lib_vulkan, "Use Vulkan"), - (lib_noavx2, "NoAVX2 Mode (Old CPU)"), - (lib_clblast_noavx2, "CLBlast NoAVX2 (Old CPU)"), - (lib_vulkan_noavx2, "Vulkan NoAVX2 (Old CPU)"), - (lib_failsafe, "Failsafe Mode (Old CPU)")] - openblas_option, default_option, clblast_option, cublas_option, hipblas_option, vulkan_option, noavx2_option, clblast_noavx2_option, vulkan_noavx2_option, failsafe_option = (opt if file_exists(lib) or (os.name == 'nt' and file_exists(opt + ".dll")) else None for lib, opt in lib_option_pairs) # slider data blasbatchsize_values = ["-1", "32", "64", "128", "256", "512", "1024", "2048"] blasbatchsize_text = ["Don't Batch BLAS","32","64","128","256","512","1024","2048"] contextsize_text = ["256", "512", "1024", "2048", "3072", "4096", "6144", "8192", "12288", "16384", "24576", "32768", "49152", "65536", "98304", "131072"] - runopts = [opt for lib, opt in lib_option_pairs if file_exists(lib)] antirunopts = [opt.replace("Use ", "") for lib, opt in lib_option_pairs if not (opt in runopts)] quantkv_text = ["F16 (Off)","8-Bit","4-Bit"] @@ -2198,8 +2223,6 @@ def show_gui(): whisper_model_var = ctk.StringVar() - kcpp_jsonembed_var = ctk.IntVar(value=0) - def tabbuttonaction(name): for t in tabcontent: if name == t: @@ -2292,10 +2315,15 @@ def show_gui(): # decided to follow yellowrose's and kalomaze's suggestions, this function will automatically try to determine GPU identifiers # run in new thread so it doesnt block. does not return anything, instead overwrites specific values and redraws GUI - def auto_set_backend(): - fetch_gpu_properties(True,True,True) - #autopick cublas if suitable, requires at least 3.5GB VRAM to auto pick + def auto_set_backend_gui(manual_select=False): global exitcounter, runmode_untouched + if manual_select: + print("\nA .kcppt template was selected - automatically selecting your backend...\n") + runmode_untouched = True + fetch_gpu_properties(False,True,True) + else: + fetch_gpu_properties(True,True,True) + #autopick cublas if suitable, requires at least 3.5GB VRAM to auto pick #we do not want to autoselect hip/cublas if the user has already changed their desired backend! if exitcounter < 100 and MaxMemory[0]>3500000000 and (("Use CuBLAS" in runopts and CUDevicesNames[0]!="") or "Use hipBLAS (ROCm)" in runopts) and (any(CUDevicesNames) or any(CLDevicesNames)) and runmode_untouched: if "Use CuBLAS" in runopts: @@ -2312,7 +2340,7 @@ def show_gui(): def on_picked_model_file(filepath): global gui_layers_untouched - if filepath.lower().endswith('.kcpps') or filepath.lower().endswith('.skcpps'): + if filepath.lower().endswith('.kcpps') or filepath.lower().endswith('.kcppt'): #load it as a config file instead with open(filepath, 'r') as f: dict = json.load(f) @@ -2678,13 +2706,28 @@ def show_gui(): audio_tab = tabcontent["Audio"] makefileentry(audio_tab, "Whisper Model (Speech-To-Text):", "Select Whisper .bin Model File", whisper_model_var, 1, width=280, filetypes=[("*.bin","*.bin")], tooltiptxt="Select a Whisper .bin model file on disk to be loaded.") + def kcpp_export_template(): + nonlocal kcpp_exporting_template + kcpp_exporting_template = True + export_vars() + kcpp_exporting_template = False + savdict = json.loads(json.dumps(args.__dict__)) + file_type = [("KoboldCpp LaunchTemplate", "*.kcppt")] + savdict["istemplate"] = True + filename = asksaveasfile(filetypes=file_type, defaultextension=file_type) + if filename == None: return + file = open(str(filename.name), 'a') + file.write(json.dumps(savdict)) + file.close() + pass + # extra tab extra_tab = tabcontent["Extra"] makelabel(extra_tab, "Unpack KoboldCpp to a local directory to modify its files.", 1, 0) makelabel(extra_tab, "You can also launch via koboldcpp.py for faster startup.", 2, 0) ctk.CTkButton(extra_tab , text = "Unpack KoboldCpp To Folder", command = unpack_to_dir ).grid(row=3,column=0, stick="w", padx= 8, pady=2) - makecheckbox(extra_tab, "Save launch settings as portable SKCPPS", kcpp_jsonembed_var, 5, tooltiptxt="Portable sharing format.\nEmbeds any selected JSON files directly into skcpps setting files when saving.") - + makelabel(extra_tab, "Export as launcher .kcppt template (Expert Only)", 4, 0,tooltiptxt="Creates a KoboldCpp launch template for others to use.\nEmbeds JSON files directly into exported file when saving.\nWhen loaded, forces the backend to be automatically determined.\nWarning! Not recommended for beginners!") + ctk.CTkButton(extra_tab , text = "Generate LaunchTemplate", command = kcpp_export_template ).grid(row=5,column=0, stick="w", padx= 8, pady=2) # launch def guilaunch(): @@ -2698,6 +2741,7 @@ def show_gui(): pass def export_vars(): + nonlocal kcpp_exporting_template args.threads = int(threads_var.get()) args.usemlock = usemlock.get() == 1 args.debugmode = debugmode.get() @@ -2768,7 +2812,7 @@ def show_gui(): args.chatcompletionsadapter = None if chatcompletionsadapter_var.get() == "" else chatcompletionsadapter_var.get() try: - if kcpp_jsonembed_var.get()==1 and isinstance(args.chatcompletionsadapter, str) and args.chatcompletionsadapter!="" and os.path.exists(args.chatcompletionsadapter): + if kcpp_exporting_template and isinstance(args.chatcompletionsadapter, str) and args.chatcompletionsadapter!="" and os.path.exists(args.chatcompletionsadapter): print(f"Embedding chat completions adapter...") # parse and save embedded preload story with open(args.chatcompletionsadapter, 'r') as f: args.chatcompletionsadapter = json.load(f) @@ -2779,7 +2823,7 @@ def show_gui(): args.lora = None if lora_var.get() == "" else ([lora_var.get()] if lora_base_var.get()=="" else [lora_var.get(), lora_base_var.get()]) args.preloadstory = None if preloadstory_var.get() == "" else preloadstory_var.get() try: - if kcpp_jsonembed_var.get()==1 and isinstance(args.preloadstory, str) and args.preloadstory!="" and os.path.exists(args.preloadstory): + if kcpp_exporting_template and isinstance(args.preloadstory, str) and args.preloadstory!="" and os.path.exists(args.preloadstory): print(f"Embedding preload story...") # parse and save embedded preload story with open(args.preloadstory, 'r') as f: args.preloadstory = json.load(f) @@ -2970,18 +3014,24 @@ def show_gui(): whisper_model_var.set(dict["whispermodel"] if ("whispermodel" in dict and dict["whispermodel"]) else "") - def save_config(): - file_type = [("KoboldCpp Settings", "*.kcpps *.skcpps")] + if "istemplate" in dict and dict["istemplate"]: + auto_set_backend_gui(True) + + def save_config_gui(): + nonlocal kcpp_exporting_template + kcpp_exporting_template = False + export_vars() + savdict = json.loads(json.dumps(args.__dict__)) + file_type = [("KoboldCpp Settings", "*.kcpps")] filename = asksaveasfile(filetypes=file_type, defaultextension=file_type) if filename == None: return - export_vars() file = open(str(filename.name), 'a') - file.write(json.dumps(args.__dict__)) + file.write(json.dumps(savdict)) file.close() pass - def load_config(): - file_type = [("KoboldCpp Settings", "*.kcpps *.skcpps")] + def load_config_gui(): #this is used to populate the GUI with a config file, whereas load_config_cli simply overwrites cli args + file_type = [("KoboldCpp Settings", "*.kcpps *.kcppt")] global runmode_untouched runmode_untouched = False filename = askopenfilename(filetypes=file_type, defaultextension=file_type, initialdir=None) @@ -3008,12 +3058,12 @@ def show_gui(): ctk.CTkButton(tabs , text = "Launch", fg_color="#2f8d3c", hover_color="#2faa3c", command = guilaunch, width=80, height = 35 ).grid(row=1,column=1, stick="se", padx= 25, pady=5) ctk.CTkButton(tabs , text = "Update", fg_color="#9900cc", hover_color="#aa11dd", command = display_updates, width=90, height = 35 ).grid(row=1,column=0, stick="sw", padx= 5, pady=5) - ctk.CTkButton(tabs , text = "Save", fg_color="#084a66", hover_color="#085a88", command = save_config, width=60, height = 35 ).grid(row=1,column=1, stick="sw", padx= 5, pady=5) - ctk.CTkButton(tabs , text = "Load", fg_color="#084a66", hover_color="#085a88", command = load_config, width=60, height = 35 ).grid(row=1,column=1, stick="sw", padx= 70, pady=5) + ctk.CTkButton(tabs , text = "Save", fg_color="#084a66", hover_color="#085a88", command = save_config_gui, width=60, height = 35 ).grid(row=1,column=1, stick="sw", padx= 5, pady=5) + ctk.CTkButton(tabs , text = "Load", fg_color="#084a66", hover_color="#085a88", command = load_config_gui, width=60, height = 35 ).grid(row=1,column=1, stick="sw", padx= 70, pady=5) ctk.CTkButton(tabs , text = "Help", fg_color="#992222", hover_color="#bb3333", command = display_help, width=60, height = 35 ).grid(row=1,column=1, stick="sw", padx= 135, pady=5) # start a thread that tries to get actual gpu names and layer counts - gpuinfo_thread = threading.Thread(target=auto_set_backend) + gpuinfo_thread = threading.Thread(target=auto_set_backend_gui) gpuinfo_thread.start() #submit job in new thread so nothing is waiting # runs main loop until closed or launch clicked @@ -3025,6 +3075,7 @@ def show_gui(): sys.exit(0) else: # processing vars + kcpp_exporting_template = False export_vars() if not args.model_param and not args.sdmodel and not args.whispermodel: @@ -3428,12 +3479,15 @@ def unload_libs(): del handle handle = None -def loadconfigfile(filename): - print("Loading kcpps configuration file...") +def load_config_cli(filename): + print("Loading .kcpps configuration file...") with open(filename, 'r') as f: config = json.load(f) + args.istemplate = False for key, value in config.items(): setattr(args, key, value) + if args.istemplate: + auto_set_backend_cli() def delete_old_pyinstaller(): @@ -3487,7 +3541,7 @@ def main(launch_args,start_server=True): if args.config and len(args.config)==1: if isinstance(args.config[0], str) and os.path.exists(args.config[0]): - loadconfigfile(args.config[0]) + load_config_cli(args.config[0]) elif args.ignoremissing: print("Ignoring missing kcpp config file...") else: @@ -3497,8 +3551,8 @@ def main(launch_args,start_server=True): args = convert_outdated_args(args) #positional handling for kcpps files (drag and drop) - if args.model_param and args.model_param!="" and (args.model_param.lower().endswith('.kcpps') or args.model_param.lower().endswith('.skcpps')): - loadconfigfile(args.model_param) + if args.model_param and args.model_param!="" and (args.model_param.lower().endswith('.kcpps') or args.model_param.lower().endswith('.kcppt')): + load_config_cli(args.model_param) #prevent quantkv from being used without flash attn if args.quantkv and args.quantkv>0 and not args.flashattention: @@ -3665,7 +3719,7 @@ def main(launch_args,start_server=True): pass elif args.gpulayers==-1 and not shouldavoidgpu and os.path.exists(args.model_param): print("Trying to automatically determine GPU layers...") - if MaxMemory[0] == 0: #try to get gpu vram for cuda + if MaxMemory[0] == 0: #try to get gpu vram for cuda if not picked yet fetch_gpu_properties(False,True,False) pass if MaxMemory[0] > 0: @@ -3673,6 +3727,10 @@ def main(launch_args,start_server=True): print(f"Auto Recommended Layers: {layeramt}") args.gpulayers = layeramt + if args.threads == -1: + args.threads = get_default_threads() + print(f"Auto Set Threads: {args.threads}") + init_library() # Note: if blas does not exist and is enabled, program will crash. print("==========") time.sleep(1) @@ -3988,14 +4046,8 @@ if __name__ == '__main__': parser.add_argument("--host", metavar=('[ipaddr]'), help="Host IP to listen on. If empty, all routable interfaces are accepted.", default="") parser.add_argument("--launch", help="Launches a web browser when load is completed.", action='store_true') parser.add_argument("--config", metavar=('[filename]'), help="Load settings from a .kcpps file. Other arguments will be ignored", type=str, nargs=1) - physical_core_limit = 1 - if os.cpu_count()!=None and os.cpu_count()>1: - physical_core_limit = os.cpu_count() // 2 - default_threads = (physical_core_limit if physical_core_limit<=3 else max(3,physical_core_limit-1)) - processor = platform.processor() - if 'Intel' in processor: - default_threads = (8 if default_threads > 8 else default_threads) #this helps avoid e-cores. - parser.add_argument("--threads", metavar=('[threads]'), help="Use a custom number of threads if specified. Otherwise, uses an amount based on CPU cores", type=int, default=default_threads) + + parser.add_argument("--threads", metavar=('[threads]'), help="Use a custom number of threads if specified. Otherwise, uses an amount based on CPU cores", type=int, default=get_default_threads()) compatgroup = parser.add_mutually_exclusive_group() compatgroup.add_argument("--usecublas", help="Use CuBLAS for GPU Acceleration. Requires CUDA. Select lowvram to not allocate VRAM scratch buffer. Enter a number afterwards to select and use 1 GPU. Leaving no number will use all GPUs. For hipBLAS binaries, please check YellowRoseCx rocm fork.", nargs='*',metavar=('[lowvram|normal] [main GPU ID] [mmq] [rowsplit]'), choices=['normal', 'lowvram', '0', '1', '2', '3', 'mmq', 'rowsplit']) compatgroup.add_argument("--usevulkan", help="Use Vulkan for GPU Acceleration. Can optionally specify GPU Device ID (e.g. --usevulkan 0).", metavar=('[Device ID]'), nargs='*', type=int, default=None)