diff --git a/expose.h b/expose.h index b583d8570..2b91bc23f 100644 --- a/expose.h +++ b/expose.h @@ -6,6 +6,7 @@ const int images_max = 8; const int audio_max = 4; const int logprobs_max = 10; const int overridekv_max = 4; +const int lora_filenames_max = 4; // match kobold's sampler list and order enum samplers @@ -188,7 +189,7 @@ struct sd_load_model_inputs const char * clip1_filename = nullptr; const char * clip2_filename = nullptr; const char * vae_filename = nullptr; - const char * lora_filename = nullptr; + const char * lora_filenames[lora_filenames_max] = {}; const float lora_multiplier = 1.0f; const int lora_apply_mode = 0; const char * photomaker_filename = nullptr; diff --git a/koboldcpp.py b/koboldcpp.py index 55e5af062..4c68638a6 100755 --- a/koboldcpp.py +++ b/koboldcpp.py @@ -59,6 +59,7 @@ default_vae_tile_threshold = 768 default_native_ctx = 16384 overridekv_max = 4 default_autofit_padding = 1024 +lora_filenames_max = 4 # abuse prevention stop_token_max = 256 @@ -311,7 +312,7 @@ class sd_load_model_inputs(ctypes.Structure): ("clip1_filename", ctypes.c_char_p), ("clip2_filename", ctypes.c_char_p), ("vae_filename", ctypes.c_char_p), - ("lora_filename", ctypes.c_char_p), + ("lora_filenames", ctypes.c_char_p * lora_filenames_max), ("lora_multiplier", ctypes.c_float), ("lora_apply_mode", ctypes.c_int), ("photomaker_filename", ctypes.c_char_p), @@ -1931,7 +1932,7 @@ def sd_quant_option(value): except Exception: return 0 -def sd_load_model(model_filename,vae_filename,lora_filename,t5xxl_filename,clip1_filename,clip2_filename,photomaker_filename,upscaler_filename): +def sd_load_model(model_filename,vae_filename,lora_filenames,t5xxl_filename,clip1_filename,clip2_filename,photomaker_filename,upscaler_filename): global args inputs = sd_load_model_inputs() inputs.model_filename = model_filename.encode("UTF-8") @@ -1954,7 +1955,12 @@ def sd_load_model(model_filename,vae_filename,lora_filename,t5xxl_filename,clip1 inputs.taesd = True if args.sdvaeauto else False inputs.tiled_vae_threshold = args.sdtiledvae inputs.vae_filename = vae_filename.encode("UTF-8") - inputs.lora_filename = lora_filename.encode("UTF-8") + for n in range(lora_filenames_max): + if n >= len(lora_filenames): + inputs.lora_filenames[n] = "".encode("UTF-8") + else: + inputs.lora_filenames[n] = lora_filenames[n].encode("UTF-8") + inputs.lora_multiplier = args.sdloramult inputs.t5xxl_filename = t5xxl_filename.encode("UTF-8") inputs.clip1_filename = clip1_filename.encode("UTF-8") @@ -5173,7 +5179,7 @@ def RunServerMultiThreaded(addr, port, server_handler): sys.exit(0) # Based on https://github.com/mathgeniuszach/xdialog/blob/main/xdialog/zenity_dialogs.py - MIT license | - Expanded version by Henk717 -def zenity(filetypes=None, initialdir="", initialfile="", **kwargs) -> Tuple[int, str]: +def zenity(filetypes=None, initialdir="", initialfile="", multiple=False, **kwargs) -> Tuple[int, object]: global zenity_recent_dir, zenity_permitted if not zenity_permitted: @@ -5238,6 +5244,10 @@ def zenity(filetypes=None, initialdir="", initialfile="", **kwargs) -> Tuple[int initialpath = os.path.join(initialdir, initialfile) args.append(f'--filename={initialpath}') + if multiple: + args.append("--multiple") + args.append("--separator=|") + clean_env = os.environ.copy() clean_env.pop("LD_LIBRARY_PATH", None) clean_env["PATH"] = "/usr/bin:/bin" @@ -5252,15 +5262,18 @@ def zenity(filetypes=None, initialdir="", initialfile="", **kwargs) -> Tuple[int result = procres.stdout.decode('utf-8').strip() if procres.returncode==0 and result: directory = result - if not os.path.isdir(result): - directory = os.path.dirname(result) + if multiple: + result = tuple(result.split("|")) + directory = result[0] + if not os.path.isdir(directory): + directory = os.path.dirname(directory) zenity_recent_dir = directory return (procres.returncode, result) # note: In this section we wrap around file dialogues to allow for zenity def zentk_askopenfilename(**options): try: - result = zenity(filetypes=options.get("filetypes"), initialdir=options.get("initialdir"), title=options.get("title"))[1] + result = zenity(filetypes=options.get("filetypes"), initialdir=options.get("initialdir"), multiple=False, title=options.get("title"))[1] if result and not os.path.isfile(result): print("A folder was selected while we need a file, ignoring selection.") return '' @@ -5269,9 +5282,21 @@ def zentk_askopenfilename(**options): result = askopenfilename(**options) return result +def zentk_askopenfilenames(**options): + try: + result = zenity(filetypes=options.get("filetypes"), initialdir=options.get("initialdir"), multiple=True, title=options.get("title"))[1] + for itm in result: + if itm and not os.path.isfile(itm): + print("A folder was selected while we need a file, ignoring selection.") + return '' + except Exception: + from tkinter.filedialog import askopenfilenames + result = askopenfilenames(**options) + return result + def zentk_askdirectory(**options): try: - result = zenity(initialdir=options.get("initialdir"), title=options.get("title"), directory=True)[1] + result = zenity(initialdir=options.get("initialdir"), multiple=False, title=options.get("title"), directory=True)[1] except Exception: from tkinter.filedialog import askdirectory result = askdirectory(**options) @@ -5279,7 +5304,7 @@ def zentk_askdirectory(**options): def zentk_asksaveasfilename(**options): try: - result = zenity(filetypes=options.get("filetypes"), initialdir=options.get("initialdir"), initialfile=options.get("initialfile"), title=options.get("title"), save=True)[1] + result = zenity(filetypes=options.get("filetypes"), initialdir=options.get("initialdir"), initialfile=options.get("initialfile"), multiple=False, title=options.get("title"), save=True)[1] except Exception: from tkinter.filedialog import asksaveasfilename result = asksaveasfilename(**options) @@ -5724,7 +5749,7 @@ def show_gui(): return entry, label #file dialog types: 0=openfile,1=savefile,2=opendir - def makefileentry(parent, text, searchtext, var, row=0, width=200, filetypes=[], onchoosefile=None, singlerow=False, singlecol=True, dialog_type=0, tooltiptxt=""): + def makefileentry(parent, text, searchtext, var, row=0, width=200, filetypes=[], onchoosefile=None, singlerow=False, singlecol=True, dialog_type=0, tooltiptxt="", multiple=False): label = makelabel(parent, text, row,0,tooltiptxt,columnspan=3) def getfilename(var, text): initialDir = os.path.dirname(var.get()) @@ -5740,7 +5765,11 @@ def show_gui(): fnam = str(fnam).strip() fnam = f"{fnam}.jsondb" if ".jsondb" not in fnam.lower() else fnam else: - fnam = zentk_askopenfilename(title=text,filetypes=filetypes, initialdir=initialDir) + if multiple: + fnam = zentk_askopenfilenames(title=text,filetypes=filetypes, initialdir=initialDir) + fnam = "|".join(fnam) + else: + fnam = zentk_askopenfilename(title=text,filetypes=filetypes, initialdir=initialDir) if fnam: var.set(fnam) if onchoosefile: @@ -6383,7 +6412,7 @@ def show_gui(): makelabelcombobox(images_tab, "Compress Weights: ", sd_quant_var, 8, width=(60), padx=(126), labelpadx=8, tooltiptxt="Quantizes the SD model weights to save memory.\nHigher levels save more memory, and cause more quality degradation.", values=sd_quant_choices) sd_quant_var.trace_add("write", changed_gpulayers_estimate) - makefileentry(images_tab, "Image LoRA:", "Select SD lora file",sd_lora_var, 20, width=160, singlerow=True, filetypes=[("*.safetensors *.gguf", "*.safetensors *.gguf")],tooltiptxt="Select a .safetensors or .gguf SD LoRA model file to be loaded. Should be unquantized!") + makefileentry(images_tab, "Image LoRA:", "Select SD lora file",sd_lora_var, 20, width=160, singlerow=True, filetypes=[("*.safetensors *.gguf", "*.safetensors *.gguf")],tooltiptxt="Select a .safetensors or .gguf SD LoRA model file to be loaded. Should be unquantized!", multiple=True) makelabelentry(images_tab, "Multiplier:" , sd_loramult_var, 20, 50,padx=(390),singleline=True,tooltip="What mutiplier value to apply the SD LoRA with.",labelpadx=(330)) makefileentry(images_tab, "T5-XXL File:", "Select T5-XXL model file (SD3, Flux, WAN)",sd_t5xxl_var, 24, width=280, singlerow=True, filetypes=[("*.safetensors *.gguf","*.safetensors *.gguf")],tooltiptxt="Select a .safetensors t5xxl file to be loaded.") @@ -6711,10 +6740,10 @@ def show_gui(): args.sdupscaler = sd_upscaler_var.get() args.sdquant = sd_quant_option(sd_quant_var.get()) if sd_lora_var.get() != "": - args.sdlora = sd_lora_var.get() + args.sdlora = [item.strip() for item in sd_lora_var.get().split("|") if item] args.sdloramult = float(sd_loramult_var.get()) else: - args.sdlora = "" + args.sdlora = None if gen_defaults_var.get() != "": args.gendefaults = gen_defaults_var.get() @@ -6959,8 +6988,13 @@ def show_gui(): sd_upscaler_var.set(dict["sdupscaler"] if ("sdupscaler" in dict and dict["sdupscaler"]) else "") sd_vaeauto_var.set(1 if ("sdvaeauto" in dict and dict["sdvaeauto"]) else 0) sd_tiled_vae_var.set(str(dict["sdtiledvae"]) if ("sdtiledvae" in dict and dict["sdtiledvae"]) else str(default_vae_tile_threshold)) - - sd_lora_var.set(dict["sdlora"] if ("sdlora" in dict and dict["sdlora"]) else "") + if "sdlora" in dict and dict["sdlora"]: + if isinstance((dict["sdlora"]), list): + sd_lora_var.set("|".join(dict["sdlora"])) + else: + sd_lora_var.set(dict["sdlora"] if ("sdlora" in dict and dict["sdlora"]) else "") + else: + sd_lora_var.set("") sd_loramult_var.set(str(dict["sdloramult"]) if ("sdloramult" in dict and dict["sdloramult"]) else "1.0") gen_defaults_var.set(dict["gendefaults"] if ("gendefaults" in dict and dict["gendefaults"]) else "") gen_defaults_overwrite_var.set(1 if "gendefaultsoverwrite" in dict and dict["gendefaultsoverwrite"] else 0) @@ -7401,6 +7435,8 @@ def convert_invalid_args(args): dict["gendefaults"] = dict["sdgendefaults"] if "flashattention" in dict and "noflashattention" not in dict: dict["noflashattention"] = not dict["flashattention"] + if "sdlora" in dict and isinstance(dict["sdlora"], str): + dict["sdlora"] = ([dict["sdlora"]] if dict["sdlora"] else None) return args def setuptunnel(global_memory, has_sd): @@ -8220,10 +8256,11 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): dlfile = download_model_from_url(args.sdvae,[".gguf",".safetensors"],min_file_size=500000) if dlfile: args.sdvae = dlfile - if args.sdlora and args.sdlora!="": - dlfile = download_model_from_url(args.sdlora,[".gguf",".safetensors"],min_file_size=500000) - if dlfile: - args.sdlora = dlfile + if args.sdlora and len(args.sdlora)>0: + for i in range(0,len(args.sdlora)): + dlfile = download_model_from_url(args.sdlora[i],[".gguf",".safetensors"],min_file_size=500000) + if dlfile: + args.sdlora[i] = dlfile if args.mmproj and args.mmproj!="": dlfile = download_model_from_url(args.mmproj,[".gguf"],min_file_size=500000) if dlfile: @@ -8499,18 +8536,20 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): exitcounter = 999 exit_with_error(2,f"Cannot find image model file: {imgmodel}") else: - imglora = "" + imgloras = [] imgvae = "" imgt5xxl = "" imgclip1 = "" imgclip2 = "" imgphotomaker = "" imgupscaler = "" - if args.sdlora: - if os.path.exists(args.sdlora): - imglora = os.path.abspath(args.sdlora) - else: - print("Missing SD LORA model file...") + if args.sdlora and len(args.sdlora)>0: + for i in range (0,len(args.sdlora)): + curr = args.sdlora[i] + if os.path.exists(curr): + imgloras.append(os.path.abspath(curr)) + else: + print(f"Missing SD LORA model file {curr}...") if args.sdvae: if os.path.exists(args.sdvae): imgvae = os.path.abspath(args.sdvae) @@ -8547,7 +8586,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False): friendlysdmodelname = os.path.basename(imgmodel) friendlysdmodelname = os.path.splitext(friendlysdmodelname)[0] friendlysdmodelname = sanitize_string(friendlysdmodelname) - loadok = sd_load_model(imgmodel,imgvae,imglora,imgt5xxl,imgclip1,imgclip2,imgphotomaker,imgupscaler) + loadok = sd_load_model(imgmodel,imgvae,imgloras,imgt5xxl,imgclip1,imgclip2,imgphotomaker,imgupscaler) print("Load Image Model OK: " + str(loadok)) if not loadok: exitcounter = 999 @@ -9008,8 +9047,8 @@ if __name__ == '__main__': sdparsergroupvae.add_argument("--sdvaeauto", help="Uses a built-in tiny VAE via TAE SD, which is very fast, and fixed bad VAEs.", action='store_true') sdparsergrouplora = sdparsergroup.add_mutually_exclusive_group() sdparsergrouplora.add_argument("--sdquant", metavar=('[quantization level 0/1/2]'), help="If specified, loads the model quantized to save memory. 0=off, 1=q8, 2=q4", type=int, choices=[0,1,2], nargs="?", const=2, default=0) - sdparsergrouplora.add_argument("--sdlora", metavar=('[filename]'), help="Specify an image generation LORA safetensors model to be applied.", default="") - sdparsergroup.add_argument("--sdloramult", metavar=('[amount]'), help="Multiplier for the image LORA model to be applied.", type=float, default=1.0) + sdparsergrouplora.add_argument("--sdlora", metavar=('[filename]'), help="Specify image generation LoRAs safetensors models to be applied. Multiple LoRAs are accepted.", nargs='+') + sdparsergroup.add_argument("--sdloramult", metavar=('[amount]'), help="Multiplier for the image LoRA model to be applied.", type=float, default=1.0) sdparsergroup.add_argument("--sdtiledvae", metavar=('[maxres]'), help="Adjust the automatic VAE tiling trigger for images above this size. 0 disables vae tiling.", type=int, default=default_vae_tile_threshold) whisperparsergroup = parser.add_argument_group('Whisper Transcription Commands') whisperparsergroup.add_argument("--whispermodel", metavar=('[filename]'), help="Specify a Whisper .bin model to enable Speech-To-Text transcription.", default="") diff --git a/otherarch/sdcpp/sdtype_adapter.cpp b/otherarch/sdcpp/sdtype_adapter.cpp index 14ac4c5dc..7a726692c 100644 --- a/otherarch/sdcpp/sdtype_adapter.cpp +++ b/otherarch/sdcpp/sdtype_adapter.cpp @@ -78,8 +78,8 @@ struct SDParams { bool chroma_use_dit_mask = true; - std::string lora_path; - sd_lora_t lora_spec; + std::vector lora_paths; + std::vector lora_specs; uint32_t lora_count; }; @@ -207,7 +207,15 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { set_sd_quiet(sd_is_quiet); executable_path = inputs.executable_path; std::string taesdpath = ""; - std::string lorafilename = inputs.lora_filename; + std::vector lorafilenames; + for(int i=0;i0) { - const char* lora_apply_mode_name = lora_apply_mode == 1 ? "immediately" - : lora_apply_mode == 2 ? "at runtime" - : "auto"; - printf("With LoRA: %s at %f power, apply mode: %s\n", - lorafilename.c_str(),inputs.lora_multiplier,lora_apply_mode_name); + for(int i=0;iclip_l_path = clip1_filename; sd_params->clip_g_path = clip2_filename; sd_params->stacked_id_embeddings_path = photomaker_filename; - sd_params->lora_path = lorafilename; + sd_params->lora_paths = lorafilenames; //if t5 is set, and model is a gguf, load it as a diffusion model path bool endswithgguf = (sd_params->model_path.rfind(".gguf") == sd_params->model_path.size() - 5); if((sd_params->t5xxl_path!="" || sd_params->clip_l_path!="" || sd_params->clip_g_path!="") && endswithgguf) @@ -405,15 +416,21 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) { std::filesystem::path mpath(inputs.model_filename); sdmodelfilename = mpath.filename().string(); - sd_params->lora_spec = {}; - sd_params->lora_spec.path = sd_params->lora_path.c_str(); - sd_params->lora_spec.multiplier = inputs.lora_multiplier; - - if(sd_params->lora_path!="" && sd_params->lora_spec.multiplier>0) + sd_params->lora_specs.clear(); + sd_params->lora_specs.reserve(lora_filenames_max*2); + for(int i=0;ilora_paths.size();++i) { - printf("\nApply LoRA...\n"); - sd_params->lora_count = 1; - sd_ctx->sd->apply_loras(&sd_params->lora_spec, sd_params->lora_count); + sd_lora_t spec = {}; + spec.path = sd_params->lora_paths[i].c_str(); + spec.multiplier = inputs.lora_multiplier; + sd_params->lora_specs.push_back(spec); + } + + if(sd_params->lora_specs.size()>0 && inputs.lora_multiplier>0) + { + printf("\nApply %d LoRAs...\n",sd_params->lora_specs.size()); + sd_params->lora_count = sd_params->lora_specs.size(); + sd_ctx->sd->apply_loras(sd_params->lora_specs.data(), sd_params->lora_count); } input_extraimage_buffers.reserve(max_extra_images); @@ -1011,7 +1028,7 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs) // needs to be "reapplied" because sdcpp tracks previously applied LoRAs // and weights, and apply/unapply the differences at each gen - params.loras = &sd_params->lora_spec; + params.loras = sd_params->lora_specs.data(); params.lora_count = sd_params->lora_count; params.ref_images = reference_imgs.data();