mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-07-10 01:18:32 +00:00
ltx wip
This commit is contained in:
Concedo
parent
2c6184054d
commit
13663730a9
4 changed files with 57 additions and 22 deletions
1
expose.h
1
expose.h
|
|
@ -201,6 +201,7 @@ struct sd_load_model_inputs
|
|||
const char * clip1_filename = nullptr;
|
||||
const char * clip2_filename = nullptr;
|
||||
const char * vae_filename = nullptr;
|
||||
const char * audio_vae_filename = nullptr;
|
||||
const int lora_len = 0;
|
||||
const char ** lora_filenames = nullptr;
|
||||
const float * lora_multipliers = nullptr;
|
||||
|
|
|
|||
32
koboldcpp.py
32
koboldcpp.py
|
|
@ -385,6 +385,7 @@ class sd_load_model_inputs(ctypes.Structure):
|
|||
("clip1_filename", ctypes.c_char_p),
|
||||
("clip2_filename", ctypes.c_char_p),
|
||||
("vae_filename", ctypes.c_char_p),
|
||||
("audio_vae_filename", ctypes.c_char_p),
|
||||
("lora_len", ctypes.c_int),
|
||||
("lora_filenames", ctypes.POINTER(ctypes.c_char_p)),
|
||||
("lora_multipliers", ctypes.POINTER(ctypes.c_float)),
|
||||
|
|
@ -2443,7 +2444,7 @@ def sd_resolve_device(name, default_=-1):
|
|||
name = str(max(name, -2))
|
||||
return sd_get_device_number(name)
|
||||
|
||||
def sd_load_model(model_filename,vae_filename,t5xxl_filename,clip1_filename,clip2_filename,photomaker_filename,upscaler_filename):
|
||||
def sd_load_model(model_filename,vae_filename,t5xxl_filename,clip1_filename,clip2_filename,photomaker_filename,upscaler_filename,audio_vae_filename):
|
||||
global args
|
||||
inputs = sd_load_model_inputs()
|
||||
inputs.model_filename = model_filename.encode("UTF-8")
|
||||
|
|
@ -2467,6 +2468,7 @@ def sd_load_model(model_filename,vae_filename,t5xxl_filename,clip1_filename,clip
|
|||
inputs.taesd = True if args.sdvaeauto else False
|
||||
inputs.tiled_vae_threshold = args.sdtiledvae
|
||||
inputs.vae_filename = vae_filename.encode("UTF-8")
|
||||
inputs.audio_vae_filename = audio_vae_filename.encode("UTF-8")
|
||||
inputs.t5xxl_filename = t5xxl_filename.encode("UTF-8")
|
||||
inputs.clip1_filename = clip1_filename.encode("UTF-8")
|
||||
inputs.clip2_filename = clip2_filename.encode("UTF-8")
|
||||
|
|
@ -7676,6 +7678,7 @@ def show_gui():
|
|||
sd_lora_var = ctk.StringVar()
|
||||
sd_loramult_var = ctk.StringVar(value="1.0")
|
||||
sd_vae_var = ctk.StringVar()
|
||||
sd_audio_vae_var = ctk.StringVar()
|
||||
sd_t5xxl_var = ctk.StringVar()
|
||||
sd_clip1_var = ctk.StringVar()
|
||||
sd_clip2_var = ctk.StringVar()
|
||||
|
|
@ -8602,6 +8605,8 @@ def show_gui():
|
|||
|
||||
|
||||
sdvaeitem1,sdvaeitem2,sdvaeitem3 = makefileentry(images_tab, "Image VAE:", "Select Optional SD VAE file",sd_vae_var, 40, width=280, singlerow=True, filetypes=[("*.safetensors *.gguf", "*.safetensors *.gguf")],tooltiptxt="Select a .safetensors or .gguf SD VAE file to be loaded.")
|
||||
makefileentry(images_tab, "Audio VAE:", "Select Audio VAE file for LTX video gen",sd_audio_vae_var, 42, width=280, singlerow=True, filetypes=[("*.safetensors *.gguf","*.safetensors *.gguf")],tooltiptxt="Select an Audio VAE file for LTX video generation")
|
||||
|
||||
def toggletaesd(a,b,c):
|
||||
if sd_vaeauto_var.get()==1:
|
||||
sdvaeitem1.grid_remove()
|
||||
|
|
@ -8612,10 +8617,10 @@ def show_gui():
|
|||
sdvaeitem1.grid()
|
||||
sdvaeitem2.grid()
|
||||
sdvaeitem3.grid()
|
||||
makecheckbox(images_tab, "Automatic VAE (TAE SD)", sd_vaeauto_var, 42,command=toggletaesd,tooltiptxt="Replace VAE with TAESD. May fix bad VAE.")
|
||||
makelabelcombobox(images_tab, "Conv2D Direct:", sd_convdirect_var, row=42, labelpadx=(220), padx=(310), width=90, tooltiptxt="Use Conv2D Direct operation. May save memory or improve performance.\nMight crash if not supported by the backend.\n", values=sd_convdirect_choices)
|
||||
makelabelentry(images_tab, "VAE Tiling Threshold:", sd_tiled_vae_var, 44, 50, padx=(144),singleline=True,tooltip="Enable VAE Tiling for images above this size, to save memory.\nSet to 0 to disable VAE tiling.")
|
||||
makecheckbox(images_tab, "SD Flash Attention", sd_flash_attention_var, 44,padx=(230), tooltiptxt="Enable Flash Attention for image diffusion. May save memory or improve performance.")
|
||||
makecheckbox(images_tab, "Automatic VAE (TAE SD)", sd_vaeauto_var, 44,command=toggletaesd,tooltiptxt="Replace VAE with TAESD. May fix bad VAE.")
|
||||
makelabelcombobox(images_tab, "Conv2D Direct:", sd_convdirect_var, row=44, labelpadx=(220), padx=(310), width=90, tooltiptxt="Use Conv2D Direct operation. May save memory or improve performance.\nMight crash if not supported by the backend.\n", values=sd_convdirect_choices)
|
||||
makelabelentry(images_tab, "VAE Tiling Threshold:", sd_tiled_vae_var, 46, 50, padx=(144),singleline=True,tooltip="Enable VAE Tiling for images above this size, to save memory.\nSet to 0 to disable VAE tiling.")
|
||||
makecheckbox(images_tab, "SD Flash Attention", sd_flash_attention_var, 46,padx=(230), tooltiptxt="Enable Flash Attention for image diffusion. May save memory or improve performance.")
|
||||
makecheckbox(images_tab, "Model Offload", sd_offload_cpu_var, 50,padx=8, tooltiptxt="Offload image weights in RAM to save VRAM, swap into VRAM when needed.")
|
||||
makelabelcombobox(images_tab, "VAE dev:", sd_vae_device_var, 50,labelpadx=(140),padx=(200), width=70, tooltiptxt="Change VAE device for image generation.", values=sd_device_choices)
|
||||
makelabelcombobox(images_tab, "CLIP dev:", sd_clip_device_var, 50,labelpadx=(280),padx=340, width=70, tooltiptxt="Change CLIP / T5 / LLM device for image generation.", values=sd_device_choices)
|
||||
|
|
@ -8961,6 +8966,7 @@ def show_gui():
|
|||
args.sdvae = ""
|
||||
if sd_vae_var.get() != "":
|
||||
args.sdvae = sd_vae_var.get()
|
||||
args.sdaudiovae = sd_audio_vae_var.get() if sd_audio_vae_var.get() != "" else ""
|
||||
args.sdconvdirect = sd_convdirect_option(sd_convdirect_var.get())
|
||||
args.sdt5xxl = sd_t5xxl_var.get() if sd_t5xxl_var.get() != "" else ""
|
||||
args.sdclip1 = sd_clip1_var.get() if sd_clip1_var.get() != "" else ""
|
||||
|
|
@ -9254,6 +9260,7 @@ def show_gui():
|
|||
sd_clip_device_var.set(sd_get_device_name(sd_resolve_device(mydict.get("sdclipdevice"), default_sdclipdevice)))
|
||||
sd_convdirect_var.set(sd_convdirect_option(mydict.get("sdconvdirect")))
|
||||
sd_vae_var.set(mydict["sdvae"] if ("sdvae" in mydict and mydict["sdvae"]) else "")
|
||||
sd_audio_vae_var.set(mydict["sdaudiovae"] if ("sdaudiovae" in mydict and mydict["sdaudiovae"]) else "")
|
||||
sd_t5xxl_var.set(mydict["sdt5xxl"] if ("sdt5xxl" in mydict and mydict["sdt5xxl"]) else "")
|
||||
sd_clip1_var.set(mydict["sdclip1"] if ("sdclip1" in mydict and mydict["sdclip1"]) else "")
|
||||
sd_clip2_var.set(mydict["sdclip2"] if ("sdclip2" in mydict and mydict["sdclip2"]) else "")
|
||||
|
|
@ -10635,7 +10642,7 @@ def disableSwappedFieldsInConfig(args, swapReqType):
|
|||
for e in ["musicllm", "musicembeddings", "musicdiffusion", "musicvae"]:
|
||||
setattr(args, e, "")
|
||||
if swapReqType != "image":
|
||||
for e in ["sdmodel", "sdt5xxl", "sdclip1", "sdclip2", "sdphotomaker", "sdupscaler", "sdvae", "sdlora"]:
|
||||
for e in ["sdmodel", "sdt5xxl", "sdclip1", "sdclip2", "sdphotomaker", "sdupscaler", "sdvae", "sdaudiovae", "sdlora"]:
|
||||
setattr(args, e, "")
|
||||
|
||||
|
||||
|
|
@ -10826,6 +10833,10 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
|||
dlfile = download_model_from_url(args.sdvae,[".gguf",".safetensors"],min_file_size=500000)
|
||||
if dlfile:
|
||||
args.sdvae = dlfile
|
||||
if args.sdaudiovae and args.sdaudiovae!="":
|
||||
dlfile = download_model_from_url(args.sdaudiovae,[".gguf",".safetensors"],min_file_size=500000)
|
||||
if dlfile:
|
||||
args.sdaudiovae = dlfile
|
||||
if args.sdlora and len(args.sdlora)>0:
|
||||
for i in range(0,len(args.sdlora)):
|
||||
dlfile = download_model_from_url(args.sdlora[i],[".gguf",".safetensors"],min_file_size=500000)
|
||||
|
|
@ -11171,6 +11182,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
|||
imgclip2 = ""
|
||||
imgphotomaker = ""
|
||||
imgupscaler = ""
|
||||
imgaudiovae = ""
|
||||
global imglora_preload, imglora_bypath, imglora_name2path
|
||||
imglora_preload, imglora_bypath, imglora_name2path = mk_lora_info(args.sdlora, args.sdloramult)
|
||||
if args.sdvae:
|
||||
|
|
@ -11178,6 +11190,11 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
|||
imgvae = os.path.abspath(args.sdvae)
|
||||
else:
|
||||
print("Missing SD VAE model file...")
|
||||
if args.sdaudiovae:
|
||||
if os.path.exists(args.sdaudiovae):
|
||||
imgaudiovae = os.path.abspath(args.sdaudiovae)
|
||||
else:
|
||||
print("Missing SD Audio VAE model file...")
|
||||
if args.sdt5xxl:
|
||||
if os.path.exists(args.sdt5xxl):
|
||||
imgt5xxl = os.path.abspath(args.sdt5xxl)
|
||||
|
|
@ -11209,7 +11226,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
|||
friendlysdmodelname = os.path.basename(imgmodel)
|
||||
friendlysdmodelname = os.path.splitext(friendlysdmodelname)[0]
|
||||
friendlysdmodelname = sanitize_string(friendlysdmodelname)
|
||||
loadok = sd_load_model(imgmodel,imgvae,imgt5xxl,imgclip1,imgclip2,imgphotomaker,imgupscaler)
|
||||
loadok = sd_load_model(imgmodel,imgvae,imgt5xxl,imgclip1,imgclip2,imgphotomaker,imgupscaler,imgaudiovae)
|
||||
cached_sd_info = sd_get_info()
|
||||
print("Load Image Model OK: " + str(loadok))
|
||||
if not loadok:
|
||||
|
|
@ -11790,6 +11807,7 @@ if __name__ == '__main__':
|
|||
sdparsergroupvae = sdparsergroup.add_mutually_exclusive_group()
|
||||
sdparsergroupvae.add_argument("--sdvae", metavar=('[filename]'), help="Specify an image generation safetensors VAE which replaces the one in the model.", default="")
|
||||
sdparsergroupvae.add_argument("--sdvaeauto", help="Uses a built-in tiny VAE via TAE SD, which is very fast, and fixed bad VAEs.", action='store_true')
|
||||
sdparsergroup.add_argument("--sdaudiovae", metavar=('[filename]'), help="Specify an image generation audio VAE for LTX2.3 video generation.", default="")
|
||||
sdparsergrouplora = sdparsergroup.add_mutually_exclusive_group()
|
||||
sdparsergrouplora.add_argument("--sdquant", metavar=('[quantization level 0/1/2]'), help="If specified, loads the model quantized to save memory. 0=off, 1=q8, 2=q4", type=int, choices=[0,1,2], nargs="?", const=2, default=0)
|
||||
sdparsergrouplora.add_argument("--sdlora", metavar=('[filename]'), help="Specify image generation LoRAs safetensors models to be applied. Multiple LoRAs are accepted.", nargs='+')
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ struct SDParams {
|
|||
std::string t5xxl_path;
|
||||
std::string diffusion_model_path;
|
||||
std::string vae_path;
|
||||
std::string audio_vae_path;
|
||||
std::string taesd_path;
|
||||
std::string stacked_id_embeddings_path;
|
||||
sd_type_t wtype = SD_TYPE_COUNT;
|
||||
|
|
@ -340,6 +341,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
|
|||
lora_map.add_lora(inputs.lora_filenames[i], inputs.lora_multipliers[i]);
|
||||
}
|
||||
std::string vaefilename = inputs.vae_filename;
|
||||
std::string audiovaefilename = inputs.audio_vae_filename;
|
||||
std::string t5xxl_filename = inputs.t5xxl_filename;
|
||||
std::string clip1_filename = inputs.clip1_filename;
|
||||
std::string clip2_filename = inputs.clip2_filename;
|
||||
|
|
@ -394,6 +396,10 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
|
|||
{
|
||||
printf("With Custom VAE: %s\n",vaefilename.c_str());
|
||||
}
|
||||
if(audiovaefilename!="")
|
||||
{
|
||||
printf("With Audio VAE: %s\n",audiovaefilename.c_str());
|
||||
}
|
||||
if(t5xxl_filename!="")
|
||||
{
|
||||
printf("With Custom T5-XXL Model: %s\n",t5xxl_filename.c_str());
|
||||
|
|
@ -457,6 +463,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
|
|||
sd_params->diffusion_conv_direct = inputs.diffusion_conv_direct;
|
||||
sd_params->vae_conv_direct = inputs.vae_conv_direct;
|
||||
sd_params->vae_path = vaefilename;
|
||||
sd_params->audio_vae_path = audiovaefilename;
|
||||
sd_params->taesd_path = taesdpath;
|
||||
sd_params->t5xxl_path = t5xxl_filename;
|
||||
sd_params->clip_l_path = clip1_filename;
|
||||
|
|
@ -490,6 +497,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
|
|||
params.t5xxl_path = sd_params->t5xxl_path.c_str();
|
||||
params.diffusion_model_path = sd_params->diffusion_model_path.c_str();
|
||||
params.vae_path = sd_params->vae_path.c_str();
|
||||
params.audio_vae_path = sd_params->audio_vae_path.c_str();
|
||||
params.taesd_path = sd_params->taesd_path.c_str();
|
||||
params.photo_maker_path = sd_params->stacked_id_embeddings_path.c_str();
|
||||
|
||||
|
|
|
|||
|
|
@ -277,13 +277,14 @@ public:
|
|||
}
|
||||
max_vram = sd::ggml_graph_cut::resolve_max_vram_gib(max_vram, backend_for(SDBackendModule::DIFFUSION));
|
||||
|
||||
std::string clip_vision_fixed = SAFE_STR(sd_ctx_params->clip_vision_path);
|
||||
std::string clipg_path_fixed = SAFE_STR(sd_ctx_params->clip_g_path);
|
||||
std::string clipl_path_fixed = SAFE_STR(sd_ctx_params->clip_l_path);
|
||||
std::string llm_path_fixed = SAFE_STR(sd_ctx_params->llm_path);
|
||||
std::string clip_vision_fixed = SAFE_STR(sd_ctx_params->clip_vision_path);
|
||||
std::string clipg_path_fixed = SAFE_STR(sd_ctx_params->clip_g_path);
|
||||
std::string clipl_path_fixed = SAFE_STR(sd_ctx_params->clip_l_path);
|
||||
std::string llm_path_fixed = SAFE_STR(sd_ctx_params->llm_path);
|
||||
std::string llm_vision_path_fixed = SAFE_STR(sd_ctx_params->llm_vision_path);
|
||||
std::string t5_path_fixed = SAFE_STR(sd_ctx_params->t5xxl_path);
|
||||
std::string taesd_path_fixed = SAFE_STR(sd_ctx_params->taesd_path);
|
||||
std::string t5_path_fixed = SAFE_STR(sd_ctx_params->t5xxl_path);
|
||||
std::string taesd_path_fixed = SAFE_STR(sd_ctx_params->taesd_path);
|
||||
std::string embed_connector_fixed = SAFE_STR(sd_ctx_params->embeddings_connectors_path);
|
||||
|
||||
ModelLoader model_loader;
|
||||
|
||||
|
|
@ -353,7 +354,8 @@ public:
|
|||
bool is_ernie = sd_version_is_ernie_image(tempver);
|
||||
bool is_longcat = sd_version_is_longcat(tempver);
|
||||
bool is_lens = sd_version_is_lens(tempver);
|
||||
bool conditioner_is_llm = (is_qwenimg || iszimg || isflux2 || is_ovis || is_anima || is_ernie || is_longcat || is_lens);
|
||||
bool is_ltx = sd_version_is_ltxav(tempver);
|
||||
bool conditioner_is_llm = (is_qwenimg || iszimg || isflux2 || is_ovis || is_anima || is_ernie || is_longcat || is_lens || is_ltx);
|
||||
|
||||
//kcpp qol fallback: if a llm was loaded as t5 by mistake
|
||||
if(conditioner_is_llm && t5_path_fixed!="")
|
||||
|
|
@ -366,7 +368,7 @@ public:
|
|||
else if(clipl_path_fixed!="" && clipg_path_fixed=="")
|
||||
{
|
||||
//very tricky case. see if we can tell if clipl is an mmproj, if so move to right place
|
||||
if(toLowerCase(clipl_path_fixed).find("mmproj") != std::string::npos)
|
||||
if(toLowerCase(clipl_path_fixed).find("mmproj") != std::string::npos || is_ltx)
|
||||
{
|
||||
clipg_path_fixed = clipl_path_fixed;
|
||||
clipl_path_fixed = t5_path_fixed;
|
||||
|
|
@ -409,6 +411,11 @@ public:
|
|||
llm_vision_path_fixed = clipg_path_fixed;
|
||||
clipg_path_fixed = "";
|
||||
}
|
||||
else if(is_ltx)
|
||||
{
|
||||
embed_connector_fixed = clipg_path_fixed;
|
||||
clipg_path_fixed = "";
|
||||
}
|
||||
}
|
||||
|
||||
//settle possible inversions for mmproj
|
||||
|
|
@ -476,13 +483,14 @@ public:
|
|||
sd_ctx_params->photo_maker_path = "";
|
||||
}
|
||||
|
||||
sd_ctx_params->clip_g_path = clipg_path_fixed.c_str();
|
||||
sd_ctx_params->clip_l_path = clipl_path_fixed.c_str();
|
||||
sd_ctx_params->clip_vision_path = clip_vision_fixed.c_str();
|
||||
sd_ctx_params->llm_path = llm_path_fixed.c_str();
|
||||
sd_ctx_params->llm_vision_path = llm_vision_path_fixed.c_str();
|
||||
sd_ctx_params->t5xxl_path = t5_path_fixed.c_str();
|
||||
sd_ctx_params->taesd_path = taesd_path_fixed.c_str();
|
||||
sd_ctx_params->clip_g_path = clipg_path_fixed.c_str();
|
||||
sd_ctx_params->clip_l_path = clipl_path_fixed.c_str();
|
||||
sd_ctx_params->clip_vision_path = clip_vision_fixed.c_str();
|
||||
sd_ctx_params->llm_path = llm_path_fixed.c_str();
|
||||
sd_ctx_params->llm_vision_path = llm_vision_path_fixed.c_str();
|
||||
sd_ctx_params->t5xxl_path = t5_path_fixed.c_str();
|
||||
sd_ctx_params->taesd_path = taesd_path_fixed.c_str();
|
||||
sd_ctx_params->embeddings_connectors_path = embed_connector_fixed.c_str();
|
||||
//debug print
|
||||
// printf("\n\nclip_g: %s\nclip_l: %s\nclip_vision: %s\nllm: %s\nllm_vision: %s\nt5xxl: %s\ntaesd: %s\n",
|
||||
// sd_ctx_params->clip_g_path, sd_ctx_params->clip_l_path, sd_ctx_params->clip_vision_path,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue