mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-04-26 10:41:25 +00:00
sd: support changing preloaded LoRA multipliers (#2041)
* sd: remove C++ support for enforcing fixed LoRA multipliers The logic at the Python level is enough. * sd: support changing preloaded LoRA multipliers We keep the same rules as before: - Any LoRA with multiplier 0 can be changed - If all LoRAs have multiplier != 0, they are fixed and optimized but tweak the corner case of LoRAs specified more than once to allow adjusting the multiplier if the same LoRA is also specified with a zero multiplier, as if they were two different LoRAs. So the following keeps working as before: - --sdlora /loras/lcm.gguf --sdloramult 1 : fixed as 1 - --sdlora /loras/lcm.gguf --sdloramult 0 : dynamic, default 0 - --sdlora /loras/ : dynamic, default 0 - --sdlora /loras/lcm.gguf /loras/lcm.gguf --sdloramult 1 1 : fixed as 2 But now we have: - --sdlora /loras/lcm.gguf /loras/lcm.gguf --sdloramult 1 0 : dynamic, default 1 - --sdlora /loras/lcm.gguf /loras/ --sdloramult 1 : dynamic, default 1
This commit is contained in:
Wagner Bruna
GitHub
parent
0c66ed863d
commit
51187d5362
3 changed files with 35 additions and 41 deletions
35
koboldcpp.py
35
koboldcpp.py
|
|
@ -89,8 +89,8 @@ ttsmodelpath = "" #if empty, not initialized
|
|||
embeddingsmodelpath = "" #if empty, not initialized
|
||||
musicllmmodelpath = "" #if empty, not initialized
|
||||
musicdiffusionmodelpath = "" #if empty, not initialized
|
||||
imglora_preload = []
|
||||
imglora_bypath = {}
|
||||
imglora_preload = [] # all preloaded LoRAs
|
||||
imglora_bypath = {} # len(imglora_bypath) == 0 <==> static loras
|
||||
imglora_name2path = {}
|
||||
imglora_cached = True
|
||||
imglora_initial_fixed = True
|
||||
|
|
@ -2070,8 +2070,7 @@ def sd_load_model(model_filename,vae_filename,t5xxl_filename,clip1_filename,clip
|
|||
if imglora_bypath:
|
||||
lora_dynamic = 1 << 3 # accept changes at runtime
|
||||
lora_cache = 1 << 4 if imglora_cached else 0 # cache the preloaded LoRAs
|
||||
lora_fixed = 1 << 5 if imglora_initial_fixed else 0 # do not allow changes to the non-zero preloaded LoRAs
|
||||
lora_apply_mode = lora_dynamic | lora_cache | lora_fixed
|
||||
lora_apply_mode = lora_dynamic | lora_cache
|
||||
inputs.lora_apply_mode = lora_apply_mode
|
||||
|
||||
inputs.img_hard_limit = args.sdclamped
|
||||
|
|
@ -2253,7 +2252,7 @@ def mk_sdapi_lora_list(imglora_bypath):
|
|||
return [
|
||||
{'name': info['name'], 'path': info['path']}
|
||||
for info in imglora_bypath.values()
|
||||
if info['multiplier'] == 0.0 # both preloaded and scanned
|
||||
if not info.get('fixed')
|
||||
]
|
||||
|
||||
def extract_loras_from_prompt(prompt):
|
||||
|
|
@ -8854,8 +8853,12 @@ def mk_lora_info(imgloras, multipliers, mock_filesystem=False):
|
|||
if not mock_filesystem:
|
||||
lora_fullpath = os.path.abspath(lora_fullpath)
|
||||
# dedup paths (e.g. preloaded and on directory)
|
||||
if lora_fullpath in lora_fullmap:
|
||||
lora_fullmap[lora_fullpath]["multiplier"] += multiplier
|
||||
info = lora_fullmap.get(lora_fullpath)
|
||||
if info:
|
||||
info["multiplier"] += multiplier
|
||||
if multiplier == 0.0 and 'fixed' in info:
|
||||
# allow changes if we see this lora again with weight 0
|
||||
del info['fixed']
|
||||
continue
|
||||
lora_name, lora_ext = os.path.splitext(lora_file)
|
||||
# ensure unique names
|
||||
|
|
@ -8867,23 +8870,25 @@ def mk_lora_info(imgloras, multipliers, mock_filesystem=False):
|
|||
unique_lora_names.add(lora_uname)
|
||||
lora_upath = lora_uname + lora_ext
|
||||
lora_entry = {
|
||||
'fullpath': lora_fullpath,
|
||||
'name': lora_uname,
|
||||
'path': lora_upath,
|
||||
'multiplier': multiplier,
|
||||
'preloaded': preloaded,
|
||||
'fullpath': lora_fullpath, # where it is on disk
|
||||
'name': lora_uname, # 'name' in api field and <lora:name:multiplier>
|
||||
'path': lora_upath, # 'path' in api field (relative), + extension
|
||||
'multiplier': multiplier, # preload multiplier
|
||||
}
|
||||
if preloaded:
|
||||
lora_entry['preloaded'] = preloaded
|
||||
if multiplier != 0.0 and imglora_initial_fixed:
|
||||
lora_entry['fixed'] = True
|
||||
lora_fullmap[lora_fullpath] = lora_entry
|
||||
# build the runtime tables
|
||||
preloaded_table = []
|
||||
lora_path_map = {}
|
||||
lora_name_map = {}
|
||||
for lora_entry in lora_fullmap.values():
|
||||
# only map LoRAs that can be changed
|
||||
if not imglora_initial_fixed or lora_entry["multiplier"] == 0.0:
|
||||
if not lora_entry.get("fixed"): # only map LoRAs that can be changed
|
||||
lora_path_map[lora_entry["path"]] = lora_entry
|
||||
lora_name_map[lora_entry["name"]] = lora_entry["path"]
|
||||
if lora_entry["preloaded"]:
|
||||
if lora_entry.get("preloaded"):
|
||||
preloaded_table.append(lora_entry)
|
||||
return preloaded_table, lora_path_map, lora_name_map
|
||||
|
||||
|
|
|
|||
|
|
@ -137,7 +137,6 @@ struct SDParams {
|
|||
|
||||
LoraMap lora_map;
|
||||
bool lora_dynamic = false;
|
||||
bool lora_fixed = false;
|
||||
|
||||
std::string cache_mode;
|
||||
std::string cache_options;
|
||||
|
|
@ -288,17 +287,14 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
|
|||
int lora_apply_mode = LORA_APPLY_AT_RUNTIME;
|
||||
bool lora_dynamic = false;
|
||||
bool lora_cache = false;
|
||||
bool lora_fixed = false;
|
||||
if(inputs.lora_apply_mode >= 0 && inputs.lora_apply_mode <= 2) {
|
||||
lora_apply_mode = inputs.lora_apply_mode;
|
||||
}
|
||||
else {
|
||||
// bit 3: LoRAs can be changed dynamically
|
||||
// bit 4: cache the initial LoRA list in VRAM
|
||||
// bit 5: do not allow multiplier changes for the initial LoRAs
|
||||
lora_dynamic = !!(inputs.lora_apply_mode & (1<<3));
|
||||
lora_cache = lora_dynamic && !!(inputs.lora_apply_mode & (1<<4));
|
||||
lora_fixed = lora_dynamic && !!(inputs.lora_apply_mode & (1<<5));
|
||||
}
|
||||
|
||||
if(lora_map.items.size() > 0)
|
||||
|
|
@ -311,8 +307,7 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
|
|||
printf("With LoRAs in apply mode %s%s%s:\n", lora_apply_mode_name, lora_dynamic_name, lora_cache_name);
|
||||
for(auto lora: lora_map.items)
|
||||
{
|
||||
const char * lora_fixed_name = lora_fixed && lora.second != 0.f ? " (fixed)" : "";
|
||||
printf(" %s at %f power%s\n", lora.first.c_str(), lora.second, lora_fixed_name);
|
||||
printf(" %s at %f power\n", lora.first.c_str(), lora.second);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -402,7 +397,6 @@ bool sdtype_load_model(const sd_load_model_inputs inputs) {
|
|||
sd_params->stacked_id_embeddings_path = photomaker_filename;
|
||||
sd_params->lora_map = lora_map;
|
||||
sd_params->lora_dynamic = lora_dynamic;
|
||||
sd_params->lora_fixed = lora_fixed;
|
||||
//if t5 is set, and model is a gguf, load it as a diffusion model path
|
||||
bool endswithgguf = (sd_params->model_path.rfind(".gguf") == sd_params->model_path.size() - 5);
|
||||
if((sd_params->t5xxl_path!="" || sd_params->clip_l_path!="" || sd_params->clip_g_path!="") && endswithgguf)
|
||||
|
|
@ -1222,12 +1216,9 @@ sd_generation_outputs sdtype_generate(const sd_generation_inputs inputs)
|
|||
LoraMap lora_map = sd_params->lora_map;
|
||||
if (sd_params->lora_dynamic) {
|
||||
for (int i = 0; i < inputs.lora_len; i++) {
|
||||
// check if it was initially fixed
|
||||
std::string path = inputs.lora_filenames[i];
|
||||
float preloaded_mult = sd_params->lora_map.get_mult(path);
|
||||
if (!sd_params->lora_fixed || preloaded_mult == 0.f) {
|
||||
lora_map.add_lora(path, inputs.lora_multipliers[i]);
|
||||
}
|
||||
lora_map.add_lora(path, inputs.lora_multipliers[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ def mk_lora_info(imgloras, multipliers):
|
|||
fake filesystem access
|
||||
fake filesystem access
|
||||
>>> pre
|
||||
[{'fullpath': '/x/lora1.safetensors', 'name': 'lora1', 'path': 'lora1.safetensors', 'multiplier': 1.0, 'preloaded': True}, {'fullpath': '/y/lora2.gguf', 'name': 'lora2', 'path': 'lora2.gguf', 'multiplier': 1.0, 'preloaded': True}]
|
||||
[{'fullpath': '/x/lora1.safetensors', 'name': 'lora1', 'path': 'lora1.safetensors', 'multiplier': 1.0, 'preloaded': True, 'fixed': True}, {'fullpath': '/y/lora2.gguf', 'name': 'lora2', 'path': 'lora2.gguf', 'multiplier': 1.0, 'preloaded': True, 'fixed': True}]
|
||||
>>> path
|
||||
{}
|
||||
>>> name
|
||||
|
|
@ -79,7 +79,7 @@ def mk_lora_info(imgloras, multipliers):
|
|||
fake filesystem access
|
||||
fake filesystem access
|
||||
>>> pre
|
||||
[{'fullpath': '/x/lora1.safetensors', 'name': 'lora1', 'path': 'lora1.safetensors', 'multiplier': 0.3, 'preloaded': True}, {'fullpath': '/y/lora1.safetensors', 'name': 'lora1_2', 'path': 'lora1_2.safetensors', 'multiplier': 0.3, 'preloaded': True}]
|
||||
[{'fullpath': '/x/lora1.safetensors', 'name': 'lora1', 'path': 'lora1.safetensors', 'multiplier': 0.3, 'preloaded': True, 'fixed': True}, {'fullpath': '/y/lora1.safetensors', 'name': 'lora1_2', 'path': 'lora1_2.safetensors', 'multiplier': 0.3, 'preloaded': True, 'fixed': True}]
|
||||
>>> path
|
||||
{}
|
||||
|
||||
|
|
@ -95,14 +95,12 @@ def mk_lora_info(imgloras, multipliers):
|
|||
... 'fullpath': '/lora/dir/lora1_makebelieve.gguf',
|
||||
... 'name': 'lora1_makebelieve',
|
||||
... 'path': 'lora1_makebelieve.gguf',
|
||||
... 'multiplier': 0.0,
|
||||
... 'preloaded': False},
|
||||
... 'multiplier': 0.0},
|
||||
... 'lora2/makebelieve.gguf': {
|
||||
... 'fullpath': '/lora/dir/lora2/makebelieve.gguf',
|
||||
... 'name': 'lora2/makebelieve',
|
||||
... 'path': 'lora2/makebelieve.gguf',
|
||||
... 'multiplier': 0.0,
|
||||
... 'preloaded': False}}
|
||||
... 'multiplier': 0.0}}
|
||||
>>> path == expected
|
||||
True
|
||||
>>> name
|
||||
|
|
@ -185,21 +183,21 @@ def mk_sdapi_lora_list(imglora_bypath):
|
|||
... 'lora_a.safetensors': {'name': 'lora_a', 'path': 'lora_a.safetensors', 'multiplier': 0.0},
|
||||
... 'lora_b.gguf' : {'name': 'lora_b', 'path': 'lora_b.gguf', 'multiplier': 0.0},
|
||||
... 'lora_c.safetensors': {'name': 'lora_c', 'path': 'lora_c.safetensors', 'multiplier': 1.0},
|
||||
... 'lora_d.safetensors': {'name': 'lora_d', 'path': 'lora_d.safetensors', 'multiplier': 1.0, 'fixed': True},
|
||||
... 'chars/waifu.gguf' : {'name': 'chars/waifu', 'path': 'chars/waifu.gguf', 'multiplier': 0.0}
|
||||
... }
|
||||
>>> mk_sdapi_lora_list(imglora_bypath)
|
||||
[{'name': 'lora_a', 'path': 'lora_a.safetensors'}, {'name': 'lora_b', 'path': 'lora_b.gguf'}, {'name': 'chars/waifu', 'path': 'chars/waifu.gguf'}]
|
||||
>>> expected = [
|
||||
... {'name': 'lora_a', 'path': 'lora_a.safetensors'},
|
||||
... {'name': 'lora_b', 'path': 'lora_b.gguf'},
|
||||
... {'name': 'lora_c', 'path': 'lora_c.safetensors'},
|
||||
... {'name': 'chars/waifu', 'path': 'chars/waifu.gguf'}
|
||||
... ]
|
||||
>>> mk_sdapi_lora_list(imglora_bypath) == expected
|
||||
True
|
||||
|
||||
>>> empty_data = {}
|
||||
>>> mk_sdapi_lora_list(empty_data)
|
||||
[]
|
||||
|
||||
>>> mixed_data = {
|
||||
... 'k1': {'name': 'X', 'path': 'p1', 'multiplier': 0.5},
|
||||
... 'k2': {'name': 'Y', 'path': 'p2', 'multiplier': 0.0}
|
||||
... }
|
||||
>>> mk_sdapi_lora_list(mixed_data)
|
||||
[{'name': 'Y', 'path': 'p2'}]
|
||||
'''
|
||||
return koboldcpp.mk_sdapi_lora_list(imglora_bypath)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue