vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 09:34:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added model switching to gguf in admin mode (auto guess layers)

Browse source
This commit is contained in:
Concedo 2025-03-14 19:45:55 +08:00
parent 30cb77a900
commit d7498e7e8a
1 changed files with 25 additions and 15 deletions
Download patch file Download diff file Expand all files Collapse all files

28
koboldcpp.py
View file

@ -2428,7 +2428,7 @@ Enter Prompt:<br>
opts = [] opts = []
if args.admin and args.admindir and os.path.exists(args.admindir) and self.check_header_password(args.adminpassword): if args.admin and args.admindir and os.path.exists(args.admindir) and self.check_header_password(args.adminpassword):
dirpath = os.path.abspath(args.admindir) dirpath = os.path.abspath(args.admindir)
opts = [f for f in sorted(os.listdir(dirpath)) if (f.endswith(".kcpps") or f.endswith(".kcppt")) and os.path.isfile(os.path.join(dirpath, f))] opts = [f for f in sorted(os.listdir(dirpath)) if (f.endswith(".kcpps") or f.endswith(".kcppt") or f.endswith(".gguf")) and os.path.isfile(os.path.join(dirpath, f))]
response_body = (json.dumps(opts).encode()) response_body = (json.dumps(opts).encode())
elif self.path.endswith(('/api/extra/perf')): elif self.path.endswith(('/api/extra/perf')):
@ -2882,7 +2882,7 @@ Enter Prompt:<br>
if targetfile and targetfile!="": if targetfile and targetfile!="":
dirpath = os.path.abspath(args.admindir) dirpath = os.path.abspath(args.admindir)
targetfilepath = os.path.join(dirpath, targetfile) targetfilepath = os.path.join(dirpath, targetfile)
opts = [f for f in os.listdir(dirpath) if (f.endswith(".kcpps") or f.endswith(".kcppt")) and os.path.isfile(os.path.join(dirpath, f))] opts = [f for f in os.listdir(dirpath) if (f.endswith(".kcpps") or f.endswith(".kcppt") or f.endswith(".gguf")) and os.path.isfile(os.path.join(dirpath, f))]
if targetfile in opts and os.path.exists(targetfilepath): if targetfile in opts and os.path.exists(targetfilepath):
print(f"Admin: Received request to reload config to {targetfile}") print(f"Admin: Received request to reload config to {targetfile}")
global_memory["restart_target"] = targetfile global_memory["restart_target"] = targetfile
@ -4882,12 +4882,10 @@ def unload_libs():
del handle del handle
handle = None handle = None
def reload_new_config(filename): #for changing config after launch def reload_from_new_args(newargs):
with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
try: try:
config = json.load(f)
args.istemplate = False args.istemplate = False
for key, value in config.items(): #do not overwrite certain values for key, value in newargs.items(): #do not overwrite certain values
if key not in ["remotetunnel","showgui","port","host","port_param","admin","adminpassword","admindir","ssl","nocertify","benchmark","prompt","config"]: if key not in ["remotetunnel","showgui","port","host","port_param","admin","adminpassword","admindir","ssl","nocertify","benchmark","prompt","config"]:
setattr(args, key, value) setattr(args, key, value)
setattr(args,"showgui",False) setattr(args,"showgui",False)
@ -4898,6 +4896,14 @@ def reload_new_config(filename): #for changing config after launch
except Exception as e: except Exception as e:
print(f"Reload New Config Failed: {e}") print(f"Reload New Config Failed: {e}")
def reload_new_config(filename): #for changing config after launch
with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
try:
config = json.load(f)
reload_from_new_args(config)
except Exception as e:
print(f"Reload New Config Failed: {e}")
def load_config_cli(filename): def load_config_cli(filename):
print("Loading .kcpps configuration file...") print("Loading .kcpps configuration file...")
with open(filename, 'r', encoding='utf-8', errors='ignore') as f: with open(filename, 'r', encoding='utf-8', errors='ignore') as f:
@ -5078,7 +5084,7 @@ def analyze_gguf_model_wrapper(filename=""):
dumpthread = threading.Thread(target=analyze_gguf_model, args=(args,filename)) dumpthread = threading.Thread(target=analyze_gguf_model, args=(args,filename))
dumpthread.start() dumpthread.start()
def main(launch_args): def main(launch_args, default_args):
global args, showdebug, kcpp_instance, exitcounter, using_gui_launcher, sslvalid, global_memory global args, showdebug, kcpp_instance, exitcounter, using_gui_launcher, sslvalid, global_memory
args = launch_args #note: these are NOT shared with the child processes! args = launch_args #note: these are NOT shared with the child processes!
@ -5216,7 +5222,7 @@ def main(launch_args):
fault_recovery_mode = False fault_recovery_mode = False
restart_target = global_memory["restart_target"] restart_target = global_memory["restart_target"]
if restart_target!="": if restart_target!="":
print(f"Reloading new config: {restart_target}") print(f"Reloading new model/config: {restart_target}")
global_memory["restart_target"] = "" global_memory["restart_target"] = ""
time.sleep(0.5) #sleep for 0.5s then restart time.sleep(0.5) #sleep for 0.5s then restart
if args.admin and args.admindir: if args.admin and args.admindir:
@ -5230,6 +5236,10 @@ def main(launch_args):
kcpp_instance = None kcpp_instance = None
print("Restarting KoboldCpp...") print("Restarting KoboldCpp...")
fault_recovery_mode = True fault_recovery_mode = True
if targetfilepath.endswith(".gguf"):
reload_from_new_args(vars(default_args))
args.model_param = targetfilepath
else:
reload_new_config(targetfilepath) reload_new_config(targetfilepath)
kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": False}) kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": False})
kcpp_instance.daemon = True kcpp_instance.daemon = True
@ -6018,4 +6028,4 @@ if __name__ == '__main__':
compatgroup.add_argument("--noblas", help=argparse.SUPPRESS, action='store_true') compatgroup.add_argument("--noblas", help=argparse.SUPPRESS, action='store_true')
compatgroup3.add_argument("--nommap", help=argparse.SUPPRESS, action='store_true') compatgroup3.add_argument("--nommap", help=argparse.SUPPRESS, action='store_true')
main(parser.parse_args()) main(launch_args=parser.parse_args(),default_args=parser.parse_args([]))