mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-10 17:14:36 +00:00
support running in single process mode without admin flag
This commit is contained in:
Concedo
parent
fb87ca5cec
commit
1cb42bf260
1 changed files with 61 additions and 51 deletions
112
koboldcpp.py
112
koboldcpp.py
|
|
@ -47,10 +47,10 @@ logit_bias_max = 512
|
|||
dry_seq_break_max = 128
|
||||
|
||||
# global vars
|
||||
KcppVersion = "1.83"
|
||||
KcppVersion = "1.83.1"
|
||||
showdebug = True
|
||||
kcpp_instance = None #global running instance
|
||||
global_memory = None
|
||||
global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False}
|
||||
using_gui_launcher = False
|
||||
|
||||
handle = None
|
||||
|
|
@ -4685,12 +4685,13 @@ def setuptunnel(global_memory, has_sd):
|
|||
found = re.findall(pattern, line)
|
||||
for x in found:
|
||||
tunneloutput = x
|
||||
print(f"Your remote Kobold API can be found at {tunneloutput}/api")
|
||||
print(f"Your remote OpenAI Compatible API can be found at {tunneloutput}/v1")
|
||||
if has_sd:
|
||||
print(f"StableUI is available at {tunneloutput}/sdui/")
|
||||
print("======\n")
|
||||
print(f"Your remote tunnel is ready, please connect to {tunneloutput}", flush=True)
|
||||
if global_memory["load_complete"]:
|
||||
print(f"Your remote Kobold API can be found at {tunneloutput}/api")
|
||||
print(f"Your remote OpenAI Compatible API can be found at {tunneloutput}/v1")
|
||||
if has_sd:
|
||||
print(f"StableUI is available at {tunneloutput}/sdui/")
|
||||
print("======\n")
|
||||
print(f"Your remote tunnel is ready, please connect to {tunneloutput}", flush=True)
|
||||
if global_memory:
|
||||
global_memory["tunnel_url"] = tunneloutput
|
||||
return
|
||||
|
|
@ -4930,7 +4931,7 @@ def analyze_gguf_model_wrapper(filename=""):
|
|||
dumpthread.start()
|
||||
|
||||
def main(launch_args):
|
||||
global args, showdebug, kcpp_instance, exitcounter, using_gui_launcher, sslvalid
|
||||
global args, showdebug, kcpp_instance, exitcounter, using_gui_launcher, sslvalid, global_memory
|
||||
args = launch_args #note: these are NOT shared with the child processes!
|
||||
|
||||
if (args.version) and len(sys.argv) <= 2:
|
||||
|
|
@ -5005,51 +5006,60 @@ def main(launch_args):
|
|||
if len(args.ssl)==2 and isinstance(args.ssl[0], str) and os.path.exists(args.ssl[0]) and isinstance(args.ssl[1], str) and os.path.exists(args.ssl[1]):
|
||||
sslvalid = True
|
||||
|
||||
# manager command queue
|
||||
with multiprocessing.Manager() as mp_manager:
|
||||
global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False})
|
||||
|
||||
if not (args.admin and args.admindir): #run in single process mode
|
||||
if args.remotetunnel and not args.prompt and not args.benchmark:
|
||||
setuptunnel(global_memory, True if args.sdmodel else False)
|
||||
|
||||
# invoke the main koboldcpp process
|
||||
kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": using_gui_launcher})
|
||||
kcpp_instance.daemon = True
|
||||
kcpp_instance.start()
|
||||
|
||||
while True: # keep the manager alive
|
||||
try:
|
||||
restart_target = ""
|
||||
if not kcpp_instance or not kcpp_instance.is_alive():
|
||||
break
|
||||
restart_target = global_memory["restart_target"]
|
||||
if restart_target!="":
|
||||
print(f"Reloading new config: {restart_target}")
|
||||
global_memory["restart_target"] = ""
|
||||
time.sleep(0.5) #sleep for 0.5s then restart
|
||||
if args.admin and args.admindir:
|
||||
dirpath = os.path.abspath(args.admindir)
|
||||
targetfilepath = os.path.join(dirpath, restart_target)
|
||||
if os.path.exists(targetfilepath):
|
||||
print("Terminating old process...")
|
||||
kcpp_instance.terminate()
|
||||
kcpp_instance.join(timeout=10) # Ensure process is stopped
|
||||
kcpp_instance = None
|
||||
print("Restarting KoboldCpp...")
|
||||
reload_new_config(targetfilepath)
|
||||
kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": using_gui_launcher})
|
||||
kcpp_instance.daemon = True
|
||||
kcpp_instance.start()
|
||||
global_memory["restart_target"] = ""
|
||||
time.sleep(1)
|
||||
else:
|
||||
time.sleep(0.2)
|
||||
except (KeyboardInterrupt,SystemExit):
|
||||
break
|
||||
setuptunnel(None, True if args.sdmodel else False)
|
||||
kcpp_main_process(args,global_memory,using_gui_launcher)
|
||||
if global_memory["input_to_exit"]:
|
||||
print("===")
|
||||
print("Press ENTER key to exit.", flush=True)
|
||||
input()
|
||||
else: # manager command queue for admin mode
|
||||
with multiprocessing.Manager() as mp_manager:
|
||||
global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False})
|
||||
|
||||
if args.remotetunnel and not args.prompt and not args.benchmark:
|
||||
setuptunnel(global_memory, True if args.sdmodel else False)
|
||||
|
||||
# invoke the main koboldcpp process
|
||||
kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": using_gui_launcher})
|
||||
kcpp_instance.daemon = True
|
||||
kcpp_instance.start()
|
||||
|
||||
while True: # keep the manager alive
|
||||
try:
|
||||
restart_target = ""
|
||||
if not kcpp_instance or not kcpp_instance.is_alive():
|
||||
break
|
||||
restart_target = global_memory["restart_target"]
|
||||
if restart_target!="":
|
||||
print(f"Reloading new config: {restart_target}")
|
||||
global_memory["restart_target"] = ""
|
||||
time.sleep(0.5) #sleep for 0.5s then restart
|
||||
if args.admin and args.admindir:
|
||||
dirpath = os.path.abspath(args.admindir)
|
||||
targetfilepath = os.path.join(dirpath, restart_target)
|
||||
if os.path.exists(targetfilepath):
|
||||
print("Terminating old process...")
|
||||
global_memory["load_complete"] = False
|
||||
kcpp_instance.terminate()
|
||||
kcpp_instance.join(timeout=10) # Ensure process is stopped
|
||||
kcpp_instance = None
|
||||
print("Restarting KoboldCpp...")
|
||||
reload_new_config(targetfilepath)
|
||||
kcpp_instance = multiprocessing.Process(target=kcpp_main_process,kwargs={"launch_args": args, "g_memory": global_memory, "gui_launcher": using_gui_launcher})
|
||||
kcpp_instance.daemon = True
|
||||
kcpp_instance.start()
|
||||
global_memory["restart_target"] = ""
|
||||
time.sleep(1)
|
||||
else:
|
||||
time.sleep(0.2)
|
||||
except (KeyboardInterrupt,SystemExit):
|
||||
break
|
||||
if global_memory["input_to_exit"]:
|
||||
print("===")
|
||||
print("Press ENTER key to exit.", flush=True)
|
||||
input()
|
||||
|
||||
def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
||||
global embedded_kailite, embedded_kcpp_docs, embedded_kcpp_sdui, start_time, exitcounter, global_memory, using_gui_launcher
|
||||
|
|
@ -5546,7 +5556,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
|||
print(f"Your remote OpenAI Compatible API can be found at {endpoint_url}/v1")
|
||||
if args.sdmodel:
|
||||
print(f"StableUI is available at {endpoint_url}/sdui/")
|
||||
|
||||
global_memory["load_complete"] = True
|
||||
if args.launch:
|
||||
LaunchWebbrowser(endpoint_url,"--launch was set, but could not launch web browser automatically.")
|
||||
|
||||
|
|
@ -5641,7 +5651,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
|
|||
file.write(f"\n{datetimestamp},{libname},{args.gpulayers},{benchmodel},{benchmaxctx},{benchlen},{t_pp:.2f},{s_pp:.2f},{t_gen:.2f},{s_gen:.2f},{(t_pp+t_gen):.2f},{result},{benchflagstr}")
|
||||
except Exception as e:
|
||||
print(f"Error writing benchmark to file: {e}")
|
||||
if using_gui_launcher and not save_to_file:
|
||||
if global_memory and using_gui_launcher and not save_to_file:
|
||||
global_memory["input_to_exit"] = True
|
||||
time.sleep(1)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue