vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

support running in single process mode without admin flag

Browse source
This commit is contained in:
Concedo 2025-02-09 13:17:14 +08:00
parent fb87ca5cec
commit 1cb42bf260
1 changed files with 61 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

24
koboldcpp.py
View file

@ -47,10 +47,10 @@ logit_bias_max = 512
dry_seq_break_max = 128 dry_seq_break_max = 128
# global vars # global vars
KcppVersion = "1.83" KcppVersion = "1.83.1"
showdebug = True showdebug = True
kcpp_instance = None #global running instance kcpp_instance = None #global running instance
global_memory = None global_memory = {"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False}
using_gui_launcher = False using_gui_launcher = False
handle = None handle = None
@ -4685,6 +4685,7 @@ def setuptunnel(global_memory, has_sd):
found = re.findall(pattern, line) found = re.findall(pattern, line)
for x in found: for x in found:
tunneloutput = x tunneloutput = x
if global_memory["load_complete"]:
print(f"Your remote Kobold API can be found at {tunneloutput}/api") print(f"Your remote Kobold API can be found at {tunneloutput}/api")
print(f"Your remote OpenAI Compatible API can be found at {tunneloutput}/v1") print(f"Your remote OpenAI Compatible API can be found at {tunneloutput}/v1")
if has_sd: if has_sd:
@ -4930,7 +4931,7 @@ def analyze_gguf_model_wrapper(filename=""):
dumpthread.start() dumpthread.start()
def main(launch_args): def main(launch_args):
global args, showdebug, kcpp_instance, exitcounter, using_gui_launcher, sslvalid global args, showdebug, kcpp_instance, exitcounter, using_gui_launcher, sslvalid, global_memory
args = launch_args #note: these are NOT shared with the child processes! args = launch_args #note: these are NOT shared with the child processes!
if (args.version) and len(sys.argv) <= 2: if (args.version) and len(sys.argv) <= 2:
@ -5005,9 +5006,17 @@ def main(launch_args):
if len(args.ssl)==2 and isinstance(args.ssl[0], str) and os.path.exists(args.ssl[0]) and isinstance(args.ssl[1], str) and os.path.exists(args.ssl[1]): if len(args.ssl)==2 and isinstance(args.ssl[0], str) and os.path.exists(args.ssl[0]) and isinstance(args.ssl[1], str) and os.path.exists(args.ssl[1]):
sslvalid = True sslvalid = True
# manager command queue if not (args.admin and args.admindir): #run in single process mode
if args.remotetunnel and not args.prompt and not args.benchmark:
setuptunnel(None, True if args.sdmodel else False)
kcpp_main_process(args,global_memory,using_gui_launcher)
if global_memory["input_to_exit"]:
print("===")
print("Press ENTER key to exit.", flush=True)
input()
else: # manager command queue for admin mode
with multiprocessing.Manager() as mp_manager: with multiprocessing.Manager() as mp_manager:
global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False}) global_memory = mp_manager.dict({"tunnel_url": "", "restart_target":"", "input_to_exit":False, "load_complete":False})
if args.remotetunnel and not args.prompt and not args.benchmark: if args.remotetunnel and not args.prompt and not args.benchmark:
setuptunnel(global_memory, True if args.sdmodel else False) setuptunnel(global_memory, True if args.sdmodel else False)
@ -5032,6 +5041,7 @@ def main(launch_args):
targetfilepath = os.path.join(dirpath, restart_target) targetfilepath = os.path.join(dirpath, restart_target)
if os.path.exists(targetfilepath): if os.path.exists(targetfilepath):
print("Terminating old process...") print("Terminating old process...")
global_memory["load_complete"] = False
kcpp_instance.terminate() kcpp_instance.terminate()
kcpp_instance.join(timeout=10) # Ensure process is stopped kcpp_instance.join(timeout=10) # Ensure process is stopped
kcpp_instance = None kcpp_instance = None
@ -5546,7 +5556,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
print(f"Your remote OpenAI Compatible API can be found at {endpoint_url}/v1") print(f"Your remote OpenAI Compatible API can be found at {endpoint_url}/v1")
if args.sdmodel: if args.sdmodel:
print(f"StableUI is available at {endpoint_url}/sdui/") print(f"StableUI is available at {endpoint_url}/sdui/")
global_memory["load_complete"] = True
if args.launch: if args.launch:
LaunchWebbrowser(endpoint_url,"--launch was set, but could not launch web browser automatically.") LaunchWebbrowser(endpoint_url,"--launch was set, but could not launch web browser automatically.")
@ -5641,7 +5651,7 @@ def kcpp_main_process(launch_args, g_memory=None, gui_launcher=False):
file.write(f"\n{datetimestamp},{libname},{args.gpulayers},{benchmodel},{benchmaxctx},{benchlen},{t_pp:.2f},{s_pp:.2f},{t_gen:.2f},{s_gen:.2f},{(t_pp+t_gen):.2f},{result},{benchflagstr}") file.write(f"\n{datetimestamp},{libname},{args.gpulayers},{benchmodel},{benchmaxctx},{benchlen},{t_pp:.2f},{s_pp:.2f},{t_gen:.2f},{s_gen:.2f},{(t_pp+t_gen):.2f},{result},{benchflagstr}")
except Exception as e: except Exception as e:
print(f"Error writing benchmark to file: {e}") print(f"Error writing benchmark to file: {e}")
if using_gui_launcher and not save_to_file: if global_memory and using_gui_launcher and not save_to_file:
global_memory["input_to_exit"] = True global_memory["input_to_exit"] = True
time.sleep(1) time.sleep(1)