vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-14 02:49:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

still refactoring, but need a checkpoint to prepare build for 1.0.7

Browse source
This commit is contained in:
Concedo 2023-04-01 08:55:14 +08:00
parent 6b86f5ea22
commit 801b178f2a
11 changed files with 452 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

6
llamacpp_for_kobold.py
View file

@ -334,9 +334,9 @@ def main(args):
print(f"Starting Kobold HTTP Server on port {args.port}")
epurl = ""
if args.host=="":
epurl = f"http://localhost:{args.port}" + ("?streaming=1" if not args.nostream else "")
epurl = f"http://localhost:{args.port}" + ("?streaming=1" if args.stream else "")
else:
epurl = f"http://{args.host}:{args.port}?host={args.host}" + ("&streaming=1" if not args.nostream else "")
epurl = f"http://{args.host}:{args.port}?host={args.host}" + ("&streaming=1" if args.stream else "")
print(f"Please connect to custom endpoint at {epurl}")
@ -351,7 +351,7 @@ if __name__ == '__main__':
parser.add_argument("--host", help="Host IP to listen on. If empty, all routable interfaces are accepted.", default="")
default_threads = (os.cpu_count() if os.cpu_count()<=6 else max(6,os.cpu_count()-2))
parser.add_argument("--threads", help="Use a custom number of threads if specified. Otherwise, uses an amount based on CPU cores", type=int, default=default_threads)
parser.add_argument("--nostream", help="Disables pseudo streaming", action='store_true')
parser.add_argument("--stream", help="Uses pseudo streaming", action='store_true')
parser.add_argument("--noblas", help="Do not use OpenBLAS for accelerated prompt ingestion", action='store_true')
args = parser.parse_args()
main(args)