vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

default to 12 tokens drafted

Browse source
This commit is contained in:
Concedo 2024-11-30 11:52:07 +08:00
parent b21d0fe3ac
commit e0c59486ee
3 changed files with 9 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

6
koboldcpp.py
View file

@ -2764,7 +2764,7 @@ def show_gui():
preloadstory_var = ctk.StringVar()
mmproj_var = ctk.StringVar()
draftmodel_var = ctk.StringVar()
draftamount_var = ctk.StringVar(value="16")
draftamount_var = ctk.StringVar(value="12")
nomodel = ctk.IntVar(value=0)
port_var = ctk.StringVar(value=defaultport)
@ -3508,7 +3508,7 @@ def show_gui():
pass
args.mmproj = None if mmproj_var.get() == "" else mmproj_var.get()
args.draftmodel = None if draftmodel_var.get() == "" else draftmodel_var.get()
args.draftamount = int(draftamount_var.get()) if draftamount_var.get()!="" else 16
args.draftamount = int(draftamount_var.get()) if draftamount_var.get()!="" else 12
args.ssl = None if (ssl_cert_var.get() == "" or ssl_key_var.get() == "") else ([ssl_cert_var.get(), ssl_key_var.get()])
args.password = None if (password_var.get() == "") else (password_var.get())
@ -4951,7 +4951,7 @@ if __name__ == '__main__':
advparser.add_argument("--nocertify", help="Allows insecure SSL connections. Use this if you have cert errors and need to bypass certificate restrictions.", action='store_true')
advparser.add_argument("--mmproj", help="Select a multimodal projector file for vision models like LLaVA.", default="")
advparser.add_argument("--draftmodel", help="Load a small draft model for speculative decoding. It will be fully offloaded. Vocab must match the main model.", default="")
advparser.add_argument("--draftamount", metavar=('[tokens]'), help="How many tokens to draft per chunk before verifying results", type=int, default=16)
advparser.add_argument("--draftamount", metavar=('[tokens]'), help="How many tokens to draft per chunk before verifying results", type=int, default=12)
advparser.add_argument("--password", help="Enter a password required to use this instance. This key will be required for all text endpoints. Image endpoints are not secured.", default=None)
advparser.add_argument("--ignoremissing", help="Ignores all missing non-essential files, just skipping them instead.", action='store_true')
advparser.add_argument("--chatcompletionsadapter", help="Select an optional ChatCompletions Adapter JSON file to force custom instruct tags.", default="")