mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2025-09-10 17:14:36 +00:00
added toggle for guidance
This commit is contained in:
Concedo
parent
41142ad67a
commit
f59b5eb561
4 changed files with 54 additions and 21 deletions
48
koboldcpp.py
48
koboldcpp.py
|
|
@ -183,6 +183,7 @@ class load_model_inputs(ctypes.Structure):
|
|||
("rope_freq_base", ctypes.c_float),
|
||||
("moe_experts", ctypes.c_int),
|
||||
("no_bos_token", ctypes.c_bool),
|
||||
("load_guidance", ctypes.c_bool),
|
||||
("override_kv", ctypes.c_char_p),
|
||||
("override_tensors", ctypes.c_char_p),
|
||||
("flash_attention", ctypes.c_bool),
|
||||
|
|
@ -1230,6 +1231,7 @@ def load_model(model_filename):
|
|||
|
||||
inputs.moe_experts = args.moeexperts
|
||||
inputs.no_bos_token = args.nobostoken
|
||||
inputs.load_guidance = args.enableguidance
|
||||
inputs.override_kv = args.overridekv.encode("UTF-8") if args.overridekv else "".encode("UTF-8")
|
||||
inputs.override_tensors = args.overridetensors.encode("UTF-8") if args.overridetensors else "".encode("UTF-8")
|
||||
inputs = set_backend_props(inputs)
|
||||
|
|
@ -1238,21 +1240,23 @@ def load_model(model_filename):
|
|||
|
||||
def generate(genparams, stream_flag=False):
|
||||
global maxctx, args, currentusergenkey, totalgens, pendingabortkey
|
||||
default_adapter = {} if chatcompl_adapter is None else chatcompl_adapter
|
||||
adapter_obj = genparams.get('adapter', default_adapter)
|
||||
|
||||
prompt = genparams.get('prompt', "")
|
||||
memory = genparams.get('memory', "")
|
||||
images = genparams.get('images', [])
|
||||
max_context_length = tryparseint(genparams.get('max_context_length', maxctx),maxctx)
|
||||
max_length = tryparseint(genparams.get('max_length', args.defaultgenamt),args.defaultgenamt)
|
||||
temperature = tryparsefloat(genparams.get('temperature', 0.75),0.75)
|
||||
top_k = tryparseint(genparams.get('top_k', 100),100)
|
||||
temperature = tryparsefloat(genparams.get('temperature', adapter_obj.get("temperature", 0.75)),0.75)
|
||||
top_k = tryparseint(genparams.get('top_k', adapter_obj.get("top_k", 100)),100)
|
||||
top_a = tryparsefloat(genparams.get('top_a', 0.0),0.0)
|
||||
top_p = tryparsefloat(genparams.get('top_p', 0.92),0.92)
|
||||
min_p = tryparsefloat(genparams.get('min_p', 0.0),0.0)
|
||||
top_p = tryparsefloat(genparams.get('top_p', adapter_obj.get("top_p", 0.92)),0.92)
|
||||
min_p = tryparsefloat(genparams.get('min_p', adapter_obj.get("min_p", 0.0)),0.0)
|
||||
typical_p = tryparsefloat(genparams.get('typical', 1.0),1.0)
|
||||
tfs = tryparsefloat(genparams.get('tfs', 1.0),1.0)
|
||||
nsigma = tryparsefloat(genparams.get('nsigma', 0.0),0.0)
|
||||
rep_pen = tryparsefloat(genparams.get('rep_pen', 1.0),1.0)
|
||||
rep_pen = tryparsefloat(genparams.get('rep_pen', adapter_obj.get("rep_pen", 1.0)),1.0)
|
||||
rep_pen_range = tryparseint(genparams.get('rep_pen_range', 320),320)
|
||||
rep_pen_slope = tryparsefloat(genparams.get('rep_pen_slope', 1.0),1.0)
|
||||
presence_penalty = tryparsefloat(genparams.get('presence_penalty', 0.0),0.0)
|
||||
|
|
@ -1268,7 +1272,8 @@ def generate(genparams, stream_flag=False):
|
|||
xtc_probability = tryparsefloat(genparams.get('xtc_probability', 0),0)
|
||||
sampler_order = genparams.get('sampler_order', [6, 0, 1, 3, 4, 2, 5])
|
||||
seed = tryparseint(genparams.get('sampler_seed', -1),-1)
|
||||
stop_sequence = genparams.get('stop_sequence', [])
|
||||
stop_sequence = (genparams.get('stop_sequence', []) if genparams.get('stop_sequence', []) is not None else [])
|
||||
stop_sequence = stop_sequence[:stop_token_max]
|
||||
ban_eos_token = genparams.get('ban_eos_token', False)
|
||||
stream_sse = stream_flag
|
||||
grammar = genparams.get('grammar', '')
|
||||
|
|
@ -1306,6 +1311,11 @@ def generate(genparams, stream_flag=False):
|
|||
memory = memory.replace("{{[INPUT]}}", assistant_message_end + user_message_start)
|
||||
memory = memory.replace("{{[OUTPUT]}}", user_message_end + assistant_message_start)
|
||||
memory = memory.replace("{{[SYSTEM]}}", system_message_start)
|
||||
for i in range(len(stop_sequence)):
|
||||
if stop_sequence[i] == "{{[INPUT]}}":
|
||||
stop_sequence[i] = user_message_start
|
||||
elif stop_sequence[i] == "{{[OUTPUT]}}":
|
||||
stop_sequence[i] = assistant_message_start
|
||||
|
||||
for tok in custom_token_bans.split(','):
|
||||
tok = tok.strip() # Remove leading/trailing whitespace
|
||||
|
|
@ -1402,9 +1412,6 @@ def generate(genparams, stream_flag=False):
|
|||
print("ERROR: sampler_order must be a list of integers: " + str(e))
|
||||
inputs.seed = seed
|
||||
|
||||
if stop_sequence is None:
|
||||
stop_sequence = []
|
||||
stop_sequence = stop_sequence[:stop_token_max]
|
||||
inputs.stop_sequence_len = len(stop_sequence)
|
||||
inputs.stop_sequence = (ctypes.c_char_p * inputs.stop_sequence_len)()
|
||||
|
||||
|
|
@ -3819,7 +3826,7 @@ def show_gui():
|
|||
import customtkinter as ctk
|
||||
nextstate = 0 #0=exit, 1=launch
|
||||
original_windowwidth = 580
|
||||
original_windowheight = 560
|
||||
original_windowheight = 580
|
||||
windowwidth = original_windowwidth
|
||||
windowheight = original_windowheight
|
||||
ctk.set_appearance_mode("dark")
|
||||
|
|
@ -3966,6 +3973,7 @@ def show_gui():
|
|||
nobostoken_var = ctk.IntVar(value=0)
|
||||
override_kv_var = ctk.StringVar(value="")
|
||||
override_tensors_var = ctk.StringVar(value="")
|
||||
enableguidance_var = ctk.IntVar(value=0)
|
||||
|
||||
model_var = ctk.StringVar()
|
||||
lora_var = ctk.StringVar()
|
||||
|
|
@ -4056,11 +4064,11 @@ def show_gui():
|
|||
quick_tab = tabcontent["Quick Launch"]
|
||||
|
||||
# helper functions
|
||||
def makecheckbox(parent, text, variable=None, row=0, column=0, command=None, onvalue=1, offvalue=0,tooltiptxt=""):
|
||||
temp = ctk.CTkCheckBox(parent, text=text,variable=variable, onvalue=onvalue, offvalue=offvalue)
|
||||
def makecheckbox(parent, text, variable=None, row=0, column=0, command=None, padx=8,tooltiptxt=""):
|
||||
temp = ctk.CTkCheckBox(parent, text=text,variable=variable, onvalue=1, offvalue=0)
|
||||
if command is not None and variable is not None:
|
||||
variable.trace("w", command)
|
||||
temp.grid(row=row,column=column, padx=8, pady=1, stick="nw")
|
||||
temp.grid(row=row,column=column, padx=padx, pady=1, stick="nw")
|
||||
if tooltiptxt!="":
|
||||
temp.bind("<Enter>", lambda event: show_tooltip(event, tooltiptxt))
|
||||
temp.bind("<Leave>", hide_tooltip)
|
||||
|
|
@ -4577,16 +4585,17 @@ def show_gui():
|
|||
item.grid_remove()
|
||||
makecheckbox(tokens_tab, "Custom RoPE Config", variable=customrope_var, row=22, command=togglerope,tooltiptxt="Override the default RoPE configuration with custom RoPE scaling.")
|
||||
use_flashattn = makecheckbox(tokens_tab, "Use FlashAttention", flashattention, 28, command=toggleflashattn, tooltiptxt="Enable flash attention for GGUF models.")
|
||||
noqkvlabel = makelabel(tokens_tab,"QuantKV works best with flash attention enabled",33,0,"WARNING: NOT RECOMMENDED.\nOnly K cache can be quantized, and performance can suffer.\nIn some cases, it might even use more VRAM when doing a full offload.")
|
||||
noqkvlabel = makelabel(tokens_tab,"(Note: QuantKV works best with flash attention)",28,0,"Only K cache can be quantized, and performance can suffer.\nIn some cases, it might even use more VRAM when doing a full offload.",padx=160)
|
||||
noqkvlabel.configure(text_color="#ff5555")
|
||||
avoidfalabel = makelabel(tokens_tab,"Flash attention discouraged with Vulkan GPU offload!",35,0,"FlashAttention is discouraged when using Vulkan GPU offload.")
|
||||
avoidfalabel = makelabel(tokens_tab,"(Note: Flash attention may be slow on Vulkan)",28,0,"FlashAttention is discouraged when using Vulkan GPU offload.",padx=160)
|
||||
avoidfalabel.configure(text_color="#ff5555")
|
||||
qkvslider,qkvlabel,qkvtitle = makeslider(tokens_tab, "Quantize KV Cache:", quantkv_text, quantkv_var, 0, 2, 30, set=0,tooltip="Enable quantization of KV cache.\nRequires FlashAttention for full effect, otherwise only K cache is quantized.")
|
||||
quantkv_var.trace("w", toggleflashattn)
|
||||
makecheckbox(tokens_tab, "No BOS Token", nobostoken_var, 43, tooltiptxt="Prevents BOS token from being added at the start of any prompt. Usually NOT recommended for most models.")
|
||||
makelabelentry(tokens_tab, "MoE Experts:", moeexperts_var, row=45, padx=120, singleline=True, tooltip="Override number of MoE experts.")
|
||||
makelabelentry(tokens_tab, "Override KV:", override_kv_var, row=47, padx=120, singleline=True, width=150, tooltip="Advanced option to override model metadata by key, same as in llama.cpp. Mainly for debugging, not intended for general use. Types: int, float, bool, str")
|
||||
makelabelentry(tokens_tab, "Override Tensors:", override_tensors_var, row=49, padx=120, singleline=True, width=150, tooltip="Advanced option to override tensor backend selection, same as in llama.cpp.")
|
||||
makecheckbox(tokens_tab, "Enable Guidance", enableguidance_var, 43,padx=140, tooltiptxt="Enables the use of Classifier-Free-Guidance, which allows the use of negative prompts. Has performance and memory impact.")
|
||||
makelabelentry(tokens_tab, "MoE Experts:", moeexperts_var, row=55, padx=120, singleline=True, tooltip="Override number of MoE experts.")
|
||||
makelabelentry(tokens_tab, "Override KV:", override_kv_var, row=57, padx=120, singleline=True, width=150, tooltip="Advanced option to override model metadata by key, same as in llama.cpp. Mainly for debugging, not intended for general use. Types: int, float, bool, str")
|
||||
makelabelentry(tokens_tab, "Override Tensors:", override_tensors_var, row=59, padx=120, singleline=True, width=150, tooltip="Advanced option to override tensor backend selection, same as in llama.cpp.")
|
||||
|
||||
# Model Tab
|
||||
model_tab = tabcontent["Loaded Files"]
|
||||
|
|
@ -4862,6 +4871,7 @@ def show_gui():
|
|||
args.moeexperts = int(moeexperts_var.get()) if moeexperts_var.get()!="" else -1
|
||||
args.defaultgenamt = int(defaultgenamt_var.get()) if defaultgenamt_var.get()!="" else 512
|
||||
args.nobostoken = (nobostoken_var.get()==1)
|
||||
args.enableguidance = (enableguidance_var.get()==1)
|
||||
args.overridekv = None if override_kv_var.get() == "" else override_kv_var.get()
|
||||
args.overridetensors = None if override_tensors_var.get() == "" else override_tensors_var.get()
|
||||
args.chatcompletionsadapter = None if chatcompletionsadapter_var.get() == "" else chatcompletionsadapter_var.get()
|
||||
|
|
@ -5057,6 +5067,7 @@ def show_gui():
|
|||
if "defaultgenamt" in dict and dict["defaultgenamt"]:
|
||||
defaultgenamt_var.set(dict["defaultgenamt"])
|
||||
nobostoken_var.set(dict["nobostoken"] if ("nobostoken" in dict) else 0)
|
||||
enableguidance_var.set(dict["enableguidance"] if ("enableguidance" in dict) else 0)
|
||||
if "overridekv" in dict and dict["overridekv"]:
|
||||
override_kv_var.set(dict["overridekv"])
|
||||
if "overridetensors" in dict and dict["overridetensors"]:
|
||||
|
|
@ -6801,6 +6812,7 @@ if __name__ == '__main__':
|
|||
advparser.add_argument("--moeexperts", metavar=('[num of experts]'), help="How many experts to use for MoE models (default=follow gguf)", type=int, default=-1)
|
||||
advparser.add_argument("--defaultgenamt", help="How many tokens to generate by default, if not specified. Must be smaller than context size. Usually, your frontend GUI will override this.", type=check_range(int,64,4096), default=512)
|
||||
advparser.add_argument("--nobostoken", help="Prevents BOS token from being added at the start of any prompt. Usually NOT recommended for most models.", action='store_true')
|
||||
advparser.add_argument("--enableguidance", help="Enables the use of Classifier-Free-Guidance, which allows the use of negative prompts. Has performance and memory impact.", action='store_true')
|
||||
advparser.add_argument("--maxrequestsize", metavar=('[size in MB]'), help="Specify a max request payload size. Any requests to the server larger than this size will be dropped. Do not change if unsure.", type=int, default=32)
|
||||
advparser.add_argument("--overridekv", metavar=('[name=type:value]'), help="Advanced option to override a metadata by key, same as in llama.cpp. Mainly for debugging, not intended for general use. Types: int, float, bool, str", default="")
|
||||
advparser.add_argument("--overridetensors", metavar=('[tensor name pattern=buffer type]'), help="Advanced option to override tensor backend selection, same as in llama.cpp.", default="")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue