vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merged, added ability to render special tokens

Browse source
This commit is contained in:
Concedo 2024-04-22 18:19:58 +08:00
commit b4d2031215
37 changed files with 335 additions and 7328 deletions
Download patch file Download diff file Expand all files Collapse all files

11
koboldcpp.py
View file

@ -81,7 +81,8 @@ class generation_inputs(ctypes.Structure):
("mirostat_eta", ctypes.c_float),
("sampler_order", ctypes.c_int * sampler_order_max),
("sampler_len", ctypes.c_int),
("unban_tokens_rt", ctypes.c_bool),
("allow_eos_token", ctypes.c_bool),
("render_special", ctypes.c_bool),
("stop_sequence", ctypes.c_char_p * stop_token_max),
("stream_sse", ctypes.c_bool),
("grammar", ctypes.c_char_p),
@ -398,7 +399,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt, memory="", images=[], max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.0, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, dynatemp_exponent=1.0, smoothing_factor=0.0, logit_biases={}):
def generate(prompt, memory="", images=[], max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.0, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, dynatemp_exponent=1.0, smoothing_factor=0.0, logit_biases={}, render_special=False):
global maxctx, args, currentusergenkey, totalgens, pendingabortkey
inputs = generation_inputs()
inputs.prompt = prompt.encode("UTF-8")
@ -436,7 +437,8 @@ def generate(prompt, memory="", images=[], max_length=32, max_context_length=512
inputs.smoothing_factor = smoothing_factor
inputs.grammar = grammar.encode("UTF-8")
inputs.grammar_retain_state = grammar_retain_state
inputs.unban_tokens_rt = not use_default_badwordsids
inputs.allow_eos_token = not use_default_badwordsids
inputs.render_special = render_special
if mirostat in (1, 2):
inputs.mirostat = mirostat
inputs.mirostat_tau = mirostat_tau
@ -806,7 +808,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
dynatemp_range=genparams.get('dynatemp_range', 0.0),
dynatemp_exponent=genparams.get('dynatemp_exponent', 1.0),
smoothing_factor=genparams.get('smoothing_factor', 0.0),
logit_biases=genparams.get('logit_bias', {})
logit_biases=genparams.get('logit_bias', {}),
render_special=genparams.get('render_special', False),
)
recvtxt = ""