vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add new sampling algorithm mirostat

Browse source
This commit is contained in:
Hendrik Langer 2023-05-05 13:23:47 +02:00
parent c8f7eeb7fd
commit 8131bc8b56
3 changed files with 20 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

14
koboldcpp.py
View file

@ -38,6 +38,9 @@ class generation_inputs(ctypes.Structure):
("tfs", ctypes.c_float),
("rep_pen", ctypes.c_float),
("rep_pen_range", ctypes.c_int),
("mirostat", ctypes.c_int),
("mirostat_tau", ctypes.c_float),
("mirostat_eta", ctypes.c_float),
("stop_sequence", ctypes.c_char_p * stop_token_max)]
class generation_outputs(ctypes.Structure):
@ -152,7 +155,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]):
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,mirostat=0,mirostat_lr=0.1,mirostat_ent=5.0,seed=-1,stop_sequence=[]):
inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
inputs.prompt = prompt.encode("UTF-8")
@ -165,6 +168,9 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=
inputs.tfs = tfs
inputs.rep_pen = rep_pen
inputs.rep_pen_range = rep_pen_range
inputs.mirostat = mirostat
inputs.mirostat_eta = mirostat_lr
inputs.mirostat_tau = mirostat_ent
inputs.seed = seed
for n in range(0,stop_token_max):
if not stop_sequence or n >= len(stop_sequence):
@ -309,6 +315,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
tfs=genparams.get('tfs', 1.0),
rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128),
mirostat=genparams.get('mirostat', 0),
mirostat_lr=genparams.get('mirostat_lr', 0.1),
mirostat_ent=genparams.get('mirostat_ent', 5.0),
seed=-1,
stop_sequence=genparams.get('stop_sequence', [])
)
@ -325,6 +334,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
tfs=genparams.get('tfs', 1.0),
rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128),
mirostat=genparams.get('mirostat', 0),
mirostat_lr=genparams.get('mirostat_lr', 0.1),
mirostat_ent=genparams.get('mirostat_ent', 5.0),
seed=-1,
stop_sequence=genparams.get('stop_sequence', [])
)