vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added top_a sampler

Browse source
This commit is contained in:
Concedo 2023-05-27 17:32:37 +08:00
parent 01a0f206df
commit abfdfb702e
3 changed files with 43 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

10
koboldcpp.py
View file

@ -35,6 +35,7 @@ class generation_inputs(ctypes.Structure):
("max_length", ctypes.c_int),
("temperature", ctypes.c_float),
("top_k", ctypes.c_int),
("top_a", ctypes.c_float),
("top_p", ctypes.c_float),
("typical_p", ctypes.c_float),
("tfs", ctypes.c_float),
@ -161,7 +162,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]):
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=300, top_a=0.0 ,top_p=0.85, typical_p=1.0, tfs=1.0 ,rep_pen=1.1,rep_pen_range=128,seed=-1,stop_sequence=[]):
inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
inputs.prompt = prompt.encode("UTF-8")
@ -169,6 +170,7 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=
inputs.max_length = max_length
inputs.temperature = temperature
inputs.top_k = top_k
inputs.top_a = top_a
inputs.top_p = top_p
inputs.typical_p = typical_p
inputs.tfs = tfs
@ -326,7 +328,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
max_context_length=genparams.get('max_context_length', maxctx),
max_length=genparams.get('max_length', 50),
temperature=genparams.get('temperature', 0.8),
top_k=genparams.get('top_k', 200),
top_k=genparams.get('top_k', 300),
top_a=genparams.get('top_a', 0.0),
top_p=genparams.get('top_p', 0.85),
typical_p=genparams.get('typical', 1.0),
tfs=genparams.get('tfs', 1.0),
@ -342,7 +345,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
prompt=newprompt,
max_length=genparams.get('max', 50),
temperature=genparams.get('temperature', 0.8),
top_k=genparams.get('top_k', 200),
top_k=genparams.get('top_k', 300),
top_a=genparams.get('top_a', 0.0),
top_p=genparams.get('top_p', 0.85),
typical_p=genparams.get('typical', 1.0),
tfs=genparams.get('tfs', 1.0),