vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-12 09:59:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

added ability to fast forward in time through partially duplicated prompts

Browse source
This commit is contained in:
Concedo 2023-03-24 18:50:16 +08:00
parent 1166fda943
commit 706e19e9b4
3 changed files with 53 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

24
llama_for_kobold.py
View file

@ -23,8 +23,7 @@ class generation_inputs(ctypes.Structure):
("top_k", ctypes.c_int),
("top_p", ctypes.c_float),
("rep_pen", ctypes.c_float),
("rep_pen_range", ctypes.c_int),
("reset_state", ctypes.c_bool)]
("rep_pen_range", ctypes.c_int)]
class generation_outputs(ctypes.Structure):
_fields_ = [("status", ctypes.c_int),
@ -48,7 +47,7 @@ def load_model(model_filename,batch_size=8,max_context_length=512,n_parts_overwr
ret = handle.load_model(inputs)
return ret
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1,reset_state=True):
def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=100,top_p=0.85,rep_pen=1.1,rep_pen_range=128,seed=-1):
inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
inputs.prompt = prompt.encode("UTF-8")
@ -60,7 +59,6 @@ def generate(prompt,max_length=20, max_context_length=512,temperature=0.8,top_k=
inputs.rep_pen = rep_pen
inputs.rep_pen_range = rep_pen_range
inputs.seed = seed
inputs.reset_state = reset_state
ret = handle.generate(inputs,outputs)
if(ret.status==1):
return ret.text.decode("UTF-8")
@ -80,7 +78,6 @@ maxctx = 2048
maxlen = 128
modelbusy = False
port = 5001
last_context = ""
embedded_kailite = None
class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
@ -130,7 +127,6 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
def do_POST(self):
global modelbusy
global last_context
content_length = int(self.headers['Content-Length'])
body = self.rfile.read(content_length)
@ -159,18 +155,14 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
self.end_headers()
return
print("\nInput: " + json.dumps(genparams))
fresh_state = True
modelbusy = True
if kai_api_flag:
fullprompt = genparams.get('prompt', "")
else:
fullprompt = genparams.get('text', "")
newprompt = fullprompt
if last_context!="" and newprompt.startswith(last_context):
fresh_state = False
newprompt = newprompt[len(last_context):]
print("Resuming state, new input len: " + str(len(newprompt)))
recvtxt = ""
if kai_api_flag:
@ -183,11 +175,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
top_p=genparams.get('top_p', 0.85),
rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128),
seed=-1,
reset_state=fresh_state
seed=-1
)
print("\nOutput: " + recvtxt)
last_context = fullprompt + recvtxt
res = {"results": [{"text": recvtxt}]}
self.send_response(200)
self.end_headers()
@ -201,11 +191,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
top_p=genparams.get('top_p', 0.85),
rep_pen=genparams.get('rep_pen', 1.1),
rep_pen_range=genparams.get('rep_pen_range', 128),
seed=-1,
reset_state=fresh_state
seed=-1
)
print("\nOutput: " + recvtxt)
last_context = fullprompt + recvtxt
res = {"data": {"seqs":[recvtxt]}}
self.send_response(200)
self.end_headers()