vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 17:14:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Quadratic Sampling UI (#652)

Browse source 
* Quadratic Sampling UI

Kalomaze's Quadratic Sampling, now has a UI within KCPP.

* remove debug prints

* cleanup, add smooth sampler to dynatemp

---------

Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
This commit is contained in:
Alexander Abushady 2024-02-04 03:26:27 -05:00 • committed by GitHub
parent 504300784f
commit 4cb956c7db
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 57 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

5
koboldcpp.py Executable file → Normal file
View file

@ -81,6 +81,7 @@ class generation_inputs(ctypes.Structure):
("quiet", ctypes.c_bool),
("dynatemp_range", ctypes.c_float),
("dynatemp_exponent", ctypes.c_float),
("smoothing_factor", ctypes.c_float),
("logit_biases", logit_bias * logit_bias_max)]
class generation_outputs(ctypes.Structure):
@ -328,7 +329,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt, memory="", max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.0, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, dynatemp_exponent=1.0, logit_biases={}):
def generate(prompt, memory="", max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.0, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, dynatemp_exponent=1.0, smoothing_factor=0.0, logit_biases={}):
global maxctx, args, currentusergenkey, totalgens, pendingabortkey
inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
@ -359,6 +360,7 @@ def generate(prompt, memory="", max_length=32, max_context_length=512, temperatu
inputs.quiet = quiet
inputs.dynatemp_range = dynatemp_range
inputs.dynatemp_exponent = dynatemp_exponent
inputs.smoothing_factor = smoothing_factor
inputs.grammar = grammar.encode("UTF-8")
inputs.grammar_retain_state = grammar_retain_state
inputs.unban_tokens_rt = not use_default_badwordsids
@ -588,6 +590,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
quiet=is_quiet,
dynatemp_range=genparams.get('dynatemp_range', 0.0),
dynatemp_exponent=genparams.get('dynatemp_exponent', 1.0),
smoothing_factor=genparams.get('smoothing_factor', 0.0),
logit_biases=genparams.get('logit_bias', {})
)