vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-11 01:24:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Full DynaTemp implementation + UI (#600)

Browse source 
* move Dynatemp changes to new branch

* fix float header

* Properly reintroduce variable expert count

Controllable through experts.txt

* first pass at DynaTemp UI

Checkbox partial implemented, Min and Max Temp implemented

* DynaTemp UI Checkbox

Trigger DynaTemp on checkbox

* DynaTemp UI checkbox edition

Hell Yeah! DynaTemp!

* Remove greedy dynatemp

* Fix race condition caused by debug print

* Fixed broken presets and miro

Fixes broken presets and mirostat

* Remove debug function + HHI temp

Also removed unnecessary softmax double precision

* Fix whitespace (?) for generate function

* epic upstream renaming scheme fix

* fix stupid indents

* Other cleanup

Reintroduce unused rep pen function, move temp functions first before entropy dynamic temp

* Slight indent fix

* revert batch pyinstaller maker to mainline

and also delete experts.txt since adjustable routing is also being removed for the PR

* compact dynatemp into a single value dynatemp_range. This is a float which represents the allowed deviation from the min and max temperature when using dynatemp. Thus, if we want a value of dynatemp_min=0.3, dynatemp_max=0.5, then we would simply set temperature=0.4 and dynatemp_range=0.1. Functionally dynatemp would operate the same, but it would simplify usage and make it a single easy to adjust value.

---------

Co-authored-by: Alexander Abushady <aabushady214@gmail.com>
Co-authored-by: Concedo <39025047+LostRuins@users.noreply.github.com>
This commit is contained in:
kalomaze 2024-01-05 21:13:16 -06:00 • committed by GitHub
parent 427ba21e62
commit 123bff9a0f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 132 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

8
koboldcpp.py
View file

@ -78,6 +78,7 @@ class generation_inputs(ctypes.Structure):
("grammar", ctypes.c_char_p),
("grammar_retain_state", ctypes.c_bool),
("quiet", ctypes.c_bool),
("dynatemp_range", ctypes.c_float),
("logit_biases", logit_bias * logit_bias_max)]
class generation_outputs(ctypes.Structure):
@ -310,7 +311,7 @@ def load_model(model_filename):
ret = handle.load_model(inputs)
return ret
def generate(prompt, memory="", max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.1, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, logit_biases={}):
def generate(prompt, memory="", max_length=32, max_context_length=512, temperature=0.7, top_k=100, top_a=0.0, top_p=0.92, min_p=0.0, typical_p=1.0, tfs=1.0, rep_pen=1.1, rep_pen_range=128, presence_penalty=0.0, mirostat=0, mirostat_tau=5.0, mirostat_eta=0.1, sampler_order=[6,0,1,3,4,2,5], seed=-1, stop_sequence=[], use_default_badwordsids=False, stream_sse=False, grammar='', grammar_retain_state=False, genkey='', trimstop=False, quiet=False, dynatemp_range=0.0, logit_biases={}):
global maxctx, args, currentusergenkey, totalgens
inputs = generation_inputs()
outputs = ctypes.create_unicode_buffer(ctypes.sizeof(generation_outputs))
@ -338,6 +339,7 @@ def generate(prompt, memory="", max_length=32, max_context_length=512, temperatu
inputs.presence_penalty = presence_penalty
inputs.stream_sse = stream_sse
inputs.quiet = quiet
inputs.dynatemp_range = dynatemp_range
inputs.grammar = grammar.encode("UTF-8")
inputs.grammar_retain_state = grammar_retain_state
inputs.unban_tokens_rt = not use_default_badwordsids
@ -547,7 +549,9 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
genkey=genparams.get('genkey', ''),
trimstop=genparams.get('trim_stop', False),
quiet=is_quiet,
logit_biases=genparams.get('logit_bias', {}))
dynatemp_range=genparams.get('dynatemp_range', 0.0),
logit_biases=genparams.get('logit_bias', {})
)
recvtxt = ""
if stream_flag: