vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-10 09:04:36 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

default trim_stop to true, which trims any tokens after a stop sequence and the stop sequence itself. This is potentially a breaking change.

Browse source
This commit is contained in:
Concedo 2024-12-03 22:44:10 +08:00
parent 7d11d2946c
commit 52cc908f7f
3 changed files with 18 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

7
kcpp_docs.embd
View file

@ -89,7 +89,7 @@
"type": "integer" "type": "integer"
}, },
"stop_sequence": { "stop_sequence": {
"description": "An array of string sequences where the API will stop generating further tokens. The returned text WILL contain the stop sequence.", "description": "An array of string sequences where the API will stop generating further tokens. The returned text WILL contain the stop sequence if trim_stop is false.",
"items": { "items": {
"type": "string" "type": "string"
}, },
@ -197,8 +197,8 @@
} }
}, },
"trim_stop": { "trim_stop": {
"default": false, "default": true,
"description": "KoboldCpp ONLY. If true, also removes detected stop_sequences from the output and truncates all text after them.", "description": "KoboldCpp ONLY. If true, also removes detected stop_sequences from the output and truncates all text after them. If false, output will also include stop sequence and potentially a few additional characters.",
"type": "boolean" "type": "boolean"
}, },
"render_special": { "render_special": {
@ -1779,7 +1779,6 @@
}; };
</script> </script>
<script> <script>
//self destruct into json if requested //self destruct into json if requested
const urlParams = new URLSearchParams(window.location.search); const urlParams = new URLSearchParams(window.location.search);

21
klite.embd
View file

@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
--> -->
<script> <script>
const LITEVER = 192; const LITEVER = 193;
const urlParams = new URLSearchParams(window.location.search); const urlParams = new URLSearchParams(window.location.search);
var localflag = true; var localflag = true;
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_"; const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@ -5402,12 +5402,12 @@ Current version indicated by LITEVER below.
image_db[imgid].imsource = 0; //0=generated,1=uploaded image_db[imgid].imsource = 0; //0=generated,1=uploaded
}else{ }else{
console.log("Generation Error!"); console.log("Generation Error!");
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running and properly configured!\n\nIt must be launched with the flag --enable-cors-header '*' to enable API access\n"); msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running at "+localsettings.saved_comfy_url+" and properly configured!\n\nIt must be launched with the flag --listen --enable-cors-header '*' to enable API access\n");
} }
}).catch((error) => { }).catch((error) => {
console.log("Generation Error: " + error); console.log("Generation Error: " + error);
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running and properly configured!\n\nIt must be launched with the flag --enable-cors-header '*' to enable API access\n"); msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running at "+localsettings.saved_comfy_url+" and properly configured!\n\nIt must be launched with the flag --listen --enable-cors-header '*' to enable API access\n");
}); });
} }
@ -8822,7 +8822,6 @@ Current version indicated by LITEVER below.
console.log(data); console.log(data);
if (data && data.models && data.models.length > 0) if (data && data.models && data.models.length > 0)
{ {
var lastOption = dropdown.lastElementChild;
for (var i = dropdown.options.length - 1; i >= 0; i--) { for (var i = dropdown.options.length - 1; i >= 0; i--) {
var option = dropdown.options[i]; var option = dropdown.options[i];
dropdown.remove(option); dropdown.remove(option);
@ -8908,12 +8907,18 @@ Current version indicated by LITEVER below.
dropdown.remove(option); dropdown.remove(option);
} }
let selidx = 0; let selidx = 0;
let sortedarr = [];
for(var i = 0; i < data.data.length; i++) { for(var i = 0; i < data.data.length; i++) {
var opt = data.data[i]; var opt = data.data[i];
sortedarr.push(opt.id);
}
sortedarr.sort();
for(var i=0;i<sortedarr.length;++i)
{
var el = document.createElement("option"); var el = document.createElement("option");
el.textContent = opt.id; el.textContent = sortedarr[i];
el.value = opt.id; el.value = sortedarr[i];
if(isOpenrouter && opt.id=="mistralai/mistral-7b-instruct") if(isOpenrouter && sortedarr[i]=="mistralai/mistral-7b-instruct")
{ {
selidx = i; selidx = i;
} }
@ -14686,7 +14691,7 @@ Current version indicated by LITEVER below.
}).catch((error) => { }).catch((error) => {
console.log("Generation Error: " + error); console.log("Generation Error: " + error);
delete image_db[key]; delete image_db[key];
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running and properly configured!\n\nIt must be launched with the flag --enable-cors-header '*' to enable API access\n"); msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running at "+localsettings.saved_comfy_url+" and properly configured!\n\nIt must be launched with the flag --listen --enable-cors-header '*' to enable API access\n");
}); });
} }
} }

4
koboldcpp.py
View file

@ -976,7 +976,7 @@ def generate(genparams, is_quiet=False, stream_flag=False):
grammar = genparams.get('grammar', '') grammar = genparams.get('grammar', '')
grammar_retain_state = genparams.get('grammar_retain_state', False) grammar_retain_state = genparams.get('grammar_retain_state', False)
genkey = genparams.get('genkey', '') genkey = genparams.get('genkey', '')
trimstop = genparams.get('trim_stop', False) trimstop = genparams.get('trim_stop', True)
quiet = is_quiet quiet = is_quiet
dynatemp_range = genparams.get('dynatemp_range', 0.0) dynatemp_range = genparams.get('dynatemp_range', 0.0)
dynatemp_exponent = genparams.get('dynatemp_exponent', 1.0) dynatemp_exponent = genparams.get('dynatemp_exponent', 1.0)
@ -1723,7 +1723,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
if tokenStr!="" or streamDone: if tokenStr!="" or streamDone:
sseq = genparams.get('stop_sequence', []) sseq = genparams.get('stop_sequence', [])
trimstop = genparams.get('trim_stop', False) trimstop = genparams.get('trim_stop', True)
if trimstop and not streamDone and string_contains_or_overlaps_sequence_substring(tokenStr,sseq): if trimstop and not streamDone and string_contains_or_overlaps_sequence_substring(tokenStr,sseq):
tokenReserve += tokenStr tokenReserve += tokenStr
await asyncio.sleep(async_sleep_short) #if a stop sequence could trigger soon, do not send output await asyncio.sleep(async_sleep_short) #if a stop sequence could trigger soon, do not send output