vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-09 16:44:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

default trim_stop to true, which trims any tokens after a stop sequence and the stop sequence itself. This is potentially a breaking change.

Browse source
This commit is contained in:
Concedo 2024-12-03 22:44:10 +08:00
parent 7d11d2946c
commit 52cc908f7f
3 changed files with 18 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

7
kcpp_docs.embd
View file

@ -89,7 +89,7 @@
"type": "integer"
},
"stop_sequence": {
"description": "An array of string sequences where the API will stop generating further tokens. The returned text WILL contain the stop sequence.",
"description": "An array of string sequences where the API will stop generating further tokens. The returned text WILL contain the stop sequence if trim_stop is false.",
"items": {
"type": "string"
},
@ -197,8 +197,8 @@
}
},
"trim_stop": {
"default": false,
"description": "KoboldCpp ONLY. If true, also removes detected stop_sequences from the output and truncates all text after them.",
"default": true,
"description": "KoboldCpp ONLY. If true, also removes detected stop_sequences from the output and truncates all text after them. If false, output will also include stop sequence and potentially a few additional characters.",
"type": "boolean"
},
"render_special": {
@ -1779,7 +1779,6 @@
};
</script>
<script>
//self destruct into json if requested
const urlParams = new URLSearchParams(window.location.search);

21
klite.embd
View file

@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
-->
<script>
const LITEVER = 192;
const LITEVER = 193;
const urlParams = new URLSearchParams(window.location.search);
var localflag = true;
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@ -5402,12 +5402,12 @@ Current version indicated by LITEVER below.
image_db[imgid].imsource = 0; //0=generated,1=uploaded
}else{
console.log("Generation Error!");
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running and properly configured!\n\nIt must be launched with the flag --enable-cors-header '*' to enable API access\n");
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running at "+localsettings.saved_comfy_url+" and properly configured!\n\nIt must be launched with the flag --listen --enable-cors-header '*' to enable API access\n");
}
}).catch((error) => {
console.log("Generation Error: " + error);
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running and properly configured!\n\nIt must be launched with the flag --enable-cors-header '*' to enable API access\n");
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running at "+localsettings.saved_comfy_url+" and properly configured!\n\nIt must be launched with the flag --listen --enable-cors-header '*' to enable API access\n");
});
}
@ -8822,7 +8822,6 @@ Current version indicated by LITEVER below.
console.log(data);
if (data && data.models && data.models.length > 0)
{
var lastOption = dropdown.lastElementChild;
for (var i = dropdown.options.length - 1; i >= 0; i--) {
var option = dropdown.options[i];
dropdown.remove(option);
@ -8908,12 +8907,18 @@ Current version indicated by LITEVER below.
dropdown.remove(option);
}
let selidx = 0;
let sortedarr = [];
for(var i = 0; i < data.data.length; i++) {
var opt = data.data[i];
sortedarr.push(opt.id);
}
sortedarr.sort();
for(var i=0;i<sortedarr.length;++i)
{
var el = document.createElement("option");
el.textContent = opt.id;
el.value = opt.id;
if(isOpenrouter && opt.id=="mistralai/mistral-7b-instruct")
el.textContent = sortedarr[i];
el.value = sortedarr[i];
if(isOpenrouter && sortedarr[i]=="mistralai/mistral-7b-instruct")
{
selidx = i;
}
@ -14686,7 +14691,7 @@ Current version indicated by LITEVER below.
}).catch((error) => {
console.log("Generation Error: " + error);
delete image_db[key];
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running and properly configured!\n\nIt must be launched with the flag --enable-cors-header '*' to enable API access\n");
msgbox("Image Generation Failed!\n\nPlease make sure ComfyUI is running at "+localsettings.saved_comfy_url+" and properly configured!\n\nIt must be launched with the flag --listen --enable-cors-header '*' to enable API access\n");
});
}
}

4
koboldcpp.py
View file

@ -976,7 +976,7 @@ def generate(genparams, is_quiet=False, stream_flag=False):
grammar = genparams.get('grammar', '')
grammar_retain_state = genparams.get('grammar_retain_state', False)
genkey = genparams.get('genkey', '')
trimstop = genparams.get('trim_stop', False)
trimstop = genparams.get('trim_stop', True)
quiet = is_quiet
dynatemp_range = genparams.get('dynatemp_range', 0.0)
dynatemp_exponent = genparams.get('dynatemp_exponent', 1.0)
@ -1723,7 +1723,7 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
if tokenStr!="" or streamDone:
sseq = genparams.get('stop_sequence', [])
trimstop = genparams.get('trim_stop', False)
trimstop = genparams.get('trim_stop', True)
if trimstop and not streamDone and string_contains_or_overlaps_sequence_substring(tokenStr,sseq):
tokenReserve += tokenStr
await asyncio.sleep(async_sleep_short) #if a stop sequence could trigger soon, do not send output