vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2025-09-09 16:44:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

readjusted mistral and oai template, fixed compile issue on termux, updated lite, show generated token ids in debug mode

Browse source
This commit is contained in:
Concedo 2025-08-07 21:14:48 +08:00
parent 61c19fea56
commit 338b1fe97e
8 changed files with 121 additions and 98 deletions
Download patch file Download diff file Expand all files Collapse all files

2
gpttype_adapter.cpp
View file

@ -4198,7 +4198,7 @@ generation_outputs gpttype_generate(const generation_inputs inputs)
TopPicksData toppick = top_picks_history[top_picks_history.size()-1];
std::string topstr = toppick.selected_token;
::utreplace(topstr, "\n", "\\n");
printf("(%s %.2f%%)", RemoveBell(topstr).c_str(), toppick.selected_probability*100);
printf("(%s <%d> %.2f%%)", RemoveBell(topstr).c_str(), toppick.selected_tokenid, toppick.selected_probability*100);
int maxtoshow = (toppick.tokenid.size()>4?4:toppick.tokenid.size());
for (int i=0;i<maxtoshow;++i)
{

24
kcpp_adapters/AutoGuess.json
View file

@ -89,18 +89,18 @@
}
}, {
"search": ["[/INST]", "[SYSTEM_PROMPT]"],
"name": "Mistral V7 (with system prompt)",
"name": "Mistral Tekken",
"adapter": {
"system_start": "[SYSTEM_PROMPT] ",
"system_start": "[SYSTEM_PROMPT]",
"system_end": "[/SYSTEM_PROMPT]",
"user_start": "[INST] ",
"user_start": "[INST]",
"user_end": "",
"assistant_start": "[/INST]",
"assistant_end": "</s>"
}
}, {
"search": ["[/INST]", "\"[INST] \" + system_message"],
"name": "Mistral V3",
"name": "Mistral Non-Tekken",
"adapter": {
"system_start": "[INST] ",
"system_end": "[/INST]",
@ -190,12 +190,12 @@
"search": ["<|start|>user<|message|>", "<|channel|>", "<|end|>"],
"name": "OpenAI Harmony",
"adapter": {
"system_start": "<|start|>system<|message|>",
"system_end": "<|end|>\n",
"system_start": "<|start|>developer<|message|>",
"system_end": "<|end|>",
"user_start": "<|start|>user<|message|>",
"user_end": "<|end|>\n",
"user_end": "<|end|>",
"assistant_start": "<|start|>assistant<|channel|>final<|message|>",
"assistant_end": "<|return|>\n"
"assistant_end": "<|return|>"
}
}, {
"search": ["rwkv_", "'User: '"],
@ -212,11 +212,11 @@
"search": ["[/INST]"],
"name": "Mistral (Generic)",
"adapter": {
"system_start": "[INST] ",
"system_end": "[/INST]\n",
"user_start": "[INST] ",
"system_start": "[INST]",
"system_end": "[/INST]",
"user_start": "[INST]",
"user_end": "",
"assistant_start": "[/INST]\n",
"assistant_start": "[/INST]",
"assistant_end": "</s>"
}
}, {

4
kcpp_adapters/Mistral-V2-V3.json → kcpp_adapters/Mistral-NonTekken.json
View file

@ -1,7 +1,7 @@
{
"system_start": "",
"system_start": "[INST] ",
"system_end": "",
"user_start": "[INST]",
"user_start": "[INST] ",
"user_end": "",
"assistant_start": "[/INST]",
"assistant_end": "</s>"

4
kcpp_adapters/Mistral-V3-Tekken-V7.json → kcpp_adapters/Mistral-Tekken.json
View file

@ -1,7 +1,7 @@
{
"system_start": "[SYSTEM_PROMPT] ",
"system_start": "[SYSTEM_PROMPT]",
"system_end": "[/SYSTEM_PROMPT]",
"user_start": "[INST] ",
"user_start": "[INST]",
"user_end": "",
"assistant_start": "[/INST]",
"assistant_end": "</s>"

8
kcpp_adapters/Mistral-V1.json
View file

@ -1,8 +0,0 @@
{
"system_start": "",
"system_end": "",
"user_start": " [INST]",
"user_end": "",
"assistant_start": " [/INST]",
"assistant_end": "</s>"
}

8
kcpp_adapters/OpenAI-Harmony.json
View file

@ -1,8 +1,8 @@
{
"system_start": "<|start|>system<|message|>",
"system_end": "<|end|>\n",
"system_start": "<|start|>developer<|message|>",
"system_end": "<|end|>",
"user_start": "<|start|>user<|message|>",
"user_end": "<|end|>\n",
"user_end": "<|end|>",
"assistant_start": "<|start|>assistant<|channel|>final<|message|>",
"assistant_end": "<|return|>\n"
"assistant_end": "<|return|>"
}

167
klite.embd
View file

@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
-->
<script id="init-config">
const LITEVER = 267;
const LITEVER = 268;
const urlParams = new URLSearchParams(window.location.search);
var localflag = urlParams.get('local'); //this will be replaced automatically in embedded kcpp
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@ -3034,7 +3034,6 @@ Current version indicated by LITEVER below.
const kobold_custom_mdl_endpoint = "/api/v1/model";
const kobold_custom_version_endpoint = "/api/v1/info/version";
const kobold_custom_maxctxlen_endpoint = "/api/v1/config/max_context_length";
const kobold_custom_genamt_endpoint = "/api/v1/config/max_length";
const koboldcpp_version_endpoint = "/api/extra/version";
const koboldcpp_abort_endpoint = "/api/extra/abort";
@ -3075,7 +3074,7 @@ Current version indicated by LITEVER below.
const default_mistralai_base = "https://api.mistral.ai/v1";
const default_featherless_base = "https://api.featherless.ai/v1";
const default_grok_base = "https://api.x.ai/v1";
const default_oai_base = "https://api.openai.com";
const default_oai_base = "https://api.openai.com/v1";
const default_claude_base = "https://api.anthropic.com";
const default_gemini_base = "https://generativelanguage.googleapis.com/v1beta/models/";
const default_gemini_suffix = ":generateContent?key=";
@ -3595,17 +3594,17 @@ Current version indicated by LITEVER below.
"system_end":"",
},
{
"name":"Mistral Legacy",
"user":" [INST] ",
"name":"Mistral Non-Tekken",
"user":"[INST] ",
"user_end":"",
"assistant":" [/INST]",
"assistant":"[/INST]",
"assistant_end":"</s>",
"system":"",
"system_end":"",
},
{
"name":"Mistral Modern",
"user":"[INST] ",
"name":"Mistral Tekken",
"user":"[INST]",
"user_end":"",
"assistant":"[/INST]",
"assistant_end":"</s>",
@ -3633,11 +3632,11 @@ Current version indicated by LITEVER below.
{
"name":"OpenAI Harmony",
"user":"<|start|>user<|message|>",
"user_end":"<|end|>\\n",
"user_end":"<|end|>",
"assistant":"<|start|>assistant<|channel|>final<|message|>",
"assistant_end":"<|return|>\\n",
"system":"<|start|>system<|message|>",
"system_end":"<|end|>\\n",
"assistant_end":"<|return|>",
"system":"<|start|>developer<|message|>",
"system_end":"<|end|>",
}
];
@ -5335,7 +5334,7 @@ Current version indicated by LITEVER below.
return;
}
hide_msgbox();
fetch(custom_kobold_endpoint + koboldcpp_check_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_check_endpoint), {
method: 'POST',
headers: get_kobold_header(),
body: JSON.stringify({
@ -5513,7 +5512,7 @@ Current version indicated by LITEVER below.
//if we wanted logprobs, try fetching them manually
if(localsettings.request_logprobs && last_response_obj==null)
{
fetch(custom_kobold_endpoint + koboldcpp_logprobs_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_logprobs_endpoint), {
method: 'POST',
headers: get_kobold_header(),
body: JSON.stringify({
@ -10202,6 +10201,10 @@ Current version indicated by LITEVER below.
document.getElementById("inputboxcontainerinputarea").classList.remove("hidden");
document.getElementById("inputboxcontainerinputarea").value = inputVal;
document.getElementById("inputboxcontainerinputarea").placeholder = escape_html(inputPlaceholder);
if(!isPassword)
{
document.getElementById("inputboxcontainerinputarea").focus();
}
}
else
{
@ -10213,15 +10216,19 @@ Current version indicated by LITEVER below.
{
inputboxIsPassword = true;
}
else
{
document.getElementById("inputboxcontainerinput").focus();
}
}
inputboxblur();
onInputboxOk = function(){document.getElementById("inputboxcontainer").classList.add("hidden");onDone();};
onInputboxCancel = null;
document.getElementById("inputboxcancel").classList.add("hidden");
}
function inputBoxOkCancel(text,title,inputVal,inputPlaceholder,onDone,onCancel,isHtml=false,isTextArea=false)
function inputBoxOkCancel(text,title,inputVal,inputPlaceholder,onDone,onCancel,isHtml=false,isTextArea=false,isPassword=false)
{
inputBox(text,title,inputVal,inputPlaceholder,onDone,isHtml,isTextArea);
inputBox(text,title,inputVal,inputPlaceholder,onDone,isHtml,isTextArea,isPassword);
document.getElementById("inputboxcancel").classList.remove("hidden");
onInputboxCancel = function(){document.getElementById("inputboxcontainer").classList.add("hidden");onCancel();};
}
@ -10604,7 +10611,13 @@ Current version indicated by LITEVER below.
autofetch_attempt_dict[desired_oai_ep] = true;
let dropdown = get_custom_ep_model_dropdown();
fetch((desired_oai_ep + oai_models_endpoint), {
let fetchurl = desired_oai_ep + oai_models_endpoint;
let epchoice = document.getElementById("customapidropdown").value;
if(epchoice==2 && !is_local_url(desired_oai_ep) && document.getElementById("corsproxy_oai").checked)
{
fetchurl = cors_proxy + "?" + fetchurl; //temp cors proxy if selected
}
fetch(fetchurl, {
method: 'GET',
headers: oaiheaders,
referrerPolicy: 'no-referrer',
@ -10865,12 +10878,12 @@ Current version indicated by LITEVER below.
let is_local = is_local_url(desired_oai_ep);
desired_oai_ep = (is_local?"http://":"https://") + desired_oai_ep;
}
//fix incorrect paths
if(desired_oai_ep!="" && desired_oai_ep.toLowerCase().endsWith("/chat/completions")) {
desired_oai_ep = desired_oai_ep.slice(0,-17);
}
if (document.getElementById("oaiaddversion").checked && !desired_oai_ep.toLowerCase().includes("text.pollinations.ai"))
{
//fix incorrect paths
if(desired_oai_ep!="" && desired_oai_ep.toLowerCase().endsWith("/chat/completions")) {
desired_oai_ep = desired_oai_ep.slice(0,-17);
}
if(desired_oai_ep!="" && desired_oai_ep.length > 4 && !desired_oai_ep.slice(-4).toLowerCase().includes("/v") && !desired_oai_ep.toLowerCase().includes("/v1/")) {
desired_oai_ep = desired_oai_ep + "/v1";
}
@ -10896,6 +10909,7 @@ Current version indicated by LITEVER below.
let epchoice = document.getElementById("customapidropdown").value;
document.getElementById("connectstatusproxied").classList.add("hidden");
document.getElementById("connectstatus").innerHTML = "Connecting";
clear_cors_proxy_flag();
if(epchoice==0) //ai horde
{
@ -10928,7 +10942,12 @@ Current version indicated by LITEVER below.
custom_kobold_endpoint = desiredkoboldendpoint;
custom_kobold_key = desiredkoboldkey;
const backup_oai_test = desiredkoboldendpoint + "/v1" + oai_models_endpoint; //last resort later
if(!is_local_url(desiredkoboldendpoint) && document.getElementById("corsproxy_kai").checked)
{
uses_cors_proxy = true;
}
const backup_oai_test = apply_proxy_url(desiredkoboldendpoint + "/v1" + oai_models_endpoint); //last resort later
let fetchedurl = apply_proxy_url(desiredkoboldendpoint + kobold_custom_mdl_endpoint);
@ -11199,22 +11218,17 @@ Current version indicated by LITEVER below.
//we will go down the fallbacks one by one until we run out of options
localmodeport = backup_localmodeport;
if(is_local && sublocalpathname!="")
if(is_local && !uses_cors_proxy && sublocalpathname!="")
{
sublocalpathname = ""; // first fallback, check subdir paths
attempt_connect(false);
}
else if(reattempt_local_port80) // second fallback, try port 80
else if(!uses_cors_proxy && reattempt_local_port80) // second fallback, try port 80
{
reattempt_local_port80 = false;
localmodeport = 80;
attempt_connect(false);
}
else if(!is_local && !uses_cors_proxy) //third fallback, use cors proxy if not a local IP
{
uses_cors_proxy = true; // fallback to cors proxy, this will remain for rest of session
connect_custom_endpoint(); //one more try
}
else //finally, we give up
{
const connectfailed = function()
@ -11282,6 +11296,11 @@ Current version indicated by LITEVER below.
localsettings.saved_oai_tts_url = custom_oai_endpoint + default_oai_tts_endpoint;
localsettings.saved_oai_embd_key = custom_oai_key;
localsettings.saved_oai_embd_url = custom_oai_endpoint + default_oai_embeddings_endpoint;
if(!is_local_url(custom_oai_endpoint) && document.getElementById("corsproxy_oai").checked)
{
uses_cors_proxy = true;
}
}
else if(epchoice==7)
{
@ -11332,6 +11351,14 @@ Current version indicated by LITEVER below.
}
document.getElementById("connectstatus").innerHTML = "OpenAI Endpoint";
render_gametext(true);
if(uses_cors_proxy && !is_local_url(desired_oai_ep))
{
document.getElementById("connectstatusproxied").classList.remove("hidden");
}else
{
document.getElementById("connectstatusproxied").classList.add("hidden");
}
}
else
{
@ -11484,7 +11511,7 @@ Current version indicated by LITEVER below.
{
header['Authorization'] = 'Bearer ' + adminkey;
}
fetch(custom_kobold_endpoint + koboldcpp_admin_list_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_admin_list_endpoint), {
method: 'GET',
headers: header,
})
@ -11583,7 +11610,7 @@ Current version indicated by LITEVER below.
{
header['Authorization'] = 'Bearer ' + last_admin_key;
}
fetch(custom_kobold_endpoint + koboldcpp_admin_reload_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_admin_reload_endpoint), {
method: 'POST',
headers: header,
body: JSON.stringify(payload)
@ -11636,7 +11663,7 @@ Current version indicated by LITEVER below.
{
header['Authorization'] = 'Bearer ' + last_admin_key;
}
fetch(custom_kobold_endpoint + koboldcpp_admin_savestate_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_admin_savestate_endpoint), {
method: 'POST',
headers: header,
body: JSON.stringify({
@ -11668,7 +11695,7 @@ Current version indicated by LITEVER below.
{
header['Authorization'] = 'Bearer ' + last_admin_key;
}
fetch(custom_kobold_endpoint + koboldcpp_admin_loadstate_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_admin_loadstate_endpoint), {
method: 'POST',
headers: header,
body: JSON.stringify({
@ -11788,7 +11815,7 @@ Current version indicated by LITEVER below.
if(is_using_kcpp_with_savedatafile())
{
//grab saves
fetch(custom_kobold_endpoint + koboldcpp_savedata_list_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_savedata_list_endpoint), {
method: 'POST', // or 'PUT'
headers: get_kobold_header(),
})
@ -11897,7 +11924,7 @@ Current version indicated by LITEVER below.
userinput = defaultsavename;
}
//complete remote save
fetch(custom_kobold_endpoint + koboldcpp_savedata_save_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_savedata_save_endpoint), {
method: 'POST', // or 'PUT'
headers: get_kobold_header(),
body: JSON.stringify({
@ -11963,7 +11990,7 @@ Current version indicated by LITEVER below.
});
} else {
last_used_saveslot = -1;
fetch(custom_kobold_endpoint + koboldcpp_savedata_load_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_savedata_load_endpoint), {
method: 'POST', // or 'PUT'
headers: get_kobold_header(),
body: JSON.stringify({
@ -12021,7 +12048,7 @@ Current version indicated by LITEVER below.
}
else
{
fetch(custom_kobold_endpoint + koboldcpp_savedata_load_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_savedata_load_endpoint), {
method: 'POST', // or 'PUT'
headers: get_kobold_header(),
body: JSON.stringify({
@ -12074,7 +12101,7 @@ Current version indicated by LITEVER below.
});
} else {
msgboxYesNo("Delete story in Server Storage Slot " + slotnumshown + "?", "Delete Server Storage Slot " + slotnumshown, () => {
fetch(custom_kobold_endpoint + koboldcpp_savedata_save_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_savedata_save_endpoint), {
method: 'POST', // or 'PUT'
headers: get_kobold_header(),
body: JSON.stringify({
@ -12175,7 +12202,11 @@ Current version indicated by LITEVER below.
function fetch_koboldcpp_perf()
{
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_perf_endpoint))
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_perf_endpoint),
{
method: 'GET',
headers: get_kobold_header(),
})
.then(x => x.json())
.then(resp => {
console.log(resp);
@ -14677,7 +14708,7 @@ Current version indicated by LITEVER below.
else if(is_using_kcpp_with_streaming())
{
//we can use abort functions
fetch(custom_kobold_endpoint + koboldcpp_abort_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_abort_endpoint), {
method: 'POST', // or 'PUT'
headers: get_kobold_header(),
body: JSON.stringify({
@ -14749,7 +14780,7 @@ Current version indicated by LITEVER below.
function add_img_btn_custom()
{
inputBox("Enter a custom prompt to generate an image with.","Generate Image Manually","","Enter a Prompt",()=>{
inputBoxOkCancel("Enter a custom prompt to generate an image with.","Generate Image Manually","","Enter a Prompt",()=>{
let userinput = getInputBoxValue();
if(userinput.trim()!="")
{
@ -16867,7 +16898,7 @@ Current version indicated by LITEVER below.
else if (custom_oai_key != "")//handle for OAI
{
let targetep = (custom_oai_endpoint + oai_submit_endpoint);
let targetep = apply_proxy_url(custom_oai_endpoint + oai_submit_endpoint);
let scaled_rep_pen = 0;
if(submit_payload.params.presence_penalty > 0)
@ -16976,7 +17007,7 @@ Current version indicated by LITEVER below.
}
oai_payload.messages = [];
targetep = (custom_oai_endpoint + oai_submit_endpoint_chat);
targetep = apply_proxy_url(custom_oai_endpoint + oai_submit_endpoint_chat);
if (document.getElementById("jailbreakprompt") && document.getElementById("jailbreakprompt").checked && document.getElementById("jailbreakprompttext").value!="") {
let addrole = document.getElementById("jailbreakprompttextrole").value;
addrole = ((addrole==2)?"system":(addrole==1?"assistant":"user"));
@ -17978,9 +18009,7 @@ Current version indicated by LITEVER below.
};
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_transcribe_endpoint), {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
headers: get_kobold_header(),
body: JSON.stringify(payload),
})
.then(x => x.json())
@ -19410,9 +19439,7 @@ Current version indicated by LITEVER below.
};
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_transcribe_endpoint), {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
headers: get_kobold_header(),
body: JSON.stringify(payload),
})
.then(x => x.json())
@ -19461,9 +19488,7 @@ Current version indicated by LITEVER below.
};
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_transcribe_endpoint), {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
headers: get_kobold_header(),
body: JSON.stringify(payload),
})
.then(x => x.json())
@ -19493,9 +19518,7 @@ Current version indicated by LITEVER below.
};
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_tokenize_endpoint), {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
headers: get_kobold_header(),
body: JSON.stringify(payload),
})
.then(x => x.json())
@ -19667,7 +19690,7 @@ Current version indicated by LITEVER below.
if (polledstreaming && (localflag?true:(poll_ticks_passed%2==0)))
{
//get in-progress results
fetch(custom_kobold_endpoint + koboldcpp_check_endpoint, {
fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_check_endpoint), {
method: 'POST',
headers: get_kobold_header(),
body: JSON.stringify({
@ -26021,37 +26044,45 @@ Current version indicated by LITEVER below.
<input class="form-control" title="Enter KoboldCpp Custom Endpoint" id="customkoboldendpoint" placeholder="https://sample-remote-address.trycloudflare.com" value="">
<input class="form-control" title="Enter KoboldCpp API Key" type="password" id="customkoboldkey" placeholder="KoboldAI API Key (Optional)" value="" onfocus="focus_api_keys()" onblur="blur_api_keys()"><br>
<div class="borderbox flex flex-push-right">
<input type="checkbox" id="corsproxy_kai" title="Use CORS Proxy">
<div class="box-label">Use CORS Proxy <span class="helpicon">?<span class="helptext">Attempts to use a CORS proxy to bypass CORS restrictions. Only use this if normal connections fail. Only works on public internet APIs.</span></span></div>
</div>
</div>
<div id="oaicustom" class="menutext hidden">
<span id="oaidesc">
Entering your OpenAI API key will allow you to use KoboldAI Lite with their API.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the OpenAI API and is not transmitted to us. Note that connecting will reset any custom DALL-E based or custom OpenAI based TTS endpoint urls to this new address.<br>Only Temperature, Top-P and Repetition Penalty samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">Please input OpenAI API URL and Key.</span><br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. Your API key is used directly with the OpenAI API, <span class="color_red">unless CORS proxy is used</span>. Reconnecting resets all custom OpenAI URLs.<br>Only Temperature, Top-P and Repetition Penalty samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">Please input OpenAI API URL and Key.</span><br>
<div style="display:inline-flex">
<div><input type="checkbox" id="corsproxy_oai" title="Use CORS Proxy">
<div class="box-label">Use CORS Proxy <span class="helpicon">?<span class="helptext">Attempts to use a CORS proxy to bypass CORS restrictions. Only use this if normal connections fail. Only works on public internet APIs.</span></span></div></div>
<div><input type="checkbox" id="oaiaddversion" title="Add Endpoint Version Number" onchange="" checked>
<div class="box-label">Add Ver. Num <span class="helpicon">?<span class="helptext">Usually OpenAI Compatible APIs end with /v1. If they do not, uncheck this box.</span></span></div></div>
</div>
</span>
<span id="openrouterdesc" class="hidden">
Entering your OpenRouter API key will allow you to use KoboldAI Lite with their API.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the OpenRouter API and is not transmitted to us.<br>Only Temperature, Top-P and Repetition Penalty samplers are used.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. Your API key is used directly with the OpenRouter API and is not transmitted to us.<br>Only Temperature, Top-P and Repetition Penalty samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">Please input OpenRouter Key.</span><br><br>
</span>
<span id="mistralaidesc" class="hidden">
Entering your MistralAI API key will allow you to use KoboldAI Lite with their API.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the MistralAI API and is not transmitted to us.<br>Only Temperature and Top-P samplers are used.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. Your API key is used directly with the MistralAI API and is not transmitted to us.<br>Only Temperature and Top-P samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">Please input MistralAI Key.</span><br><br>
</span>
<span id="featherlessdesc" class="hidden">
Entering your Featherless API key will allow you to use KoboldAI Lite with their API.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the Featherless API and is not transmitted to us.<br>Only Temperature, Top-P, Top-K, Min-P and Repetition Penalty samplers are used.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. Your API key is used directly with the Featherless API and is not transmitted to us.<br>Only Temperature, Top-P, Top-K, Min-P and Repetition Penalty samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">Please input Featherless Key.</span><br><br>
</span>
<span id="grokdesc" class="hidden">
Entering your Grok API key will allow you to use KoboldAI Lite with their API.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the Grok API and is not transmitted to us.<br>Only Temperature, Top-P, Top-K, Min-P and Repetition Penalty samplers are used.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. Your API key is used directly with the Grok API and is not transmitted to us.<br>Only Temperature, Top-P, Top-K, Min-P and Repetition Penalty samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">Please input Grok Key.</span><br><br>
</span>
<span id="pollinationsdesc" class="hidden">
Pollinations.ai API is free to use without any key required.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. This service is ad driven, ads may appear in the output.<br>Only Temperature, Top-P, Top-K and Repetition Penalty samplers are used.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. This service is ad driven, ads may appear in the output.<br>Only Temperature, Top-P, Top-K and Repetition Penalty samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">No Key Required.</span><br><br>
</span>
@ -26159,8 +26190,6 @@ Current version indicated by LITEVER below.
<div class="box-label">Emulate Completions API with Prefill</div></div>
</div>
<div style="display:inline-flex">
<div><input type="checkbox" id="oaiaddversion" title="Add Endpoint Version Number" onchange="" checked>
<div class="box-label">Add Ver. Num</div></div>
<div><input type="checkbox" id="useoaichatcompl" title="Use ChatCompletions API" onchange="toggleoaichatcompl()">
<div class="box-label">Chat-Completions API</div></div>
<div><input type="checkbox" id="useoainonstandard" title="Send Non-Standard Fields">
@ -26208,7 +26237,7 @@ Current version indicated by LITEVER below.
</div>
<div id="claudecustom" class="menutext hidden">
Entering your Claude API key will allow you to use KoboldAI Lite with their API.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. <br><span class="color_red">At this time, the official Claude API has CORS restrictions and must be accessed with a CORS proxy. Your connection WILL be proxied.</span><br>Only Temperature, Top-P and Top-K samplers are used.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. <br><span class="color_red">At this time, the official Claude API has CORS restrictions and must be accessed with a CORS proxy. Your connection WILL be proxied.</span><br>Only Temperature, Top-P and Top-K samplers are used.<br><br>
<span class="color_green" style="font-weight: bold;">Please input Claude API URL and Key.</span><br><br>
<input class="form-control" type="text" id="custom_claude_endpoint" placeholder="Claude API URL" value="">
<input class="form-control" type="password" id="custom_claude_key" placeholder="Claude API Key (Required)" value="" onfocus="focus_api_keys()" onblur="blur_api_keys()"><br>
@ -26253,7 +26282,7 @@ Current version indicated by LITEVER below.
</div>
<div id="geminicustom" class="menutext hidden">
Uses Gemini by Google.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the Gemini API and is not transmitted to us.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. Your API key is used directly with the Gemini API and is not transmitted to us.<br><br>
<div>
<select title="Gemini AI Model Selection" style="padding:4px; width:calc(100% - 110px); display:inline-block" class="form-control" id="custom_gemini_model" onchange="togglegeminimodel()">
<option value="gemini-1.5-flash-latest" selected="selected">gemini-1.5-flash-latest</option>
@ -26307,7 +26336,7 @@ Current version indicated by LITEVER below.
</div>
<div id="coherecustom" class="menutext hidden">
Uses Cohere's models through their own API.<br><br>
Note that KoboldAI Lite takes no responsibility for your usage or consequences of this feature. Your API key is used directly with the Cohere API and is not transmitted to us.<br><br>
KoboldAI Lite takes no responsibility for your usage of this feature. Your API key is used directly with the Cohere API and is not transmitted to us.<br><br>
<select title="Cohere AI Model Selection" style="padding:4px;" class="form-control" id="custom_cohere_model">
<option value="command" selected="selected">command</option>
<option value="command-r">command-r</option>
@ -26329,7 +26358,7 @@ Current version indicated by LITEVER below.
</span>
</div>
<div class="popupfooter">
<button type="button" class="btn btn-primary" onclick="clear_cors_proxy_flag();connect_custom_endpoint()">Ok</button>
<button type="button" class="btn btn-primary" onclick="connect_custom_endpoint()">Ok</button>
<button type="button" class="btn btn-primary" onclick="dismiss_endpoint_container()">Cancel</button>
</div>
</div>

2
src/llama-mmap.cpp
View file

@ -290,6 +290,7 @@ struct llama_mmap::impl {
throw std::runtime_error(format("mmap failed: %s", strerror(errno)));
}
#ifndef __ANDROID__
if (prefetch > 0) {
if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {
LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",
@ -302,6 +303,7 @@ struct llama_mmap::impl {
strerror(errno));
}
}
#endif
mapped_fragments.emplace_back(0, file->size());
}