diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index f4322c6ee..000000000 --- a/SECURITY.md +++ /dev/null @@ -1,67 +0,0 @@ -# Security Policy - - - [**Using llama.cpp securely**](#using-llamacpp-securely) - - [Untrusted models](#untrusted-models) - - [Untrusted inputs](#untrusted-inputs) - - [Data privacy](#data-privacy) - - [Untrusted environments or networks](#untrusted-environments-or-networks) - - [Multi-Tenant environments](#multi-tenant-environments) - - [**Reporting a vulnerability**](#reporting-a-vulnerability) - -## Using llama.cpp securely - -### Untrusted models -Be careful when running untrusted models. This classification includes models created by unknown developers or utilizing data obtained from unknown sources. - -*Always execute untrusted models within a secure, isolated environment such as a sandbox* (e.g., containers, virtual machines). This helps protect your system from potentially malicious code. - -> [!NOTE] -> The trustworthiness of a model is not binary. You must always determine the proper level of caution depending on the specific model and how it matches your use case and risk tolerance. - -### Untrusted inputs - -Some models accept various input formats (text, images, audio, etc.). The libraries converting these inputs have varying security levels, so it's crucial to isolate the model and carefully pre-process inputs to mitigate script injection risks. - -For maximum security when handling untrusted inputs, you may need to employ the following: - -* Sandboxing: Isolate the environment where the inference happens. -* Pre-analysis: Check how the model performs by default when exposed to prompt injection (e.g. using [fuzzing for prompt injection](https://github.com/FonduAI/awesome-prompt-injection?tab=readme-ov-file#tools)). This will give you leads on how hard you will have to work on the next topics. -* Updates: Keep both LLaMA C++ and your libraries updated with the latest security patches. -* Input Sanitation: Before feeding data to the model, sanitize inputs rigorously. This involves techniques such as: - * Validation: Enforce strict rules on allowed characters and data types. - * Filtering: Remove potentially malicious scripts or code fragments. - * Encoding: Convert special characters into safe representations. - * Verification: Run tooling that identifies potential script injections (e.g. [models that detect prompt injection attempts](https://python.langchain.com/docs/guides/safety/hugging_face_prompt_injection)). - -### Data privacy - -To protect sensitive data from potential leaks or unauthorized access, it is crucial to sandbox the model execution. This means running the model in a secure, isolated environment, which helps mitigate many attack vectors. - -### Untrusted environments or networks - -If you can't run your models in a secure and isolated environment or if it must be exposed to an untrusted network, make sure to take the following security precautions: -* Confirm the hash of any downloaded artifact (e.g. pre-trained model weights) matches a known-good value -* Encrypt your data if sending it over the network. - -### Multi-Tenant environments - -If you intend to run multiple models in parallel with shared memory, it is your responsibility to ensure the models do not interact or access each other's data. The primary areas of concern are tenant isolation, resource allocation, model sharing and hardware attacks. - -1. Tenant Isolation: Models should run separately with strong isolation methods to prevent unwanted data access. Separating networks is crucial for isolation, as it prevents unauthorized access to data or models and malicious users from sending graphs to execute under another tenant's identity. - -2. Resource Allocation: A denial of service caused by one model can impact the overall system health. Implement safeguards like rate limits, access controls, and health monitoring. - -3. Model Sharing: In a multitenant model sharing design, tenants and users must understand the security risks of running code provided by others. Since there are no reliable methods to detect malicious models, sandboxing the model execution is the recommended approach to mitigate the risk. - -4. Hardware Attacks: GPUs or TPUs can also be attacked. [Researches](https://scholar.google.com/scholar?q=gpu+side+channel) has shown that side channel attacks on GPUs are possible, which can make data leak from other models or processes running on the same system at the same time. - -## Reporting a vulnerability - -Beware that none of the topics under [Using llama.cpp securely](#using-llamacpp-securely) are considered vulnerabilities of LLaMA C++. - - -However, If you have discovered a security vulnerability in this project, please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released. - -Please disclose it as a private [security advisory](https://github.com/ggerganov/llama.cpp/security/advisories/new). - -A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure. diff --git a/kcpp_docs.embd b/kcpp_docs.embd index fba70ea64..a67db8fab 100644 --- a/kcpp_docs.embd +++ b/kcpp_docs.embd @@ -532,7 +532,7 @@ "description": "Successful request" } }, - "summary": "Retrieve the current model string", + "summary": "Retrieve the current model string.", "tags": [ "api/v1" ] @@ -878,6 +878,50 @@ ] } }, + "/api/extra/transcribe": { + "post": { + "description": "Uses Whisper to perform a Speech-To-Text transcription.", + "requestBody": { + "content": { + "application/json": { + "example": { + "prompt": "", + "audio_data": "base64_wav_data", + }, + "schema": { + "properties": { + "audio_data": { + "type": "string", + "description": "Base64 respresentation of a 16-bit 16kHz wave file to be transcribed to text." + } + }, + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "example": { + "text": "Hello world" + }, + "schema": { + "$ref": "#/components/schemas/ValueResult" + } + } + }, + "description": "Successful request" + } + }, + "summary": "Uses Whisper to perform a Speech-To-Text transcription.", + "tags": [ + "api/extra" + ] + } + }, "/sdapi/v1/sd-models": { "get": { "description": "Gets a list of image generation models. For koboldcpp, only one model will be returned. If no model is loaded, the list is empty.", diff --git a/klite.embd b/klite.embd index 745def80f..a5de46e7f 100644 --- a/klite.embd +++ b/klite.embd @@ -7,7 +7,7 @@ Just copy this single static HTML file anywhere and open it in a browser, or fro Please go to https://github.com/LostRuins/lite.koboldai.net for updates on Kobold Lite. If you are submitting a pull request for Lite, PLEASE use the above repo, not the KoboldCpp one. Kobold Lite is under the AGPL v3.0 License unless otherwise exempted. Please do not remove this line. -Current version: 141 +Current version: 142 -Concedo --> @@ -52,6 +52,9 @@ Current version: 141 --img_load:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAMAAADXqc3KAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAAnQAAAJ0Bj3LnbgAAADxQTFRF8MQZKbmZ8p0fAAAA8bIc8MIa8MQZq7NIKrmY7cQa0sIs8d2BebRn87wcLL+aJrmZ//8AdbhpAP//c69phZ/jMwAAABR0Uk5T//3/AP+y8v6kbP///xcZUAGrAUn40tQBAAAAp0lEQVR4nHWSiw6DIAwAy1peojLd///rKm3ZZOOiCXLpBRIhB2CCf+JeXUr1RAECATGQ+WN3TFUBKsJmJu0ijOCZo5m7EJzVRrE2c3ZBhoiXCXoYsYm4qejbUSbWo94HXCfZALXF0kVWQVAKP6Xo/hK0RHZcmVnAa+lzketYESBb6dvwPmxDqdX49TiWlIw/JeG6+ViCVsJZCWclnJVwVkLwfwau/+INBncEwpxiohQAAAAASUVORK5CYII="); --img_delete:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAMAAADXqc3KAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAAnQAAAJ0Bj3LnbgAAADxQTFRF////AAAA////////////////////////////////////////////////////////////////////////npt1BAAAABR0Uk5T/wB3M/ryVkUE3GQ8bhvOlb+s6oPyVnGNAAAAt0lEQVR4nHWR2RaDIBBDGXZlVf//X8sMsQtt88BxuJpIUDTVrBLZhg1F0WutL/XUNUYfB/Dqh/Z/wA9AbLVIrIjMlySc6FyNHMC2gg2grqAC9BV0gIOHs4+z2y55B0DjvELJuUDFLYDzUsCfNAApKs/ustQFkFHP0C6PGkAmxz4j536HgdQYplWwUuEEAeF7xmkDgOGbSxyVo+EvEkCZzT0XZT7AmwpAXEEEWOutdANT7WvbVo6gB7iKBNvL+guBAAAAAElFTkSuQmCC"); --img_download:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABgAAAAYCAMAAADXqc3KAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAADxQTFRFAAAA////////////+f783ffv/v//3ffv3ffv5Pnz8fz56vr16/r23ffv9v377/v4////3vfw5Pny3ffvbBfD6AAAABR0Uk5TAP+TRfsGWm8m6uDv0U/LfyiIepVDO0gQAAAAbElEQVR4nNWPuQ6AIBAFF+S+j///V0FjWJTEwsqpyEzxFoBXKGnQvwcuQw9B8kkLIRzrgbn2ROGQFwwFbYe3GgUfR4gejyRqzg1D0+2q2gszdb6ql9x2bH54AFW0Lmrxc1BSPvw2gQKZ+BQW7MaRAtfJQ2l0AAAAAElFTkSuQmCC"); + --img_mic:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACQAAAAkAgMAAACcbnALAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAAxQTFRFAQEB/f39W1tbsrKyhr4L4QAAAAR0Uk5TBv+O8t7TK14AAACVSURBVHicY2BAAatgDK7/C6As3dAKKGtqaByEwXQ1NLyBtizuJiCr6QGQxXsIyGouQGU5AVmNIBZ3EpDVCVLHXTA1NIwXxGKO2Boap3oAZFycduiLrWADpzfta7oGtsP0AQN3DMRrdau2QjzHFBoaCnEBw9bQKKh/maY1wIJjGgM6i+n/9f8Yrt///x9Ui9aqFQzIAACxbkd5KhPnwgAAAABJRU5ErkJggg=="); + --img_mic_live:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACQAAAAkAgMAAACcbnALAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAAxQTFRFAAAAHvoBSldJTaRCcSDH5wAAAAR0Uk5TBv1875jbENYAAACVSURBVHicY2BAAatgDO7/D6As3dAKKGtqaBiEwXQ1NLyBtizuJiCrCeQG3kNAVnMBkMUKZgWAWE5AViOIxZ0EZHWC1HEWAFm8CUAWc8TW0DjVAyDj4oCu3wo28HrX/0XXwXaYPtDgj4F4LXzVVYjnmENDQw9APLc3tA7qX6brDbDgmMaAzsLq+v///0O1rFoFD0owAADWKEefP5UQnwAAAABJRU5ErkJggg=="); + --img_mic_off:url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACQAAAAkAgMAAACcbnALAAAAAXNSR0IB2cksfwAAAAlwSFlzAAAOxAAADsQBlSsOGwAAAAxQTFRFAQEBoKCgW1tbmpqa1zTSqAAAAAR0Uk5TBfGLNb6PCuwAAADJSURBVHicXdCxDoIwEAbgi4YBjeEpGHRnNzqbOHgtMSRldOQFCFAnHwBmR8M7mPASfZSOJN7RMmCnL7n/mrsDWLx+xgbfEA2sGFNIjqwWBRjWChGfOHh9M3CSpvBSU5E1SnCStvDyRVIpvaStaieF46RQl9LqnBScrMLmyjqb0esirVNolcEX57amRBGwIiqKmAdN1B4fLX9oKt1pQVijzSG8kXZZLPo25VjBe3AsGWjLOw8Q0TBdPZ+jg3/NHQvRhXzLof8sDvsDMJVHBpRtjD4AAAAASUVORK5CYII="); } body { @@ -315,6 +318,31 @@ Current version: 141 background-color: #98989a; } + .showmicbig{ + width: 32px; + height: 32px; + margin:auto; + background-repeat: no-repeat !important; + background-position: center !important; + background-image: var(--img_mic) !important; + } + .showmiclivebig{ + width: 32px; + height: 32px; + margin:auto; + background-repeat: no-repeat !important; + background-position: center !important; + background-image: var(--img_mic_live) !important; + } + .showmicoffbig{ + width: 32px; + height: 32px; + margin:auto; + background-repeat: no-repeat !important; + background-position: center !important; + background-image: var(--img_mic_off) !important; + } + #anoterowcontainer { display: none; } @@ -1918,6 +1946,16 @@ Current version: 141 .chat_msg_send_btn:disabled { background: #838383 none repeat scroll 0 0; } + .chat_msg_send_btn.showmic{ + background-image: var(--img_mic) !important; + } + .chat_msg_send_btn.showmiclive{ + background-image: var(--img_mic_live) !important; + } + .chat_msg_send_btn.showmicoff{ + background-image: var(--img_mic_off) !important; + } + .chat_msg_send_btn_abort { background: #b73333 none repeat scroll 0 0; @@ -2656,6 +2694,10 @@ Current version: 141 } function escapeHtml(unsafe) { + if(localsettings.no_escape_html) + { + return unsafe; + } return unsafe .replace(/&/g, "&") .replace(/= 0 && koboldcpp_has_vision); } + function is_using_kcpp_with_whisper() + { + return (custom_kobold_endpoint!="" && koboldcpp_version && koboldcpp_version!="" && compare_version_str(koboldcpp_version, "1.66") >= 0 && koboldcpp_has_whisper); + } + //0 is none, 1 is pseudostreaming, 2 is true poll-streaming, 3 is sse-streaming function determine_streaming_type() @@ -5673,65 +5730,91 @@ Current version: 141 } function load_tavern_obj(obj) { - console.log("Loading tavern obj"); - if(obj.spec=="chara_card_v2" && obj.data!=null) + let load_tav_obj_confirm = function(usechatmode) { - obj = obj.data; - } - let chatopponent = obj.name?obj.name:defaultchatopponent; - let myname = ((localsettings.chatname && localsettings.chatname!="")?localsettings.chatname:"User"); - let memory = obj.description?("Persona: "+obj.description):""; - memory += obj.personality?("\nPersonality: "+obj.personality):""; - let scenario = obj.scenario?obj.scenario:""; - let examplemsg = obj.mes_example?obj.mes_example:""; - let greeting = obj.first_mes?obj.first_mes:""; - let sysprompt = obj.system_prompt?obj.system_prompt:""; - - //post process - if(scenario!="") - { - scenario = "\n[Scenario: "+scenario+"]"; - } - if(examplemsg!="") - { - examplemsg = "\n"+examplemsg; - } - if(sysprompt!="") - { - sysprompt = sysprompt+"\n"; - } - let combinedmem = sysprompt + memory + scenario + examplemsg; - let agnaidatafieldsempty = scenario + examplemsg + (obj.personality?obj.personality:"") + greeting; - //check if it's a world info only card, if so, do not restart game - if(combinedmem.trim()=="" && greeting=="" && obj.entries) - { - current_wi = load_tavern_wi(obj,chatopponent,myname); - } - else if(agnaidatafieldsempty.trim()=="" && obj.entries && obj.kind=="memory") - { - current_wi = load_agnai_wi(obj,chatopponent,myname); - } - else - { - restart_new_game(false); - localsettings.chatname = myname; - localsettings.chatopponent = chatopponent; - gametext_arr.push("\n"+chatopponent+": "+greeting); - current_memory = combinedmem + "\n***"; - localsettings.opmode = 3; - localsettings.gui_type_chat = 2; - localsettings.multiline_replies = true; - //handle character book - if(obj.character_book && obj.character_book.entries && obj.character_book.entries.length>0) + console.log("Loading tavern obj"); + if(obj.spec=="chara_card_v2" && obj.data!=null) { - current_wi = load_tavern_wi(obj.character_book,chatopponent,myname); + obj = obj.data; } - else if(obj.entries && obj.entries.length>0) + let chatopponent = obj.name?obj.name:defaultchatopponent; + let myname = ((localsettings.chatname && localsettings.chatname!="")?localsettings.chatname:"User"); + let memory = obj.description?("Persona: "+obj.description):""; + memory += obj.personality?("\nPersonality: "+obj.personality):""; + let scenario = obj.scenario?obj.scenario:""; + let examplemsg = obj.mes_example?obj.mes_example:""; + let greeting = obj.first_mes?obj.first_mes:""; + let sysprompt = obj.system_prompt?obj.system_prompt:""; + + //post process + if(scenario!="") + { + scenario = "\n[Scenario: "+scenario+"]"; + } + if(examplemsg!="") + { + examplemsg = "\n"+examplemsg; + } + if(sysprompt!="") + { + sysprompt = sysprompt+"\n"; + } + let combinedmem = sysprompt + memory + scenario + examplemsg; + let agnaidatafieldsempty = scenario + examplemsg + (obj.personality?obj.personality:"") + greeting; + //check if it's a world info only card, if so, do not restart game + if(combinedmem.trim()=="" && greeting=="" && obj.entries) + { + current_wi = load_tavern_wi(obj,chatopponent,myname); + } + else if(agnaidatafieldsempty.trim()=="" && obj.entries && obj.kind=="memory") { current_wi = load_agnai_wi(obj,chatopponent,myname); } + else + { + restart_new_game(false); + localsettings.chatname = myname; + localsettings.chatopponent = chatopponent; + current_memory = combinedmem + "\n***"; + localsettings.multiline_replies = true; + if(usechatmode) + { + localsettings.opmode = 3; + localsettings.gui_type_chat = 2; + gametext_arr.push("\n"+chatopponent+": "+greeting); + } + else + { + localsettings.opmode = 4; + localsettings.gui_type_instruct = 2; + localsettings.inject_chatnames_instruct = true; + gametext_arr.push(instructendplaceholder+chatopponent+": "+greeting); + } + //handle character book + if(obj.character_book && obj.character_book.entries && obj.character_book.entries.length>0) + { + current_wi = load_tavern_wi(obj.character_book,chatopponent,myname); + } + else if(obj.entries && obj.entries.length>0) + { + current_wi = load_agnai_wi(obj,chatopponent,myname); + } + } + render_gametext(true); + } + + if(localsettings.show_advanced_load) + { + msgboxYesNo("Import Character Card in Instruct Mode?\n\nYes = Instruct Mode Used\nNo = Chat Mode Used\n\nIf unsure, select 'No'.","Import Tavern Card", ()=>{ + load_tav_obj_confirm(false); + },()=>{ + load_tav_obj_confirm(true); + }); + } + else + { + load_tav_obj_confirm(true); } - render_gametext(true); } function load_ooba_obj(obj) { @@ -6761,6 +6844,13 @@ Current version: 141 document.getElementById("workercontainer").classList.add("hidden"); document.getElementById("myownworkercontainer").classList.add("hidden"); } + function is_aesthetic_ui() + { + return (localsettings.gui_type_story!=0 && localsettings.opmode==1) + ||(localsettings.gui_type_adventure!=0 && localsettings.opmode==2) + ||(localsettings.gui_type_chat!=0 && localsettings.opmode==3) + ||(localsettings.gui_type_instruct!=0 && localsettings.opmode==4); + } function is_popup_open() { return !( @@ -7154,7 +7244,7 @@ Current version: 141 function togglepalmmodel() { let mdlname = document.getElementById("custom_palm_model").value; - if(mdlname=="gemini-1.5-pro-latest") + if(mdlname=="gemini-1.5-pro-latest" || mdlname=="gemini-1.5-flash-latest") { document.getElementById("gemini_system_instruction").classList.remove("hidden"); if(localsettings.saved_palm_jailbreak=="") @@ -7536,6 +7626,7 @@ Current version: 141 koboldcpp_version = data.version; console.log("KoboldCpp Detected: " + koboldcpp_version); koboldcpp_has_vision = (data.vision?true:false); + koboldcpp_has_whisper = (data.transcribe?true:false); let has_password = (data.protected?true:false); //also check against kcpp's max true context length @@ -7627,7 +7718,7 @@ Current version: 141 if (userinput != null && userinput!="") { custom_kobold_key = document.getElementById("customkoboldkey").value = localmodekey = localsettings.saved_kai_key = userinput.trim(); } - },false,true); + },false,false,true); } }else{ @@ -8500,6 +8591,7 @@ Current version: 141 } document.getElementById("setgrammar").disabled = !is_using_kcpp_with_grammar(); + document.getElementById("voice_typing_mode").disabled = !is_using_kcpp_with_whisper(); document.getElementById("grammar_retain_state").disabled = document.getElementById("setgrammar").disabled; if(custom_kobold_endpoint!="") @@ -8574,6 +8666,7 @@ Current version: 141 toggle_tts_mode(); document.getElementById("beep_on").checked = localsettings.beep_on; document.getElementById("notify_on").checked = localsettings.notify_on; + document.getElementById("no_escape_html").checked = localsettings.no_escape_html; document.getElementById("narrate_both_sides").checked = localsettings.narrate_both_sides; document.getElementById("narrate_only_dialog").checked = localsettings.narrate_only_dialog; toggle_opmode(); @@ -8823,6 +8916,7 @@ Current version: 141 localsettings.xtts_voice = document.getElementById("xtts_voices").value; localsettings.beep_on = (document.getElementById("beep_on").checked?true:false); localsettings.notify_on = (document.getElementById("notify_on").checked?true:false); + localsettings.no_escape_html = (document.getElementById("no_escape_html").checked?true:false); localsettings.narrate_both_sides = (document.getElementById("narrate_both_sides").checked?true:false); localsettings.narrate_only_dialog = (document.getElementById("narrate_only_dialog").checked?true:false); localsettings.auto_ctxlen = (document.getElementById("auto_ctxlen").checked ? true : false); @@ -8863,10 +8957,7 @@ Current version: 141 localsettings.img_aspect = defaultsettings.img_aspect; } - if((localsettings.gui_type_story!=0 && localsettings.opmode==1) - ||(localsettings.gui_type_adventure!=0 && localsettings.opmode==2) - ||(localsettings.gui_type_chat!=0 && localsettings.opmode==3) - ||(localsettings.gui_type_instruct!=0 && localsettings.opmode==4)) + if(is_aesthetic_ui()) { //kick out of edit mode if(document.getElementById("allowediting")) @@ -8904,6 +8995,12 @@ Current version: 141 localsettings.sampler_seed = cleannum(localsettings.sampler_seed, -1, 999999); toggle_invert_colors(); + voice_typing_enabled = (document.getElementById("voice_typing_mode").checked?true:false); + if(voice_typing_enabled && is_using_kcpp_with_whisper()) + { + init_voice_typing(); + } + hide_popups(); autosave();//need to always autosave, so that we can switch back to non persistent sessions render_gametext(false); @@ -9390,6 +9487,7 @@ Current version: 141 } function restart_new_game(save = true, keep_memory = false) { + xtts_is_playing = false; idle_timer = 0; gametext_arr = []; redo_arr = []; @@ -9419,6 +9517,7 @@ Current version: 141 welcome = ""; last_known_filename = "saved_story.json"; is_impersonate_user = false; + voice_is_processing = false; if (!keep_memory) { personal_notes = ""; @@ -9489,7 +9588,7 @@ Current version: 141 render_gametext(false); } - function replace_placeholders_direct(inputtxt) + function replace_placeholders_direct(inputtxt,escape=false) { inputtxt = replaceAll(inputtxt,instructstartplaceholder,get_instruct_starttag(false)); inputtxt = replaceAll(inputtxt,instructendplaceholder,get_instruct_endtag(false)); @@ -9497,8 +9596,16 @@ Current version: 141 inputtxt = replaceAll(inputtxt,instructstartplaceholder.trim(),get_instruct_starttag(false)); inputtxt = replaceAll(inputtxt,instructendplaceholder.trim(),get_instruct_endtag(false)); - inputtxt = replaceAll(inputtxt,"{{user}}",localsettings.chatname?localsettings.chatname:"User",true); - inputtxt = replaceAll(inputtxt,"{{char}}",localsettings.chatopponent?localsettings.chatopponent:defaultchatopponent,true); + if(escape) + { + inputtxt = replaceAll(inputtxt,"{{user}}",escapeHtml(localsettings.chatname?localsettings.chatname:"User"),true); + inputtxt = replaceAll(inputtxt,"{{char}}",escapeHtml(localsettings.chatopponent?localsettings.chatopponent:defaultchatopponent),true); + } + else + { + inputtxt = replaceAll(inputtxt,"{{user}}",(localsettings.chatname?localsettings.chatname:"User"),true); + inputtxt = replaceAll(inputtxt,"{{char}}",(localsettings.chatopponent?localsettings.chatopponent:defaultchatopponent),true); + } for(let i=0;i { console.log("XTTS Speak Error: " + error); }); @@ -10014,7 +10127,12 @@ Current version: 141 const playSound = audioContext.createBufferSource(); playSound.buffer = decodedData; playSound.connect(audioContext.destination); + xtts_is_playing = true; playSound.start(audioContext.currentTime); + playSound.onended = function() { + xtts_is_playing = false; + console.log("Audio finished playing"); + }; }).catch((error) => { console.log("AllTalk Speak Error: " + error); }); @@ -10706,6 +10824,22 @@ Current version: 141 return seqs; } + function cleanup_story_completion(resp) + { + if(!gametext_arr[gametext_arr.length-1].endsWith(" ") && !gametext_arr[gametext_arr.length-1].endsWith("\n")) + { + if(/^\.\.\.[a-zA-Z0-9]/.test(resp)) + { + resp = resp.slice(3); + } + if(/^[^\p{P}\p{Z}\s]/u.test(resp)) + { + resp = " "+resp; + } + } + return resp; + } + function dispatch_submit_generation(submit_payload, input_was_empty) //if input is not empty, always unban eos { console.log(submit_payload); @@ -10929,20 +11063,30 @@ Current version: 141 claude_payload = { "model": custom_claude_model, - "messages": [{"role": "user", "content": submit_payload.prompt}], + "messages": [], "max_tokens": submit_payload.params.max_length, "top_k": (submit_payload.params.top_k<1?300:submit_payload.params.top_k), "temperature": submit_payload.params.temperature, "top_p": submit_payload.params.top_p, }; + claude_payload.messages.push({"role": "user", "content": submit_payload.prompt}) if(sysprompt) { claude_payload.system = sysprompt; } + if(localsettings.opmode==1) + { + claude_payload.system = "Always respond with a direct partial continuation of the story immediately from the latest word."; + if(sysprompt) + { + claude_payload.system = sysprompt +"\n"+ claude_payload.system; + } + } if(assistantprompt) { claude_payload.messages.push({"role": "assistant", "content": assistantprompt}); } + } else { @@ -11008,6 +11152,10 @@ Current version: 141 if(custom_claude_key != "" && data.content && data.content.length > 0 && data.content[0].text) { data.completion = data.content[0].text; //for claudev3 + if(localsettings.opmode==1 && gametext_arr.length>0 && data.completion!="") + { + data.completion = cleanup_story_completion(data.completion); + } } if (custom_claude_key != "" && data.completion != null && data.completion != "") { @@ -11113,7 +11261,7 @@ Current version: 141 }; let sysinst = document.getElementById("gemini_system_instruction").value; - if(sysinst!="" && mdlname=="gemini-1.5-pro-latest") + if(sysinst!="" && (mdlname=="gemini-1.5-pro-latest" || mdlname=="gemini-1.5-flash-latest")) { payload["systemInstruction"] = { "role": "system", @@ -11144,6 +11292,11 @@ Current version: 141 synchro_polled_response = data.candidates[0].output; }else if (custom_palm_key != "" && data.candidates != null && data.candidates.length>0 && data.candidates[0].content && data.candidates[0].content.parts != null && data.candidates[0].content.parts.length>0) { synchro_polled_response = data.candidates[0].content.parts[0].text; + //try to handle the stripping of spaces + if(localsettings.opmode==1 && gametext_arr.length>0 && synchro_polled_response!="") + { + synchro_polled_response = cleanup_story_completion(synchro_polled_response); + } } else { //error occurred, maybe captcha failed @@ -12296,7 +12449,7 @@ Current version: 141 var idle_timer = 0; //used in chat mode to send multi replies var idle_triggered_counter = 0; var idle_backoff_array = [15000,60000,300000,1200000,14400000]; - function poll_background_tasks() + function poll_idle_responses() { let idle_timer_max = 0; if(localsettings.idle_duration>0) @@ -12337,6 +12490,239 @@ Current version: 141 } else { idle_timer = 0; } + + } + + function ready_to_record() + { + let currentlySpeaking = false; + if ('speechSynthesis' in window) { + currentlySpeaking = window.speechSynthesis.speaking; + } + return (voice_typing_enabled && is_using_kcpp_with_whisper() + && !document.getElementById("btnsend").disabled + && !voice_is_processing && !voice_is_recording && isVoiceInputConfigured + && !currentlySpeaking && !xtts_is_playing && !is_popup_open()); + } + + var isVoiceInputConfigured = false; + function init_voice_typing() + { + if (isVoiceInputConfigured) { + return; + } + isVoiceInputConfigured = true; + + //under BSD-3-Clause license + //original source https://github.com/kdavis-mozilla/vad.js Copyright (c) 2015, Kelly Daviss + let VAD=function(t){for(var e in this.options={fftSize:256,bufferLen:256,voice_stop:function(){},voice_start:function(){},smoothingTimeConstant:.99,energy_offset:1e-8,energy_threshold_ratio_pos:2,energy_threshold_ratio_neg:.5,energy_integration:1,filter:[{f:200,v:0},{f:2e3,v:1}],source:null,context:null},t)t.hasOwnProperty(e)&&(this.options[e]=t[e]);if(!this.options.source)throw Error("The options must specify a MediaStreamAudioSourceNode.");this.options.context=this.options.source.context,this.hertzPerBin=this.options.context.sampleRate/this.options.fftSize,this.iterationFrequency=this.options.context.sampleRate/this.options.bufferLen,this.iterationPeriod=1/this.iterationFrequency,this.setFilter=function(t){this.filter=[];for(var e=0,i=this.options.fftSize/2;ethis.energy_threshold_pos?this.voiceTrend=this.voiceTrend+1>this.voiceTrendMax?this.voiceTrendMax:this.voiceTrend+1:t<-this.energy_threshold_neg?this.voiceTrend=this.voiceTrend-10?this.voiceTrend--:this.voiceTrend<0&&this.voiceTrend++;var e=!1,i=!1;this.voiceTrend>this.voiceTrendStart?e=!0:this.voiceTrend0||!i?this.energy_offset+=s:this.energy_offset+=10*s,this.energy_offset=this.energy_offset<0?0:this.energy_offset,this.energy_threshold_pos=this.energy_offset*this.options.energy_threshold_ratio_pos,this.energy_threshold_neg=this.energy_offset*this.options.energy_threshold_ratio_neg,e&&!this.vadState&&(this.vadState=!0,this.options.voice_start()),i&&this.vadState&&(this.vadState=!1,this.options.voice_stop()),t}}; + + let audioBufferToWavBlob = function (audioBuffer) { + let writeWavString = function (view, offset, string) { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } + } + const numOfChan = audioBuffer.numberOfChannels, + length = audioBuffer.length * numOfChan * 2 + 44, + buffer = new ArrayBuffer(length), + view = new DataView(buffer), + channels = [], + sampleRate = 16000, + bitDepth = 16; + writeWavString(view, 0, 'RIFF'); + view.setUint32(4, 44 + audioBuffer.length * 2 - 8, true); + writeWavString(view, 8, 'WAVE'); + writeWavString(view, 12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, numOfChan, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * numOfChan * 2, true); + view.setUint16(32, numOfChan * 2, true); + view.setUint16(34, bitDepth, true); + writeWavString(view, 36, 'data'); + view.setUint32(40, audioBuffer.length * numOfChan * 2, true); + for (let i = 0; i < audioBuffer.numberOfChannels; i++) { + channels.push(audioBuffer.getChannelData(i)); + } + let offset = 44; + for (let i = 0; i < audioBuffer.length; i++) { + for (let channel = 0; channel < numOfChan; channel++) { + const sample = Math.max(-1, Math.min(1, channels[channel][i])); + view.setInt16(offset, sample < 0 ? sample * 0x8000 : sample * 0x7FFF, true); + offset += 2; + } + } + return new Blob([buffer], { type: 'audio/wav' }); + } + + let audioBlobToDecodedAudioBuffer = function(inBlob, onDone) + { + let reader = new window.FileReader(); + reader.readAsArrayBuffer(inBlob); + reader.onloadend = function() { + let arrayBuffer = reader.result; + window.AudioContext = window.AudioContext || window.webkitAudioContext; + let audioContext = new AudioContext({ sampleRate: 16000 }); + audioContext.decodeAudioData(arrayBuffer, (buffer)=>{ + onDone(buffer); + }); + } + } + + let concatenateAudioBuffers = function(buffer1, buffer2) { + const audioContext = new (window.AudioContext || window.webkitAudioContext)(); + const numberOfChannels = Math.min(buffer1.numberOfChannels, buffer2.numberOfChannels); + const tmp = audioContext.createBuffer( + numberOfChannels, + buffer1.length + buffer2.length, + buffer1.sampleRate + ); + for (let i = 0; i < numberOfChannels; i++) { + const channel = tmp.getChannelData(i); + channel.set(buffer1.getChannelData(i), 0); + channel.set(buffer2.getChannelData(i), buffer1.length); + } + return tmp; + } + + let prerecorder, preaudiobuffers = [], preaudioblobs = []; //will store 2 preblobs at a time + let onRecordingReady = function (e) { + let completeRecording = new Blob([e.data], { type: 'audio/webm' }); + let audiodatareader = new window.FileReader(); + + if(preaudioblobs.length<2) + { + audioBlobToDecodedAudioBuffer(completeRecording,(buffer)=>{ + let wavblob = audioBufferToWavBlob(finalbuf); + audiodatareader.readAsDataURL(wavblob); + }); + } else { + audioBlobToDecodedAudioBuffer(completeRecording,(buffer)=>{ + audioBlobToDecodedAudioBuffer(preaudioblobs[0],(buffer2)=>{ + audioBlobToDecodedAudioBuffer(preaudioblobs[1],(buffer3)=>{ + let prefix = concatenateAudioBuffers(buffer2,buffer3); + let finalbuf = concatenateAudioBuffers(prefix,buffer); + let wavblob = audioBufferToWavBlob(finalbuf); + audiodatareader.readAsDataURL(wavblob); + }); + }); + }); + } + + audiodatareader.onloadend = function () { + let dataurl = audiodatareader.result; + dispatch_transcribe(dataurl); + } + } + + + let recorder, is_speaking = false, speaking_counter = 0; + // get audio stream from user's mic + navigator.mediaDevices.getUserMedia({ + audio: true + }).then(function (stream) { + prerecorder = new MediaRecorder(stream); + prerecorder.addEventListener('dataavailable', (ev)=>{ + preaudiobuffers.push(ev.data); + if(preaudiobuffers.length>2) + { + preaudiobuffers.shift(); + } + }); + setInterval(()=>{ + if (prerecorder.state !== "inactive") { + prerecorder.stop(); + } + if(ready_to_record()){ + prerecorder.start(); + } + }, 500); + + recorder = new MediaRecorder(stream); + recorder.addEventListener('dataavailable', onRecordingReady); + window.AudioContext = window.AudioContext || window.webkitAudioContext; + let audioContext = new AudioContext({ sampleRate: 16000 }); + let source = audioContext.createMediaStreamSource(stream); + let options = { + source: source, + voice_stop: function () { + is_speaking = false; + let check_speak_counter = speaking_counter; + setTimeout(() => { + if (voice_is_recording && !is_speaking && speaking_counter == check_speak_counter) { + //generate prerecorder blobs (prebuffer 1sec) + preaudioblobs = []; + for(let i=0;i x.json()) + .then(resp => { + console.log(resp); + voice_is_processing = false; + update_submit_button(false); + if(resp && resp.text && resp.text!="") + { + let trimmed = resp.text.trim(); + let noise = trimmed.startsWith("(") && trimmed.endsWith(")"); + let blank = trimmed.startsWith("[") && trimmed.endsWith("]"); + let willsubmit = (document.getElementById("btnsend").disabled ? false : true); + if(willsubmit && trimmed && !noise && !blank) + { + document.getElementById("input_text").value = trimmed; + submit_generation(); + } + } + }).catch((error) => { + console.log("Transcribe Error: " + error); + voice_is_processing = false; + update_submit_button(false); + }); } //clock speed is 500ms per tick @@ -12836,6 +13222,25 @@ Current version: 141 else { document.getElementById("btnsend").innerHTML = "Submit"; } + document.getElementById("chat_msg_send_btn").classList.remove("showmic"); + document.getElementById("chat_msg_send_btn").classList.remove("showmiclive"); + document.getElementById("chat_msg_send_btn").classList.remove("showmicoff"); + if(voice_typing_enabled && is_using_kcpp_with_whisper()) + { + if (voice_is_processing) { + document.getElementById("chat_msg_send_btn").classList.add("showmicoff"); + document.getElementById("btnsend").innerHTML = "
Busy"; + } else if (voice_is_recording) { + document.getElementById("chat_msg_send_btn").classList.add("showmiclive"); + document.getElementById("btnsend").innerHTML = "
Record"; + } else if (ready_to_record()) { + document.getElementById("chat_msg_send_btn").classList.add("showmic"); + document.getElementById("btnsend").innerHTML = "
Standby"; + } else { + document.getElementById("chat_msg_send_btn").classList.add("showmicoff"); + document.getElementById("btnsend").innerHTML = "
Busy"; + } + } } else { if(full_update) @@ -12851,6 +13256,8 @@ Current version: 141 } } } + document.getElementById("chat_msg_send_btn").disabled = document.getElementById("btnsend").disabled; + } function handle_autoscroll(alwaysscroll=true) @@ -12971,7 +13378,7 @@ Current version: 141 fulltxt = concat_gametext(false, "", "%SpnStg%", "%SpnEtg%",true); } else { fulltxt = concat_gametext(false, "", "", "",true); - fulltxt = replace_placeholders(fulltxt); + fulltxt = replace_placeholders(fulltxt,true); } @@ -13184,19 +13591,14 @@ Current version: 141 else { document.getElementById("fvico").href = favicon_busy; } - update_submit_button(true); //full update for submit button, otherwise just text when not generating - // Render onto enhanced chat interface if selected. Currently only applicable to Chat & Instruct modes. - let isStyleApplicable = ( - (localsettings.opmode==1 && localsettings.gui_type_story!=0) || - (localsettings.opmode==2 && localsettings.gui_type_adventure!=0) || - (localsettings.opmode==3 && localsettings.gui_type_chat!=0) || - (localsettings.opmode==4 && localsettings.gui_type_instruct!=0)); + // Render onto enhanced chat interface if selected. + let isStyleApplicable = is_aesthetic_ui(); if (!inEditMode && isStyleApplicable) { let textToRender = (gametext_arr.length == 0 ? document.getElementById("gametext").innerHTML : concat_gametext(false, "", "", "", true)); - textToRender = replace_placeholders(textToRender); + textToRender = replace_placeholders(textToRender,true); if(localsettings.opmode==3 && localsettings.gui_type_chat==1) { @@ -13253,6 +13655,8 @@ Current version: 141 document.getElementById("normalinterface").classList.remove("hidden"); } + update_submit_button(true); //full update for submit button, otherwise just text when not generating + document.getElementById("btnautogenmem").disabled = document.getElementById("btnsend").disabled; if (localsettings.persist_session && save) { @@ -14105,6 +14509,23 @@ Current version: 141 },false); } } + function add_another_participant() + { + inputBox("Turn it into a group chat by adding more AI characters.\n\nInput name of additional character:","Add Another Participant","","[Enter Character Name]", ()=>{ + let userinput = getInputBoxValue(); + userinput = userinput.trim(); + if(userinput!="") + { + if(document.getElementById("chatopponent").value=="") + { + document.getElementById("chatopponent").value = userinput; + }else{ + document.getElementById("chatopponent").value += "||$||"+userinput; + handle_bot_name_onchange(); + } + } + },false); + } function confirm_groupchat_select() { groupchat_removals = []; @@ -14498,9 +14919,9 @@ Current version: 141 input = input.replaceAll(mynameregex3, '{{userplaceholder}}'); if(as.show_chat_names) { - input = input.replaceAll("{{userplaceholder}}", `{{userplaceholder}}

`+localsettings.chatname+`

`); + input = input.replaceAll("{{userplaceholder}}", `{{userplaceholder}}

`+escapeHtml(localsettings.chatname)+`

`); input = input.replaceAll(othernamesregex, function(match) { - return "{{botplaceholder}}

" + match.substring(0,match.length-2).trim() + "

"; + return "{{botplaceholder}}

" + escapeHtml(match.substring(0,match.length-2).trim()) + "

"; }); } else @@ -14511,6 +14932,13 @@ Current version: 141 you = "{{userplaceholder}}"; bot = "{{botplaceholder}}"; } + if(localsettings.opmode==4 && localsettings.inject_chatnames_instruct && localsettings.chatname!="" && localsettings.chatopponent!="") + { + let m_name = localsettings.chatname + ": "; + let m_opp = localsettings.chatopponent + ": "; + input = replaceAll(input, m_name, `

` + escapeHtml(localsettings.chatname) + `

`); + input = replaceAll(input, m_opp, `

` + escapeHtml(localsettings.chatopponent) + `

`); + } let portraitsStyling = // Also, implement portraits as css classes. Now chat entries can reuse them instead of recreating them. `