diff --git a/README.md b/README.md index ebc279593..9cf6475da 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Finally, obtain and load a GGUF model. See [here](#Obtaining-a-GGUF-model) ## MacOS (Precompiled Binary) - PyInstaller binaries for Modern ARM64 MacOS (M1, M2, M3) are now available! **[Simply download the MacOS binary](https://github.com/LostRuins/koboldcpp/releases/latest)** - In a MacOS terminal window, set the file to executable `chmod +x koboldcpp-mac-arm64` and run it with `./koboldcpp-mac-arm64`. +- In newer MacOS you may also have to whitelist it in security settings if it's blocked. [Here's a video guide](https://youtube.com/watch?v=NOW5dyA_JgY). - Alternatively, or for older x86 MacOS computers, you can clone the repo and compile from source code, see Compiling for MacOS below. - Finally, obtain and load a GGUF model. See [here](#Obtaining-a-GGUF-model) @@ -120,7 +121,8 @@ when you can't use the precompiled binary directly, we provide an automated buil - GPU acceleration for Termux may be possible but I have not explored it. If you find a good cross-device solution, do share or PR it. ## AMD Users -- Please check out https://github.com/YellowRoseCx/koboldcpp-rocm +- For most users, you can get very decent speeds by selecting the **Vulkan** option instead, which supports both Nvidia and AMD GPUs. +- Alternatively, you can try the ROCM fork at https://github.com/YellowRoseCx/koboldcpp-rocm ## Third Party Resources - These unofficial resources have been contributed by the community, and may be outdated or unmaintained. No official support will be provided for them! diff --git a/klite.embd b/klite.embd index 230dffb85..96d0d48ea 100644 --- a/klite.embd +++ b/klite.embd @@ -4300,6 +4300,7 @@ Current version indicated by LITEVER below. const default_openrouter_base = "https://openrouter.ai/api/v1"; const default_mistralai_base = "https://api.mistral.ai/v1"; const default_featherless_base = "https://api.featherless.ai/v1"; + const default_grok_base = "https://api.x.ai/v1"; const default_oai_base = "https://api.openai.com"; const default_claude_base = "https://api.anthropic.com"; const default_palm_base = "https://generativelanguage.googleapis.com/v1beta2/models/text-bison-001:generateText?key="; @@ -4446,6 +4447,7 @@ Current version indicated by LITEVER below. saved_openrouter_key: "", saved_mistralai_key: "", saved_featherless_key: "", + saved_grok_key:"", saved_claude_key: "", //do not ever share this in save files! saved_claude_addr: default_claude_base, //do not ever share this in save files! saved_palm_key: "", //do not ever share this in save files! @@ -4513,7 +4515,7 @@ Current version indicated by LITEVER below. img_img2imgstr: 0.6, img_steps: 20, img_sampler: "Euler a", - img_aspect:0, //0=square,1=portrait,2=landscape + img_aspect:0, //0=square,1=portrait,2=landscape,3=bigsquare save_images: true, save_remote_images: false, prompt_for_savename: false, @@ -6387,6 +6389,7 @@ Current version indicated by LITEVER below. new_save_storyobj.savedsettings.saved_openrouter_key = ""; new_save_storyobj.savedsettings.saved_mistralai_key = ""; new_save_storyobj.savedsettings.saved_featherless_key = ""; + new_save_storyobj.savedsettings.saved_grok_key = ""; new_save_storyobj.savedsettings.saved_palm_key = ""; new_save_storyobj.savedsettings.saved_cohere_key = ""; @@ -6730,6 +6733,7 @@ Current version indicated by LITEVER below. let tmp_or1 = localsettings.saved_openrouter_key; let tmp_mai = localsettings.saved_mistralai_key; let tmp_fai = localsettings.saved_featherless_key; + let tmp_grok = localsettings.saved_grok_key; let tmp_claude1 = localsettings.saved_claude_key; let tmp_claude2 = localsettings.saved_claude_addr; let tmp_palm1 = localsettings.saved_palm_key; @@ -6770,6 +6774,7 @@ Current version indicated by LITEVER below. localsettings.saved_openrouter_key = tmp_or1; localsettings.saved_mistralai_key = tmp_mai; localsettings.saved_featherless_key = tmp_fai; + localsettings.saved_grok_key = tmp_grok; localsettings.saved_claude_key = tmp_claude1; localsettings.saved_claude_addr = tmp_claude2; localsettings.saved_palm_key = tmp_palm1; @@ -8782,6 +8787,8 @@ Current version indicated by LITEVER below. return document.getElementById("custom_mistralai_model"); case "8": return document.getElementById("custom_featherless_model"); + case "9": + return document.getElementById("custom_grok_model"); default: return document.getElementById("custom_oai_model"); } @@ -8887,7 +8894,12 @@ Current version indicated by LITEVER below. } else { - msgbox(JSON.stringify(data.error.message),"Error Encountered",false,false); + let errmsg = data.error.message; + if(!errmsg) + { + errmsg = data.error; + } + msgbox(JSON.stringify(errmsg),"Error Encountered",false,false); } }) .catch(error => { @@ -8944,6 +8956,7 @@ Current version indicated by LITEVER below. document.getElementById("custom_openrouter_model").classList.add("hidden"); document.getElementById("custom_mistralai_model").classList.add("hidden"); document.getElementById("custom_featherless_model").classList.add("hidden"); + document.getElementById("custom_grok_model").classList.add("hidden"); document.getElementById("hordeloadmodelcontainer").classList.add("hidden"); document.getElementById("coherecustom").classList.add("hidden"); @@ -8961,12 +8974,13 @@ Current version indicated by LITEVER below. document.getElementById("customkoboldkey").value = localsettings.saved_kai_key; } } - else if(epchoice==2 || epchoice==3 || epchoice==7 || epchoice==8) + else if(epchoice==2 || epchoice==3 || epchoice==7 || epchoice==8 || epchoice==9) { document.getElementById("oaicustom").classList.remove("hidden"); document.getElementById("openrouterdesc").classList.add("hidden"); document.getElementById("mistralaidesc").classList.add("hidden"); document.getElementById("featherlessdesc").classList.add("hidden"); + document.getElementById("grokdesc").classList.add("hidden"); document.getElementById("oaidesc").classList.add("hidden"); if(epchoice==2) { @@ -8998,6 +9012,14 @@ Current version indicated by LITEVER below. document.getElementById("custom_oai_endpoint").value = default_featherless_base; try_fetch_oai_models_auto(); } + else if(epchoice==9) + { + document.getElementById("custom_grok_model").classList.remove("hidden"); + document.getElementById("grokdesc").classList.remove("hidden"); + document.getElementById("custom_oai_endpoint").classList.add("hidden"); + document.getElementById("custom_oai_key").value = localsettings.saved_grok_key; + document.getElementById("custom_oai_endpoint").value = default_grok_base; + } else //openrouter supports autofetch { document.getElementById("openrouterdesc").classList.remove("hidden"); @@ -9408,7 +9430,7 @@ Current version indicated by LITEVER below. }); } } - else if(epchoice==2 || epchoice==3 || epchoice==7 || epchoice==8) //connect to OAI / OpenRouter / MistralAI / Featherless Endpoint + else if(epchoice==2 || epchoice==3 || epchoice==7 || epchoice==8 || epchoice==9) //connect to OAI / OpenRouter / MistralAI / Featherless / Grok Endpoint { let desired_oai_key = document.getElementById("custom_oai_key").value.trim(); let desired_oai_ep = document.getElementById("custom_oai_endpoint").value.trim(); @@ -9444,6 +9466,10 @@ Current version indicated by LITEVER below. { localsettings.saved_featherless_key = custom_oai_key; } + else if(epchoice==9) + { + localsettings.saved_grok_key = custom_oai_key; + } else { localsettings.saved_openrouter_key = custom_oai_key; @@ -13117,13 +13143,13 @@ Current version indicated by LITEVER below. "temperature": submit_payload.params.temperature, "top_p": submit_payload.params.top_p, } - if(localsettings.request_logprobs && !targetep.toLowerCase().includes("api.mistral.ai")) + if(localsettings.request_logprobs && !targetep.toLowerCase().includes("api.x.ai") && !targetep.toLowerCase().includes("api.mistral.ai")) { - if(document.getElementById("useoaichatcompl").checked) + if(document.getElementById("useoaichatcompl").checked || targetep.toLowerCase().includes("api.x.ai")) { oai_payload.logprobs = true; oai_payload.top_logprobs = 5; - }else{ + } else { oai_payload.logprobs = 5; } } @@ -13781,6 +13807,11 @@ Current version indicated by LITEVER below. { iwidth = 768; } + else if(localsettings.img_aspect==3) + { + iwidth = 768; + iheight = 768; + } let genimg_payload = { "prompt": (sentence + negprompt), @@ -16911,6 +16942,9 @@ Current version indicated by LITEVER below. }else if(custom_oai_endpoint.toLowerCase().includes("featherless.ai")) { localsettings.prev_custom_endpoint_type = 8; + }else if(custom_oai_endpoint.toLowerCase().includes("api.x.ai")) + { + localsettings.prev_custom_endpoint_type = 9; } } else if(custom_claude_key!="") @@ -18239,7 +18273,7 @@ Current version indicated by LITEVER below.