From 6b8b50b3500ee14699b37dc50ab677a8f7f2588e Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Tue, 6 Aug 2024 15:31:41 +0800
Subject: [PATCH] try fix ipv6 (+1 squashed commits)

Squashed commits:

[8d95a639] try fix ipv6
---
 klite.embd   | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++--
 koboldcpp.py | 14 ++++++++---
 2 files changed, 78 insertions(+), 5 deletions(-)

diff --git a/klite.embd b/klite.embd
index 43c0ed64c..2eb0ca3c2 100644
--- a/klite.embd
+++ b/klite.embd
@@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
 -->
 
 <script>
-	const LITEVER = 161;
+	const LITEVER = 162;
 	const urlParams = new URLSearchParams(window.location.search);
 	const localflag = true;
 	const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
@@ -4069,6 +4069,7 @@ Current version indicated by LITEVER below.
 	const koboldcpp_preloadstory_endpoint = "/api/extra/preloadstory";
 	const koboldcpp_transcribe_endpoint = "/api/extra/transcribe";
 	const koboldcpp_tokenize_endpoint = "/api/extra/tokencount";
+	const koboldcpp_perf_endpoint = "/api/extra/perf";
 
 	const oai_models_endpoint = "/models";
 	const oai_submit_endpoint = "/completions";
@@ -4187,6 +4188,7 @@ Current version indicated by LITEVER below.
 	var localmodekey = "";
 	var kobold_endpoint_version = ""; //used to track problematic versions to avoid sending extra fields
 	var koboldcpp_version = ""; //detect if we are using koboldcpp
+	var koboldcpp_version_obj = {};
 	var koboldcpp_has_vision = false;
 	var last_request_str = "No Requests Available"; //full context of last submitted request
 	var lastcheckgenkey = ""; //for checking polled-streaming unique id when generating in kcpp
@@ -4241,6 +4243,7 @@ Current version indicated by LITEVER below.
 		trimwhitespace: false, //trim trailing whitespace
 		compressnewlines: false, //compress multiple newlines
 		eos_ban_mode: 0, //allow the EOS token when using locally 0=auto,1=unban,2=ban,3=bypass
+		token_count_multiplier: 100, //100 means 1x
 		opmode: 4, //what mode are we in? 1=story, 2=adventure, 3=chat, 4=instruct
 		adventure_is_action: false, //in adventure mode, determine story or action
 		adventure_context_mod: true, //extra injection for adventure mode
@@ -8476,9 +8479,10 @@ Current version indicated by LITEVER below.
 								.then(data => {
 									if(data && data!="" && data.version && data.version!="")
 									{
+										koboldcpp_version_obj = data;
 										koboldcpp_version = data.version;
 										console.log("KoboldCpp Detected: " + koboldcpp_version);
-										document.getElementById("connectstatus").innerHTML = ("KoboldCpp "+koboldcpp_version);
+										document.getElementById("connectstatus").innerHTML = (`<span style='cursor: pointer;' onclick='fetch_koboldcpp_perf()'>KoboldCpp ${koboldcpp_version}</a>`);
 										koboldcpp_has_vision = (data.vision?true:false);
 										koboldcpp_has_whisper = (data.transcribe?true:false);
 										let has_password = (data.protected?true:false);
@@ -9005,6 +9009,49 @@ Current version indicated by LITEVER below.
 		});
 	}
 
+	function fetch_koboldcpp_perf()
+	{
+		fetch(apply_proxy_url(custom_kobold_endpoint + koboldcpp_perf_endpoint))
+		.then(x => x.json())
+		.then(resp => {
+			console.log(resp);
+			if(resp)
+			{
+				let perfs = "";
+				if(koboldcpp_version_obj)
+				{
+					for (let key in koboldcpp_version_obj) {
+						if (koboldcpp_version_obj.hasOwnProperty(key)) {
+							let val = koboldcpp_version_obj[key];
+							if (typeof val === 'number' && !Number.isInteger(val)) {
+								val = val.toFixed(2);
+							}
+							perfs += (perfs==""?``:`\n`);
+							perfs += `${key}: ${val}`;
+						}
+					}
+				}
+				for (let key in resp) {
+					if (resp.hasOwnProperty(key)) {
+						let val = resp[key];
+						if (typeof val === 'number' && !Number.isInteger(val)) {
+							val = val.toFixed(2);
+						}
+						perfs += `\n${key}: ${val}`;
+					}
+				}
+
+
+				msgbox(perfs,"KoboldCpp Server Status");
+			}else{
+				msgbox("KoboldCpp Server Error","KoboldCpp Server Status");
+			}
+		}).catch((error) => {
+			console.log("Perf Error: " + error);
+			msgbox("KoboldCpp Server Inaccessible","KoboldCpp Server Status");
+		});
+	}
+
 	//function to allow selection of models
 	function display_horde_models() {
 		document.getElementById("pickedmodel").innerHTML = "";
@@ -9481,6 +9528,7 @@ Current version indicated by LITEVER below.
 		document.getElementById("dry_multiplier").value = localsettings.dry_multiplier;
 		document.getElementById("dry_base").value = localsettings.dry_base;
 		document.getElementById("dry_allowed_length").value = localsettings.dry_allowed_length;
+		document.getElementById("token_count_multiplier").value = localsettings.token_count_multiplier;
 
 		if(is_using_kcpp_with_mirostat())
 		{
@@ -9868,6 +9916,7 @@ Current version indicated by LITEVER below.
 		localsettings.dry_base = document.getElementById("dry_base").value;
 		localsettings.dry_allowed_length = document.getElementById("dry_allowed_length").value;
 		localsettings.dry_sequence_breakers = pendingsequencebreakers;
+		localsettings.token_count_multiplier = document.getElementById("token_count_multiplier").value;
 
 		localsettings.speech_synth = document.getElementById("ttsselect").value;
 		localsettings.xtts_voice = document.getElementById("xtts_voices").value;
@@ -9967,6 +10016,7 @@ Current version indicated by LITEVER below.
 		localsettings.dry_base = cleannum(localsettings.dry_base, 0.0, 8.0);
 		localsettings.dry_allowed_length = cleannum(Math.floor(localsettings.dry_allowed_length), 0, 100);
 		localsettings.sampler_seed = cleannum(localsettings.sampler_seed, -1, 999999);
+		localsettings.token_count_multiplier = cleannum(localsettings.token_count_multiplier, 70, 130);
 		toggle_invert_colors();
 
 		voice_typing_mode = document.getElementById("voice_typing_mode").value;
@@ -11430,6 +11480,7 @@ Current version indicated by LITEVER below.
 			{
 				chars_per_token = 6;
 			}
+			chars_per_token = chars_per_token * (localsettings.token_count_multiplier*0.01);
 			let max_allowed_characters = Math.max(1, Math.floor((maxctxlen-maxgenamt) * chars_per_token) - 12);
 
 			//for adventure mode, inject hidden context, even more if there's nothing in memory
@@ -17373,6 +17424,7 @@ Current version indicated by LITEVER below.
 					<option value="gemini-pro" selected="selected">gemini-pro</option>
 					<option value="gemini-1.5-pro-latest">gemini-1.5-pro-latest</option>
 					<option value="gemini-1.5-flash-latest">gemini-1.5-flash-latest</option>
+					<option value="gemini-1.5-pro-exp-0801">gemini-1.5-pro-exp-0801</option>
 					<option value="text-bison-001">text-bison-001</option>
 				</select>
 				<span class="color_green" style="font-weight: bold;">Please input Gemini or PaLM API Key.</span><br><br>
@@ -17838,6 +17890,19 @@ Current version indicated by LITEVER below.
 									<option value="3">Bypass</option>
 								</select>
 							</div>
+							<div class="settinglabel settingcell">
+								<div class="justifyleft settingsmall">Token Limit Multiplier <span class="helpicon">?<span
+									class="helptext">Adds a multiplier to token estimations, useful if token counts are inaccurate. Influences text truncation. Higher values increases token allowance.</span></span></div>
+									<select title="Token Limit Multiplier" style="padding:1px; height:auto; margin:0px 0px 0px auto;" class="form-control" id="token_count_multiplier">
+									<option value="70">0.7x</option>
+									<option value="80">0.8x</option>
+									<option value="90">0.9x</option>
+									<option value="100">1.0x</option>
+									<option value="110">1.1x</option>
+									<option value="120">1.2x</option>
+									<option value="130">1.3x</option>
+								</select>
+							</div>
 							<div class="settinglabel settingcell">
 								<div title="Sampler Order" class="justifyleft settingsmall" style="width:100%">Sampler Order <span class="helpicon">?<span class="helptext">
 								The order by which all 7 samplers are applied, separated by commas. 0=top_k, 1=top_a, 2=top_p, 3=tfs, 4=typ, 5=temp, 6=rep_pen</span></span></div>
diff --git a/koboldcpp.py b/koboldcpp.py
index c77841a42..4fb1156a5 100644
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -2042,9 +2042,11 @@ def is_ipv6_supported():
     try:
         # Attempt to create an IPv6 socket
         sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
         sock.close()
         return True
-    except socket.error:
+    except Exception as ex:
         return False
 
 def RunServerMultiThreaded(addr, port):
@@ -2058,6 +2060,7 @@ def RunServerMultiThreaded(addr, port):
     if is_ipv6_supported():
         ipv6_sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
         ipv6_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        ipv6_sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
 
     if args.ssl and sslvalid:
         import ssl
@@ -2072,9 +2075,14 @@ def RunServerMultiThreaded(addr, port):
     numThreads = 24
     ipv4_sock.bind((addr, port))
     ipv4_sock.listen(numThreads)
+
     if ipv6_sock:
-        ipv6_sock.bind((addr, port))
-        ipv6_sock.listen(numThreads)
+        try:
+            ipv6_sock.bind((addr, port))
+            ipv6_sock.listen(numThreads)
+        except Exception as ex:
+            ipv6_sock = None
+            print("IPv6 Socket Failed to Bind. IPv6 will be unavailable.")
 
     class Thread(threading.Thread):
         def __init__(self, i):