From 43c55bb7e2fc6933c9ffbd8566413a31f86225fc Mon Sep 17 00:00:00 2001
From: Concedo <39025047+LostRuins@users.noreply.github.com>
Date: Tue, 30 Jul 2024 22:18:22 +0800
Subject: [PATCH] hack to fix bad unicode fragments corrupting streamed output

---
 koboldcpp.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/koboldcpp.py b/koboldcpp.py
index befdad114..9af8c9ddf 100644
--- a/koboldcpp.py
+++ b/koboldcpp.py
@@ -1435,7 +1435,8 @@ class ServerRequestHandler(http.server.SimpleHTTPRequestHandler):
                     newbyte = ctypes.string_at(token)
                     incomplete_token_buffer += bytearray(newbyte)
                     tokenSeg = incomplete_token_buffer.decode("UTF-8","ignore")
-                    if tokenSeg!="":
+                    badFragment = (tokenSeg==" " and len(incomplete_token_buffer)>1) #partial incomplete unicode
+                    if tokenSeg!="" and not badFragment:
                         incomplete_token_buffer.clear()
                         tokenStr += tokenSeg