fix(memory): preserve UTF-16 chunk boundaries (#101574)

* fix(memory): preserve UTF-16 chunk boundaries

* test(memory): require coarse boundary split

Co-authored-by: jensenwang560-blip <267012233+jensenwang560-blip@users.noreply.github.com>

* fix(memory): retain chunk budget at surrogate boundaries

---------

Co-authored-by: jensenwang560-blip <267012233+jensenwang560-blip@users.noreply.github.com>

packages/memory-host-sdk/src/host/internal.test.ts

@@ -226,7 +226,19 @@ describe("memory host SDK package internals", () => {
       overlap: 0,
     });
     for (const chunk of surrogateChunks) {
-      expect(chunk.text).not.toContain("\uFFFD");
+      expect(() => encodeURIComponent(chunk.text)).not.toThrow();
+    }
+  });
+
+  it("preserves a surrogate pair at the coarse split boundary", () => {
+    const text = `${"a".repeat(39)}🌸${"b".repeat(39)}`;
+
+    const chunks = chunkMarkdown(text, { tokens: 10, overlap: 0 });
+
+    expect(chunks.length).toBeGreaterThan(1);
+    expect(chunks.map((chunk) => chunk.text).join("")).toBe(text);
+    for (const chunk of chunks) {
+      expect(() => encodeURIComponent(chunk.text)).not.toThrow();
     }
   });
 

packages/memory-host-sdk/src/host/internal.ts

@@ -25,6 +25,7 @@ import {
   detectMime,
   estimateStringChars,
   runTasksWithConcurrency,
+  truncateUtf16Safe,
 } from "./openclaw-runtime-io.js";
 import {
   resolveCanonicalRootMemoryFile,
@@ -454,25 +455,23 @@ export function chunkMarkdown(
       // Second pass: if a segment's *weighted* size still exceeds the budget
       // (happens for CJK-heavy text where 1 char ≈ 1 token), re-split it at
       // chunking.tokens so the chunk stays within the token budget.
-      for (let start = 0; start < line.length; start += maxChars) {
-        const coarse = line.slice(start, start + maxChars);
+      for (let start = 0; start < line.length; ) {
+        const coarse = truncateUtf16Safe(line.slice(start), maxChars);
         if (estimateStringChars(coarse) > maxChars) {
           const fineStep = Math.max(1, chunking.tokens);
           for (let j = 0; j < coarse.length; ) {
             let end = Math.min(j + fineStep, coarse.length);
-            // Avoid splitting inside a UTF-16 surrogate pair (CJK Extension B+).
-            if (end < coarse.length) {
-              const code = coarse.charCodeAt(end - 1);
-              if (code >= 0xd800 && code <= 0xdbff) {
-                end += 1; // include the low surrogate
-              }
+            const lastCodeUnit = coarse.charCodeAt(end - 1);
+            if (lastCodeUnit >= 0xd800 && lastCodeUnit <= 0xdbff && end < coarse.length) {
+              end += 1;
             }
             segments.push(coarse.slice(j, end));
-            j = end; // advance cursor to the adjusted boundary
+            j = end;
           }
         } else {
           segments.push(coarse);
         }
+        start += coarse.length;
       }
     }
     for (const segment of segments) {