diff --git a/packages/memory-host-sdk/src/host/internal.test.ts b/packages/memory-host-sdk/src/host/internal.test.ts index 648fad0452d..c37904a8336 100644 --- a/packages/memory-host-sdk/src/host/internal.test.ts +++ b/packages/memory-host-sdk/src/host/internal.test.ts @@ -226,7 +226,19 @@ describe("memory host SDK package internals", () => { overlap: 0, }); for (const chunk of surrogateChunks) { - expect(chunk.text).not.toContain("\uFFFD"); + expect(() => encodeURIComponent(chunk.text)).not.toThrow(); + } + }); + + it("preserves a surrogate pair at the coarse split boundary", () => { + const text = `${"a".repeat(39)}🌸${"b".repeat(39)}`; + + const chunks = chunkMarkdown(text, { tokens: 10, overlap: 0 }); + + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.map((chunk) => chunk.text).join("")).toBe(text); + for (const chunk of chunks) { + expect(() => encodeURIComponent(chunk.text)).not.toThrow(); } }); diff --git a/packages/memory-host-sdk/src/host/internal.ts b/packages/memory-host-sdk/src/host/internal.ts index dad3a2e2774..4fe7bb512e1 100644 --- a/packages/memory-host-sdk/src/host/internal.ts +++ b/packages/memory-host-sdk/src/host/internal.ts @@ -25,6 +25,7 @@ import { detectMime, estimateStringChars, runTasksWithConcurrency, + truncateUtf16Safe, } from "./openclaw-runtime-io.js"; import { resolveCanonicalRootMemoryFile, @@ -454,25 +455,23 @@ export function chunkMarkdown( // Second pass: if a segment's *weighted* size still exceeds the budget // (happens for CJK-heavy text where 1 char ≈ 1 token), re-split it at // chunking.tokens so the chunk stays within the token budget. - for (let start = 0; start < line.length; start += maxChars) { - const coarse = line.slice(start, start + maxChars); + for (let start = 0; start < line.length; ) { + const coarse = truncateUtf16Safe(line.slice(start), maxChars); if (estimateStringChars(coarse) > maxChars) { const fineStep = Math.max(1, chunking.tokens); for (let j = 0; j < coarse.length; ) { let end = Math.min(j + fineStep, coarse.length); - // Avoid splitting inside a UTF-16 surrogate pair (CJK Extension B+). - if (end < coarse.length) { - const code = coarse.charCodeAt(end - 1); - if (code >= 0xd800 && code <= 0xdbff) { - end += 1; // include the low surrogate - } + const lastCodeUnit = coarse.charCodeAt(end - 1); + if (lastCodeUnit >= 0xd800 && lastCodeUnit <= 0xdbff && end < coarse.length) { + end += 1; } segments.push(coarse.slice(j, end)); - j = end; // advance cursor to the adjusted boundary + j = end; } } else { segments.push(coarse); } + start += coarse.length; } } for (const segment of segments) {