diff --git a/packages/core/src/tool/html-markdown.ts b/packages/core/src/tool/html-markdown.ts index ea18530f17e..057d08b6738 100644 --- a/packages/core/src/tool/html-markdown.ts +++ b/packages/core/src/tool/html-markdown.ts @@ -95,7 +95,7 @@ export function convertHTMLToMarkdown(html: string) { const remaining = limit - outputBytes const next = bytes.byteLength <= remaining ? value : sliceBytes(value, remaining) output.push(next) - outputBytes += encoder.encode(next).byteLength + outputBytes += bytes.byteLength <= remaining ? bytes.byteLength : encoder.encode(next).byteLength last = next.at(-1) ?? last } const appendRaw = (value: string) => { diff --git a/packages/core/test/tool-webfetch.test.ts b/packages/core/test/tool-webfetch.test.ts index 1d5d73c1762..dcc8895fd9f 100644 --- a/packages/core/test/tool-webfetch.test.ts +++ b/packages/core/test/tool-webfetch.test.ts @@ -128,6 +128,15 @@ describe("WebFetchTool helpers", () => { expect(output).toHaveLength(WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024) }) + test.each(["x", "\u00e9", "\u{1f600}"])("preserves UTF-8 boundaries at the content limit for %s", (character) => { + const budget = WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024 + const fitting = "aa" + character.repeat(Math.floor((budget - 2) / Buffer.byteLength(character))) + expect(WebFetchTool.convertHTMLToMarkdown(fitting)).toBe(fitting) + const truncated = WebFetchTool.convertHTMLToMarkdown(fitting + character) + expect(truncated).toBe(fitting) + expect(Buffer.byteLength(truncated)).toBe(Buffer.byteLength(fitting)) + }) + test("bounds deeply nested list output and fragmented code fences", () => { const lists = `${"".repeat(2_000)}` const quotes = `${"

item".repeat(2_000)}${"

".repeat(2_000)}`