vrr/opencode
2
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-09-01 12:54:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 6

refactor(core): reuse Markdown chunk byte counts (#46079)

Browse source
This commit is contained in:
Kit Langton 2026-08-31 11:30:49 -04:00 • committed by GitHub
parent 8be3ce8b6c
commit e9f7331516
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 10 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
packages/core/src/tool/html-markdown.ts
View file

@ -95,7 +95,7 @@ export function convertHTMLToMarkdown(html: string) {
const remaining = limit - outputBytes
const next = bytes.byteLength <= remaining ? value : sliceBytes(value, remaining)
output.push(next)
outputBytes += encoder.encode(next).byteLength
outputBytes += bytes.byteLength <= remaining ? bytes.byteLength : encoder.encode(next).byteLength
last = next.at(-1) ?? last
}
const appendRaw = (value: string) => {

9
packages/core/test/tool-webfetch.test.ts
View file

@ -128,6 +128,15 @@ describe("WebFetchTool helpers", () => {
expect(output).toHaveLength(WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024)
})
test.each(["x", "\u00e9", "\u{1f600}"])("preserves UTF-8 boundaries at the content limit for %s", (character) => {
const budget = WebFetchTool.MAX_RESPONSE_BYTES - 64 * 1024
const fitting = "aa" + character.repeat(Math.floor((budget - 2) / Buffer.byteLength(character)))
expect(WebFetchTool.convertHTMLToMarkdown(fitting)).toBe(fitting)
const truncated = WebFetchTool.convertHTMLToMarkdown(fitting + character)
expect(truncated).toBe(fitting)
expect(Buffer.byteLength(truncated)).toBe(Buffer.byteLength(fitting))
})
test("bounds deeply nested list output and fragmented code fences", () => {
const lists = `${"<ul><li>item".repeat(2_000)}${"</li></ul>".repeat(2_000)}`
const quotes = `${"<blockquote><p>item".repeat(2_000)}${"</p></blockquote>".repeat(2_000)}`