diff --git a/.changeset/cli-autolink-cjk-boundary.md b/.changeset/cli-autolink-cjk-boundary.md new file mode 100644 index 000000000..56defb58a --- /dev/null +++ b/.changeset/cli-autolink-cjk-boundary.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/kimi-code": patch +--- + +Fix bare URLs in chat output absorbing the CJK characters that follow them, which made the link unclickable or open a broken address. diff --git a/.changeset/pi-tui-autolink-cjk-boundary.md b/.changeset/pi-tui-autolink-cjk-boundary.md new file mode 100644 index 000000000..fe5a19c08 --- /dev/null +++ b/.changeset/pi-tui-autolink-cjk-boundary.md @@ -0,0 +1,5 @@ +--- +"@moonshot-ai/pi-tui": patch +--- + +Stop GFM autolinks at CJK/full-width punctuation so a bare URL followed by CJK text no longer swallows the CJK characters into the link target. diff --git a/packages/pi-tui/AGENTS.md b/packages/pi-tui/AGENTS.md index c25bd9efe..4a5833437 100644 --- a/packages/pi-tui/AGENTS.md +++ b/packages/pi-tui/AGENTS.md @@ -11,6 +11,7 @@ Never overwrite this directory wholesale when syncing from upstream. Each of the 3. **`src/tui.ts` — truncate overwide lines instead of throwing**: `doRender` truncates overwide lines with `sliceByColumn` while building the per-line processed output (upstream had a "write crash log + throw" block in the differential render path — do not bring it back when syncing). Performance constraint: the width check must go through the `asciiVisibleWidth` fast path in `utils.ts` first (ANSI-aware ASCII scan with an early exit past the limit) and only fall back to `visibleWidth` for non-ASCII lines; `WIDTH_CACHE_SIZE` is 4096 to match. Since divergence 5 below, this check only runs for lines whose raw string reference changed since the previous frame, so the every-line-every-frame scan (and its width-cache FIFO thrash beyond 4096 distinct non-ASCII lines) no longer occurs. Guarding tests: "TUI overwide line handling" in `test/tui-render.test.ts` (exact viewport assertions) and "asciiVisibleWidth" in `test/truncate-to-width.test.ts`. 4. **`src/components/text.ts` / `markdown.ts` / `truncated-text.ts` / `editor.ts` — negative-width `repeat` guards**: the `repeat` counts for blank lines, horizontal rules, and the editor's top/bottom borders are clamped to ≥ 0 (two editor border sites; markdown's emptyLine and hr — the hr site is currently unreachable from the render entry and is purely defensive). Guarding tests: the "negative width safety" cases — Text's lives in `test/tui-render.test.ts` (Text has no dedicated test file), Markdown's and TruncatedText's live in their own test files; the editor's is "does not throw at zero or negative widths" inside the "Editor narrow width rendering" group in `test/editor.test.ts`. 5. **`src/tui.ts` — per-frame processed-line reuse**: `doRender` keeps the previous frame's raw lines (`previousRawLines`), their processed output (`previousLines`), and per-line kitty image ids (`previousLineImageIds`). A line whose raw string is reference-identical to the previous frame's reuses its processed output (truncation + `normalizeTerminalOutput` + trailing `SEGMENT_RESET`) verbatim, so a steady-state frame costs O(total lines) pointer comparisons plus O(changed lines) real work instead of re-normalizing every line. Image-id consumers (`expandChangedRangeForKittyImages`, `deleteChangedKittyImages`, the frame-end id union) read the cached per-line ids rather than re-scanning line text; upstream has no such cache and re-processes every line every frame (upstream also has `applyLineResets`, which this divergence inlines into the processed-line build). Reuse is only valid when the terminal width is unchanged; width changes re-render everything at the new width so references never match. Guarding tests: "TUI steady-frame processed-line reuse" in `test/tui-render.test.ts`. +6. **`src/components/markdown.ts` — `CjkBoundaryUrlTokenizer` autolink CJK boundary**: marked's GFM autolink accepts any non-space characters after the domain and its backpedal strips only ASCII trailing punctuation, so CJK/full-width punctuation right after a bare URL is absorbed into the link text and href (`.../pull/232（本地` renders as one anchor with a CJK target). The `CjkBoundaryUrlTokenizer` subclass (the tokenizer actually registered on the parser) cuts the match at the first CJK punctuation character before the ASCII backpedal; full-width parentheses follow GFM's ASCII-paren rule — balanced pairs stay in the URL (`.../wiki/中华人民共和国（1949年）`, punctuation inside them included), only unbalanced ones terminate the match. `StrictStrikethroughTokenizer` itself stays byte-identical to upstream. Guarding tests: the bare-URL CJK cases in the "Links" group in `test/markdown.test.ts`. ## Acceptance after syncing from upstream diff --git a/packages/pi-tui/src/components/markdown.ts b/packages/pi-tui/src/components/markdown.ts index 9b47a31c2..aa53a8c0f 100644 --- a/packages/pi-tui/src/components/markdown.ts +++ b/packages/pi-tui/src/components/markdown.ts @@ -23,6 +23,98 @@ class StrictStrikethroughTokenizer extends Tokenizer { } } +// Local divergence (not upstream; keep StrictStrikethroughTokenizer untouched +// for re-vendoring): marked's GFM autolink accepts any non-space characters +// after the domain and its backpedal only strips ASCII trailing punctuation, +// so CJK/full-width punctuation right after a bare URL is absorbed into the +// link text and href (`.../pull/232（本地` becomes one anchor). Cut the match +// at the first CJK punctuation character BEFORE the ASCII backpedal so trailing +// ASCII punctuation left by the cut is still normalized away. Full-width +// parentheses are handled like GFM handles ASCII ones: balanced pairs stay +// part of the URL (`.../wiki/中华人民共和国（1949年）`, punctuation inside +// them included), and only an unbalanced `（` / `）` terminates the match. +// Guarded by the "CJK punctuation after bare URLs" tests in +// test/markdown.test.ts. +const FULLWIDTH_LEFT_PAREN = 0xff08; // （ +const FULLWIDTH_RIGHT_PAREN = 0xff09; // ） +const CJK_URL_TERMINATOR_REGEX = + /[\u3000-\u303f\uff01-\uff07\uff0a-\uff0f\uff1a-\uff20\uff3b-\uff40\uff5b-\uff65\u2013\u2014\u2018\u2019\u201c\u201d\u2026]/; + +/** + * Index at which to cut an autolink match, or -1 to keep it whole. Non-paren + * CJK punctuation terminates the URL outside of full-width parens; full-width + * parens only terminate it when unbalanced. Punctuation inside a balanced + * parenthetical (e.g. the ，in （北京，1949年）) stays part of the URL — + * prose parentheticals contain spaces and never survive marked's match this + * far, so a balanced group is almost always deliberate URL content. + */ +function findCjkUrlBoundary(match: string): number { + let parenDepth = 0; + let unmatchedOpen = -1; + for (let i = 0; i < match.length; i++) { + const code = match.charCodeAt(i); + if (code === FULLWIDTH_LEFT_PAREN) { + parenDepth++; + if (unmatchedOpen === -1) { + unmatchedOpen = i; + } + } else if (code === FULLWIDTH_RIGHT_PAREN) { + if (parenDepth === 0) { + return i; + } + parenDepth--; + if (parenDepth === 0) { + unmatchedOpen = -1; + } + } else if (parenDepth === 0 && CJK_URL_TERMINATOR_REGEX.test(match[i]!)) { + return i; + } + } + return parenDepth > 0 ? unmatchedOpen : -1; +} + +class CjkBoundaryUrlTokenizer extends StrictStrikethroughTokenizer { + override url(src: string): Tokens.Link | undefined { + const cap = this.rules.inline.url.exec(src); + if (!cap) { + return undefined; + } + // Autolinked emails (cap[2] === "@") skip the backpedal upstream; keep + // that behavior exactly. + if (cap[2] === "@") { + const text = cap[0]; + return { + type: "link", + raw: text, + text, + href: `mailto:${text}`, + tokens: [{ type: "text", raw: text, text }], + }; + } + const boundary = findCjkUrlBoundary(cap[0]); + if (boundary !== -1) { + cap[0] = cap[0].slice(0, boundary); + } + if (!cap[0]) { + return undefined; + } + let previous: string; + do { + previous = cap[0]; + cap[0] = this.rules.inline._backpedal.exec(cap[0])?.[0] ?? ""; + } while (previous !== cap[0]); + const text = cap[0]; + const href = cap[1] === "www." ? `http://${text}` : text; + return { + type: "link", + raw: text, + text, + href, + tokens: [{ type: "text", raw: text, text }], + }; + } +} + interface LatexToken extends Tokens.Generic { type: "latex" | "latexBlock"; text: string; @@ -170,7 +262,7 @@ function trimPartialClosingFences(tokens: readonly Token[]): void { const markdownParser = new Marked(); markdownParser.setOptions({ - tokenizer: new StrictStrikethroughTokenizer(), + tokenizer: new CjkBoundaryUrlTokenizer(), }); markdownParser.use({ extensions: [...LATEX_MARKDOWN_EXTENSIONS] }); diff --git a/packages/pi-tui/test/markdown.test.ts b/packages/pi-tui/test/markdown.test.ts index ace5c2163..396f5d91e 100644 --- a/packages/pi-tui/test/markdown.test.ts +++ b/packages/pi-tui/test/markdown.test.ts @@ -1603,6 +1603,98 @@ bar`, ); assert.ok(!rawPlain.join("").includes("(https://example.com)"), "URL should not appear twice"); }); + + it("should not absorb CJK punctuation after bare URLs into the link", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const markdown = new Markdown( + "PR 已开：https://example.com/app/pull/232（本地 main 已退回 origin/main 保持干净）。", + 0, + 0, + defaultMarkdownTheme, + ); + + const lines = markdown.render(80); + const joined = lines.join(""); + + // The hyperlink target must stop at the CJK boundary… + assert.ok( + joined.includes("\x1b]8;;https://example.com/app/pull/232\x1b\\"), + "OSC 8 target should end at the URL, before the full-width parenthesis", + ); + // …and the CJK text must not be part of any hyperlink target. + assert.ok(!joined.includes("%EF%BC%88"), "OSC 8 target should not contain encoded CJK"); + assert.ok(!/\x1b\]8;;[^\x1b]*（/.test(joined), "No hyperlink target should contain CJK characters"); + // The full source text still renders visibly. + const rawPlain = lines.map((line) => + line.replace(/\x1b\]8;;[^\x1b]*\x1b\\/g, "").replace(/\x1b\[[0-9;]*m/g, ""), + ); + assert.ok( + rawPlain.join("").includes("https://example.com/app/pull/232（本地 main 已退回"), + "URL and following CJK text should both render", + ); + }); + + it("should strip a trailing full-width parenthesis after a bare URL", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const markdown = new Markdown("看这个（https://example.com/page）就知道", 0, 0, defaultMarkdownTheme); + + const lines = markdown.render(80); + const joined = lines.join(""); + + assert.ok( + joined.includes("\x1b]8;;https://example.com/page\x1b\\"), + "OSC 8 target should exclude the wrapping full-width parenthesis", + ); + }); + + it("should keep CJK characters inside the URL path", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const markdown = new Markdown("见 https://example.com/wiki/测试页面 的说明", 0, 0, defaultMarkdownTheme); + + const lines = markdown.render(80); + const joined = lines.join(""); + + assert.ok( + joined.includes("\x1b]8;;https://example.com/wiki/测试页面\x1b\\"), + "CJK path characters remain part of the hyperlink target", + ); + }); + + it("should keep balanced full-width parentheses inside the URL path", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const markdown = new Markdown( + "见 https://example.com/wiki/中华人民共和国（1949年） 的说明", + 0, + 0, + defaultMarkdownTheme, + ); + + const lines = markdown.render(80); + const joined = lines.join(""); + + assert.ok( + joined.includes("\x1b]8;;https://example.com/wiki/中华人民共和国（1949年）\x1b\\"), + "Balanced full-width parens remain part of the hyperlink target", + ); + }); + + it("should keep CJK punctuation inside balanced full-width parentheses", () => { + setCapabilities({ images: null, trueColor: false, hyperlinks: true }); + const markdown = new Markdown( + "见 https://example.com/wiki/中华人民共和国（北京，1949年） 的说明", + 0, + 0, + defaultMarkdownTheme, + ); + + const lines = markdown.render(80); + const joined = lines.join(""); + + assert.ok( + joined.includes("\x1b]8;;https://example.com/wiki/中华人民共和国（北京，1949年）\x1b\\"), + "Punctuation inside balanced full-width parens remains part of the hyperlink target", + ); + }); }); describe("HTML-like tags in text", () => {