From 59097783e2d0273c07d7ecfbec3d982340f9fd57 Mon Sep 17 00:00:00 2001 From: NickNMorty Date: Mon, 6 Jul 2026 21:43:42 -0700 Subject: [PATCH] =?UTF-8?q?fix(markdown-core):=20CJK-friendly=20emphasis?= =?UTF-8?q?=20flanking=20so=20**=E6=A0=87=E7=AD=BE=EF=BC=9A**=E6=AD=A3?= =?UTF-8?q?=E6=96=87=20renders=20bold=20(#101120)=20(#101230)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(telegram): reproduce CJK emphasis flanking bug * fix(markdown-core): support CJK emphasis flanking * fix(markdown-core): type CJK delimiter state (#101120) * fix(markdown-core): mirror markdown-it Unicode whitespace in CJK delimiter override Preserves markdown-it isWhiteSpace classification (U+3000, U+00A0, U+2000-200A, etc.) before forcing CJK-adjacent delimiter flags, and adds U+3000/U+2009 regressions. Addresses clawsweeper P1 finding on #101230. * refactor(markdown): use maintained CJK flanking plugin --------- Co-authored-by: Peter Steinberger --- CHANGELOG.md | 1 + extensions/telegram/src/format.test.ts | 6 ++ packages/markdown-core/package.json | 1 + .../markdown-core/src/ir.emphasis-cjk.test.ts | 68 +++++++++++++++++++ packages/markdown-core/src/ir.ts | 2 + pnpm-lock.yaml | 20 ++++++ 6 files changed, 98 insertions(+) create mode 100644 packages/markdown-core/src/ir.emphasis-cjk.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d4cc5d0632c..6a490b9add4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ Docs: https://docs.openclaw.ai ### Fixes - **Android hardware keyboard chat:** send with unmodified Enter on physical keyboards while preserving Shift+Enter and other modified Enter combinations for multiline input. (#101239) Thanks @3ninyt3nin-creator. +- **CJK Markdown emphasis:** render adjacent Chinese, Japanese, and Korean emphasis punctuation through the shared Markdown pipeline instead of leaking literal markers across channels. (#101230, #101120) Thanks @nicknmorty. - **Codex yielded native subagents:** keep the parent app-server subscription and shared client alive until yielded native subagent completion delivery settles, preventing lost wakeups and leaked one-shot cleanup. - **Discord streamed finals:** send completion replies as fresh messages so inactive channels become unread, while preserving targeted mentions without escalating `@everyone` or `@here`. (#99711, #99662) Thanks @davelutztx. - **OpenAI-compatible SSE parsing:** recognize event streams mislabeled as JSON without prepending a second `data:` prefix, preserving valid streamed responses from non-conforming providers. (#96503) Thanks @ZengWen-DT. diff --git a/extensions/telegram/src/format.test.ts b/extensions/telegram/src/format.test.ts index 70cab9bc090..edf9ded2113 100644 --- a/extensions/telegram/src/format.test.ts +++ b/extensions/telegram/src/format.test.ts @@ -94,6 +94,12 @@ describe("markdownToTelegramHtml", () => { expect(markdownToTelegramRichHtml("1")).toBe("1"); }); + it("renders bold spans that close before CJK punctuation in rich markdown", () => { + expect(markdownToTelegramRichHtml("边界：**社区显示：**Fable 与 **有效**。")).toBe( + "边界：社区显示：Fable 与 有效。", + ); + }); + it("materializes inline and paragraph newlines as
for rich messages", () => { // The exact reported symptom: literal "• " bullets (not Markdown list markers) // joined by soft breaks, which Bot API 10.1 rich messages collapse without
. diff --git a/packages/markdown-core/package.json b/packages/markdown-core/package.json index 60e63a7b284..8e19aa8f52c 100644 --- a/packages/markdown-core/package.json +++ b/packages/markdown-core/package.json @@ -57,6 +57,7 @@ }, "dependencies": { "markdown-it": "14.3.0", + "markdown-it-cjk-friendly": "2.0.2", "yaml": "2.9.0" } } diff --git a/packages/markdown-core/src/ir.emphasis-cjk.test.ts b/packages/markdown-core/src/ir.emphasis-cjk.test.ts new file mode 100644 index 00000000000..03a9b7cf34c --- /dev/null +++ b/packages/markdown-core/src/ir.emphasis-cjk.test.ts @@ -0,0 +1,68 @@ +// Markdown Core tests cover CJK-friendly emphasis flanking. +import { describe, expect, it } from "vitest"; +import { markdownToIR } from "./ir.js"; + +function styledText(markdown: string, style: "bold" | "italic" = "bold"): string[] { + const ir = markdownToIR(markdown); + return ir.styles + .filter((span) => span.style === style) + .map((span) => ir.text.slice(span.start, span.end)); +} + +describe("markdownToIR CJK emphasis flanking", () => { + it("closes strong emphasis before adjacent Chinese text", () => { + expect(styledText("前**加粗：**后")).toEqual(["加粗："]); + }); + + it("closes strong emphasis before adjacent Japanese text", () => { + expect(styledText("これは**強調。**です")).toEqual(["強調。"]); + }); + + it("closes strong emphasis before adjacent Korean text", () => { + expect(styledText("이것은 **강조:**입니다")).toEqual(["강조:"]); + }); + + it("handles supplementary CJK and variation selectors at emphasis boundaries", () => { + expect(styledText("𰻞𰻞**（ビャンビャン）**麺")).toEqual(["（ビャンビャン）"]); + expect(styledText("葛󠄀**(こちらが正式表記)**城市")).toEqual(["(こちらが正式表記)"]); + }); + + it("supports CJK-friendly underscore flanking without enabling Latin intraword emphasis", () => { + expect(styledText("__注意__：注意事項")).toEqual(["注意"]); + expect(styledText("foo_bar_baz", "italic")).toEqual([]); + }); + + it("keeps ASCII CommonMark emphasis behavior", () => { + expect(styledText("**bold** text")).toEqual(["bold"]); + expect(styledText("foo**bar**baz")).toEqual(["bar"]); + }); + + it("treats ideographic space (U+3000) as whitespace, not a flanking CJK char", () => { + // Opening delimiter followed by U+3000 must not be forced open. + expect(styledText("前**\u3000加粗**后")).toEqual([]); + // U+3000-separated emphasis keeps normal CommonMark behavior. + expect(styledText("前\u3000**加粗**\u3000后")).toEqual(["加粗"]); + }); + + it("treats Unicode thin space (U+2009) as whitespace in delimiter scanning", () => { + // Opening delimiter followed by U+2009 must not be forced open. + expect(styledText("前**\u2009加粗**后")).toEqual([]); + // Closing delimiter after CJK punctuation still closes when followed by U+2009. + expect(styledText("前**加粗：**\u2009后")).toEqual(["加粗："]); + }); + + it("leaves code spans and links on their existing paths", () => { + const code = markdownToIR("`前**加粗：**后`"); + expect(code.text).toBe("前**加粗：**后"); + expect(code.styles.map((span) => span.style)).toEqual(["code"]); + + const linked = markdownToIR("[前**加粗：**后](https://example.com)"); + expect(linked.text).toBe("前加粗：后"); + expect(linked.links).toEqual([ + { start: 0, end: linked.text.length, href: "https://example.com" }, + ]); + expect(linked.styles.filter((span) => span.style === "bold")).toEqual([ + { start: 1, end: 4, style: "bold" }, + ]); + }); +}); diff --git a/packages/markdown-core/src/ir.ts b/packages/markdown-core/src/ir.ts index 552e5bb9207..9187a2d19f4 100644 --- a/packages/markdown-core/src/ir.ts +++ b/packages/markdown-core/src/ir.ts @@ -1,5 +1,6 @@ // Markdown Core module implements ir behavior. import MarkdownIt from "markdown-it"; +import markdownItCjkFriendly from "markdown-it-cjk-friendly"; import { chunkText } from "./chunk-text.js"; import type { MarkdownTableMode } from "./types.js"; @@ -150,6 +151,7 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt { breaks: false, typographer: false, }); + md.use(markdownItCjkFriendly); md.enable("strikethrough"); if (options.tableMode && options.tableMode !== "off") { md.enable("table"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 06e9a50e1fb..bbd62d1050b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1918,6 +1918,9 @@ importers: markdown-it: specifier: 14.3.0 version: 14.3.0 + markdown-it-cjk-friendly: + specifier: 2.0.2 + version: 2.0.2(@types/markdown-it@14.1.2)(markdown-it@14.3.0) yaml: specifier: 2.9.0 version: 2.9.0 @@ -5999,6 +6002,16 @@ packages: resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==} engines: {node: '>=16'} + markdown-it-cjk-friendly@2.0.2: + resolution: {integrity: sha512-KXCl6sd129UqkAiRDb+NcAHrxC9xRa2WsGIsMMvtp2y1YlbeIaNYzArX2zfDoGhOjsyNMfJrGO7xGBss27YQSA==} + engines: {node: '>=18'} + peerDependencies: + '@types/markdown-it': '*' + markdown-it: '*' + peerDependenciesMeta: + '@types/markdown-it': + optional: true + markdown-it-task-lists@2.1.1: resolution: {integrity: sha512-TxFAc76Jnhb2OUu+n3yz9RMu4CwGfaT788br6HhEDlvWfdeJcLUsxk1Hgw2yJio0OXsxv7pyIPmvECY7bMbluA==} @@ -11810,6 +11823,13 @@ snapshots: markdown-extensions@2.0.0: {} + markdown-it-cjk-friendly@2.0.2(@types/markdown-it@14.1.2)(markdown-it@14.3.0): + dependencies: + get-east-asian-width: 1.6.0 + markdown-it: 14.3.0 + optionalDependencies: + '@types/markdown-it': 14.1.2 + markdown-it-task-lists@2.1.1: {} markdown-it@14.3.0: