mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
* test(telegram): reproduce CJK emphasis flanking bug * fix(markdown-core): support CJK emphasis flanking * fix(markdown-core): type CJK delimiter state (#101120) * fix(markdown-core): mirror markdown-it Unicode whitespace in CJK delimiter override Preserves markdown-it isWhiteSpace classification (U+3000, U+00A0, U+2000-200A, etc.) before forcing CJK-adjacent delimiter flags, and adds U+3000/U+2009 regressions. Addresses clawsweeper P1 finding on #101230. * refactor(markdown): use maintained CJK flanking plugin --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
parent
bbd01c169e
commit
59097783e2
6 changed files with 98 additions and 0 deletions
|
|
@ -24,6 +24,7 @@ Docs: https://docs.openclaw.ai
|
|||
### Fixes
|
||||
|
||||
- **Android hardware keyboard chat:** send with unmodified Enter on physical keyboards while preserving Shift+Enter and other modified Enter combinations for multiline input. (#101239) Thanks @3ninyt3nin-creator.
|
||||
- **CJK Markdown emphasis:** render adjacent Chinese, Japanese, and Korean emphasis punctuation through the shared Markdown pipeline instead of leaking literal markers across channels. (#101230, #101120) Thanks @nicknmorty.
|
||||
- **Codex yielded native subagents:** keep the parent app-server subscription and shared client alive until yielded native subagent completion delivery settles, preventing lost wakeups and leaked one-shot cleanup.
|
||||
- **Discord streamed finals:** send completion replies as fresh messages so inactive channels become unread, while preserving targeted mentions without escalating `@everyone` or `@here`. (#99711, #99662) Thanks @davelutztx.
|
||||
- **OpenAI-compatible SSE parsing:** recognize event streams mislabeled as JSON without prepending a second `data:` prefix, preserving valid streamed responses from non-conforming providers. (#96503) Thanks @ZengWen-DT.
|
||||
|
|
|
|||
|
|
@ -94,6 +94,12 @@ describe("markdownToTelegramHtml", () => {
|
|||
expect(markdownToTelegramRichHtml("<sup>1</sup>")).toBe("<sup>1</sup>");
|
||||
});
|
||||
|
||||
it("renders bold spans that close before CJK punctuation in rich markdown", () => {
|
||||
expect(markdownToTelegramRichHtml("边界:**社区显示:**Fable 与 **有效**。")).toBe(
|
||||
"边界:<b>社区显示:</b>Fable 与 <b>有效</b>。",
|
||||
);
|
||||
});
|
||||
|
||||
it("materializes inline and paragraph newlines as <br> for rich messages", () => {
|
||||
// The exact reported symptom: literal "• " bullets (not Markdown list markers)
|
||||
// joined by soft breaks, which Bot API 10.1 rich messages collapse without <br>.
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"markdown-it": "14.3.0",
|
||||
"markdown-it-cjk-friendly": "2.0.2",
|
||||
"yaml": "2.9.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
68
packages/markdown-core/src/ir.emphasis-cjk.test.ts
Normal file
68
packages/markdown-core/src/ir.emphasis-cjk.test.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
// Markdown Core tests cover CJK-friendly emphasis flanking.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { markdownToIR } from "./ir.js";
|
||||
|
||||
function styledText(markdown: string, style: "bold" | "italic" = "bold"): string[] {
|
||||
const ir = markdownToIR(markdown);
|
||||
return ir.styles
|
||||
.filter((span) => span.style === style)
|
||||
.map((span) => ir.text.slice(span.start, span.end));
|
||||
}
|
||||
|
||||
describe("markdownToIR CJK emphasis flanking", () => {
|
||||
it("closes strong emphasis before adjacent Chinese text", () => {
|
||||
expect(styledText("前**加粗:**后")).toEqual(["加粗:"]);
|
||||
});
|
||||
|
||||
it("closes strong emphasis before adjacent Japanese text", () => {
|
||||
expect(styledText("これは**強調。**です")).toEqual(["強調。"]);
|
||||
});
|
||||
|
||||
it("closes strong emphasis before adjacent Korean text", () => {
|
||||
expect(styledText("이것은 **강조:**입니다")).toEqual(["강조:"]);
|
||||
});
|
||||
|
||||
it("handles supplementary CJK and variation selectors at emphasis boundaries", () => {
|
||||
expect(styledText("𰻞𰻞**(ビャンビャン)**麺")).toEqual(["(ビャンビャン)"]);
|
||||
expect(styledText("葛󠄀**(こちらが正式表記)**城市")).toEqual(["(こちらが正式表記)"]);
|
||||
});
|
||||
|
||||
it("supports CJK-friendly underscore flanking without enabling Latin intraword emphasis", () => {
|
||||
expect(styledText("__注意__:注意事項")).toEqual(["注意"]);
|
||||
expect(styledText("foo_bar_baz", "italic")).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps ASCII CommonMark emphasis behavior", () => {
|
||||
expect(styledText("**bold** text")).toEqual(["bold"]);
|
||||
expect(styledText("foo**bar**baz")).toEqual(["bar"]);
|
||||
});
|
||||
|
||||
it("treats ideographic space (U+3000) as whitespace, not a flanking CJK char", () => {
|
||||
// Opening delimiter followed by U+3000 must not be forced open.
|
||||
expect(styledText("前**\u3000加粗**后")).toEqual([]);
|
||||
// U+3000-separated emphasis keeps normal CommonMark behavior.
|
||||
expect(styledText("前\u3000**加粗**\u3000后")).toEqual(["加粗"]);
|
||||
});
|
||||
|
||||
it("treats Unicode thin space (U+2009) as whitespace in delimiter scanning", () => {
|
||||
// Opening delimiter followed by U+2009 must not be forced open.
|
||||
expect(styledText("前**\u2009加粗**后")).toEqual([]);
|
||||
// Closing delimiter after CJK punctuation still closes when followed by U+2009.
|
||||
expect(styledText("前**加粗:**\u2009后")).toEqual(["加粗:"]);
|
||||
});
|
||||
|
||||
it("leaves code spans and links on their existing paths", () => {
|
||||
const code = markdownToIR("`前**加粗:**后`");
|
||||
expect(code.text).toBe("前**加粗:**后");
|
||||
expect(code.styles.map((span) => span.style)).toEqual(["code"]);
|
||||
|
||||
const linked = markdownToIR("[前**加粗:**后](https://example.com)");
|
||||
expect(linked.text).toBe("前加粗:后");
|
||||
expect(linked.links).toEqual([
|
||||
{ start: 0, end: linked.text.length, href: "https://example.com" },
|
||||
]);
|
||||
expect(linked.styles.filter((span) => span.style === "bold")).toEqual([
|
||||
{ start: 1, end: 4, style: "bold" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
// Markdown Core module implements ir behavior.
|
||||
import MarkdownIt from "markdown-it";
|
||||
import markdownItCjkFriendly from "markdown-it-cjk-friendly";
|
||||
import { chunkText } from "./chunk-text.js";
|
||||
import type { MarkdownTableMode } from "./types.js";
|
||||
|
||||
|
|
@ -150,6 +151,7 @@ function createMarkdownIt(options: MarkdownParseOptions): MarkdownIt {
|
|||
breaks: false,
|
||||
typographer: false,
|
||||
});
|
||||
md.use(markdownItCjkFriendly);
|
||||
md.enable("strikethrough");
|
||||
if (options.tableMode && options.tableMode !== "off") {
|
||||
md.enable("table");
|
||||
|
|
|
|||
20
pnpm-lock.yaml
generated
20
pnpm-lock.yaml
generated
|
|
@ -1918,6 +1918,9 @@ importers:
|
|||
markdown-it:
|
||||
specifier: 14.3.0
|
||||
version: 14.3.0
|
||||
markdown-it-cjk-friendly:
|
||||
specifier: 2.0.2
|
||||
version: 2.0.2(@types/markdown-it@14.1.2)(markdown-it@14.3.0)
|
||||
yaml:
|
||||
specifier: 2.9.0
|
||||
version: 2.9.0
|
||||
|
|
@ -5999,6 +6002,16 @@ packages:
|
|||
resolution: {integrity: sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q==}
|
||||
engines: {node: '>=16'}
|
||||
|
||||
markdown-it-cjk-friendly@2.0.2:
|
||||
resolution: {integrity: sha512-KXCl6sd129UqkAiRDb+NcAHrxC9xRa2WsGIsMMvtp2y1YlbeIaNYzArX2zfDoGhOjsyNMfJrGO7xGBss27YQSA==}
|
||||
engines: {node: '>=18'}
|
||||
peerDependencies:
|
||||
'@types/markdown-it': '*'
|
||||
markdown-it: '*'
|
||||
peerDependenciesMeta:
|
||||
'@types/markdown-it':
|
||||
optional: true
|
||||
|
||||
markdown-it-task-lists@2.1.1:
|
||||
resolution: {integrity: sha512-TxFAc76Jnhb2OUu+n3yz9RMu4CwGfaT788br6HhEDlvWfdeJcLUsxk1Hgw2yJio0OXsxv7pyIPmvECY7bMbluA==}
|
||||
|
||||
|
|
@ -11810,6 +11823,13 @@ snapshots:
|
|||
|
||||
markdown-extensions@2.0.0: {}
|
||||
|
||||
markdown-it-cjk-friendly@2.0.2(@types/markdown-it@14.1.2)(markdown-it@14.3.0):
|
||||
dependencies:
|
||||
get-east-asian-width: 1.6.0
|
||||
markdown-it: 14.3.0
|
||||
optionalDependencies:
|
||||
'@types/markdown-it': 14.1.2
|
||||
|
||||
markdown-it-task-lists@2.1.1: {}
|
||||
|
||||
markdown-it@14.3.0:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue