mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
788 lines
28 KiB
JavaScript
788 lines
28 KiB
JavaScript
import fs from "node:fs";
|
||
import path from "node:path";
|
||
import slugify, { slugifyWithCounter } from "@sindresorhus/slugify";
|
||
import { Parser } from "htmlparser2";
|
||
import MarkdownIt from "markdown-it";
|
||
import anchor from "markdown-it-anchor";
|
||
import { parse } from "yaml";
|
||
|
||
/** @public Consumed by openclaw/docs mdx-ish.mjs through the docs-sync support contract. */
|
||
export const markerPrefix = "OPENCLAW_DOCS_MARKER";
|
||
/** @public Consumed by openclaw/docs mdx-ish.mjs through the docs-sync support contract. */
|
||
export const inlineMarkerPrefix = "OPENCLAW_DOCS_INLINE";
|
||
const knownBlocks = new Map([
|
||
["AccordionGroup", ["accordion-group", ""]],
|
||
["Tabs", ["tabs", ""]],
|
||
["CodeGroup", ["code-group", ""]],
|
||
["TileGroup", ["tile-group", ""]],
|
||
["CTAGroup", ["cta-grid", ""]],
|
||
["StatGrid", ["stat-grid", ""]],
|
||
]);
|
||
const callouts = new Map([
|
||
["Note", "Note"],
|
||
["Warning", "Warning"],
|
||
["Tip", "Tip"],
|
||
["Info", "Info"],
|
||
["Check", "Check"],
|
||
["Say", "Say"],
|
||
["Banner", "Banner"],
|
||
["Update", "Update"],
|
||
]);
|
||
|
||
const components = new Map([
|
||
["Card", ["cardOpen", "cardClose", "cardSelf"]],
|
||
["CTA", ["ctaOpen", "ctaClose"]],
|
||
["CTACard", ["ctaCardOpen", "ctaCardClose", "ctaCardSelf"]],
|
||
["Lead", ["leadOpen", "leadClose"]],
|
||
["PullQuote", ["pullQuoteOpen", "pullQuoteClose"]],
|
||
["Stat", ["statOpen", "statClose", "statSelf"]],
|
||
["Steps", ["stepsOpen", "stepsClose"]],
|
||
["Step", ["stepOpen", "stepClose"]],
|
||
["Tab", ["tabOpen", "tabClose"]],
|
||
["Accordion", ["accordionOpen", "accordionClose"]],
|
||
["Expandable", ["accordionOpen", "accordionClose"]],
|
||
["Frame", ["frameOpen", "frameClose"]],
|
||
["Panel", ["panelOpen", "panelClose"]],
|
||
["Prompt", ["promptOpen", "promptClose"]],
|
||
["ParamField", ["paramOpen", "paramClose"]],
|
||
["Param", ["paramOpen", "paramClose"]],
|
||
["Field", ["paramOpen", "paramClose"]],
|
||
["Property", ["paramOpen", "paramClose"]],
|
||
["ResponseField", ["paramOpen", "paramClose"]],
|
||
["Tile", ["tileOpen", "tileClose", "tileSelf"]],
|
||
["Badge", ["badgeOpen", "badgeClose", "badgeSelf"]],
|
||
["Tooltip", ["tooltipOpen", "tooltipClose"]],
|
||
]);
|
||
const inlineComponents = new Set(["Badge", "Tooltip"]);
|
||
const gridComponents = new Set(["CardGroup", "Columns"]);
|
||
// Quoted values can contain placeholders such as <section>; only unquoted > closes a tag.
|
||
const componentAttrsPattern = String.raw`(?:"[^"]*"|'[^']*'|[^'">])*`;
|
||
const componentTag = new RegExp(
|
||
String.raw`<(\/)?([A-Z][A-Za-z0-9_.-]*)\b(${componentAttrsPattern})>`,
|
||
"g",
|
||
);
|
||
|
||
// Track source lines through the existing rewrites; inserted snippet content has
|
||
// no location in the including file. This metadata never enters rendered tokens.
|
||
class DocsSource {
|
||
constructor(text, firstLine) {
|
||
this.text = text;
|
||
if (firstLine === undefined) {
|
||
this.origins = [];
|
||
return;
|
||
}
|
||
let line = firstLine;
|
||
this.origins = Array.from({ length: text.length }, (_, index) => {
|
||
const origin = line;
|
||
if (text[index] === "\n") {
|
||
line++;
|
||
}
|
||
return origin;
|
||
});
|
||
}
|
||
|
||
replace(pattern, replacement, mapped = true) {
|
||
if (!this.origins.length) {
|
||
this.text = this.text.replace(pattern, replacement);
|
||
return this;
|
||
}
|
||
const origins = [];
|
||
let end = 0;
|
||
this.text = this.text.replace(pattern, (...args) => {
|
||
const offset = args.at(-2);
|
||
const value = typeof replacement === "function" ? replacement(...args) : replacement;
|
||
for (let i = end; i < offset; i++) {
|
||
origins.push(this.origins[i]);
|
||
}
|
||
for (let i = 0; i < value.length; i++) {
|
||
origins.push(
|
||
value === args[0] ? this.origins[offset + i] : mapped ? this.origins[offset] : undefined,
|
||
);
|
||
}
|
||
end = offset + args[0].length;
|
||
return value;
|
||
});
|
||
for (let i = end; i < this.origins.length; i++) {
|
||
origins.push(this.origins[i]);
|
||
}
|
||
this.origins = origins;
|
||
return this;
|
||
}
|
||
|
||
lines() {
|
||
if (!this.origins.length) {
|
||
return [];
|
||
}
|
||
let offset = 0;
|
||
return this.text.split("\n").map((line) => {
|
||
const origins = new Set(
|
||
this.origins.slice(offset, offset + line.length).filter((origin) => origin !== undefined),
|
||
);
|
||
offset += line.length + 1;
|
||
return origins.size === 1 ? origins.values().next().value : undefined;
|
||
});
|
||
}
|
||
}
|
||
|
||
function preprocess(input, restore) {
|
||
let out = input.replace(/\r\n/g, "\n").replace(/^import\s+.+?;?\s*$/gm, "");
|
||
out = out.replace(
|
||
new RegExp(String.raw`<Mermaid\b${componentAttrsPattern}>([\s\S]*?)<\/Mermaid>`, "g"),
|
||
(_, body) => `\n${marker("mermaidBlock", restore(body))}\n`,
|
||
);
|
||
out = out.replace(
|
||
new RegExp(String.raw`<Chart\b(${componentAttrsPattern})\/>`, "g"),
|
||
(_, attrs) => `\n${marker("chart", JSON.stringify({ attrs: restore(attrs), body: "" }))}\n`,
|
||
);
|
||
out = out.replace(
|
||
new RegExp(String.raw`<Chart\b(${componentAttrsPattern})>([\s\S]*?)<\/Chart>`, "g"),
|
||
(_, attrs, body) =>
|
||
`\n${marker("chart", JSON.stringify({ attrs: restore(attrs), body: restore(body) }))}\n`,
|
||
);
|
||
out = out.replace(/<br\s*\/?>/gi, "\n");
|
||
return out.replace(componentTag, (tag, closing, name, attrs) => {
|
||
let kind;
|
||
// Restore literal attributes before encoding hides their placeholders.
|
||
let value = closing ? "" : restore(attrs);
|
||
if (gridComponents.has(name) || knownBlocks.has(name)) {
|
||
kind = closing ? "blockClose" : "blockOpen";
|
||
value = gridComponents.has(name) ? cardGridClass(attrs) : knownBlocks.get(name)[0];
|
||
} else if (callouts.has(name)) {
|
||
kind = closing ? "calloutClose" : "calloutOpen";
|
||
value = callouts.get(name);
|
||
} else {
|
||
const definition = components.get(name);
|
||
if (!definition) {
|
||
return escapeHtml(tag);
|
||
}
|
||
kind = closing
|
||
? definition[1]
|
||
: attrs.endsWith("/") && definition[2]
|
||
? definition[2]
|
||
: definition[0];
|
||
}
|
||
// Block boundaries stay separate even for adjacent components on one line.
|
||
return inlineComponents.has(name)
|
||
? inlineMarker(kind, value)
|
||
: `\n\n${marker(kind, value)}\n\n`;
|
||
});
|
||
}
|
||
|
||
function marker(kind, payload = "") {
|
||
return `${markerPrefix}:${kind}:${Buffer.from(payload, "utf8").toString("base64url")}`;
|
||
}
|
||
|
||
function inlineMarker(kind, payload = "") {
|
||
return `${inlineMarkerPrefix}:${kind}:${Buffer.from(payload, "utf8").toString("base64url")}:`;
|
||
}
|
||
|
||
function cardGridClass(rawAttrs) {
|
||
const attrs = parseAttrs(rawAttrs);
|
||
const cols = Math.max(1, Math.min(4, Number.parseInt(attrs.cols ?? "", 10) || 2));
|
||
return `card-grid oc-card-cols-${cols}`;
|
||
}
|
||
|
||
/** @public Consumed by openclaw/docs mdx-ish.mjs through the docs-sync support contract. */
|
||
export function parseAttrs(raw) {
|
||
/** @type {Record<string, string>} */
|
||
const attrs = {};
|
||
for (const match of raw.matchAll(
|
||
/([A-Za-z0-9_-]+)(?:=(?:"([^"]*)"|'([^']*)'|\{([^}]*)\}|([^\s>]+)))?/g,
|
||
)) {
|
||
attrs[match[1]] = codeParser.utils.unescapeAll(
|
||
match[2] ?? match[3] ?? match[4]?.replace(/^['"]|['"]$/g, "") ?? match[5] ?? "",
|
||
);
|
||
}
|
||
return attrs;
|
||
}
|
||
|
||
function escapeHtml(value) {
|
||
return String(value)
|
||
.replaceAll("&", "&")
|
||
.replaceAll("<", "<")
|
||
.replaceAll(">", ">")
|
||
.replaceAll('"', """);
|
||
}
|
||
|
||
function escapeAttr(value) {
|
||
return escapeHtml(value).replaceAll("'", "'");
|
||
}
|
||
|
||
// Inline tokens lack block maps. Capture the parser cursor before link rules
|
||
// normalize hrefs or replace reference metadata, only for diagnostic projections.
|
||
const inlineLines = new WeakMap();
|
||
|
||
/** @public Consumed by openclaw/docs mdx-ish.mjs through the docs-sync support contract. */
|
||
export function createDocsMarkdown(options = {}) {
|
||
const md = new MarkdownIt({ html: true, linkify: false, typographer: false, ...options }).use(
|
||
anchor,
|
||
);
|
||
md.inline.State = class extends md.inline.State {
|
||
push(type, tag, nesting) {
|
||
const token = super.push(type, tag, nesting);
|
||
if (this.env.trackSourceLines && ["link_open", "image", "html_inline"].includes(type)) {
|
||
inlineLines.set(token, this.src.slice(0, this.pos).split("\n").length - 1);
|
||
}
|
||
return token;
|
||
}
|
||
};
|
||
return md;
|
||
}
|
||
|
||
const codeParser = new MarkdownIt({ html: false });
|
||
|
||
// Normalize the renderer's component indentation before CommonMark identifies code.
|
||
// Literal regions are restored only after preprocessing, so examples cannot become components.
|
||
function prepareDocument(input, { sourceFile, root, seen = new Set() }, firstLine) {
|
||
const saved = [];
|
||
let prefix = "OPENCLAWVERBATIM";
|
||
while (input.includes(prefix)) {
|
||
prefix += "X";
|
||
}
|
||
const placeholder = new RegExp(`${prefix}(\\d+)END`, "g");
|
||
const restore = (text) => text.replace(placeholder, (_, index) => saved[Number(index)]);
|
||
const hold = (value) => {
|
||
const key = `${prefix}${saved.length}END`;
|
||
saved.push(restore(value));
|
||
return key;
|
||
};
|
||
const jsxComments = new Map();
|
||
const literalContinuationLines = new Set();
|
||
const literals =
|
||
/(`+)(?:(?!\n(?:[ \t]*\r?\n|[ \t]*<(?:pre|script|style|textarea)\b))[^])*?\1|<!--[^]*?-->|\{\/\*[^]*?\*\/\}|<(pre|code|script|style|textarea)\b[^>]*>[^]*?<\/\2\s*>/g;
|
||
let literalEnd = 0;
|
||
let nextLiteral;
|
||
let inlineCode = false;
|
||
let jsxComment = false;
|
||
let literalLine = 0;
|
||
let depth = 0;
|
||
let fenceEndLine = 0;
|
||
const projectionLines = [];
|
||
const recordLine = (line, projection = line) => {
|
||
projectionLines.push(projection);
|
||
return line;
|
||
};
|
||
// Keep split("\n") semantics: CRLF must not create extra code-line entries.
|
||
let text = new DocsSource(input, firstLine).replace(
|
||
/(?<=^|\n)[^\n]*/g,
|
||
(line, offset, source) => {
|
||
const insideLiteral = offset < literalEnd;
|
||
let normalized =
|
||
depth && (!insideLiteral || inlineCode)
|
||
? line.replace(new RegExp(`^ {1,${depth * 2}}`), "")
|
||
: line;
|
||
const lineIndex = projectionLines.length;
|
||
if (insideLiteral && !inlineCode) {
|
||
literalContinuationLines.add(lineIndex);
|
||
}
|
||
if (lineIndex < fenceEndLine) {
|
||
return recordLine(normalized);
|
||
}
|
||
if (!insideLiteral && /`{3,}|~{3,}/.test(normalized)) {
|
||
const suffix = source.slice(offset + line.length);
|
||
// Keep the processed prefix for list/quote context. Component depth is
|
||
// fixed inside this fence; CommonMark owns its closing/container boundary.
|
||
const projection = [
|
||
...projectionLines,
|
||
normalized +
|
||
(depth ? suffix.replace(new RegExp(`^ {1,${depth * 2}}`, "gm"), "") : suffix),
|
||
].join("\n");
|
||
const token = codeParser
|
||
.parse(projection, {})
|
||
.find((entry) => entry.type === "fence" && entry.map[0] === lineIndex);
|
||
if (token) {
|
||
fenceEndLine = token.map[1];
|
||
return recordLine(normalized);
|
||
}
|
||
}
|
||
// The first opener owns its span: fenced examples cannot open raw HTML,
|
||
// and raw HTML cannot open fences. Keep each held piece on its source line.
|
||
let cursor = offset + line.length - normalized.length;
|
||
const end = offset + line.length;
|
||
const pieces = [];
|
||
const projectedPieces = [];
|
||
while (cursor < end) {
|
||
if (cursor < literalEnd) {
|
||
const value = source.slice(cursor, Math.min(literalEnd, end));
|
||
if (jsxComment) {
|
||
jsxComments.set(saved.length, { openerLine: literalLine, line: lineIndex });
|
||
}
|
||
const held = inlineCode ? value : hold(value);
|
||
pieces.push(held);
|
||
const structuralPrefix = inlineCode ? "" : value.match(/^[ \t]*(?:>[ \t]*)*/)[0];
|
||
projectedPieces.push(
|
||
(depth
|
||
? structuralPrefix.replace(new RegExp(`^ {1,${depth * 2}}`), "")
|
||
: structuralPrefix) + (structuralPrefix.length < value.length ? held : ""),
|
||
);
|
||
cursor += value.length;
|
||
continue;
|
||
}
|
||
if (nextLiteral === undefined || (nextLiteral && nextLiteral.index < cursor)) {
|
||
literals.lastIndex = cursor;
|
||
nextLiteral = literals.exec(source);
|
||
}
|
||
const literal = nextLiteral;
|
||
if (!literal || literal.index >= end) {
|
||
pieces.push(source.slice(cursor, end));
|
||
projectedPieces.push(source.slice(cursor, end));
|
||
break;
|
||
}
|
||
pieces.push(source.slice(cursor, literal.index));
|
||
projectedPieces.push(source.slice(cursor, literal.index));
|
||
cursor = literal.index;
|
||
literalEnd = cursor + literal[0].length;
|
||
inlineCode = literal[0].startsWith("`");
|
||
jsxComment = literal[0].startsWith("{/*");
|
||
literalLine = lineIndex;
|
||
}
|
||
normalized = pieces.join("");
|
||
for (const [, closing, name, attrs] of normalized
|
||
.replace(/(`+)[^\n]*?\1/g, "")
|
||
.matchAll(componentTag)) {
|
||
if (
|
||
inlineComponents.has(name) ||
|
||
!(
|
||
components.has(name) ||
|
||
knownBlocks.has(name) ||
|
||
callouts.has(name) ||
|
||
gridComponents.has(name)
|
||
)
|
||
) {
|
||
continue;
|
||
}
|
||
if (closing) {
|
||
depth = Math.max(0, depth - 1);
|
||
} else if (!attrs.endsWith("/")) {
|
||
depth++;
|
||
}
|
||
}
|
||
return recordLine(normalized, projectedPieces.join(""));
|
||
},
|
||
);
|
||
const codeLines = new Set();
|
||
const quoteRanges = [];
|
||
for (const token of codeParser.parse(projectionLines.join("\n"), {})) {
|
||
// Quote markers inside raw literals cannot establish a later comment's container.
|
||
if (token.type === "blockquote_open" && !literalContinuationLines.has(token.map[0])) {
|
||
quoteRanges.push(token.map);
|
||
}
|
||
if ((token.type === "fence" || token.type === "code_block") && token.map) {
|
||
for (let i = token.map[0]; i < token.map[1]; i++) {
|
||
codeLines.add(i);
|
||
}
|
||
}
|
||
}
|
||
// A comment's opener owns whether its bytes are code. Remove ordinary JSX
|
||
// comments before code holding can preserve their indented continuation lines.
|
||
text = text.replace(placeholder, (key, index) => {
|
||
const comment = jsxComments.get(Number(index));
|
||
if (!comment || codeLines.has(comment.openerLine)) {
|
||
return key;
|
||
}
|
||
const value = saved[Number(index)];
|
||
const quoteDepth = quoteRanges.filter(
|
||
([start, end]) => start <= comment.openerLine && comment.line < end,
|
||
).length;
|
||
const quotePrefix = value.match(new RegExp(`^(?:[ \\t]*>){0,${quoteDepth}}`))[0];
|
||
return quotePrefix + value.slice(quotePrefix.length).replace(/[^\n]/g, " ");
|
||
});
|
||
let sourceLine = 0;
|
||
text = text
|
||
.replace(/(?<=^|\n)[^\n]*/g, (line) => (codeLines.has(sourceLine++) ? hold(line) : line))
|
||
.replace(/(`+)([^]*?)\1/g, hold);
|
||
if (sourceFile) {
|
||
text = text.replace(
|
||
new RegExp(String.raw`<Snippet\b(${componentAttrsPattern})\/>`, "g"),
|
||
(_, rawAttrs) => {
|
||
const attrs = parseAttrs(rawAttrs);
|
||
const ref = attrs.file ?? attrs.src;
|
||
if (!ref) {
|
||
return "";
|
||
}
|
||
const target = path.resolve(path.dirname(sourceFile), ref);
|
||
const relative = path.relative(root, target);
|
||
if (
|
||
relative.startsWith("..") ||
|
||
path.isAbsolute(relative) ||
|
||
seen.has(target) ||
|
||
!fs.existsSync(target)
|
||
) {
|
||
return "";
|
||
}
|
||
const content = parseFrontmatter(fs.readFileSync(target, "utf8")).content;
|
||
return `\n${prepareDocument(content, { sourceFile: target, root, seen: new Set([...seen, target]) }).text.trim()}\n`;
|
||
},
|
||
false,
|
||
);
|
||
}
|
||
text = preprocess(text, restore);
|
||
return text.replace(placeholder, (_, index) => saved[Number(index)], false);
|
||
}
|
||
|
||
// Published documentation anchors retain these transliterations across slugify upgrades.
|
||
const publishedSlugReplacements = [
|
||
["'", "-"],
|
||
["إ", "i"],
|
||
["Ə", "-"],
|
||
["ə", "-"],
|
||
["Œ", "-"],
|
||
["œ", "-"],
|
||
["ẞ", "SS"],
|
||
["ու", "vo-"],
|
||
["ՈՒ", "Vo-"],
|
||
["Ու", "Vo-"],
|
||
["𝓀", "h"],
|
||
["𝕆", "N"],
|
||
["ⓒ", "(b)"],
|
||
["ⓓ", "(c)"],
|
||
];
|
||
|
||
function publishedBaseSlug(title, options) {
|
||
let value = slugify(title, {
|
||
...options,
|
||
lowercase: false,
|
||
decamelize: false,
|
||
customReplacements: publishedSlugReplacements,
|
||
});
|
||
if (options.decamelize) {
|
||
// The published acronym rule keeps APIUsage joined but splits APISection.
|
||
value = value
|
||
.replace(/([A-Z]{2})(\d)/g, "$1-$2")
|
||
.replace(/([a-z\d])([A-Z])/g, "$1-$2")
|
||
.replace(/([A-Z])([A-Z][a-rt-z\d]+)/g, "$1-$2");
|
||
}
|
||
if (options.lowercase !== false) {
|
||
value = value.toLowerCase();
|
||
}
|
||
// Published anchors join these suffixes before counting duplicates.
|
||
return value.replace(/([a-zA-Z\d]+)-([ts])(-|$)/g, "$1$2$3");
|
||
}
|
||
function publishedSlug(title, counter = slugifyWithCounter()) {
|
||
const encoded = anchor.defaults.slugify(title);
|
||
const options = /%[0-9A-F]{2}/.test(encoded)
|
||
? { decamelize: false, preserveCharacters: ["%", "_"], lowercase: false }
|
||
: { decamelize: false, preserveCharacters: ["_"] };
|
||
const base = publishedBaseSlug(encoded, options);
|
||
return counter(base, options);
|
||
}
|
||
function cleanPublishedId(id) {
|
||
return decodeURIComponent(id.replace(/%(?![0-9A-Fa-f]{2})/g, "%25"))
|
||
.replace(/[?,;:!'"()[\]{}]/g, "")
|
||
.replace(
|
||
/\p{Emoji_Modifier}|\p{Emoji_Modifier_Base}|\p{Emoji_Presentation}|\p{Extended_Pictographic}|\u200D|\uFE0E|\uFE0F/gu,
|
||
"",
|
||
);
|
||
}
|
||
function deduplicatePublishedId(id, seen) {
|
||
const count = seen.get(id) ?? 0;
|
||
seen.set(id, count + 1);
|
||
if (!count) {
|
||
return id;
|
||
}
|
||
let suffix = count + 1;
|
||
while (seen.has(`${id}-${suffix}`)) {
|
||
suffix++;
|
||
}
|
||
const result = `${id}-${suffix}`;
|
||
seen.set(result, 1);
|
||
return result;
|
||
}
|
||
|
||
function readHtmlTargets(chunks) {
|
||
const ids = [];
|
||
const links = [];
|
||
let verbatim = 0;
|
||
let chunk;
|
||
let offset = 0;
|
||
const parser = new Parser({
|
||
onopentag(name, attrs) {
|
||
if (attrs.id) {
|
||
ids.push(attrs.id);
|
||
}
|
||
if (["pre", "code", "script", "style", "textarea"].includes(name)) {
|
||
verbatim++;
|
||
}
|
||
if (!verbatim) {
|
||
for (const key of ["href", "src", "data-href"]) {
|
||
if (attrs[key]) {
|
||
const line = chunk.lineAt?.(
|
||
chunk.html.slice(0, parser.startIndex - offset).split("\n").length - 1,
|
||
);
|
||
links.push({ href: attrs[key], line });
|
||
}
|
||
}
|
||
}
|
||
},
|
||
onclosetag(name) {
|
||
if (["pre", "code", "script", "style", "textarea"].includes(name)) {
|
||
verbatim--;
|
||
}
|
||
},
|
||
});
|
||
for (chunk of chunks) {
|
||
parser.write(chunk.html);
|
||
offset += chunk.html.length;
|
||
}
|
||
parser.end();
|
||
return { ids, links };
|
||
}
|
||
|
||
/** @public Consumed by openclaw/docs assets.mjs through the docs-sync support contract. */
|
||
export function resolveDocsFragment(hash, ids) {
|
||
const raw = hash.startsWith("#") ? hash.slice(1) : hash;
|
||
if (ids.has(raw)) {
|
||
return raw;
|
||
}
|
||
try {
|
||
const decoded = decodeURIComponent(raw);
|
||
return ids.has(decoded) ? decoded : undefined;
|
||
} catch {
|
||
return undefined;
|
||
}
|
||
}
|
||
|
||
/** @public Consumed by openclaw/docs mdx-ish.mjs through the docs-sync support contract. */
|
||
export function parseDocsDocument(markdown, md = createDocsMarkdown(), options = {}) {
|
||
const env = options.mapLink ? { trackSourceLines: true } : {};
|
||
const parsed = parseFrontmatter(markdown);
|
||
const firstLine = options.mapLink
|
||
? markdown.slice(0, markdown.length - parsed.content.length).split("\n").length
|
||
: undefined;
|
||
const source = prepareDocument(parsed.content, options, firstLine);
|
||
const sourceLines = source.lines();
|
||
const tokens = md.parse(source.text, env);
|
||
const reserved = new Set();
|
||
const ids = [];
|
||
const links = [];
|
||
const collisions = [];
|
||
const componentTargets = [];
|
||
const candidates = [];
|
||
const headings = slugifyWithCounter();
|
||
const tabs = slugifyWithCounter();
|
||
const toc = new Map();
|
||
/** @type {Map<string, number>} */
|
||
const componentCounts = new Map();
|
||
const stack = [];
|
||
const reserve = (id) => {
|
||
if (id) {
|
||
if (reserved.has(id)) {
|
||
collisions.push({ id, reason: "duplicate authored/canonical ID" });
|
||
}
|
||
reserved.add(id);
|
||
ids.push(id);
|
||
}
|
||
};
|
||
const scanHtml = (chunks) => {
|
||
const found = readHtmlTargets(chunks);
|
||
found.ids.forEach(reserve);
|
||
links.push(...found.links);
|
||
};
|
||
for (let i = 0; i < tokens.length; i++) {
|
||
const token = tokens[i];
|
||
const lineAt =
|
||
options.mapLink && token.map ? (offset) => sourceLines[token.map[0] + offset] : undefined;
|
||
if (token.type === "heading_open") {
|
||
const id = token.attrGet("id");
|
||
reserve(id);
|
||
if (
|
||
!["custom", "frame"].includes(parsed.data.mode) &&
|
||
Number(token.tag.slice(1)) <= 4 &&
|
||
!/\{[^}]*\}/.test(tokens[i + 1].content)
|
||
) {
|
||
let alias = publishedSlug(
|
||
tokens[i + 1].children
|
||
.map((child) =>
|
||
child.type === "softbreak"
|
||
? "\n"
|
||
: ["text", "code_inline"].includes(child.type)
|
||
? child.content
|
||
: "",
|
||
)
|
||
.join(""),
|
||
headings,
|
||
);
|
||
if (
|
||
!stack.some(({ kind }) =>
|
||
["accordionOpen", "accordion-group", "Update", "promptOpen"].includes(kind),
|
||
)
|
||
) {
|
||
alias = deduplicatePublishedId(cleanPublishedId(alias), toc);
|
||
}
|
||
candidates.push({ token, id, alias });
|
||
}
|
||
}
|
||
if (token.type === "html_block") {
|
||
const original = token.content;
|
||
token.content = token.content.replace(
|
||
/(<div class="maturity-category-docs">)([\s\S]*?)(<\/div>)/g,
|
||
(_, open, body, close) => `${open}${normalizeEmbeddedMarkdownLinks(body)}${close}`,
|
||
);
|
||
scanHtml([{ html: token.content, lineAt: token.content === original ? lineAt : undefined }]);
|
||
}
|
||
if (token.type !== "inline") {
|
||
continue;
|
||
}
|
||
const match = token.content.match(new RegExp(`^${markerPrefix}:([^:]+):([A-Za-z0-9_-]*)$`));
|
||
if (
|
||
match &&
|
||
tokens[i - 1]?.type === "paragraph_open" &&
|
||
tokens[i + 1]?.type === "paragraph_close"
|
||
) {
|
||
const kind = match[1];
|
||
const value = Buffer.from(match[2], "base64url").toString("utf8");
|
||
const attrs = parseAttrs(value);
|
||
/** @type {{ token: typeof token, kind: string, attrs: Record<string, string>, id?: string, alias?: string }} */
|
||
const component = { token, kind, attrs, id: attrs.id, alias: undefined };
|
||
if (kind.endsWith("Close")) {
|
||
stack.pop();
|
||
} else if (kind.endsWith("Open")) {
|
||
stack.push({ kind: kind === "blockOpen" || kind === "calloutOpen" ? value : kind, attrs });
|
||
}
|
||
if (["accordionOpen", "stepOpen", "tabOpen", "paramOpen"].includes(kind)) {
|
||
reserve(attrs.id);
|
||
if (!attrs.id) {
|
||
const title = attrs.title;
|
||
if (kind === "tabOpen" && title) {
|
||
component.alias = publishedSlug(title, tabs);
|
||
} else if (kind === "stepOpen" && title && !["", "true"].includes(attrs.noAnchor)) {
|
||
const parent = stack.at(-2);
|
||
const size =
|
||
attrs.titleSize ??
|
||
(parent?.kind === "stepsOpen" ? parent.attrs.titleSize : undefined);
|
||
component.alias = ["h2", "h3"].includes(size)
|
||
? deduplicatePublishedId(publishedSlug(title), toc)
|
||
: publishedSlug(title);
|
||
} else if (kind === "accordionOpen" && title) {
|
||
component.alias = publishedBaseSlug(title.replace(":", "-"), { decamelize: false });
|
||
} else if (kind === "paramOpen") {
|
||
const name = attrs.query ?? attrs.path ?? attrs.body ?? attrs.header ?? attrs.name;
|
||
if (name) {
|
||
component.alias = publishedBaseSlug(`param-${name}`, { decamelize: true });
|
||
}
|
||
}
|
||
}
|
||
componentTargets.push(component);
|
||
}
|
||
for (const key of ["href", "primaryHref", "secondaryHref"]) {
|
||
if (attrs[key]) {
|
||
links.push({ href: md.utils.unescapeAll(attrs[key]), line: lineAt?.(0) });
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
const children = token.children ?? [];
|
||
scanHtml(
|
||
children.map((child, index) => ({
|
||
html: md.renderer.rules[child.type]
|
||
? md.renderer.rules[child.type](children, index, md.options, env, md.renderer)
|
||
: md.renderer.renderToken(children, index, md.options),
|
||
lineAt:
|
||
lineAt && inlineLines.has(child)
|
||
? (offset) => lineAt(inlineLines.get(child) + offset)
|
||
: undefined,
|
||
})),
|
||
);
|
||
}
|
||
// Published headings and authored HTML/component IDs win regardless of order.
|
||
// Aliases never steal them; ambiguous aliases are omitted and reported.
|
||
const aliasOwners = new Map();
|
||
for (const candidate of candidates) {
|
||
if (candidate.alias === candidate.id || !candidate.alias) {
|
||
continue;
|
||
}
|
||
const owners = aliasOwners.get(candidate.alias) ?? [];
|
||
owners.push(candidate);
|
||
aliasOwners.set(candidate.alias, owners);
|
||
}
|
||
for (const [alias, owners] of aliasOwners) {
|
||
if (reserved.has(alias) || owners.length !== 1) {
|
||
collisions.push({ id: alias, reason: "compatibility alias collision" });
|
||
continue;
|
||
}
|
||
reserve(alias);
|
||
const token = owners[0].token;
|
||
token.meta = { ...token.meta, anchorAlias: alias };
|
||
}
|
||
for (const component of componentTargets) {
|
||
if (!component.id && component.alias) {
|
||
const base = component.alias;
|
||
let count = componentCounts.get(base) ?? 0;
|
||
let id = count ? `${base}-${count}` : base;
|
||
while (reserved.has(id)) {
|
||
id = `${base}-${++count}`;
|
||
}
|
||
componentCounts.set(base, count + 1);
|
||
component.id = id;
|
||
reserve(id);
|
||
}
|
||
if (component.id) {
|
||
const encoded = Buffer.from(component.id).toString("base64url");
|
||
component.token.content += `:${encoded}`;
|
||
component.token.children[0].content = component.token.content;
|
||
}
|
||
}
|
||
return {
|
||
tokens,
|
||
env,
|
||
ids,
|
||
// The publisher consumes href strings. Audits project each occurrence here,
|
||
// before its source location is discarded, without a parallel return field.
|
||
links: links.map(({ href, line }) => {
|
||
const target = relativeDocsHref(href, options);
|
||
return options.mapLink ? options.mapLink(target, line) : target;
|
||
}),
|
||
collisions,
|
||
};
|
||
}
|
||
|
||
const openingDelimiter = /^---[ \t]*\r?\n/u;
|
||
const closingDelimiter = /\r?\n---[ \t]*(?:\r?\n|$)/u;
|
||
|
||
/** @public Consumed by openclaw/docs build.mjs and smoke.mjs through docs sync. */
|
||
export function parseFrontmatter(source) {
|
||
const input = String(source).replace(/^\uFEFF/u, "");
|
||
const opening = input.match(openingDelimiter);
|
||
if (!opening) {
|
||
return { data: {}, content: input };
|
||
}
|
||
|
||
const frontmatterStart = opening[0].length;
|
||
const closing = closingDelimiter.exec(input.slice(frontmatterStart));
|
||
if (!closing || closing.index === undefined) {
|
||
return { data: {}, content: input };
|
||
}
|
||
|
||
const data = parse(input.slice(frontmatterStart, frontmatterStart + closing.index)) ?? {};
|
||
const contentStart = frontmatterStart + closing.index + closing[0].length;
|
||
return { data, content: input.slice(contentStart) };
|
||
}
|
||
|
||
function normalizeEmbeddedMarkdownLinks(body) {
|
||
return body.replace(
|
||
/\[([^\]]+)\]\(((?:\/|https?:\/\/)[^)\s]+)\)/g,
|
||
(_, label, href) => `<a href="${escapeAttr(href)}">${escapeHtml(label)}</a>`,
|
||
);
|
||
}
|
||
|
||
function relativeDocsHref(href, { pageRoute } = {}) {
|
||
if (!pageRoute || /^(?:[#/?]|[a-z][a-z0-9+.-]*:)/i.test(href)) {
|
||
return href;
|
||
}
|
||
const url = new URL(href, `https://docs.openclaw.ai${pageRoute}`);
|
||
if (/\.[^/]+$/.test(url.pathname) && !/\.mdx?$/.test(url.pathname)) {
|
||
return href;
|
||
}
|
||
const target = url.pathname.replace(/\.mdx?$/, "").replace(/\/index$/, "") || "/";
|
||
return `${target}${url.search}${url.hash}`;
|
||
}
|
||
|
||
/** @public Consumed by openclaw/docs mdx-ish.mjs through docs sync. */
|
||
export function rewriteDocsRelativeLinks(html, options) {
|
||
return html.replace(/<(?:a|span)\b[^>]*>/g, (tag) =>
|
||
tag.replace(/\b(href|data-href)=(['"])(.*?)\2/g, (_, attr, quote, href) => {
|
||
const target = relativeDocsHref(codeParser.utils.unescapeAll(href), options);
|
||
return `${attr}=${quote}${escapeAttr(target)}${quote}`;
|
||
}),
|
||
);
|
||
}
|