diff --git a/src/agents/tools/web-fetch-utils.test.ts b/src/agents/tools/web-fetch-utils.test.ts index b0f004462e5..bf3860cb23a 100644 --- a/src/agents/tools/web-fetch-utils.test.ts +++ b/src/agents/tools/web-fetch-utils.test.ts @@ -1,6 +1,11 @@ // web_fetch extraction utility tests cover HTML entity decoding. import { describe, expect, it } from "vitest"; -import { htmlToMarkdown, truncateText } from "./web-fetch-utils.js"; +import { + extractBasicHtmlContent, + htmlToMarkdown, + markdownToText, + truncateText, +} from "./web-fetch-utils.js"; describe("web-fetch-utils htmlToMarkdown entity decoding", () => { const grin = String.fromCodePoint(0x1f600); // 😀 — an astral (> U+FFFF) code point @@ -40,6 +45,261 @@ describe("web-fetch-utils htmlToMarkdown entity decoding", () => { expect(htmlToMarkdown(`

'x; end

`).text).toBe("'x; end"); }); + it("renders basic HTML structure with a forward-only scanner", () => { + const rendered = htmlToMarkdown( + `My & Page

Intro

Go there

`, + ); + + expect(rendered.title).toBe("My & Page"); + expect(rendered.text).toBe("# Intro\nGo [there](/docs?x=1&y=2)\n\n- One\n- Two"); + }); + + it("drops script, style, and noscript raw-text blocks even when one is unterminated", () => { + const payload = "x".repeat(10_000); + + expect( + htmlToMarkdown( + `

Before

After

`, + ).text, + ).toBe("Before\nAfter"); + expect(htmlToMarkdown(`

Before

Shown

`).text).toBe( + "Visible\nShown", + ); + }); + + it("does not end raw-text blocks inside opener attributes", () => { + const rendered = htmlToMarkdown( + `

Visible

`, + ); + + expect(rendered.text).toBe("Visible"); + expect(rendered.text).not.toContain("Ignore previous instructions"); + }); + + it("ignores raw-text-looking openers inside closed quoted attributes", () => { + const rendered = htmlToMarkdown( + `Read

After

`, + ); + + expect(rendered.text).toBe("[Read](/real)After"); + }); + + it("re-enters raw-text parsing when an invalid tag span contains a raw-text opener", () => { + const rendered = htmlToMarkdown(`<

Visible

`); + + expect(rendered.text).toBe(" { + const rendered = htmlToMarkdown( + ` { + const rendered = htmlToMarkdown(`2 < 3

Visible

`); + + expect(rendered.text).toBe("2 < 3 Visible"); + expect(rendered.text).not.toContain("Ignore"); + }); + + it("skips comments without leaking raw-text-looking content", () => { + const rendered = htmlToMarkdown( + `

Visible

`, + ); + + expect(rendered.text).toBe("Visible"); + expect(rendered.text).not.toContain("Ignore previous instructions"); + }); + + it("continues after abruptly closed empty comments", () => { + expect(htmlToMarkdown(`

Before

After

`).text).toBe("Before\nAfter"); + expect(htmlToMarkdown(`

Before

After

`).text).toBe("Before\nAfter"); + }); + + it("does not treat underscore tag names as raw-text tags", () => { + expect(htmlToMarkdown(`Visible

After

`).text).toBe( + "VisibleAfter", + ); + }); + + it("does not treat dotted tag names as raw-text tags", () => { + expect(htmlToMarkdown(`Visible

After

`).text).toBe( + "VisibleAfter", + ); + }); + + it("skips raw-text blocks without reusing indices from a lowercased copy", () => { + expect(htmlToMarkdown(`İ

After

`).text).toBe("İAfter"); + }); + + it("reads href attributes without matching quoted text from another attribute", () => { + expect(htmlToMarkdown(`
Read`).text).toBe( + "[Read](/real)", + ); + }); + + it("continues href scanning after unsupported framework-style attributes", () => { + expect(htmlToMarkdown(`Read`).text).toBe("[Read](/real)"); + expect(htmlToMarkdown(`Read`).text).toBe( + "[Read](/real)", + ); + }); + + it("preserves slashes in unquoted href attributes", () => { + expect(htmlToMarkdown(`Read`).text).toBe( + "[Read](https://example.com/path)", + ); + expect(htmlToMarkdown(`Read`).text).toBe("[Read](/docs/path)"); + expect(htmlToMarkdown(`Docs`).text).toBe("[Docs](/docs/)"); + expect(htmlToMarkdown(`Docs`).text).toBe( + "[Docs](https://example.com/)", + ); + }); + + it("preserves hrefs when anchor labels strip to empty", () => { + expect(htmlToMarkdown(`

See

`).text).toBe( + "See /next", + ); + expect(htmlToMarkdown(``).text).toBe("/next"); + }); + + it("treats quoted self-closing anchors as closed", () => { + expect(htmlToMarkdown(`after`).text).toBe("after"); + }); + + it("keeps bare less-than text from swallowing later closing tags", () => { + expect(htmlToMarkdown(`my <3 story rest`).text).toBe("[my <3 story](/x) rest"); + expect(htmlToMarkdown(`2 < 3

Body

`)).toEqual({ + text: "Body", + title: "2 < 3", + }); + }); + + it("closes titles when literal title text looks like nested markup", () => { + expect(htmlToMarkdown(`My <a> Site

Hello

`)).toEqual({ + text: "Hello", + title: "My Site", + }); + expect(htmlToMarkdown(`My <h1> Site</h1>

Hello

`)).toEqual({ + text: "Hello", + title: "My Site", + }); + }); + + it("bounds nested render contexts from malformed repeated anchors", () => { + const rendered = htmlToMarkdown(`t`.repeat(100)).text; + + expect(rendered.match(/\[t]\(\/x\)/g)).toHaveLength(100); + }); + + it("does not rescan empty anchor text on each block open", () => { + const rendered = htmlToMarkdown(`${"

".repeat(1_000)}`).text; + + expect(rendered).toBe("/x"); + }); + + it("closes stale anchors before structural content claims the rest of the page", () => { + expect( + htmlToMarkdown(`
deal

Para one.

Head

Para two.

`).text, + ).toBe("[deal](/promo) Para one.\n\n# Head\nPara two."); + }); + + it("drops bogus closing tags instead of exposing hidden text", () => { + const rendered = htmlToMarkdown(`

Hi

Bye

`); + + expect(rendered.text).toBe("Hi\nBye"); + expect(rendered.text).not.toContain("IGNORE PREVIOUS INSTRUCTIONS"); + }); + + it("preserves card-style anchors around block content", () => { + expect(htmlToMarkdown(`

Title

`).text).toBe("[Title](/post)"); + expect(htmlToMarkdown(`
Card text
`).text).toBe("[Card text](/x)"); + }); + + it("closes anchors through unclosed nested contexts", () => { + expect(htmlToMarkdown(`

Card

Body text

`).text).toBe( + "[Card](/p)Body text", + ); + }); + + it("closes heading and list contexts through nested anchors", () => { + expect(htmlToMarkdown(`

Head link

Body one.

`).text).toBe( + "# Head [link](/x)\nBody one.", + ); + expect(htmlToMarkdown(`
  • Item link
  • Body

    `).text).toBe( + "- Item [link](/x)Body", + ); + }); + + it("uses the title as fallback content when an HTML shell has no body text", async () => { + await expect( + extractBasicHtmlContent({ html: `Shell Page`, extractMode: "markdown" }), + ).resolves.toEqual({ text: "Shell Page", title: "Shell Page" }); + await expect( + extractBasicHtmlContent({ html: `Shell Page`, extractMode: "text" }), + ).resolves.toEqual({ text: "Shell Page", title: "Shell Page" }); + }); + + it("consumes a malformed tag tail once instead of rescanning every later less-than", () => { + const payload = ` { + const payload = `${`"`.repeat(1_000)}

    Visible

    `; + const rendered = htmlToMarkdown(payload).text; + + expect(rendered).toContain("Visible"); + expect(rendered).not.toContain("script"); + }); + + it("resyncs raw-text openers from repeated unterminated quoted tags", () => { + const payload = `${`
    { + const rendered = htmlToMarkdown(` { + const rendered = htmlToMarkdown( + `

    Visible

    After

    `, + ); + + expect(rendered.text).toBe("Visible\nAfter"); + expect(rendered.text).not.toContain("INJECTED PROMPT"); + }); + + it("consumes repeated invalid tags before a later close bracket in one span", () => { + const payload = `${"<".repeat(20_000)}>`; + + expect(htmlToMarkdown(payload).text).toBe(payload); + }); + + it("strips markdown fences in a forward pass without changing adjacent fence output", () => { + const fenced = `${"```js\nx\n```".repeat(1_000)}after`; + + expect(markdownToText(fenced)).toBe(`${"x\n".repeat(1_000)}after`); + }); + it("truncates without splitting a boundary emoji", () => { const prefix = "a".repeat(79); const result = truncateText(`${prefix}${grin}tail`, 80); diff --git a/src/agents/tools/web-fetch-utils.ts b/src/agents/tools/web-fetch-utils.ts index 2499ee550f3..30ddaa7228d 100644 --- a/src/agents/tools/web-fetch-utils.ts +++ b/src/agents/tools/web-fetch-utils.ts @@ -10,19 +10,46 @@ import { sanitizeHtml, stripInvisibleUnicode } from "./web-fetch-visibility.js"; /** Output mode requested by web_fetch extraction. */ export type ExtractMode = "markdown" | "text"; -const HTML_TAG_RE = /<[^>]+>/g; -const SCRIPT_TAG_BLOCK_RE = //gi; -const STYLE_TAG_BLOCK_RE = //gi; -const NOSCRIPT_TAG_BLOCK_RE = //gi; -const SCRIPT_STYLE_NOSCRIPT_OPEN_RE = /<(?:script|style|noscript)\b/i; -const ANCHOR_OPEN_RE = /]*href=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi; -const HEADING_OPEN_RE = /]*>([\s\S]*?)<\/h\1>/gi; -const LIST_ITEM_OPEN_RE = /]*>([\s\S]*?)<\/li>/gi; -const BLOCK_BREAK_RE = - /<(?:br|hr)\s*\/?>|<\/(?:p|div|section|article|header|footer|table|tr|ul|ol)>/gi; +const RAW_TEXT_TAGS = new Set(["script", "style", "noscript"]); +const BLOCK_BREAK_TAGS = new Set([ + "p", + "div", + "section", + "article", + "header", + "footer", + "table", + "tr", + "ul", + "ol", +]); +// Keep malformed nested markup from making end-of-document context unwind quadratic. +// web_fetch favors bounded, auditable text over preserving deep broken HTML structure. +const MAX_RENDER_CONTEXT_DEPTH = 32; + +type RenderContext = + | { kind: "root"; parts: string[] } + | { kind: "title"; parts: string[] } + | { kind: "anchor"; href: string | undefined; hasText: boolean; parts: string[] } + | { kind: "heading"; level: number; parts: string[] } + | { kind: "list-item"; parts: string[] }; + +type HtmlTagToken = { + closing: boolean; + name: string; + raw: string; + selfClosing: boolean; +}; + +type ReadTagResult = { + token: HtmlTagToken | null; + next: number; +}; + +type TagEndResult = { + end: number; + rawTextStart?: number; +}; // Decode entities through the canonical shared decoder (agents/utils/html.ts) so web_fetch and the // renderer share one entity contract — the divergent hand-rolled copy here was what truncated astral @@ -53,9 +80,544 @@ function decodeEntities(value: string): string { return out; } +function isAsciiWhitespace(value: string): boolean { + return value === " " || value === "\n" || value === "\r" || value === "\t" || value === "\f"; +} + +function isTagNameChar(value: string): boolean { + const code = value.charCodeAt(0); + return ( + (code >= 48 && code <= 57) || + (code >= 65 && code <= 90) || + (code >= 97 && code <= 122) || + value === "." || + value === "-" || + value === "_" || + value === ":" + ); +} + +function isTagNameStartChar(value: string): boolean { + const code = value.charCodeAt(0); + return (code >= 65 && code <= 90) || (code >= 97 && code <= 122); +} + +function isTagBoundary(value: string | undefined): boolean { + return !value || isAsciiWhitespace(value) || value === ">" || value === "/"; +} + +function asciiLower(value: string): string { + const code = value.charCodeAt(0); + return code >= 65 && code <= 90 ? String.fromCharCode(code + 32) : value; +} + +function startsWithClosingTag(html: string, start: number, tagName: string): boolean { + if (html[start] !== "<" || html[start + 1] !== "/") { + return false; + } + for (let offset = 0; offset < tagName.length; offset += 1) { + if (asciiLower(html[start + 2 + offset] ?? "") !== tagName[offset]) { + return false; + } + } + return isTagBoundary(html[start + 2 + tagName.length]); +} + +function readRawTextOpenTagName(html: string, start: number): string | undefined { + if (html[start] !== "<" || html[start + 1] === "/") { + return undefined; + } + for (const tagName of RAW_TEXT_TAGS) { + let matches = true; + for (let offset = 0; offset < tagName.length; offset += 1) { + if (asciiLower(html[start + 1 + offset] ?? "") !== tagName[offset]) { + matches = false; + break; + } + } + if (matches && isTagBoundary(html[start + 1 + tagName.length])) { + return tagName; + } + } + return undefined; +} + +function findRawTextOpenTagStart(html: string, start: number, end: number): number { + for (let i = start; i < end; i += 1) { + if (readRawTextOpenTagName(html, i)) { + return i; + } + } + return -1; +} + +function startsLikeHtmlTag(html: string, start: number): boolean { + const next = html[start + 1]; + return next === "!" || next === "?" || next === "/" || isTagNameStartChar(next ?? ""); +} + +function findTagEnd(html: string, start: number): TagEndResult { + let quote: string | null = null; + let afterEquals = false; + let rawTextStartInQuote: number | undefined; + for (let i = start + 1; i < html.length; i += 1) { + const ch = html[i]; + if (quote) { + if (rawTextStartInQuote === undefined && readRawTextOpenTagName(html, i)) { + rawTextStartInQuote = i; + } + if (ch === quote) { + quote = null; + } + continue; + } + if (afterEquals && isAsciiWhitespace(ch)) { + continue; + } + if (afterEquals && (ch === '"' || ch === "'")) { + quote = ch; + afterEquals = false; + continue; + } + afterEquals = false; + if (readRawTextOpenTagName(html, i)) { + return { end: -1, rawTextStart: i }; + } + if (ch === ">") { + return { end: i }; + } + if (ch === "=") { + afterEquals = true; + } + } + return { end: -1, rawTextStart: rawTextStartInQuote }; +} + +function isSelfClosingTagRaw(raw: string): boolean { + const trimmed = raw.trimEnd(); + if (!trimmed.endsWith("/")) { + return false; + } + const beforeSlash = trimmed[trimmed.length - 2]; + const tagBody = trimmed.slice(0, -1); + let hasAttributeSeparator = false; + for (const ch of tagBody) { + if (isAsciiWhitespace(ch)) { + hasAttributeSeparator = true; + break; + } + } + return ( + !beforeSlash || + isAsciiWhitespace(beforeSlash) || + beforeSlash === '"' || + beforeSlash === "'" || + !hasAttributeSeparator + ); +} + +function readTagToken(html: string, start: number): ReadTagResult | null { + if (html.startsWith("", start + 4); + return { + token: null, + next: commentEnd === -1 ? html.length : commentEnd + 3, + }; + } + + const tagEnd = findTagEnd(html, start); + const end = tagEnd.end; + if (end === -1) { + if (tagEnd.rawTextStart !== undefined) { + return { + token: null, + next: tagEnd.rawTextStart, + }; + } + return null; + } + + let pos = start + 1; + while (pos < end && isAsciiWhitespace(html[pos])) { + pos += 1; + } + const closing = html[pos] === "/"; + if (closing) { + pos += 1; + while (pos < end && isAsciiWhitespace(html[pos])) { + pos += 1; + } + } + + if (pos >= end || html[pos] === "!" || html[pos] === "?") { + return { + token: null, + next: end + 1, + }; + } + + const nameStart = pos; + while (pos < end && isTagNameChar(html[pos])) { + pos += 1; + } + if (pos === nameStart || !isTagNameStartChar(html[nameStart] ?? "")) { + const rawTextStart = findRawTextOpenTagStart(html, start + 1, end + 1); + if (rawTextStart !== -1) { + return { + token: null, + next: rawTextStart, + }; + } + return { + token: null, + next: end + 1, + }; + } + + const raw = html.slice(start + 1, end); + return { + token: { + closing, + name: html.slice(nameStart, pos).toLowerCase(), + raw, + selfClosing: isSelfClosingTagRaw(raw), + }, + next: end + 1, + }; +} + +function readAttributeValue(rawTag: string, name: string): string | undefined { + const target = name.toLowerCase(); + let pos = 0; + while (pos < rawTag.length && !isAsciiWhitespace(rawTag[pos])) { + pos += 1; + } + while (pos < rawTag.length) { + while (pos < rawTag.length && (isAsciiWhitespace(rawTag[pos]) || rawTag[pos] === "/")) { + pos += 1; + } + const attrStart = pos; + while (pos < rawTag.length && isTagNameChar(rawTag[pos])) { + pos += 1; + } + if (pos === attrStart) { + pos = skipUnsupportedAttribute(rawTag, pos); + continue; + } + const attrName = rawTag.slice(attrStart, pos).toLowerCase(); + while (pos < rawTag.length && isAsciiWhitespace(rawTag[pos])) { + pos += 1; + } + let value = ""; + if (rawTag[pos] === "=") { + pos += 1; + while (pos < rawTag.length && isAsciiWhitespace(rawTag[pos])) { + pos += 1; + } + const quote = rawTag[pos]; + if (quote === '"' || quote === "'") { + const valueStart = pos + 1; + const valueEnd = rawTag.indexOf(quote, valueStart); + if (valueEnd === -1) { + value = rawTag.slice(valueStart); + pos = rawTag.length; + } else { + value = rawTag.slice(valueStart, valueEnd); + pos = valueEnd + 1; + } + } else { + const valueStart = pos; + while ( + pos < rawTag.length && + !isAsciiWhitespace(rawTag[pos]) && + rawTag[pos] !== '"' && + rawTag[pos] !== "'" && + rawTag[pos] !== "=" && + rawTag[pos] !== "<" && + rawTag[pos] !== ">" && + rawTag[pos] !== "`" + ) { + pos += 1; + } + value = rawTag.slice(valueStart, pos); + } + } + if (attrName === target) { + return decodeEntities(value); + } + } + return undefined; +} + +function skipUnsupportedAttribute(rawTag: string, start: number): number { + let pos = start; + while (pos < rawTag.length && !isAsciiWhitespace(rawTag[pos])) { + const quote = rawTag[pos]; + if (quote === '"' || quote === "'") { + const valueEnd = rawTag.indexOf(quote, pos + 1); + pos = valueEnd === -1 ? rawTag.length : valueEnd + 1; + continue; + } + pos += 1; + } + return pos; +} + +function contextText(context: RenderContext): string { + return context.parts.join(""); +} + +function appendText(stack: RenderContext[], value: string): void { + const context = stack[stack.length - 1]; + context?.parts.push(value); + if (context?.kind === "anchor" && /\S/.test(value)) { + context.hasText = true; + } +} + +function closeContext( + context: RenderContext, + parent: RenderContext, + state: { title?: string }, +): void { + const label = normalizeWhitespace(contextText(context)); + if (!label && context.kind !== "title" && !(context.kind === "anchor" && context.href)) { + return; + } + switch (context.kind) { + case "title": + state.title ??= label || undefined; + return; + case "anchor": + if (parent.kind === "title") { + parent.parts.push(label); + } else { + parent.parts.push( + context.href && label ? `[${label}](${context.href})` : label || context.href || "", + ); + } + return; + case "heading": + if (parent.kind === "title") { + parent.parts.push(label); + } else if (parent.kind === "anchor") { + parent.parts.push(label); + parent.hasText ||= Boolean(label); + } else { + parent.parts.push(`\n${"#".repeat(context.level)} ${label}\n`); + } + return; + case "list-item": + if (parent.kind === "title") { + parent.parts.push(label); + } else { + if (parent.kind === "anchor") { + parent.hasText ||= Boolean(label); + } + parent.parts.push(`\n- ${label}`); + } + return; + case "root": + parent.parts.push(label); + } +} + +function closeTopContext(stack: RenderContext[], state: { title?: string }): boolean { + if (stack.length < 2) { + return false; + } + const context = stack.pop(); + const parent = stack[stack.length - 1]; + if (!context || !parent) { + return false; + } + closeContext(context, parent, state); + return true; +} + +function closeThroughContext( + stack: RenderContext[], + kind: RenderContext["kind"], + state: { title?: string }, +): boolean { + for (let i = stack.length - 1; i > 0; i -= 1) { + if (stack[i]?.kind === kind) { + while (stack.length > i) { + closeTopContext(stack, state); + } + return true; + } + } + return false; +} + +function pushContext( + stack: RenderContext[], + context: Exclude, + state: { title?: string }, +): void { + while (stack.length >= MAX_RENDER_CONTEXT_DEPTH) { + closeTopContext(stack, state); + } + stack.push(context); +} + +function closeOpenAnchorWithText(stack: RenderContext[], state: { title?: string }): boolean { + for (let i = stack.length - 1; i > 0; i -= 1) { + const context = stack[i]; + if (context?.kind === "anchor") { + if (!context.hasText) { + return false; + } + while (stack.length > i) { + closeTopContext(stack, state); + } + return true; + } + } + return false; +} + +function closeRawTextTagEnd(html: string, tagName: string, contentStart: number): number { + let closeStart = html.indexOf(" 1) { + closeTopContext(stack, state); + } + + return { + text: normalizeWhitespace(contextText(root)), + title: state.title, + }; +} + function stripTags(value: string): string { - const withoutTags = value.includes("<") ? value.replace(HTML_TAG_RE, "") : value; - return decodeEntities(withoutTags); + return htmlFragmentToMarkdown(value).text; } /** Collapses display whitespace while preserving paragraph breaks. */ @@ -70,42 +632,7 @@ export function normalizeWhitespace(value: string): string { /** Converts sanitized HTML into coarse markdown plus an optional title. */ export function htmlToMarkdown(html: string): { text: string; title?: string } { - const titleMatch = html.match(/]*>([\s\S]*?)<\/title>/i); - const title = titleMatch ? normalizeWhitespace(stripTags(titleMatch[1])) : undefined; - let text = html; - if (SCRIPT_STYLE_NOSCRIPT_OPEN_RE.test(text)) { - text = text - .replace(SCRIPT_TAG_BLOCK_RE, "") - .replace(STYLE_TAG_BLOCK_RE, "") - .replace(NOSCRIPT_TAG_BLOCK_RE, ""); - } - if (ANCHOR_OPEN_RE.test(text)) { - text = text.replace(ANCHOR_RE, (_, href, body) => { - const label = normalizeWhitespace(stripTags(body)); - if (!label) { - return href; - } - // Preserve link targets in markdown mode so fetched pages remain source-auditable. - return `[${label}](${href})`; - }); - } - if (HEADING_OPEN_RE.test(text)) { - text = text.replace(HEADING_RE, (_, level, body) => { - const prefix = "#".repeat(Math.max(1, Math.min(6, Number.parseInt(level, 10)))); - const label = normalizeWhitespace(stripTags(body)); - return `\n${prefix} ${label}\n`; - }); - } - if (LIST_ITEM_OPEN_RE.test(text)) { - text = text.replace(LIST_ITEM_RE, (_, body) => { - const label = normalizeWhitespace(stripTags(body)); - return label ? `\n- ${label}` : ""; - }); - } - text = text.replace(BLOCK_BREAK_RE, "\n"); - text = stripTags(text); - text = normalizeWhitespace(text); - return { text, title }; + return htmlFragmentToMarkdown(html); } /** Removes markdown decoration for plain text extraction. */ @@ -113,9 +640,27 @@ export function markdownToText(markdown: string): string { let text = markdown; text = text.replace(/!\[[^\]]*]\([^)]+\)/g, ""); text = text.replace(/\[([^\]]+)]\([^)]+\)/g, "$1"); - text = text.replace(/```[\s\S]*?```/g, (block) => - block.replace(/```[^\n]*\n?/g, "").replace(/```/g, ""), - ); + let unfenced = ""; + let pos = 0; + while (pos < text.length) { + const open = text.indexOf("```", pos); + if (open === -1) { + unfenced += text.slice(pos); + break; + } + unfenced += text.slice(pos, open); + const afterOpen = open + 3; + const close = text.indexOf("```", afterOpen); + if (close === -1) { + unfenced += text.slice(open); + break; + } + const firstLineEnd = text.indexOf("\n", afterOpen); + const contentStart = firstLineEnd === -1 || firstLineEnd > close ? afterOpen : firstLineEnd + 1; + unfenced += text.slice(contentStart, close); + pos = close + 3; + } + text = unfenced; text = text.replace(/`([^`]+)`/g, "$1"); text = text.replace(/^#{1,6}\s+/gm, ""); text = text.replace(/^\s*[-*+]\s+/gm, ""); @@ -144,9 +689,10 @@ export async function extractBasicHtmlContent(params: { if (params.extractMode === "text") { const text = stripInvisibleUnicode(markdownToText(rendered.text)) || + stripInvisibleUnicode(rendered.title ?? "") || stripInvisibleUnicode(normalizeWhitespace(stripTags(cleanHtml))); return text ? { text, title: rendered.title } : null; } - const text = stripInvisibleUnicode(rendered.text); + const text = stripInvisibleUnicode(rendered.text) || stripInvisibleUnicode(rendered.title ?? ""); return text ? { text, title: rendered.title } : null; }