mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-10 00:11:19 +00:00
test(firecrawl): add unit tests for firecrawl client functions (#97714)
This commit is contained in:
solodmd
GitHub
parent
d21ddd3f7a
commit
86b9b88f40
2 changed files with 739 additions and 4 deletions
720
extensions/firecrawl/src/firecrawl-client.test.ts
Normal file
720
extensions/firecrawl/src/firecrawl-client.test.ts
Normal file
|
|
@ -0,0 +1,720 @@
|
|||
// Firecrawl tests cover firecrawl client behavior — URL safety,
|
||||
// scrape payload parsing, and search-item extraction.
|
||||
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
|
||||
let firecrawlClient: typeof import("./firecrawl-client.js").testing;
|
||||
|
||||
beforeAll(async () => {
|
||||
firecrawlClient = (
|
||||
await vi.importActual<typeof import("./firecrawl-client.js")>("./firecrawl-client.js")
|
||||
).testing;
|
||||
});
|
||||
|
||||
afterAll(() => {
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// assertFirecrawlScrapeTargetAllowed
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("assertFirecrawlScrapeTargetAllowed", () => {
|
||||
it("allows valid public HTTPS URLs", () => {
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("https://example.com/page"),
|
||||
).not.toThrow();
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("https://api.firecrawl.dev/v1/scrape"),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects invalid URL strings", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("not a url")).toThrow(
|
||||
"Invalid URL",
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("")).toThrow("Invalid URL");
|
||||
});
|
||||
|
||||
it("rejects non-HTTP(S) protocols", () => {
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("ftp://example.com/file"),
|
||||
).toThrow(/Blocked non-HTTP\(S\) protocol/);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("file:///etc/passwd")).toThrow(
|
||||
/Blocked non-HTTP\(S\) protocol/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("data:text/html,<x>")).toThrow(
|
||||
/Blocked non-HTTP\(S\) protocol/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects private and loopback IP addresses", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://127.0.0.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://10.0.0.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://192.168.1.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://172.16.0.1")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects blocked hostnames like localhost", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://localhost")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://LOCALHOST")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
});
|
||||
|
||||
it("allows HTTP URLs to public hosts (SSRF check targets the hostname, not the scheme)", () => {
|
||||
// Plain HTTP to a public hostname is not blocked here — the SSRF
|
||||
// layer resolves the hostname to decide if it targets a private network.
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://example.com"),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("rejects IPv6 loopback and private addresses", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://[::1]")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("https://[::1]")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://[fc00::]")).toThrow(
|
||||
/Blocked/,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects URL with embedded credentials targeting a blocked host", () => {
|
||||
// Credentials in the URL do not bypass the hostname/IP check.
|
||||
expect(() =>
|
||||
firecrawlClient.assertFirecrawlScrapeTargetAllowed("http://user:pass@127.0.0.1"),
|
||||
).toThrow(/Blocked/);
|
||||
});
|
||||
|
||||
it("rejects bare hostname strings without a scheme as invalid", () => {
|
||||
expect(() => firecrawlClient.assertFirecrawlScrapeTargetAllowed("example.com")).toThrow(
|
||||
"Invalid URL",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// resolveSearchItems
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("resolveSearchItems", () => {
|
||||
it("extracts items from a top-level data array (Firecrawl Search API)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com", title: "Example" },
|
||||
{ url: "https://openclaw.ai", title: "OpenClaw" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0]).toMatchObject({ url: "https://example.com", title: "Example" });
|
||||
expect(result[1]).toMatchObject({ url: "https://openclaw.ai", title: "OpenClaw" });
|
||||
});
|
||||
|
||||
it("extracts items from a results array", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
results: [{ url: "https://example.org", title: "Org" }],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.org");
|
||||
expect(result[0].title).toBe("Org");
|
||||
});
|
||||
|
||||
it("extracts items from data.results (nested)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: {
|
||||
results: [
|
||||
{ url: "https://example.com/a", title: "A" },
|
||||
{ url: "https://example.com/b", title: "B" },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("extracts items from data.data (doubly nested)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: {
|
||||
data: [{ url: "https://example.com/nested", title: "Nested" }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.com/nested");
|
||||
});
|
||||
|
||||
it("extracts items from data.web array (Firecrawl web search format)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: {
|
||||
web: [{ url: "https://example.com/web", title: "Web Result" }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.com/web");
|
||||
expect(result[0].title).toBe("Web Result");
|
||||
});
|
||||
|
||||
it("extracts items from web.results (top-level)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
web: {
|
||||
results: [{ url: "https://example.com/top-web", title: "Top Web" }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://example.com/top-web");
|
||||
});
|
||||
|
||||
it("returns an empty array when no search items are present", () => {
|
||||
expect(firecrawlClient.resolveSearchItems({})).toEqual([]);
|
||||
expect(firecrawlClient.resolveSearchItems({ data: "not-an-array" })).toEqual([]);
|
||||
expect(firecrawlClient.resolveSearchItems({ data: [] })).toEqual([]);
|
||||
expect(firecrawlClient.resolveSearchItems({ data: { items: [] } })).toEqual([]);
|
||||
});
|
||||
|
||||
it("skips entries without a resolvable URL", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com/ok", title: "OK" },
|
||||
{ title: "No URL" },
|
||||
{},
|
||||
null,
|
||||
"string entry",
|
||||
42,
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("OK");
|
||||
});
|
||||
|
||||
it("resolves URL from alternate fields: sourceURL, sourceUrl, metadata.sourceURL", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", title: "A" },
|
||||
{ sourceURL: "https://b.com", title: "B" },
|
||||
{ sourceUrl: "https://c.com", title: "C" },
|
||||
{ metadata: { sourceURL: "https://d.com" }, title: "D" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(4);
|
||||
expect(result.map((r) => r.url)).toEqual([
|
||||
"https://a.com",
|
||||
"https://b.com",
|
||||
"https://c.com",
|
||||
"https://d.com",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reads description from multiple possible fields", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", description: "explicit desc" },
|
||||
{ url: "https://b.com", snippet: "snippet text" },
|
||||
{ url: "https://c.com", summary: "summary text" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].description).toBe("explicit desc");
|
||||
expect(result[1].description).toBe("snippet text");
|
||||
expect(result[2].description).toBe("summary text");
|
||||
});
|
||||
|
||||
it("reads content from multiple possible fields", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", markdown: "# md" },
|
||||
{ url: "https://b.com", content: "plain content" },
|
||||
{ url: "https://c.com", text: "raw text" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].content).toBe("# md");
|
||||
expect(result[1].content).toBe("plain content");
|
||||
expect(result[2].content).toBe("raw text");
|
||||
});
|
||||
|
||||
it("reads published date from multiple possible fields", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://a.com", publishedDate: "2025-01-01" },
|
||||
{ url: "https://b.com", published: "2025-02-02" },
|
||||
{ url: "https://c.com", metadata: { publishedTime: "2025-03-03" } },
|
||||
{ url: "https://d.com", metadata: { publishedDate: "2025-04-04" } },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].published).toBe("2025-01-01");
|
||||
expect(result[1].published).toBe("2025-02-02");
|
||||
expect(result[2].published).toBe("2025-03-03");
|
||||
expect(result[3].published).toBe("2025-04-04");
|
||||
});
|
||||
|
||||
it("resolves siteName by stripping www. prefix from URL hostname", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://www.example.com/page", title: "WWW" },
|
||||
{ url: "https://example.org", title: "No WWW" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result[0].siteName).toBe("example.com");
|
||||
expect(result[1].siteName).toBe("example.org");
|
||||
});
|
||||
|
||||
it("sets description and content to undefined when absent", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [{ url: "https://example.com", title: "Minimal" }],
|
||||
});
|
||||
|
||||
expect(result[0].description).toBeUndefined();
|
||||
expect(result[0].content).toBeUndefined();
|
||||
expect(result[0].published).toBeUndefined();
|
||||
});
|
||||
|
||||
it("falls back from empty url to sourceURL within the same entry", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "", sourceURL: "https://fallback.com", title: "Fallback" },
|
||||
{ sourceURL: "https://only-source.com", title: "Only Source" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].url).toBe("https://fallback.com");
|
||||
expect(result[1].url).toBe("https://only-source.com");
|
||||
});
|
||||
|
||||
it("includes entries with empty title (title defaults to empty string)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com/no-title" },
|
||||
{ url: "https://example.com/with-title", title: "Has Title" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
expect(result[0].title).toBe("");
|
||||
expect(result[1].title).toBe("Has Title");
|
||||
});
|
||||
|
||||
it("picks the first candidate array when multiple are present", () => {
|
||||
// The candidates list checks data before results. Both are arrays here,
|
||||
// so data wins and results is ignored.
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [{ url: "https://from-data.com", title: "From Data" }],
|
||||
results: [{ url: "https://from-results.com", title: "From Results" }],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].url).toBe("https://from-data.com");
|
||||
});
|
||||
|
||||
it("treats non-object metadata as absent (number, string)", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "https://example.com/meta-num", metadata: 42 },
|
||||
{ url: "https://example.com/meta-str", metadata: "oops" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(2);
|
||||
// Both should still be resolved; metadata fallback should not crash.
|
||||
expect(result[0].url).toBe("https://example.com/meta-num");
|
||||
expect(result[1].url).toBe("https://example.com/meta-str");
|
||||
});
|
||||
|
||||
it("sets siteName to undefined when url is not a valid URL", () => {
|
||||
// resolveSiteName uses new URL() internally and catches errors.
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{ url: "not-a-valid-url", title: "Invalid" },
|
||||
{ url: "", title: "Empty URL" }, // will be skipped
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].siteName).toBeUndefined();
|
||||
});
|
||||
|
||||
it("prefers record.title over metadata.title when both are present", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{
|
||||
url: "https://example.com",
|
||||
title: "record title",
|
||||
metadata: { title: "metadata title" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("record title");
|
||||
});
|
||||
|
||||
it("falls back to metadata.title when record.title is absent", () => {
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{
|
||||
url: "https://example.com",
|
||||
metadata: { title: "metadata title" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("metadata title");
|
||||
});
|
||||
|
||||
it("falls back to metadata.title when record.title is empty string", () => {
|
||||
// typeof "" === "string" && "" → falsy → falls through to metadata.title
|
||||
const result = firecrawlClient.resolveSearchItems({
|
||||
data: [
|
||||
{
|
||||
url: "https://example.com",
|
||||
title: "",
|
||||
metadata: { title: "metadata title" },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result).toHaveLength(1);
|
||||
expect(result[0].title).toBe("metadata title");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// parseFirecrawlScrapePayload
|
||||
// ---------------------------------------------------------------------------
|
||||
describe("parseFirecrawlScrapePayload", () => {
|
||||
const baseOpts = {
|
||||
url: "https://example.com/page",
|
||||
extractMode: "markdown" as const,
|
||||
maxChars: 50_000,
|
||||
};
|
||||
|
||||
it("parses a standard markdown scrape response", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "# Hello\n\nThis is page content.",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.url).toBe("https://example.com/page");
|
||||
expect(result.extractor).toBe("firecrawl");
|
||||
expect(result.extractMode).toBe("markdown");
|
||||
expect(result.text).toContain("# Hello");
|
||||
expect(result.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("falls back to content field when markdown is absent", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
content: "Fallback content body",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.text).toContain("Fallback content body");
|
||||
});
|
||||
|
||||
it("throws when no content is returned", () => {
|
||||
expect(() =>
|
||||
firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: { data: {} },
|
||||
}),
|
||||
).toThrow(/no content/i);
|
||||
|
||||
expect(() =>
|
||||
firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {},
|
||||
}),
|
||||
).toThrow(/no content/i);
|
||||
});
|
||||
|
||||
it("converts markdown to plain text in text mode", () => {
|
||||
const markdownResult = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 50_000,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "# Heading\n\n**bold** and `code`",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const textResult = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "text",
|
||||
maxChars: 50_000,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "# Heading\n\n**bold** and `code`",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(markdownResult.extractMode).toBe("markdown");
|
||||
expect(textResult.extractMode).toBe("text");
|
||||
// text mode strips markdown syntax: heading markers should be removed
|
||||
expect(textResult.text).not.toContain("# Heading");
|
||||
// The raw lengths differ because text mode strips markdown characters
|
||||
expect(textResult.rawLength as number).toBeLessThan(markdownResult.rawLength as number);
|
||||
});
|
||||
|
||||
it("includes metadata: finalUrl, title, and statusCode", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content with metadata",
|
||||
url: "https://redirected.example.com",
|
||||
statusCode: 200,
|
||||
metadata: {
|
||||
sourceURL: "https://final.example.com/page",
|
||||
title: "Page Title",
|
||||
statusCode: 200,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://final.example.com/page");
|
||||
expect(result.title).toContain("Page Title");
|
||||
expect(result.status).toBe(200);
|
||||
});
|
||||
|
||||
it("falls back to data.url for finalUrl when metadata.sourceURL is absent", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content",
|
||||
url: "https://direct.example.com",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://direct.example.com");
|
||||
});
|
||||
|
||||
it("uses the requested url as finalUrl when no redirect is present", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "no redirect info" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.finalUrl).toBe("https://example.com/page");
|
||||
});
|
||||
|
||||
it("sets title to undefined when metadata title is absent", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "no title" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.title).toBeUndefined();
|
||||
});
|
||||
|
||||
it("sets status to undefined when no statusCode is available", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "no status" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("truncates content when it exceeds maxChars", () => {
|
||||
const longContent = "a".repeat(200);
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 50,
|
||||
payload: {
|
||||
data: { markdown: longContent },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.rawLength as number).toBe(200);
|
||||
});
|
||||
|
||||
it("does not truncate content within maxChars limit", () => {
|
||||
const shortContent = "short content here";
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: shortContent },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.rawLength as number).toBe(shortContent.length);
|
||||
});
|
||||
|
||||
it("handles truncation at exact boundary (not truncated)", () => {
|
||||
const content = "x".repeat(100);
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 100,
|
||||
payload: {
|
||||
data: { markdown: content },
|
||||
},
|
||||
});
|
||||
|
||||
// When raw length equals maxChars, truncateText returns the full text.
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.rawLength as number).toBe(100);
|
||||
});
|
||||
|
||||
it("truncates content one character over maxChars", () => {
|
||||
const content = "x".repeat(101);
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 100,
|
||||
payload: {
|
||||
data: { markdown: content },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.rawLength as number).toBe(101);
|
||||
});
|
||||
|
||||
it("handles maxChars of 0 (truncates everything)", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
url: "https://example.com",
|
||||
extractMode: "markdown",
|
||||
maxChars: 0,
|
||||
payload: {
|
||||
data: { markdown: "some content" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.rawLength as number).toBe("some content".length);
|
||||
});
|
||||
|
||||
it("preserves warning string from the response payload", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content with warning" },
|
||||
warning: "Proxy fallback was used for this request",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toContain("Proxy fallback was used");
|
||||
});
|
||||
|
||||
it("omits warning when response has no warning field", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content without warning" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toBeUndefined();
|
||||
});
|
||||
|
||||
it("handles non-string warning gracefully", () => {
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content" },
|
||||
warning: 42,
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toBeUndefined();
|
||||
});
|
||||
|
||||
it("ignores non-numeric statusCode values", () => {
|
||||
// Firecrawl may return statusCode as a string in some response shapes.
|
||||
// The check is `typeof ... === "number"`, so strings are treated as absent.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content",
|
||||
statusCode: "200",
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBeUndefined();
|
||||
});
|
||||
|
||||
it("treats whitespace-only markdown as valid content", () => {
|
||||
// typeof " " === "string" && " " → truthy → treated as content.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: " " },
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.rawLength as number).toBe(3);
|
||||
expect(result.truncated).toBe(false);
|
||||
});
|
||||
|
||||
it("silently drops empty-string warning", () => {
|
||||
// typeof "" === "string" && "" → "" is falsy → warning = undefined.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: { markdown: "content" },
|
||||
warning: "",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.warning).toBeUndefined();
|
||||
});
|
||||
|
||||
it("drops empty-string metadata.title (treated as absent)", () => {
|
||||
// typeof "" === "string" && "" → falsy → title = undefined.
|
||||
const result = firecrawlClient.parseFirecrawlScrapePayload({
|
||||
...baseOpts,
|
||||
payload: {
|
||||
data: {
|
||||
markdown: "content",
|
||||
metadata: { title: "" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.title).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
|
@ -479,7 +479,7 @@ describe("scripts/test-extension.mjs", () => {
|
|||
expect(batch.extensionCount).toBe(2);
|
||||
expect(batch.noTestExtensionIds).toEqual([extensionId]);
|
||||
expect(batch.hasTests).toBe(true);
|
||||
expect(batch.testFileCount).toBe(1);
|
||||
expect(batch.testFileCount).toBe(2);
|
||||
expect(batch.planGroups.flatMap((group) => group.extensionIds)).toEqual(["firecrawl"]);
|
||||
});
|
||||
|
||||
|
|
@ -835,7 +835,12 @@ describe("scripts/test-extension.mjs", () => {
|
|||
resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds: ["firecrawl"] }),
|
||||
{
|
||||
runGroup,
|
||||
vitestArgs: ["--exclude", bundledPluginFile("firecrawl", "src/firecrawl-tools.test.ts")],
|
||||
vitestArgs: [
|
||||
"--exclude",
|
||||
bundledPluginFile("firecrawl", "src/firecrawl-tools.test.ts"),
|
||||
"--exclude",
|
||||
bundledPluginFile("firecrawl", "src/firecrawl-client.test.ts"),
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -849,7 +854,12 @@ describe("scripts/test-extension.mjs", () => {
|
|||
resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds: ["firecrawl"] }),
|
||||
{
|
||||
runGroup,
|
||||
vitestArgs: ["--exclude", "firecrawl/src/firecrawl-tools.test.ts"],
|
||||
vitestArgs: [
|
||||
"--exclude",
|
||||
"firecrawl/src/firecrawl-tools.test.ts",
|
||||
"--exclude",
|
||||
"firecrawl/src/firecrawl-client.test.ts",
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -864,7 +874,12 @@ describe("scripts/test-extension.mjs", () => {
|
|||
{
|
||||
allowEmptyAfterExclude: true,
|
||||
runGroup,
|
||||
vitestArgs: ["--exclude", bundledPluginFile("firecrawl", "src/firecrawl-tools.test.ts")],
|
||||
vitestArgs: [
|
||||
"--exclude",
|
||||
bundledPluginFile("firecrawl", "src/firecrawl-tools.test.ts"),
|
||||
"--exclude",
|
||||
bundledPluginFile("firecrawl", "src/firecrawl-client.test.ts"),
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue