diff --git a/docs/developers/tools/web-fetch.md b/docs/developers/tools/web-fetch.md index 21fdc54d4..88f0b6daa 100644 --- a/docs/developers/tools/web-fetch.md +++ b/docs/developers/tools/web-fetch.md @@ -4,20 +4,30 @@ This document describes the `web_fetch` tool for Qwen Code. ## Description -Use `web_fetch` to fetch content from a specified URL and process it using an AI model. The tool takes a URL and a prompt as input, fetches the URL content, converts HTML to markdown, and processes the content with the prompt using a small, fast model. +Use `web_fetch` to fetch content from a specified URL and process it using an AI model. The tool takes a URL and a prompt as input, fetches the URL content, and processes the content with the prompt using a small, fast model. ### Arguments -`web_fetch` takes two arguments: +`web_fetch` takes three arguments: - `url` (string, required): The URL to fetch content from. Must be a fully-formed valid URL starting with `http://` or `https://`. - `prompt` (string, required): The prompt describing what information you want to extract from the page content. +- `format` (string, optional): Controls only the `Accept` header sent to the server, indicating your content preference. **All fetched content is normalized to plain text for LLM processing**, regardless of the format specified. Defaults to `"auto"` if not specified. + - `"auto"` (default): Prefers markdown via content negotiation (`Accept: text/markdown, text/html`), accepts HTML as fallback. **Recommended for most use cases** as it can reduce token usage by up to 80% for servers that support markdown. + - `"markdown"`: Sends `Accept: text/markdown`. Use when you explicitly need markdown content. + - `"html"`: Sends `Accept: text/html`. Use when the server requires HTML in the Accept header. Content is still converted to plain text for LLM processing. + - `"text"`: Sends `Accept: text/plain`. Use when you specifically need plain text content. ## How to use `web_fetch` with Qwen Code To use `web_fetch` with Qwen Code, provide a URL and a prompt describing what you want to extract from that URL. The tool will ask for confirmation before fetching the URL. Once confirmed, the tool will fetch the content directly and process it using an AI model. -The tool automatically converts HTML to text, handles GitHub blob URLs (converting them to raw URLs), and upgrades HTTP URLs to HTTPS for security. +The tool automatically: + +- Converts HTML to text when necessary +- Handles GitHub blob URLs (converting them to raw URLs) +- Upgrades HTTP URLs to HTTPS for security +- Supports content negotiation for markdown (reduces token usage significantly) Usage: @@ -25,6 +35,12 @@ Usage: web_fetch(url="https://example.com", prompt="Summarize the main points of this article") ``` +With format specification: + +``` +web_fetch(url="https://example.com", prompt="Get the raw content", format="markdown") +``` + ## `web_fetch` examples Summarize a single article: @@ -45,10 +61,45 @@ Analyze GitHub documentation: web_fetch(url="https://github.com/QwenLM/Qwen/blob/main/README.md", prompt="What are the installation steps and main features?") ``` +Get markdown content (for servers supporting Markdown for Agents): + +``` +web_fetch(url="https://developers.cloudflare.com/fundamentals/reference/markdown-for-agents/", prompt="Extract the key information", format="markdown") +``` + ## Important notes - **Single URL processing:** `web_fetch` processes one URL at a time. To analyze multiple URLs, make separate calls to the tool. - **URL format:** The tool automatically upgrades HTTP URLs to HTTPS and converts GitHub blob URLs to raw format for better content access. -- **Content processing:** The tool fetches content directly and processes it using an AI model, converting HTML to readable text format. +- **Content negotiation:** The tool supports "Markdown for Agents" content negotiation. When using `format="auto"` (default), it sends `Accept: text/markdown, text/html` headers, allowing servers that support markdown to return it directly instead of HTML. This can reduce token usage by up to 80%. +- **Content processing:** The tool fetches content directly and processes it using an AI model. When the server returns HTML, it converts it to readable text format. When the server returns markdown or plain text, it uses the content as-is. - **Output quality:** The quality of the output will depend on the clarity of the instructions in the prompt. - **MCP tools:** If an MCP-provided web fetch tool is available (starting with "mcp\_\_"), prefer using that tool as it may have fewer restrictions. + +## Markdown for Agents Support + +Qwen Code's `web_fetch` tool implements support for [Cloudflare's Markdown for Agents](https://blog.cloudflare.com/markdown-for-agents/) specification. This feature allows websites to serve markdown content directly to AI agents, significantly reducing token usage compared to parsing HTML. + +### How it works + +1. The `format` parameter controls **only** the `Accept` header sent to the server (it does not affect the output format): + - `format="auto"`: sends `Accept: text/markdown, text/html` + - `format="markdown"`: sends `Accept: text/markdown` + - `format="html"`: sends `Accept: text/html` + - `format="text"`: sends `Accept: text/plain` +2. If the server supports markdown, it returns `Content-Type: text/markdown` +3. The tool uses markdown or plain text content directly without conversion +4. If the server returns HTML, it converts to readable text format for LLM processing +5. All content is normalized to text before being processed by the AI model + +### Benefits + +- **Token efficiency:** Markdown content typically uses 80% fewer tokens than equivalent HTML +- **Better structure:** Markdown preserves semantic structure (headings, lists, etc.) +- **Backward compatible:** Works with all websites, enhanced experience for supporting servers + +### Example servers supporting markdown + +- Cloudflare Developer Documentation +- Cloudflare Blog +- Any website using Cloudflare's "Markdown for Agents" feature diff --git a/packages/core/src/tools/web-fetch.test.ts b/packages/core/src/tools/web-fetch.test.ts index 93ef2826e..67901be79 100644 --- a/packages/core/src/tools/web-fetch.test.ts +++ b/packages/core/src/tools/web-fetch.test.ts @@ -37,6 +37,7 @@ describe('WebFetchTool', () => { getProxy: vi.fn(), getGeminiClient: mockGetGeminiClient, getSessionId: vi.fn(() => 'test-session-id'), + getModel: vi.fn(() => 'qwen-coder'), } as unknown as Config; }); @@ -66,6 +67,7 @@ describe('WebFetchTool', () => { vi.spyOn(fetchUtils, 'isPrivateIp').mockReturnValue(false); vi.spyOn(fetchUtils, 'fetchWithTimeout').mockResolvedValue({ ok: true, + headers: new Headers({ 'content-type': 'text/html' }), text: () => Promise.resolve('
Test content'), } as Response); mockGenerateContent.mockRejectedValue(new Error('API error')); @@ -77,6 +79,169 @@ describe('WebFetchTool', () => { }); }); + describe('format parameter', () => { + it('should default to auto format when not specified', async () => { + const fetchSpy = vi + .spyOn(fetchUtils, 'fetchWithTimeout') + .mockResolvedValue({ + ok: true, + headers: new Headers({ 'content-type': 'text/html' }), + text: () => Promise.resolve('Test content'), + } as Response); + + mockGenerateContent.mockResolvedValue({ + response: { text: () => 'Summary' }, + }); + + const tool = new WebFetchTool(mockConfig); + const params = { url: 'https://example.com', prompt: 'summarize' }; + const invocation = tool.build(params); + await invocation.execute(new AbortController().signal); + + expect(fetchSpy).toHaveBeenCalledWith( + 'https://example.com', + expect.any(Number), + { Accept: 'text/markdown, text/html, text/plain' }, + ); + }); + + it('should request only markdown when format is markdown', async () => { + const fetchSpy = vi + .spyOn(fetchUtils, 'fetchWithTimeout') + .mockResolvedValue({ + ok: true, + headers: new Headers({ 'content-type': 'text/markdown' }), + text: () => Promise.resolve('# Test Content'), + } as Response); + + mockGenerateContent.mockResolvedValue({ + response: { text: () => 'Summary' }, + }); + + const tool = new WebFetchTool(mockConfig); + const params = { + url: 'https://example.com', + prompt: 'summarize', + format: 'markdown' as const, + }; + const invocation = tool.build(params); + await invocation.execute(new AbortController().signal); + + expect(fetchSpy).toHaveBeenCalledWith( + 'https://example.com', + expect.any(Number), + { Accept: 'text/markdown' }, + ); + }); + + it('should request only HTML when format is html', async () => { + const fetchSpy = vi + .spyOn(fetchUtils, 'fetchWithTimeout') + .mockResolvedValue({ + ok: true, + headers: new Headers({ 'content-type': 'text/html' }), + text: () => Promise.resolve('Test content'), + } as Response); + + mockGenerateContent.mockResolvedValue({ + response: { text: () => 'Summary' }, + }); + + const tool = new WebFetchTool(mockConfig); + const params = { + url: 'https://example.com', + prompt: 'summarize', + format: 'html' as const, + }; + const invocation = tool.build(params); + await invocation.execute(new AbortController().signal); + + expect(fetchSpy).toHaveBeenCalledWith( + 'https://example.com', + expect.any(Number), + { Accept: 'text/html' }, + ); + }); + + it('should request plain text when format is text', async () => { + const fetchSpy = vi + .spyOn(fetchUtils, 'fetchWithTimeout') + .mockResolvedValue({ + ok: true, + headers: new Headers({ 'content-type': 'text/plain' }), + text: () => Promise.resolve('Plain text content'), + } as Response); + + mockGenerateContent.mockResolvedValue({ + response: { text: () => 'Summary' }, + }); + + const tool = new WebFetchTool(mockConfig); + const params = { + url: 'https://example.com', + prompt: 'summarize', + format: 'text' as const, + }; + const invocation = tool.build(params); + await invocation.execute(new AbortController().signal); + + expect(fetchSpy).toHaveBeenCalledWith( + 'https://example.com', + expect.any(Number), + { Accept: 'text/plain' }, + ); + }); + + it('should include markdown content in prompt when server returns markdown', async () => { + let receivedContent = ''; + vi.spyOn(fetchUtils, 'fetchWithTimeout').mockResolvedValue({ + ok: true, + headers: new Headers({ + 'content-type': 'text/markdown; charset=utf-8', + }), + text: () => + Promise.resolve('# Hello World\n\nThis is markdown content.'), + } as Response); + + mockGenerateContent.mockImplementation((messages) => { + receivedContent = messages[0].parts[0].text; + return Promise.resolve({ response: { text: () => 'Processed' } }); + }); + + const tool = new WebFetchTool(mockConfig); + const params = { url: 'https://example.com', prompt: 'summarize' }; + const invocation = tool.build(params); + await invocation.execute(new AbortController().signal); + + expect(receivedContent).toContain('# Hello World'); + }); + + it('should include plain text content in prompt when server returns plain text', async () => { + let receivedContent = ''; + vi.spyOn(fetchUtils, 'fetchWithTimeout').mockResolvedValue({ + ok: true, + headers: new Headers({ 'content-type': 'text/plain' }), + text: () => Promise.resolve('Plain text content here'), + } as Response); + + mockGenerateContent.mockImplementation((messages) => { + receivedContent = messages[0].parts[0].text; + return Promise.resolve({ response: { text: () => 'Processed' } }); + }); + + const tool = new WebFetchTool(mockConfig); + const params = { + url: 'https://example.com', + prompt: 'summarize', + format: 'text' as const, + }; + const invocation = tool.build(params); + await invocation.execute(new AbortController().signal); + + expect(receivedContent).toContain('Plain text content here'); + }); + }); + describe('getConfirmationDetails', () => { it('should return confirmation details with the correct prompt and urls', async () => { const tool = new WebFetchTool(mockConfig); diff --git a/packages/core/src/tools/web-fetch.ts b/packages/core/src/tools/web-fetch.ts index f77168a5c..30bdef4c0 100644 --- a/packages/core/src/tools/web-fetch.ts +++ b/packages/core/src/tools/web-fetch.ts @@ -37,6 +37,15 @@ export interface WebFetchToolParams { * The prompt to run on the fetched content */ prompt: string; + /** + * Preferred content format (controls only the Accept header) + * All content is normalized to plain text for LLM processing + * - auto: Prefers markdown via content negotiation (default) + * - markdown: Request markdown format only + * - html: Request HTML format only (still converted to text) + * - text: Request plain text format + */ + format?: 'auto' | 'markdown' | 'html' | 'text'; } /** @@ -56,6 +65,21 @@ class WebFetchToolInvocation extends BaseToolInvocation< this.debugLogger = createDebugLogger('WEB_FETCH'); } + private getAcceptHeader(): string { + const format = this.params.format ?? 'auto'; + switch (format) { + case 'markdown': + return 'text/markdown'; + case 'html': + return 'text/html'; + case 'text': + return 'text/plain'; + case 'auto': + default: + return 'text/markdown, text/html, text/plain'; + } + } + private async executeDirectFetch(signal: AbortSignal): Promise