diff --git a/packages/cli/src/acp-integration/service/filesystem.test.ts b/packages/cli/src/acp-integration/service/filesystem.test.ts index d53c512a45..ba31dc6de5 100644 --- a/packages/cli/src/acp-integration/service/filesystem.test.ts +++ b/packages/cli/src/acp-integration/service/filesystem.test.ts @@ -107,6 +107,39 @@ describe('AcpFileSystemService', () => { }); }); + it('converts core-only read params at the ACP boundary', async () => { + const mockResponse = { + content: 'slice', + _meta: { bom: false, encoding: 'utf-8' }, + }; + const client = { + readTextFile: vi.fn().mockResolvedValue(mockResponse), + } as unknown as AgentSideConnection; + const signal = new AbortController().signal; + + const svc = new AcpFileSystemService( + client, + 'session-1', + { readTextFile: true, writeTextFile: true }, + createFallback(), + ); + + await svc.readTextFile({ + path: '/some/file.txt', + line: 0, + limit: 5, + maxOutputBytes: 1024, + signal, + }); + + expect(client.readTextFile).toHaveBeenCalledWith({ + path: '/some/file.txt', + line: 1, + limit: 5, + sessionId: 'session-1', + }); + }); + it('converts RESOURCE_NOT_FOUND error to ENOENT', async () => { const resourceNotFoundError = { code: RESOURCE_NOT_FOUND_CODE, @@ -953,11 +986,20 @@ describe('AcpFileSystemService', () => { fallback, ); - const result = await svc.readTextFile({ path: '/some/file.txt' }); + const signal = new AbortController().signal; + const result = await svc.readTextFile({ + path: '/some/file.txt', + line: 0, + maxOutputBytes: 2048, + signal, + }); expect(result).toEqual(fallbackResponse); expect(fallback.readTextFile).toHaveBeenCalledWith({ path: '/some/file.txt', + line: 0, + maxOutputBytes: 2048, + signal, }); expect(client.readTextFile).not.toHaveBeenCalled(); }); diff --git a/packages/cli/src/acp-integration/service/filesystem.ts b/packages/cli/src/acp-integration/service/filesystem.ts index 95fb1d14b6..bcc010f716 100644 --- a/packages/cli/src/acp-integration/service/filesystem.ts +++ b/packages/cli/src/acp-integration/service/filesystem.ts @@ -13,6 +13,7 @@ import type { } from '@agentclientprotocol/sdk'; import { RequestError } from '@agentclientprotocol/sdk'; import type { + CoreReadTextFileRequest, FileSystemService, ReadTextFileResponse, } from '@qwen-code/qwen-code-core'; @@ -105,6 +106,29 @@ async function resolveRealPath(value: string): Promise { } } +function toAcpReadTextFileRequest( + params: CoreReadTextFileRequest, + sessionId: string, +): ReadTextFileRequest { + // `maxOutputBytes`, `signal`, and `stats` are core-local concerns that the + // current ACP schema cannot represent. Keep this boundary explicit if the + // schema grows. + const request: ReadTextFileRequest = { + path: params.path, + sessionId, + }; + if (params._meta !== undefined) { + request._meta = params._meta; + } + if (params.limit !== undefined) { + request.limit = params.limit; + } + if (params.line != null) { + request.line = params.line + 1; + } + return request; +} + export class AcpFileSystemService implements FileSystemService { constructor( private readonly connection: AgentSideConnection, @@ -115,7 +139,7 @@ export class AcpFileSystemService implements FileSystemService { ) {} async readTextFile( - params: Omit, + params: CoreReadTextFileRequest, ): Promise { if (!this.capabilities.readTextFile) { return this.fallback.readTextFile(params); @@ -123,10 +147,9 @@ export class AcpFileSystemService implements FileSystemService { let response: ReadTextFileResponse; try { - response = await this.connection.readTextFile({ - ...params, - sessionId: this.sessionId, - }); + response = await this.connection.readTextFile( + toAcpReadTextFileRequest(params, this.sessionId), + ); } catch (error) { const errorCode = getErrorCode(error); diff --git a/packages/cli/src/serve/fs/policy.ts b/packages/cli/src/serve/fs/policy.ts index 1cd7cfe5c3..0061b6cd98 100644 --- a/packages/cli/src/serve/fs/policy.ts +++ b/packages/cli/src/serve/fs/policy.ts @@ -17,12 +17,12 @@ import type { Intent, ResolvedPath } from './paths.js'; * typical source files, small enough that an SSE replay buffer * doesn't fill on a single read. * - * Files **above** this cap are refused with `file_too_large` rather - * than truncated — the underlying `readFileWithLineAndLimit` - * reads the whole file into memory before slicing lines, so soft - * truncation past the cap would still OOM the daemon. Files - * **at or below** the cap honor a tighter `opts.maxBytes` via - * post-decode truncation (`enforceReadSize`); that's where the + * Full-snapshot reads above this cap are refused with `file_too_large` + * rather than truncated. `readText` can serve explicit line windows + * from larger files, but the default read/edit contract still needs a + * bounded snapshot for hash stability, SSE buffering, and oldText + * matching. Files at or below the cap honor a tighter `opts.maxBytes` + * via post-decode truncation (`enforceReadSize`); that's where the * `meta.truncated = true` flag fires. * * `enforceReadBytesSize` (the `readBytes` gate) and `edit()` use the diff --git a/packages/cli/src/serve/fs/workspace-file-system.test.ts b/packages/cli/src/serve/fs/workspace-file-system.test.ts index 51222c23ed..a6f9fd73d0 100644 --- a/packages/cli/src/serve/fs/workspace-file-system.test.ts +++ b/packages/cli/src/serve/fs/workspace-file-system.test.ts @@ -735,6 +735,20 @@ describe('WorkspaceFileSystem - write/edit', () => { expect(after).toBe('foo=42\nbar=2\n'); }); + it('edit() preserves the tail of files larger than the default range cap', async () => { + const target = path.join(h.workspace, 'large-edit.txt'); + const tail = 'tail-marker\n'; + const content = `foo=1\n${'body\n'.repeat(6_000)}${tail}`; + await fsp.writeFile(target, content); + const r = await h.fs.resolve('large-edit.txt', 'edit'); + + const out = await h.fs.edit(r, 'foo=1', 'foo=42'); + + expect(out.writtenBytes).toBeGreaterThan(25_000); + const after = await fsp.readFile(target, 'utf-8'); + expect(after).toBe(`foo=42\n${'body\n'.repeat(6_000)}${tail}`); + }); + it('throws parse_error when oldText is not present', async () => { const target = path.join(h.workspace, 'c.txt'); await fsp.writeFile(target, 'abc'); diff --git a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts index 4ead67f5a8..52437f6400 100644 --- a/packages/cli/src/ui/hooks/atCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/atCommandProcessor.test.ts @@ -156,6 +156,40 @@ describe('handleAtCommand', () => { expect(result.toolDisplays![0].status).toBe(ToolCallStatus.Success); }); + it('should attach a truncated text file larger than 10MB', async () => { + const filePath = await createTestFile( + path.join(testRootDir, 'large.log'), + 'x'.repeat(11 * 1024 * 1024), + ); + + const result = await handleAtCommand({ + query: `@${filePath}`, + config: mockConfig, + onDebugMessage: mockOnDebugMessage, + messageId: 626, + signal: abortController.signal, + }); + + const processedText = Array.isArray(result.processedQuery) + ? result.processedQuery + .map((part) => + typeof part === 'string' + ? part + : 'text' in part + ? part.text + : JSON.stringify(part), + ) + .join('') + : ''; + + expect(processedText).toContain( + 'Showing lines 1-1 of at least 1 total lines', + ); + expect(processedText).toContain('... [truncated]'); + expect(result.shouldProceed).toBe(true); + expect(result.toolDisplays![0].status).toBe(ToolCallStatus.Success); + }); + it('should only allow actual temp directory paths outside the workspace', async () => { const tempParentDir = await fsPromises.mkdtemp( path.join(os.tmpdir(), 'at-command-temp-'), diff --git a/packages/core/src/services/fileSystemService.test.ts b/packages/core/src/services/fileSystemService.test.ts index c89b40dff0..495f669662 100644 --- a/packages/core/src/services/fileSystemService.test.ts +++ b/packages/core/src/services/fileSystemService.test.ts @@ -87,7 +87,6 @@ describe('StandardFileSystemService', () => { expect(readFileWithLineAndLimit).toHaveBeenCalledWith({ path: '/test/file.txt', limit: Infinity, - line: 0, }); expect(result.content).toBe('Hello, World!'); expect(result._meta?.bom).toBe(false); @@ -116,6 +115,74 @@ describe('StandardFileSystemService', () => { expect(result._meta?.originalLineCount).toBe(100); }); + it('should preserve explicit line zero for offset reads', async () => { + vi.mocked(readFileWithLineAndLimit).mockResolvedValue({ + content: 'line 1', + bom: false, + encoding: 'utf-8', + originalLineCount: 100, + }); + + await fileSystem.readTextFile({ + path: '/test/file.txt', + line: 0, + }); + + expect(readFileWithLineAndLimit).toHaveBeenCalledWith({ + path: '/test/file.txt', + limit: Infinity, + line: 0, + }); + }); + + it('should pass maxOutputBytes and return byte-truncation metadata', async () => { + vi.mocked(readFileWithLineAndLimit).mockResolvedValue({ + content: 'partial', + bom: false, + encoding: 'utf-8', + originalLineCount: 100, + truncatedByBytes: true, + }); + + const result = await fileSystem.readTextFile({ + path: '/test/file.txt', + limit: 10, + line: 5, + maxOutputBytes: 128, + }); + + expect(readFileWithLineAndLimit).toHaveBeenCalledWith({ + path: '/test/file.txt', + limit: 10, + line: 5, + maxOutputBytes: 128, + }); + expect(result._meta?.truncatedByBytes).toBe(true); + }); + + it('should pass cached stats to readFileWithLineAndLimit', async () => { + const stats = { size: 123 } as import('node:fs').Stats; + vi.mocked(readFileWithLineAndLimit).mockResolvedValue({ + content: 'line 1', + bom: false, + encoding: 'utf-8', + originalLineCount: 1, + }); + + await fileSystem.readTextFile({ + path: '/test/file.txt', + maxOutputBytes: 128, + stats, + }); + + expect(readFileWithLineAndLimit).toHaveBeenCalledWith({ + path: '/test/file.txt', + limit: Infinity, + maxOutputBytes: 128, + stats, + }); + }); + it('should return encoding info for GBK file', async () => { vi.mocked(readFileWithLineAndLimit).mockResolvedValue({ content: '你好世界', diff --git a/packages/core/src/services/fileSystemService.ts b/packages/core/src/services/fileSystemService.ts index 4ecfd0ed7d..62189c7d36 100644 --- a/packages/core/src/services/fileSystemService.ts +++ b/packages/core/src/services/fileSystemService.ts @@ -5,6 +5,7 @@ */ import os from 'node:os'; +import type { Stats } from 'node:fs'; import * as path from 'node:path'; import { globSync } from 'glob'; import { atomicWriteFile } from '../utils/atomicFileWrite.js'; @@ -29,10 +30,26 @@ export type ReadTextFileResponse = { bom?: boolean; encoding?: string; originalLineCount?: number; + originalLineCountExact?: boolean; lineEnding?: LineEnding; + truncatedByBytes?: boolean; }; }; +export type CoreReadTextFileRequest = Omit< + ReadTextFileRequest, + 'sessionId' | 'line' +> & { + /** + * Core-local callers use 0-based line offsets. ACP protocol boundaries remain + * 1-based and convert explicitly before remote calls. + */ + line?: number | null; + maxOutputBytes?: number; + signal?: AbortSignal; + stats?: Stats; +}; + /** * Supported file encodings for new files. */ @@ -50,9 +67,7 @@ export type FileEncodingType = (typeof FileEncoding)[keyof typeof FileEncoding]; * Interface for file system operations that may be delegated to different implementations */ export interface FileSystemService { - readTextFile( - params: Omit, - ): Promise; + readTextFile(params: CoreReadTextFileRequest): Promise; writeTextFile( params: Omit, @@ -261,18 +276,32 @@ export function encodeTextFileContent( */ export class StandardFileSystemService implements FileSystemService { async readTextFile( - params: Omit, + params: CoreReadTextFileRequest, ): Promise { - const { path, limit, line } = params; - // Use encoding-aware reader that handles BOM and non-UTF-8 encodings (e.g. GBK) - const { content, bom, encoding, originalLineCount } = - await readFileWithLineAndLimit({ - path, - limit: limit ?? Number.POSITIVE_INFINITY, - line: line || 0, - }); - const lineEnding = detectLineEnding(content); - return { content, _meta: { bom, encoding, originalLineCount, lineEnding } }; + const { path, limit, line, maxOutputBytes, signal, stats } = params; + const readResult = await readFileWithLineAndLimit({ + path, + limit: limit ?? Number.POSITIVE_INFINITY, + ...(line !== undefined && line !== null ? { line } : {}), + ...(maxOutputBytes !== undefined ? { maxOutputBytes } : {}), + ...(signal !== undefined ? { signal } : {}), + ...(stats !== undefined ? { stats } : {}), + }); + const detectedLineEnding = + readResult.lineEnding ?? detectLineEnding(readResult.content); + return { + content: readResult.content, + _meta: { + bom: readResult.bom, + encoding: readResult.encoding, + originalLineCount: readResult.originalLineCount, + originalLineCountExact: readResult.originalLineCountExact, + lineEnding: detectedLineEnding, + ...(readResult.truncatedByBytes !== undefined + ? { truncatedByBytes: readResult.truncatedByBytes } + : {}), + }, + }; } async writeTextFile( diff --git a/packages/core/src/tools/artifact/artifact-tool.test.ts b/packages/core/src/tools/artifact/artifact-tool.test.ts index 84b5ec8fad..9f68d4f151 100644 --- a/packages/core/src/tools/artifact/artifact-tool.test.ts +++ b/packages/core/src/tools/artifact/artifact-tool.test.ts @@ -134,6 +134,31 @@ describe('ArtifactTool', () => { expect(res.error?.type).toBe(ToolErrorType.FILE_NOT_FOUND); }); + it('forwards cancellation signals to source file reads', async () => { + const file = path.join(workdir, 'page.html'); + const controller = new AbortController(); + const readTextFile = vi.fn(async () => ({ content: '

x

' })); + const signalAwareTool = new ArtifactTool( + { + ...makeConfig(), + getFileSystemService: () => ({ readTextFile }), + } as unknown as Config, + { + kind: 'oss', + publish: async () => ({ id: 'x', url: 'https://h/x' }), + }, + openSpy as unknown as UrlOpener, + ); + + await signalAwareTool.build({ file_path: file }).execute(controller.signal); + + expect(readTextFile).toHaveBeenCalledWith({ + path: file, + maxOutputBytes: MAX_ARTIFACT_BYTES, + signal: controller.signal, + }); + }); + it('returns EXECUTION_FAILED when the publisher throws', async () => { const file = await writeFragment('page.html', '

x

'); const failingTool = new ArtifactTool( @@ -154,11 +179,22 @@ describe('ArtifactTool', () => { expect(openSpy).not.toHaveBeenCalled(); }); - it('enforces the size cap', async () => { + it('rejects source fragments that exceed the artifact byte budget', async () => { const big = '

' + 'a'.repeat(MAX_ARTIFACT_BYTES) + '

'; const file = await writeFragment('big.html', big); const res = await tool.build({ file_path: file }).execute(signal); expect(res.error?.type).toBe(ToolErrorType.FILE_TOO_LARGE); + expect(res.error?.message).toContain('source exceeds'); + expect(res.error?.message).toContain(`${MAX_ARTIFACT_BYTES} byte limit`); + expect(openSpy).not.toHaveBeenCalled(); + }); + + it('enforces the published artifact size cap', async () => { + const almostTooBig = 'a'.repeat(MAX_ARTIFACT_BYTES - 1); + const file = await writeFragment('wrapped-big.html', almostTooBig); + const res = await tool.build({ file_path: file }).execute(signal); + expect(res.error?.type).toBe(ToolErrorType.FILE_TOO_LARGE); + expect(res.error?.message).toContain('Artifact is too large'); expect(openSpy).not.toHaveBeenCalled(); }); diff --git a/packages/core/src/tools/artifact/artifact-tool.ts b/packages/core/src/tools/artifact/artifact-tool.ts index 5dce543c61..691a88295a 100644 --- a/packages/core/src/tools/artifact/artifact-tool.ts +++ b/packages/core/src/tools/artifact/artifact-tool.ts @@ -62,6 +62,14 @@ Set artifact.autoOpen=false in settings.json, or QWEN_ARTIFACT_NO_AUTO_OPEN=1, t const debugLogger = createDebugLogger('artifact'); +function cancelledArtifactResult(): ToolResult { + const message = 'Artifact publishing was cancelled.'; + return { + llmContent: message, + returnDisplay: message, + }; +} + class ArtifactToolInvocation extends BaseToolInvocation< ArtifactToolParams, ToolResult @@ -130,11 +138,26 @@ class ArtifactToolInvocation extends BaseToolInvocation< // Read the fragment the model wrote. let fragment: string; try { - const { content } = await this.config + const { content, _meta } = await this.config .getFileSystemService() - .readTextFile({ path: file_path }); + .readTextFile({ + path: file_path, + maxOutputBytes: MAX_ARTIFACT_BYTES, + signal, + }); + if (_meta?.truncatedByBytes === true) { + const message = `Artifact is too large (source exceeds the ${MAX_ARTIFACT_BYTES} byte limit). Trim the content or split it across multiple artifacts.`; + return { + llmContent: message, + returnDisplay: message, + error: { message, type: ToolErrorType.FILE_TOO_LARGE }, + }; + } fragment = content; } catch (err) { + if (signal.aborted || isAbortError(err)) { + return cancelledArtifactResult(); + } const notFound = isNodeError(err) && err.code === 'ENOENT'; const message = notFound ? `Artifact source file not found: ${file_path}. Write the page content to this file first.` @@ -193,11 +216,7 @@ class ArtifactToolInvocation extends BaseToolInvocation< // A user-initiated cancel (Esc / aborted signal) is not a failure — // surface it as a cancellation rather than a publish error. if (signal.aborted || isAbortError(err)) { - const message = 'Artifact publishing was cancelled.'; - return { - llmContent: message, - returnDisplay: message, - }; + return cancelledArtifactResult(); } const message = `Failed to publish artifact: ${getErrorMessage(err)}`; return { diff --git a/packages/core/src/tools/edit.test.ts b/packages/core/src/tools/edit.test.ts index 15fb31ddcb..2aed6e775a 100644 --- a/packages/core/src/tools/edit.test.ts +++ b/packages/core/src/tools/edit.test.ts @@ -1313,6 +1313,39 @@ describe('EditTool', () => { expect(fs.readFileSync(filePath, 'utf8')).toBe('X\nline b\nline c\n'); }); + it('allows editing a large text file after a ranged read', async () => { + const initialContent = [ + 'target', + 'context 1', + 'context 2', + 'context 3', + 'context 4', + 'x'.repeat(11 * 1024 * 1024), + ].join('\n'); + fs.writeFileSync(filePath, initialContent, 'utf8'); + const stats = fs.statSync(filePath); + fileReadCache.recordRead(filePath, stats, { + full: false, + cacheable: true, + }); + (mockConfig.getApprovalMode as Mock).mockReturnValueOnce( + ApprovalMode.AUTO_EDIT, + ); + + const result = await tool + .build({ + file_path: filePath, + old_string: 'target', + new_string: 'updated', + }) + .execute(abortSignal); + + expect(result.error).toBeUndefined(); + expect(fs.readFileSync(filePath, 'utf8').startsWith('updated\n')).toBe( + true, + ); + }); + it('rejects an edit when the previous read was non-cacheable (binary / pdf / image)', async () => { // ReadFile records every successful read into the cache, // including binary / PDF / image reads that produce a diff --git a/packages/core/src/tools/read-file.test.ts b/packages/core/src/tools/read-file.test.ts index 276e1744bf..9b126433a0 100644 --- a/packages/core/src/tools/read-file.test.ts +++ b/packages/core/src/tools/read-file.test.ts @@ -423,9 +423,8 @@ describe('ReadFileTool', () => { }); }); - it('should return error for a file that is too large', async () => { + it('should read and truncate a text file larger than 10MB', async () => { const filePath = path.join(tempRootDir, 'largefile.txt'); - // 11MB of content exceeds 10MB limit const largeContent = 'x'.repeat(11 * 1024 * 1024); await fsp.writeFile(filePath, largeContent, 'utf-8'); const params: ReadFileToolParams = { file_path: filePath }; @@ -435,10 +434,28 @@ describe('ReadFileTool', () => { >; const result = await invocation.execute(abortSignal); - expect(result).toHaveProperty('error'); - expect(result.error?.type).toBe(ToolErrorType.FILE_TOO_LARGE); - expect(result.error?.message).toContain( - 'File size exceeds the 10MB limit', + expect(result.error).toBeUndefined(); + expect(result.returnDisplay).toBe( + 'Read lines 1-1 of at least 1 from largefile.txt (truncated)', + ); + expect(result.llmContent).toContain( + 'Showing lines 1-1 of at least 1 total lines', + ); + expect(result.llmContent).toContain('... [truncated]'); + }); + + it('should propagate an aborted signal before reading', async () => { + const filePath = path.join(tempRootDir, 'abort.txt'); + await fsp.writeFile(filePath, 'content', 'utf-8'); + const invocation = tool.build({ file_path: filePath }) as ToolInvocation< + ReadFileToolParams, + ToolResult + >; + const controller = new AbortController(); + controller.abort(); + + await expect(invocation.execute(controller.signal)).rejects.toThrow( + /abort/i, ); }); diff --git a/packages/core/src/tools/read-file.ts b/packages/core/src/tools/read-file.ts index eca595532a..c4ccc77b69 100644 --- a/packages/core/src/tools/read-file.ts +++ b/packages/core/src/tools/read-file.ts @@ -132,7 +132,7 @@ class ReadFileToolInvocation extends BaseToolInvocation< return 'ask'; } - async execute(): Promise { + async execute(signal: AbortSignal): Promise { const absPath = path.resolve(this.params.file_path); const projectRoot = this.config.getTargetDir(); // Auto-memory files (AGENTS.md and friends under the auto-memory @@ -206,6 +206,7 @@ class ReadFileToolInvocation extends BaseToolInvocation< offset: this.params.offset, limit: this.params.limit, pages: this.params.pages, + signal, }, ); @@ -284,7 +285,9 @@ class ReadFileToolInvocation extends BaseToolInvocation< ) { const [start, end] = result.linesShown!; const total = result.originalLineCount!; - llmContent = `Showing lines ${start}-${end} of ${total} total lines.\n\n---\n\n${result.llmContent}`; + const totalLabel = + result.originalLineCountExact === false ? `at least ${total}` : total; + llmContent = `Showing lines ${start}-${end} of ${totalLabel} total lines.\n\n---\n\n${result.llmContent}`; } else { llmContent = result.llmContent || ''; } diff --git a/packages/core/src/utils/fileUtils.test.ts b/packages/core/src/utils/fileUtils.test.ts index 0ff64f1b6f..c0cef12435 100644 --- a/packages/core/src/utils/fileUtils.test.ts +++ b/packages/core/src/utils/fileUtils.test.ts @@ -29,11 +29,14 @@ import { processSingleFileContent, detectBOM, decodeBufferWithEncodingInfo, + readFileWithLineAndLimit, readFileWithEncoding, readFileWithEncodingInfo, detectFileEncoding, fileExists, } from './fileUtils.js'; +import { iconvEncode } from './iconvHelper.js'; +import { LargeNonUtf8TextError } from './read-text-range.js'; import type { Config } from '../config/config.js'; import { StandardFileSystemService } from '../services/fileSystemService.js'; import { ToolErrorType } from '../tools/tool-error.js'; @@ -1993,6 +1996,149 @@ describe('fileUtils', () => { expect(result.linesShown).toEqual([1, 2]); }); + it('should preserve default full file-system reads for large text files', async () => { + const content = `head\n${'x'.repeat(11 * 1024 * 1024)}`; + actualNodeFs.writeFileSync(testTextFilePath, content); + + const result = await fsService.readTextFile({ path: testTextFilePath }); + + expect(result.content).toBe(content); + expect(result._meta?.originalLineCount).toBe(2); + expect(result._meta?.originalLineCountExact).toBe(true); + expect(result._meta?.truncatedByBytes).not.toBe(true); + }); + + it('should stream explicit offset reads for large text files', async () => { + actualNodeFs.writeFileSync( + testTextFilePath, + `skip\n${'line\n'.repeat(3 * 1024 * 1024)}`, + ); + + const result = await fsService.readTextFile({ + path: testTextFilePath, + line: 1, + }); + + expect(result.content.startsWith('line\n')).toBe(true); + expect(result.content.startsWith('skip\n')).toBe(false); + expect(result._meta?.originalLineCountExact).toBe(false); + expect(result._meta?.truncatedByBytes).toBe(true); + }); + + it('should preserve unbounded explicit line-zero reads below the large-file threshold', async () => { + const content = `head\n${'body\n'.repeat(6_000)}tail\n`; + actualNodeFs.writeFileSync(testTextFilePath, content); + + const result = await fsService.readTextFile({ + path: testTextFilePath, + line: 0, + }); + + expect(result.content).toBe(content); + expect(result._meta?.truncatedByBytes).not.toBe(true); + }); + + it('should enforce maxOutputBytes for default file-system reads below the large-file threshold', async () => { + actualNodeFs.writeFileSync(testTextFilePath, 'x'.repeat(100)); + + const result = await fsService.readTextFile({ + path: testTextFilePath, + maxOutputBytes: 10, + }); + + expect(result.content).toBe('x'.repeat(10)); + expect(result._meta?.originalLineCount).toBe(1); + expect(result._meta?.originalLineCountExact).toBe(true); + expect(result._meta?.truncatedByBytes).toBe(true); + }); + + it('should propagate large non-UTF-8 errors through bounded reads', async () => { + const gbkLine = iconvEncode('中文日志行\n', 'gbk'); + const gbkChunk = Buffer.concat( + Array.from({ length: 1024 }, () => gbkLine), + ); + const repeatCount = Math.ceil((11 * 1024 * 1024) / gbkChunk.length); + actualNodeFs.writeFileSync( + testTextFilePath, + Buffer.concat(Array.from({ length: repeatCount }, () => gbkChunk)), + ); + + await expect( + readFileWithLineAndLimit({ + path: testTextFilePath, + limit: 10, + maxOutputBytes: 10_000, + }), + ).rejects.toThrow(LargeNonUtf8TextError); + }); + + it('should propagate aborts from unbounded full reads', async () => { + actualNodeFs.writeFileSync(testTextFilePath, 'hello\nworld'); + const controller = new AbortController(); + controller.abort(); + + await expect( + readFileWithLineAndLimit({ + path: testTextFilePath, + limit: Number.POSITIVE_INFINITY, + signal: controller.signal, + }), + ).rejects.toThrow(/abort/i); + }); + + it('should propagate aborts before large unbounded full reads', async () => { + actualNodeFs.writeFileSync( + testTextFilePath, + 'x'.repeat(11 * 1024 * 1024), + ); + const controller = new AbortController(); + controller.abort(); + + await expect( + readFileWithLineAndLimit({ + path: testTextFilePath, + limit: Number.POSITIVE_INFINITY, + signal: controller.signal, + }), + ).rejects.toThrow(/abort/i); + }); + + it('should use provided stats when reading with line and byte limits', async () => { + actualNodeFs.writeFileSync(testTextFilePath, 'hello\nworld'); + const stats = actualNodeFs.statSync(testTextFilePath); + const statSpy = vi + .spyOn(fs.promises, 'stat') + .mockRejectedValueOnce(new Error('unexpected stat')); + + try { + const result = await readFileWithLineAndLimit({ + path: testTextFilePath, + limit: 1, + maxOutputBytes: 100, + stats, + }); + + expect(result.content).toBe('hello'); + expect(statSpy).not.toHaveBeenCalled(); + } finally { + statSpy.mockRestore(); + } + }); + + it('should not byte-truncate multibyte text before the character limit', async () => { + const content = '你'.repeat(1000); + actualNodeFs.writeFileSync(testTextFilePath, content, 'utf-8'); + + const result = await processSingleFileContent( + testTextFilePath, + mockConfig, + ); + + expect(result.llmContent).toBe(content); + expect(result.returnDisplay).toBe(''); + expect(result.isTruncated).toBe(false); + }); + it('should truncate long lines in text files', async () => { const longLine = 'a'.repeat(2500); actualNodeFs.writeFileSync( @@ -2073,31 +2219,152 @@ describe('fileUtils', () => { ); }); - it('should return an error if the file size exceeds 10MB', async () => { - // Create a small test file - actualNodeFs.writeFileSync(testTextFilePath, 'test content'); + it('should read large text files through bounded truncation instead of the 10MB gate', async () => { + const lines = Array.from( + { length: 65_000 }, + (_, index) => `Line ${index + 1} ${'x'.repeat(180)}`, + ); + actualNodeFs.writeFileSync(testTextFilePath, lines.join('\n')); - // Spy on fs.promises.stat to return a large file size - const statSpy = vi.spyOn(fs.promises, 'stat').mockResolvedValueOnce({ - size: 11 * 1024 * 1024, - isDirectory: () => false, - isFile: () => true, - } as fs.Stats); + const result = await processSingleFileContent( + testTextFilePath, + mockConfig, + ); - try { - const result = await processSingleFileContent( - testTextFilePath, - mockConfig, - ); + expect(result.error).toBeUndefined(); + expect(result.llmContent).toContain('Line 1'); + expect(result.returnDisplay).toContain('Read lines 1-'); + expect(result.isTruncated).toBe(true); + expect(result.originalLineCount).toBeGreaterThanOrEqual( + result.linesShown?.[1] ?? 1, + ); + expect(result.originalLineCount).toBeLessThan(65_000); + expect(result.originalLineCountExact).toBe(false); + expect(result.linesShown?.[0]).toBe(1); + }); - expect(result.error).toContain('File size exceeds the 10MB limit'); - expect(result.returnDisplay).toContain( - 'File size exceeds the 10MB limit', - ); - expect(result.llmContent).toContain('File size exceeds the 10MB limit'); - } finally { - statSpy.mockRestore(); - } + it('should stream large text files when line truncation is disabled', async () => { + actualNodeFs.writeFileSync( + testTextFilePath, + 'x'.repeat(11 * 1024 * 1024), + ); + const noLineLimitConfig = { + ...mockConfig, + getTruncateToolOutputLines: () => Number.POSITIVE_INFINITY, + } as unknown as Config; + + const result = await processSingleFileContent( + testTextFilePath, + noLineLimitConfig, + ); + + expect(result.error).toBeUndefined(); + expect(typeof result.llmContent).toBe('string'); + expect(result.llmContent).toContain('... [truncated]'); + expect(result.returnDisplay).toBe( + 'Read lines 1-1 of at least 1 from test.txt (truncated)', + ); + expect(result.isTruncated).toBe(true); + expect(result.originalLineCountExact).toBe(false); + }); + + it('should mark byte truncation metadata without character truncation', async () => { + actualNodeFs.writeFileSync(testTextFilePath, 'visible'); + const byteTruncatedConfig = { + ...mockConfig, + getTruncateToolOutputThreshold: () => Number.POSITIVE_INFINITY, + getFileSystemService: () => ({ + readTextFile: vi.fn().mockResolvedValue({ + content: 'visible', + _meta: { + originalLineCount: 1, + originalLineCountExact: false, + truncatedByBytes: true, + }, + }), + }), + } as unknown as Config; + + const result = await processSingleFileContent( + testTextFilePath, + byteTruncatedConfig, + ); + + expect(typeof result.llmContent).toBe('string'); + const llmContent = result.llmContent as string; + expect(llmContent).toBe('visible\n... [truncated]'); + expect(llmContent.match(/\.\.\. \[truncated\]/g)).toHaveLength(1); + expect(result.returnDisplay).toBe( + 'Read lines 1-1 of at least 1 from test.txt (truncated)', + ); + expect(result.isTruncated).toBe(true); + }); + + it('should use selected range as a lower bound when large file metadata is missing', async () => { + actualNodeFs.writeFileSync( + testTextFilePath, + 'x'.repeat(11 * 1024 * 1024), + ); + const missingMetadataConfig = { + ...mockConfig, + getFileSystemService: () => ({ + readTextFile: vi.fn().mockResolvedValue({ + content: 'visible\nnext', + }), + }), + } as unknown as Config; + + const result = await processSingleFileContent( + testTextFilePath, + missingMetadataConfig, + { offset: 9, limit: 2 }, + ); + + expect(result.originalLineCount).toBe(11); + expect(result.originalLineCountExact).toBe(false); + expect(result.returnDisplay).toBe( + 'Read lines 10-11 of at least 11 from test.txt', + ); + }); + + it('should preserve disabled output truncation for large text files', async () => { + const byteLength = 11 * 1024 * 1024; + actualNodeFs.writeFileSync(testTextFilePath, 'x'.repeat(byteLength)); + const noCharacterLimitConfig = { + ...mockConfig, + getTruncateToolOutputThreshold: () => Number.POSITIVE_INFINITY, + } as unknown as Config; + + const result = await processSingleFileContent( + testTextFilePath, + noCharacterLimitConfig, + ); + + expect(typeof result.llmContent).toBe('string'); + const llmContent = result.llmContent as string; + expect(llmContent).toHaveLength(byteLength); + expect(llmContent).not.toContain('... [truncated]'); + expect(result.returnDisplay).toBe(''); + expect(result.isTruncated).toBe(false); + }); + + it('should still return an error if an inline media file exceeds 10MB', async () => { + mockMimeGetType.mockReturnValue('image/png'); + actualNodeFs.writeFileSync( + testImageFilePath, + Buffer.alloc(11 * 1024 * 1024), + ); + + const result = await processSingleFileContent( + testImageFilePath, + mockConfig, + ); + + expect(result.error).toContain('File size exceeds the 10MB limit'); + expect(result.returnDisplay).toContain( + 'File size exceeds the 10MB limit', + ); + expect(result.llmContent).toContain('File size exceeds the 10MB limit'); }); it('should allow explicit page ranges above the full-PDF text-extraction size cap', async () => { diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index c6e03416e3..6cb62bf205 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -18,7 +18,7 @@ import { ToolErrorType } from '../tools/tool-error.js'; import { BINARY_EXTENSIONS } from './ignorePatterns.js'; import type { Config } from '../config/config.js'; import { createDebugLogger } from './debugLogger.js'; -import { getErrorMessage, isNodeError } from './errors.js'; +import { getErrorMessage, isAbortError, isNodeError } from './errors.js'; import type { InputModalities } from '../core/contentGenerator.js'; import { detectEncodingFromBuffer } from './systemEncoding.js'; import { @@ -35,6 +35,11 @@ import { shouldRequirePDFPageRange, } from './pdf.js'; import { readNotebookWithMetadata } from './notebook.js'; +import { readTextRange } from './read-text-range.js'; +import { + DEFAULT_RANGE_READ_BYTES, + TEXT_RANGE_FAST_PATH_MAX_SIZE, +} from './text-range-constants.js'; const debugLogger = createDebugLogger('FILE_UTILS'); @@ -273,9 +278,13 @@ function bomEncodingToName(bomEncoding: UnicodeEncoding): string { */ export async function readFileWithEncodingInfo( filePath: string, + signal?: AbortSignal, ): Promise { // Read the file once; detect BOM and decode from the single buffer. - const full = await fs.promises.readFile(filePath); + const full = await fs.promises.readFile( + filePath, + signal === undefined ? undefined : { signal }, + ); return decodeBufferWithEncodingInfo(full); } @@ -299,14 +308,42 @@ export async function readFileWithLineAndLimit(params: { path: string; limit: number; line?: number; + maxOutputBytes?: number; + signal?: AbortSignal; + stats?: import('node:fs').Stats; }): Promise<{ content: string; bom?: boolean; encoding?: string; originalLineCount: number; + originalLineCountExact?: boolean; + lineEnding?: 'crlf' | 'lf'; + truncatedByBytes?: boolean; }> { - const { path: filePath, limit, line } = params; - const { content, encoding, bom } = await readFileWithEncodingInfo(filePath); + const { path: filePath, limit, line, maxOutputBytes, signal } = params; + const stats = params.stats ?? (await fs.promises.stat(filePath)); + if ( + (line !== undefined && line > 0) || + Number.isFinite(limit) || + maxOutputBytes !== undefined + ) { + return readTextRange({ + path: filePath, + offset: line || 0, + limit, + maxOutputBytes: normalizeRangeReadByteLimit(maxOutputBytes), + stats, + ...(signal !== undefined ? { signal } : {}), + }); + } + + signal?.throwIfAborted(); + + const { content, encoding, bom } = await readFileWithEncodingInfo( + filePath, + signal, + ); + signal?.throwIfAborted(); const lines = content.split('\n'); const originalLineCount = lines.length; const startLine = line || 0; @@ -321,6 +358,7 @@ export async function readFileWithLineAndLimit(params: { bom, encoding, originalLineCount, + originalLineCountExact: true, }; } @@ -829,6 +867,7 @@ export interface ProcessedFileReadResult { error?: string; // Optional error message for the LLM if file processing failed errorType?: ToolErrorType; // Structured error type originalLineCount?: number; // For text files, the total number of lines in the original file + originalLineCountExact?: boolean; // False when a large range read stopped before EOF isTruncated?: boolean; // Indicates if displayed content was truncated linesShown?: [number, number]; // For text files [startLine, endLine] (1-based for display) /** @@ -867,6 +906,7 @@ export interface ProcessSingleFileContentOptions { * sets this after deciding the vision bridge should handle the image. */ preserveUnsupportedImage?: boolean; + signal?: AbortSignal; /** * Large full-PDF text fallback returns a tool error by default. `@`-attached * PDFs use `reference` so the model gets guidance without a failed read. @@ -945,10 +985,12 @@ export async function processSingleFileContent( limit, pages, preserveUnsupportedImage = false, + signal, largePdfBehavior = 'error', } = options; const rootDirectory = config.getTargetDir(); try { + signal?.throwIfAborted(); let stats: import('node:fs').Stats; try { // Async stat doubles as the existence check — ENOENT is handled below @@ -1101,7 +1143,7 @@ export async function processSingleFileContent( }; } } - if (fileSizeInMB > 9.9 && !willExtractPdfText) { + if (fileSizeInMB > 9.9 && !willExtractPdfText && fileType !== 'text') { return { llmContent: 'File size exceeds the 10MB limit.', returnDisplay: 'File size exceeds the 10MB limit.', @@ -1182,11 +1224,25 @@ export async function processSingleFileContent( path: filePath, limit: limit ?? config.getTruncateToolOutputLines(), line: offset, + maxOutputBytes: getRangeReadByteLimit(config), + stats, + ...(signal !== undefined ? { signal } : {}), }); - const originalLineCount = - _meta?.originalLineCount ?? (await countFileLines(filePath)); const selectedLines = content.split('\n').map((line) => line.trimEnd()); const startLine = offset || 0; + const selectedLineCount = + content.length === 0 ? 0 : selectedLines.length; + const hasOriginalLineCount = _meta?.originalLineCount !== undefined; + const originalLineCount = + _meta?.originalLineCount ?? + (stats.size >= TEXT_RANGE_FAST_PATH_MAX_SIZE + ? startLine + selectedLineCount + : await countFileLines(filePath)); + const originalLineCountExact = + _meta?.originalLineCountExact === false + ? false + : hasOriginalLineCount || + stats.size < TEXT_RANGE_FAST_PATH_MAX_SIZE; const configCharLimit = config.getTruncateToolOutputThreshold(); // Apply character limit truncation @@ -1227,17 +1283,38 @@ export async function processSingleFileContent( linesIncluded = selectedLines.length; } + if (_meta?.truncatedByBytes === true) { + const marker = '... [truncated]'; + if (!llmContent.endsWith(marker)) { + const prefix = Number.isFinite(configCharLimit) + ? llmContent.slice( + 0, + Math.max(configCharLimit - marker.length - 1, 0), + ) + : llmContent; + const separator = + prefix.length === 0 || prefix.endsWith('\n') ? '' : '\n'; + llmContent = prefix + separator + marker; + } + contentLengthTruncated = true; + } + const actualEndLine = startLine + linesIncluded; const contentRangeTruncated = - startLine > 0 || actualEndLine < originalLineCount; + startLine > 0 || + !originalLineCountExact || + actualEndLine < originalLineCount; const isTruncated = contentRangeTruncated || contentLengthTruncated; + const lineCountLabel = originalLineCountExact + ? `${originalLineCount}` + : `at least ${originalLineCount}`; // By default, return nothing to streamline the common case of a successful read_file. let returnDisplay = ''; if (isTruncated) { returnDisplay = `Read lines ${ startLine + 1 - }-${actualEndLine} of ${originalLineCount} from ${relativePathForDisplay}`; + }-${actualEndLine} of ${lineCountLabel} from ${relativePathForDisplay}`; if (contentLengthTruncated) { returnDisplay += ' (truncated)'; } @@ -1248,6 +1325,7 @@ export async function processSingleFileContent( returnDisplay, isTruncated, originalLineCount, + originalLineCountExact, linesShown: [startLine + 1, actualEndLine], stats, }; @@ -1385,6 +1463,9 @@ export async function processSingleFileContent( } } } catch (error) { + if (signal?.aborted || isAbortError(error)) { + throw error; + } const errorMessage = getErrorMessage(error); const displayPath = path .relative(rootDirectory, filePath) @@ -1398,6 +1479,28 @@ export async function processSingleFileContent( } } +function getRangeReadByteLimit(config: Config): number { + const charLimit = config.getTruncateToolOutputThreshold(); + if (charLimit === Number.POSITIVE_INFINITY) { + return Number.POSITIVE_INFINITY; + } + if (!Number.isFinite(charLimit)) { + return DEFAULT_RANGE_READ_BYTES; + } + // Leave enough byte headroom for UTF-8 text so the character budget remains + // the primary truncation control for normal source/log content. + return Math.max(DEFAULT_RANGE_READ_BYTES, Math.floor(charLimit) * 4); +} + +function normalizeRangeReadByteLimit(maxOutputBytes: number | undefined) { + if (maxOutputBytes === Number.POSITIVE_INFINITY) { + return Number.POSITIVE_INFINITY; + } + return typeof maxOutputBytes === 'number' && Number.isFinite(maxOutputBytes) + ? maxOutputBytes + : DEFAULT_RANGE_READ_BYTES; +} + export async function fileExists(filePath: string): Promise { try { await fsPromises.access(filePath, fs.constants.F_OK); diff --git a/packages/core/src/utils/pathReader.test.ts b/packages/core/src/utils/pathReader.test.ts index c6b8040d39..5b5e951acb 100644 --- a/packages/core/src/utils/pathReader.test.ts +++ b/packages/core/src/utils/pathReader.test.ts @@ -428,8 +428,7 @@ describe('readPathFromWorkspace', () => { }, ); - it('should return an error string for files exceeding the size limit', async () => { - // Mock a file slightly larger than the 10MB limit defined in fileUtils.ts + it('should truncate text files exceeding 10MB instead of failing', async () => { const largeContent = 'a'.repeat(11 * 1024 * 1024); // 11MB mock({ [CWD]: { @@ -442,7 +441,7 @@ describe('readPathFromWorkspace', () => { const config = createMockConfig(CWD, [], mockFileService); const result = await readPathFromWorkspace('large.txt', config); const textResult = result[0] as string; - // The error message comes directly from processSingleFileContent - expect(textResult).toBe('File size exceeds the 10MB limit.'); + expect(textResult).toContain('a'.repeat(100)); + expect(textResult).toContain('... [truncated]'); }); }); diff --git a/packages/core/src/utils/read-text-range.test.ts b/packages/core/src/utils/read-text-range.test.ts new file mode 100644 index 0000000000..2812045580 --- /dev/null +++ b/packages/core/src/utils/read-text-range.test.ts @@ -0,0 +1,294 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import * as fs from 'node:fs/promises'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { iconvEncode } from './iconvHelper.js'; +import { LargeNonUtf8TextError, readTextRange } from './read-text-range.js'; + +describe('readTextRange', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'read-text-range-')); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + async function writeFile( + name: string, + data: string | Buffer, + ): Promise { + const filePath = path.join(tempDir, name); + await fs.writeFile(filePath, data); + return filePath; + } + + function largeUtf8Lines(lineCount: number): string { + return Array.from( + { length: lineCount }, + (_, index) => `line-${index + 1} ${'x'.repeat(180)}`, + ).join('\n'); + } + + it('preserves split newline semantics on the fast path', async () => { + const emptyPath = await writeFile('empty.txt', ''); + await expect( + readTextRange({ + path: emptyPath, + offset: 0, + limit: 10, + maxOutputBytes: 100, + }), + ).resolves.toMatchObject({ + content: '', + originalLineCount: 1, + truncatedByBytes: false, + }); + + const trailingPath = await writeFile('trailing.txt', 'a\n'); + await expect( + readTextRange({ + path: trailingPath, + offset: 0, + limit: 10, + maxOutputBytes: 100, + }), + ).resolves.toMatchObject({ + content: 'a\n', + originalLineCount: 2, + truncatedByBytes: false, + }); + }); + + it('streams a large UTF-8 file and returns the requested range', async () => { + const filePath = await writeFile('large.log', largeUtf8Lines(65_000)); + + const result = await readTextRange({ + path: filePath, + offset: 42_000, + limit: 3, + maxOutputBytes: 10_000, + }); + + expect(result.content.split('\n')).toEqual([ + expect.stringContaining('line-42001'), + expect.stringContaining('line-42002'), + expect.stringContaining('line-42003'), + ]); + expect(result.originalLineCount).toBeGreaterThanOrEqual(42_004); + expect(result.originalLineCount).toBeLessThan(65_000); + expect(result.originalLineCountExact).toBe(false); + expect(result.encoding).toBe('utf-8'); + expect(result.bom).toBe(false); + expect(result.truncatedByBytes).toBe(false); + }); + + it('streams a large UTF-8 file from the beginning when no range is provided', async () => { + const filePath = await writeFile('large.log', largeUtf8Lines(65_000)); + + const result = await readTextRange({ + path: filePath, + maxOutputBytes: 10_000, + }); + + expect(result.content).toContain('line-1'); + expect(result.content).toContain('line-2'); + expect(result.content).not.toContain('line-65000'); + expect(result.originalLineCount).toBeGreaterThan(1); + expect(result.originalLineCountExact).toBe(false); + expect(result.truncatedByBytes).toBe(true); + }); + + it('returns empty content when a large UTF-8 range starts beyond EOF', async () => { + const filePath = await writeFile('large.log', largeUtf8Lines(65_000)); + + const result = await readTextRange({ + path: filePath, + offset: 100_000, + limit: 10, + maxOutputBytes: 10_000, + }); + + expect(result.content).toBe(''); + expect(result.originalLineCount).toBe(65_000); + expect(result.originalLineCountExact).toBe(true); + expect(result.truncatedByBytes).toBe(false); + }); + + it('preserves CRLF content and line-ending metadata for large files', async () => { + const content = Array.from( + { length: 65_000 }, + (_, index) => `line-${index + 1} ${'x'.repeat(180)}`, + ).join('\r\n'); + const filePath = await writeFile('crlf.log', content); + + const result = await readTextRange({ + path: filePath, + offset: 1, + limit: 2, + maxOutputBytes: 10_000, + }); + + expect(result.content).toContain('\r\n'); + expect(result.content.split('\n')[0]).toMatch(/\r$/); + expect(result.lineEnding).toBe('crlf'); + expect(result.originalLineCount).toBeGreaterThanOrEqual(4); + expect(result.originalLineCount).toBeLessThan(65_000); + expect(result.originalLineCountExact).toBe(false); + }); + + it('detects CRLF when the pair crosses a stream chunk boundary', async () => { + const highWaterMark = 512 * 1024; + const firstChunk = `${'a'.repeat(highWaterMark - 1)}\r`; + const body = Buffer.concat([ + Buffer.from(firstChunk), + Buffer.from('\nsecond\n'), + Buffer.alloc(11 * 1024 * 1024, 'x'), + ]); + const filePath = await writeFile('split-crlf.log', body); + + const result = await readTextRange({ + path: filePath, + offset: 0, + limit: 2, + maxOutputBytes: highWaterMark + 100, + }); + + expect(result.lineEnding).toBe('crlf'); + expect(result.content).toContain('\r\nsecond'); + }); + + it('strips UTF-8 BOM from large file content and reports BOM metadata', async () => { + const body = largeUtf8Lines(65_000); + const filePath = await writeFile( + 'bom.log', + Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(body)]), + ); + + const result = await readTextRange({ + path: filePath, + offset: 0, + limit: 1, + maxOutputBytes: 10_000, + }); + + expect(result.content.charCodeAt(0)).not.toBe(0xfeff); + expect(result.content).toContain('line-1'); + expect(result.bom).toBe(true); + expect(result.encoding).toBe('utf-8'); + expect(result.originalLineCountExact).toBe(false); + }); + + it('does not split a UTF-8 character when byte-truncating', async () => { + const filePath = await writeFile('emoji.txt', `a🙂b`); + + const result = await readTextRange({ + path: filePath, + offset: 0, + limit: 1, + maxOutputBytes: 4, + }); + + expect(result.content).toBe('a'); + expect(result.content).not.toContain('\uFFFD'); + expect(result.truncatedByBytes).toBe(true); + expect(result.originalLineCountExact).toBe(true); + }); + + it('rejects large non-UTF-8 files with a targeted error', async () => { + const gbkLine = iconvEncode('中文日志行\n', 'gbk'); + const repeatCount = Math.ceil((11 * 1024 * 1024) / gbkLine.length); + const filePath = await writeFile( + 'gbk.log', + Buffer.concat(Array.from({ length: repeatCount }, () => gbkLine)), + ); + + await expect( + readTextRange({ + path: filePath, + offset: 0, + limit: 10, + maxOutputBytes: 10_000, + }), + ).rejects.toThrow(LargeNonUtf8TextError); + }); + + it('rejects large files with invalid UTF-8 beyond the encoding sample', async () => { + const mostlyAsciiThenGbk = Buffer.concat([ + Buffer.alloc(9 * 1024, 'a'), + iconvEncode('你好', 'gbk'), + Buffer.alloc(11 * 1024 * 1024, 'b'), + ]); + const filePath = await writeFile('late-gbk.log', mostlyAsciiThenGbk); + + const promise = readTextRange({ + path: filePath, + offset: 0, + limit: 500, + maxOutputBytes: 20_000, + }); + + await expect(promise).rejects.toThrow(LargeNonUtf8TextError); + await expect(promise).rejects.toThrow(/invalid UTF-8 byte sequence/); + await expect(promise).rejects.toMatchObject({ reason: 'invalid-utf8' }); + }); + + it('bounds selected output for a large single-line file', async () => { + const filePath = await writeFile( + 'single-line.log', + 'x'.repeat(11 * 1024 * 1024), + ); + + const result = await readTextRange({ + path: filePath, + offset: 0, + limit: 1, + maxOutputBytes: 1024, + }); + + expect(result.content).toBe('x'.repeat(1024)); + expect(result.originalLineCount).toBe(1); + expect(result.originalLineCountExact).toBe(false); + expect(result.truncatedByBytes).toBe(true); + }); + + it('propagates aborts before reading large files', async () => { + const filePath = await writeFile('large.log', largeUtf8Lines(65_000)); + const controller = new AbortController(); + controller.abort(); + + await expect( + readTextRange({ + path: filePath, + offset: 0, + limit: 10, + maxOutputBytes: 10_000, + signal: controller.signal, + }), + ).rejects.toThrow(/abort/i); + }); + + it('propagates aborts while streaming large files', async () => { + const filePath = await writeFile('large.log', largeUtf8Lines(80_000)); + const controller = new AbortController(); + const promise = readTextRange({ + path: filePath, + offset: 70_000, + limit: 10, + maxOutputBytes: 10_000, + signal: controller.signal, + }); + + setTimeout(() => controller.abort(), 0); + + await expect(promise).rejects.toThrow(/abort/i); + }); +}); diff --git a/packages/core/src/utils/read-text-range.ts b/packages/core/src/utils/read-text-range.ts new file mode 100644 index 0000000000..d675372893 --- /dev/null +++ b/packages/core/src/utils/read-text-range.ts @@ -0,0 +1,281 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { createReadStream, type Stats } from 'node:fs'; +import { stat } from 'node:fs/promises'; +import { TextDecoder } from 'node:util'; +import { detectFileEncoding, readFileWithEncodingInfo } from './fileUtils.js'; +import { isUtf8CompatibleEncoding } from './iconvHelper.js'; +import { + DEFAULT_RANGE_READ_BYTES, + TEXT_RANGE_FAST_PATH_MAX_SIZE, +} from './text-range-constants.js'; + +export interface ReadTextRangeRequest { + path: string; + offset?: number; + limit?: number; + maxOutputBytes: number; + signal?: AbortSignal; + stats?: Stats; +} + +export interface ReadTextRangeResult { + content: string; + originalLineCount: number; + encoding?: string; + bom?: boolean; + lineEnding?: 'crlf' | 'lf'; + originalLineCountExact: boolean; + truncatedByBytes: boolean; +} + +export class LargeNonUtf8TextError extends Error { + constructor( + readonly encoding: string, + readonly reason?: 'invalid-utf8', + ) { + super( + reason === 'invalid-utf8' + ? 'Large text file contains invalid UTF-8 byte sequence beyond the initial encoding sample. Convert or extract a smaller UTF-8 slice and read that instead.' + : `Large non-UTF-8 text files are not supported for streaming reads (detected ${encoding}). Convert or extract a smaller UTF-8 slice and read that instead.`, + ); + this.name = 'LargeNonUtf8TextError'; + } +} + +export async function readTextRange( + request: ReadTextRangeRequest, +): Promise { + request.signal?.throwIfAborted(); + const stats = request.stats ?? (await stat(request.path)); + const maxOutputBytes = normalizeMaxBytes(request.maxOutputBytes); + + if (stats.size < TEXT_RANGE_FAST_PATH_MAX_SIZE) { + const { content, encoding, bom } = await readFileWithEncodingInfo( + request.path, + request.signal, + ); + request.signal?.throwIfAborted(); + const range = sliceDecodedContent( + content, + request.offset, + request.limit, + maxOutputBytes, + ); + return { + ...range, + encoding, + bom, + lineEnding: detectLineEndingFromContent(content), + }; + } + + return readLargeUtf8Range(request, maxOutputBytes); +} + +function normalizeMaxBytes(maxOutputBytes: number): number { + if (maxOutputBytes === Number.POSITIVE_INFINITY) { + return Number.POSITIVE_INFINITY; + } + if (!Number.isFinite(maxOutputBytes)) { + return DEFAULT_RANGE_READ_BYTES; + } + return Math.max(0, Math.floor(maxOutputBytes)); +} + +function sliceDecodedContent( + content: string, + offset: number | undefined, + limit: number | undefined, + maxOutputBytes: number, +): Pick< + ReadTextRangeResult, + | 'content' + | 'originalLineCount' + | 'originalLineCountExact' + | 'truncatedByBytes' +> { + const lines = content.split('\n'); + const originalLineCount = lines.length; + const start = Math.min(Math.max(0, offset ?? 0), originalLineCount); + const end = + limit === undefined + ? originalLineCount + : Math.min(start + Math.max(0, limit), originalLineCount); + const selected = lines.slice(start, end).join('\n'); + const truncated = truncateUtf8(selected, maxOutputBytes); + + return { + content: truncated.content, + originalLineCount, + originalLineCountExact: true, + truncatedByBytes: truncated.truncated, + }; +} + +async function readLargeUtf8Range( + request: ReadTextRangeRequest, + maxOutputBytes: number, +): Promise { + const encoding = await detectFileEncoding(request.path); + if (!isUtf8CompatibleEncoding(encoding)) { + throw new LargeNonUtf8TextError(encoding); + } + + const offset = Math.max(0, request.offset ?? 0); + const endLine = + offset + Math.max(0, request.limit ?? Number.POSITIVE_INFINITY); + let currentLine = 0; + let output = ''; + let outputBytes = 0; + let truncatedByBytes = false; + let bom = false; + let firstChunk = true; + let lineEnding: 'crlf' | 'lf' = 'lf'; + let previousChunkEndedWithCR = false; + let originalLineCountExact = true; + let stoppedEarly = false; + const decoder = new TextDecoder('utf-8', { + fatal: true, + ignoreBOM: true, + }); + + const stream = createReadStream(request.path, { + highWaterMark: 512 * 1024, + signal: request.signal, + }); + + function appendSelected(fragment: string): void { + if (fragment.length === 0 || truncatedByBytes) { + return; + } + + const available = maxOutputBytes - outputBytes; + if (available <= 0) { + truncatedByBytes = true; + return; + } + + const truncated = truncateUtf8(fragment, available); + output += truncated.content; + outputBytes += Buffer.byteLength(truncated.content, 'utf8'); + if (truncated.truncated) { + truncatedByBytes = true; + } + } + + function isSelectedLine(): boolean { + return currentLine >= offset && currentLine < endLine; + } + + function decodeUtf8Chunk( + chunk?: Buffer, + options?: TextDecodeOptions, + ): string { + try { + return decoder.decode(chunk, options); + } catch { + throw new LargeNonUtf8TextError(encoding, 'invalid-utf8'); + } + } + + try { + for await (const rawChunk of stream) { + request.signal?.throwIfAborted(); + let chunk = decodeUtf8Chunk(rawChunk as Buffer, { stream: true }); + if (firstChunk) { + firstChunk = false; + if (chunk.charCodeAt(0) === 0xfeff) { + chunk = chunk.slice(1); + bom = true; + } + } + + if ( + (previousChunkEndedWithCR && chunk.startsWith('\n')) || + chunk.includes('\r\n') + ) { + lineEnding = 'crlf'; + } + previousChunkEndedWithCR = chunk.endsWith('\r'); + + let start = 0; + let newline = chunk.indexOf('\n', start); + while (newline !== -1) { + if (isSelectedLine()) { + appendSelected(chunk.slice(start, newline)); + if (currentLine + 1 < endLine) { + appendSelected('\n'); + } + } + currentLine++; + start = newline + 1; + if (currentLine >= endLine || truncatedByBytes) { + originalLineCountExact = false; + stoppedEarly = true; + break; + } + newline = chunk.indexOf('\n', start); + } + + if (start < chunk.length && isSelectedLine()) { + appendSelected(chunk.slice(start)); + } + if (currentLine >= endLine || truncatedByBytes) { + originalLineCountExact = false; + stoppedEarly = true; + break; + } + } + } finally { + stream.destroy(); + } + + if (!stoppedEarly) { + decodeUtf8Chunk(); + } + + return { + content: output, + originalLineCount: currentLine + 1, + encoding: 'utf-8', + bom, + lineEnding, + originalLineCountExact, + truncatedByBytes, + }; +} + +function truncateUtf8( + content: string, + maxBytes: number, +): { content: string; truncated: boolean } { + const bytes = Buffer.byteLength(content, 'utf8'); + if (bytes <= maxBytes) { + return { content, truncated: false }; + } + if (maxBytes <= 0) { + return { content: '', truncated: true }; + } + + const buffer = Buffer.from(content, 'utf8'); + let end = Math.min(maxBytes, buffer.length); + // `end` is the first excluded byte. If it lands inside a multi-byte UTF-8 + // sequence, the byte at `end` is a continuation byte, so back up until the + // prefix ends before the incomplete character. + while (end > 0 && (buffer[end] & 0xc0) === 0x80) { + end--; + } + return { + content: buffer.subarray(0, end).toString('utf8'), + truncated: true, + }; +} + +function detectLineEndingFromContent(content: string): 'crlf' | 'lf' { + return content.includes('\r\n') ? 'crlf' : 'lf'; +} diff --git a/packages/core/src/utils/readManyFiles.test.ts b/packages/core/src/utils/readManyFiles.test.ts index 20d654e253..4351432c80 100644 --- a/packages/core/src/utils/readManyFiles.test.ts +++ b/packages/core/src/utils/readManyFiles.test.ts @@ -147,6 +147,22 @@ describe('readManyFiles', () => { expect(content).toContain('--- End of content ---'); }); + it('should include truncated large text files instead of reporting a size error', async () => { + const relativePath = 'large.log'; + const absolutePath = path.join(tempRootDir, relativePath); + await fs.writeFile(absolutePath, 'x'.repeat(11 * 1024 * 1024), 'utf-8'); + const mockConfig = createMockConfig(tempRootDir); + + const result = await readManyFiles(mockConfig, { paths: [relativePath] }); + + const content = contentToString(result.contentParts); + expect(content).toContain('Showing lines 1-1 of at least 1 total lines'); + expect(content).toContain('... [truncated]'); + expect(result.files).toHaveLength(1); + expect(result.files[0]!.error).toBeUndefined(); + expect(result.files[0]!.filePath).toBe(absolutePath); + }); + it('should include truncated notebooks that do not expose text line ranges', async () => { const relativePath = 'large.ipynb'; const absolutePath = path.join(tempRootDir, relativePath); @@ -276,6 +292,20 @@ describe('readManyFiles', () => { expect(content).not.toContain('Content of file1.txt'); }); + it('should propagate aborts before reading a directory', async () => { + await createTestFile('mydir', 'file1.txt'); + const mockConfig = createMockConfig(tempRootDir); + const controller = new AbortController(); + controller.abort(); + + await expect( + readManyFiles(mockConfig, { + paths: ['mydir'], + signal: controller.signal, + }), + ).rejects.toThrow(/abort/i); + }); + it('should handle directory with trailing slash', async () => { await createTestFile('mydir', 'file1.txt'); const mockConfig = createMockConfig(tempRootDir); @@ -439,6 +469,20 @@ describe('readManyFiles', () => { }); describe('per-file error surfacing', () => { + it('should propagate aborts from file reads instead of returning an error message', async () => { + const { relativePath } = await createTestFile('cancel.txt'); + const mockConfig = createMockConfig(tempRootDir); + const controller = new AbortController(); + controller.abort(); + + await expect( + readManyFiles(mockConfig, { + paths: [relativePath], + signal: controller.signal, + }), + ).rejects.toThrow(/abort/i); + }); + it('should surface processSingleFileContent errors instead of silently skipping the file', async () => { // Trigger the >10MB file-size error path in processSingleFileContent. const relativePath = 'huge.bin'; diff --git a/packages/core/src/utils/readManyFiles.ts b/packages/core/src/utils/readManyFiles.ts index 80bc835cb1..7d12b26dca 100644 --- a/packages/core/src/utils/readManyFiles.ts +++ b/packages/core/src/utils/readManyFiles.ts @@ -8,7 +8,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import type { Part, PartListUnion } from '@google/genai'; import type { Config } from '../config/config.js'; -import { getErrorMessage } from './errors.js'; +import { getErrorMessage, isAbortError } from './errors.js'; import type { ProcessedFileReadResult } from './fileUtils.js'; import { isCacheableReadResult, @@ -104,7 +104,11 @@ export async function readManyFiles( config: Config, options: ReadManyFilesOptions, ): Promise { - const { paths: inputPatterns, preserveUnsupportedImageForBridge } = options; + const { + paths: inputPatterns, + preserveUnsupportedImageForBridge, + signal, + } = options; const seenFiles = new Set(); const contentParts: Part[] = []; @@ -114,6 +118,7 @@ export async function readManyFiles( const projectRoot = config.getProjectRoot(); for (const rawPattern of inputPatterns) { + signal?.throwIfAborted(); const normalizedPattern = rawPattern.replace(/\\/g, '/'); const fullPath = path.resolve(projectRoot, normalizedPattern); const stats = fs.existsSync(fullPath) ? fs.statSync(fullPath) : null; @@ -122,6 +127,7 @@ export async function readManyFiles( const { contentParts: dirParts, info } = await readDirectory( config, fullPath, + signal, ); contentParts.push(...dirParts); files.push(info); @@ -134,6 +140,7 @@ export async function readManyFiles( config, fullPath, preserveUnsupportedImageForBridge, + signal, ); if (readResult) { contentParts.push(...readResult.contentParts); @@ -142,6 +149,9 @@ export async function readManyFiles( } } } catch (error) { + if (signal?.aborted || isAbortError(error)) { + throw error; + } const errorMessage = `Error during file search: ${getErrorMessage(error)}`; return { contentParts: [errorMessage], @@ -165,11 +175,14 @@ export async function readManyFiles( async function readDirectory( config: Config, directoryPath: string, + signal?: AbortSignal, ): Promise<{ contentParts: Part[]; info: FileReadInfo }> { + signal?.throwIfAborted(); const structure = await getFolderStructure(directoryPath, { fileService: config.getFileService(), fileFilteringOptions: config.getFileFilteringOptions(), }); + signal?.throwIfAborted(); const contentParts: Part[] = [ { text: `\nContent from ${directoryPath}:\n` }, @@ -190,10 +203,12 @@ async function readFileContent( config: Config, filePath: string, preserveUnsupportedImage = false, + signal?: AbortSignal, ): Promise<{ contentParts: Part[]; info: FileReadInfo } | null> { try { const fileReadResult = await processSingleFileContent(filePath, config, { preserveUnsupportedImage, + ...(signal !== undefined ? { signal } : {}), largePdfBehavior: 'reference', }); @@ -233,7 +248,11 @@ async function readFileContent( ) { const [start, end] = fileReadResult.linesShown!; const total = fileReadResult.originalLineCount!; - fileContentForLlm = `Showing lines ${start}-${end} of ${total} total lines.\n---\n${fileReadResult.llmContent}`; + const totalLabel = + fileReadResult.originalLineCountExact === false + ? `at least ${total}` + : total; + fileContentForLlm = `Showing lines ${start}-${end} of ${totalLabel} total lines.\n---\n${fileReadResult.llmContent}`; } else { fileContentForLlm = fileReadResult.llmContent; } @@ -258,7 +277,10 @@ async function readFileContent( isDirectory: false, }, }; - } catch { + } catch (error) { + if (signal?.aborted || isAbortError(error)) { + throw error; + } return null; } } diff --git a/packages/core/src/utils/text-range-constants.ts b/packages/core/src/utils/text-range-constants.ts new file mode 100644 index 0000000000..57c2b2bc8b --- /dev/null +++ b/packages/core/src/utils/text-range-constants.ts @@ -0,0 +1,8 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +export const TEXT_RANGE_FAST_PATH_MAX_SIZE = 10 * 1024 * 1024; +export const DEFAULT_RANGE_READ_BYTES = 25_000;