mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-07-09 17:19:02 +00:00
fix(core): Support large text range reads (#6404)
* fix(core): support large text range reads Allow text reads to stream bounded line ranges for files larger than the previous 10MB guard, while preserving media size limits and forwarding cancellation through read_file/read_many_files/ACP paths. Refs #6403 Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): address large text review feedback Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): propagate abort signals in text reads Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): validate streamed utf8 reads Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): handle disabled line truncation for large reads Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): use kebab-case for text range reader Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): preserve artifact size errors for large sources Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): address large text review follow-up Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): allow default large text reads Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): honor text read byte caps Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): clarify invalid utf8 range read errors Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): prevent truncated full large reads Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): preserve unbounded line-zero reads Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): forward artifact read cancellation Pass artifact execution abort signals into source file reads and preserve cancellation semantics when the read is aborted. Add regression coverage for unbounded large UTF-8 range reads and offsets beyond EOF. Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): address file read review feedback Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> * fix(core): preserve large text mutation reads Allow default unbounded readTextFile calls to keep reading full large text files so mutation tools can prepare complete snapshots after a prior ranged read. Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> --------- Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com> Co-authored-by: qwen-code-dev-bot <qwen-code-dev-bot@users.noreply.github.com>
This commit is contained in:
jinye
GitHub
parent
17138b525f
commit
6fdd0fc710
20 changed files with 1417 additions and 82 deletions
|
|
@ -107,6 +107,39 @@ describe('AcpFileSystemService', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('converts core-only read params at the ACP boundary', async () => {
|
||||
const mockResponse = {
|
||||
content: 'slice',
|
||||
_meta: { bom: false, encoding: 'utf-8' },
|
||||
};
|
||||
const client = {
|
||||
readTextFile: vi.fn().mockResolvedValue(mockResponse),
|
||||
} as unknown as AgentSideConnection;
|
||||
const signal = new AbortController().signal;
|
||||
|
||||
const svc = new AcpFileSystemService(
|
||||
client,
|
||||
'session-1',
|
||||
{ readTextFile: true, writeTextFile: true },
|
||||
createFallback(),
|
||||
);
|
||||
|
||||
await svc.readTextFile({
|
||||
path: '/some/file.txt',
|
||||
line: 0,
|
||||
limit: 5,
|
||||
maxOutputBytes: 1024,
|
||||
signal,
|
||||
});
|
||||
|
||||
expect(client.readTextFile).toHaveBeenCalledWith({
|
||||
path: '/some/file.txt',
|
||||
line: 1,
|
||||
limit: 5,
|
||||
sessionId: 'session-1',
|
||||
});
|
||||
});
|
||||
|
||||
it('converts RESOURCE_NOT_FOUND error to ENOENT', async () => {
|
||||
const resourceNotFoundError = {
|
||||
code: RESOURCE_NOT_FOUND_CODE,
|
||||
|
|
@ -953,11 +986,20 @@ describe('AcpFileSystemService', () => {
|
|||
fallback,
|
||||
);
|
||||
|
||||
const result = await svc.readTextFile({ path: '/some/file.txt' });
|
||||
const signal = new AbortController().signal;
|
||||
const result = await svc.readTextFile({
|
||||
path: '/some/file.txt',
|
||||
line: 0,
|
||||
maxOutputBytes: 2048,
|
||||
signal,
|
||||
});
|
||||
|
||||
expect(result).toEqual(fallbackResponse);
|
||||
expect(fallback.readTextFile).toHaveBeenCalledWith({
|
||||
path: '/some/file.txt',
|
||||
line: 0,
|
||||
maxOutputBytes: 2048,
|
||||
signal,
|
||||
});
|
||||
expect(client.readTextFile).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ import type {
|
|||
} from '@agentclientprotocol/sdk';
|
||||
import { RequestError } from '@agentclientprotocol/sdk';
|
||||
import type {
|
||||
CoreReadTextFileRequest,
|
||||
FileSystemService,
|
||||
ReadTextFileResponse,
|
||||
} from '@qwen-code/qwen-code-core';
|
||||
|
|
@ -105,6 +106,29 @@ async function resolveRealPath(value: string): Promise<string | undefined> {
|
|||
}
|
||||
}
|
||||
|
||||
function toAcpReadTextFileRequest(
|
||||
params: CoreReadTextFileRequest,
|
||||
sessionId: string,
|
||||
): ReadTextFileRequest {
|
||||
// `maxOutputBytes`, `signal`, and `stats` are core-local concerns that the
|
||||
// current ACP schema cannot represent. Keep this boundary explicit if the
|
||||
// schema grows.
|
||||
const request: ReadTextFileRequest = {
|
||||
path: params.path,
|
||||
sessionId,
|
||||
};
|
||||
if (params._meta !== undefined) {
|
||||
request._meta = params._meta;
|
||||
}
|
||||
if (params.limit !== undefined) {
|
||||
request.limit = params.limit;
|
||||
}
|
||||
if (params.line != null) {
|
||||
request.line = params.line + 1;
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
||||
export class AcpFileSystemService implements FileSystemService {
|
||||
constructor(
|
||||
private readonly connection: AgentSideConnection,
|
||||
|
|
@ -115,7 +139,7 @@ export class AcpFileSystemService implements FileSystemService {
|
|||
) {}
|
||||
|
||||
async readTextFile(
|
||||
params: Omit<ReadTextFileRequest, 'sessionId'>,
|
||||
params: CoreReadTextFileRequest,
|
||||
): Promise<ReadTextFileResponse> {
|
||||
if (!this.capabilities.readTextFile) {
|
||||
return this.fallback.readTextFile(params);
|
||||
|
|
@ -123,10 +147,9 @@ export class AcpFileSystemService implements FileSystemService {
|
|||
|
||||
let response: ReadTextFileResponse;
|
||||
try {
|
||||
response = await this.connection.readTextFile({
|
||||
...params,
|
||||
sessionId: this.sessionId,
|
||||
});
|
||||
response = await this.connection.readTextFile(
|
||||
toAcpReadTextFileRequest(params, this.sessionId),
|
||||
);
|
||||
} catch (error) {
|
||||
const errorCode = getErrorCode(error);
|
||||
|
||||
|
|
|
|||
|
|
@ -17,12 +17,12 @@ import type { Intent, ResolvedPath } from './paths.js';
|
|||
* typical source files, small enough that an SSE replay buffer
|
||||
* doesn't fill on a single read.
|
||||
*
|
||||
* Files **above** this cap are refused with `file_too_large` rather
|
||||
* than truncated — the underlying `readFileWithLineAndLimit`
|
||||
* reads the whole file into memory before slicing lines, so soft
|
||||
* truncation past the cap would still OOM the daemon. Files
|
||||
* **at or below** the cap honor a tighter `opts.maxBytes` via
|
||||
* post-decode truncation (`enforceReadSize`); that's where the
|
||||
* Full-snapshot reads above this cap are refused with `file_too_large`
|
||||
* rather than truncated. `readText` can serve explicit line windows
|
||||
* from larger files, but the default read/edit contract still needs a
|
||||
* bounded snapshot for hash stability, SSE buffering, and oldText
|
||||
* matching. Files at or below the cap honor a tighter `opts.maxBytes`
|
||||
* via post-decode truncation (`enforceReadSize`); that's where the
|
||||
* `meta.truncated = true` flag fires.
|
||||
*
|
||||
* `enforceReadBytesSize` (the `readBytes` gate) and `edit()` use the
|
||||
|
|
|
|||
|
|
@ -735,6 +735,20 @@ describe('WorkspaceFileSystem - write/edit', () => {
|
|||
expect(after).toBe('foo=42\nbar=2\n');
|
||||
});
|
||||
|
||||
it('edit() preserves the tail of files larger than the default range cap', async () => {
|
||||
const target = path.join(h.workspace, 'large-edit.txt');
|
||||
const tail = 'tail-marker\n';
|
||||
const content = `foo=1\n${'body\n'.repeat(6_000)}${tail}`;
|
||||
await fsp.writeFile(target, content);
|
||||
const r = await h.fs.resolve('large-edit.txt', 'edit');
|
||||
|
||||
const out = await h.fs.edit(r, 'foo=1', 'foo=42');
|
||||
|
||||
expect(out.writtenBytes).toBeGreaterThan(25_000);
|
||||
const after = await fsp.readFile(target, 'utf-8');
|
||||
expect(after).toBe(`foo=42\n${'body\n'.repeat(6_000)}${tail}`);
|
||||
});
|
||||
|
||||
it('throws parse_error when oldText is not present', async () => {
|
||||
const target = path.join(h.workspace, 'c.txt');
|
||||
await fsp.writeFile(target, 'abc');
|
||||
|
|
|
|||
|
|
@ -156,6 +156,40 @@ describe('handleAtCommand', () => {
|
|||
expect(result.toolDisplays![0].status).toBe(ToolCallStatus.Success);
|
||||
});
|
||||
|
||||
it('should attach a truncated text file larger than 10MB', async () => {
|
||||
const filePath = await createTestFile(
|
||||
path.join(testRootDir, 'large.log'),
|
||||
'x'.repeat(11 * 1024 * 1024),
|
||||
);
|
||||
|
||||
const result = await handleAtCommand({
|
||||
query: `@${filePath}`,
|
||||
config: mockConfig,
|
||||
onDebugMessage: mockOnDebugMessage,
|
||||
messageId: 626,
|
||||
signal: abortController.signal,
|
||||
});
|
||||
|
||||
const processedText = Array.isArray(result.processedQuery)
|
||||
? result.processedQuery
|
||||
.map((part) =>
|
||||
typeof part === 'string'
|
||||
? part
|
||||
: 'text' in part
|
||||
? part.text
|
||||
: JSON.stringify(part),
|
||||
)
|
||||
.join('')
|
||||
: '';
|
||||
|
||||
expect(processedText).toContain(
|
||||
'Showing lines 1-1 of at least 1 total lines',
|
||||
);
|
||||
expect(processedText).toContain('... [truncated]');
|
||||
expect(result.shouldProceed).toBe(true);
|
||||
expect(result.toolDisplays![0].status).toBe(ToolCallStatus.Success);
|
||||
});
|
||||
|
||||
it('should only allow actual temp directory paths outside the workspace', async () => {
|
||||
const tempParentDir = await fsPromises.mkdtemp(
|
||||
path.join(os.tmpdir(), 'at-command-temp-'),
|
||||
|
|
|
|||
|
|
@ -87,7 +87,6 @@ describe('StandardFileSystemService', () => {
|
|||
expect(readFileWithLineAndLimit).toHaveBeenCalledWith({
|
||||
path: '/test/file.txt',
|
||||
limit: Infinity,
|
||||
line: 0,
|
||||
});
|
||||
expect(result.content).toBe('Hello, World!');
|
||||
expect(result._meta?.bom).toBe(false);
|
||||
|
|
@ -116,6 +115,74 @@ describe('StandardFileSystemService', () => {
|
|||
expect(result._meta?.originalLineCount).toBe(100);
|
||||
});
|
||||
|
||||
it('should preserve explicit line zero for offset reads', async () => {
|
||||
vi.mocked(readFileWithLineAndLimit).mockResolvedValue({
|
||||
content: 'line 1',
|
||||
bom: false,
|
||||
encoding: 'utf-8',
|
||||
originalLineCount: 100,
|
||||
});
|
||||
|
||||
await fileSystem.readTextFile({
|
||||
path: '/test/file.txt',
|
||||
line: 0,
|
||||
});
|
||||
|
||||
expect(readFileWithLineAndLimit).toHaveBeenCalledWith({
|
||||
path: '/test/file.txt',
|
||||
limit: Infinity,
|
||||
line: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('should pass maxOutputBytes and return byte-truncation metadata', async () => {
|
||||
vi.mocked(readFileWithLineAndLimit).mockResolvedValue({
|
||||
content: 'partial',
|
||||
bom: false,
|
||||
encoding: 'utf-8',
|
||||
originalLineCount: 100,
|
||||
truncatedByBytes: true,
|
||||
});
|
||||
|
||||
const result = await fileSystem.readTextFile({
|
||||
path: '/test/file.txt',
|
||||
limit: 10,
|
||||
line: 5,
|
||||
maxOutputBytes: 128,
|
||||
});
|
||||
|
||||
expect(readFileWithLineAndLimit).toHaveBeenCalledWith({
|
||||
path: '/test/file.txt',
|
||||
limit: 10,
|
||||
line: 5,
|
||||
maxOutputBytes: 128,
|
||||
});
|
||||
expect(result._meta?.truncatedByBytes).toBe(true);
|
||||
});
|
||||
|
||||
it('should pass cached stats to readFileWithLineAndLimit', async () => {
|
||||
const stats = { size: 123 } as import('node:fs').Stats;
|
||||
vi.mocked(readFileWithLineAndLimit).mockResolvedValue({
|
||||
content: 'line 1',
|
||||
bom: false,
|
||||
encoding: 'utf-8',
|
||||
originalLineCount: 1,
|
||||
});
|
||||
|
||||
await fileSystem.readTextFile({
|
||||
path: '/test/file.txt',
|
||||
maxOutputBytes: 128,
|
||||
stats,
|
||||
});
|
||||
|
||||
expect(readFileWithLineAndLimit).toHaveBeenCalledWith({
|
||||
path: '/test/file.txt',
|
||||
limit: Infinity,
|
||||
maxOutputBytes: 128,
|
||||
stats,
|
||||
});
|
||||
});
|
||||
|
||||
it('should return encoding info for GBK file', async () => {
|
||||
vi.mocked(readFileWithLineAndLimit).mockResolvedValue({
|
||||
content: '你好世界',
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
*/
|
||||
|
||||
import os from 'node:os';
|
||||
import type { Stats } from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { globSync } from 'glob';
|
||||
import { atomicWriteFile } from '../utils/atomicFileWrite.js';
|
||||
|
|
@ -29,10 +30,26 @@ export type ReadTextFileResponse = {
|
|||
bom?: boolean;
|
||||
encoding?: string;
|
||||
originalLineCount?: number;
|
||||
originalLineCountExact?: boolean;
|
||||
lineEnding?: LineEnding;
|
||||
truncatedByBytes?: boolean;
|
||||
};
|
||||
};
|
||||
|
||||
export type CoreReadTextFileRequest = Omit<
|
||||
ReadTextFileRequest,
|
||||
'sessionId' | 'line'
|
||||
> & {
|
||||
/**
|
||||
* Core-local callers use 0-based line offsets. ACP protocol boundaries remain
|
||||
* 1-based and convert explicitly before remote calls.
|
||||
*/
|
||||
line?: number | null;
|
||||
maxOutputBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
stats?: Stats;
|
||||
};
|
||||
|
||||
/**
|
||||
* Supported file encodings for new files.
|
||||
*/
|
||||
|
|
@ -50,9 +67,7 @@ export type FileEncodingType = (typeof FileEncoding)[keyof typeof FileEncoding];
|
|||
* Interface for file system operations that may be delegated to different implementations
|
||||
*/
|
||||
export interface FileSystemService {
|
||||
readTextFile(
|
||||
params: Omit<ReadTextFileRequest, 'sessionId'>,
|
||||
): Promise<ReadTextFileResponse>;
|
||||
readTextFile(params: CoreReadTextFileRequest): Promise<ReadTextFileResponse>;
|
||||
|
||||
writeTextFile(
|
||||
params: Omit<WriteTextFileRequest, 'sessionId'>,
|
||||
|
|
@ -261,18 +276,32 @@ export function encodeTextFileContent(
|
|||
*/
|
||||
export class StandardFileSystemService implements FileSystemService {
|
||||
async readTextFile(
|
||||
params: Omit<ReadTextFileRequest, 'sessionId'>,
|
||||
params: CoreReadTextFileRequest,
|
||||
): Promise<ReadTextFileResponse> {
|
||||
const { path, limit, line } = params;
|
||||
// Use encoding-aware reader that handles BOM and non-UTF-8 encodings (e.g. GBK)
|
||||
const { content, bom, encoding, originalLineCount } =
|
||||
await readFileWithLineAndLimit({
|
||||
path,
|
||||
limit: limit ?? Number.POSITIVE_INFINITY,
|
||||
line: line || 0,
|
||||
});
|
||||
const lineEnding = detectLineEnding(content);
|
||||
return { content, _meta: { bom, encoding, originalLineCount, lineEnding } };
|
||||
const { path, limit, line, maxOutputBytes, signal, stats } = params;
|
||||
const readResult = await readFileWithLineAndLimit({
|
||||
path,
|
||||
limit: limit ?? Number.POSITIVE_INFINITY,
|
||||
...(line !== undefined && line !== null ? { line } : {}),
|
||||
...(maxOutputBytes !== undefined ? { maxOutputBytes } : {}),
|
||||
...(signal !== undefined ? { signal } : {}),
|
||||
...(stats !== undefined ? { stats } : {}),
|
||||
});
|
||||
const detectedLineEnding =
|
||||
readResult.lineEnding ?? detectLineEnding(readResult.content);
|
||||
return {
|
||||
content: readResult.content,
|
||||
_meta: {
|
||||
bom: readResult.bom,
|
||||
encoding: readResult.encoding,
|
||||
originalLineCount: readResult.originalLineCount,
|
||||
originalLineCountExact: readResult.originalLineCountExact,
|
||||
lineEnding: detectedLineEnding,
|
||||
...(readResult.truncatedByBytes !== undefined
|
||||
? { truncatedByBytes: readResult.truncatedByBytes }
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async writeTextFile(
|
||||
|
|
|
|||
|
|
@ -134,6 +134,31 @@ describe('ArtifactTool', () => {
|
|||
expect(res.error?.type).toBe(ToolErrorType.FILE_NOT_FOUND);
|
||||
});
|
||||
|
||||
it('forwards cancellation signals to source file reads', async () => {
|
||||
const file = path.join(workdir, 'page.html');
|
||||
const controller = new AbortController();
|
||||
const readTextFile = vi.fn(async () => ({ content: '<p>x</p>' }));
|
||||
const signalAwareTool = new ArtifactTool(
|
||||
{
|
||||
...makeConfig(),
|
||||
getFileSystemService: () => ({ readTextFile }),
|
||||
} as unknown as Config,
|
||||
{
|
||||
kind: 'oss',
|
||||
publish: async () => ({ id: 'x', url: 'https://h/x' }),
|
||||
},
|
||||
openSpy as unknown as UrlOpener,
|
||||
);
|
||||
|
||||
await signalAwareTool.build({ file_path: file }).execute(controller.signal);
|
||||
|
||||
expect(readTextFile).toHaveBeenCalledWith({
|
||||
path: file,
|
||||
maxOutputBytes: MAX_ARTIFACT_BYTES,
|
||||
signal: controller.signal,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns EXECUTION_FAILED when the publisher throws', async () => {
|
||||
const file = await writeFragment('page.html', '<p>x</p>');
|
||||
const failingTool = new ArtifactTool(
|
||||
|
|
@ -154,11 +179,22 @@ describe('ArtifactTool', () => {
|
|||
expect(openSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('enforces the size cap', async () => {
|
||||
it('rejects source fragments that exceed the artifact byte budget', async () => {
|
||||
const big = '<p>' + 'a'.repeat(MAX_ARTIFACT_BYTES) + '</p>';
|
||||
const file = await writeFragment('big.html', big);
|
||||
const res = await tool.build({ file_path: file }).execute(signal);
|
||||
expect(res.error?.type).toBe(ToolErrorType.FILE_TOO_LARGE);
|
||||
expect(res.error?.message).toContain('source exceeds');
|
||||
expect(res.error?.message).toContain(`${MAX_ARTIFACT_BYTES} byte limit`);
|
||||
expect(openSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('enforces the published artifact size cap', async () => {
|
||||
const almostTooBig = 'a'.repeat(MAX_ARTIFACT_BYTES - 1);
|
||||
const file = await writeFragment('wrapped-big.html', almostTooBig);
|
||||
const res = await tool.build({ file_path: file }).execute(signal);
|
||||
expect(res.error?.type).toBe(ToolErrorType.FILE_TOO_LARGE);
|
||||
expect(res.error?.message).toContain('Artifact is too large');
|
||||
expect(openSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -62,6 +62,14 @@ Set artifact.autoOpen=false in settings.json, or QWEN_ARTIFACT_NO_AUTO_OPEN=1, t
|
|||
|
||||
const debugLogger = createDebugLogger('artifact');
|
||||
|
||||
function cancelledArtifactResult(): ToolResult {
|
||||
const message = 'Artifact publishing was cancelled.';
|
||||
return {
|
||||
llmContent: message,
|
||||
returnDisplay: message,
|
||||
};
|
||||
}
|
||||
|
||||
class ArtifactToolInvocation extends BaseToolInvocation<
|
||||
ArtifactToolParams,
|
||||
ToolResult
|
||||
|
|
@ -130,11 +138,26 @@ class ArtifactToolInvocation extends BaseToolInvocation<
|
|||
// Read the fragment the model wrote.
|
||||
let fragment: string;
|
||||
try {
|
||||
const { content } = await this.config
|
||||
const { content, _meta } = await this.config
|
||||
.getFileSystemService()
|
||||
.readTextFile({ path: file_path });
|
||||
.readTextFile({
|
||||
path: file_path,
|
||||
maxOutputBytes: MAX_ARTIFACT_BYTES,
|
||||
signal,
|
||||
});
|
||||
if (_meta?.truncatedByBytes === true) {
|
||||
const message = `Artifact is too large (source exceeds the ${MAX_ARTIFACT_BYTES} byte limit). Trim the content or split it across multiple artifacts.`;
|
||||
return {
|
||||
llmContent: message,
|
||||
returnDisplay: message,
|
||||
error: { message, type: ToolErrorType.FILE_TOO_LARGE },
|
||||
};
|
||||
}
|
||||
fragment = content;
|
||||
} catch (err) {
|
||||
if (signal.aborted || isAbortError(err)) {
|
||||
return cancelledArtifactResult();
|
||||
}
|
||||
const notFound = isNodeError(err) && err.code === 'ENOENT';
|
||||
const message = notFound
|
||||
? `Artifact source file not found: ${file_path}. Write the page content to this file first.`
|
||||
|
|
@ -193,11 +216,7 @@ class ArtifactToolInvocation extends BaseToolInvocation<
|
|||
// A user-initiated cancel (Esc / aborted signal) is not a failure —
|
||||
// surface it as a cancellation rather than a publish error.
|
||||
if (signal.aborted || isAbortError(err)) {
|
||||
const message = 'Artifact publishing was cancelled.';
|
||||
return {
|
||||
llmContent: message,
|
||||
returnDisplay: message,
|
||||
};
|
||||
return cancelledArtifactResult();
|
||||
}
|
||||
const message = `Failed to publish artifact: ${getErrorMessage(err)}`;
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1313,6 +1313,39 @@ describe('EditTool', () => {
|
|||
expect(fs.readFileSync(filePath, 'utf8')).toBe('X\nline b\nline c\n');
|
||||
});
|
||||
|
||||
it('allows editing a large text file after a ranged read', async () => {
|
||||
const initialContent = [
|
||||
'target',
|
||||
'context 1',
|
||||
'context 2',
|
||||
'context 3',
|
||||
'context 4',
|
||||
'x'.repeat(11 * 1024 * 1024),
|
||||
].join('\n');
|
||||
fs.writeFileSync(filePath, initialContent, 'utf8');
|
||||
const stats = fs.statSync(filePath);
|
||||
fileReadCache.recordRead(filePath, stats, {
|
||||
full: false,
|
||||
cacheable: true,
|
||||
});
|
||||
(mockConfig.getApprovalMode as Mock).mockReturnValueOnce(
|
||||
ApprovalMode.AUTO_EDIT,
|
||||
);
|
||||
|
||||
const result = await tool
|
||||
.build({
|
||||
file_path: filePath,
|
||||
old_string: 'target',
|
||||
new_string: 'updated',
|
||||
})
|
||||
.execute(abortSignal);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(fs.readFileSync(filePath, 'utf8').startsWith('updated\n')).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects an edit when the previous read was non-cacheable (binary / pdf / image)', async () => {
|
||||
// ReadFile records every successful read into the cache,
|
||||
// including binary / PDF / image reads that produce a
|
||||
|
|
|
|||
|
|
@ -423,9 +423,8 @@ describe('ReadFileTool', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('should return error for a file that is too large', async () => {
|
||||
it('should read and truncate a text file larger than 10MB', async () => {
|
||||
const filePath = path.join(tempRootDir, 'largefile.txt');
|
||||
// 11MB of content exceeds 10MB limit
|
||||
const largeContent = 'x'.repeat(11 * 1024 * 1024);
|
||||
await fsp.writeFile(filePath, largeContent, 'utf-8');
|
||||
const params: ReadFileToolParams = { file_path: filePath };
|
||||
|
|
@ -435,10 +434,28 @@ describe('ReadFileTool', () => {
|
|||
>;
|
||||
|
||||
const result = await invocation.execute(abortSignal);
|
||||
expect(result).toHaveProperty('error');
|
||||
expect(result.error?.type).toBe(ToolErrorType.FILE_TOO_LARGE);
|
||||
expect(result.error?.message).toContain(
|
||||
'File size exceeds the 10MB limit',
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Read lines 1-1 of at least 1 from largefile.txt (truncated)',
|
||||
);
|
||||
expect(result.llmContent).toContain(
|
||||
'Showing lines 1-1 of at least 1 total lines',
|
||||
);
|
||||
expect(result.llmContent).toContain('... [truncated]');
|
||||
});
|
||||
|
||||
it('should propagate an aborted signal before reading', async () => {
|
||||
const filePath = path.join(tempRootDir, 'abort.txt');
|
||||
await fsp.writeFile(filePath, 'content', 'utf-8');
|
||||
const invocation = tool.build({ file_path: filePath }) as ToolInvocation<
|
||||
ReadFileToolParams,
|
||||
ToolResult
|
||||
>;
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(invocation.execute(controller.signal)).rejects.toThrow(
|
||||
/abort/i,
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -132,7 +132,7 @@ class ReadFileToolInvocation extends BaseToolInvocation<
|
|||
return 'ask';
|
||||
}
|
||||
|
||||
async execute(): Promise<ToolResult> {
|
||||
async execute(signal: AbortSignal): Promise<ToolResult> {
|
||||
const absPath = path.resolve(this.params.file_path);
|
||||
const projectRoot = this.config.getTargetDir();
|
||||
// Auto-memory files (AGENTS.md and friends under the auto-memory
|
||||
|
|
@ -206,6 +206,7 @@ class ReadFileToolInvocation extends BaseToolInvocation<
|
|||
offset: this.params.offset,
|
||||
limit: this.params.limit,
|
||||
pages: this.params.pages,
|
||||
signal,
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -284,7 +285,9 @@ class ReadFileToolInvocation extends BaseToolInvocation<
|
|||
) {
|
||||
const [start, end] = result.linesShown!;
|
||||
const total = result.originalLineCount!;
|
||||
llmContent = `Showing lines ${start}-${end} of ${total} total lines.\n\n---\n\n${result.llmContent}`;
|
||||
const totalLabel =
|
||||
result.originalLineCountExact === false ? `at least ${total}` : total;
|
||||
llmContent = `Showing lines ${start}-${end} of ${totalLabel} total lines.\n\n---\n\n${result.llmContent}`;
|
||||
} else {
|
||||
llmContent = result.llmContent || '';
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,11 +29,14 @@ import {
|
|||
processSingleFileContent,
|
||||
detectBOM,
|
||||
decodeBufferWithEncodingInfo,
|
||||
readFileWithLineAndLimit,
|
||||
readFileWithEncoding,
|
||||
readFileWithEncodingInfo,
|
||||
detectFileEncoding,
|
||||
fileExists,
|
||||
} from './fileUtils.js';
|
||||
import { iconvEncode } from './iconvHelper.js';
|
||||
import { LargeNonUtf8TextError } from './read-text-range.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { StandardFileSystemService } from '../services/fileSystemService.js';
|
||||
import { ToolErrorType } from '../tools/tool-error.js';
|
||||
|
|
@ -1993,6 +1996,149 @@ describe('fileUtils', () => {
|
|||
expect(result.linesShown).toEqual([1, 2]);
|
||||
});
|
||||
|
||||
it('should preserve default full file-system reads for large text files', async () => {
|
||||
const content = `head\n${'x'.repeat(11 * 1024 * 1024)}`;
|
||||
actualNodeFs.writeFileSync(testTextFilePath, content);
|
||||
|
||||
const result = await fsService.readTextFile({ path: testTextFilePath });
|
||||
|
||||
expect(result.content).toBe(content);
|
||||
expect(result._meta?.originalLineCount).toBe(2);
|
||||
expect(result._meta?.originalLineCountExact).toBe(true);
|
||||
expect(result._meta?.truncatedByBytes).not.toBe(true);
|
||||
});
|
||||
|
||||
it('should stream explicit offset reads for large text files', async () => {
|
||||
actualNodeFs.writeFileSync(
|
||||
testTextFilePath,
|
||||
`skip\n${'line\n'.repeat(3 * 1024 * 1024)}`,
|
||||
);
|
||||
|
||||
const result = await fsService.readTextFile({
|
||||
path: testTextFilePath,
|
||||
line: 1,
|
||||
});
|
||||
|
||||
expect(result.content.startsWith('line\n')).toBe(true);
|
||||
expect(result.content.startsWith('skip\n')).toBe(false);
|
||||
expect(result._meta?.originalLineCountExact).toBe(false);
|
||||
expect(result._meta?.truncatedByBytes).toBe(true);
|
||||
});
|
||||
|
||||
it('should preserve unbounded explicit line-zero reads below the large-file threshold', async () => {
|
||||
const content = `head\n${'body\n'.repeat(6_000)}tail\n`;
|
||||
actualNodeFs.writeFileSync(testTextFilePath, content);
|
||||
|
||||
const result = await fsService.readTextFile({
|
||||
path: testTextFilePath,
|
||||
line: 0,
|
||||
});
|
||||
|
||||
expect(result.content).toBe(content);
|
||||
expect(result._meta?.truncatedByBytes).not.toBe(true);
|
||||
});
|
||||
|
||||
it('should enforce maxOutputBytes for default file-system reads below the large-file threshold', async () => {
|
||||
actualNodeFs.writeFileSync(testTextFilePath, 'x'.repeat(100));
|
||||
|
||||
const result = await fsService.readTextFile({
|
||||
path: testTextFilePath,
|
||||
maxOutputBytes: 10,
|
||||
});
|
||||
|
||||
expect(result.content).toBe('x'.repeat(10));
|
||||
expect(result._meta?.originalLineCount).toBe(1);
|
||||
expect(result._meta?.originalLineCountExact).toBe(true);
|
||||
expect(result._meta?.truncatedByBytes).toBe(true);
|
||||
});
|
||||
|
||||
it('should propagate large non-UTF-8 errors through bounded reads', async () => {
|
||||
const gbkLine = iconvEncode('中文日志行\n', 'gbk');
|
||||
const gbkChunk = Buffer.concat(
|
||||
Array.from({ length: 1024 }, () => gbkLine),
|
||||
);
|
||||
const repeatCount = Math.ceil((11 * 1024 * 1024) / gbkChunk.length);
|
||||
actualNodeFs.writeFileSync(
|
||||
testTextFilePath,
|
||||
Buffer.concat(Array.from({ length: repeatCount }, () => gbkChunk)),
|
||||
);
|
||||
|
||||
await expect(
|
||||
readFileWithLineAndLimit({
|
||||
path: testTextFilePath,
|
||||
limit: 10,
|
||||
maxOutputBytes: 10_000,
|
||||
}),
|
||||
).rejects.toThrow(LargeNonUtf8TextError);
|
||||
});
|
||||
|
||||
it('should propagate aborts from unbounded full reads', async () => {
|
||||
actualNodeFs.writeFileSync(testTextFilePath, 'hello\nworld');
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
readFileWithLineAndLimit({
|
||||
path: testTextFilePath,
|
||||
limit: Number.POSITIVE_INFINITY,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow(/abort/i);
|
||||
});
|
||||
|
||||
it('should propagate aborts before large unbounded full reads', async () => {
|
||||
actualNodeFs.writeFileSync(
|
||||
testTextFilePath,
|
||||
'x'.repeat(11 * 1024 * 1024),
|
||||
);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
readFileWithLineAndLimit({
|
||||
path: testTextFilePath,
|
||||
limit: Number.POSITIVE_INFINITY,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow(/abort/i);
|
||||
});
|
||||
|
||||
it('should use provided stats when reading with line and byte limits', async () => {
|
||||
actualNodeFs.writeFileSync(testTextFilePath, 'hello\nworld');
|
||||
const stats = actualNodeFs.statSync(testTextFilePath);
|
||||
const statSpy = vi
|
||||
.spyOn(fs.promises, 'stat')
|
||||
.mockRejectedValueOnce(new Error('unexpected stat'));
|
||||
|
||||
try {
|
||||
const result = await readFileWithLineAndLimit({
|
||||
path: testTextFilePath,
|
||||
limit: 1,
|
||||
maxOutputBytes: 100,
|
||||
stats,
|
||||
});
|
||||
|
||||
expect(result.content).toBe('hello');
|
||||
expect(statSpy).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
statSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('should not byte-truncate multibyte text before the character limit', async () => {
|
||||
const content = '你'.repeat(1000);
|
||||
actualNodeFs.writeFileSync(testTextFilePath, content, 'utf-8');
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
mockConfig,
|
||||
);
|
||||
|
||||
expect(result.llmContent).toBe(content);
|
||||
expect(result.returnDisplay).toBe('');
|
||||
expect(result.isTruncated).toBe(false);
|
||||
});
|
||||
|
||||
it('should truncate long lines in text files', async () => {
|
||||
const longLine = 'a'.repeat(2500);
|
||||
actualNodeFs.writeFileSync(
|
||||
|
|
@ -2073,31 +2219,152 @@ describe('fileUtils', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('should return an error if the file size exceeds 10MB', async () => {
|
||||
// Create a small test file
|
||||
actualNodeFs.writeFileSync(testTextFilePath, 'test content');
|
||||
it('should read large text files through bounded truncation instead of the 10MB gate', async () => {
|
||||
const lines = Array.from(
|
||||
{ length: 65_000 },
|
||||
(_, index) => `Line ${index + 1} ${'x'.repeat(180)}`,
|
||||
);
|
||||
actualNodeFs.writeFileSync(testTextFilePath, lines.join('\n'));
|
||||
|
||||
// Spy on fs.promises.stat to return a large file size
|
||||
const statSpy = vi.spyOn(fs.promises, 'stat').mockResolvedValueOnce({
|
||||
size: 11 * 1024 * 1024,
|
||||
isDirectory: () => false,
|
||||
isFile: () => true,
|
||||
} as fs.Stats);
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
mockConfig,
|
||||
);
|
||||
|
||||
try {
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
mockConfig,
|
||||
);
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.llmContent).toContain('Line 1');
|
||||
expect(result.returnDisplay).toContain('Read lines 1-');
|
||||
expect(result.isTruncated).toBe(true);
|
||||
expect(result.originalLineCount).toBeGreaterThanOrEqual(
|
||||
result.linesShown?.[1] ?? 1,
|
||||
);
|
||||
expect(result.originalLineCount).toBeLessThan(65_000);
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
expect(result.linesShown?.[0]).toBe(1);
|
||||
});
|
||||
|
||||
expect(result.error).toContain('File size exceeds the 10MB limit');
|
||||
expect(result.returnDisplay).toContain(
|
||||
'File size exceeds the 10MB limit',
|
||||
);
|
||||
expect(result.llmContent).toContain('File size exceeds the 10MB limit');
|
||||
} finally {
|
||||
statSpy.mockRestore();
|
||||
}
|
||||
it('should stream large text files when line truncation is disabled', async () => {
|
||||
actualNodeFs.writeFileSync(
|
||||
testTextFilePath,
|
||||
'x'.repeat(11 * 1024 * 1024),
|
||||
);
|
||||
const noLineLimitConfig = {
|
||||
...mockConfig,
|
||||
getTruncateToolOutputLines: () => Number.POSITIVE_INFINITY,
|
||||
} as unknown as Config;
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
noLineLimitConfig,
|
||||
);
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(typeof result.llmContent).toBe('string');
|
||||
expect(result.llmContent).toContain('... [truncated]');
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Read lines 1-1 of at least 1 from test.txt (truncated)',
|
||||
);
|
||||
expect(result.isTruncated).toBe(true);
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
});
|
||||
|
||||
it('should mark byte truncation metadata without character truncation', async () => {
|
||||
actualNodeFs.writeFileSync(testTextFilePath, 'visible');
|
||||
const byteTruncatedConfig = {
|
||||
...mockConfig,
|
||||
getTruncateToolOutputThreshold: () => Number.POSITIVE_INFINITY,
|
||||
getFileSystemService: () => ({
|
||||
readTextFile: vi.fn().mockResolvedValue({
|
||||
content: 'visible',
|
||||
_meta: {
|
||||
originalLineCount: 1,
|
||||
originalLineCountExact: false,
|
||||
truncatedByBytes: true,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
} as unknown as Config;
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
byteTruncatedConfig,
|
||||
);
|
||||
|
||||
expect(typeof result.llmContent).toBe('string');
|
||||
const llmContent = result.llmContent as string;
|
||||
expect(llmContent).toBe('visible\n... [truncated]');
|
||||
expect(llmContent.match(/\.\.\. \[truncated\]/g)).toHaveLength(1);
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Read lines 1-1 of at least 1 from test.txt (truncated)',
|
||||
);
|
||||
expect(result.isTruncated).toBe(true);
|
||||
});
|
||||
|
||||
it('should use selected range as a lower bound when large file metadata is missing', async () => {
|
||||
actualNodeFs.writeFileSync(
|
||||
testTextFilePath,
|
||||
'x'.repeat(11 * 1024 * 1024),
|
||||
);
|
||||
const missingMetadataConfig = {
|
||||
...mockConfig,
|
||||
getFileSystemService: () => ({
|
||||
readTextFile: vi.fn().mockResolvedValue({
|
||||
content: 'visible\nnext',
|
||||
}),
|
||||
}),
|
||||
} as unknown as Config;
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
missingMetadataConfig,
|
||||
{ offset: 9, limit: 2 },
|
||||
);
|
||||
|
||||
expect(result.originalLineCount).toBe(11);
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
expect(result.returnDisplay).toBe(
|
||||
'Read lines 10-11 of at least 11 from test.txt',
|
||||
);
|
||||
});
|
||||
|
||||
it('should preserve disabled output truncation for large text files', async () => {
|
||||
const byteLength = 11 * 1024 * 1024;
|
||||
actualNodeFs.writeFileSync(testTextFilePath, 'x'.repeat(byteLength));
|
||||
const noCharacterLimitConfig = {
|
||||
...mockConfig,
|
||||
getTruncateToolOutputThreshold: () => Number.POSITIVE_INFINITY,
|
||||
} as unknown as Config;
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testTextFilePath,
|
||||
noCharacterLimitConfig,
|
||||
);
|
||||
|
||||
expect(typeof result.llmContent).toBe('string');
|
||||
const llmContent = result.llmContent as string;
|
||||
expect(llmContent).toHaveLength(byteLength);
|
||||
expect(llmContent).not.toContain('... [truncated]');
|
||||
expect(result.returnDisplay).toBe('');
|
||||
expect(result.isTruncated).toBe(false);
|
||||
});
|
||||
|
||||
it('should still return an error if an inline media file exceeds 10MB', async () => {
|
||||
mockMimeGetType.mockReturnValue('image/png');
|
||||
actualNodeFs.writeFileSync(
|
||||
testImageFilePath,
|
||||
Buffer.alloc(11 * 1024 * 1024),
|
||||
);
|
||||
|
||||
const result = await processSingleFileContent(
|
||||
testImageFilePath,
|
||||
mockConfig,
|
||||
);
|
||||
|
||||
expect(result.error).toContain('File size exceeds the 10MB limit');
|
||||
expect(result.returnDisplay).toContain(
|
||||
'File size exceeds the 10MB limit',
|
||||
);
|
||||
expect(result.llmContent).toContain('File size exceeds the 10MB limit');
|
||||
});
|
||||
|
||||
it('should allow explicit page ranges above the full-PDF text-extraction size cap', async () => {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import { ToolErrorType } from '../tools/tool-error.js';
|
|||
import { BINARY_EXTENSIONS } from './ignorePatterns.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { createDebugLogger } from './debugLogger.js';
|
||||
import { getErrorMessage, isNodeError } from './errors.js';
|
||||
import { getErrorMessage, isAbortError, isNodeError } from './errors.js';
|
||||
import type { InputModalities } from '../core/contentGenerator.js';
|
||||
import { detectEncodingFromBuffer } from './systemEncoding.js';
|
||||
import {
|
||||
|
|
@ -35,6 +35,11 @@ import {
|
|||
shouldRequirePDFPageRange,
|
||||
} from './pdf.js';
|
||||
import { readNotebookWithMetadata } from './notebook.js';
|
||||
import { readTextRange } from './read-text-range.js';
|
||||
import {
|
||||
DEFAULT_RANGE_READ_BYTES,
|
||||
TEXT_RANGE_FAST_PATH_MAX_SIZE,
|
||||
} from './text-range-constants.js';
|
||||
|
||||
const debugLogger = createDebugLogger('FILE_UTILS');
|
||||
|
||||
|
|
@ -273,9 +278,13 @@ function bomEncodingToName(bomEncoding: UnicodeEncoding): string {
|
|||
*/
|
||||
export async function readFileWithEncodingInfo(
|
||||
filePath: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FileReadResult> {
|
||||
// Read the file once; detect BOM and decode from the single buffer.
|
||||
const full = await fs.promises.readFile(filePath);
|
||||
const full = await fs.promises.readFile(
|
||||
filePath,
|
||||
signal === undefined ? undefined : { signal },
|
||||
);
|
||||
return decodeBufferWithEncodingInfo(full);
|
||||
}
|
||||
|
||||
|
|
@ -299,14 +308,42 @@ export async function readFileWithLineAndLimit(params: {
|
|||
path: string;
|
||||
limit: number;
|
||||
line?: number;
|
||||
maxOutputBytes?: number;
|
||||
signal?: AbortSignal;
|
||||
stats?: import('node:fs').Stats;
|
||||
}): Promise<{
|
||||
content: string;
|
||||
bom?: boolean;
|
||||
encoding?: string;
|
||||
originalLineCount: number;
|
||||
originalLineCountExact?: boolean;
|
||||
lineEnding?: 'crlf' | 'lf';
|
||||
truncatedByBytes?: boolean;
|
||||
}> {
|
||||
const { path: filePath, limit, line } = params;
|
||||
const { content, encoding, bom } = await readFileWithEncodingInfo(filePath);
|
||||
const { path: filePath, limit, line, maxOutputBytes, signal } = params;
|
||||
const stats = params.stats ?? (await fs.promises.stat(filePath));
|
||||
if (
|
||||
(line !== undefined && line > 0) ||
|
||||
Number.isFinite(limit) ||
|
||||
maxOutputBytes !== undefined
|
||||
) {
|
||||
return readTextRange({
|
||||
path: filePath,
|
||||
offset: line || 0,
|
||||
limit,
|
||||
maxOutputBytes: normalizeRangeReadByteLimit(maxOutputBytes),
|
||||
stats,
|
||||
...(signal !== undefined ? { signal } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
signal?.throwIfAborted();
|
||||
|
||||
const { content, encoding, bom } = await readFileWithEncodingInfo(
|
||||
filePath,
|
||||
signal,
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
const lines = content.split('\n');
|
||||
const originalLineCount = lines.length;
|
||||
const startLine = line || 0;
|
||||
|
|
@ -321,6 +358,7 @@ export async function readFileWithLineAndLimit(params: {
|
|||
bom,
|
||||
encoding,
|
||||
originalLineCount,
|
||||
originalLineCountExact: true,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -829,6 +867,7 @@ export interface ProcessedFileReadResult {
|
|||
error?: string; // Optional error message for the LLM if file processing failed
|
||||
errorType?: ToolErrorType; // Structured error type
|
||||
originalLineCount?: number; // For text files, the total number of lines in the original file
|
||||
originalLineCountExact?: boolean; // False when a large range read stopped before EOF
|
||||
isTruncated?: boolean; // Indicates if displayed content was truncated
|
||||
linesShown?: [number, number]; // For text files [startLine, endLine] (1-based for display)
|
||||
/**
|
||||
|
|
@ -867,6 +906,7 @@ export interface ProcessSingleFileContentOptions {
|
|||
* sets this after deciding the vision bridge should handle the image.
|
||||
*/
|
||||
preserveUnsupportedImage?: boolean;
|
||||
signal?: AbortSignal;
|
||||
/**
|
||||
* Large full-PDF text fallback returns a tool error by default. `@`-attached
|
||||
* PDFs use `reference` so the model gets guidance without a failed read.
|
||||
|
|
@ -945,10 +985,12 @@ export async function processSingleFileContent(
|
|||
limit,
|
||||
pages,
|
||||
preserveUnsupportedImage = false,
|
||||
signal,
|
||||
largePdfBehavior = 'error',
|
||||
} = options;
|
||||
const rootDirectory = config.getTargetDir();
|
||||
try {
|
||||
signal?.throwIfAborted();
|
||||
let stats: import('node:fs').Stats;
|
||||
try {
|
||||
// Async stat doubles as the existence check — ENOENT is handled below
|
||||
|
|
@ -1101,7 +1143,7 @@ export async function processSingleFileContent(
|
|||
};
|
||||
}
|
||||
}
|
||||
if (fileSizeInMB > 9.9 && !willExtractPdfText) {
|
||||
if (fileSizeInMB > 9.9 && !willExtractPdfText && fileType !== 'text') {
|
||||
return {
|
||||
llmContent: 'File size exceeds the 10MB limit.',
|
||||
returnDisplay: 'File size exceeds the 10MB limit.',
|
||||
|
|
@ -1182,11 +1224,25 @@ export async function processSingleFileContent(
|
|||
path: filePath,
|
||||
limit: limit ?? config.getTruncateToolOutputLines(),
|
||||
line: offset,
|
||||
maxOutputBytes: getRangeReadByteLimit(config),
|
||||
stats,
|
||||
...(signal !== undefined ? { signal } : {}),
|
||||
});
|
||||
const originalLineCount =
|
||||
_meta?.originalLineCount ?? (await countFileLines(filePath));
|
||||
const selectedLines = content.split('\n').map((line) => line.trimEnd());
|
||||
const startLine = offset || 0;
|
||||
const selectedLineCount =
|
||||
content.length === 0 ? 0 : selectedLines.length;
|
||||
const hasOriginalLineCount = _meta?.originalLineCount !== undefined;
|
||||
const originalLineCount =
|
||||
_meta?.originalLineCount ??
|
||||
(stats.size >= TEXT_RANGE_FAST_PATH_MAX_SIZE
|
||||
? startLine + selectedLineCount
|
||||
: await countFileLines(filePath));
|
||||
const originalLineCountExact =
|
||||
_meta?.originalLineCountExact === false
|
||||
? false
|
||||
: hasOriginalLineCount ||
|
||||
stats.size < TEXT_RANGE_FAST_PATH_MAX_SIZE;
|
||||
const configCharLimit = config.getTruncateToolOutputThreshold();
|
||||
|
||||
// Apply character limit truncation
|
||||
|
|
@ -1227,17 +1283,38 @@ export async function processSingleFileContent(
|
|||
linesIncluded = selectedLines.length;
|
||||
}
|
||||
|
||||
if (_meta?.truncatedByBytes === true) {
|
||||
const marker = '... [truncated]';
|
||||
if (!llmContent.endsWith(marker)) {
|
||||
const prefix = Number.isFinite(configCharLimit)
|
||||
? llmContent.slice(
|
||||
0,
|
||||
Math.max(configCharLimit - marker.length - 1, 0),
|
||||
)
|
||||
: llmContent;
|
||||
const separator =
|
||||
prefix.length === 0 || prefix.endsWith('\n') ? '' : '\n';
|
||||
llmContent = prefix + separator + marker;
|
||||
}
|
||||
contentLengthTruncated = true;
|
||||
}
|
||||
|
||||
const actualEndLine = startLine + linesIncluded;
|
||||
const contentRangeTruncated =
|
||||
startLine > 0 || actualEndLine < originalLineCount;
|
||||
startLine > 0 ||
|
||||
!originalLineCountExact ||
|
||||
actualEndLine < originalLineCount;
|
||||
const isTruncated = contentRangeTruncated || contentLengthTruncated;
|
||||
const lineCountLabel = originalLineCountExact
|
||||
? `${originalLineCount}`
|
||||
: `at least ${originalLineCount}`;
|
||||
|
||||
// By default, return nothing to streamline the common case of a successful read_file.
|
||||
let returnDisplay = '';
|
||||
if (isTruncated) {
|
||||
returnDisplay = `Read lines ${
|
||||
startLine + 1
|
||||
}-${actualEndLine} of ${originalLineCount} from ${relativePathForDisplay}`;
|
||||
}-${actualEndLine} of ${lineCountLabel} from ${relativePathForDisplay}`;
|
||||
if (contentLengthTruncated) {
|
||||
returnDisplay += ' (truncated)';
|
||||
}
|
||||
|
|
@ -1248,6 +1325,7 @@ export async function processSingleFileContent(
|
|||
returnDisplay,
|
||||
isTruncated,
|
||||
originalLineCount,
|
||||
originalLineCountExact,
|
||||
linesShown: [startLine + 1, actualEndLine],
|
||||
stats,
|
||||
};
|
||||
|
|
@ -1385,6 +1463,9 @@ export async function processSingleFileContent(
|
|||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted || isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const errorMessage = getErrorMessage(error);
|
||||
const displayPath = path
|
||||
.relative(rootDirectory, filePath)
|
||||
|
|
@ -1398,6 +1479,28 @@ export async function processSingleFileContent(
|
|||
}
|
||||
}
|
||||
|
||||
function getRangeReadByteLimit(config: Config): number {
|
||||
const charLimit = config.getTruncateToolOutputThreshold();
|
||||
if (charLimit === Number.POSITIVE_INFINITY) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
if (!Number.isFinite(charLimit)) {
|
||||
return DEFAULT_RANGE_READ_BYTES;
|
||||
}
|
||||
// Leave enough byte headroom for UTF-8 text so the character budget remains
|
||||
// the primary truncation control for normal source/log content.
|
||||
return Math.max(DEFAULT_RANGE_READ_BYTES, Math.floor(charLimit) * 4);
|
||||
}
|
||||
|
||||
function normalizeRangeReadByteLimit(maxOutputBytes: number | undefined) {
|
||||
if (maxOutputBytes === Number.POSITIVE_INFINITY) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
return typeof maxOutputBytes === 'number' && Number.isFinite(maxOutputBytes)
|
||||
? maxOutputBytes
|
||||
: DEFAULT_RANGE_READ_BYTES;
|
||||
}
|
||||
|
||||
export async function fileExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await fsPromises.access(filePath, fs.constants.F_OK);
|
||||
|
|
|
|||
|
|
@ -428,8 +428,7 @@ describe('readPathFromWorkspace', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it('should return an error string for files exceeding the size limit', async () => {
|
||||
// Mock a file slightly larger than the 10MB limit defined in fileUtils.ts
|
||||
it('should truncate text files exceeding 10MB instead of failing', async () => {
|
||||
const largeContent = 'a'.repeat(11 * 1024 * 1024); // 11MB
|
||||
mock({
|
||||
[CWD]: {
|
||||
|
|
@ -442,7 +441,7 @@ describe('readPathFromWorkspace', () => {
|
|||
const config = createMockConfig(CWD, [], mockFileService);
|
||||
const result = await readPathFromWorkspace('large.txt', config);
|
||||
const textResult = result[0] as string;
|
||||
// The error message comes directly from processSingleFileContent
|
||||
expect(textResult).toBe('File size exceeds the 10MB limit.');
|
||||
expect(textResult).toContain('a'.repeat(100));
|
||||
expect(textResult).toContain('... [truncated]');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
294
packages/core/src/utils/read-text-range.test.ts
Normal file
294
packages/core/src/utils/read-text-range.test.ts
Normal file
|
|
@ -0,0 +1,294 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2026 Qwen Team
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import * as fs from 'node:fs/promises';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { iconvEncode } from './iconvHelper.js';
|
||||
import { LargeNonUtf8TextError, readTextRange } from './read-text-range.js';
|
||||
|
||||
describe('readTextRange', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'read-text-range-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function writeFile(
|
||||
name: string,
|
||||
data: string | Buffer,
|
||||
): Promise<string> {
|
||||
const filePath = path.join(tempDir, name);
|
||||
await fs.writeFile(filePath, data);
|
||||
return filePath;
|
||||
}
|
||||
|
||||
function largeUtf8Lines(lineCount: number): string {
|
||||
return Array.from(
|
||||
{ length: lineCount },
|
||||
(_, index) => `line-${index + 1} ${'x'.repeat(180)}`,
|
||||
).join('\n');
|
||||
}
|
||||
|
||||
it('preserves split newline semantics on the fast path', async () => {
|
||||
const emptyPath = await writeFile('empty.txt', '');
|
||||
await expect(
|
||||
readTextRange({
|
||||
path: emptyPath,
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
maxOutputBytes: 100,
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
content: '',
|
||||
originalLineCount: 1,
|
||||
truncatedByBytes: false,
|
||||
});
|
||||
|
||||
const trailingPath = await writeFile('trailing.txt', 'a\n');
|
||||
await expect(
|
||||
readTextRange({
|
||||
path: trailingPath,
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
maxOutputBytes: 100,
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
content: 'a\n',
|
||||
originalLineCount: 2,
|
||||
truncatedByBytes: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('streams a large UTF-8 file and returns the requested range', async () => {
|
||||
const filePath = await writeFile('large.log', largeUtf8Lines(65_000));
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
offset: 42_000,
|
||||
limit: 3,
|
||||
maxOutputBytes: 10_000,
|
||||
});
|
||||
|
||||
expect(result.content.split('\n')).toEqual([
|
||||
expect.stringContaining('line-42001'),
|
||||
expect.stringContaining('line-42002'),
|
||||
expect.stringContaining('line-42003'),
|
||||
]);
|
||||
expect(result.originalLineCount).toBeGreaterThanOrEqual(42_004);
|
||||
expect(result.originalLineCount).toBeLessThan(65_000);
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
expect(result.encoding).toBe('utf-8');
|
||||
expect(result.bom).toBe(false);
|
||||
expect(result.truncatedByBytes).toBe(false);
|
||||
});
|
||||
|
||||
it('streams a large UTF-8 file from the beginning when no range is provided', async () => {
|
||||
const filePath = await writeFile('large.log', largeUtf8Lines(65_000));
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
maxOutputBytes: 10_000,
|
||||
});
|
||||
|
||||
expect(result.content).toContain('line-1');
|
||||
expect(result.content).toContain('line-2');
|
||||
expect(result.content).not.toContain('line-65000');
|
||||
expect(result.originalLineCount).toBeGreaterThan(1);
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
expect(result.truncatedByBytes).toBe(true);
|
||||
});
|
||||
|
||||
it('returns empty content when a large UTF-8 range starts beyond EOF', async () => {
|
||||
const filePath = await writeFile('large.log', largeUtf8Lines(65_000));
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
offset: 100_000,
|
||||
limit: 10,
|
||||
maxOutputBytes: 10_000,
|
||||
});
|
||||
|
||||
expect(result.content).toBe('');
|
||||
expect(result.originalLineCount).toBe(65_000);
|
||||
expect(result.originalLineCountExact).toBe(true);
|
||||
expect(result.truncatedByBytes).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves CRLF content and line-ending metadata for large files', async () => {
|
||||
const content = Array.from(
|
||||
{ length: 65_000 },
|
||||
(_, index) => `line-${index + 1} ${'x'.repeat(180)}`,
|
||||
).join('\r\n');
|
||||
const filePath = await writeFile('crlf.log', content);
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
offset: 1,
|
||||
limit: 2,
|
||||
maxOutputBytes: 10_000,
|
||||
});
|
||||
|
||||
expect(result.content).toContain('\r\n');
|
||||
expect(result.content.split('\n')[0]).toMatch(/\r$/);
|
||||
expect(result.lineEnding).toBe('crlf');
|
||||
expect(result.originalLineCount).toBeGreaterThanOrEqual(4);
|
||||
expect(result.originalLineCount).toBeLessThan(65_000);
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
});
|
||||
|
||||
it('detects CRLF when the pair crosses a stream chunk boundary', async () => {
|
||||
const highWaterMark = 512 * 1024;
|
||||
const firstChunk = `${'a'.repeat(highWaterMark - 1)}\r`;
|
||||
const body = Buffer.concat([
|
||||
Buffer.from(firstChunk),
|
||||
Buffer.from('\nsecond\n'),
|
||||
Buffer.alloc(11 * 1024 * 1024, 'x'),
|
||||
]);
|
||||
const filePath = await writeFile('split-crlf.log', body);
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
offset: 0,
|
||||
limit: 2,
|
||||
maxOutputBytes: highWaterMark + 100,
|
||||
});
|
||||
|
||||
expect(result.lineEnding).toBe('crlf');
|
||||
expect(result.content).toContain('\r\nsecond');
|
||||
});
|
||||
|
||||
it('strips UTF-8 BOM from large file content and reports BOM metadata', async () => {
|
||||
const body = largeUtf8Lines(65_000);
|
||||
const filePath = await writeFile(
|
||||
'bom.log',
|
||||
Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(body)]),
|
||||
);
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
maxOutputBytes: 10_000,
|
||||
});
|
||||
|
||||
expect(result.content.charCodeAt(0)).not.toBe(0xfeff);
|
||||
expect(result.content).toContain('line-1');
|
||||
expect(result.bom).toBe(true);
|
||||
expect(result.encoding).toBe('utf-8');
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
});
|
||||
|
||||
it('does not split a UTF-8 character when byte-truncating', async () => {
|
||||
const filePath = await writeFile('emoji.txt', `a🙂b`);
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
maxOutputBytes: 4,
|
||||
});
|
||||
|
||||
expect(result.content).toBe('a');
|
||||
expect(result.content).not.toContain('\uFFFD');
|
||||
expect(result.truncatedByBytes).toBe(true);
|
||||
expect(result.originalLineCountExact).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects large non-UTF-8 files with a targeted error', async () => {
|
||||
const gbkLine = iconvEncode('中文日志行\n', 'gbk');
|
||||
const repeatCount = Math.ceil((11 * 1024 * 1024) / gbkLine.length);
|
||||
const filePath = await writeFile(
|
||||
'gbk.log',
|
||||
Buffer.concat(Array.from({ length: repeatCount }, () => gbkLine)),
|
||||
);
|
||||
|
||||
await expect(
|
||||
readTextRange({
|
||||
path: filePath,
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
maxOutputBytes: 10_000,
|
||||
}),
|
||||
).rejects.toThrow(LargeNonUtf8TextError);
|
||||
});
|
||||
|
||||
it('rejects large files with invalid UTF-8 beyond the encoding sample', async () => {
|
||||
const mostlyAsciiThenGbk = Buffer.concat([
|
||||
Buffer.alloc(9 * 1024, 'a'),
|
||||
iconvEncode('你好', 'gbk'),
|
||||
Buffer.alloc(11 * 1024 * 1024, 'b'),
|
||||
]);
|
||||
const filePath = await writeFile('late-gbk.log', mostlyAsciiThenGbk);
|
||||
|
||||
const promise = readTextRange({
|
||||
path: filePath,
|
||||
offset: 0,
|
||||
limit: 500,
|
||||
maxOutputBytes: 20_000,
|
||||
});
|
||||
|
||||
await expect(promise).rejects.toThrow(LargeNonUtf8TextError);
|
||||
await expect(promise).rejects.toThrow(/invalid UTF-8 byte sequence/);
|
||||
await expect(promise).rejects.toMatchObject({ reason: 'invalid-utf8' });
|
||||
});
|
||||
|
||||
it('bounds selected output for a large single-line file', async () => {
|
||||
const filePath = await writeFile(
|
||||
'single-line.log',
|
||||
'x'.repeat(11 * 1024 * 1024),
|
||||
);
|
||||
|
||||
const result = await readTextRange({
|
||||
path: filePath,
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
maxOutputBytes: 1024,
|
||||
});
|
||||
|
||||
expect(result.content).toBe('x'.repeat(1024));
|
||||
expect(result.originalLineCount).toBe(1);
|
||||
expect(result.originalLineCountExact).toBe(false);
|
||||
expect(result.truncatedByBytes).toBe(true);
|
||||
});
|
||||
|
||||
it('propagates aborts before reading large files', async () => {
|
||||
const filePath = await writeFile('large.log', largeUtf8Lines(65_000));
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
readTextRange({
|
||||
path: filePath,
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
maxOutputBytes: 10_000,
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow(/abort/i);
|
||||
});
|
||||
|
||||
it('propagates aborts while streaming large files', async () => {
|
||||
const filePath = await writeFile('large.log', largeUtf8Lines(80_000));
|
||||
const controller = new AbortController();
|
||||
const promise = readTextRange({
|
||||
path: filePath,
|
||||
offset: 70_000,
|
||||
limit: 10,
|
||||
maxOutputBytes: 10_000,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
setTimeout(() => controller.abort(), 0);
|
||||
|
||||
await expect(promise).rejects.toThrow(/abort/i);
|
||||
});
|
||||
});
|
||||
281
packages/core/src/utils/read-text-range.ts
Normal file
281
packages/core/src/utils/read-text-range.ts
Normal file
|
|
@ -0,0 +1,281 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2026 Qwen Team
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { createReadStream, type Stats } from 'node:fs';
|
||||
import { stat } from 'node:fs/promises';
|
||||
import { TextDecoder } from 'node:util';
|
||||
import { detectFileEncoding, readFileWithEncodingInfo } from './fileUtils.js';
|
||||
import { isUtf8CompatibleEncoding } from './iconvHelper.js';
|
||||
import {
|
||||
DEFAULT_RANGE_READ_BYTES,
|
||||
TEXT_RANGE_FAST_PATH_MAX_SIZE,
|
||||
} from './text-range-constants.js';
|
||||
|
||||
export interface ReadTextRangeRequest {
|
||||
path: string;
|
||||
offset?: number;
|
||||
limit?: number;
|
||||
maxOutputBytes: number;
|
||||
signal?: AbortSignal;
|
||||
stats?: Stats;
|
||||
}
|
||||
|
||||
export interface ReadTextRangeResult {
|
||||
content: string;
|
||||
originalLineCount: number;
|
||||
encoding?: string;
|
||||
bom?: boolean;
|
||||
lineEnding?: 'crlf' | 'lf';
|
||||
originalLineCountExact: boolean;
|
||||
truncatedByBytes: boolean;
|
||||
}
|
||||
|
||||
export class LargeNonUtf8TextError extends Error {
|
||||
constructor(
|
||||
readonly encoding: string,
|
||||
readonly reason?: 'invalid-utf8',
|
||||
) {
|
||||
super(
|
||||
reason === 'invalid-utf8'
|
||||
? 'Large text file contains invalid UTF-8 byte sequence beyond the initial encoding sample. Convert or extract a smaller UTF-8 slice and read that instead.'
|
||||
: `Large non-UTF-8 text files are not supported for streaming reads (detected ${encoding}). Convert or extract a smaller UTF-8 slice and read that instead.`,
|
||||
);
|
||||
this.name = 'LargeNonUtf8TextError';
|
||||
}
|
||||
}
|
||||
|
||||
export async function readTextRange(
|
||||
request: ReadTextRangeRequest,
|
||||
): Promise<ReadTextRangeResult> {
|
||||
request.signal?.throwIfAborted();
|
||||
const stats = request.stats ?? (await stat(request.path));
|
||||
const maxOutputBytes = normalizeMaxBytes(request.maxOutputBytes);
|
||||
|
||||
if (stats.size < TEXT_RANGE_FAST_PATH_MAX_SIZE) {
|
||||
const { content, encoding, bom } = await readFileWithEncodingInfo(
|
||||
request.path,
|
||||
request.signal,
|
||||
);
|
||||
request.signal?.throwIfAborted();
|
||||
const range = sliceDecodedContent(
|
||||
content,
|
||||
request.offset,
|
||||
request.limit,
|
||||
maxOutputBytes,
|
||||
);
|
||||
return {
|
||||
...range,
|
||||
encoding,
|
||||
bom,
|
||||
lineEnding: detectLineEndingFromContent(content),
|
||||
};
|
||||
}
|
||||
|
||||
return readLargeUtf8Range(request, maxOutputBytes);
|
||||
}
|
||||
|
||||
function normalizeMaxBytes(maxOutputBytes: number): number {
|
||||
if (maxOutputBytes === Number.POSITIVE_INFINITY) {
|
||||
return Number.POSITIVE_INFINITY;
|
||||
}
|
||||
if (!Number.isFinite(maxOutputBytes)) {
|
||||
return DEFAULT_RANGE_READ_BYTES;
|
||||
}
|
||||
return Math.max(0, Math.floor(maxOutputBytes));
|
||||
}
|
||||
|
||||
function sliceDecodedContent(
|
||||
content: string,
|
||||
offset: number | undefined,
|
||||
limit: number | undefined,
|
||||
maxOutputBytes: number,
|
||||
): Pick<
|
||||
ReadTextRangeResult,
|
||||
| 'content'
|
||||
| 'originalLineCount'
|
||||
| 'originalLineCountExact'
|
||||
| 'truncatedByBytes'
|
||||
> {
|
||||
const lines = content.split('\n');
|
||||
const originalLineCount = lines.length;
|
||||
const start = Math.min(Math.max(0, offset ?? 0), originalLineCount);
|
||||
const end =
|
||||
limit === undefined
|
||||
? originalLineCount
|
||||
: Math.min(start + Math.max(0, limit), originalLineCount);
|
||||
const selected = lines.slice(start, end).join('\n');
|
||||
const truncated = truncateUtf8(selected, maxOutputBytes);
|
||||
|
||||
return {
|
||||
content: truncated.content,
|
||||
originalLineCount,
|
||||
originalLineCountExact: true,
|
||||
truncatedByBytes: truncated.truncated,
|
||||
};
|
||||
}
|
||||
|
||||
async function readLargeUtf8Range(
|
||||
request: ReadTextRangeRequest,
|
||||
maxOutputBytes: number,
|
||||
): Promise<ReadTextRangeResult> {
|
||||
const encoding = await detectFileEncoding(request.path);
|
||||
if (!isUtf8CompatibleEncoding(encoding)) {
|
||||
throw new LargeNonUtf8TextError(encoding);
|
||||
}
|
||||
|
||||
const offset = Math.max(0, request.offset ?? 0);
|
||||
const endLine =
|
||||
offset + Math.max(0, request.limit ?? Number.POSITIVE_INFINITY);
|
||||
let currentLine = 0;
|
||||
let output = '';
|
||||
let outputBytes = 0;
|
||||
let truncatedByBytes = false;
|
||||
let bom = false;
|
||||
let firstChunk = true;
|
||||
let lineEnding: 'crlf' | 'lf' = 'lf';
|
||||
let previousChunkEndedWithCR = false;
|
||||
let originalLineCountExact = true;
|
||||
let stoppedEarly = false;
|
||||
const decoder = new TextDecoder('utf-8', {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
});
|
||||
|
||||
const stream = createReadStream(request.path, {
|
||||
highWaterMark: 512 * 1024,
|
||||
signal: request.signal,
|
||||
});
|
||||
|
||||
function appendSelected(fragment: string): void {
|
||||
if (fragment.length === 0 || truncatedByBytes) {
|
||||
return;
|
||||
}
|
||||
|
||||
const available = maxOutputBytes - outputBytes;
|
||||
if (available <= 0) {
|
||||
truncatedByBytes = true;
|
||||
return;
|
||||
}
|
||||
|
||||
const truncated = truncateUtf8(fragment, available);
|
||||
output += truncated.content;
|
||||
outputBytes += Buffer.byteLength(truncated.content, 'utf8');
|
||||
if (truncated.truncated) {
|
||||
truncatedByBytes = true;
|
||||
}
|
||||
}
|
||||
|
||||
function isSelectedLine(): boolean {
|
||||
return currentLine >= offset && currentLine < endLine;
|
||||
}
|
||||
|
||||
function decodeUtf8Chunk(
|
||||
chunk?: Buffer,
|
||||
options?: TextDecodeOptions,
|
||||
): string {
|
||||
try {
|
||||
return decoder.decode(chunk, options);
|
||||
} catch {
|
||||
throw new LargeNonUtf8TextError(encoding, 'invalid-utf8');
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
for await (const rawChunk of stream) {
|
||||
request.signal?.throwIfAborted();
|
||||
let chunk = decodeUtf8Chunk(rawChunk as Buffer, { stream: true });
|
||||
if (firstChunk) {
|
||||
firstChunk = false;
|
||||
if (chunk.charCodeAt(0) === 0xfeff) {
|
||||
chunk = chunk.slice(1);
|
||||
bom = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
(previousChunkEndedWithCR && chunk.startsWith('\n')) ||
|
||||
chunk.includes('\r\n')
|
||||
) {
|
||||
lineEnding = 'crlf';
|
||||
}
|
||||
previousChunkEndedWithCR = chunk.endsWith('\r');
|
||||
|
||||
let start = 0;
|
||||
let newline = chunk.indexOf('\n', start);
|
||||
while (newline !== -1) {
|
||||
if (isSelectedLine()) {
|
||||
appendSelected(chunk.slice(start, newline));
|
||||
if (currentLine + 1 < endLine) {
|
||||
appendSelected('\n');
|
||||
}
|
||||
}
|
||||
currentLine++;
|
||||
start = newline + 1;
|
||||
if (currentLine >= endLine || truncatedByBytes) {
|
||||
originalLineCountExact = false;
|
||||
stoppedEarly = true;
|
||||
break;
|
||||
}
|
||||
newline = chunk.indexOf('\n', start);
|
||||
}
|
||||
|
||||
if (start < chunk.length && isSelectedLine()) {
|
||||
appendSelected(chunk.slice(start));
|
||||
}
|
||||
if (currentLine >= endLine || truncatedByBytes) {
|
||||
originalLineCountExact = false;
|
||||
stoppedEarly = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
stream.destroy();
|
||||
}
|
||||
|
||||
if (!stoppedEarly) {
|
||||
decodeUtf8Chunk();
|
||||
}
|
||||
|
||||
return {
|
||||
content: output,
|
||||
originalLineCount: currentLine + 1,
|
||||
encoding: 'utf-8',
|
||||
bom,
|
||||
lineEnding,
|
||||
originalLineCountExact,
|
||||
truncatedByBytes,
|
||||
};
|
||||
}
|
||||
|
||||
function truncateUtf8(
|
||||
content: string,
|
||||
maxBytes: number,
|
||||
): { content: string; truncated: boolean } {
|
||||
const bytes = Buffer.byteLength(content, 'utf8');
|
||||
if (bytes <= maxBytes) {
|
||||
return { content, truncated: false };
|
||||
}
|
||||
if (maxBytes <= 0) {
|
||||
return { content: '', truncated: true };
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(content, 'utf8');
|
||||
let end = Math.min(maxBytes, buffer.length);
|
||||
// `end` is the first excluded byte. If it lands inside a multi-byte UTF-8
|
||||
// sequence, the byte at `end` is a continuation byte, so back up until the
|
||||
// prefix ends before the incomplete character.
|
||||
while (end > 0 && (buffer[end] & 0xc0) === 0x80) {
|
||||
end--;
|
||||
}
|
||||
return {
|
||||
content: buffer.subarray(0, end).toString('utf8'),
|
||||
truncated: true,
|
||||
};
|
||||
}
|
||||
|
||||
function detectLineEndingFromContent(content: string): 'crlf' | 'lf' {
|
||||
return content.includes('\r\n') ? 'crlf' : 'lf';
|
||||
}
|
||||
|
|
@ -147,6 +147,22 @@ describe('readManyFiles', () => {
|
|||
expect(content).toContain('--- End of content ---');
|
||||
});
|
||||
|
||||
it('should include truncated large text files instead of reporting a size error', async () => {
|
||||
const relativePath = 'large.log';
|
||||
const absolutePath = path.join(tempRootDir, relativePath);
|
||||
await fs.writeFile(absolutePath, 'x'.repeat(11 * 1024 * 1024), 'utf-8');
|
||||
const mockConfig = createMockConfig(tempRootDir);
|
||||
|
||||
const result = await readManyFiles(mockConfig, { paths: [relativePath] });
|
||||
|
||||
const content = contentToString(result.contentParts);
|
||||
expect(content).toContain('Showing lines 1-1 of at least 1 total lines');
|
||||
expect(content).toContain('... [truncated]');
|
||||
expect(result.files).toHaveLength(1);
|
||||
expect(result.files[0]!.error).toBeUndefined();
|
||||
expect(result.files[0]!.filePath).toBe(absolutePath);
|
||||
});
|
||||
|
||||
it('should include truncated notebooks that do not expose text line ranges', async () => {
|
||||
const relativePath = 'large.ipynb';
|
||||
const absolutePath = path.join(tempRootDir, relativePath);
|
||||
|
|
@ -276,6 +292,20 @@ describe('readManyFiles', () => {
|
|||
expect(content).not.toContain('Content of file1.txt');
|
||||
});
|
||||
|
||||
it('should propagate aborts before reading a directory', async () => {
|
||||
await createTestFile('mydir', 'file1.txt');
|
||||
const mockConfig = createMockConfig(tempRootDir);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
readManyFiles(mockConfig, {
|
||||
paths: ['mydir'],
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow(/abort/i);
|
||||
});
|
||||
|
||||
it('should handle directory with trailing slash', async () => {
|
||||
await createTestFile('mydir', 'file1.txt');
|
||||
const mockConfig = createMockConfig(tempRootDir);
|
||||
|
|
@ -439,6 +469,20 @@ describe('readManyFiles', () => {
|
|||
});
|
||||
|
||||
describe('per-file error surfacing', () => {
|
||||
it('should propagate aborts from file reads instead of returning an error message', async () => {
|
||||
const { relativePath } = await createTestFile('cancel.txt');
|
||||
const mockConfig = createMockConfig(tempRootDir);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(
|
||||
readManyFiles(mockConfig, {
|
||||
paths: [relativePath],
|
||||
signal: controller.signal,
|
||||
}),
|
||||
).rejects.toThrow(/abort/i);
|
||||
});
|
||||
|
||||
it('should surface processSingleFileContent errors instead of silently skipping the file', async () => {
|
||||
// Trigger the >10MB file-size error path in processSingleFileContent.
|
||||
const relativePath = 'huge.bin';
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import * as fs from 'node:fs';
|
|||
import * as path from 'node:path';
|
||||
import type { Part, PartListUnion } from '@google/genai';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { getErrorMessage } from './errors.js';
|
||||
import { getErrorMessage, isAbortError } from './errors.js';
|
||||
import type { ProcessedFileReadResult } from './fileUtils.js';
|
||||
import {
|
||||
isCacheableReadResult,
|
||||
|
|
@ -104,7 +104,11 @@ export async function readManyFiles(
|
|||
config: Config,
|
||||
options: ReadManyFilesOptions,
|
||||
): Promise<ReadManyFilesResult> {
|
||||
const { paths: inputPatterns, preserveUnsupportedImageForBridge } = options;
|
||||
const {
|
||||
paths: inputPatterns,
|
||||
preserveUnsupportedImageForBridge,
|
||||
signal,
|
||||
} = options;
|
||||
|
||||
const seenFiles = new Set<string>();
|
||||
const contentParts: Part[] = [];
|
||||
|
|
@ -114,6 +118,7 @@ export async function readManyFiles(
|
|||
const projectRoot = config.getProjectRoot();
|
||||
|
||||
for (const rawPattern of inputPatterns) {
|
||||
signal?.throwIfAborted();
|
||||
const normalizedPattern = rawPattern.replace(/\\/g, '/');
|
||||
const fullPath = path.resolve(projectRoot, normalizedPattern);
|
||||
const stats = fs.existsSync(fullPath) ? fs.statSync(fullPath) : null;
|
||||
|
|
@ -122,6 +127,7 @@ export async function readManyFiles(
|
|||
const { contentParts: dirParts, info } = await readDirectory(
|
||||
config,
|
||||
fullPath,
|
||||
signal,
|
||||
);
|
||||
contentParts.push(...dirParts);
|
||||
files.push(info);
|
||||
|
|
@ -134,6 +140,7 @@ export async function readManyFiles(
|
|||
config,
|
||||
fullPath,
|
||||
preserveUnsupportedImageForBridge,
|
||||
signal,
|
||||
);
|
||||
if (readResult) {
|
||||
contentParts.push(...readResult.contentParts);
|
||||
|
|
@ -142,6 +149,9 @@ export async function readManyFiles(
|
|||
}
|
||||
}
|
||||
} catch (error) {
|
||||
if (signal?.aborted || isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const errorMessage = `Error during file search: ${getErrorMessage(error)}`;
|
||||
return {
|
||||
contentParts: [errorMessage],
|
||||
|
|
@ -165,11 +175,14 @@ export async function readManyFiles(
|
|||
async function readDirectory(
|
||||
config: Config,
|
||||
directoryPath: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ contentParts: Part[]; info: FileReadInfo }> {
|
||||
signal?.throwIfAborted();
|
||||
const structure = await getFolderStructure(directoryPath, {
|
||||
fileService: config.getFileService(),
|
||||
fileFilteringOptions: config.getFileFilteringOptions(),
|
||||
});
|
||||
signal?.throwIfAborted();
|
||||
|
||||
const contentParts: Part[] = [
|
||||
{ text: `\nContent from ${directoryPath}:\n` },
|
||||
|
|
@ -190,10 +203,12 @@ async function readFileContent(
|
|||
config: Config,
|
||||
filePath: string,
|
||||
preserveUnsupportedImage = false,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ contentParts: Part[]; info: FileReadInfo } | null> {
|
||||
try {
|
||||
const fileReadResult = await processSingleFileContent(filePath, config, {
|
||||
preserveUnsupportedImage,
|
||||
...(signal !== undefined ? { signal } : {}),
|
||||
largePdfBehavior: 'reference',
|
||||
});
|
||||
|
||||
|
|
@ -233,7 +248,11 @@ async function readFileContent(
|
|||
) {
|
||||
const [start, end] = fileReadResult.linesShown!;
|
||||
const total = fileReadResult.originalLineCount!;
|
||||
fileContentForLlm = `Showing lines ${start}-${end} of ${total} total lines.\n---\n${fileReadResult.llmContent}`;
|
||||
const totalLabel =
|
||||
fileReadResult.originalLineCountExact === false
|
||||
? `at least ${total}`
|
||||
: total;
|
||||
fileContentForLlm = `Showing lines ${start}-${end} of ${totalLabel} total lines.\n---\n${fileReadResult.llmContent}`;
|
||||
} else {
|
||||
fileContentForLlm = fileReadResult.llmContent;
|
||||
}
|
||||
|
|
@ -258,7 +277,10 @@ async function readFileContent(
|
|||
isDirectory: false,
|
||||
},
|
||||
};
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (signal?.aborted || isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
8
packages/core/src/utils/text-range-constants.ts
Normal file
8
packages/core/src/utils/text-range-constants.ts
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2026 Qwen Team
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
export const TEXT_RANGE_FAST_PATH_MAX_SIZE = 10 * 1024 * 1024;
|
||||
export const DEFAULT_RANGE_READ_BYTES = 25_000;
|
||||
Loading…
Add table
Add a link
Reference in a new issue