diff --git a/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx b/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx index bbebc1361..a5931119b 100644 --- a/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx +++ b/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx @@ -6,7 +6,7 @@ import type React from 'react'; import { useMemo } from 'react'; -import { Box, Text } from 'ink'; +import { Box } from 'ink'; import type { IndividualToolCallDisplay } from '../../types.js'; import { ToolCallStatus } from '../../types.js'; import { ToolMessage } from './ToolMessage.js'; @@ -136,13 +136,6 @@ export const ToolGroupMessage: React.FC = ({ contentWidth={innerWidth} /> )} - {tool.outputFile && ( - - - Output too long and was saved to: {tool.outputFile} - - - )} ); })} diff --git a/packages/cli/src/ui/hooks/useReactToolScheduler.ts b/packages/cli/src/ui/hooks/useReactToolScheduler.ts index 56992f678..966c6adff 100644 --- a/packages/cli/src/ui/hooks/useReactToolScheduler.ts +++ b/packages/cli/src/ui/hooks/useReactToolScheduler.ts @@ -252,7 +252,6 @@ export function mapToDisplay( status: mapCoreStatusToDisplayStatus(trackedCall.status), resultDisplay: trackedCall.response.resultDisplay, confirmationDetails: undefined, - outputFile: trackedCall.response.outputFile, }; case 'error': return { diff --git a/packages/cli/src/ui/types.ts b/packages/cli/src/ui/types.ts index d2483f371..8f4c41f6d 100644 --- a/packages/cli/src/ui/types.ts +++ b/packages/cli/src/ui/types.ts @@ -68,7 +68,6 @@ export interface IndividualToolCallDisplay { confirmationDetails: ToolCallConfirmationDetails | undefined; renderOutputAsMarkdown?: boolean; ptyId?: number; - outputFile?: string; } export interface CompressionProps { diff --git a/packages/core/src/core/coreToolScheduler.test.ts b/packages/core/src/core/coreToolScheduler.test.ts index 145e8ace1..3411fff50 100644 --- a/packages/core/src/core/coreToolScheduler.test.ts +++ b/packages/core/src/core/coreToolScheduler.test.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import type { Mock } from 'vitest'; import type { Config, @@ -29,7 +29,6 @@ import type { ToolCall, WaitingToolCall } from './coreToolScheduler.js'; import { CoreToolScheduler, convertToFunctionResponse, - truncateAndSaveToFile, } from './coreToolScheduler.js'; import type { Part, PartListUnion } from '@google/genai'; import { @@ -37,13 +36,6 @@ import { MockTool, MOCK_TOOL_SHOULD_CONFIRM_EXECUTE, } from '../test-utils/mock-tool.js'; -import * as fs from 'node:fs/promises'; -import * as path from 'node:path'; - -vi.mock('fs/promises', () => ({ - writeFile: vi.fn(), -})); - class TestApprovalTool extends BaseDeclarativeTool<{ id: string }, ToolResult> { static readonly Name = 'testApprovalTool'; @@ -2290,227 +2282,6 @@ describe('CoreToolScheduler Sequential Execution', () => { }); }); -describe('truncateAndSaveToFile', () => { - const mockWriteFile = vi.mocked(fs.writeFile); - const THRESHOLD = 40_000; - const TRUNCATE_LINES = 1000; - - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('should return content unchanged if below threshold', async () => { - const content = 'Short content'; - const callId = 'test-call-id'; - const projectTempDir = '/tmp'; - - const result = await truncateAndSaveToFile( - content, - callId, - projectTempDir, - THRESHOLD, - TRUNCATE_LINES, - ); - - expect(result).toEqual({ content }); - expect(mockWriteFile).not.toHaveBeenCalled(); - }); - - it('should truncate content by lines when content has many lines', async () => { - // Create content that exceeds 100,000 character threshold with many lines - const lines = Array(2000).fill('x'.repeat(100)); // 100 chars per line * 2000 lines = 200,000 chars - const content = lines.join('\n'); - const callId = 'test-call-id'; - const projectTempDir = '/tmp'; - - mockWriteFile.mockResolvedValue(undefined); - - const result = await truncateAndSaveToFile( - content, - callId, - projectTempDir, - THRESHOLD, - TRUNCATE_LINES, - ); - - expect(result.outputFile).toBe( - path.join(projectTempDir, `${callId}.output`), - ); - expect(mockWriteFile).toHaveBeenCalledWith( - path.join(projectTempDir, `${callId}.output`), - content, - ); - - // Should contain the first and last lines with 1/5 head and 4/5 tail - const head = Math.floor(TRUNCATE_LINES / 5); - const beginning = lines.slice(0, head); - const end = lines.slice(-(TRUNCATE_LINES - head)); - const expectedTruncated = - beginning.join('\n') + '\n... [CONTENT TRUNCATED] ...\n' + end.join('\n'); - - expect(result.content).toContain( - 'Tool output was too large and has been truncated', - ); - expect(result.content).toContain('Truncated part of the output:'); - expect(result.content).toContain(expectedTruncated); - }); - - it('should wrap and truncate content when content has few but long lines', async () => { - const content = 'a'.repeat(200_000); // A single very long line - const callId = 'test-call-id'; - const projectTempDir = '/tmp'; - const wrapWidth = 120; - - mockWriteFile.mockResolvedValue(undefined); - - // Manually wrap the content to generate the expected file content - const wrappedLines: string[] = []; - for (let i = 0; i < content.length; i += wrapWidth) { - wrappedLines.push(content.substring(i, i + wrapWidth)); - } - const expectedFileContent = wrappedLines.join('\n'); - - const result = await truncateAndSaveToFile( - content, - callId, - projectTempDir, - THRESHOLD, - TRUNCATE_LINES, - ); - - expect(result.outputFile).toBe( - path.join(projectTempDir, `${callId}.output`), - ); - // Check that the file was written with the wrapped content - expect(mockWriteFile).toHaveBeenCalledWith( - path.join(projectTempDir, `${callId}.output`), - expectedFileContent, - ); - - // Should contain the first and last lines with 1/5 head and 4/5 tail of the wrapped content - const head = Math.floor(TRUNCATE_LINES / 5); - const beginning = wrappedLines.slice(0, head); - const end = wrappedLines.slice(-(TRUNCATE_LINES - head)); - const expectedTruncated = - beginning.join('\n') + '\n... [CONTENT TRUNCATED] ...\n' + end.join('\n'); - expect(result.content).toContain( - 'Tool output was too large and has been truncated', - ); - expect(result.content).toContain('Truncated part of the output:'); - expect(result.content).toContain(expectedTruncated); - }); - - it('should handle file write errors gracefully', async () => { - const content = 'a'.repeat(2_000_000); - const callId = 'test-call-id'; - const projectTempDir = '/tmp'; - - mockWriteFile.mockRejectedValue(new Error('File write failed')); - - const result = await truncateAndSaveToFile( - content, - callId, - projectTempDir, - THRESHOLD, - TRUNCATE_LINES, - ); - - expect(result.outputFile).toBeUndefined(); - expect(result.content).toContain( - '[Note: Could not save full output to file]', - ); - expect(mockWriteFile).toHaveBeenCalled(); - }); - - it('should save to correct file path with call ID', async () => { - const content = 'a'.repeat(200_000); - const callId = 'unique-call-123'; - const projectTempDir = '/custom/temp/dir'; - const wrapWidth = 120; - - mockWriteFile.mockResolvedValue(undefined); - - // Manually wrap the content to generate the expected file content - const wrappedLines: string[] = []; - for (let i = 0; i < content.length; i += wrapWidth) { - wrappedLines.push(content.substring(i, i + wrapWidth)); - } - const expectedFileContent = wrappedLines.join('\n'); - - const result = await truncateAndSaveToFile( - content, - callId, - projectTempDir, - THRESHOLD, - TRUNCATE_LINES, - ); - - const expectedPath = path.join(projectTempDir, `${callId}.output`); - expect(result.outputFile).toBe(expectedPath); - expect(mockWriteFile).toHaveBeenCalledWith( - expectedPath, - expectedFileContent, - ); - }); - - it('should include helpful instructions in truncated message', async () => { - const content = 'a'.repeat(2_000_000); - const callId = 'test-call-id'; - const projectTempDir = '/tmp'; - - mockWriteFile.mockResolvedValue(undefined); - - const result = await truncateAndSaveToFile( - content, - callId, - projectTempDir, - THRESHOLD, - TRUNCATE_LINES, - ); - - expect(result.content).toContain( - 'Tool output was too large and has been truncated', - ); - expect(result.content).toContain('The full output has been saved to:'); - expect(result.content).toContain( - 'To read the complete output, use the read_file tool with the absolute file path above', - ); - expect(result.content).toContain( - 'The truncated output below shows the beginning and end of the content', - ); - }); - - it('should sanitize callId to prevent path traversal', async () => { - const content = 'a'.repeat(200_000); - const callId = '../../../../../etc/passwd'; - const projectTempDir = '/tmp/safe_dir'; - const wrapWidth = 120; - - mockWriteFile.mockResolvedValue(undefined); - - // Manually wrap the content to generate the expected file content - const wrappedLines: string[] = []; - for (let i = 0; i < content.length; i += wrapWidth) { - wrappedLines.push(content.substring(i, i + wrapWidth)); - } - const expectedFileContent = wrappedLines.join('\n'); - - await truncateAndSaveToFile( - content, - callId, - projectTempDir, - THRESHOLD, - TRUNCATE_LINES, - ); - - const expectedPath = path.join(projectTempDir, 'passwd.output'); - expect(mockWriteFile).toHaveBeenCalledWith( - expectedPath, - expectedFileContent, - ); - }); -}); - describe('CoreToolScheduler plan mode with ask_user_question', () => { function createAskUserQuestionMockTool() { let wasAnswered = false; diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index 60e0e1e0e..4f0f361e6 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -25,12 +25,8 @@ import { ToolConfirmationOutcome, ApprovalMode, logToolCall, - ReadFileTool, ToolErrorType, ToolCallEvent, - ShellTool, - logToolOutputTruncated, - ToolOutputTruncatedEvent, InputFormat, Kind, SkillTool, @@ -49,8 +45,6 @@ import { modifyWithEditor, } from '../tools/modifiable-tool.js'; import * as Diff from 'diff'; -import * as fs from 'node:fs/promises'; -import * as path from 'node:path'; import { doesToolInvocationMatch } from '../utils/tool-utils.js'; import levenshtein from 'fast-levenshtein'; import { getPlanModeSystemReminder } from './prompts.js'; @@ -306,67 +300,6 @@ const createErrorResponse = ( contentLength: error.message.length, }); -export async function truncateAndSaveToFile( - content: string, - callId: string, - projectTempDir: string, - threshold: number, - truncateLines: number, -): Promise<{ content: string; outputFile?: string }> { - if (content.length <= threshold) { - return { content }; - } - - let lines = content.split('\n'); - let fileContent = content; - - // If the content is long but has few lines, wrap it to enable line-based truncation. - if (lines.length <= truncateLines) { - const wrapWidth = 120; // A reasonable width for wrapping. - const wrappedLines: string[] = []; - for (const line of lines) { - if (line.length > wrapWidth) { - for (let i = 0; i < line.length; i += wrapWidth) { - wrappedLines.push(line.substring(i, i + wrapWidth)); - } - } else { - wrappedLines.push(line); - } - } - lines = wrappedLines; - fileContent = lines.join('\n'); - } - - const head = Math.floor(truncateLines / 5); - const beginning = lines.slice(0, head); - const end = lines.slice(-(truncateLines - head)); - const truncatedContent = - beginning.join('\n') + '\n... [CONTENT TRUNCATED] ...\n' + end.join('\n'); - - // Sanitize callId to prevent path traversal. - const safeFileName = `${path.basename(callId)}.output`; - const outputFile = path.join(projectTempDir, safeFileName); - try { - await fs.writeFile(outputFile, fileContent); - - return { - content: `Tool output was too large and has been truncated. -The full output has been saved to: ${outputFile} -To read the complete output, use the ${ReadFileTool.Name} tool with the absolute file path above. -The truncated output below shows the beginning and end of the content. The marker '... [CONTENT TRUNCATED] ...' indicates where content was removed. -This allows you to efficiently examine different parts of the output without loading the entire file. -Truncated part of the output: -${truncatedContent}`, - outputFile, - }; - } catch (_error) { - return { - content: - truncatedContent + `\n[Note: Could not save full output to file]`, - }; - } -} - interface CoreToolSchedulerOptions { config: Config; outputUpdateHandler?: OutputUpdateHandler; @@ -1213,42 +1146,9 @@ export class CoreToolScheduler { } if (toolResult.error === undefined) { - let content = toolResult.llmContent; - let outputFile: string | undefined = undefined; + const content = toolResult.llmContent; const contentLength = typeof content === 'string' ? content.length : undefined; - if ( - typeof content === 'string' && - toolName === ShellTool.Name && - this.config.getTruncateToolOutputThreshold() > 0 && - this.config.getTruncateToolOutputLines() > 0 - ) { - const originalContentLength = content.length; - const threshold = this.config.getTruncateToolOutputThreshold(); - const lines = this.config.getTruncateToolOutputLines(); - const truncatedResult = await truncateAndSaveToFile( - content, - callId, - this.config.storage.getProjectTempDir(), - threshold, - lines, - ); - content = truncatedResult.content; - outputFile = truncatedResult.outputFile; - - if (outputFile) { - logToolOutputTruncated( - this.config, - new ToolOutputTruncatedEvent(scheduledCall.request.prompt_id, { - toolName, - originalContentLength, - truncatedContentLength: content.length, - threshold, - lines, - }), - ); - } - } const response = convertToFunctionResponse(toolName, callId, content); const successResponse: ToolCallResponseInfo = { @@ -1257,7 +1157,6 @@ export class CoreToolScheduler { resultDisplay: toolResult.returnDisplay, error: undefined, errorType: undefined, - outputFile, contentLength, }; this.setStatusInternal(callId, 'success', successResponse); diff --git a/packages/core/src/core/nonInteractiveToolExecutor.test.ts b/packages/core/src/core/nonInteractiveToolExecutor.test.ts index 989b61c37..29bcf99b8 100644 --- a/packages/core/src/core/nonInteractiveToolExecutor.test.ts +++ b/packages/core/src/core/nonInteractiveToolExecutor.test.ts @@ -94,7 +94,6 @@ describe('executeToolCall', () => { callId: 'call1', error: undefined, errorType: undefined, - outputFile: undefined, resultDisplay: 'Success!', contentLength: typeof toolResult.llmContent === 'string' @@ -299,7 +298,6 @@ describe('executeToolCall', () => { callId: 'call6', error: undefined, errorType: undefined, - outputFile: undefined, resultDisplay: 'Image processed', contentLength: undefined, responseParts: [ diff --git a/packages/core/src/core/turn.ts b/packages/core/src/core/turn.ts index 08f379d68..2037081ff 100644 --- a/packages/core/src/core/turn.ts +++ b/packages/core/src/core/turn.ts @@ -109,7 +109,6 @@ export interface ToolCallResponseInfo { resultDisplay: ToolResultDisplay | undefined; error: Error | undefined; errorType: ToolErrorType | undefined; - outputFile?: string | undefined; contentLength?: number; } diff --git a/packages/core/src/tools/shell.test.ts b/packages/core/src/tools/shell.test.ts index d03509451..fae07ae91 100644 --- a/packages/core/src/tools/shell.test.ts +++ b/packages/core/src/tools/shell.test.ts @@ -61,7 +61,10 @@ describe('ShellTool', () => { .mockReturnValue(createMockWorkspaceContext('/test/dir')), storage: { getUserSkillsDir: vi.fn().mockReturnValue('/test/dir/.qwen/skills'), + getProjectTempDir: vi.fn().mockReturnValue('/tmp/qwen-temp'), }, + getTruncateToolOutputThreshold: vi.fn().mockReturnValue(0), + getTruncateToolOutputLines: vi.fn().mockReturnValue(0), getGeminiClient: vi.fn(), getGitCoAuthor: vi.fn().mockReturnValue({ enabled: true, diff --git a/packages/core/src/tools/shell.ts b/packages/core/src/tools/shell.ts index 01a9ac5cf..d8c205d67 100644 --- a/packages/core/src/tools/shell.ts +++ b/packages/core/src/tools/shell.ts @@ -27,6 +27,9 @@ import { } from './tools.js'; import { getErrorMessage } from '../utils/errors.js'; import { summarizeToolOutput } from '../utils/summarizer.js'; +import { truncateAndSaveToFile } from '../utils/truncation.js'; +import { logToolOutputTruncated } from '../telemetry/loggers.js'; +import { ToolOutputTruncatedEvent } from '../telemetry/types.js'; import type { ShellExecutionConfig, ShellOutputEvent, @@ -378,6 +381,43 @@ export class ShellToolInvocation extends BaseToolInvocation< } } + // Truncate large output and save full content to a temp file. + const truncateThreshold = this.config.getTruncateToolOutputThreshold(); + const truncateLines = this.config.getTruncateToolOutputLines(); + if ( + typeof llmContent === 'string' && + truncateThreshold > 0 && + truncateLines > 0 + ) { + const originalContentLength = llmContent.length; + const fileName = `shell_${crypto.randomBytes(6).toString('hex')}`; + const truncatedResult = await truncateAndSaveToFile( + llmContent, + fileName, + this.config.storage.getProjectTempDir(), + truncateThreshold, + truncateLines, + ); + + if (truncatedResult.outputFile) { + llmContent = truncatedResult.content; + returnDisplayMessage += + (returnDisplayMessage ? '\n' : '') + + `Output too long and was saved to: ${truncatedResult.outputFile}`; + + logToolOutputTruncated( + this.config, + new ToolOutputTruncatedEvent('', { + toolName: ShellTool.Name, + originalContentLength, + truncatedContentLength: truncatedResult.content.length, + threshold: truncateThreshold, + lines: truncateLines, + }), + ); + } + } + const summarizeConfig = this.config.getSummarizeToolOutputConfig(); const executionError = result.error ? { diff --git a/packages/core/src/utils/truncation.test.ts b/packages/core/src/utils/truncation.test.ts new file mode 100644 index 000000000..26db3256d --- /dev/null +++ b/packages/core/src/utils/truncation.test.ts @@ -0,0 +1,319 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, it, expect, vi, beforeEach } from 'vitest'; +import { truncateAndSaveToFile } from './truncation.js'; +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; + +vi.mock('node:fs/promises'); + +describe('truncateAndSaveToFile', () => { + const mockWriteFile = vi.mocked(fs.writeFile); + const THRESHOLD = 40_000; + const TRUNCATE_LINES = 1000; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('should return content unchanged if below both threshold and line limit', async () => { + const content = 'Short content'; + const fileName = 'test-file'; + const projectTempDir = '/tmp'; + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + expect(result).toEqual({ content }); + expect(mockWriteFile).not.toHaveBeenCalled(); + }); + + it('should truncate when line limit exceeded even if under character threshold', async () => { + // 2000 short lines, well under the 40,000 char threshold + const lines = Array(2000).fill('short'); + const content = lines.join('\n'); // ~12,000 chars, under THRESHOLD + const fileName = 'test-file'; + const projectTempDir = '/tmp'; + + expect(content.length).toBeLessThan(THRESHOLD); + + mockWriteFile.mockResolvedValue(undefined); + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + expect(result.outputFile).toBe( + path.join(projectTempDir, `${fileName}.output`), + ); + + const head = Math.floor(TRUNCATE_LINES / 5); + const beginning = lines.slice(0, head); + const end = lines.slice(-(TRUNCATE_LINES - head)); + const expectedTruncated = + beginning.join('\n') + + '\n\n---\n... [CONTENT TRUNCATED] ...\n---\n\n' + + end.join('\n'); + + expect(result.content).toContain( + 'Tool output was too large and has been truncated', + ); + expect(result.content).toContain(expectedTruncated); + }); + + it('should reduce effective lines when line content would exceed character threshold', async () => { + // 2000 lines of 100 chars each = 200,000 chars, well over THRESHOLD (40,000) + // Even after truncating to TRUNCATE_LINES (1000), that's 100,000 chars — still over. + // The effective line count should be reduced to fit within the threshold. + const lines = Array(2000).fill('x'.repeat(100)); + const content = lines.join('\n'); + const fileName = 'test-file'; + const projectTempDir = '/tmp'; + + mockWriteFile.mockResolvedValue(undefined); + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + expect(result.outputFile).toBeDefined(); + expect(result.content).toContain('... [CONTENT TRUNCATED] ...'); + + // Extract just the truncated part (after the instructions) + const truncatedPart = result.content.split( + 'Truncated part of the output:\n', + )[1]; + // The truncated content (excluding the instructions header) should + // be roughly within the character threshold. + expect(truncatedPart.length).toBeLessThan(THRESHOLD * 1.5); + + // With 100 chars/line and 40,000 threshold, effective lines ≈ 400. + // Verify we have fewer lines than the default TRUNCATE_LINES. + const truncatedLines = truncatedPart.split('\n'); + expect(truncatedLines.length).toBeLessThan(TRUNCATE_LINES); + }); + + it('should truncate content by lines when line limit is the binding constraint', async () => { + // 2000 lines of 5 chars each = ~12,000 chars, well under THRESHOLD (40,000) + // so the line limit (1000) is the binding constraint, not the char threshold. + const lines = Array(2000).fill('hello'); + const content = lines.join('\n'); + const fileName = 'test-file'; + const projectTempDir = '/tmp'; + + expect(content.length).toBeLessThan(THRESHOLD); + + mockWriteFile.mockResolvedValue(undefined); + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + expect(result.outputFile).toBe( + path.join(projectTempDir, `${fileName}.output`), + ); + expect(mockWriteFile).toHaveBeenCalledWith( + path.join(projectTempDir, `${fileName}.output`), + content, + ); + + // Effective lines = min(1000, 40000/5) = 1000 (line limit is binding) + const head = Math.floor(TRUNCATE_LINES / 5); + const beginning = lines.slice(0, head); + const end = lines.slice(-(TRUNCATE_LINES - head)); + const expectedTruncated = + beginning.join('\n') + + '\n\n---\n... [CONTENT TRUNCATED] ...\n---\n\n' + + end.join('\n'); + + expect(result.content).toContain( + 'Tool output was too large and has been truncated', + ); + expect(result.content).toContain('Truncated part of the output:'); + expect(result.content).toContain(expectedTruncated); + }); + + it('should wrap and truncate content when content has few but long lines', async () => { + const content = 'a'.repeat(200_000); // A single very long line + const fileName = 'test-file'; + const projectTempDir = '/tmp'; + const wrapWidth = 120; + + mockWriteFile.mockResolvedValue(undefined); + + // Manually wrap the content to generate the expected file content + const wrappedLines: string[] = []; + for (let i = 0; i < content.length; i += wrapWidth) { + wrappedLines.push(content.substring(i, i + wrapWidth)); + } + const expectedFileContent = wrappedLines.join('\n'); + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + expect(result.outputFile).toBe( + path.join(projectTempDir, `${fileName}.output`), + ); + // Check that the file was written with the wrapped content + expect(mockWriteFile).toHaveBeenCalledWith( + path.join(projectTempDir, `${fileName}.output`), + expectedFileContent, + ); + + // After wrapping, avg line length is 120 chars. Effective lines = + // min(1000, floor(40000/120)) = min(1000, 333) = 333. + const avgLineLength = 120; + const effectiveLines = Math.min( + TRUNCATE_LINES, + Math.floor(THRESHOLD / avgLineLength), + ); + const head = Math.floor(effectiveLines / 5); + const beginning = wrappedLines.slice(0, head); + const end = wrappedLines.slice(-(effectiveLines - head)); + const expectedTruncated = + beginning.join('\n') + + '\n\n---\n... [CONTENT TRUNCATED] ...\n---\n\n' + + end.join('\n'); + expect(result.content).toContain( + 'Tool output was too large and has been truncated', + ); + expect(result.content).toContain('Truncated part of the output:'); + expect(result.content).toContain(expectedTruncated); + }); + + it('should handle file write errors gracefully', async () => { + const content = 'a'.repeat(2_000_000); + const fileName = 'test-file'; + const projectTempDir = '/tmp'; + + mockWriteFile.mockRejectedValue(new Error('File write failed')); + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + expect(result.outputFile).toBeUndefined(); + expect(result.content).toContain( + '[Note: Could not save full output to file]', + ); + expect(mockWriteFile).toHaveBeenCalled(); + }); + + it('should save to correct file path with file name', async () => { + const content = 'a'.repeat(200_000); + const fileName = 'unique-file-123'; + const projectTempDir = '/custom/temp/dir'; + const wrapWidth = 120; + + mockWriteFile.mockResolvedValue(undefined); + + // Manually wrap the content to generate the expected file content + const wrappedLines: string[] = []; + for (let i = 0; i < content.length; i += wrapWidth) { + wrappedLines.push(content.substring(i, i + wrapWidth)); + } + const expectedFileContent = wrappedLines.join('\n'); + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + const expectedPath = path.join(projectTempDir, `${fileName}.output`); + expect(result.outputFile).toBe(expectedPath); + expect(mockWriteFile).toHaveBeenCalledWith( + expectedPath, + expectedFileContent, + ); + }); + + it('should include helpful instructions in truncated message', async () => { + const content = 'a'.repeat(2_000_000); + const fileName = 'test-file'; + const projectTempDir = '/tmp'; + + mockWriteFile.mockResolvedValue(undefined); + + const result = await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + expect(result.content).toContain( + 'Tool output was too large and has been truncated', + ); + expect(result.content).toContain('The full output has been saved to:'); + expect(result.content).toContain( + 'To read the complete output, use the read_file tool with the absolute file path above', + ); + expect(result.content).toContain( + 'The truncated output below shows the beginning and end of the content', + ); + }); + + it('should sanitize fileName to prevent path traversal', async () => { + const content = 'a'.repeat(200_000); + const fileName = '../../../../../etc/passwd'; + const projectTempDir = '/tmp/safe_dir'; + const wrapWidth = 120; + + mockWriteFile.mockResolvedValue(undefined); + + // Manually wrap the content to generate the expected file content + const wrappedLines: string[] = []; + for (let i = 0; i < content.length; i += wrapWidth) { + wrappedLines.push(content.substring(i, i + wrapWidth)); + } + const expectedFileContent = wrappedLines.join('\n'); + + await truncateAndSaveToFile( + content, + fileName, + projectTempDir, + THRESHOLD, + TRUNCATE_LINES, + ); + + const expectedPath = path.join(projectTempDir, 'passwd.output'); + expect(mockWriteFile).toHaveBeenCalledWith( + expectedPath, + expectedFileContent, + ); + }); +}); diff --git a/packages/core/src/utils/truncation.ts b/packages/core/src/utils/truncation.ts new file mode 100644 index 000000000..fe947c67a --- /dev/null +++ b/packages/core/src/utils/truncation.ts @@ -0,0 +1,100 @@ +/** + * @license + * Copyright 2025 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; +import { ReadFileTool } from '../tools/read-file.js'; + +/** + * Truncates large tool output and saves the full content to a temp file. + * Used by the shell tool to prevent excessively large outputs from being + * sent to the LLM context. + * + * If content length is within the threshold, returns it unchanged. + * Otherwise, saves full content to a file and returns a truncated version + * with head/tail lines and a pointer to the saved file. + */ +export async function truncateAndSaveToFile( + content: string, + fileName: string, + projectTempDir: string, + threshold: number, + truncateLines: number, +): Promise<{ content: string; outputFile?: string }> { + let lines = content.split('\n'); + let fileContent = content; + + // Check both constraints: character threshold and line limit. + const exceedsThreshold = content.length > threshold; + const exceedsLineLimit = lines.length > truncateLines; + + if (!exceedsThreshold && !exceedsLineLimit) { + return { content }; + } + + // If the content is long but has few lines, wrap it to enable line-based truncation. + if (exceedsThreshold && !exceedsLineLimit) { + const wrapWidth = 120; // A reasonable width for wrapping. + const wrappedLines: string[] = []; + for (const line of lines) { + if (line.length > wrapWidth) { + for (let i = 0; i < line.length; i += wrapWidth) { + wrappedLines.push(line.substring(i, i + wrapWidth)); + } + } else { + wrappedLines.push(line); + } + } + lines = wrappedLines; + fileContent = lines.join('\n'); + } + + // Compute effective line limit that respects both constraints. + // If the average line length would cause truncateLines to exceed the + // character threshold, reduce the number of lines to fit. + let effectiveLines = truncateLines; + if (lines.length > 0) { + const totalChars = lines.reduce((sum, line) => sum + line.length, 0); + const avgLineLength = totalChars / lines.length; + if (avgLineLength > 0) { + const linesFittingThreshold = Math.floor(threshold / avgLineLength); + effectiveLines = Math.min(truncateLines, linesFittingThreshold); + // Ensure at least a small number of lines are kept. + effectiveLines = Math.max(effectiveLines, 10); + } + } + + const head = Math.floor(effectiveLines / 5); + const beginning = lines.slice(0, head); + const end = lines.slice(-(effectiveLines - head)); + const truncatedContent = + beginning.join('\n') + + '\n\n---\n... [CONTENT TRUNCATED] ...\n---\n\n' + + end.join('\n'); + + // Sanitize fileName to prevent path traversal. + const safeFileName = `${path.basename(fileName)}.output`; + const outputFile = path.join(projectTempDir, safeFileName); + try { + await fs.writeFile(outputFile, fileContent); + + return { + content: `Tool output was too large and has been truncated. +The full output has been saved to: ${outputFile} +To read the complete output, use the ${ReadFileTool.Name} tool with the absolute file path above. +The truncated output below shows the beginning and end of the content. The marker '... [CONTENT TRUNCATED] ...' indicates where content was removed. + +Truncated part of the output: +${truncatedContent}`, + outputFile, + }; + } catch (_error) { + return { + content: + truncatedContent + `\n[Note: Could not save full output to file]`, + }; + } +}