diff --git a/docs/plans/memory-diagnostics-reference-design.md b/docs/plans/memory-diagnostics-reference-design.md new file mode 100644 index 0000000000..c4b754eae2 --- /dev/null +++ b/docs/plans/memory-diagnostics-reference-design.md @@ -0,0 +1,56 @@ +# Memory Diagnostics Reference Design + +## Context + +Issue #3000 tracks memory and performance diagnostics for long-running Qwen +Code sessions. The first PR should establish a small, low-risk diagnostic +surface before adding heavier profiling or retention changes. + +The design is reference-first: + +- Claude Code keeps memory diagnostics separate from heap snapshot generation. + Its diagnostics include process memory, V8 heap statistics, heap spaces, + resource usage, active handles/requests, file descriptors, Linux + `smaps_rollup`, and leak hints. +- Codex focuses heavily on bounded retention and lazy loading for long-lived + process state. Those ideas should guide later PRs that address conversation, + command output, and history retention. + +## First PR Scope + +Add a `/doctor memory` diagnostic path that captures a single point-in-time +snapshot: + +- `process.memoryUsage()` +- V8 heap statistics and heap spaces +- `process.resourceUsage()` +- active handle/request counts +- open file descriptor count when `/proc/self/fd` is available +- Linux `smaps_rollup` when available +- basic risk hints for heap pressure, detached contexts, excessive handles, + excessive requests, high file descriptor count, and native memory pressure + +This command should be cheap enough to run in normal sessions and safe on +platforms where Linux-only probes are unavailable. + +## Non-Goals + +This PR intentionally does not: + +- write heap snapshots +- run continuous polling +- change prompt/history retention +- change tool output retention +- alter module loading behavior + +Those are follow-up PRs after the diagnostic baseline exists. + +## Follow-Up PRs + +1. Add explicit snapshot/export support for deeper local investigation. +2. Add bounded retention for large command/tool outputs, using Codex's capped + output retention as the main reference. +3. Audit lazy loading and module startup paths after measurements identify + hot spots. +4. Add repeatable memory/performance benchmark scenarios for long-running + sessions. diff --git a/packages/cli/src/acp-integration/session/Session.test.ts b/packages/cli/src/acp-integration/session/Session.test.ts index 7127cbf035..f1124e3e6a 100644 --- a/packages/cli/src/acp-integration/session/Session.test.ts +++ b/packages/cli/src/acp-integration/session/Session.test.ts @@ -650,6 +650,71 @@ describe('Session', () => { }); }); + it('honors explicit no-input override for built-in commands with subCommands', async () => { + getAvailableCommandsSpy.mockResolvedValueOnce([ + { + name: 'doctor', + description: 'Run installation and environment diagnostics', + kind: 'built-in', + acceptsInput: false, + subCommands: [ + { + name: 'memory', + description: 'Show current process memory diagnostics', + kind: 'built-in', + }, + ], + }, + ]); + + await session.sendAvailableCommandsUpdate(); + + expect(mockClient.sessionUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: 'test-session-id', + update: expect.objectContaining({ + sessionUpdate: 'available_commands_update', + availableCommands: expect.arrayContaining([ + expect.objectContaining({ + name: 'doctor', + description: 'Run installation and environment diagnostics', + input: null, + }), + ]), + }), + }), + ); + }); + + it('honors explicit input override for built-in commands without input metadata', async () => { + getAvailableCommandsSpy.mockResolvedValueOnce([ + { + name: 'diagnostics', + description: 'Run diagnostics', + kind: 'built-in', + acceptsInput: true, + }, + ]); + + await session.sendAvailableCommandsUpdate(); + + expect(mockClient.sessionUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: 'test-session-id', + update: expect.objectContaining({ + sessionUpdate: 'available_commands_update', + availableCommands: expect.arrayContaining([ + expect.objectContaining({ + name: 'diagnostics', + description: 'Run diagnostics', + input: { hint: '' }, + }), + ]), + }), + }), + ); + }); + it('attaches available skills to available_commands_update metadata', async () => { getAvailableCommandsSpy.mockResolvedValueOnce([ { diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index db408553c3..8a47c8ced6 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -1448,17 +1448,21 @@ export class Session implements SessionContext { // let users type arguments before submitting. Commands with no argument // support get input: null so the client auto-submits them on selection. // - // A command is considered to accept arguments when any of: - // - it is not a BUILT_IN command (skills, file commands, etc.) - // - it has a completion function - // - it declares an argumentHint - // - it has subCommands + // acceptsInput is determined by: + // 1. cmd.acceptsInput, if explicitly set (true or false overrides + // inference) + // 2. Otherwise, a command accepts arguments when any of: + // - it is not a BUILT_IN command (skills, file commands, etc.) + // - it has a completion function + // - it declares an argumentHint + // - it has subCommands const availableCommands: AvailableCommand[] = slashCommands.map((cmd) => { const acceptsInput = - cmd.kind !== CommandKind.BUILT_IN || - cmd.completion != null || - cmd.argumentHint != null || - (cmd.subCommands != null && cmd.subCommands.length > 0); + cmd.acceptsInput ?? + (cmd.kind !== CommandKind.BUILT_IN || + cmd.completion != null || + cmd.argumentHint != null || + (cmd.subCommands != null && cmd.subCommands.length > 0)); return { name: cmd.name, description: cmd.description, diff --git a/packages/cli/src/i18n/locales/en.js b/packages/cli/src/i18n/locales/en.js index 29db173bc2..fb4ff837ba 100644 --- a/packages/cli/src/i18n/locales/en.js +++ b/packages/cli/src/i18n/locales/en.js @@ -1894,6 +1894,8 @@ export default { // === Core: added from PR #3328 === 'Open the memory manager.': 'Open the memory manager.', + 'Show current process memory diagnostics': + 'Show current process memory diagnostics', 'Save a durable memory to the memory system.': 'Save a durable memory to the memory system.', 'Ask a quick side question without affecting the main conversation': diff --git a/packages/cli/src/i18n/locales/zh-TW.js b/packages/cli/src/i18n/locales/zh-TW.js index b6ddc78cbe..17af2229a4 100644 --- a/packages/cli/src/i18n/locales/zh-TW.js +++ b/packages/cli/src/i18n/locales/zh-TW.js @@ -1482,6 +1482,7 @@ export default { // === Core: added from PR #3328 === 'Open the memory manager.': '打開記憶管理器。', + 'Show current process memory diagnostics': '顯示目前程序的內存診斷。', 'Save a durable memory to the memory system.': '將持久記憶保存到記憶系統。', 'Ask a quick side question without affecting the main conversation': '在不影響主對話的情況下快速提問旁支問題', diff --git a/packages/cli/src/i18n/locales/zh.js b/packages/cli/src/i18n/locales/zh.js index bb804745c3..ce91586739 100644 --- a/packages/cli/src/i18n/locales/zh.js +++ b/packages/cli/src/i18n/locales/zh.js @@ -1717,6 +1717,7 @@ export default { '[{{label}}] failed: {{error}}': '[{{label}}] 失败:{{error}}', 'Loading suggestions...': '正在加载建议...', 'Open the memory manager.': '打开记忆管理器。', + 'Show current process memory diagnostics': '显示当前进程的内存诊断。', 'Save a durable memory to the memory system.': '将一条持久记忆保存到记忆系统。', 'Show per-item context usage breakdown.': '显示按项目划分的上下文使用详情。', diff --git a/packages/cli/src/nonInteractiveCliCommands.ts b/packages/cli/src/nonInteractiveCliCommands.ts index d575cd1c83..180e94e888 100644 --- a/packages/cli/src/nonInteractiveCliCommands.ts +++ b/packages/cli/src/nonInteractiveCliCommands.ts @@ -48,13 +48,13 @@ export type NonInteractiveSlashCommandResult = } | { type: 'message'; - messageType: 'info' | 'error'; + messageType: 'info' | 'warning' | 'error'; content: string; } | { type: 'stream_messages'; messages: AsyncGenerator< - { messageType: 'info' | 'error'; content: string }, + { messageType: 'info' | 'warning' | 'error'; content: string }, void, unknown >; diff --git a/packages/cli/src/ui/commands/doctorCommand.test.ts b/packages/cli/src/ui/commands/doctorCommand.test.ts index 188645b00b..f9afbd969c 100644 --- a/packages/cli/src/ui/commands/doctorCommand.test.ts +++ b/packages/cli/src/ui/commands/doctorCommand.test.ts @@ -10,14 +10,27 @@ import { type CommandContext } from './types.js'; import { createMockCommandContext } from '../../test-utils/mockCommandContext.js'; import * as doctorChecksModule from '../../utils/doctorChecks.js'; import * as memoryDiagnosticsModule from '../../utils/memoryDiagnostics.js'; +import { collectMemoryDiagnostics } from '@qwen-code/qwen-code-core'; import type { DoctorCheckResult } from '../types.js'; vi.mock('../../utils/doctorChecks.js'); vi.mock('../../utils/memoryDiagnostics.js'); +vi.mock('@qwen-code/qwen-code-core', async (importOriginal) => ({ + ...(await importOriginal()), + collectMemoryDiagnostics: vi.fn(), +})); describe('doctorCommand', () => { let mockContext: CommandContext; + const getMemoryCommand = () => { + const memoryCommand = doctorCommand.subCommands?.find( + (command) => command.name === 'memory', + ); + expect(memoryCommand).toBeDefined(); + return memoryCommand!; + }; + const mockChecks: DoctorCheckResult[] = [ { category: 'System', @@ -96,6 +109,55 @@ describe('doctorCommand', () => { vi.mocked(doctorChecksModule.runDoctorChecks).mockResolvedValue(mockChecks); mockMemoryDiagnostics(); + vi.mocked(collectMemoryDiagnostics).mockResolvedValue({ + timestamp: '2026-05-01T10:00:00.000Z', + uptimeSeconds: 60, + memoryUsage: { + heapUsed: 1_000, + heapTotal: 2_000, + rss: 3_000, + external: 100, + arrayBuffers: 50, + }, + v8HeapStats: { + heapSizeLimit: 4_000, + totalHeapSize: 2_000, + usedHeapSize: 1_000, + mallocedMemory: 2_048, + peakMallocedMemory: 4_096, + detachedContexts: 0, + nativeContexts: 1, + }, + v8HeapSpaces: [ + { + name: 'old_space', + size: 4_096, + used: 2_048, + available: 2_048, + }, + { + name: 'new_space', + size: 2_048, + used: 1_024, + available: 1_024, + }, + ], + resourceUsage: { + maxRSS: 4_000, + userCPUTime: 10, + systemCPUTime: 20, + }, + activeHandles: 2, + activeRequests: 0, + openFileDescriptors: null, + smapsRollup: 'Rss: 5000 kB\nPss: 1000 kB\n', + platform: 'darwin', + nodeVersion: 'v20.19.0', + analysis: { + risks: [], + recommendation: 'No obvious leak indicators.', + }, + }); }); afterEach(() => { @@ -610,4 +672,367 @@ describe('doctorCommand', () => { // setPendingItem(null) should still be called via finally expect(mockContext.ui.setPendingItem).toHaveBeenCalledWith(null); }); + + it('should return memory diagnostics as JSON for /doctor memory --json', async () => { + mockContext = createMockCommandContext({ + executionMode: 'non_interactive', + ui: { + addItem: vi.fn(), + setPendingItem: vi.fn(), + }, + } as unknown as CommandContext); + + const result = await getMemoryCommand().action!(mockContext, '--json'); + + expect(doctorChecksModule.runDoctorChecks).not.toHaveBeenCalled(); + expect(collectMemoryDiagnostics).toHaveBeenCalledTimes(1); + expect(result).toEqual( + expect.objectContaining({ + type: 'message', + messageType: 'info', + }), + ); + expect( + JSON.parse(result?.type === 'message' ? result.content : '{}'), + ).toMatchObject({ + memoryUsage: { + heapUsed: 1_000, + }, + analysis: { + risks: [], + }, + }); + }); + + it('should return a readable memory diagnostics summary for /doctor memory', async () => { + mockContext = createMockCommandContext({ + executionMode: 'non_interactive', + ui: { + addItem: vi.fn(), + setPendingItem: vi.fn(), + }, + } as unknown as CommandContext); + + const result = await getMemoryCommand().action!(mockContext, ''); + + expect(result).toEqual( + expect.objectContaining({ + type: 'message', + messageType: 'info', + content: expect.stringContaining('Memory Diagnostics'), + }), + ); + expect(result?.type === 'message' ? result.content : '').toContain( + 'heapUsed', + ); + expect(result?.type === 'message' ? result.content : '').toContain( + 'v8MallocedMemory: 2.0 KB', + ); + }); + + it('should render small memory values without rounding to zero MiB', async () => { + const result = await getMemoryCommand().action!(mockContext, ''); + + expect(result?.type === 'message' ? result.content : '').toContain( + 'heapUsed: 1.0 KB', + ); + expect(result?.type === 'message' ? result.content : '').not.toContain( + 'heapUsed: 0.0 MB', + ); + }); + + it('should pass session metadata to memory diagnostics', async () => { + const getSessionId = vi.fn(() => 'session-123'); + const getCliVersion = vi.fn(() => '0.15.11'); + mockContext = createMockCommandContext({ + services: { + config: { + getSessionId, + getCliVersion, + }, + }, + } as unknown as CommandContext); + + await getMemoryCommand().action!(mockContext, '--json'); + + expect(collectMemoryDiagnostics).toHaveBeenCalledWith({ + sessionId: 'session-123', + qwenVersion: '0.15.11', + }); + }); + + it('should register memory as a real doctor subcommand', () => { + expect(doctorCommand.subCommands?.map((command) => command.name)).toContain( + 'memory', + ); + expect(getMemoryCommand().argumentHint).toBe( + '[--json] [--sample] [--snapshot]', + ); + }); + + it('should support sampled memory diagnostics through the memory subcommand', async () => { + mockContext = createMockCommandContext({ + executionMode: 'non_interactive', + ui: { + addItem: vi.fn(), + setPendingItem: vi.fn(), + }, + } as unknown as CommandContext); + + const result = await getMemoryCommand().action!(mockContext, '--sample'); + + expect( + memoryDiagnosticsModule.collectMemoryPressureSamples, + ).toHaveBeenCalledWith({ + sampleCount: 3, + intervalMs: 1000, + signal: undefined, + }); + expect(collectMemoryDiagnostics).not.toHaveBeenCalled(); + expect(result).toEqual({ + type: 'message', + messageType: 'info', + content: + 'Memory diagnostics\nRSS: 100.0 MiB\nActive handles: 3\n\nMemory pressure samples\nSample count: 1', + }); + }); + + it('should support heap snapshots through the memory subcommand', async () => { + mockContext = createMockCommandContext({ + executionMode: 'non_interactive', + ui: { + addItem: vi.fn(), + setPendingItem: vi.fn(), + }, + } as unknown as CommandContext); + + const result = await getMemoryCommand().action!(mockContext, '--snapshot'); + + expect(memoryDiagnosticsModule.writeMemoryHeapSnapshot).toHaveBeenCalled(); + expect(collectMemoryDiagnostics).not.toHaveBeenCalled(); + expect(result).toEqual({ + type: 'message', + messageType: 'info', + content: + 'Memory diagnostics\nRSS: 100.0 MiB\nActive handles: 3\n\nHeap snapshot written: /tmp/qwen-code-heap.heapsnapshot\nHeap snapshot may contain prompts, file contents, tool results, and other sensitive data. Do not share it publicly without reviewing it first.', + }); + }); + + it('should render risk indicators without failing memory diagnostics', async () => { + vi.mocked(collectMemoryDiagnostics).mockResolvedValue({ + timestamp: '2026-05-01T10:00:00.000Z', + uptimeSeconds: 60, + memoryUsage: { + heapUsed: 3_500, + heapTotal: 4_000, + rss: 8_000, + external: 100, + arrayBuffers: 50, + }, + v8HeapStats: { + heapSizeLimit: 4_000, + totalHeapSize: 4_000, + usedHeapSize: 3_500, + mallocedMemory: 10, + peakMallocedMemory: 20, + detachedContexts: 0, + nativeContexts: 1, + }, + resourceUsage: { + maxRSS: 8_000, + userCPUTime: 10, + systemCPUTime: 20, + }, + activeHandles: 2, + activeRequests: 0, + v8HeapSpaces: null, + openFileDescriptors: null, + smapsRollup: null, + platform: 'darwin', + nodeVersion: 'v20.19.0', + analysis: { + risks: [{ type: 'heap-pressure', message: 'Heap pressure detected.' }], + recommendation: '1 potential leak indicator(s) found.', + }, + }); + const result = await getMemoryCommand().action!(mockContext, ''); + + expect(result).toEqual( + expect.objectContaining({ + type: 'message', + messageType: 'warning', + }), + ); + expect(result?.type === 'message' ? result.content : '').toContain( + 'heap-pressure: Heap pressure detected.', + ); + expect(result?.type === 'message' ? result.content : '').toContain( + 'recommendation: 1 potential leak indicator(s) found.', + ); + expect(result?.type === 'message' ? result.content : '').not.toContain( + 'recommendation: WARNING:', + ); + }); + + it('should skip memory diagnostics when already aborted', async () => { + const abortController = new AbortController(); + abortController.abort(); + mockContext = createMockCommandContext({ + executionMode: 'non_interactive', + abortSignal: abortController.signal, + ui: { + addItem: vi.fn(), + setPendingItem: vi.fn(), + }, + } as unknown as CommandContext); + + const result = await getMemoryCommand().action!(mockContext, ''); + + expect(result).toBeUndefined(); + expect(collectMemoryDiagnostics).not.toHaveBeenCalled(); + }); + + it('should return an error message when memory diagnostics fail', async () => { + vi.mocked(collectMemoryDiagnostics).mockRejectedValueOnce( + new Error('probe failed'), + ); + + const result = await getMemoryCommand().action!(mockContext, ''); + + expect(result).toEqual( + expect.objectContaining({ + type: 'message', + messageType: 'error', + content: expect.stringContaining('probe failed'), + }), + ); + }); + + it('should reject unknown arguments with a usage hint', async () => { + const result = await getMemoryCommand().action!(mockContext, '--bogus'); + + expect(collectMemoryDiagnostics).not.toHaveBeenCalled(); + expect(result).toEqual( + expect.objectContaining({ + type: 'message', + messageType: 'error', + content: expect.stringContaining('--bogus'), + }), + ); + expect(result?.type === 'message' ? result.content : '').toContain( + '/doctor memory [--json] [--sample] [--snapshot]', + ); + }); + + it('should show a parse error marker for malformed smaps rollup data', async () => { + vi.mocked(collectMemoryDiagnostics).mockResolvedValueOnce({ + timestamp: '2026-05-01T10:00:00.000Z', + uptimeSeconds: 60, + memoryUsage: { + heapUsed: 1_000, + heapTotal: 2_000, + rss: 3_000, + external: 100, + arrayBuffers: 50, + }, + v8HeapStats: { + heapSizeLimit: 4_000, + totalHeapSize: 2_000, + usedHeapSize: 1_000, + mallocedMemory: 2_048, + peakMallocedMemory: 4_096, + detachedContexts: 0, + nativeContexts: 1, + }, + v8HeapSpaces: null, + resourceUsage: { + maxRSS: 4_000, + userCPUTime: 10, + systemCPUTime: 20, + }, + activeHandles: 2, + activeRequests: 0, + openFileDescriptors: null, + smapsRollup: 'Pss: 1000 kB\n', + platform: 'linux', + nodeVersion: 'v20.19.0', + analysis: { + risks: [], + recommendation: 'No obvious leak indicators.', + }, + }); + + const result = await getMemoryCommand().action!(mockContext, ''); + const content = result?.type === 'message' ? result.content : ''; + + expect(content).toContain('smapsRollup: parse error: Pss:'); + expect(content).not.toContain('smapsRollup: available'); + }); + + it('should suppress JSON output when aborted between probe and return', async () => { + const abortController = new AbortController(); + vi.mocked(collectMemoryDiagnostics).mockImplementationOnce(async () => { + abortController.abort(); + return { + timestamp: '2026-05-01T10:00:00.000Z', + uptimeSeconds: 1, + memoryUsage: { + heapUsed: 1, + heapTotal: 1, + rss: 1, + external: 0, + arrayBuffers: 0, + }, + v8HeapStats: { + heapSizeLimit: 1, + totalHeapSize: 1, + usedHeapSize: 1, + mallocedMemory: 0, + peakMallocedMemory: 0, + detachedContexts: 0, + nativeContexts: 1, + }, + resourceUsage: { maxRSS: 0, userCPUTime: 0, systemCPUTime: 0 }, + activeHandles: 0, + activeRequests: 0, + v8HeapSpaces: null, + openFileDescriptors: null, + smapsRollup: null, + platform: 'darwin', + nodeVersion: 'v20.19.0', + analysis: { risks: [], recommendation: '' }, + }; + }); + + mockContext = createMockCommandContext({ + executionMode: 'non_interactive', + abortSignal: abortController.signal, + ui: { + addItem: vi.fn(), + setPendingItem: vi.fn(), + }, + } as unknown as CommandContext); + + const result = await getMemoryCommand().action!(mockContext, '--json'); + + expect(result).toBeUndefined(); + }); + + it('should render expanded fields in readable summary', async () => { + const result = await getMemoryCommand().action!(mockContext, ''); + const content = result?.type === 'message' ? result.content : ''; + + expect(content).toContain('detachedContexts: 0'); + expect(content).toContain('nativeContexts: 1'); + expect(content).toContain('maxRSS:'); + expect(content).toContain('userCPUTime:'); + expect(content).toContain('systemCPUTime:'); + expect(content).toContain('smapsRollup: Rss: 5000 kB'); + expect(content).toContain('v8HeapSpaces:'); + expect(content).toContain('old_space: used 2.0 KB'); + }); + + it('should advertise the memory subcommand on the parent doctor argumentHint', () => { + expect(doctorCommand.argumentHint).toBe('[memory] [--sample] [--snapshot]'); + }); }); diff --git a/packages/cli/src/ui/commands/doctorCommand.ts b/packages/cli/src/ui/commands/doctorCommand.ts index 9c89cac6e0..bb90c38997 100644 --- a/packages/cli/src/ui/commands/doctorCommand.ts +++ b/packages/cli/src/ui/commands/doctorCommand.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import type { SlashCommand } from './types.js'; +import type { CommandContext, SlashCommand } from './types.js'; import { CommandKind } from './types.js'; import type { HistoryItemDoctor } from '../types.js'; import { runDoctorChecks } from '../../utils/doctorChecks.js'; @@ -17,6 +17,11 @@ import { writeMemoryHeapSnapshot, } from '../../utils/memoryDiagnostics.js'; import { t } from '../../i18n/index.js'; +import { + collectMemoryDiagnostics, + type MemoryDiagnostics, +} from '@qwen-code/qwen-code-core'; +import { formatMemoryUsage } from '../utils/formatters.js'; const MEMORY_SUBCOMMAND = 'memory'; const DOCTOR_SUBCOMMANDS = [MEMORY_SUBCOMMAND] as const; @@ -217,4 +222,163 @@ export const doctorCommand: SlashCommand = { } } }, + subCommands: [ + { + name: 'memory', + get description() { + return t('Show current process memory diagnostics'); + }, + kind: CommandKind.BUILT_IN, + supportedModes: ['interactive', 'non_interactive', 'acp'] as const, + argumentHint: '[--json] [--sample] [--snapshot]', + action: memoryDoctorAction, + }, + ], }; + +const MEMORY_USAGE_HINT = '/doctor memory [--json] [--sample] [--snapshot]'; + +async function memoryDoctorAction(context: CommandContext, args = '') { + if (context.abortSignal?.aborted) { + return; + } + + const tokens = args.trim().split(/\s+/).filter(Boolean); + const unknown = tokens.filter( + (token) => + token !== '--json' && token !== '--sample' && token !== '--snapshot', + ); + if (unknown.length > 0) { + return { + type: 'message' as const, + messageType: 'error' as const, + content: `${t('Unknown argument(s)')}: ${unknown.join(', ')}. ${t('Usage')}: ${MEMORY_USAGE_HINT}`, + }; + } + + const shouldSampleMemory = tokens.includes('--sample'); + const shouldWriteHeapSnapshot = tokens.includes('--snapshot'); + if (shouldSampleMemory || shouldWriteHeapSnapshot) { + return doctorCommand.action?.( + context, + [MEMORY_SUBCOMMAND, ...tokens.filter((token) => token !== '--json')].join( + ' ', + ), + ); + } + + try { + const diagnostics = await collectMemoryDiagnostics({ + sessionId: context.services.config?.getSessionId(), + qwenVersion: context.services.config?.getCliVersion(), + }); + + if (context.abortSignal?.aborted) { + return; + } + + return { + type: 'message' as const, + messageType: + diagnostics.analysis.risks.length > 0 + ? ('warning' as const) + : ('info' as const), + content: tokens.includes('--json') + ? JSON.stringify(diagnostics, null, 2) + : formatCoreDiagnostics(diagnostics), + }; + } catch (error) { + if (context.abortSignal?.aborted) { + return; + } + + return { + type: 'message' as const, + messageType: 'error' as const, + content: `${t('Failed to collect memory diagnostics')}: ${formatErrorMessage(error)}`, + }; + } +} + +// resourceUsage CPU times are microseconds; convert to seconds for display. +function formatCpuMicroseconds(micros: number): string { + return `${(micros / 1_000_000).toFixed(2)}s`; +} + +function formatHeapSpaces( + spaces: MemoryDiagnostics['v8HeapSpaces'], +): string | undefined { + if (!spaces || spaces.length === 0) { + return undefined; + } + const top = [...spaces].sort((a, b) => b.used - a.used).slice(0, 4); + const lines = top.map( + (space) => + ` - ${space.name}: used ${formatMemoryUsage(space.used)} / size ${formatMemoryUsage(space.size)}`, + ); + if (spaces.length > top.length) { + lines.push(` - … ${spaces.length - top.length} more`); + } + return lines.join('\n'); +} + +function formatSmapsRollup(smapsRollup: string | null): string { + if (!smapsRollup) { + return t('unavailable'); + } + + const rssLine = smapsRollup + .split(/\r?\n/) + .map((line) => line.trim().replace(/\s+/g, ' ')) + .find((line) => line.startsWith('Rss:')); + if (rssLine) { + return rssLine; + } + + const preview = smapsRollup.slice(0, 80).trim().replace(/\s+/g, ' '); + return `${t('parse error')}: ${preview}`; +} + +function formatCoreDiagnostics(diagnostics: MemoryDiagnostics): string { + const risks = + diagnostics.analysis.risks.length > 0 + ? diagnostics.analysis.risks + .map((risk) => ` - ${risk.type}: ${risk.message}`) + .join('\n') + : ` ${t('none')}`; + + const lines: string[] = [ + t('Memory Diagnostics'), + `timestamp: ${diagnostics.timestamp}`, + `uptimeSeconds: ${diagnostics.uptimeSeconds.toFixed(1)}`, + `heapUsed: ${formatMemoryUsage(diagnostics.memoryUsage.heapUsed)}`, + `heapTotal: ${formatMemoryUsage(diagnostics.memoryUsage.heapTotal)}`, + `rss: ${formatMemoryUsage(diagnostics.memoryUsage.rss)}`, + `external: ${formatMemoryUsage(diagnostics.memoryUsage.external)}`, + `arrayBuffers: ${formatMemoryUsage(diagnostics.memoryUsage.arrayBuffers)}`, + `v8HeapLimit: ${formatMemoryUsage(diagnostics.v8HeapStats.heapSizeLimit)}`, + `v8MallocedMemory: ${formatMemoryUsage(diagnostics.v8HeapStats.mallocedMemory)}`, + `v8PeakMallocedMemory: ${formatMemoryUsage(diagnostics.v8HeapStats.peakMallocedMemory)}`, + `detachedContexts: ${diagnostics.v8HeapStats.detachedContexts}`, + `nativeContexts: ${diagnostics.v8HeapStats.nativeContexts}`, + `maxRSS: ${formatMemoryUsage(diagnostics.resourceUsage.maxRSS)}`, + `userCPUTime: ${formatCpuMicroseconds(diagnostics.resourceUsage.userCPUTime)}`, + `systemCPUTime: ${formatCpuMicroseconds(diagnostics.resourceUsage.systemCPUTime)}`, + `activeHandles: ${diagnostics.activeHandles}`, + `activeRequests: ${diagnostics.activeRequests}`, + `openFileDescriptors: ${diagnostics.openFileDescriptors ?? t('unavailable')}`, + `smapsRollup: ${formatSmapsRollup(diagnostics.smapsRollup)}`, + ]; + + const heapSpaces = formatHeapSpaces(diagnostics.v8HeapSpaces); + if (heapSpaces) { + lines.push('v8HeapSpaces:', heapSpaces); + } + + lines.push( + 'risks:', + risks, + `recommendation: ${diagnostics.analysis.recommendation}`, + ); + return lines.join('\n'); +} diff --git a/packages/cli/src/ui/commands/insightCommand.test.ts b/packages/cli/src/ui/commands/insightCommand.test.ts index 43752ec4c4..22f230fed7 100644 --- a/packages/cli/src/ui/commands/insightCommand.test.ts +++ b/packages/cli/src/ui/commands/insightCommand.test.ts @@ -120,7 +120,7 @@ describe('insightCommand', () => { const messagesPromise = (async () => { const messages: Array<{ - messageType: 'info' | 'error'; + messageType: 'info' | 'warning' | 'error'; content: string; }> = []; for await (const message of result.messages) { diff --git a/packages/cli/src/ui/commands/types.ts b/packages/cli/src/ui/commands/types.ts index ee2c1836ef..a7454ea975 100644 --- a/packages/cli/src/ui/commands/types.ts +++ b/packages/cli/src/ui/commands/types.ts @@ -131,7 +131,7 @@ export interface QuitActionReturn { */ export interface MessageActionReturn { type: 'message'; - messageType: 'info' | 'error'; + messageType: 'info' | 'warning' | 'error'; content: string; } @@ -142,7 +142,7 @@ export interface MessageActionReturn { export interface StreamMessagesActionReturn { type: 'stream_messages'; messages: AsyncGenerator< - { messageType: 'info' | 'error'; content: string }, + { messageType: 'info' | 'warning' | 'error'; content: string }, void, unknown >; @@ -362,6 +362,12 @@ export interface SlashCommand { */ argumentHint?: string; + /** + * Whether command-picker clients should wait for additional user input before + * submitting this command. Defaults are inferred from command metadata. + */ + acceptsInput?: boolean; + /** * Describes when to use this command — injected into the model-visible * description for modelInvocable commands. diff --git a/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts b/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts index 57cc7048ba..82e2beecd5 100644 --- a/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts @@ -334,6 +334,28 @@ describe('useSlashCommandProcessor', () => { ); }); + it('should display warning message command results as warnings', async () => { + const command = createTestCommand({ + name: 'warn', + action: vi.fn().mockResolvedValue({ + type: 'message', + messageType: 'warning', + content: 'Check diagnostics.', + }), + }); + const result = setupProcessorHook([command]); + await waitFor(() => expect(result.current.slashCommands).toHaveLength(1)); + + await act(async () => { + await result.current.handleSlashCommand('/warn'); + }); + + expect(mockAddItem).toHaveBeenCalledWith( + { type: MessageType.WARNING, text: 'Check diagnostics.' }, + expect.any(Number), + ); + }); + it('should correctly find and execute a nested subcommand', async () => { const childAction = vi.fn(); const parentCommand: SlashCommand = { diff --git a/packages/cli/src/ui/hooks/slashCommandProcessor.ts b/packages/cli/src/ui/hooks/slashCommandProcessor.ts index a4bfe44b52..0607778cdd 100644 --- a/packages/cli/src/ui/hooks/slashCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/slashCommandProcessor.ts @@ -640,6 +640,12 @@ export const useSlashCommandProcessor = ( content: result.content, timestamp: new Date(), }); + } else if (result.messageType === 'warning') { + addMessage({ + type: MessageType.WARNING, + content: result.content, + timestamp: new Date(), + }); } else { addMessage({ type: MessageType.ERROR, diff --git a/packages/cli/src/ui/types.ts b/packages/cli/src/ui/types.ts index f040b3d2dc..5a180130b5 100644 --- a/packages/cli/src/ui/types.ts +++ b/packages/cli/src/ui/types.ts @@ -606,7 +606,11 @@ export interface InsightProgressProps { // Simplified message structure for internal feedback export type Message = | { - type: MessageType.INFO | MessageType.ERROR | MessageType.USER; + type: + | MessageType.INFO + | MessageType.WARNING + | MessageType.ERROR + | MessageType.USER; content: string; // Renamed from text for clarity in this context timestamp: Date; } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 5ebd1d4d89..bae6ca877c 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -287,6 +287,7 @@ export * from './utils/gitIgnoreParser.js'; export * from './utils/gitUtils.js'; export * from './utils/ignorePatterns.js'; export * from './utils/jsonl-utils.js'; +export * from './utils/memoryDiagnostics.js'; export * from './utils/memoryDiscovery.js'; export * from './utils/modelId.js'; export { ConditionalRulesRegistry } from './utils/rulesDiscovery.js'; diff --git a/packages/core/src/utils/memoryDiagnostics.test.ts b/packages/core/src/utils/memoryDiagnostics.test.ts new file mode 100644 index 0000000000..0e7c3de4a0 --- /dev/null +++ b/packages/core/src/utils/memoryDiagnostics.test.ts @@ -0,0 +1,594 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import { afterEach, describe, expect, it, vi } from 'vitest'; +import process from 'node:process'; + +const debugLogger = vi.hoisted(() => ({ + debug: vi.fn(), +})); + +vi.mock('./debugLogger.js', () => ({ + createDebugLogger: () => debugLogger, +})); + +import { collectMemoryDiagnostics } from './memoryDiagnostics.js'; + +describe('collectMemoryDiagnostics', () => { + afterEach(() => { + debugLogger.debug.mockReset(); + vi.restoreAllMocks(); + }); + + it('captures memory, V8, resource, handle, fd, smaps, and risk data', async () => { + const diagnostics = await collectMemoryDiagnostics({ + now: () => new Date('2026-05-01T10:00:00.000Z'), + sessionId: 'session-123', + qwenVersion: '0.15.6', + memoryUsage: () => ({ + heapUsed: 32 * 1024 * 1024, + heapTotal: 40 * 1024 * 1024, + rss: 100 * 1024 * 1024, + external: 700, + arrayBuffers: 300, + }), + heapStatistics: () => ({ + heap_size_limit: 40 * 1024 * 1024, + total_heap_size: 40 * 1024 * 1024, + total_heap_size_executable: 0, + total_physical_size: 40 * 1024 * 1024, + used_heap_size: 32 * 1024 * 1024, + malloced_memory: 80 * 1024 * 1024, + peak_malloced_memory: 90 * 1024 * 1024, + does_zap_garbage: 0, + number_of_native_contexts: 2, + number_of_detached_contexts: 1, + total_available_size: 400, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 700, + }), + heapSpaceStatistics: () => [ + { + space_name: 'old_space', + space_size: 1_000, + space_used_size: 800, + space_available_size: 200, + physical_space_size: 1_000, + }, + ], + resourceUsage: () => ({ + userCPUTime: 10, + systemCPUTime: 20, + maxRSS: 6, + sharedMemorySize: 0, + unsharedDataSize: 0, + unsharedStackSize: 0, + minorPageFault: 0, + majorPageFault: 0, + swappedOut: 0, + fsRead: 0, + fsWrite: 0, + ipcSent: 0, + ipcReceived: 0, + signalsCount: 0, + voluntaryContextSwitches: 0, + involuntaryContextSwitches: 0, + }), + uptimeSeconds: () => 60, + activeHandles: () => 300, + activeRequests: () => 3, + openFileDescriptors: async () => 501, + smapsRollup: async () => 'Rss: 5000 kB', + platform: 'linux', + nodeVersion: 'v20.19.0', + }); + + expect(diagnostics).toMatchObject({ + timestamp: '2026-05-01T10:00:00.000Z', + sessionId: 'session-123', + qwenVersion: '0.15.6', + uptimeSeconds: 60, + memoryUsage: { + heapUsed: 32 * 1024 * 1024, + heapTotal: 40 * 1024 * 1024, + rss: 100 * 1024 * 1024, + external: 700, + arrayBuffers: 300, + }, + v8HeapStats: { + heapSizeLimit: 40 * 1024 * 1024, + totalHeapSize: 40 * 1024 * 1024, + usedHeapSize: 32 * 1024 * 1024, + mallocedMemory: 80 * 1024 * 1024, + peakMallocedMemory: 90 * 1024 * 1024, + detachedContexts: 1, + nativeContexts: 2, + }, + v8HeapSpaces: [ + { + name: 'old_space', + size: 1_000, + used: 800, + available: 200, + }, + ], + resourceUsage: { + maxRSS: 6, + userCPUTime: 10, + systemCPUTime: 20, + }, + activeHandles: 300, + activeRequests: 3, + openFileDescriptors: 501, + smapsRollup: 'Rss: 5000 kB', + platform: 'linux', + nodeVersion: 'v20.19.0', + }); + + expect('memoryGrowthRate' in diagnostics).toBe(false); + + expect(diagnostics.analysis.risks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'heap-pressure' }), + expect.objectContaining({ type: 'detached-contexts' }), + expect.objectContaining({ type: 'active-handles' }), + expect.objectContaining({ type: 'fd-leak' }), + expect.objectContaining({ type: 'native-memory-pressure' }), + ]), + ); + + const nativeRisk = diagnostics.analysis.risks.find( + (risk) => risk.type === 'native-memory-pressure', + ); + expect(nativeRisk?.message).toContain('80.0 MB'); + expect(nativeRisk?.message).toContain('32.0 MB'); + expect(diagnostics.analysis.recommendation).toBe( + '5 potential leak indicator(s) found.', + ); + expect(diagnostics.analysis.recommendation).not.toContain('WARNING:'); + }); + + it('does not flag native pressure when malloced memory is below the absolute floor', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 1_600, + heapTotal: 2_000, + rss: 5_000, + external: 700, + arrayBuffers: 300, + }), + heapStatistics: () => ({ + heap_size_limit: 2_000, + total_heap_size: 2_000, + total_heap_size_executable: 0, + total_physical_size: 2_000, + used_heap_size: 1_600, + // 32 MB malloced, well above 2× the tiny heap but below the 64 MB + // floor — should not flag as a leak indicator. + malloced_memory: 32 * 1024 * 1024, + peak_malloced_memory: 32 * 1024 * 1024, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 400, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 700, + }), + activeHandles: () => 0, + activeRequests: () => 0, + }); + + expect(diagnostics.analysis.risks).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'native-memory-pressure' }), + ]), + ); + }); + + it('does not flag active-handles below the 256 threshold', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 100, + heapTotal: 200, + rss: 300, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 1_000, + total_heap_size: 200, + total_heap_size_executable: 0, + total_physical_size: 200, + used_heap_size: 100, + malloced_memory: 0, + peak_malloced_memory: 0, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 900, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + activeHandles: () => 200, + activeRequests: () => 0, + }); + + expect(diagnostics.analysis.risks).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'active-handles' }), + ]), + ); + }); + + it('treats maxRSS as bytes on all platforms', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 100, + heapTotal: 200, + rss: 300, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 1_000, + total_heap_size: 200, + total_heap_size_executable: 0, + total_physical_size: 200, + used_heap_size: 100, + malloced_memory: 0, + peak_malloced_memory: 0, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 900, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + resourceUsage: () => ({ + userCPUTime: 10, + systemCPUTime: 20, + maxRSS: 4_096, + sharedMemorySize: 0, + unsharedDataSize: 0, + unsharedStackSize: 0, + minorPageFault: 0, + majorPageFault: 0, + swappedOut: 0, + fsRead: 0, + fsWrite: 0, + ipcSent: 0, + ipcReceived: 0, + signalsCount: 0, + voluntaryContextSwitches: 0, + involuntaryContextSwitches: 0, + }), + platform: 'darwin', + nodeVersion: 'v20.19.0', + }); + + // Node.js >=14.10.0 returns maxRSS in bytes on all platforms. + expect(diagnostics.resourceUsage.maxRSS).toBe(4_096); + }); + + it('treats unsupported optional probes as unavailable instead of failing', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 100, + heapTotal: 200, + rss: 300, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 1_000, + total_heap_size: 200, + total_heap_size_executable: 0, + total_physical_size: 200, + used_heap_size: 100, + malloced_memory: 0, + peak_malloced_memory: 0, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 900, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + heapSpaceStatistics: () => { + throw new Error('not available'); + }, + activeHandles: () => 0, + activeRequests: () => 0, + openFileDescriptors: async () => { + throw new Error('not available'); + }, + smapsRollup: async () => { + throw new Error('not available'); + }, + }); + + expect(diagnostics.v8HeapSpaces).toBeNull(); + expect(diagnostics.openFileDescriptors).toBeNull(); + expect(diagnostics.smapsRollup).toBeNull(); + expect(diagnostics.analysis.risks).toEqual([]); + expect(diagnostics.analysis.recommendation).toBe( + 'No obvious leak indicators detected.', + ); + expect(diagnostics.analysis.recommendation).not.toContain('heap snapshot'); + expect(debugLogger.debug).toHaveBeenCalledWith( + expect.stringContaining('heapSpaceStatistics'), + expect.any(Error), + ); + expect(debugLogger.debug).toHaveBeenCalledWith( + expect.stringContaining('openFileDescriptors'), + expect.any(Error), + ); + expect(debugLogger.debug).toHaveBeenCalledWith( + expect.stringContaining('smapsRollup'), + expect.any(Error), + ); + }); + + it('treats active handle and request probe failures as zero counts', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 100, + heapTotal: 200, + rss: 300, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 1_000, + total_heap_size: 200, + total_heap_size_executable: 0, + total_physical_size: 200, + used_heap_size: 100, + malloced_memory: 0, + peak_malloced_memory: 0, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 900, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + activeHandles: () => { + throw new Error('handles unavailable'); + }, + activeRequests: () => { + throw new Error('requests unavailable'); + }, + }); + + expect(diagnostics.activeHandles).toBe(0); + expect(diagnostics.activeRequests).toBe(0); + expect(diagnostics.analysis.risks).toEqual([]); + }); + + it('logs unavailable Node.js internal active probes before returning zero counts', async () => { + const internals = process as typeof process & { + _getActiveHandles?: () => unknown[]; + _getActiveRequests?: () => unknown[]; + }; + const originalGetActiveHandles = internals._getActiveHandles; + const originalGetActiveRequests = internals._getActiveRequests; + internals._getActiveHandles = undefined; + internals._getActiveRequests = undefined; + + try { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 100, + heapTotal: 200, + rss: 300, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 1_000, + total_heap_size: 200, + total_heap_size_executable: 0, + total_physical_size: 200, + used_heap_size: 100, + malloced_memory: 0, + peak_malloced_memory: 0, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 900, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + }); + + expect(diagnostics.activeHandles).toBe(0); + expect(diagnostics.activeRequests).toBe(0); + expect(debugLogger.debug).toHaveBeenCalledWith( + expect.stringContaining('activeHandles'), + expect.any(Error), + ); + expect(debugLogger.debug).toHaveBeenCalledWith( + expect.stringContaining('activeRequests'), + expect.any(Error), + ); + } finally { + internals._getActiveHandles = originalGetActiveHandles; + internals._getActiveRequests = originalGetActiveRequests; + } + }); + + it('starts independent optional probes before awaiting slow probes', async () => { + let resolveFileDescriptors: ((count: number) => void) | undefined; + const fileDescriptors = new Promise((resolve) => { + resolveFileDescriptors = resolve; + }); + let smapsStarted = false; + let heapSpacesStarted = false; + + const diagnosticsPromise = collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 100, + heapTotal: 200, + rss: 300, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 1_000, + total_heap_size: 200, + total_heap_size_executable: 0, + total_physical_size: 200, + used_heap_size: 100, + malloced_memory: 0, + peak_malloced_memory: 0, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 900, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + heapSpaceStatistics: () => { + heapSpacesStarted = true; + return []; + }, + activeHandles: () => 0, + activeRequests: () => 0, + openFileDescriptors: () => fileDescriptors, + smapsRollup: async () => { + smapsStarted = true; + return 'Rss: 300 kB'; + }, + }); + + await Promise.resolve(); + expect(smapsStarted).toBe(true); + expect(heapSpacesStarted).toBe(true); + + resolveFileDescriptors?.(4); + const diagnostics = await diagnosticsPromise; + + expect(diagnostics.openFileDescriptors).toBe(4); + expect(diagnostics.smapsRollup).toBe('Rss: 300 kB'); + }); + + it('flags unusually high active requests', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 100, + heapTotal: 200, + rss: 300, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 1_000, + total_heap_size: 200, + total_heap_size_executable: 0, + total_physical_size: 200, + used_heap_size: 100, + malloced_memory: 0, + peak_malloced_memory: 0, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 900, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + activeRequests: () => 101, + }); + + expect(diagnostics.analysis.risks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'active-requests' }), + ]), + ); + }); + + it('does not flag native pressure from normal RSS overhead alone', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 5 * 1024 * 1024, + heapTotal: 8 * 1024 * 1024, + rss: 50 * 1024 * 1024, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 512 * 1024 * 1024, + total_heap_size: 8 * 1024 * 1024, + total_heap_size_executable: 0, + total_physical_size: 8 * 1024 * 1024, + used_heap_size: 5 * 1024 * 1024, + malloced_memory: 512 * 1024, + peak_malloced_memory: 1024 * 1024, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 500 * 1024 * 1024, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + }); + + expect(diagnostics.analysis.risks).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'native-memory-pressure' }), + ]), + ); + }); + + it('flags RSS that is much larger than JS heap with a high floor', async () => { + const diagnostics = await collectMemoryDiagnostics({ + memoryUsage: () => ({ + heapUsed: 50 * 1024 * 1024, + heapTotal: 64 * 1024 * 1024, + rss: 800 * 1024 * 1024, + external: 10, + arrayBuffers: 5, + }), + heapStatistics: () => ({ + heap_size_limit: 512 * 1024 * 1024, + total_heap_size: 64 * 1024 * 1024, + total_heap_size_executable: 0, + total_physical_size: 64 * 1024 * 1024, + used_heap_size: 50 * 1024 * 1024, + malloced_memory: 512 * 1024, + peak_malloced_memory: 1024 * 1024, + does_zap_garbage: 0, + number_of_native_contexts: 1, + number_of_detached_contexts: 0, + total_available_size: 450 * 1024 * 1024, + total_global_handles_size: 0, + used_global_handles_size: 0, + external_memory: 10, + }), + activeHandles: () => 0, + activeRequests: () => 0, + }); + + expect(diagnostics.analysis.risks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'rss-heap-gap', + message: expect.stringContaining('800.0 MB'), + }), + ]), + ); + }); +}); diff --git a/packages/core/src/utils/memoryDiagnostics.ts b/packages/core/src/utils/memoryDiagnostics.ts new file mode 100644 index 0000000000..2ebe3c88e6 --- /dev/null +++ b/packages/core/src/utils/memoryDiagnostics.ts @@ -0,0 +1,346 @@ +/** + * @license + * Copyright 2025 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import { readdir, readFile } from 'node:fs/promises'; +import process from 'node:process'; +import v8 from 'node:v8'; +import { createDebugLogger } from './debugLogger.js'; +import { formatMemoryUsage } from './formatters.js'; + +const RSS_HEAP_GAP_RATIO = 10; +const RSS_HEAP_GAP_MIN_BYTES = 256 * 1024 * 1024; +// Native pressure can look extreme during early startup when heap is tiny. +// Require an absolute floor before the ratio check so cold processes don't +// flag spurious risks. +const NATIVE_MEMORY_PRESSURE_MIN_BYTES = 64 * 1024 * 1024; +const ACTIVE_HANDLES_THRESHOLD = 256; +const ACTIVE_REQUESTS_THRESHOLD = 100; +const OPEN_FD_THRESHOLD = 500; +const debugLogger = createDebugLogger('MEMORY_DIAGNOSTICS'); + +export interface MemoryDiagnostics { + timestamp: string; + sessionId?: string; + qwenVersion?: string; + uptimeSeconds: number; + memoryUsage: NodeJS.MemoryUsage; + v8HeapStats: V8HeapStats; + v8HeapSpaces: V8HeapSpaceStats[] | null; + resourceUsage: MemoryResourceUsage; + activeHandles: number; + activeRequests: number; + openFileDescriptors: number | null; + smapsRollup: string | null; + platform: NodeJS.Platform; + nodeVersion: string; + analysis: MemoryDiagnosticsAnalysis; +} + +export interface V8HeapStats { + heapSizeLimit: number; + totalHeapSize: number; + usedHeapSize: number; + mallocedMemory: number; + peakMallocedMemory: number; + detachedContexts: number; + nativeContexts: number; +} + +export interface V8HeapSpaceStats { + name: string; + size: number; + used: number; + available: number; +} + +export interface MemoryResourceUsage { + maxRSS: number; + userCPUTime: number; + systemCPUTime: number; +} + +export interface MemoryDiagnosticsAnalysis { + risks: MemoryRisk[]; + recommendation: string; +} + +export interface MemoryRisk { + type: + | 'heap-pressure' + | 'detached-contexts' + | 'active-handles' + | 'active-requests' + | 'fd-leak' + | 'native-memory-pressure' + | 'rss-heap-gap'; + message: string; +} + +export interface MemoryDiagnosticsOptions { + now?: () => Date; + sessionId?: string; + qwenVersion?: string; + memoryUsage?: () => NodeJS.MemoryUsage; + heapStatistics?: () => v8.HeapInfo; + heapSpaceStatistics?: () => v8.HeapSpaceInfo[]; + resourceUsage?: () => NodeJS.ResourceUsage; + uptimeSeconds?: () => number; + activeHandles?: () => number; + activeRequests?: () => number; + openFileDescriptors?: () => Promise; + smapsRollup?: () => Promise; + platform?: NodeJS.Platform; + nodeVersion?: string; +} + +// `_getActiveHandles` / `_getActiveRequests` are undocumented Node internals. +// They've been stable for years but are not part of the public API and could +// change in a future Node release. Both call sites guard with try/catch and +// fall back to 0, so a removal degrades gracefully. +interface ProcessInternals { + _getActiveHandles?: () => unknown; + _getActiveRequests?: () => unknown; +} + +export async function collectMemoryDiagnostics( + options: MemoryDiagnosticsOptions = {}, +): Promise { + const now = options.now ?? (() => new Date()); + const platform = options.platform ?? process.platform; + const memoryUsage = options.memoryUsage?.() ?? process.memoryUsage(); + const heapStatistics = options.heapStatistics?.() ?? v8.getHeapStatistics(); + const resourceUsage = options.resourceUsage?.() ?? process.resourceUsage(); + const uptimeSeconds = options.uptimeSeconds?.() ?? process.uptime(); + const [openFileDescriptors, smapsRollup, heapSpaceStatistics] = + await Promise.all([ + optionalProbe( + 'openFileDescriptors', + options.openFileDescriptors ?? countOpenFileDescriptors, + ), + optionalProbe('smapsRollup', options.smapsRollup ?? readProcSmapsRollup), + optionalSyncProbe( + 'heapSpaceStatistics', + options.heapSpaceStatistics ?? (() => v8.getHeapSpaceStatistics()), + ), + ]); + const v8HeapSpaces = mapHeapSpaces(heapSpaceStatistics); + + // Node.js >=14.10.0 returns maxRSS in bytes on all platforms. + // This project requires Node >=22. + const maxRSSBytes = resourceUsage.maxRSS; + + const diagnostics = { + timestamp: now().toISOString(), + sessionId: options.sessionId, + qwenVersion: options.qwenVersion, + uptimeSeconds, + memoryUsage, + v8HeapStats: mapHeapStats(heapStatistics), + v8HeapSpaces, + resourceUsage: { + maxRSS: maxRSSBytes, + userCPUTime: resourceUsage.userCPUTime, + systemCPUTime: resourceUsage.systemCPUTime, + }, + activeHandles: getProcessInternalCount( + 'activeHandles', + '_getActiveHandles', + options.activeHandles, + ), + activeRequests: getProcessInternalCount( + 'activeRequests', + '_getActiveRequests', + options.activeRequests, + ), + openFileDescriptors, + smapsRollup, + platform, + nodeVersion: options.nodeVersion ?? process.version, + } satisfies Omit; + + return { + ...diagnostics, + analysis: analyzeMemoryDiagnostics(diagnostics), + }; +} + +function mapHeapStats(heapInfo: v8.HeapInfo): V8HeapStats { + return { + heapSizeLimit: heapInfo.heap_size_limit, + totalHeapSize: heapInfo.total_heap_size, + usedHeapSize: heapInfo.used_heap_size, + mallocedMemory: heapInfo.malloced_memory, + peakMallocedMemory: heapInfo.peak_malloced_memory, + detachedContexts: heapInfo.number_of_detached_contexts, + nativeContexts: heapInfo.number_of_native_contexts, + }; +} + +function mapHeapSpaces( + heapSpaces: v8.HeapSpaceInfo[] | null, +): V8HeapSpaceStats[] | null { + if (!heapSpaces) { + return null; + } + return heapSpaces.map((space) => ({ + name: space.space_name, + size: space.space_size, + used: space.space_used_size, + available: space.space_available_size, + })); +} + +function getProcessInternalCount( + name: 'activeHandles' | 'activeRequests', + internalMethod: '_getActiveHandles' | '_getActiveRequests', + probe?: () => number, +): number { + try { + if (probe) { + return probe(); + } + const internals = process as unknown as ProcessInternals; + const internalProbe = internals[internalMethod]; + if (typeof internalProbe !== 'function') { + logProbeFailure(name, new Error(`${internalMethod} is unavailable`)); + return 0; + } + + const result = internalProbe(); + if (!Array.isArray(result)) { + logProbeFailure( + name, + new Error(`${internalMethod} returned a non-array result`), + ); + return 0; + } + + return result.length; + } catch (error) { + logProbeFailure(name, error); + return 0; + } +} + +async function countOpenFileDescriptors(): Promise { + return (await readdir('/proc/self/fd')).length; +} + +async function readProcSmapsRollup(): Promise { + return readFile('/proc/self/smaps_rollup', 'utf8'); +} + +async function optionalProbe( + name: string, + probe: () => Promise, +): Promise { + try { + return await probe(); + } catch (error) { + logProbeFailure(name, error); + return null; + } +} + +async function optionalSyncProbe( + name: string, + probe: () => T, +): Promise { + try { + return probe(); + } catch (error) { + logProbeFailure(name, error); + return null; + } +} + +function logProbeFailure(name: string, error: unknown): void { + debugLogger.debug(`memory diagnostics probe failed: ${name}`, error); +} + +function analyzeMemoryDiagnostics( + diagnostics: Omit, +): MemoryDiagnosticsAnalysis { + const risks: MemoryRisk[] = []; + const heapRatio = + diagnostics.v8HeapStats.heapSizeLimit > 0 + ? diagnostics.v8HeapStats.usedHeapSize / + diagnostics.v8HeapStats.heapSizeLimit + : 0; + + if (heapRatio >= 0.75) { + risks.push({ + type: 'heap-pressure', + message: `Heap usage is ${(heapRatio * 100).toFixed(1)}% of the V8 limit.`, + }); + } + + if (diagnostics.v8HeapStats.detachedContexts > 0) { + risks.push({ + type: 'detached-contexts', + message: `${diagnostics.v8HeapStats.detachedContexts} detached V8 context(s) detected.`, + }); + } + + if (diagnostics.activeHandles > ACTIVE_HANDLES_THRESHOLD) { + risks.push({ + type: 'active-handles', + message: `${diagnostics.activeHandles} active handle(s) detected.`, + }); + } + + if (diagnostics.activeRequests > ACTIVE_REQUESTS_THRESHOLD) { + risks.push({ + type: 'active-requests', + message: `${diagnostics.activeRequests} active request(s) detected.`, + }); + } + + if ( + diagnostics.openFileDescriptors !== null && + diagnostics.openFileDescriptors > OPEN_FD_THRESHOLD + ) { + risks.push({ + type: 'fd-leak', + message: `${diagnostics.openFileDescriptors} open file descriptor(s) detected.`, + }); + } + + // Use mallocedMemory instead of rss - heapUsed. RSS includes normal process + // overhead such as code segments, shared libraries, stacks, and mapped files, + // which creates false positives on healthy Node.js processes. Also gate on + // an absolute floor so tiny startup heaps don't trip the 2× ratio. + const nativeMemory = diagnostics.v8HeapStats.mallocedMemory; + if ( + nativeMemory >= NATIVE_MEMORY_PRESSURE_MIN_BYTES && + nativeMemory > diagnostics.memoryUsage.heapUsed * 2 + ) { + risks.push({ + type: 'native-memory-pressure', + message: `V8 native malloced memory (${formatMemoryUsage(nativeMemory)}) is more than 2× heap used (${formatMemoryUsage(diagnostics.memoryUsage.heapUsed)}).`, + }); + } + + if ( + diagnostics.memoryUsage.heapUsed > 0 && + diagnostics.memoryUsage.rss >= RSS_HEAP_GAP_MIN_BYTES && + diagnostics.memoryUsage.rss > + diagnostics.memoryUsage.heapUsed * RSS_HEAP_GAP_RATIO + ) { + risks.push({ + type: 'rss-heap-gap', + message: `RSS (${formatMemoryUsage(diagnostics.memoryUsage.rss)}) is more than ${RSS_HEAP_GAP_RATIO}× heap used (${formatMemoryUsage(diagnostics.memoryUsage.heapUsed)}). Check native addons, libuv buffers, mapped files, or retained tool output.`, + }); + } + + return { + risks, + recommendation: + risks.length > 0 + ? `${risks.length} potential leak indicator(s) found.` + : 'No obvious leak indicators detected.', + }; +}