diff --git a/.qwen/skills/terminal-capture/SKILL.md b/.qwen/skills/terminal-capture/SKILL.md index adf8fff13..7fc99a18d 100644 --- a/.qwen/skills/terminal-capture/SKILL.md +++ b/.qwen/skills/terminal-capture/SKILL.md @@ -109,6 +109,38 @@ Supported key names: `ArrowUp`, `ArrowDown`, `ArrowLeft`, `ArrowRight`, `Enter`, Auto-screenshot is triggered after the key sequence ends (when the next step is not a `key`). +### `streaming` — Capture During Execution + +Capture multiple screenshots at intervals during long-running output (e.g., progress bars). Optionally generates an animated GIF. + +```typescript +{ + type: 'Run this command: bash progress.sh', + streaming: { + delayMs: 7000, // Wait before first capture (skip initial waiting phase) + intervalMs: 500, // Interval between captures + count: 20, // Maximum number of captures + gif: true, // Generate animated GIF (default: true, requires ffmpeg) + }, +} +``` + +- `delayMs` (optional): Milliseconds to wait after pressing Enter before starting captures. Useful for skipping model thinking/approval time. +- Captures stop early if terminal output is unchanged for 3 consecutive intervals. +- Duplicate frames (no output change) are automatically skipped. + +**GIF prerequisite**: If the scenario uses `streaming` with GIF enabled (default), check if `ffmpeg` is installed before running. If not, ask the user whether they'd like to install it: + +```bash +# Check +which ffmpeg + +# Install (macOS) +brew install ffmpeg +``` + +If the user declines, the scenario still runs — GIF generation is skipped with a warning. + ### `capture` / `captureFull` — Explicit Screenshot Use as a standalone step, or override automatic naming: @@ -178,20 +210,32 @@ This tool is commonly used for visual verification during PR reviews. For the co ## Full ScenarioConfig Type ```typescript -interface ScenarioConfig { - name: string; // Scenario name (also used as screenshot subdirectory name) - spawn: string[]; // Launch command ["node", "dist/cli.js", "--yolo"] - flow: FlowStep[]; // Interaction steps - terminal?: { - // Terminal configuration (all optional) - cols?: number; // Number of columns, default 100 - rows?: number; // Number of rows, default 28 - theme?: string; // Theme: dracula|one-dark|github-dark|monokai|night-owl - chrome?: boolean; // macOS window decorations, default true - title?: string; // Window title, default "Terminal" - fontSize?: number; // Font size - cwd?: string; // Working directory (relative to config file) +interface FlowStep { + type?: string; // Input text + key?: string | string[]; // Key press(es) + capture?: string; // Viewport screenshot filename + captureFull?: string; // Full scrollback screenshot filename + streaming?: { + delayMs?: number; // Delay before first capture (default: 0) + intervalMs: number; // Interval between captures in ms + count: number; // Maximum number of captures + gif?: boolean; // Generate animated GIF (default: true) }; - outputDir?: string; // Screenshot output directory (relative to config file) +} + +interface ScenarioConfig { + name: string; // Scenario name (also used as screenshot subdirectory name) + spawn: string[]; // Launch command ["node", "dist/cli.js", "--yolo"] + flow: FlowStep[]; // Interaction steps + terminal?: { + cols?: number; // Number of columns, default 100 + rows?: number; // Number of rows, default 28 + theme?: string; // Theme: dracula|one-dark|github-dark|monokai|night-owl + chrome?: boolean; // macOS window decorations, default true + title?: string; // Window title, default "Terminal" + fontSize?: number; // Font size + cwd?: string; // Working directory (relative to config file) + }; + outputDir?: string; // Screenshot output directory (relative to config file) } ``` diff --git a/integration-tests/terminal-capture/motivation.md b/integration-tests/terminal-capture/motivation.md index 388019369..3d004ddee 100644 --- a/integration-tests/terminal-capture/motivation.md +++ b/integration-tests/terminal-capture/motivation.md @@ -40,6 +40,10 @@ Playwright element screenshot | WYSIWYG | xterm.js fully renders ANSI, no manual output cleaning needed | | Theme Support | Built-in 5 themes (Dracula, One Dark, GitHub Dark, Monokai, Night Owl) | | Full-length | `captureFull()` supports capturing scrollback buffer content | +| Streaming Capture | Capture multiple frames at intervals during execution (e.g., progress bars) | +| Animated GIF | Auto-generate GIF from streaming frames via ffmpeg | +| Early Stop | Streaming stops early if output stabilizes; duplicate frames are skipped | +| Auto Cleanup | Output directory is cleared before each run to prevent stale screenshots | | Deterministic Naming | Screenshot filenames auto-generated by step sequence for easy regression comparison | | Batch Execution | `run.ts` executes all scenarios in one command | @@ -90,8 +94,14 @@ scenarios/screenshots/ 02-01.png # Step 2 input state 02-02.png # Step 2 result full-flow.png # Final state full-length image - context/ + streaming-shell/ + 01-01.png # Input state + 01-streaming-01.png # Streaming frame 1 + 01-streaming-02.png # Streaming frame 2 ... + 01-02.png # Final result + streaming.gif # Animated GIF (requires ffmpeg) + full-flow.png # Final state full-length image ``` ## 4. Position in Testing System diff --git a/integration-tests/terminal-capture/scenario-runner.ts b/integration-tests/terminal-capture/scenario-runner.ts index 4bd858fd4..b77108dd4 100644 --- a/integration-tests/terminal-capture/scenario-runner.ts +++ b/integration-tests/terminal-capture/scenario-runner.ts @@ -10,7 +10,9 @@ */ import { TerminalCapture, THEMES } from './terminal-capture.js'; -import { dirname, resolve, isAbsolute } from 'node:path'; +import { dirname, resolve, isAbsolute, join } from 'node:path'; +import { execSync } from 'node:child_process'; +import { writeFileSync, unlinkSync, rmSync, existsSync } from 'node:fs'; // ───────────────────────────────────────────── // Schema — Minimal @@ -29,6 +31,20 @@ export interface FlowStep { capture?: string; /** Explicit screenshot: full scrollback buffer long image (standalone capture when no type) */ captureFull?: string; + /** + * Streaming capture: capture multiple screenshots during execution at intervals. + * Useful for demonstrating real-time output like progress bars. + */ + streaming?: { + /** Delay before starting captures in milliseconds (skip initial waiting phase) */ + delayMs?: number; + /** Interval between captures in milliseconds */ + intervalMs: number; + /** Maximum number of captures */ + count: number; + /** Generate animated GIF from captured frames (default: true) */ + gif?: boolean; + }; } export interface ScenarioConfig { @@ -105,6 +121,11 @@ export async function runScenario( ? resolve(basedir, config.outputDir, scenarioDir) : resolve(basedir, 'screenshots', scenarioDir); + // Clean previous screenshots + if (existsSync(outputDir)) { + rmSync(outputDir, { recursive: true }); + } + console.log(`\n${'═'.repeat(60)}`); console.log(`▶ ${config.name}`); console.log('═'.repeat(60)); @@ -171,13 +192,95 @@ export async function runScenario( if (autoEnter) { // ── Auto-press Enter → Wait for stabilization → 02 screenshot ── await terminal.type('\n'); - console.log(` ⏳ waiting for output to settle...`); - await terminal.idle(2000, 60000); - console.log(` ✅ settled`); - const resultName = step.capture ?? `${pad(seq)}-02.png`; - console.log(` ${label} 📸 result: ${resultName}`); - screenshots.push(await terminal.capture(resultName)); + // Streaming capture: capture multiple screenshots during execution + if (step.streaming) { + const { + delayMs = 0, + intervalMs, + count, + gif = true, + } = step.streaming; + console.log( + ` 🎬 streaming capture: ${count} shots @ ${intervalMs}ms intervals${delayMs ? ` (delay ${delayMs}ms)` : ''}`, + ); + + // Wait before starting captures (skip initial waiting phase) + if (delayMs > 0) { + await sleep(delayMs); + } + + // Capture frames at intervals (stop early if output stabilizes) + const streamingShots: string[] = []; + let prevOutputLen = terminal.getRawOutput().length; + let stableCount = 0; + let shotNum = 0; + for (let j = 0; j < count; j++) { + await sleep(intervalMs); + const curOutputLen = terminal.getRawOutput().length; + if (curOutputLen === prevOutputLen) { + stableCount++; + if (stableCount >= 3) { + console.log( + ` ⏹️ streaming stopped early: output stable for ${stableCount} intervals`, + ); + break; + } + continue; // skip duplicate frame + } + stableCount = 0; + prevOutputLen = curOutputLen; + shotNum++; + const shotName = `${pad(seq)}-streaming-${pad(shotNum)}.png`; + console.log( + ` 📸 streaming [${shotNum}/${count}]: ${shotName}`, + ); + const shot = await terminal.capture(shotName); + streamingShots.push(shot); + screenshots.push(shot); + } + + // Wait for completion after streaming captures + console.log(` ⏳ waiting for output to settle...`); + await terminal.idle(2000, 60000); + console.log(` ✅ settled`); + + const resultName = step.capture ?? `${pad(seq)}-02.png`; + console.log(` ${label} 📸 result: ${resultName}`); + const resultShot = await terminal.capture(resultName); + screenshots.push(resultShot); + + // Generate animated GIF: input -> streaming frames -> result + if (gif && streamingShots.length > 0) { + // Include input and result in the GIF for complete story + const inputShot = screenshots.find((s) => + s.endsWith(`${pad(seq)}-01.png`), + ); + const gifFrames = [ + ...(inputShot ? [inputShot] : []), + ...streamingShots, + resultShot, + ]; + const gifPath = generateGif( + gifFrames, + outputDir, + intervalMs, + inputShot ? 2000 : 0, // Hold input for 2s + 2000, // Hold result for 2s + ); + if (gifPath) { + console.log(` 🎞️ GIF: ${gifPath}`); + } + } + } else { + console.log(` ⏳ waiting for output to settle...`); + await terminal.idle(2000, 60000); + console.log(` ✅ settled`); + + const resultName = step.capture ?? `${pad(seq)}-02.png`; + console.log(` ${label} 📸 result: ${resultName}`); + screenshots.push(await terminal.capture(resultName)); + } // full-flow: Only the last type step auto-captures full-length image const isLastType = !config.flow.slice(i + 1).some((s) => s.type); @@ -302,3 +405,47 @@ const KEY_MAP: Record = { function resolveKey(key: string): string { return KEY_MAP[key] ?? key; } + +/** Generate animated GIF from PNG frames using ffmpeg (concat demuxer). */ +function generateGif( + frames: string[], + outputDir: string, + intervalMs: number, + holdFirstMs: number = 0, + holdLastMs: number = 0, +): string | null { + if (frames.length === 0) return null; + + const gifPath = join(outputDir, 'streaming.gif'); + const listFile = join(outputDir, 'frames.txt'); + + try { + const lines: string[] = []; + for (let i = 0; i < frames.length; i++) { + const absPath = resolve(frames[i]); + let duration = intervalMs / 1000; + if (i === 0 && holdFirstMs > 0) duration = holdFirstMs / 1000; + else if (i === frames.length - 1 && holdLastMs > 0) + duration = holdLastMs / 1000; + lines.push(`file '${absPath}'`, `duration ${duration}`); + } + // Concat demuxer requires last frame repeated without duration + lines.push(`file '${resolve(frames[frames.length - 1])}'`); + writeFileSync(listFile, lines.join('\n')); + + execSync( + `ffmpeg -y -f concat -safe 0 -i "${listFile}" -vf "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 "${gifPath}"`, + { stdio: 'pipe' }, + ); + return gifPath; + } catch { + console.log(' ⚠️ GIF generation requires ffmpeg'); + return null; + } finally { + try { + unlinkSync(listFile); + } catch { + // ignore + } + } +} diff --git a/integration-tests/terminal-capture/scenarios/streaming-shell.ts b/integration-tests/terminal-capture/scenarios/streaming-shell.ts new file mode 100644 index 000000000..e166d9a0d --- /dev/null +++ b/integration-tests/terminal-capture/scenarios/streaming-shell.ts @@ -0,0 +1,24 @@ +import type { ScenarioConfig } from '../scenario-runner.js'; + +/** + * Demonstrates streaming shell execution output with PTY enabled by default. + * Tests the render throttle behavior and progress bar handling. + * Captures multiple screenshots during execution to show real-time output. + */ +export default { + name: 'streaming-shell', + spawn: ['node', 'dist/cli.js', '--yolo'], + terminal: { title: 'qwen-code', cwd: '../../..' }, + flow: [ + { + type: 'Run this command: bash integration-tests/terminal-capture/scenarios/progress.sh', + // Capture 20 screenshots at 500ms intervals during execution + // The progress.sh script takes ~10 seconds (20 iterations * 0.5s each) + streaming: { + delayMs: 7000, + intervalMs: 500, + count: 20, + }, + }, + ], +} satisfies ScenarioConfig; diff --git a/package-lock.json b/package-lock.json index 96b91827f..44590ebd4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "@types/uuid": "^10.0.0", "@vitest/coverage-v8": "^3.1.1", "@vitest/eslint-plugin": "^1.3.4", + "@xterm/xterm": "^6.0.0", "cross-env": "^7.0.3", "esbuild": "^0.25.0", "eslint": "^9.24.0", @@ -5629,6 +5630,16 @@ "integrity": "sha512-5xXB7kdQlFBP82ViMJTwwEc3gKCLGKR/eoxQm4zge7GPBl86tCdI0IdPJjoKd8mUSFXz5V7i/25sfsEkP4j46g==", "license": "MIT" }, + "node_modules/@xterm/xterm": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-6.0.0.tgz", + "integrity": "sha512-TQwDdQGtwwDt+2cgKDLn0IRaSxYu1tSUjgKarSDkUM0ZNiSRXFpjxEsvc/Zgc5kq5omJ+V0a8/kIM2WD3sMOYg==", + "dev": true, + "license": "MIT", + "workspaces": [ + "addons/*" + ] + }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", diff --git a/package.json b/package.json index 3b01bc667..e7caedb81 100644 --- a/package.json +++ b/package.json @@ -84,6 +84,7 @@ "@types/uuid": "^10.0.0", "@vitest/coverage-v8": "^3.1.1", "@vitest/eslint-plugin": "^1.3.4", + "@xterm/xterm": "^6.0.0", "cross-env": "^7.0.3", "esbuild": "^0.25.0", "eslint": "^9.24.0",