mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-04-28 03:30:40 +00:00
feat(terminal-capture): add streaming capture with GIF generation
Add ability to capture multiple screenshots at intervals during long-running terminal output (e.g., progress bars). Optionally generates animated GIFs from captured frames using ffmpeg. Features: - Streaming capture at configurable intervals - Early stop when output stabilizes (3 consecutive unchanged frames) - Duplicate frame skipping - Animated GIF generation via ffmpeg concat demuxer - Auto-cleanup of output directory before each run - Configurable delay before starting captures Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
tanzhenxin
parent
991ae9febc
commit
b8a7ac830d
6 changed files with 259 additions and 22 deletions
|
|
@ -109,6 +109,38 @@ Supported key names: `ArrowUp`, `ArrowDown`, `ArrowLeft`, `ArrowRight`, `Enter`,
|
|||
|
||||
Auto-screenshot is triggered after the key sequence ends (when the next step is not a `key`).
|
||||
|
||||
### `streaming` — Capture During Execution
|
||||
|
||||
Capture multiple screenshots at intervals during long-running output (e.g., progress bars). Optionally generates an animated GIF.
|
||||
|
||||
```typescript
|
||||
{
|
||||
type: 'Run this command: bash progress.sh',
|
||||
streaming: {
|
||||
delayMs: 7000, // Wait before first capture (skip initial waiting phase)
|
||||
intervalMs: 500, // Interval between captures
|
||||
count: 20, // Maximum number of captures
|
||||
gif: true, // Generate animated GIF (default: true, requires ffmpeg)
|
||||
},
|
||||
}
|
||||
```
|
||||
|
||||
- `delayMs` (optional): Milliseconds to wait after pressing Enter before starting captures. Useful for skipping model thinking/approval time.
|
||||
- Captures stop early if terminal output is unchanged for 3 consecutive intervals.
|
||||
- Duplicate frames (no output change) are automatically skipped.
|
||||
|
||||
**GIF prerequisite**: If the scenario uses `streaming` with GIF enabled (default), check if `ffmpeg` is installed before running. If not, ask the user whether they'd like to install it:
|
||||
|
||||
```bash
|
||||
# Check
|
||||
which ffmpeg
|
||||
|
||||
# Install (macOS)
|
||||
brew install ffmpeg
|
||||
```
|
||||
|
||||
If the user declines, the scenario still runs — GIF generation is skipped with a warning.
|
||||
|
||||
### `capture` / `captureFull` — Explicit Screenshot
|
||||
|
||||
Use as a standalone step, or override automatic naming:
|
||||
|
|
@ -178,20 +210,32 @@ This tool is commonly used for visual verification during PR reviews. For the co
|
|||
## Full ScenarioConfig Type
|
||||
|
||||
```typescript
|
||||
interface ScenarioConfig {
|
||||
name: string; // Scenario name (also used as screenshot subdirectory name)
|
||||
spawn: string[]; // Launch command ["node", "dist/cli.js", "--yolo"]
|
||||
flow: FlowStep[]; // Interaction steps
|
||||
terminal?: {
|
||||
// Terminal configuration (all optional)
|
||||
cols?: number; // Number of columns, default 100
|
||||
rows?: number; // Number of rows, default 28
|
||||
theme?: string; // Theme: dracula|one-dark|github-dark|monokai|night-owl
|
||||
chrome?: boolean; // macOS window decorations, default true
|
||||
title?: string; // Window title, default "Terminal"
|
||||
fontSize?: number; // Font size
|
||||
cwd?: string; // Working directory (relative to config file)
|
||||
interface FlowStep {
|
||||
type?: string; // Input text
|
||||
key?: string | string[]; // Key press(es)
|
||||
capture?: string; // Viewport screenshot filename
|
||||
captureFull?: string; // Full scrollback screenshot filename
|
||||
streaming?: {
|
||||
delayMs?: number; // Delay before first capture (default: 0)
|
||||
intervalMs: number; // Interval between captures in ms
|
||||
count: number; // Maximum number of captures
|
||||
gif?: boolean; // Generate animated GIF (default: true)
|
||||
};
|
||||
outputDir?: string; // Screenshot output directory (relative to config file)
|
||||
}
|
||||
|
||||
interface ScenarioConfig {
|
||||
name: string; // Scenario name (also used as screenshot subdirectory name)
|
||||
spawn: string[]; // Launch command ["node", "dist/cli.js", "--yolo"]
|
||||
flow: FlowStep[]; // Interaction steps
|
||||
terminal?: {
|
||||
cols?: number; // Number of columns, default 100
|
||||
rows?: number; // Number of rows, default 28
|
||||
theme?: string; // Theme: dracula|one-dark|github-dark|monokai|night-owl
|
||||
chrome?: boolean; // macOS window decorations, default true
|
||||
title?: string; // Window title, default "Terminal"
|
||||
fontSize?: number; // Font size
|
||||
cwd?: string; // Working directory (relative to config file)
|
||||
};
|
||||
outputDir?: string; // Screenshot output directory (relative to config file)
|
||||
}
|
||||
```
|
||||
|
|
|
|||
|
|
@ -40,6 +40,10 @@ Playwright element screenshot
|
|||
| WYSIWYG | xterm.js fully renders ANSI, no manual output cleaning needed |
|
||||
| Theme Support | Built-in 5 themes (Dracula, One Dark, GitHub Dark, Monokai, Night Owl) |
|
||||
| Full-length | `captureFull()` supports capturing scrollback buffer content |
|
||||
| Streaming Capture | Capture multiple frames at intervals during execution (e.g., progress bars) |
|
||||
| Animated GIF | Auto-generate GIF from streaming frames via ffmpeg |
|
||||
| Early Stop | Streaming stops early if output stabilizes; duplicate frames are skipped |
|
||||
| Auto Cleanup | Output directory is cleared before each run to prevent stale screenshots |
|
||||
| Deterministic Naming | Screenshot filenames auto-generated by step sequence for easy regression comparison |
|
||||
| Batch Execution | `run.ts` executes all scenarios in one command |
|
||||
|
||||
|
|
@ -90,8 +94,14 @@ scenarios/screenshots/
|
|||
02-01.png # Step 2 input state
|
||||
02-02.png # Step 2 result
|
||||
full-flow.png # Final state full-length image
|
||||
context/
|
||||
streaming-shell/
|
||||
01-01.png # Input state
|
||||
01-streaming-01.png # Streaming frame 1
|
||||
01-streaming-02.png # Streaming frame 2
|
||||
...
|
||||
01-02.png # Final result
|
||||
streaming.gif # Animated GIF (requires ffmpeg)
|
||||
full-flow.png # Final state full-length image
|
||||
```
|
||||
|
||||
## 4. Position in Testing System
|
||||
|
|
|
|||
|
|
@ -10,7 +10,9 @@
|
|||
*/
|
||||
|
||||
import { TerminalCapture, THEMES } from './terminal-capture.js';
|
||||
import { dirname, resolve, isAbsolute } from 'node:path';
|
||||
import { dirname, resolve, isAbsolute, join } from 'node:path';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { writeFileSync, unlinkSync, rmSync, existsSync } from 'node:fs';
|
||||
|
||||
// ─────────────────────────────────────────────
|
||||
// Schema — Minimal
|
||||
|
|
@ -29,6 +31,20 @@ export interface FlowStep {
|
|||
capture?: string;
|
||||
/** Explicit screenshot: full scrollback buffer long image (standalone capture when no type) */
|
||||
captureFull?: string;
|
||||
/**
|
||||
* Streaming capture: capture multiple screenshots during execution at intervals.
|
||||
* Useful for demonstrating real-time output like progress bars.
|
||||
*/
|
||||
streaming?: {
|
||||
/** Delay before starting captures in milliseconds (skip initial waiting phase) */
|
||||
delayMs?: number;
|
||||
/** Interval between captures in milliseconds */
|
||||
intervalMs: number;
|
||||
/** Maximum number of captures */
|
||||
count: number;
|
||||
/** Generate animated GIF from captured frames (default: true) */
|
||||
gif?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export interface ScenarioConfig {
|
||||
|
|
@ -105,6 +121,11 @@ export async function runScenario(
|
|||
? resolve(basedir, config.outputDir, scenarioDir)
|
||||
: resolve(basedir, 'screenshots', scenarioDir);
|
||||
|
||||
// Clean previous screenshots
|
||||
if (existsSync(outputDir)) {
|
||||
rmSync(outputDir, { recursive: true });
|
||||
}
|
||||
|
||||
console.log(`\n${'═'.repeat(60)}`);
|
||||
console.log(`▶ ${config.name}`);
|
||||
console.log('═'.repeat(60));
|
||||
|
|
@ -171,13 +192,95 @@ export async function runScenario(
|
|||
if (autoEnter) {
|
||||
// ── Auto-press Enter → Wait for stabilization → 02 screenshot ──
|
||||
await terminal.type('\n');
|
||||
console.log(` ⏳ waiting for output to settle...`);
|
||||
await terminal.idle(2000, 60000);
|
||||
console.log(` ✅ settled`);
|
||||
|
||||
const resultName = step.capture ?? `${pad(seq)}-02.png`;
|
||||
console.log(` ${label} 📸 result: ${resultName}`);
|
||||
screenshots.push(await terminal.capture(resultName));
|
||||
// Streaming capture: capture multiple screenshots during execution
|
||||
if (step.streaming) {
|
||||
const {
|
||||
delayMs = 0,
|
||||
intervalMs,
|
||||
count,
|
||||
gif = true,
|
||||
} = step.streaming;
|
||||
console.log(
|
||||
` 🎬 streaming capture: ${count} shots @ ${intervalMs}ms intervals${delayMs ? ` (delay ${delayMs}ms)` : ''}`,
|
||||
);
|
||||
|
||||
// Wait before starting captures (skip initial waiting phase)
|
||||
if (delayMs > 0) {
|
||||
await sleep(delayMs);
|
||||
}
|
||||
|
||||
// Capture frames at intervals (stop early if output stabilizes)
|
||||
const streamingShots: string[] = [];
|
||||
let prevOutputLen = terminal.getRawOutput().length;
|
||||
let stableCount = 0;
|
||||
let shotNum = 0;
|
||||
for (let j = 0; j < count; j++) {
|
||||
await sleep(intervalMs);
|
||||
const curOutputLen = terminal.getRawOutput().length;
|
||||
if (curOutputLen === prevOutputLen) {
|
||||
stableCount++;
|
||||
if (stableCount >= 3) {
|
||||
console.log(
|
||||
` ⏹️ streaming stopped early: output stable for ${stableCount} intervals`,
|
||||
);
|
||||
break;
|
||||
}
|
||||
continue; // skip duplicate frame
|
||||
}
|
||||
stableCount = 0;
|
||||
prevOutputLen = curOutputLen;
|
||||
shotNum++;
|
||||
const shotName = `${pad(seq)}-streaming-${pad(shotNum)}.png`;
|
||||
console.log(
|
||||
` 📸 streaming [${shotNum}/${count}]: ${shotName}`,
|
||||
);
|
||||
const shot = await terminal.capture(shotName);
|
||||
streamingShots.push(shot);
|
||||
screenshots.push(shot);
|
||||
}
|
||||
|
||||
// Wait for completion after streaming captures
|
||||
console.log(` ⏳ waiting for output to settle...`);
|
||||
await terminal.idle(2000, 60000);
|
||||
console.log(` ✅ settled`);
|
||||
|
||||
const resultName = step.capture ?? `${pad(seq)}-02.png`;
|
||||
console.log(` ${label} 📸 result: ${resultName}`);
|
||||
const resultShot = await terminal.capture(resultName);
|
||||
screenshots.push(resultShot);
|
||||
|
||||
// Generate animated GIF: input -> streaming frames -> result
|
||||
if (gif && streamingShots.length > 0) {
|
||||
// Include input and result in the GIF for complete story
|
||||
const inputShot = screenshots.find((s) =>
|
||||
s.endsWith(`${pad(seq)}-01.png`),
|
||||
);
|
||||
const gifFrames = [
|
||||
...(inputShot ? [inputShot] : []),
|
||||
...streamingShots,
|
||||
resultShot,
|
||||
];
|
||||
const gifPath = generateGif(
|
||||
gifFrames,
|
||||
outputDir,
|
||||
intervalMs,
|
||||
inputShot ? 2000 : 0, // Hold input for 2s
|
||||
2000, // Hold result for 2s
|
||||
);
|
||||
if (gifPath) {
|
||||
console.log(` 🎞️ GIF: ${gifPath}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(` ⏳ waiting for output to settle...`);
|
||||
await terminal.idle(2000, 60000);
|
||||
console.log(` ✅ settled`);
|
||||
|
||||
const resultName = step.capture ?? `${pad(seq)}-02.png`;
|
||||
console.log(` ${label} 📸 result: ${resultName}`);
|
||||
screenshots.push(await terminal.capture(resultName));
|
||||
}
|
||||
|
||||
// full-flow: Only the last type step auto-captures full-length image
|
||||
const isLastType = !config.flow.slice(i + 1).some((s) => s.type);
|
||||
|
|
@ -302,3 +405,47 @@ const KEY_MAP: Record<string, string> = {
|
|||
function resolveKey(key: string): string {
|
||||
return KEY_MAP[key] ?? key;
|
||||
}
|
||||
|
||||
/** Generate animated GIF from PNG frames using ffmpeg (concat demuxer). */
|
||||
function generateGif(
|
||||
frames: string[],
|
||||
outputDir: string,
|
||||
intervalMs: number,
|
||||
holdFirstMs: number = 0,
|
||||
holdLastMs: number = 0,
|
||||
): string | null {
|
||||
if (frames.length === 0) return null;
|
||||
|
||||
const gifPath = join(outputDir, 'streaming.gif');
|
||||
const listFile = join(outputDir, 'frames.txt');
|
||||
|
||||
try {
|
||||
const lines: string[] = [];
|
||||
for (let i = 0; i < frames.length; i++) {
|
||||
const absPath = resolve(frames[i]);
|
||||
let duration = intervalMs / 1000;
|
||||
if (i === 0 && holdFirstMs > 0) duration = holdFirstMs / 1000;
|
||||
else if (i === frames.length - 1 && holdLastMs > 0)
|
||||
duration = holdLastMs / 1000;
|
||||
lines.push(`file '${absPath}'`, `duration ${duration}`);
|
||||
}
|
||||
// Concat demuxer requires last frame repeated without duration
|
||||
lines.push(`file '${resolve(frames[frames.length - 1])}'`);
|
||||
writeFileSync(listFile, lines.join('\n'));
|
||||
|
||||
execSync(
|
||||
`ffmpeg -y -f concat -safe 0 -i "${listFile}" -vf "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse" -loop 0 "${gifPath}"`,
|
||||
{ stdio: 'pipe' },
|
||||
);
|
||||
return gifPath;
|
||||
} catch {
|
||||
console.log(' ⚠️ GIF generation requires ffmpeg');
|
||||
return null;
|
||||
} finally {
|
||||
try {
|
||||
unlinkSync(listFile);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,24 @@
|
|||
import type { ScenarioConfig } from '../scenario-runner.js';
|
||||
|
||||
/**
|
||||
* Demonstrates streaming shell execution output with PTY enabled by default.
|
||||
* Tests the render throttle behavior and progress bar handling.
|
||||
* Captures multiple screenshots during execution to show real-time output.
|
||||
*/
|
||||
export default {
|
||||
name: 'streaming-shell',
|
||||
spawn: ['node', 'dist/cli.js', '--yolo'],
|
||||
terminal: { title: 'qwen-code', cwd: '../../..' },
|
||||
flow: [
|
||||
{
|
||||
type: 'Run this command: bash integration-tests/terminal-capture/scenarios/progress.sh',
|
||||
// Capture 20 screenshots at 500ms intervals during execution
|
||||
// The progress.sh script takes ~10 seconds (20 iterations * 0.5s each)
|
||||
streaming: {
|
||||
delayMs: 7000,
|
||||
intervalMs: 500,
|
||||
count: 20,
|
||||
},
|
||||
},
|
||||
],
|
||||
} satisfies ScenarioConfig;
|
||||
11
package-lock.json
generated
11
package-lock.json
generated
|
|
@ -27,6 +27,7 @@
|
|||
"@types/uuid": "^10.0.0",
|
||||
"@vitest/coverage-v8": "^3.1.1",
|
||||
"@vitest/eslint-plugin": "^1.3.4",
|
||||
"@xterm/xterm": "^6.0.0",
|
||||
"cross-env": "^7.0.3",
|
||||
"esbuild": "^0.25.0",
|
||||
"eslint": "^9.24.0",
|
||||
|
|
@ -5629,6 +5630,16 @@
|
|||
"integrity": "sha512-5xXB7kdQlFBP82ViMJTwwEc3gKCLGKR/eoxQm4zge7GPBl86tCdI0IdPJjoKd8mUSFXz5V7i/25sfsEkP4j46g==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@xterm/xterm": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-6.0.0.tgz",
|
||||
"integrity": "sha512-TQwDdQGtwwDt+2cgKDLn0IRaSxYu1tSUjgKarSDkUM0ZNiSRXFpjxEsvc/Zgc5kq5omJ+V0a8/kIM2WD3sMOYg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"workspaces": [
|
||||
"addons/*"
|
||||
]
|
||||
},
|
||||
"node_modules/abort-controller": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@
|
|||
"@types/uuid": "^10.0.0",
|
||||
"@vitest/coverage-v8": "^3.1.1",
|
||||
"@vitest/eslint-plugin": "^1.3.4",
|
||||
"@xterm/xterm": "^6.0.0",
|
||||
"cross-env": "^7.0.3",
|
||||
"esbuild": "^0.25.0",
|
||||
"eslint": "^9.24.0",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue