mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-08-01 12:24:51 +00:00
7918717ade
Co-authored-by: 钉萁 <dingqi.jww@alibaba-inc.com> Co-authored-by: Shaojin Wen <shaojin.wensj@alibaba-inc.com>
449 lines
18 KiB
JavaScript
449 lines
18 KiB
JavaScript
/**
|
|
* @license
|
|
* Copyright 2025 Qwen Team
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*/
|
|
|
|
import { spawnSync } from 'node:child_process';
|
|
import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
import { createRequire } from 'node:module';
|
|
import { tmpdir } from 'node:os';
|
|
import path from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { describe, expect, it } from 'vitest';
|
|
|
|
const repoRoot = path.resolve(
|
|
path.dirname(fileURLToPath(import.meta.url)),
|
|
'../..',
|
|
);
|
|
const HELPER = path.join(repoRoot, 'scripts/verify-capture.mjs');
|
|
const ESC = String.fromCharCode(27);
|
|
|
|
// Every assertion here runs the real helper and inspects the real PNG. The
|
|
// point of this script is that the browser pipeline it replaces never produced
|
|
// an image in four live /verify runs — it needed a browser and a route that is
|
|
// not installed as a unit. A test that only checked the skill's wording would
|
|
// have passed through all four of those rounds.
|
|
describe('verify-capture helper', () => {
|
|
const run = (args, opts = {}) =>
|
|
spawnSync('node', [HELPER, ...args], {
|
|
cwd: repoRoot,
|
|
encoding: 'utf8',
|
|
...opts,
|
|
});
|
|
|
|
/** PNG signature — proves sharp actually rasterised, not that a file exists. */
|
|
const isPng = (file) => {
|
|
const head = readFileSync(file).subarray(0, 8);
|
|
return head.equals(
|
|
Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
|
|
);
|
|
};
|
|
|
|
const withDir = (fn) => {
|
|
const dir = mkdtempSync(path.join(tmpdir(), 'verify-capture-'));
|
|
try {
|
|
return fn(dir);
|
|
} finally {
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
};
|
|
|
|
/** A harness that emits a coloured A/B table, like a real one would. */
|
|
const harness = (dir) => {
|
|
const file = path.join(dir, 'harness.mjs');
|
|
writeFileSync(
|
|
file,
|
|
[
|
|
`const E = String.fromCharCode(27);`,
|
|
`const g = (s) => \`\${E}[32m\${s}\${E}[0m\`;`,
|
|
`const r = (s) => \`\${E}[1;31m\${s}\${E}[0m\`;`,
|
|
`console.log('cell base head');`,
|
|
`console.log(\`noisy \${r('FAIL')} \${g('PASS')}\`);`,
|
|
`console.log(\`clean \${g('PASS')} \${g('PASS')}\`);`,
|
|
].join('\n'),
|
|
);
|
|
return file;
|
|
};
|
|
|
|
it('captures a command to a real PNG', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'evidence/01-ab.png');
|
|
const res = run(
|
|
[
|
|
'--out',
|
|
out,
|
|
'--cols',
|
|
'40',
|
|
'--title',
|
|
'A/B: gate flips',
|
|
'--',
|
|
'node',
|
|
harness(dir),
|
|
],
|
|
{
|
|
env: { ...process.env, NO_COLOR: 'true' },
|
|
},
|
|
);
|
|
expect(res.status).toBe(0);
|
|
// isPng reads from evidence/01-ab.png, so a pass also proves the helper
|
|
// created the parent dir the skill tells the agent to write into.
|
|
expect(isPng(out)).toBe(true);
|
|
// Geometry is reported so a caller can spot a blank or clipped capture.
|
|
// The canvas size is deterministic — cols fix the width, the row count
|
|
// the height — so assert it exactly; the compressed byte length varies
|
|
// with the platform's font rasterisation and flaked on Linux (846B for a
|
|
// render that was >1000B on macOS).
|
|
expect(res.stdout).toMatch(/360x96 \d+B 3 rows/);
|
|
}));
|
|
|
|
it('accepts piped input as well as a command', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'piped.png');
|
|
const res = run(['--out', out, '--cols', '40'], {
|
|
input: `${ESC}[32mPASS${ESC}[0m 64/64\n`,
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
}));
|
|
|
|
// Capturing a failing base arm is the normal case for an A/B cell, so a
|
|
// non-zero exit from the captured command must still produce an image.
|
|
it('still captures when the captured command fails', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'failing.png');
|
|
const res = run([
|
|
'--out',
|
|
out,
|
|
'--',
|
|
'node',
|
|
'-e',
|
|
'console.log("boom"); process.exit(3)',
|
|
]);
|
|
expect(res.status).toBe(0);
|
|
expect(res.stderr).toContain('command exited 3');
|
|
expect(isPng(out)).toBe(true);
|
|
}));
|
|
|
|
// stdout and stderr are collected separately by spawnSync; joining them with
|
|
// an empty separator glues a partial stdout line to the first stderr line,
|
|
// producing a line that never appeared on the real terminal.
|
|
it('separates stdout and stderr with a newline', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'both-streams.png');
|
|
const res = run([
|
|
'--out',
|
|
out,
|
|
'--cols',
|
|
'40',
|
|
'--',
|
|
'node',
|
|
'-e',
|
|
'process.stdout.write("result: 42"); process.stderr.write("warning: flaky")',
|
|
]);
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
// Two separate lines, not one glued "result: 42warning: flaky".
|
|
expect(res.stdout).toContain('2 rows');
|
|
}));
|
|
|
|
// console.log-terminated stdout already ends with \n; the old join('\n')
|
|
// inserted a phantom blank row between stdout and stderr.
|
|
it('does not insert a phantom blank row when stdout ends with a newline', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'no-phantom.png');
|
|
const res = run([
|
|
'--out',
|
|
out,
|
|
'--cols',
|
|
'40',
|
|
'--',
|
|
'node',
|
|
'-e',
|
|
'console.log("result: 42"); process.stderr.write("warning: flaky\\n")',
|
|
]);
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
expect(res.stdout).toContain('2 rows');
|
|
}));
|
|
|
|
// Colour and weight are the whole reason to render rather than paste text:
|
|
// a red FAIL beside a green PASS is what makes the cell readable at a glance.
|
|
//
|
|
// Each attribute is isolated against the SAME plain baseline. A single
|
|
// "coloured and bold vs plain" comparison passes while EITHER attribute
|
|
// survives — verified: mutating colour away, and mutating bold away, both
|
|
// left that version green. This is the wrong-reason trap the skill warns
|
|
// about, met in this file's own test.
|
|
it('preserves colour and bold independently', () =>
|
|
withDir((dir) => {
|
|
const render = (name, input) => {
|
|
const out = path.join(dir, `${name}.png`);
|
|
expect(run(['--out', out, '--cols', '30'], { input }).status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
return readFileSync(out);
|
|
};
|
|
const plain = render('plain', 'FAIL PASS\n');
|
|
// Green, no bold — differs from plain ONLY if colour is applied.
|
|
const colourOnly = render('colour', `${ESC}[32mFAIL PASS${ESC}[0m\n`);
|
|
// Bold, no colour — differs from plain ONLY if weight is applied.
|
|
const boldOnly = render('bold', `${ESC}[1mFAIL PASS${ESC}[0m\n`);
|
|
expect(colourOnly.equals(plain), 'colour was dropped').toBe(false);
|
|
expect(boldOnly.equals(plain), 'bold was dropped').toBe(false);
|
|
// ...and the two attributes must not collapse onto the same rendering.
|
|
expect(colourOnly.equals(boldOnly)).toBe(false);
|
|
}));
|
|
|
|
// 256-colour and truecolor sequences produce getFgColor() values >= 16,
|
|
// which the bounds guard maps to FG_DEFAULT (#d4d4d4). Decode pixels and
|
|
// assert the fallback grey is present, so deleting the guard cannot ship
|
|
// fill="undefined" (which librsvg paints black) while the test stays green.
|
|
it('renders 256-colour and truecolor via the default-grey fallback', async () => {
|
|
let png;
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'fallback.png');
|
|
const res = run(['--out', out], {
|
|
input: `${ESC}[38;5;200mFAIL${ESC}[0m ${ESC}[38;2;255;100;0mFAIL${ESC}[0m\n`,
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
png = readFileSync(out);
|
|
});
|
|
const sharp = createRequire(import.meta.url)('sharp');
|
|
const { data, info } = await sharp(png)
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
// FG_DEFAULT is #d4d4d4 — look for a pixel matching it.
|
|
let hasFallback = false;
|
|
for (let i = 0; i + 2 < data.length; i += info.channels) {
|
|
if (data[i] === 0xd4 && data[i + 1] === 0xd4 && data[i + 2] === 0xd4) {
|
|
hasFallback = true;
|
|
break;
|
|
}
|
|
}
|
|
expect(hasFallback, '256-colour text did not render as #d4d4d4').toBe(true);
|
|
});
|
|
|
|
// SGR 30 maps to #1e1e1e — identical to the canvas BG — so black-foreground
|
|
// text (the normal way to label a coloured badge, e.g. vitest's project
|
|
// badge) vanished as black-on-black. The helper lifts it to the default
|
|
// grey; assert the rendered pixels contain something other than the background
|
|
// colour, which a black-on-black render cannot. sharp's PNG bytes are not
|
|
// run-to-run deterministic, so compare pixels, not file bytes.
|
|
it('keeps black-foreground text readable instead of black-on-black', async () => {
|
|
let png;
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'black.png');
|
|
expect(
|
|
run(['--out', out, '--cols', '30'], {
|
|
input: `${ESC}[30mFAIL PASS${ESC}[0m\n`,
|
|
}).status,
|
|
).toBe(0);
|
|
// Read the bytes before withDir removes the dir; decode them below.
|
|
png = readFileSync(out);
|
|
});
|
|
const sharp = createRequire(import.meta.url)('sharp');
|
|
const { data, info } = await sharp(png)
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
const BG = 0x1e; // #1e1e1e canvas background
|
|
let visible = false;
|
|
for (let i = 0; i + 2 < data.length; i += info.channels) {
|
|
if (data[i] !== BG || data[i + 1] !== BG || data[i + 2] !== BG) {
|
|
visible = true;
|
|
break;
|
|
}
|
|
}
|
|
expect(visible, 'SGR 30 rendered as black-on-black').toBe(true);
|
|
});
|
|
|
|
// A bare LF leaves xterm's cursor in the old column, so line 2 renders
|
|
// indented by line 1's width and the capture looks like a staircase. The
|
|
// helper normalises to CRLF; assert the rendered height matches the line
|
|
// count rather than trusting that.
|
|
it('renders one row per line, not a staircase', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'rows.png');
|
|
const res = run(['--out', out, '--cols', '20'], {
|
|
input: 'aaa\nbbb\nccc\nddd\n',
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(res.stdout).toContain('4 rows');
|
|
}));
|
|
|
|
it('trims trailing blank rows instead of padding to --rows', () =>
|
|
withDir((dir) => {
|
|
const tall = run(['--out', path.join(dir, 'a.png'), '--rows', '40'], {
|
|
input: 'one\ntwo\n',
|
|
});
|
|
const short = run(['--out', path.join(dir, 'b.png'), '--rows', '4'], {
|
|
input: 'one\ntwo\n',
|
|
});
|
|
expect(tall.stdout).toContain('2 rows');
|
|
// Same content, same image, regardless of the row cap.
|
|
expect(tall.stdout.split(' ')[1]).toBe(short.stdout.split(' ')[1]);
|
|
}));
|
|
|
|
// Regression: U+FE0F (the emoji variation selector in ⚠️) sent Pango looking
|
|
// for a colour-emoji font and abort()ing in native code on macOS — no PNG, no
|
|
// diagnostic, exit 133. The helper strips it and the base codepoint renders.
|
|
// No other test fed the helper a non-ASCII byte.
|
|
it('renders non-ASCII and emoji without aborting', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'emoji.png');
|
|
const res = run(['--out', out, '--cols', '40'], {
|
|
input: 'warn ⚠️ history gap 中文 café\n',
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
}));
|
|
|
|
// escapeXml guards every rendered cell and the title; feed it XML-hostile
|
|
// content (common in test output: expect(a < b), foo & bar) so a deleted
|
|
// escape rule cannot silently corrupt the SVG and ship a broken image.
|
|
it('escapes XML special characters in rendered text', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'xml.png');
|
|
const res = run(
|
|
['--out', out, '--cols', '40', '--title', 'a < b & "c"'],
|
|
{ input: 'expect(a < b) & foo > bar "q"\n' },
|
|
);
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
}));
|
|
|
|
// scrollback: 0 keeps only the last --rows rows, so a taller input loses its
|
|
// header with no visible sign. The helper must say so on stderr rather than
|
|
// ship an image that looks complete but starts halfway down.
|
|
it('warns when input is taller than --rows', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'tall.png');
|
|
const res = run(['--out', out, '--rows', '4'], {
|
|
input: 'l1\nl2\nl3\nl4\nl5\nl6\n',
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
expect(res.stderr).toContain('warning: input occupies 6 terminal rows');
|
|
expect(res.stderr).toContain('dropped the top 3');
|
|
}));
|
|
|
|
// A line wider than --cols wraps into ceil(len / cols) terminal rows, so
|
|
// the top can be dropped even when the newline count is under --rows.
|
|
it('warns when wrapped lines exceed --rows', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'wrapped.png');
|
|
const longLine = 'a'.repeat(20);
|
|
const res = run(['--out', out, '--cols', '10', '--rows', '4'], {
|
|
input: `${longLine}\n${longLine}\n${longLine}\n`,
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
expect(res.stderr).toContain('warning: input occupies 6 terminal rows');
|
|
expect(res.stderr).toContain('dropped the top 3');
|
|
}));
|
|
|
|
// The defect the guard exists for: at EXACTLY --rows newline-terminated lines
|
|
// the final CRLF still scrolls the top line off the scrollback-less viewport,
|
|
// yet the old `wrappedRows > opts.rows` test stayed silent. Usable capacity is
|
|
// rows - 1, so exactly --rows lines must warn that one was dropped.
|
|
it('warns at exactly --rows lines, where the top line is already gone', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'boundary.png');
|
|
const res = run(['--out', out, '--rows', '4'], {
|
|
input: 'l1\nl2\nl3\nl4\n',
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(isPng(out)).toBe(true);
|
|
expect(res.stderr).toContain('warning: input occupies 4 terminal rows');
|
|
expect(res.stderr).toContain('dropped the top 1');
|
|
}));
|
|
|
|
// Non-newline-terminated input of exactly --rows lines fits: no final CRLF
|
|
// scrolls the viewport, so the usable capacity is rows, not rows - 1.
|
|
it('does not warn for non-newline-terminated input that fits --rows', () =>
|
|
withDir((dir) => {
|
|
const out = path.join(dir, 'no-nl.png');
|
|
const res = run(['--out', out, '--rows', '4'], {
|
|
input: 'l1\nl2\nl3\nl4',
|
|
});
|
|
expect(res.status).toBe(0);
|
|
expect(res.stderr).not.toContain('warning');
|
|
}));
|
|
|
|
describe('fails loudly rather than writing a broken image', () => {
|
|
it('rejects a missing --out', () => {
|
|
const res = run(['--', 'echo', 'hi']);
|
|
expect(res.status).toBe(1);
|
|
expect(res.stderr).toContain('--out is required');
|
|
});
|
|
|
|
it('rejects a flag with no value', () => {
|
|
for (const flag of ['--out', '--cols', '--rows', '--title']) {
|
|
const res = run([flag]);
|
|
expect(res.status, `${flag} accepted without a value`).toBe(1);
|
|
expect(res.stderr).toContain('needs a value');
|
|
}
|
|
});
|
|
|
|
it('rejects nonsense geometry', () =>
|
|
withDir((dir) => {
|
|
// Both flags share one validation loop; feed both so a mutation
|
|
// dropping 'rows' from it cannot survive the suite.
|
|
for (const flag of ['--cols', '--rows']) {
|
|
for (const bad of ['0', '-5', 'abc', '9999']) {
|
|
const res = run(['--out', path.join(dir, 'x.png'), flag, bad]);
|
|
expect(res.status, `${flag} ${bad} was accepted`).toBe(1);
|
|
expect(res.stderr).toContain('must be an integer');
|
|
}
|
|
}
|
|
}));
|
|
|
|
it('rejects an unknown option instead of ignoring it', () =>
|
|
withDir((dir) => {
|
|
const res = run(['--out', path.join(dir, 'x.png'), '--width', '80']);
|
|
expect(res.status).toBe(1);
|
|
expect(res.stderr).toContain('unknown option --width');
|
|
}));
|
|
|
|
// A blank capture is worse than none: it looks like evidence and shows
|
|
// nothing. Exit 1 so the agent notices rather than publishing it.
|
|
it('refuses to write an empty capture', () =>
|
|
withDir((dir) => {
|
|
const res = run(['--out', path.join(dir, 'empty.png')], { input: '' });
|
|
expect(res.status).toBe(1);
|
|
expect(res.stderr).toContain('nothing to render');
|
|
}));
|
|
|
|
it('reports a command that does not exist', () =>
|
|
withDir((dir) => {
|
|
const res = run([
|
|
'--out',
|
|
path.join(dir, 'x.png'),
|
|
'--',
|
|
'definitely-not-a-real-binary-xyz',
|
|
]);
|
|
expect(res.status).toBe(1);
|
|
}));
|
|
});
|
|
|
|
// The helper only helps if the skill points at it. Three rounds of rewording
|
|
// failed because the named pipeline never produced an image; assert the slow
|
|
// browser route is no longer the recommendation and the live command is named.
|
|
it('is the route the skill actually names', () => {
|
|
const skill = readFileSync(
|
|
path.join(repoRoot, '.qwen/skills/verify-pr/SKILL.md'),
|
|
'utf8',
|
|
);
|
|
// Flatten whitespace so a prose reflow cannot break these, like the sibling
|
|
// test in qwen-triage-workflow.test.js.
|
|
const flat = skill.replace(/\s+/g, ' ');
|
|
expect(flat).toContain('node scripts/verify-capture.mjs --out');
|
|
expect(flat).toContain('no browser and no pseudo-terminal');
|
|
// The slow browser pipeline must not be the recommended route again...
|
|
expect(flat).not.toContain('Route: `terminal-capture` skill');
|
|
expect(flat).not.toMatch(/node-pty → xterm\.js → Playwright PNG/);
|
|
// ...but the skill must still point at it for the captures this helper
|
|
// cannot do (an ink TUI or a browser page), so the route stays discoverable.
|
|
expect(flat).toContain('see the `terminal-capture` skill');
|
|
});
|
|
});
|