docs: document test wrapper scripts

This commit is contained in:
Peter Steinberger 2026-06-04 23:54:19 -04:00
parent 29746cf7a9
commit 9b1a01e4f9
8 changed files with 69 additions and 1 deletions

View file

@ -1,8 +1,10 @@
// Builds grouped Vitest duration reports or compares two grouped reports.
import { spawn } from "node:child_process"; import { spawn } from "node:child_process";
import fs from "node:fs"; import fs from "node:fs";
import os from "node:os"; import os from "node:os";
import path from "node:path"; import path from "node:path";
import { pathToFileURL } from "node:url"; import { pathToFileURL } from "node:url";
import { parsePositiveInt } from "./lib/numeric-options.mjs";
import { import {
buildGroupedTestComparison, buildGroupedTestComparison,
buildGroupedTestReport, buildGroupedTestReport,
@ -11,7 +13,6 @@ import {
renderGroupedTestComparison, renderGroupedTestComparison,
renderGroupedTestReport, renderGroupedTestReport,
} from "./lib/test-group-report.mjs"; } from "./lib/test-group-report.mjs";
import { parsePositiveInt } from "./lib/numeric-options.mjs";
import { formatMs } from "./lib/vitest-report-cli-utils.mjs"; import { formatMs } from "./lib/vitest-report-cli-utils.mjs";
import { resolveVitestNodeArgs } from "./run-vitest.mjs"; import { resolveVitestNodeArgs } from "./run-vitest.mjs";
import { import {
@ -56,6 +57,9 @@ function usage() {
].join("\n"); ].join("\n");
} }
/**
* Parses report, compare, and Vitest-run options for grouped test reports.
*/
export function parseTestGroupReportArgs(argv) { export function parseTestGroupReportArgs(argv) {
const args = { const args = {
allowFailures: false, allowFailures: false,
@ -212,6 +216,9 @@ function formatSpawnError(error) {
return error instanceof Error ? error.message : String(error); return error instanceof Error ? error.message : String(error);
} }
/**
* Runs a command, captures text output, and terminates timed-out process groups.
*/
export function spawnText(command, args, options) { export function spawnText(command, args, options) {
const maxBuffer = 1024 * 1024 * 64; const maxBuffer = 1024 * 1024 * 64;
const timeoutMs = options.timeoutMs ?? DEFAULT_RUN_TIMEOUT_MS; const timeoutMs = options.timeoutMs ?? DEFAULT_RUN_TIMEOUT_MS;
@ -411,6 +418,9 @@ function readGroupedReport(reportPath) {
return JSON.parse(fs.readFileSync(reportPath, "utf8")); return JSON.parse(fs.readFileSync(reportPath, "utf8"));
} }
/**
* Resolves JSON report and per-run artifact directories from an output path.
*/
export function resolveReportArtifactDirs(outputPath) { export function resolveReportArtifactDirs(outputPath) {
const outputDir = path.dirname(outputPath); const outputDir = path.dirname(outputPath);
const outputExt = path.extname(outputPath); const outputExt = path.extname(outputPath);
@ -456,6 +466,9 @@ function buildFullSuiteLeafRunPlans() {
} }
} }
/**
* Resolves explicit or full-suite Vitest config plans for report generation.
*/
export function resolveRunPlans(args) { export function resolveRunPlans(args) {
if (args.reports.length > 0) { if (args.reports.length > 0) {
return []; return [];
@ -477,6 +490,9 @@ export function resolveRunPlans(args) {
})); }));
} }
/**
* Builds env for full-suite report runs, including per-config cache paths.
*/
export function resolveFullSuiteVitestEnv(args, env = process.env, label = "") { export function resolveFullSuiteVitestEnv(args, env = process.env, label = "") {
if ( if (
!args.fullSuite || !args.fullSuite ||
@ -491,6 +507,9 @@ export function resolveFullSuiteVitestEnv(args, env = process.env, label = "") {
}; };
} }
/**
* Resolves bounded concurrency for grouped report run plans.
*/
export function resolveRunPlanConcurrency(args, runPlanCount) { export function resolveRunPlanConcurrency(args, runPlanCount) {
if (runPlanCount <= 1) { if (runPlanCount <= 1) {
return 1; return 1;
@ -504,6 +523,9 @@ export function resolveRunPlanConcurrency(args, runPlanCount) {
return Math.min(2, runPlanCount); return Math.min(2, runPlanCount);
} }
/**
* Builds concrete report run specs from parsed args and config plans.
*/
export function resolveReportRunSpecs(args, runPlans, params = {}) { export function resolveReportRunSpecs(args, runPlans, params = {}) {
const concurrency = params.concurrency ?? resolveRunPlanConcurrency(args, runPlans.length); const concurrency = params.concurrency ?? resolveRunPlanConcurrency(args, runPlans.length);
const env = params.env ?? process.env; const env = params.env ?? process.env;

View file

@ -1,3 +1,4 @@
// Prints the slowest test files from a Vitest JSON report.
import { import {
formatMs, formatMs,
loadVitestReportFromArgs, loadVitestReportFromArgs,

View file

@ -1,4 +1,5 @@
#!/usr/bin/env node #!/usr/bin/env node
// Runs one named live-test shard with OPENCLAW_LIVE_TEST enabled.
import { spawnSync } from "node:child_process"; import { spawnSync } from "node:child_process";
import fs from "node:fs"; import fs from "node:fs";
import path from "node:path"; import path from "node:path";
@ -11,6 +12,7 @@ import {
const LIVE_TEST_SUFFIX = ".live.test.ts"; const LIVE_TEST_SUFFIX = ".live.test.ts";
/** Live-test shards included in release validation. */
export const RELEASE_LIVE_TEST_SHARDS = Object.freeze([ export const RELEASE_LIVE_TEST_SHARDS = Object.freeze([
"native-live-src-agents", "native-live-src-agents",
"native-live-src-gateway-core", "native-live-src-gateway-core",
@ -30,6 +32,7 @@ export const RELEASE_LIVE_TEST_SHARDS = Object.freeze([
"native-live-extensions-media-video", "native-live-extensions-media-video",
]); ]);
/** All live-test shards, including broader local-only shard aliases. */
export const LIVE_TEST_SHARDS = Object.freeze([ export const LIVE_TEST_SHARDS = Object.freeze([
...RELEASE_LIVE_TEST_SHARDS, ...RELEASE_LIVE_TEST_SHARDS,
"native-live-extensions-o-z", "native-live-extensions-o-z",
@ -68,6 +71,9 @@ function walkFiles(rootDir) {
return files; return files;
} }
/**
* Lists all live test files from git/find fallback paths.
*/
export function collectAllLiveTestFiles(repoRoot = process.cwd()) { export function collectAllLiveTestFiles(repoRoot = process.cwd()) {
const externalFiles = listExternalLiveTestFiles(repoRoot); const externalFiles = listExternalLiveTestFiles(repoRoot);
if (externalFiles) { if (externalFiles) {
@ -211,6 +217,9 @@ function isMoonshotLiveTest(file) {
return file.startsWith("extensions/moonshot/"); return file.startsWith("extensions/moonshot/");
} }
/**
* Selects the live test files belonging to one shard name.
*/
export function selectLiveShardFiles(shard, files = collectAllLiveTestFiles()) { export function selectLiveShardFiles(shard, files = collectAllLiveTestFiles()) {
switch (shard) { switch (shard) {
case "native-live-src-agents": case "native-live-src-agents":
@ -290,6 +299,9 @@ function usage(stream = process.stderr) {
); );
} }
/**
* Parses live-shard CLI args into shard name and Vitest passthrough args.
*/
export function parseLiveShardArgs(args) { export function parseLiveShardArgs(args) {
const separatorIndex = args.indexOf("--"); const separatorIndex = args.indexOf("--");
const optionArgs = separatorIndex >= 0 ? args.slice(0, separatorIndex) : args; const optionArgs = separatorIndex >= 0 ? args.slice(0, separatorIndex) : args;
@ -312,10 +324,16 @@ export function parseLiveShardArgs(args) {
return { shard, listOnly, passthroughArgs }; return { shard, listOnly, passthroughArgs };
} }
/**
* Builds pnpm/vitest args for selected live test files.
*/
export function buildLiveShardPnpmArgs(files, passthroughArgs) { export function buildLiveShardPnpmArgs(files, passthroughArgs) {
return ["test:live", "--", ...files, ...passthroughArgs]; return ["test:live", "--", ...files, ...passthroughArgs];
} }
/**
* Builds spawn options for the live-shard Vitest child.
*/
export function buildLiveShardSpawnParams(env = process.env, platform = process.platform) { export function buildLiveShardSpawnParams(env = process.env, platform = process.platform) {
return { return {
detached: shouldUseDetachedVitestProcessGroup(platform), detached: shouldUseDetachedVitestProcessGroup(platform),

View file

@ -1,9 +1,13 @@
// Runs the full live Vitest suite with live-test env and heartbeat output.
import { spawnPnpmRunner } from "./pnpm-runner.mjs"; import { spawnPnpmRunner } from "./pnpm-runner.mjs";
import { import {
installVitestProcessGroupCleanup, installVitestProcessGroupCleanup,
shouldUseDetachedVitestProcessGroup, shouldUseDetachedVitestProcessGroup,
} from "./vitest-process-group.mjs"; } from "./vitest-process-group.mjs";
/**
* Renders CLI usage for the live-test wrapper.
*/
export function testLiveUsage() { export function testLiveUsage() {
return [ return [
"Usage: node scripts/test-live.mjs [options] [--] [vitest targets/args...]", "Usage: node scripts/test-live.mjs [options] [--] [vitest targets/args...]",
@ -19,6 +23,9 @@ export function testLiveUsage() {
].join("\n"); ].join("\n");
} }
/**
* Parses live-test wrapper flags and forwarded Vitest args.
*/
export function parseTestLiveArgs(argv) { export function parseTestLiveArgs(argv) {
const forwardedArgs = []; const forwardedArgs = [];
let quietOverride; let quietOverride;
@ -61,6 +68,9 @@ export function parseTestLiveArgs(argv) {
}; };
} }
/**
* Builds env for live tests, including quiet mode and Codex harness opt-in.
*/
export function buildTestLiveEnv(args, baseEnv = process.env) { export function buildTestLiveEnv(args, baseEnv = process.env) {
return { return {
...baseEnv, ...baseEnv,
@ -73,6 +83,9 @@ export function buildTestLiveEnv(args, baseEnv = process.env) {
}; };
} }
/**
* Reads the live-test heartbeat interval.
*/
export function resolveTestLiveHeartbeatMs(baseEnv = process.env) { export function resolveTestLiveHeartbeatMs(baseEnv = process.env) {
const value = baseEnv.OPENCLAW_LIVE_WRAPPER_HEARTBEAT_MS; const value = baseEnv.OPENCLAW_LIVE_WRAPPER_HEARTBEAT_MS;
if (value === undefined || value === "") { if (value === undefined || value === "") {
@ -89,6 +102,9 @@ export function resolveTestLiveHeartbeatMs(baseEnv = process.env) {
return parsed; return parsed;
} }
/**
* Builds pnpm/vitest args for full live test execution.
*/
export function buildTestLivePnpmArgs(args) { export function buildTestLivePnpmArgs(args) {
return [ return [
"exec", "exec",
@ -100,6 +116,9 @@ export function buildTestLivePnpmArgs(args) {
]; ];
} }
/**
* Builds spawn options for the live-test Vitest child.
*/
export function buildTestLiveSpawnParams(env, platform = process.platform) { export function buildTestLiveSpawnParams(env, platform = process.platform) {
return { return {
detached: shouldUseDetachedVitestProcessGroup(platform), detached: shouldUseDetachedVitestProcessGroup(platform),
@ -108,6 +127,9 @@ export function buildTestLiveSpawnParams(env, platform = process.platform) {
}; };
} }
/**
* Runs the live-test wrapper process.
*/
export function main(argv = process.argv.slice(2), baseEnv = process.env) { export function main(argv = process.argv.slice(2), baseEnv = process.env) {
const args = parseTestLiveArgs(argv); const args = parseTestLiveArgs(argv);
if (args.help) { if (args.help) {

View file

@ -1,3 +1,4 @@
// Runs a Vitest config and enforces wall-time regression budgets.
import { pathToFileURL } from "node:url"; import { pathToFileURL } from "node:url";
import { parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs"; import { parseFlagArgs, stringFlag } from "./lib/arg-utils.mjs";
import { import {
@ -90,6 +91,7 @@ function main() {
} }
} }
/** Test-facing parser helpers for budget validation. */
export const testing = { export const testing = {
parseArgs, parseArgs,
parseBudgetNumber, parseBudgetNumber,

View file

@ -1,3 +1,4 @@
// Runs test-projects with Vitest import duration diagnostics enabled.
process.env.OPENCLAW_VITEST_IMPORT_DURATIONS = "1"; process.env.OPENCLAW_VITEST_IMPORT_DURATIONS = "1";
process.env.OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN = "1"; process.env.OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN = "1";

View file

@ -1,3 +1,4 @@
// Runs test-projects with an explicit high worker budget for max-throughput checks.
process.env.OPENCLAW_VITEST_MAX_WORKERS = "8"; process.env.OPENCLAW_VITEST_MAX_WORKERS = "8";
await import("./test-projects.mjs"); await import("./test-projects.mjs");

View file

@ -1,3 +1,4 @@
// Runs test-projects serially with a one-worker Vitest budget.
process.env.OPENCLAW_TEST_PROJECTS_SERIAL = "1"; process.env.OPENCLAW_TEST_PROJECTS_SERIAL = "1";
process.env.OPENCLAW_VITEST_MAX_WORKERS = "1"; process.env.OPENCLAW_VITEST_MAX_WORKERS = "1";