mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-11 09:56:29 +00:00
fa03d9b913
* refactor: consolidate coercion helpers * fix: remove duplicate coercion imports * fix: preserve serialized coercion guard * chore: ratchet coercion helper carve-outs * fix(test): keep gauntlet subprocess startup lean * fix: preserve imported session timestamp semantics * fix: preserve catalog timestamp string semantics * chore: align plugin SDK surface ratchet * fix: preserve trajectory and SDK string contracts * fix(test): preserve QA record assertion semantics * fix: complete standalone record guard rename * refactor(cron): use canonical string coercion * fix(acpx): preserve Pi timestamp parsing * test(channels): adapt custody test harnesses * test(telegram): classify media harness as test support * test(acpx): split timestamp contract coverage * test(channels): support generated custody contracts * chore: ban the full coercion helper name set Extends the declaration guard to all eleven consolidated helper names and renames the cron schedule-identity readNumber wrapper to readScheduleInteger so the banned generic name cannot regrow. * fix(scripts): repair release-validation guard drift and lint cause Restores the renamed isJsonRecord guard in assertTrustedWorkflowHarness after main added isRecord call sites in parallel, and attaches the caught YAML error as the thrown error cause (preserve-caught-error was red on main). * fix: preserve Claude timestamp string semantics * fix: preserve persisted timestamp string semantics * fix: preserve date-first timestamp contracts * fix(openai): harden delegation failure formatting * chore: close coercion helper guard gaps * test(openai): model non-error delegation rejection * chore: refresh plugin SDK API contract * fix(tasks): use canonical string field reader * fix(ai): use canonical provider error field coercion * fix(browser): migrate native bootstrap coercion * docs(plugin-sdk): clarify text record export compatibility * fix(gateway): normalize approval execution identity * test(outbound): isolate message action poll harness
1300 lines
45 KiB
TypeScript
1300 lines
45 KiB
TypeScript
#!/usr/bin/env node
|
|
|
|
// Runs plugin lifecycle and gateway QA gauntlet probes with timing metrics.
|
|
import { spawn } from "node:child_process";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import process from "node:process";
|
|
import { fileURLToPath } from "node:url";
|
|
import { stripLeadingPackageManagerSeparator } from "./lib/arg-utils.mts";
|
|
import {
|
|
parseNonNegativeInt,
|
|
parsePositiveInt,
|
|
parsePositiveNumber,
|
|
} from "./lib/numeric-options.mjs";
|
|
import {
|
|
buildGauntletPrebuildEnv,
|
|
collectGatewayCpuObservations,
|
|
collectMetricObservations,
|
|
collectPluginsWithRequiredEntries,
|
|
collectRequiredPluginEntries,
|
|
collectQaBaselineRegressionObservations,
|
|
detectCommandDiagnosticFailure,
|
|
discoverBundledPluginManifests,
|
|
readQaSuiteSummary,
|
|
selectPluginEntries,
|
|
} from "./lib/plugin-gateway-gauntlet.mts";
|
|
// Termination tests import this entrypoint in a child before publishing readiness.
|
|
// Keep its record guard on the dependency-light script seam to avoid startup skew.
|
|
import { isRecord } from "./lib/record-shared.mjs";
|
|
|
|
const DEFAULT_QA_SCENARIOS = [
|
|
"channel-chat-baseline",
|
|
"memory-failure-fallback",
|
|
"gateway-restart-inflight-run",
|
|
];
|
|
const DEFAULT_CPU_CORE_WARN = 0.9;
|
|
const DEFAULT_HOT_WALL_WARN_MS = 30_000;
|
|
const DEFAULT_MAX_RSS_WARN_MB = 1536;
|
|
const DEFAULT_QA_PLUGIN_CHUNK_SIZE = 12;
|
|
const SINGLE_VALUE_FLAGS = new Set([
|
|
"--build-timeout-ms",
|
|
"--command-timeout-ms",
|
|
"--cpu-core-warn",
|
|
"--hot-wall-warn-ms",
|
|
"--limit",
|
|
"--max-rss-warn-mb",
|
|
"--output-dir",
|
|
"--qa-cpu-regression-multiplier",
|
|
"--qa-plugin-chunk-size",
|
|
"--qa-timeout-ms",
|
|
"--qa-wall-regression-multiplier",
|
|
"--repo-root",
|
|
"--rss-anomaly-multiplier",
|
|
"--shard-index",
|
|
"--shard-total",
|
|
"--wall-anomaly-multiplier",
|
|
]);
|
|
const COMMAND_OUTPUT_MAX_BUFFER_BYTES = 16 * 1024 * 1024;
|
|
const MAX_TIMER_TIMEOUT_MS = 2_147_000_000;
|
|
const ANSI_PATTERN = new RegExp(String.raw`\u001B\[[0-9;]*m`, "gu");
|
|
|
|
type ProcessSignal = `SIG${string}`;
|
|
type TimerHandle = ReturnType<typeof setTimeout>;
|
|
type PluginEntry = ReturnType<typeof discoverBundledPluginManifests>[number];
|
|
type CommandAlias = PluginEntry["cliCommandAliases"][number];
|
|
export type GauntletMeasuredRow = {
|
|
label: string;
|
|
phase: string;
|
|
pluginId: string | null;
|
|
status: number;
|
|
diagnosticFailure: string | null;
|
|
diagnosticDetail?: string;
|
|
signal: ProcessSignal | null;
|
|
timedOut: boolean;
|
|
spawnError: { code: string | null; message: string } | null;
|
|
logPath: string | null;
|
|
logWriteError?: string;
|
|
qaSummaryPath?: string;
|
|
qaMetrics?: unknown;
|
|
} & ReturnType<typeof parseTimedMetrics>;
|
|
export type GauntletMeasuredCommandParams = {
|
|
args: string[];
|
|
command: string;
|
|
consoleOutputMaxBytes?: number;
|
|
cwd: string;
|
|
env: NodeJS.ProcessEnv;
|
|
killProcessGroup?: boolean;
|
|
label: string;
|
|
logDir: string;
|
|
maxBufferBytes?: number;
|
|
phase: string;
|
|
pluginId?: string;
|
|
spawnOptions?: Parameters<typeof spawn>[2];
|
|
timeoutKillGraceMs?: number;
|
|
timeoutMs: number;
|
|
timeMode?: "none";
|
|
};
|
|
type GauntletObservation = Record<string, unknown> & {
|
|
coldStart?: boolean;
|
|
kind?: string;
|
|
phase?: string;
|
|
};
|
|
type GauntletContext = {
|
|
repoRoot: string;
|
|
outputDir: string;
|
|
env: NodeJS.ProcessEnv;
|
|
matrix: PluginEntry[];
|
|
plugins: PluginEntry[];
|
|
rows: GauntletMeasuredRow[];
|
|
commandTimeoutMs: number;
|
|
skipSlashHelp: boolean;
|
|
includePluginOwnedCliAliases: boolean;
|
|
qaBaseline: boolean;
|
|
qaScenarios: string[];
|
|
qaPluginChunkSize: number;
|
|
qaTimeoutMs: number;
|
|
};
|
|
type QaSummary = NonNullable<ReturnType<typeof readQaSuiteSummary>["summary"]>;
|
|
|
|
/**
|
|
* Parses plugin gateway gauntlet CLI arguments and env defaults.
|
|
*/
|
|
export function parseArgs(argv: string[]) {
|
|
const args = stripLeadingPackageManagerSeparator(argv);
|
|
const pluginIds: string[] = [];
|
|
const qaScenarios: string[] = [];
|
|
const options = {
|
|
repoRoot: process.cwd(),
|
|
outputDir: path.join(
|
|
process.cwd(),
|
|
".artifacts",
|
|
"plugin-gateway-gauntlet",
|
|
new Date().toISOString().replace(/[:.]/g, "-"),
|
|
),
|
|
pluginIds,
|
|
shardTotal: readOptionalPositiveIntEnv("OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_TOTAL") ?? 1,
|
|
shardIndex: readOptionalNonNegativeIntEnv("OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_INDEX") ?? 0,
|
|
limit: undefined as number | undefined,
|
|
skipPrebuild: false,
|
|
skipLifecycle: false,
|
|
skipQa: false,
|
|
qaBaseline: false,
|
|
skipSlashHelp: false,
|
|
qaScenarios,
|
|
qaPluginChunkSize: DEFAULT_QA_PLUGIN_CHUNK_SIZE,
|
|
cpuCoreWarn: DEFAULT_CPU_CORE_WARN,
|
|
hotWallWarnMs: DEFAULT_HOT_WALL_WARN_MS,
|
|
maxRssWarnMb: DEFAULT_MAX_RSS_WARN_MB,
|
|
wallAnomalyMultiplier: 3,
|
|
rssAnomalyMultiplier: 2.5,
|
|
qaCpuRegressionMultiplier: 2,
|
|
qaWallRegressionMultiplier: 2,
|
|
commandTimeoutMs: 120_000,
|
|
buildTimeoutMs: 600_000,
|
|
qaTimeoutMs: 900_000,
|
|
allowEmpty: false,
|
|
failOnObservation: process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_FAIL_ON_OBSERVATION === "1",
|
|
keepRunRoot: process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_KEEP_RUN_ROOT === "1",
|
|
};
|
|
const envIds = normalizeCsv(process.env.OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_IDS);
|
|
options.pluginIds.push(...envIds);
|
|
const seenSingleValueFlags = new Set<string>();
|
|
parseArgv: for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index];
|
|
if (arg === undefined) {
|
|
break;
|
|
}
|
|
if (SINGLE_VALUE_FLAGS.has(arg)) {
|
|
if (seenSingleValueFlags.has(arg)) {
|
|
throw new Error(`${arg} was provided more than once`);
|
|
}
|
|
seenSingleValueFlags.add(arg);
|
|
}
|
|
const readValue = () => {
|
|
const value = args[index + 1];
|
|
if (!value || value.startsWith("-")) {
|
|
throw new Error(`Missing value for ${arg}`);
|
|
}
|
|
index += 1;
|
|
return value;
|
|
};
|
|
switch (arg) {
|
|
case "--":
|
|
break parseArgv;
|
|
case "--repo-root":
|
|
options.repoRoot = path.resolve(readValue());
|
|
break;
|
|
case "--output-dir":
|
|
options.outputDir = path.resolve(readValue());
|
|
break;
|
|
case "--plugin":
|
|
options.pluginIds.push(readValue());
|
|
break;
|
|
case "--shard-total":
|
|
options.shardTotal = parsePositiveInt(readValue(), "--shard-total");
|
|
break;
|
|
case "--shard-index":
|
|
options.shardIndex = parseNonNegativeInt(readValue(), "--shard-index");
|
|
break;
|
|
case "--limit":
|
|
options.limit = parsePositiveInt(readValue(), "--limit");
|
|
break;
|
|
case "--qa-scenario":
|
|
options.qaScenarios.push(readValue());
|
|
break;
|
|
case "--qa-plugin-chunk-size":
|
|
options.qaPluginChunkSize = parsePositiveInt(readValue(), "--qa-plugin-chunk-size");
|
|
break;
|
|
case "--qa-baseline":
|
|
options.qaBaseline = true;
|
|
break;
|
|
case "--cpu-core-warn":
|
|
options.cpuCoreWarn = parsePositiveNumber(readValue(), "--cpu-core-warn");
|
|
break;
|
|
case "--hot-wall-warn-ms":
|
|
options.hotWallWarnMs = parsePositiveInt(readValue(), "--hot-wall-warn-ms");
|
|
break;
|
|
case "--max-rss-warn-mb":
|
|
options.maxRssWarnMb = parsePositiveNumber(readValue(), "--max-rss-warn-mb");
|
|
break;
|
|
case "--wall-anomaly-multiplier":
|
|
options.wallAnomalyMultiplier = parsePositiveNumber(
|
|
readValue(),
|
|
"--wall-anomaly-multiplier",
|
|
);
|
|
break;
|
|
case "--rss-anomaly-multiplier":
|
|
options.rssAnomalyMultiplier = parsePositiveNumber(readValue(), "--rss-anomaly-multiplier");
|
|
break;
|
|
case "--qa-cpu-regression-multiplier":
|
|
options.qaCpuRegressionMultiplier = parsePositiveNumber(
|
|
readValue(),
|
|
"--qa-cpu-regression-multiplier",
|
|
);
|
|
break;
|
|
case "--qa-wall-regression-multiplier":
|
|
options.qaWallRegressionMultiplier = parsePositiveNumber(
|
|
readValue(),
|
|
"--qa-wall-regression-multiplier",
|
|
);
|
|
break;
|
|
case "--command-timeout-ms":
|
|
options.commandTimeoutMs = parsePositiveInt(readValue(), "--command-timeout-ms");
|
|
break;
|
|
case "--build-timeout-ms":
|
|
options.buildTimeoutMs = parsePositiveInt(readValue(), "--build-timeout-ms");
|
|
break;
|
|
case "--qa-timeout-ms":
|
|
options.qaTimeoutMs = parsePositiveInt(readValue(), "--qa-timeout-ms");
|
|
break;
|
|
case "--skip-prebuild":
|
|
options.skipPrebuild = true;
|
|
break;
|
|
case "--skip-lifecycle":
|
|
options.skipLifecycle = true;
|
|
break;
|
|
case "--skip-qa":
|
|
options.skipQa = true;
|
|
break;
|
|
case "--skip-slash-help":
|
|
options.skipSlashHelp = true;
|
|
break;
|
|
case "--keep-run-root":
|
|
options.keepRunRoot = true;
|
|
break;
|
|
case "--allow-empty":
|
|
options.allowEmpty = true;
|
|
break;
|
|
case "--fail-on-observation":
|
|
options.failOnObservation = true;
|
|
break;
|
|
case "--help":
|
|
printHelp();
|
|
process.exit(0);
|
|
break;
|
|
default:
|
|
throw new Error(`Unknown argument: ${arg}`);
|
|
}
|
|
}
|
|
if (options.qaScenarios.length === 0) {
|
|
options.qaScenarios = [...DEFAULT_QA_SCENARIOS];
|
|
}
|
|
assertNoDuplicateValues(options.pluginIds, "--plugin");
|
|
assertNoDuplicateValues(options.qaScenarios, "--qa-scenario");
|
|
return options;
|
|
}
|
|
|
|
function printHelp() {
|
|
console.log(`Usage: pnpm test:plugins:gateway-gauntlet [options]
|
|
|
|
Runs a shardable bundled-plugin lifecycle, slash inventory, and QA gateway perf gauntlet.
|
|
|
|
Options:
|
|
--plugin <id> Plugin id to include, repeatable
|
|
--shard-total <count> Total plugin shards (default: env or 1)
|
|
--shard-index <index> Zero-based shard index (default: env or 0)
|
|
--limit <count> Limit selected plugins after sharding
|
|
--output-dir <path> Artifact directory
|
|
--qa-scenario <id> QA Lab scenario id, repeatable
|
|
--qa-plugin-chunk-size <count> Plugins enabled per QA run (default: 12)
|
|
--qa-baseline Run a no-extra-plugin QA baseline before plugin chunks
|
|
--cpu-core-warn <ratio> Hot CPU threshold (default: 0.9)
|
|
--hot-wall-warn-ms <ms> Minimum wall time for hot CPU observations (default: 30000)
|
|
--max-rss-warn-mb <mb> Maximum RSS warning threshold (default: 1536)
|
|
--wall-anomaly-multiplier <n> Wall-time anomaly multiplier (default: 3)
|
|
--rss-anomaly-multiplier <n> RSS anomaly multiplier (default: 2.5)
|
|
--qa-cpu-regression-multiplier <n> QA baseline CPU regression multiplier (default: 2)
|
|
--qa-wall-regression-multiplier <n> QA baseline wall regression multiplier (default: 2)
|
|
--command-timeout-ms <ms> Lifecycle/slash command timeout (default: 120000)
|
|
--build-timeout-ms <ms> Prebuild command timeout (default: 600000)
|
|
--qa-timeout-ms <ms> QA chunk timeout (default: 900000)
|
|
--skip-prebuild Skip the upfront build used to avoid per-command rebuild noise
|
|
--skip-lifecycle Skip plugin install/inspect/disable/enable/doctor/uninstall
|
|
--skip-qa Skip QA Lab RPC conversation runs
|
|
--skip-slash-help Skip CLI help probes for plugin-declared command aliases
|
|
--allow-empty Allow zero-command runs when every active phase is skipped
|
|
--fail-on-observation Treat RSS/CPU/wall observation rows as guard failures
|
|
--keep-run-root Preserve isolated HOME/state/log temp root after success
|
|
|
|
Environment:
|
|
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_IDS Comma-separated plugin ids to include
|
|
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_TOTAL Total plugin shards
|
|
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_INDEX Zero-based shard index
|
|
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_FAIL_ON_OBSERVATION=1
|
|
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_KEEP_RUN_ROOT=1
|
|
OPENCLAW_PLUGIN_GATEWAY_GAUNTLET_QA_SUMMARY_MAX_BYTES QA summary read ceiling
|
|
`);
|
|
}
|
|
|
|
function normalizeCsv(raw: string | undefined) {
|
|
return raw
|
|
? raw
|
|
.split(",")
|
|
.map((entry) => entry.trim())
|
|
.filter((entry) => entry.length > 0)
|
|
: [];
|
|
}
|
|
|
|
function assertNoDuplicateValues(values: string[], label: string) {
|
|
const seen = new Set();
|
|
for (const value of values) {
|
|
const normalized = value.trim();
|
|
if (!normalized) {
|
|
continue;
|
|
}
|
|
if (seen.has(normalized)) {
|
|
throw new Error(`Duplicate ${label} value: ${normalized}`);
|
|
}
|
|
seen.add(normalized);
|
|
}
|
|
}
|
|
|
|
function readOptionalPositiveIntEnv(name: string) {
|
|
const raw = process.env[name];
|
|
return raw ? parsePositiveInt(raw, name) : undefined;
|
|
}
|
|
|
|
function readOptionalNonNegativeIntEnv(name: string) {
|
|
const raw = process.env[name];
|
|
return raw ? parseNonNegativeInt(raw, name) : undefined;
|
|
}
|
|
|
|
function shouldPromoteObservationGuardFailure(observation: GauntletObservation) {
|
|
// Setup and the first cold work command are still reported, but they are not
|
|
// stable enough to fail the gauntlet's steady-state regression guard.
|
|
return observation?.phase !== "prebuild" && observation?.coldStart !== true;
|
|
}
|
|
|
|
export function buildObservationGuardFailures(
|
|
observations: GauntletObservation[],
|
|
enabled = false,
|
|
) {
|
|
if (!enabled) {
|
|
return [];
|
|
}
|
|
return observations
|
|
.filter((observation) => shouldPromoteObservationGuardFailure(observation))
|
|
.map((observation) => ({
|
|
kind: `observation:${observation.kind ?? "unknown"}`,
|
|
message: `Gauntlet observation threshold exceeded: ${observation.kind ?? "unknown"}`,
|
|
observation,
|
|
}));
|
|
}
|
|
|
|
/**
|
|
* Builds the command that prepares QA runtime artifacts before gauntlet probes.
|
|
*/
|
|
export function createGauntletPrebuildCommand(repoRoot: string) {
|
|
return {
|
|
command: process.execPath,
|
|
args: ["--import", "tsx", path.join(repoRoot, "scripts", "build-all.mts"), "qaRuntime"],
|
|
};
|
|
}
|
|
|
|
function openclawCommand(repoRoot: string, args: string[]) {
|
|
return {
|
|
command: process.execPath,
|
|
args: [path.join(repoRoot, "dist", "entry.js"), ...args],
|
|
};
|
|
}
|
|
|
|
function builtEntryPath(repoRoot: string) {
|
|
return path.join(repoRoot, "dist", "entry.js");
|
|
}
|
|
|
|
function selectSlashHelpAliases(plugin: PluginEntry, includePluginOwnedCliAliases: boolean) {
|
|
return includePluginOwnedCliAliases
|
|
? plugin.cliCommandAliases
|
|
: plugin.cliCommandAliases.filter((entry) => !isPluginOwnedCliAlias(entry));
|
|
}
|
|
|
|
function requiresBuiltEntry(options: ReturnType<typeof parseArgs>, selectedPlugins: PluginEntry[]) {
|
|
if (selectedPlugins.length === 0) {
|
|
return false;
|
|
}
|
|
if (!options.skipLifecycle) {
|
|
return true;
|
|
}
|
|
if (options.skipSlashHelp) {
|
|
return false;
|
|
}
|
|
return selectedPlugins.some((plugin) => selectSlashHelpAliases(plugin, true).length > 0);
|
|
}
|
|
|
|
function sourceOpenclawCommand(repoRoot: string, args: string[]) {
|
|
return {
|
|
command: process.execPath,
|
|
args: [path.join(repoRoot, "scripts", "run-node.mjs"), ...args],
|
|
};
|
|
}
|
|
|
|
function chunkArray<Value>(values: Value[], chunkSize: number) {
|
|
const chunks: Value[][] = [];
|
|
for (let index = 0; index < values.length; index += chunkSize) {
|
|
chunks.push(values.slice(index, index + chunkSize));
|
|
}
|
|
return chunks;
|
|
}
|
|
|
|
/**
|
|
* Converts an output path to a repo-relative path, rejecting paths outside the repo.
|
|
*/
|
|
export function toRepoRelativePath(repoRoot: string, absolutePath: string) {
|
|
const relativePath = path.relative(repoRoot, absolutePath);
|
|
if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
|
|
throw new Error(`Output path must stay inside repo root: ${absolutePath}`);
|
|
}
|
|
return relativePath;
|
|
}
|
|
|
|
function validateOutputDir(options: ReturnType<typeof parseArgs>, repoRoot: string) {
|
|
if (!options.skipQa) {
|
|
toRepoRelativePath(repoRoot, path.join(options.outputDir, "qa-suite"));
|
|
}
|
|
}
|
|
|
|
function createIsolatedEnv(repoRoot: string, runRoot: string) {
|
|
const home = path.join(runRoot, "home");
|
|
const stateDir = path.join(runRoot, "state");
|
|
fs.mkdirSync(home, { recursive: true });
|
|
fs.mkdirSync(stateDir, { recursive: true });
|
|
return {
|
|
...process.env,
|
|
HOME: home,
|
|
XDG_CONFIG_HOME: path.join(home, ".config"),
|
|
XDG_CACHE_HOME: path.join(home, ".cache"),
|
|
XDG_DATA_HOME: path.join(home, ".local", "share"),
|
|
OPENCLAW_STATE_DIR: stateDir,
|
|
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
|
|
OPENCLAW_LOG_DIR: path.join(runRoot, "logs"),
|
|
OPENCLAW_QA_SUITE_PROGRESS: process.env.OPENCLAW_QA_SUITE_PROGRESS ?? "1",
|
|
PATH: process.env.PATH,
|
|
PWD: repoRoot,
|
|
};
|
|
}
|
|
|
|
function hasUsrBinTime() {
|
|
return fs.existsSync("/usr/bin/time");
|
|
}
|
|
|
|
function timeWrapperArgs(command: string, args: string[]) {
|
|
if (!hasUsrBinTime()) {
|
|
return { command, args, mode: "none" };
|
|
}
|
|
if (process.platform === "darwin") {
|
|
return { command: "/usr/bin/time", args: ["-l", command, ...args], mode: "bsd" };
|
|
}
|
|
return { command: "/usr/bin/time", args: ["-v", command, ...args], mode: "gnu" };
|
|
}
|
|
|
|
/**
|
|
* Parses `/usr/bin/time` output into wall, CPU, and RSS metrics.
|
|
*/
|
|
export function parseTimedMetrics(stderr: string, wallMs: number, mode: string) {
|
|
let userSeconds: number | null = null;
|
|
let systemSeconds: number | null = null;
|
|
let maxRssMb: number | null = null;
|
|
if (mode === "gnu") {
|
|
userSeconds = parseLastFloat(stderr, /^\s*User time \(seconds\):\s*([0-9.]+)\s*$/gmu);
|
|
systemSeconds = parseLastFloat(stderr, /^\s*System time \(seconds\):\s*([0-9.]+)\s*$/gmu);
|
|
const maxRssKb = parseLastFloat(
|
|
stderr,
|
|
/^\s*Maximum resident set size \(kbytes\):\s*([0-9.]+)\s*$/gmu,
|
|
);
|
|
maxRssMb = maxRssKb == null ? null : maxRssKb / 1024;
|
|
} else if (mode === "bsd") {
|
|
const cpuLine = parseLastMatch(
|
|
stderr,
|
|
/^\s*[0-9.]+\s+real\s+([0-9.]+)\s+user\s+([0-9.]+)\s+sys\s*$/gmu,
|
|
);
|
|
userSeconds = parseMatchFloat(cpuLine, 1);
|
|
systemSeconds = parseMatchFloat(cpuLine, 2);
|
|
const maxRssBytes = parseLastFloat(stderr, /^\s*([0-9]+)\s+maximum resident set size\s*$/gmu);
|
|
maxRssMb = maxRssBytes == null ? null : maxRssBytes / 1024 / 1024;
|
|
}
|
|
const cpuMs =
|
|
userSeconds == null && systemSeconds == null
|
|
? null
|
|
: ((userSeconds ?? 0) + (systemSeconds ?? 0)) * 1000;
|
|
return {
|
|
wallMs,
|
|
cpuMs,
|
|
cpuCoreRatio: cpuMs == null || wallMs <= 0 ? null : cpuMs / wallMs,
|
|
maxRssMb,
|
|
};
|
|
}
|
|
|
|
function parseLastMatch(value: string, pattern: RegExp) {
|
|
let lastMatch: RegExpMatchArray | null = null;
|
|
for (const match of value.matchAll(pattern)) {
|
|
lastMatch = match;
|
|
}
|
|
return lastMatch;
|
|
}
|
|
|
|
function parseMatchFloat(match: RegExpMatchArray | null, index: number) {
|
|
if (!match) {
|
|
return null;
|
|
}
|
|
const parsed = Number(match[index]);
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function parseLastFloat(value: string, pattern: RegExp) {
|
|
return parseMatchFloat(parseLastMatch(value, pattern), 1);
|
|
}
|
|
|
|
function stripAnsi(value: string) {
|
|
return value.replace(ANSI_PATTERN, "");
|
|
}
|
|
|
|
function resolveTimerTimeoutMs(valueMs: number) {
|
|
const value = Number.isFinite(valueMs) ? Math.floor(valueMs) : MAX_TIMER_TIMEOUT_MS;
|
|
return Math.min(Math.max(value, 1), MAX_TIMER_TIMEOUT_MS);
|
|
}
|
|
|
|
function resolveOptionalTimerTimeoutMs(valueMs: number | undefined) {
|
|
return valueMs === undefined || valueMs <= 0 ? null : resolveTimerTimeoutMs(valueMs);
|
|
}
|
|
|
|
function writeCommandLog(params: {
|
|
logDir: string;
|
|
label: string;
|
|
command: string[];
|
|
stdout: string;
|
|
stderr: string;
|
|
}) {
|
|
const { logDir, label, stdout, stderr } = params;
|
|
fs.mkdirSync(logDir, { recursive: true });
|
|
const safeLabel = label.replace(/[^a-zA-Z0-9_.-]+/gu, "_");
|
|
const logPath = path.join(logDir, `${safeLabel}.log`);
|
|
fs.writeFileSync(
|
|
logPath,
|
|
[`$ ${params.command.join(" ")}`, "", stripAnsi(stdout), stripAnsi(stderr)].join("\n"),
|
|
"utf8",
|
|
);
|
|
return logPath;
|
|
}
|
|
|
|
/**
|
|
* Runs a measured command through the live process implementation.
|
|
*/
|
|
export async function runMeasuredCommand(params: GauntletMeasuredCommandParams) {
|
|
return await runMeasuredCommandLive(params);
|
|
}
|
|
|
|
/**
|
|
* Runs one command with optional timing wrapper, bounded output, and log capture.
|
|
*/
|
|
export function runMeasuredCommandLive(params: GauntletMeasuredCommandParams) {
|
|
const { command, args, mode } =
|
|
params.timeMode === "none"
|
|
? { command: params.command, args: params.args, mode: "none" }
|
|
: timeWrapperArgs(params.command, params.args);
|
|
const started = performance.now();
|
|
return new Promise<GauntletMeasuredRow>((resolve) => {
|
|
let stdout = "";
|
|
let stderr = "";
|
|
let stdoutBytes = 0;
|
|
let stderrBytes = 0;
|
|
let stdoutRelayBytes = 0;
|
|
let stderrRelayBytes = 0;
|
|
let stdoutTruncated = false;
|
|
let stderrTruncated = false;
|
|
let stdoutRelayTruncated = false;
|
|
let stderrRelayTruncated = false;
|
|
let spawnError: GauntletMeasuredRow["spawnError"] = null;
|
|
let timedOut = false;
|
|
let settled = false;
|
|
let forceKillTimeout: TimerHandle | null = null;
|
|
let forceKillAt = 0;
|
|
let parentTerminationSignal: ProcessSignal | null = null;
|
|
const maxBufferBytes = params.maxBufferBytes ?? COMMAND_OUTPUT_MAX_BUFFER_BYTES;
|
|
const maxRelayBytes = params.consoleOutputMaxBytes ?? maxBufferBytes;
|
|
const timeoutMs = resolveOptionalTimerTimeoutMs(params.timeoutMs);
|
|
const timeoutKillGraceMs = resolveTimerTimeoutMs(params.timeoutKillGraceMs ?? 5_000);
|
|
const spawnOptions = mode === "none" ? (params.spawnOptions ?? {}) : {};
|
|
const useProcessGroup =
|
|
process.platform !== "win32" &&
|
|
params.killProcessGroup !== false &&
|
|
spawnOptions.detached !== false;
|
|
const child = spawn(command, args, {
|
|
cwd: params.cwd,
|
|
env: params.env,
|
|
...spawnOptions,
|
|
...(useProcessGroup ? { detached: true } : {}),
|
|
});
|
|
const killMeasuredProcess = (signal: ProcessSignal = "SIGTERM") => {
|
|
if (useProcessGroup && child.pid) {
|
|
try {
|
|
process.kill(-child.pid, signal as NodeJS.Signals);
|
|
return;
|
|
} catch {}
|
|
}
|
|
child.kill(signal as NodeJS.Signals);
|
|
};
|
|
const processGroupAlive = () => {
|
|
if (!useProcessGroup || !child.pid) {
|
|
return false;
|
|
}
|
|
try {
|
|
process.kill(-child.pid, 0);
|
|
return true;
|
|
} catch (error) {
|
|
return Boolean(
|
|
error && typeof error === "object" && "code" in error && error.code === "EPERM",
|
|
);
|
|
}
|
|
};
|
|
const waitForProcessGroupExit = async (timeoutBudgetMs: number) => {
|
|
const deadlineAt = Date.now() + timeoutBudgetMs;
|
|
while (Date.now() < deadlineAt) {
|
|
if (!processGroupAlive()) {
|
|
return true;
|
|
}
|
|
await new Promise((resolvePoll) => {
|
|
setTimeout(resolvePoll, 25);
|
|
});
|
|
}
|
|
return !processGroupAlive();
|
|
};
|
|
const scheduleForceKill = () => {
|
|
forceKillAt = Date.now() + timeoutKillGraceMs;
|
|
forceKillTimeout ??= setTimeout(() => {
|
|
forceKillTimeout = null;
|
|
killMeasuredProcess("SIGKILL");
|
|
}, timeoutKillGraceMs);
|
|
forceKillTimeout.unref?.();
|
|
};
|
|
const parentSignalHandlers = new Map<ProcessSignal, () => void>();
|
|
const removeParentSignalHandlers = () => {
|
|
for (const [signal, handler] of parentSignalHandlers) {
|
|
process.off(signal, handler);
|
|
}
|
|
parentSignalHandlers.clear();
|
|
};
|
|
const parentSignals: ProcessSignal[] =
|
|
process.platform === "win32" ? ["SIGINT", "SIGTERM"] : ["SIGINT", "SIGTERM", "SIGHUP"];
|
|
for (const signal of parentSignals) {
|
|
const handler = () => {
|
|
if (parentTerminationSignal) {
|
|
return;
|
|
}
|
|
parentTerminationSignal = signal;
|
|
killMeasuredProcess(signal);
|
|
scheduleForceKill();
|
|
};
|
|
parentSignalHandlers.set(signal, handler);
|
|
process.once(signal, handler);
|
|
}
|
|
const appendCapturedOutput = (streamName: "stdout" | "stderr", chunk: string | Uint8Array) => {
|
|
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
const currentBytes = streamName === "stdout" ? stdoutBytes : stderrBytes;
|
|
const alreadyTruncated = streamName === "stdout" ? stdoutTruncated : stderrTruncated;
|
|
if (alreadyTruncated) {
|
|
return;
|
|
}
|
|
const remainingBytes = maxBufferBytes - currentBytes;
|
|
const appendTruncation = () => {
|
|
const message = `\n[${streamName} truncated after ${maxBufferBytes} bytes]\n`;
|
|
if (streamName === "stdout") {
|
|
stdout += message;
|
|
stdoutTruncated = true;
|
|
} else {
|
|
stderr += message;
|
|
stderrTruncated = true;
|
|
}
|
|
};
|
|
if (remainingBytes <= 0) {
|
|
appendTruncation();
|
|
return;
|
|
}
|
|
const capturedBuffer =
|
|
buffer.length > remainingBytes ? buffer.subarray(0, remainingBytes) : buffer;
|
|
if (streamName === "stdout") {
|
|
stdout += capturedBuffer.toString("utf8");
|
|
stdoutBytes += capturedBuffer.length;
|
|
} else {
|
|
stderr += capturedBuffer.toString("utf8");
|
|
stderrBytes += capturedBuffer.length;
|
|
}
|
|
if (buffer.length > remainingBytes) {
|
|
appendTruncation();
|
|
}
|
|
};
|
|
const relayOutput = (streamName: "stdout" | "stderr", chunk: string | Uint8Array) => {
|
|
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
const currentBytes = streamName === "stdout" ? stdoutRelayBytes : stderrRelayBytes;
|
|
const alreadyTruncated =
|
|
streamName === "stdout" ? stdoutRelayTruncated : stderrRelayTruncated;
|
|
if (alreadyTruncated) {
|
|
return;
|
|
}
|
|
const write =
|
|
streamName === "stdout"
|
|
? process.stdout.write.bind(process.stdout)
|
|
: process.stderr.write.bind(process.stderr);
|
|
const markTruncated = () => {
|
|
write(`\n[${streamName} relay truncated after ${maxRelayBytes} bytes]\n`);
|
|
if (streamName === "stdout") {
|
|
stdoutRelayTruncated = true;
|
|
} else {
|
|
stderrRelayTruncated = true;
|
|
}
|
|
};
|
|
const remainingBytes = maxRelayBytes - currentBytes;
|
|
if (remainingBytes <= 0) {
|
|
markTruncated();
|
|
return;
|
|
}
|
|
const relayedBuffer =
|
|
buffer.length > remainingBytes ? buffer.subarray(0, remainingBytes) : buffer;
|
|
if (relayedBuffer.length > 0) {
|
|
write(relayedBuffer.toString("utf8"));
|
|
}
|
|
if (streamName === "stdout") {
|
|
stdoutRelayBytes += relayedBuffer.length;
|
|
} else {
|
|
stderrRelayBytes += relayedBuffer.length;
|
|
}
|
|
if (buffer.length > remainingBytes) {
|
|
markTruncated();
|
|
}
|
|
};
|
|
const appendOutput = (streamName: "stdout" | "stderr", chunk: string | Uint8Array) => {
|
|
relayOutput(streamName, chunk);
|
|
appendCapturedOutput(streamName, chunk);
|
|
};
|
|
child.stdout?.on("data", (chunk) => appendOutput("stdout", chunk));
|
|
child.stderr?.on("data", (chunk) => appendOutput("stderr", chunk));
|
|
const timeout =
|
|
timeoutMs !== null
|
|
? setTimeout(() => {
|
|
timedOut = true;
|
|
spawnError = {
|
|
code: "ETIMEDOUT",
|
|
message: `Command timed out after ${timeoutMs}ms`,
|
|
};
|
|
killMeasuredProcess();
|
|
scheduleForceKill();
|
|
}, timeoutMs)
|
|
: null;
|
|
timeout?.unref?.();
|
|
const finish = (status: number | null, signal: ProcessSignal | null) => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
settled = true;
|
|
if (timeout) {
|
|
clearTimeout(timeout);
|
|
}
|
|
if (forceKillTimeout) {
|
|
clearTimeout(forceKillTimeout);
|
|
}
|
|
removeParentSignalHandlers();
|
|
const wallMs = performance.now() - started;
|
|
const finalStatus = status ?? (signal || spawnError ? 1 : 0);
|
|
const finalStderr = [
|
|
stderr,
|
|
spawnError ? `[spawn error] ${spawnError.code ?? "unknown"} ${spawnError.message}` : "",
|
|
]
|
|
.filter(Boolean)
|
|
.join("\n");
|
|
let logPath: string | null = null;
|
|
let logWriteError: string | null = null;
|
|
try {
|
|
logPath = writeCommandLog({
|
|
logDir: params.logDir,
|
|
label: params.label,
|
|
command: [params.command, ...params.args],
|
|
stdout,
|
|
stderr: finalStderr,
|
|
});
|
|
} catch (error) {
|
|
logWriteError = error instanceof Error ? error.message : String(error);
|
|
}
|
|
const outputDiagnosticFailure = detectCommandDiagnosticFailure(stdout, finalStderr);
|
|
const diagnosticFailure =
|
|
outputDiagnosticFailure ?? (logWriteError ? "command-log-write-failure" : null);
|
|
resolve({
|
|
label: params.label,
|
|
phase: params.phase,
|
|
pluginId: params.pluginId ?? null,
|
|
status: logWriteError ? 1 : finalStatus,
|
|
diagnosticFailure,
|
|
signal: signal ?? null,
|
|
timedOut,
|
|
spawnError,
|
|
logPath,
|
|
...(logWriteError ? { logWriteError } : {}),
|
|
...parseTimedMetrics(finalStderr, wallMs, mode),
|
|
});
|
|
};
|
|
const waitForTerminationCleanup = async () => {
|
|
const remainingGraceMs = Math.max(0, forceKillAt - Date.now());
|
|
if (remainingGraceMs > 0) {
|
|
await waitForProcessGroupExit(remainingGraceMs);
|
|
}
|
|
if (processGroupAlive()) {
|
|
killMeasuredProcess("SIGKILL");
|
|
await waitForProcessGroupExit(100);
|
|
}
|
|
};
|
|
const rethrowParentTermination = (signal: ProcessSignal) => {
|
|
settled = true;
|
|
if (timeout) {
|
|
clearTimeout(timeout);
|
|
}
|
|
if (forceKillTimeout) {
|
|
clearTimeout(forceKillTimeout);
|
|
}
|
|
removeParentSignalHandlers();
|
|
process.kill(process.pid, signal as NodeJS.Signals);
|
|
};
|
|
const finishAfterTimeoutTeardown = async (
|
|
status: number | null,
|
|
signal: ProcessSignal | null,
|
|
) => {
|
|
await waitForTerminationCleanup();
|
|
if (parentTerminationSignal) {
|
|
rethrowParentTermination(parentTerminationSignal);
|
|
return;
|
|
}
|
|
finish(status, signal);
|
|
};
|
|
const finishAfterParentTermination = async (signal: ProcessSignal) => {
|
|
await waitForTerminationCleanup();
|
|
rethrowParentTermination(signal);
|
|
};
|
|
child.on("error", (error) => {
|
|
spawnError = {
|
|
code: "code" in error && typeof error.code === "string" ? error.code : null,
|
|
message: error.message,
|
|
};
|
|
finish(null, null);
|
|
});
|
|
child.on("close", (status, signal) => {
|
|
if (parentTerminationSignal) {
|
|
void finishAfterParentTermination(parentTerminationSignal);
|
|
return;
|
|
}
|
|
if (timedOut) {
|
|
void finishAfterTimeoutTeardown(status, signal);
|
|
return;
|
|
}
|
|
finish(status, signal);
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Reports whether gauntlet result rows contain work beyond the prebuild step.
|
|
*/
|
|
export function hasGauntletWorkRows(rows: Array<Pick<GauntletMeasuredRow, "phase">>) {
|
|
return rows.some((row) => row.phase !== "prebuild");
|
|
}
|
|
|
|
function isPluginOwnedCliAlias(alias: CommandAlias) {
|
|
return alias.kind === "runtime-slash" && alias.cliCommand === alias.name;
|
|
}
|
|
|
|
function buildSlashHelpProbe(
|
|
params: GauntletContext & {
|
|
plugin: PluginEntry;
|
|
alias: CommandAlias;
|
|
},
|
|
) {
|
|
const command = params.alias.cliCommand ?? params.alias.name;
|
|
return {
|
|
cwd: params.repoRoot,
|
|
env: params.env,
|
|
logDir: path.join(params.outputDir, "logs", "slash-help"),
|
|
...openclawCommand(params.repoRoot, [command, "--help"]),
|
|
label: `${params.plugin.id}-slash-${params.alias.name}`,
|
|
phase: "slash:help",
|
|
pluginId: params.plugin.id,
|
|
timeoutMs: params.commandTimeoutMs,
|
|
};
|
|
}
|
|
|
|
async function runPluginLifecycleCommand(
|
|
params: GauntletContext & {
|
|
logPluginId: string;
|
|
label: string;
|
|
phase: string;
|
|
args: string[];
|
|
pluginId: string;
|
|
},
|
|
) {
|
|
process.stderr.write(`[plugin-gauntlet] ${params.logPluginId} ${params.phase}\n`);
|
|
params.rows.push(
|
|
await runMeasuredCommand({
|
|
cwd: params.repoRoot,
|
|
env: params.env,
|
|
logDir: path.join(params.outputDir, "logs", "lifecycle"),
|
|
...openclawCommand(params.repoRoot, ["plugins", ...params.args]),
|
|
label: params.label,
|
|
phase: `lifecycle:${params.phase}`,
|
|
pluginId: params.pluginId,
|
|
timeoutMs: params.commandTimeoutMs,
|
|
}),
|
|
);
|
|
}
|
|
|
|
async function runPluginLifecycle(params: GauntletContext) {
|
|
for (const plugin of params.plugins) {
|
|
const requiredPlugins = collectRequiredPluginEntries(params.matrix, [plugin]);
|
|
for (const requiredPlugin of requiredPlugins) {
|
|
await runPluginLifecycleCommand({
|
|
...params,
|
|
logPluginId: plugin.id,
|
|
label: `${plugin.id}-requires-${requiredPlugin.id}-install`,
|
|
phase: `requires:${requiredPlugin.id}:install`,
|
|
args: ["install", requiredPlugin.id],
|
|
pluginId: requiredPlugin.id,
|
|
});
|
|
}
|
|
type LifecycleCommand =
|
|
| { phase: string; args: string[] }
|
|
| { phase: string; alias: CommandAlias };
|
|
const commands = [
|
|
{
|
|
phase: "install",
|
|
args: ["install", plugin.id],
|
|
},
|
|
{ phase: "inspect", args: ["inspect", plugin.id, "--json"] },
|
|
...(params.skipSlashHelp
|
|
? []
|
|
: plugin.cliCommandAliases
|
|
.filter(isPluginOwnedCliAlias)
|
|
.map((alias) => ({ phase: `slash-help:${alias.name}`, alias }))),
|
|
{ phase: "disable", args: ["disable", plugin.id] },
|
|
...(plugin.hasRequiredConfigFields ? [] : [{ phase: "enable", args: ["enable", plugin.id] }]),
|
|
{ phase: "doctor", args: ["doctor"] },
|
|
{ phase: "uninstall", args: ["uninstall", plugin.id, "--force"] },
|
|
] satisfies LifecycleCommand[];
|
|
for (const command of commands) {
|
|
if ("alias" in command) {
|
|
process.stderr.write(`[plugin-gauntlet] ${plugin.id} ${command.phase}\n`);
|
|
params.rows.push(
|
|
await runMeasuredCommand({
|
|
...buildSlashHelpProbe({
|
|
...params,
|
|
plugin,
|
|
alias: command.alias,
|
|
}),
|
|
label: `${plugin.id}-${command.phase}`,
|
|
}),
|
|
);
|
|
continue;
|
|
}
|
|
await runPluginLifecycleCommand({
|
|
...params,
|
|
logPluginId: plugin.id,
|
|
label: `${plugin.id}-${command.phase}`,
|
|
phase: command.phase,
|
|
args: command.args,
|
|
pluginId: plugin.id,
|
|
});
|
|
}
|
|
for (const requiredPlugin of requiredPlugins.toReversed()) {
|
|
await runPluginLifecycleCommand({
|
|
...params,
|
|
logPluginId: plugin.id,
|
|
label: `${plugin.id}-requires-${requiredPlugin.id}-uninstall`,
|
|
phase: `requires:${requiredPlugin.id}:uninstall`,
|
|
args: ["uninstall", requiredPlugin.id, "--force"],
|
|
pluginId: requiredPlugin.id,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
async function runSlashHelpProbes(params: GauntletContext) {
|
|
for (const plugin of params.plugins) {
|
|
const aliases = selectSlashHelpAliases(plugin, params.includePluginOwnedCliAliases);
|
|
for (const alias of aliases) {
|
|
process.stderr.write(`[plugin-gauntlet] ${plugin.id} slash-help /${alias.name}\n`);
|
|
params.rows.push(
|
|
await runMeasuredCommand({
|
|
...buildSlashHelpProbe({
|
|
...params,
|
|
plugin,
|
|
alias,
|
|
}),
|
|
}),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function runQaChunks(params: GauntletContext) {
|
|
const chunks = [
|
|
...(params.qaBaseline ? [{ label: "baseline", plugins: [] }] : []),
|
|
...chunkArray(params.plugins, params.qaPluginChunkSize).map((plugins, index) => ({
|
|
label: `chunk-${String(index).padStart(2, "0")}`,
|
|
plugins,
|
|
})),
|
|
];
|
|
const summaries: QaSummary[] = [];
|
|
for (const [index, chunk] of chunks.entries()) {
|
|
const outputDir = path.join(params.outputDir, "qa-suite", chunk.label);
|
|
const outputArg = toRepoRelativePath(params.repoRoot, outputDir);
|
|
const pluginIds = chunk.plugins.map((plugin) => plugin.id);
|
|
const enabledPluginIds = collectPluginsWithRequiredEntries(params.matrix, chunk.plugins).map(
|
|
(plugin) => plugin.id,
|
|
);
|
|
const pluginIdLabel = pluginIds.length > 0 ? pluginIds.join(",") : "<baseline>";
|
|
process.stderr.write(
|
|
`[plugin-gauntlet] qa chunk ${index + 1}/${chunks.length}: ${pluginIdLabel}\n`,
|
|
);
|
|
const row = await runMeasuredCommand({
|
|
cwd: params.repoRoot,
|
|
env: params.env,
|
|
logDir: path.join(params.outputDir, "logs", "qa-suite"),
|
|
...sourceOpenclawCommand(params.repoRoot, [
|
|
"qa",
|
|
"suite",
|
|
"--provider-mode",
|
|
"mock-openai",
|
|
"--concurrency",
|
|
"1",
|
|
"--output-dir",
|
|
outputArg,
|
|
...params.qaScenarios.flatMap((scenario) => ["--scenario", scenario]),
|
|
...enabledPluginIds.flatMap((pluginId) => ["--enable-plugin", pluginId]),
|
|
]),
|
|
label: `qa-${chunk.label}`,
|
|
phase: "qa:rpc",
|
|
timeoutMs: params.qaTimeoutMs,
|
|
});
|
|
const summaryPath = path.join(outputDir, "qa-suite-summary.json");
|
|
const qaSummaryResult = readQaSuiteSummary(summaryPath);
|
|
const qaDiagnosticFailure =
|
|
row.status === 0 && !row.timedOut ? qaSummaryResult.diagnosticFailure : null;
|
|
params.rows.push({
|
|
...row,
|
|
pluginId: pluginIdLabel,
|
|
qaSummaryPath: summaryPath,
|
|
...(qaDiagnosticFailure ? { diagnosticFailure: qaDiagnosticFailure } : {}),
|
|
...(qaSummaryResult.diagnosticDetail
|
|
? { diagnosticDetail: qaSummaryResult.diagnosticDetail }
|
|
: {}),
|
|
...(qaSummaryResult.summary?.metrics ? { qaMetrics: qaSummaryResult.summary.metrics } : {}),
|
|
});
|
|
if (qaSummaryResult.summary) {
|
|
summaries.push(qaSummaryResult.summary);
|
|
}
|
|
}
|
|
return summaries;
|
|
}
|
|
|
|
async function main() {
|
|
const options = parseArgs(process.argv.slice(2));
|
|
const repoRoot = path.resolve(options.repoRoot);
|
|
validateOutputDir(options, repoRoot);
|
|
fs.mkdirSync(options.outputDir, { recursive: true });
|
|
const runRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-gauntlet-"));
|
|
let preserveRunRoot = options.keepRunRoot;
|
|
const env = createIsolatedEnv(repoRoot, runRoot);
|
|
try {
|
|
const matrix = discoverBundledPluginManifests(repoRoot);
|
|
const selectedPlugins = selectPluginEntries(matrix, {
|
|
ids: options.pluginIds,
|
|
shardTotal: options.shardTotal,
|
|
shardIndex: options.shardIndex,
|
|
limit: options.limit,
|
|
});
|
|
const selectedPluginsWithRequired = collectPluginsWithRequiredEntries(matrix, selectedPlugins);
|
|
const rows: GauntletMeasuredRow[] = [];
|
|
const commandEnv = buildGauntletPrebuildEnv(env, {
|
|
includePrivateQa: !options.skipQa,
|
|
buildIds: selectedPluginsWithRequired.map((plugin) => plugin.buildId),
|
|
skipDeclarationBuild: true,
|
|
});
|
|
const context = {
|
|
repoRoot,
|
|
outputDir: options.outputDir,
|
|
env: commandEnv,
|
|
matrix,
|
|
plugins: selectedPlugins,
|
|
rows,
|
|
commandTimeoutMs: options.commandTimeoutMs,
|
|
skipSlashHelp: options.skipSlashHelp,
|
|
includePluginOwnedCliAliases: options.skipLifecycle,
|
|
qaBaseline: options.qaBaseline,
|
|
qaScenarios: options.qaScenarios,
|
|
qaPluginChunkSize: options.qaPluginChunkSize,
|
|
qaTimeoutMs: options.qaTimeoutMs,
|
|
};
|
|
if (!options.skipPrebuild && (selectedPlugins.length > 0 || !options.skipQa)) {
|
|
process.stderr.write("[plugin-gauntlet] prebuild\n");
|
|
const prebuildCommand = createGauntletPrebuildCommand(repoRoot);
|
|
rows.push(
|
|
await runMeasuredCommandLive({
|
|
cwd: repoRoot,
|
|
env: commandEnv,
|
|
logDir: path.join(options.outputDir, "logs", "prebuild"),
|
|
command: prebuildCommand.command,
|
|
args: prebuildCommand.args,
|
|
label: "prebuild",
|
|
phase: "prebuild",
|
|
timeoutMs: options.buildTimeoutMs,
|
|
}),
|
|
);
|
|
}
|
|
const prebuildFailed = rows.some(
|
|
(row) => row.phase === "prebuild" && (row.status !== 0 || row.timedOut),
|
|
);
|
|
const entryPath = builtEntryPath(repoRoot);
|
|
const missingSkippedPrebuildEntry =
|
|
selectedPlugins.length > 0 &&
|
|
options.skipPrebuild &&
|
|
requiresBuiltEntry(options, selectedPlugins) &&
|
|
!fs.existsSync(entryPath);
|
|
if (!prebuildFailed && !missingSkippedPrebuildEntry && !options.skipLifecycle) {
|
|
await runPluginLifecycle(context);
|
|
}
|
|
if (!prebuildFailed && !missingSkippedPrebuildEntry && !options.skipSlashHelp) {
|
|
await runSlashHelpProbes(context);
|
|
}
|
|
const qaSummaries = options.skipQa || prebuildFailed ? [] : await runQaChunks(context);
|
|
const metricObservations = collectMetricObservations(rows, {
|
|
cpuCoreWarn: options.cpuCoreWarn,
|
|
hotWallWarnMs: options.hotWallWarnMs,
|
|
maxRssWarnMb: options.maxRssWarnMb,
|
|
wallAnomalyMultiplier: options.wallAnomalyMultiplier,
|
|
rssAnomalyMultiplier: options.rssAnomalyMultiplier,
|
|
});
|
|
const qaBaselineObservations = collectQaBaselineRegressionObservations(rows, {
|
|
cpuRegressionMultiplier: options.qaCpuRegressionMultiplier,
|
|
wallRegressionMultiplier: options.qaWallRegressionMultiplier,
|
|
});
|
|
const gatewayObservations = qaSummaries.flatMap((qa) =>
|
|
collectGatewayCpuObservations({
|
|
startup: null,
|
|
qa: isRecord(qa.metrics) ? { metrics: qa.metrics } : undefined,
|
|
cpuCoreWarn: options.cpuCoreWarn,
|
|
hotWallWarnMs: options.hotWallWarnMs,
|
|
}),
|
|
);
|
|
const failures = rows.filter(
|
|
(row) => row.status !== 0 || row.timedOut || row.diagnosticFailure,
|
|
);
|
|
const observations = [...metricObservations, ...qaBaselineObservations, ...gatewayObservations];
|
|
const guardFailures: Array<{ kind: string; message: string }> = [];
|
|
if (missingSkippedPrebuildEntry) {
|
|
guardFailures.push({
|
|
kind: "missing-built-entry",
|
|
message:
|
|
`${path.relative(repoRoot, entryPath)} is missing; ` +
|
|
"run without --skip-prebuild or build the gauntlet runtime first.",
|
|
});
|
|
}
|
|
if (!hasGauntletWorkRows(rows) && !options.allowEmpty && guardFailures.length === 0) {
|
|
guardFailures.push({
|
|
kind: "empty-run",
|
|
message:
|
|
"No lifecycle, slash-help, or QA gauntlet commands ran; remove a skip flag or pass --allow-empty for intentional dry runs.",
|
|
});
|
|
}
|
|
guardFailures.push(...buildObservationGuardFailures(observations, options.failOnObservation));
|
|
const hasFailures = failures.length > 0 || guardFailures.length > 0;
|
|
preserveRunRoot = preserveRunRoot || hasFailures;
|
|
let cleanupError = null;
|
|
if (!preserveRunRoot) {
|
|
try {
|
|
fs.rmSync(runRoot, { recursive: true, force: true });
|
|
} catch (error) {
|
|
cleanupError = error instanceof Error ? error.message : String(error);
|
|
preserveRunRoot = true;
|
|
}
|
|
}
|
|
const summary = {
|
|
generatedAt: new Date().toISOString(),
|
|
repoRoot,
|
|
outputDir: options.outputDir,
|
|
isolatedRunRoot: runRoot,
|
|
isolatedRunRootPreserved: preserveRunRoot,
|
|
isolatedRunRootCleanupError: cleanupError,
|
|
selectedPluginCount: selectedPlugins.length,
|
|
totalPluginCount: matrix.length,
|
|
options: {
|
|
pluginIds: options.pluginIds,
|
|
shardTotal: options.shardTotal,
|
|
shardIndex: options.shardIndex,
|
|
limit: options.limit ?? null,
|
|
qaScenarios: options.qaScenarios,
|
|
qaPluginChunkSize: options.qaPluginChunkSize,
|
|
qaBaseline: options.qaBaseline,
|
|
allowEmpty: options.allowEmpty,
|
|
failOnObservation: options.failOnObservation,
|
|
keepRunRoot: options.keepRunRoot,
|
|
skipLifecycle: options.skipLifecycle,
|
|
skipQa: options.skipQa,
|
|
skipSlashHelp: options.skipSlashHelp,
|
|
skipPrebuild: options.skipPrebuild,
|
|
thresholds: {
|
|
cpuCoreWarn: options.cpuCoreWarn,
|
|
hotWallWarnMs: options.hotWallWarnMs,
|
|
maxRssWarnMb: options.maxRssWarnMb,
|
|
wallAnomalyMultiplier: options.wallAnomalyMultiplier,
|
|
rssAnomalyMultiplier: options.rssAnomalyMultiplier,
|
|
qaCpuRegressionMultiplier: options.qaCpuRegressionMultiplier,
|
|
qaWallRegressionMultiplier: options.qaWallRegressionMultiplier,
|
|
},
|
|
},
|
|
matrix,
|
|
selectedPlugins,
|
|
rows,
|
|
observations,
|
|
failures,
|
|
guardFailures,
|
|
};
|
|
const summaryPath = path.join(options.outputDir, "plugin-gateway-gauntlet-summary.json");
|
|
fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
|
process.stdout.write(`[plugin-gauntlet] summary: ${summaryPath}\n`);
|
|
process.stdout.write(
|
|
`[plugin-gauntlet] plugins=${selectedPlugins.length}/${matrix.length} rows=${rows.length} failures=${failures.length} observations=${summary.observations.length}\n`,
|
|
);
|
|
if (preserveRunRoot) {
|
|
process.stdout.write(`[plugin-gauntlet] isolated run root preserved: ${runRoot}\n`);
|
|
}
|
|
for (const failure of failures) {
|
|
process.stdout.write(
|
|
`[plugin-gauntlet] failure phase=${failure.phase} plugin=${failure.pluginId ?? "<none>"} status=${failure.status} timedOut=${failure.timedOut} diagnostic=${failure.diagnosticFailure ?? ""} wallMs=${Math.round(failure.wallMs)} log=${failure.logPath}\n`,
|
|
);
|
|
}
|
|
for (const failure of guardFailures) {
|
|
process.stdout.write(`[plugin-gauntlet] failure ${failure.kind}: ${failure.message}\n`);
|
|
}
|
|
for (const observation of summary.observations.slice(0, 20)) {
|
|
process.stdout.write(`[plugin-gauntlet] observation ${JSON.stringify(observation)}\n`);
|
|
}
|
|
if (hasFailures) {
|
|
process.exitCode = 1;
|
|
}
|
|
} catch (error) {
|
|
if (!options.keepRunRoot) {
|
|
try {
|
|
fs.rmSync(runRoot, { recursive: true, force: true });
|
|
} catch (cleanupError) {
|
|
process.stderr.write(
|
|
`[plugin-gauntlet] failed to clean isolated run root ${runRoot}: ${
|
|
cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
|
|
}\n`,
|
|
);
|
|
}
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
if (process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1])) {
|
|
main().catch((error: unknown) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exitCode = 1;
|
|
});
|
|
}
|