mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
392 lines
14 KiB
TypeScript
392 lines
14 KiB
TypeScript
// Compares CLI startup benchmark reports against checked-in budgets.
|
|
import { spawnSync } from "node:child_process";
|
|
import fs from "node:fs";
|
|
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
|
import { z } from "zod";
|
|
import { booleanFlag, intFlag, parseFlagArgs, stringFlag } from "./lib/arg-utils.mts";
|
|
import { budgetFloatFlag, readBudgetEnvNumber } from "./lib/budget-number-args.mts";
|
|
import { reportLimitViolations } from "./lib/check-limits.mts";
|
|
import {
|
|
assertCompatibleCliStartupExecutionModes,
|
|
assertCompatibleCliStartupMemoryMetrics,
|
|
cliStartupExecutionMode,
|
|
cliStartupMemoryMetric,
|
|
} from "./lib/cli-startup-memory-contract.mts";
|
|
import { readJsonFile } from "./test-report-utils.mts";
|
|
|
|
const CLI_STARTUP_BENCH_FIXTURE_PATH = "test/fixtures/cli-startup-bench.json";
|
|
|
|
const finiteNumberSchema = z.number().finite();
|
|
const metricSchema = z.object({ avg: finiteNumberSchema, max: finiteNumberSchema }).partial();
|
|
const startupCaseSchema = z.object({
|
|
contract: z
|
|
.object({
|
|
exitBudgetMs: finiteNumberSchema.nullable(),
|
|
firstOutputBudgetMs: finiteNumberSchema.nullable(),
|
|
})
|
|
.partial()
|
|
.nullable()
|
|
.optional(),
|
|
id: z.string(),
|
|
name: z.string(),
|
|
samples: z
|
|
.array(
|
|
z
|
|
.object({
|
|
exitCode: finiteNumberSchema.nullable(),
|
|
maxRssMb: finiteNumberSchema.nullable(),
|
|
signal: z.string().nullable(),
|
|
timedOut: z.boolean(),
|
|
})
|
|
.partial(),
|
|
)
|
|
.optional(),
|
|
summary: z
|
|
.object({
|
|
durationMs: metricSchema,
|
|
firstOutputMs: metricSchema.nullable(),
|
|
maxRssMb: metricSchema.nullable(),
|
|
})
|
|
.partial()
|
|
.optional(),
|
|
});
|
|
type StartupCase = z.infer<typeof startupCaseSchema>;
|
|
|
|
function formatMs(value: number) {
|
|
return `${value.toFixed(1)}ms`;
|
|
}
|
|
|
|
function formatMb(value: number) {
|
|
return `${value.toFixed(1)}MiB`;
|
|
}
|
|
|
|
if (process.argv.slice(2).includes("--help")) {
|
|
console.log(
|
|
[
|
|
"Usage: node --import tsx scripts/test-cli-startup-bench-budget.mts [options]",
|
|
"",
|
|
"Compare current CLI benchmark results against the checked-in fixture.",
|
|
"",
|
|
"Options:",
|
|
" --baseline <path> Baseline fixture path",
|
|
" --report <path> Reuse an existing current benchmark report",
|
|
" --entry <path> CLI entry to benchmark when report is omitted",
|
|
" --preset <name> startup | real | all (default: all)",
|
|
" --runs <n> Measured runs per case (default: 1)",
|
|
" --warmup <n> Warmup runs per case (default: 0)",
|
|
" --timeout-ms <ms> Per-run timeout (default: 30000)",
|
|
" --max-duration-regression-pct <n>",
|
|
" Fail if avg duration regresses more than this percent",
|
|
" --max-first-output-regression-pct <n>",
|
|
" Fail if avg first-output time regresses more than this percent",
|
|
" --max-rss-regression-pct <n> Fail if avg RSS regresses more than this percent",
|
|
" --skip-baseline Skip fixture regression checks and enforce case contracts only",
|
|
" --skip-response-budgets Skip response first-output and exit budget contracts",
|
|
"",
|
|
"Non-x64 runs skip fixture regression checks by default because the",
|
|
"checked-in startup fixture is a canonical x64 budget. Response contracts still run. Set",
|
|
"OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH=1 to force them.",
|
|
" --help Show this help text",
|
|
"",
|
|
"Example:",
|
|
" node --import tsx scripts/test-cli-startup-bench-budget.mts --preset real --max-duration-regression-pct 15",
|
|
].join("\n"),
|
|
);
|
|
process.exit(0);
|
|
}
|
|
|
|
function parseOptions() {
|
|
try {
|
|
return parseFlagArgs(
|
|
process.argv.slice(2),
|
|
{
|
|
baseline: CLI_STARTUP_BENCH_FIXTURE_PATH,
|
|
report: "",
|
|
entry: "openclaw.mjs",
|
|
preset: "all",
|
|
runs: 1,
|
|
warmup: 0,
|
|
timeoutMs: 30_000,
|
|
maxDurationRegressionPct:
|
|
readBudgetEnvNumber("OPENCLAW_STARTUP_BENCH_MAX_DURATION_REGRESSION_PCT") ?? 20,
|
|
maxFirstOutputRegressionPct:
|
|
readBudgetEnvNumber("OPENCLAW_STARTUP_BENCH_MAX_FIRST_OUTPUT_REGRESSION_PCT") ?? 20,
|
|
maxRssRegressionPct:
|
|
readBudgetEnvNumber("OPENCLAW_STARTUP_BENCH_MAX_RSS_REGRESSION_PCT") ?? 20,
|
|
skipBaseline: false,
|
|
skipResponseBudgets: false,
|
|
},
|
|
[
|
|
stringFlag("--baseline", "baseline"),
|
|
stringFlag("--report", "report"),
|
|
stringFlag("--entry", "entry"),
|
|
stringFlag("--preset", "preset"),
|
|
intFlag("--runs", "runs", { min: 1 }),
|
|
intFlag("--warmup", "warmup", { min: 0 }),
|
|
intFlag("--timeout-ms", "timeoutMs", { min: 1 }),
|
|
budgetFloatFlag("--max-duration-regression-pct", "maxDurationRegressionPct"),
|
|
budgetFloatFlag("--max-first-output-regression-pct", "maxFirstOutputRegressionPct"),
|
|
budgetFloatFlag("--max-rss-regression-pct", "maxRssRegressionPct"),
|
|
booleanFlag("--skip-baseline", "skipBaseline"),
|
|
booleanFlag("--skip-response-budgets", "skipResponseBudgets"),
|
|
],
|
|
);
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
return process.exit(1);
|
|
}
|
|
}
|
|
let opts = parseOptions();
|
|
|
|
const shouldAutoSkipNonCanonicalBaselineChecks =
|
|
process.arch !== "x64" && process.env.OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH !== "1";
|
|
if (shouldAutoSkipNonCanonicalBaselineChecks && !opts.skipBaseline) {
|
|
console.warn(
|
|
`[test-cli-startup-bench-budget] skipping x64 startup fixture budgets on ${process.arch}; response contracts and sample output validation still ran. Set OPENCLAW_STARTUP_BENCH_ENFORCE_NONCANONICAL_ARCH=1 to force fixture checks.`,
|
|
);
|
|
opts = {
|
|
...opts,
|
|
skipBaseline: true,
|
|
};
|
|
}
|
|
|
|
function resolveCurrentReportPath() {
|
|
if (opts.report) {
|
|
return opts.report;
|
|
}
|
|
const build = spawnSync(
|
|
process.execPath,
|
|
["--import", "tsx", "scripts/ensure-cli-startup-build.mts"],
|
|
{
|
|
cwd: process.cwd(),
|
|
stdio: "inherit",
|
|
env: process.env,
|
|
},
|
|
);
|
|
if (build.status !== 0) {
|
|
process.exit(build.status ?? 1);
|
|
}
|
|
const reportPath = `.artifacts/cli-startup-bench.current.json`;
|
|
fs.mkdirSync(".artifacts", { recursive: true });
|
|
const args = [
|
|
"--import",
|
|
"tsx",
|
|
"scripts/bench-cli-startup.ts",
|
|
"--entry",
|
|
opts.entry,
|
|
"--preset",
|
|
opts.preset,
|
|
"--runs",
|
|
String(opts.runs),
|
|
"--warmup",
|
|
String(opts.warmup),
|
|
"--timeout-ms",
|
|
String(opts.timeoutMs),
|
|
"--output",
|
|
reportPath,
|
|
];
|
|
const run = spawnSync(process.execPath, args, {
|
|
cwd: process.cwd(),
|
|
stdio: "inherit",
|
|
env: process.env,
|
|
});
|
|
if (run.status !== 0) {
|
|
process.exit(run.status ?? 1);
|
|
}
|
|
return reportPath;
|
|
}
|
|
|
|
function indexCases(report: unknown) {
|
|
const cases = isRecord(report) && isRecord(report.primary) ? report.primary.cases : undefined;
|
|
if (!Array.isArray(cases)) {
|
|
return new Map<string, StartupCase>();
|
|
}
|
|
return new Map<string, StartupCase>(
|
|
cases.flatMap((entry): Array<[string, StartupCase]> => {
|
|
const parsed = startupCaseSchema.safeParse(entry);
|
|
return parsed.success ? [[parsed.data.id, parsed.data]] : [];
|
|
}),
|
|
);
|
|
}
|
|
|
|
const baseline = readJsonFile(opts.baseline);
|
|
const current = readJsonFile(resolveCurrentReportPath());
|
|
const baselineCases = indexCases(baseline);
|
|
const currentCases = indexCases(current);
|
|
const shouldRequireEveryBaselineCase = opts.preset === "all";
|
|
const matchedBaselineCaseIds = [...baselineCases.keys()].filter((id) => currentCases.has(id));
|
|
|
|
let failed = false;
|
|
const limitViolations: string[] = [];
|
|
|
|
try {
|
|
cliStartupExecutionMode(isRecord(current) ? current.primary : undefined);
|
|
cliStartupMemoryMetric(isRecord(current) ? current.primary : undefined);
|
|
if (!opts.skipBaseline) {
|
|
assertCompatibleCliStartupExecutionModes(
|
|
isRecord(baseline) ? baseline.primary : undefined,
|
|
isRecord(current) ? current.primary : undefined,
|
|
);
|
|
assertCompatibleCliStartupMemoryMetrics(
|
|
isRecord(baseline) ? baseline.primary : undefined,
|
|
isRecord(current) ? current.primary : undefined,
|
|
);
|
|
}
|
|
} catch (error) {
|
|
console.error(
|
|
`[test-cli-startup-bench-budget] ${error instanceof Error ? error.message : String(error)}`,
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (currentCases.size === 0) {
|
|
console.error(
|
|
`[test-cli-startup-bench-budget] current report has no cases for preset ${opts.preset}`,
|
|
);
|
|
failed = true;
|
|
}
|
|
|
|
for (const [id] of baselineCases) {
|
|
if (shouldRequireEveryBaselineCase && !currentCases.has(id)) {
|
|
console.error(`[test-cli-startup-bench-budget] missing current case ${id}`);
|
|
failed = true;
|
|
}
|
|
}
|
|
if (
|
|
!opts.skipBaseline &&
|
|
!shouldRequireEveryBaselineCase &&
|
|
baselineCases.size > 0 &&
|
|
matchedBaselineCaseIds.length === 0
|
|
) {
|
|
console.error(
|
|
`[test-cli-startup-bench-budget] no current cases matched the baseline for preset ${opts.preset}`,
|
|
);
|
|
failed = true;
|
|
}
|
|
|
|
for (const currentCase of currentCases.values()) {
|
|
const samples = currentCase.samples ?? [];
|
|
if (samples.length === 0) {
|
|
console.error(`[test-cli-startup-bench-budget] ${currentCase.name} has no measured samples.`);
|
|
failed = true;
|
|
}
|
|
if (samples.some((sample) => sample.timedOut === true)) {
|
|
console.error(`[test-cli-startup-bench-budget] ${currentCase.name} timed out.`);
|
|
failed = true;
|
|
}
|
|
if (samples.some((sample) => sample.maxRssMb == null)) {
|
|
console.error(`[test-cli-startup-bench-budget] ${currentCase.name} did not report max RSS.`);
|
|
failed = true;
|
|
}
|
|
}
|
|
|
|
if (!opts.skipBaseline) {
|
|
for (const [id, baselineCase] of baselineCases) {
|
|
const currentCase = currentCases.get(id);
|
|
if (!currentCase) {
|
|
continue;
|
|
}
|
|
|
|
const baselineDuration = baselineCase.summary?.durationMs?.avg;
|
|
const currentDuration = currentCase.summary?.durationMs?.avg;
|
|
const baselineFirstOutput = baselineCase.summary?.firstOutputMs?.avg;
|
|
const currentFirstOutput = currentCase.summary?.firstOutputMs?.avg;
|
|
const baselineRss = baselineCase.summary?.maxRssMb?.avg;
|
|
const currentRss = currentCase.summary?.maxRssMb?.avg;
|
|
for (const [metric, label, regressionPct, format] of [
|
|
["durationMs", "duration", opts.maxDurationRegressionPct, formatMs],
|
|
["firstOutputMs", "first output", opts.maxFirstOutputRegressionPct, formatMs],
|
|
["maxRssMb", "RSS", opts.maxRssRegressionPct, formatMb],
|
|
] as const) {
|
|
const baselineValue = baselineCase.summary?.[metric]?.avg;
|
|
const currentValue = currentCase.summary?.[metric]?.avg;
|
|
if (baselineValue === undefined || currentValue === undefined || baselineValue <= 0) {
|
|
continue;
|
|
}
|
|
const allowed = baselineValue * (1 + regressionPct / 100);
|
|
if (currentValue > allowed) {
|
|
limitViolations.push(
|
|
`[test-cli-startup-bench-budget] ${baselineCase.name} avg ${label} ${format(
|
|
currentValue,
|
|
)} exceeded ${format(allowed)} (baseline ${format(baselineValue)}, +${String(regressionPct)}%).`,
|
|
);
|
|
}
|
|
}
|
|
|
|
if (currentDuration === undefined || baselineDuration === undefined) {
|
|
throw new Error(`startup benchmark case ${baselineCase.name} is missing duration metrics`);
|
|
}
|
|
console.log(
|
|
`[test-cli-startup-bench-budget] ${baselineCase.name} duration=${formatMs(
|
|
currentDuration,
|
|
)} baseline=${formatMs(baselineDuration)} firstOutput=${
|
|
currentFirstOutput !== undefined ? formatMs(currentFirstOutput) : "n/a"
|
|
} baselineFirstOutput=${
|
|
baselineFirstOutput !== undefined ? formatMs(baselineFirstOutput) : "n/a"
|
|
} rss=${
|
|
currentRss !== undefined ? formatMb(currentRss) : "n/a"
|
|
} baselineRss=${baselineRss !== undefined ? formatMb(baselineRss) : "n/a"}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
for (const currentCase of currentCases.values()) {
|
|
const contract = currentCase.contract;
|
|
if (!contract) {
|
|
continue;
|
|
}
|
|
|
|
const badSample = (currentCase.samples ?? []).find(
|
|
(sample) => sample.timedOut === true || sample.exitCode !== 0 || sample.signal != null,
|
|
);
|
|
if (badSample) {
|
|
console.error(
|
|
`[test-cli-startup-bench-budget] ${currentCase.name} exited ${String(
|
|
badSample.timedOut === true ? "timeout" : (badSample.signal ?? badSample.exitCode),
|
|
)}; response contract requires a clean exit.`,
|
|
);
|
|
failed = true;
|
|
}
|
|
|
|
if (!opts.skipResponseBudgets) {
|
|
const firstOutputBudgetMs = contract.firstOutputBudgetMs;
|
|
const firstOutputMax = currentCase.summary?.firstOutputMs?.max;
|
|
if (firstOutputBudgetMs != null) {
|
|
if (firstOutputMax === undefined) {
|
|
console.error(
|
|
`[test-cli-startup-bench-budget] ${currentCase.name} produced no stdout/stderr before exit; response contract requires first output within ${formatMs(
|
|
firstOutputBudgetMs,
|
|
)}.`,
|
|
);
|
|
failed = true;
|
|
} else if (firstOutputMax > firstOutputBudgetMs) {
|
|
limitViolations.push(
|
|
`[test-cli-startup-bench-budget] ${currentCase.name} first output ${formatMs(
|
|
firstOutputMax,
|
|
)} exceeded contract ${formatMs(firstOutputBudgetMs)}.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
const exitBudgetMs = contract.exitBudgetMs;
|
|
const durationMax = currentCase.summary?.durationMs?.max;
|
|
if (exitBudgetMs != null && durationMax !== undefined && durationMax > exitBudgetMs) {
|
|
limitViolations.push(
|
|
`[test-cli-startup-bench-budget] ${currentCase.name} exit ${formatMs(
|
|
durationMax,
|
|
)} exceeded contract ${formatMs(exitBudgetMs)}.`,
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
const limitsFailed = reportLimitViolations(
|
|
limitViolations.map((message) => ({
|
|
file: "scripts/test-cli-startup-bench-budget.mts",
|
|
title: "CLI startup budget",
|
|
message,
|
|
})),
|
|
);
|
|
if (failed || limitsFailed) {
|
|
process.exit(1);
|
|
}
|