openclaw/scripts/test-perf-budget.mts

// Runs a Vitest config and enforces wall-time regression budgets.
import { pathToFileURL } from "node:url";
import { isRecord } from "@openclaw/normalization-core/record-coerce";
import {
  booleanFlag,
  isStrictAffirmativeValue,
  parseFlagArgs,
  stringFlag,
} from "./lib/arg-utils.mts";
import { budgetFloatFlag, readBudgetEnvNumber } from "./lib/budget-number-args.mts";
import { reportLimitViolations } from "./lib/check-limits.mts";
import { coerceErrorMessage } from "./lib/error-format.mts";
import { formatMs } from "./lib/vitest-report-cli-utils.mts";
import { readJsonFile, runVitestJsonReport } from "./test-report-utils.mts";

function parseArgs(argv: readonly string[], env = process.env) {
  const opts = parseFlagArgs(
    argv,
    {
      config: "test/vitest/vitest.unit.config.ts",
      maxWallMs: readBudgetEnvNumber("OPENCLAW_TEST_PERF_MAX_WALL_MS", env),
      baselineWallMs: readBudgetEnvNumber("OPENCLAW_TEST_PERF_BASELINE_WALL_MS", env),
      maxRegressionPct: readBudgetEnvNumber("OPENCLAW_TEST_PERF_MAX_REGRESSION_PCT", env) ?? 10,
      reportOnly: isStrictAffirmativeValue(env.OPENCLAW_TEST_PERF_REPORT_ONLY),
    },
    [
      stringFlag("--config", "config"),
      budgetFloatFlag("--max-wall-ms", "maxWallMs"),
      budgetFloatFlag("--baseline-wall-ms", "baselineWallMs"),
      budgetFloatFlag("--max-regression-pct", "maxRegressionPct"),
      booleanFlag("--report-only", "reportOnly", true),
    ],
  );
  if (opts.maxWallMs === null && opts.baselineWallMs === null && !opts.reportOnly) {
    throw new Error(
      "[test-perf-budget] provide --max-wall-ms, --baseline-wall-ms, or set --report-only for an explicit timing-only run",
    );
  }
  return opts;
}

function collectPerfReportStats(reportPath: string) {
  let report: unknown;
  try {
    report = readJsonFile(reportPath);
  } catch (error) {
    throw new Error(
      `[test-perf-budget] failed to read Vitest JSON report ${reportPath}: ${coerceErrorMessage(
        error,
      )}`,
      { cause: error },
    );
  }

  let totalFileDurationMs = 0;
  let fileCount = 0;
  const testResults =
    isRecord(report) && Array.isArray(report.testResults) ? report.testResults : [];
  for (const result of testResults.filter(isRecord)) {
    if (typeof result.startTime === "number" && typeof result.endTime === "number") {
      totalFileDurationMs += Math.max(0, result.endTime - result.startTime);
      fileCount += 1;
    }
  }
  if (fileCount === 0) {
    throw new Error(`[test-perf-budget] Vitest JSON report contained no timed file results`);
  }
  return { fileCount, totalFileDurationMs };
}

function main() {
  let opts;
  try {
    opts = parseArgs(process.argv.slice(2));
  } catch (error) {
    console.error(coerceErrorMessage(error));
    process.exit(1);
  }

  const startedAt = process.hrtime.bigint();
  const reportPath = runVitestJsonReport({
    config: opts.config,
    prefix: "openclaw-vitest-perf",
  });
  const elapsedMs = Number.parseFloat(String(process.hrtime.bigint() - startedAt)) / 1_000_000;

  let reportStats;
  try {
    reportStats = collectPerfReportStats(reportPath);
  } catch (error) {
    console.error(coerceErrorMessage(error));
    process.exit(1);
  }

  const allowedByBaseline =
    opts.baselineWallMs !== null
      ? opts.baselineWallMs * (1 + (opts.maxRegressionPct ?? 0) / 100)
      : null;

  const violations: string[] = [];
  if (opts.maxWallMs !== null && elapsedMs > opts.maxWallMs) {
    violations.push(
      `[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded max ${formatMs(
        opts.maxWallMs,
      )}.`,
    );
  }
  if (allowedByBaseline !== null && elapsedMs > allowedByBaseline) {
    violations.push(
      `[test-perf-budget] wall time ${formatMs(elapsedMs)} exceeded baseline budget ${formatMs(
        allowedByBaseline,
      )} (baseline ${formatMs(opts.baselineWallMs ?? 0)}, +${String(opts.maxRegressionPct)}%).`,
    );
  }

  console.log(
    `[test-perf-budget] config=${opts.config} wall=${formatMs(elapsedMs)} file-sum=${formatMs(
      reportStats.totalFileDurationMs,
    )} files=${String(reportStats.fileCount)}`,
  );

  if (
    reportLimitViolations(
      violations.map((message) => ({ file: opts.config, title: "Test wall-time budget", message })),
    )
  ) {
    process.exit(1);
  }
}

/** Test-facing parser helpers for budget validation. */
export const testing = {
  collectPerfReportStats,
  parseArgs,
};

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  main();
}