From 2accfeedc72a447770957eb377c291b349825f18 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 7 Jun 2026 01:11:13 +0200 Subject: [PATCH] fix(test): require resource evidence in perf scripts --- scripts/bench-test-changed.mjs | 29 ++++++++++++++++-- scripts/test-hotspots.mjs | 4 +++ test/scripts/bench-test-changed.test.ts | 37 ++++++++++++++++++++++- test/scripts/test-hotspots.test.ts | 40 +++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 test/scripts/test-hotspots.test.ts diff --git a/scripts/bench-test-changed.mjs b/scripts/bench-test-changed.mjs index 087b73522dd..6c8ec9cb144 100644 --- a/scripts/bench-test-changed.mjs +++ b/scripts/bench-test-changed.mjs @@ -119,6 +119,21 @@ export function formatRss(valueBytes) { return `${(valueBytes / 1024 / 1024).toFixed(1)}MB`; } +export function resolveBenchRssResult({ label, output, rss, status }) { + if (!rss) { + return { maxRssBytes: null, output, status }; + } + const maxRssBytes = parseMaxRssBytes(output); + if (status === 0 && maxRssBytes === null) { + return { + maxRssBytes, + output: `${output}${output.endsWith("\n") ? "" : "\n"}[bench-test-changed] ${label} missing maximum resident set size from /usr/bin/time -l output\n`, + status: 1, + }; + } + return { maxRssBytes, output, status }; +} + function runBenchCommand(params) { const env = { ...process.env }; if (typeof params.maxWorkers === "number") { @@ -138,11 +153,17 @@ function runBenchCommand(params) { ); const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000; const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + const normalized = resolveBenchRssResult({ + label: params.label, + output, + rss: params.rss, + status: result.status ?? 1, + }); return { elapsedMs, - maxRssBytes: params.rss ? parseMaxRssBytes(output) : null, - status: result.status ?? 1, - output, + maxRssBytes: normalized.maxRssBytes, + status: normalized.status, + output: normalized.output, }; } @@ -199,6 +220,7 @@ function main() { const routed = runBenchCommand({ command: routedCommand, cwd: opts.cwd, + label: "routed", rss: opts.rss, ...(typeof opts.maxWorkers === "number" ? { maxWorkers: opts.maxWorkers } : {}), }); @@ -211,6 +233,7 @@ function main() { const root = runBenchCommand({ command: rootCommand, cwd: opts.cwd, + label: "root", rss: opts.rss, ...(typeof opts.maxWorkers === "number" ? { maxWorkers: opts.maxWorkers } : {}), }); diff --git a/scripts/test-hotspots.mjs b/scripts/test-hotspots.mjs index 272b7758895..1442807fb76 100644 --- a/scripts/test-hotspots.mjs +++ b/scripts/test-hotspots.mjs @@ -36,6 +36,10 @@ const report = loadVitestReportFromArgs(opts, "openclaw-vitest-hotspots"); const fileResults = collectVitestFileDurations(report).toSorted( (a, b) => b.durationMs - a.durationMs, ); +if (fileResults.length === 0) { + console.error("[test-hotspots] Vitest JSON report contained no timed file results."); + process.exit(1); +} const top = fileResults.slice(0, opts.limit); const totalDurationMs = fileResults.reduce((sum, item) => sum + item.durationMs, 0); diff --git a/test/scripts/bench-test-changed.test.ts b/test/scripts/bench-test-changed.test.ts index cc349f35a1f..35d8abfbe5b 100644 --- a/test/scripts/bench-test-changed.test.ts +++ b/test/scripts/bench-test-changed.test.ts @@ -1,7 +1,11 @@ // Bench Test Changed tests cover bench test changed script behavior. import { spawnSync } from "node:child_process"; import { describe, expect, it } from "vitest"; -import { formatRss, parseMaxRssBytes } from "../../scripts/bench-test-changed.mjs"; +import { + formatRss, + parseMaxRssBytes, + resolveBenchRssResult, +} from "../../scripts/bench-test-changed.mjs"; function runBenchTestChanged(args: string[]) { return spawnSync(process.execPath, ["scripts/bench-test-changed.mjs", ...args], { @@ -17,6 +21,37 @@ describe("bench-test-changed script", () => { expect(formatRss(-1_048_576)).toBe("-1.0MB"); }); + it("fails RSS-enabled runs when macOS time omits max RSS", () => { + expect( + resolveBenchRssResult({ + label: "routed", + output: "child completed\n", + rss: true, + status: 0, + }), + ).toEqual({ + maxRssBytes: null, + output: + "child completed\n[bench-test-changed] routed missing maximum resident set size from /usr/bin/time -l output\n", + status: 1, + }); + }); + + it("does not require RSS evidence when RSS collection is disabled", () => { + expect( + resolveBenchRssResult({ + label: "root", + output: "child completed\n", + rss: false, + status: 0, + }), + ).toEqual({ + maxRssBytes: null, + output: "child completed\n", + status: 0, + }); + }); + it("rejects malformed max worker values before inspecting git state", () => { const malformed = runBenchTestChanged(["--max-workers", "2abc"]); diff --git a/test/scripts/test-hotspots.test.ts b/test/scripts/test-hotspots.test.ts new file mode 100644 index 00000000000..6189d569b8d --- /dev/null +++ b/test/scripts/test-hotspots.test.ts @@ -0,0 +1,40 @@ +// Test Hotspots tests cover hotspot report evidence validation. +import { spawnSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +const tempRoots: string[] = []; + +function mkTempRoot() { + const root = mkdtempSync(join(tmpdir(), "openclaw-test-hotspots-")); + tempRoots.push(root); + return root; +} + +function runTestHotspots(args: string[]) { + return spawnSync(process.execPath, ["scripts/test-hotspots.mjs", ...args], { + cwd: process.cwd(), + encoding: "utf8", + }); +} + +afterEach(() => { + for (const root of tempRoots.splice(0)) { + rmSync(root, { force: true, recursive: true }); + } +}); + +describe("test-hotspots script", () => { + it("rejects Vitest reports without timed file results", () => { + const reportPath = join(mkTempRoot(), "empty-report.json"); + writeFileSync(reportPath, `${JSON.stringify({ testResults: [] })}\n`, "utf8"); + + const result = runTestHotspots(["--report", reportPath]); + + expect(result.status).toBe(1); + expect(result.stdout).toBe(""); + expect(result.stderr).toContain("Vitest JSON report contained no timed file results."); + }); +});