From 7e2d287db5fb4641608f8c032e118814490a601f Mon Sep 17 00:00:00 2001 From: Vincent Koc <25068+vincentkoc@users.noreply.github.com> Date: Thu, 9 Jul 2026 07:22:17 -0700 Subject: [PATCH] fix(ci): ignore profiling-only resource gate failures --- .github/workflows/openclaw-performance.yml | 2 +- scripts/lib/kova-report-gate.mjs | 616 +++++++++-- test/scripts/kova-report-gate.test.ts | 954 +++++++++++++++--- .../openclaw-performance-workflow.test.ts | 3 + 4 files changed, 1335 insertions(+), 240 deletions(-) diff --git a/.github/workflows/openclaw-performance.yml b/.github/workflows/openclaw-performance.yml index dde34619051..27459413436 100644 --- a/.github/workflows/openclaw-performance.yml +++ b/.github/workflows/openclaw-performance.yml @@ -372,7 +372,7 @@ jobs: then effective_status=0 { - echo "Kova returned a partial release-gate verdict for filtered performance coverage, but all selected scenarios passed and no baseline regression was reported." + echo "Kova returned a non-zero release-gate verdict, but the trusted report adapter found only filtered coverage or profiling-affected resource thresholds with no baseline regression." echo } >> "$GITHUB_STEP_SUMMARY" fi diff --git a/scripts/lib/kova-report-gate.mjs b/scripts/lib/kova-report-gate.mjs index 543f3c29f3a..60c8f7a1858 100644 --- a/scripts/lib/kova-report-gate.mjs +++ b/scripts/lib/kova-report-gate.mjs @@ -1,102 +1,542 @@ import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; - -function numericCount(value) { - if (typeof value !== "number") { - return undefined; +const SCHEMA = { + report: "kova.report.v1", + gate: "kova.gate.v1", + performance: "kova.performance.v1", + profiling: "kova.profiling.v1", + baseline: "kova.baselineComparison.v1", + gateBaseline: "kova.gateBaselineSummary.v1", +}; +const PROFILED_INTERPRETATION = + "instrumented run; CPU/RSS can include profiler and diagnostic overhead"; +const RSS_METRICS = ["peakRssMb", "resourcePeakGatewayRssMb"]; +const CPU_METRICS = ["cpuPercentMax"]; +const DIRECT_VIOLATIONS = new Set(["cpuPercentMax", "peakRssMb"]); +const SOAK_VIOLATIONS = new Set(["gatewayRssGrowthMb", "rssGrowthMb"]); +const ROLE_VIOLATION = /^resourceByRole\.([^.]+)\.(maxCpuPercent|peakRssMb)$/u; +function check(condition, reason) { + if (!condition) { + throw new Error(reason); } - return Number.isFinite(value) ? value : undefined; } -const rssMetricIds = ["peakRssMb", "resourcePeakGatewayRssMb"]; -const cpuMetricIds = ["cpuPercentMax"]; +function object(value, label) { + check(value !== null && typeof value === "object" && !Array.isArray(value), `invalid ${label}`); + return value; +} -function hasSampledMetric(group, metricIds) { - return metricIds.some((metricId) => { - const metric = group?.metrics?.[metricId]; - return metric && Number(metric.count) > 0; - }); +function array(value, label) { + check(Array.isArray(value), `invalid ${label}`); + return value; +} +function count(value, label, { positive = false } = {}) { + check(Number.isSafeInteger(value) && value >= (positive ? 1 : 0), `invalid ${label}`); + return value; +} + +function text(value, label) { + check(typeof value === "string" && value.trim().length > 0, `invalid ${label}`); + return value; +} + +function finite(value, label) { + check(typeof value === "number" && Number.isFinite(value), `invalid ${label}`); + return value; +} + +function stateId(record) { + return text(object(record.state, "record state").id, "record state id"); +} + +function recordKey(record) { + return [ + text(record.scenario, "record scenario"), + text(record.surface, "record surface"), + stateId(record), + ].join("\u0000"); +} + +function statusCounts(records) { + const statuses = {}; + for (const record of records) { + text(record.status, "record status"); + statuses[record.status] = (statuses[record.status] ?? 0) + 1; + } + return statuses; +} + +function exactCounts(actualValue, expected, label) { + const actual = object(actualValue, label); + check(Object.keys(actual).length === Object.keys(expected).length, `${label} keys did not match`); + for (const [key, expectedCount] of Object.entries(expected)) { + check(count(actual[key], `${label}.${key}`) === expectedCount, `${label}.${key} did not match`); + } +} + +function exactStrings(actualValue, expected, label) { + const actual = array(actualValue, label); + check( + actual.length === expected.length && actual.every((value, index) => value === expected[index]), + `${label} did not match`, + ); +} + +function validateCommandResult(value, label) { + const result = object(value, label); + check(result.status === 0 && result.timedOut === false, `${label} failed`); +} + +function alreadyAbsent(result, noun) { + check( + Number.isSafeInteger(result.status) && result.status !== 0, + `${noun} cleanup status was invalid`, + ); + check(result.timedOut === false, `${noun} cleanup timed out`); + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + const missing = new RegExp(`\\b${noun}\\b[\\s\\S]*\\b(?:does not exist|not found)\\b`, "iu").test( + output, + ); + check(missing, `${noun} cleanup lacked already-absent evidence`); +} + +function validateCleanup(status, resultValue, successStatus, noun, label) { + const result = object(resultValue, `${label} result`); + if (status === successStatus) { + validateCommandResult(result, `${label} result`); + return; + } + check(status === "already-absent", `${label} was not completed`); + alreadyAbsent(result, noun); +} + +function validateRecord(recordValue) { + const record = object(recordValue, "record"); + recordKey(record); + object(record.measurements, "record measurements"); + const profiling = object(record.profiling, "record profiling"); + check(profiling.schemaVersion === SCHEMA.profiling, "wrong profiling schema"); + const phases = array(record.phases, "record phases"); + check(phases.length > 0, "record had no phases"); + for (const phaseValue of phases) { + const results = array(object(phaseValue, "phase").results, "phase results"); + check(results.length > 0, "phase had no command results"); + results.forEach((result, index) => validateCommandResult(result, `phase result ${index}`)); + } + validateCleanup( + record.cleanup, + record.cleanupResult, + "destroyed", + "environment", + "record cleanup", + ); + return record; +} + +function validateTargetCleanup(report) { + if (!text(report.target, "report target").startsWith("local-build:")) { + check(report.targetCleanup === null, "non-local target had cleanup metadata"); + return; + } + const cleanup = object(report.targetCleanup, "target cleanup"); + validateCleanup(cleanup.status, cleanup.result, "removed", "runtime", "target cleanup"); +} + +function validateBaselines(report, gate) { + const reportBaseline = report.baseline; + const gateBaseline = gate.baseline; + check( + (reportBaseline === null || reportBaseline === undefined) === + (gateBaseline === null || gateBaseline === undefined), + "baseline evidence was one-sided", + ); + if (reportBaseline === null || reportBaseline === undefined) { + return; + } + const comparison = object( + object(reportBaseline, "report baseline").comparison, + "baseline comparison", + ); + const summary = object(gateBaseline, "gate baseline"); + check(comparison.schemaVersion === SCHEMA.baseline, "wrong baseline comparison schema"); + check(summary.schemaVersion === SCHEMA.gateBaseline, "wrong gate baseline schema"); + for (const [field, listField] of [ + ["regressionCount", "regressions"], + ["missingBaselineCount", "missing"], + ]) { + check(count(comparison[field], `baseline ${field}`) === 0, `baseline ${field} was nonzero`); + check( + array(comparison[listField], `baseline ${listField}`).length === 0, + `baseline ${listField} present`, + ); + check( + count(summary[field], `gate baseline ${field}`) === 0, + `gate baseline ${field} was nonzero`, + ); + const gateList = field === "regressionCount" ? "regressedGroups" : listField; + check( + array(summary[gateList], `gate baseline ${gateList}`).length === 0, + `gate baseline ${gateList} present`, + ); + } + check(comparison.ok === true && summary.ok === true, "baseline was not ok"); + check( + count(comparison.baselineEntryCount, "baseline entry count") === + count(summary.baselineEntryCount, "gate baseline entry count"), + "baseline entry counts did not match", + ); +} + +function validateSummary(report, records) { + const summary = object(report.summary, "report summary"); + check( + count(summary.total, "summary total") === records.length, + "summary total did not match records", + ); + exactCounts(summary.statuses, statusCounts(records), "summary statuses"); +} + +function validateMetric(metricValue, sampleCount, label) { + const metric = object(metricValue, label); + const samples = array(metric.samples, `${label} samples`); + const metricCount = count(metric.count, `${label} count`, { positive: true }); + check(metricCount === samples.length && metricCount <= sampleCount, `${label} count drift`); + check( + metric.classification === "stable" || metric.classification === "unstable", + `${label} classification was invalid`, + ); + samples.forEach((sample) => finite(sample, `${label} sample`)); + return samples; +} + +function sampledMetric(metrics, ids, sampleCount, label) { + for (const id of ids) { + if (metrics[id] !== undefined) { + const samples = validateMetric(metrics[id], sampleCount, `${label}.${id}`); + check(samples.length === sampleCount, `${label}.${id} did not cover every record`); + return { id, samples }; + } + } + throw new Error(`${label} was not sampled`); +} + +function measured(record, ids, label) { + for (const id of ids) { + if (record.measurements[id] !== undefined) { + return finite(record.measurements[id], `${label}.${id}`); + } + } + throw new Error(`${label} was not measured`); +} + +function validatePerformance(report, records, repeat) { + const performance = object(report.performance, "performance"); + check(performance.schemaVersion === SCHEMA.performance, "wrong performance schema"); + check(performance.repeat === repeat, "performance repeat did not match controls"); + const groups = array(performance.groups, "performance groups"); + check( + count(performance.groupCount, "performance group count") === groups.length, + "group count drift", + ); + + const recordsByKey = new Map(); + for (const record of records) { + const key = recordKey(record); + const matching = recordsByKey.get(key) ?? []; + matching.push(record); + recordsByKey.set(key, matching); + } + check(groups.length === recordsByKey.size, "performance groups did not map to records"); + + const groupsByKey = new Map(); + let profiledCount = 0; + for (const groupValue of groups) { + const group = object(groupValue, "performance group"); + const identity = [ + text(group.scenario, "group scenario"), + text(group.surface, "group surface"), + text(group.state, "group state"), + ]; + const key = identity.join("\u0000"); + check( + group.key === identity.join("|") && !groupsByKey.has(key), + "performance group identity was invalid", + ); + const matching = recordsByKey.get(key); + check(matching?.length > 0, "performance group had no records"); + check( + count(group.sampleCount, "group sample count", { positive: true }) === matching.length, + "sample count drift", + ); + exactCounts(group.statuses, statusCounts(matching), "group statuses"); + const matchingProfiled = matching.filter((record) => record.profiling.enabled === true).length; + check( + count(group.profiledRunCount, "group profiled count") === matchingProfiled, + "profile count drift", + ); + check( + group.resourceInterpretation === (matchingProfiled > 0 ? "instrumented" : "normal"), + "group resource interpretation was invalid", + ); + profiledCount += matchingProfiled; + const metrics = object(group.metrics, "group metrics"); + Object.entries(metrics).forEach(([id, metric]) => + validateMetric(metric, group.sampleCount, `group metric ${id}`), + ); + const rss = sampledMetric(metrics, RSS_METRICS, group.sampleCount, "group RSS"); + const cpu = sampledMetric(metrics, CPU_METRICS, group.sampleCount, "group CPU"); + for (const record of matching) { + check( + rss.samples.includes(measured(record, RSS_METRICS, "record RSS")), + "record RSS was not sampled", + ); + check( + cpu.samples.includes(measured(record, CPU_METRICS, "record CPU")), + "record CPU was not sampled", + ); + } + groupsByKey.set(key, group); + } + check( + count(performance.profiledRunCount, "performance profiled count") === profiledCount, + "profile total drift", + ); + const unstableCount = groups.filter((group) => + Object.values(group.metrics).some((metric) => metric.classification === "unstable"), + ).length; + check( + count(performance.unstableGroupCount, "unstable group count") === unstableCount, + "unstable group count drift", + ); + return groupsByKey; +} + +function validateGateCards(gate) { + const cards = array(gate.cards, "gate cards"); + const severities = { blocking: 0, warning: 0, info: 0 }; + for (const cardValue of cards) { + const card = object(cardValue, "gate card"); + const severity = card.severity; + check(Object.hasOwn(severities, severity), "unknown gate card severity"); + if (severity === "info") { + check( + (card.kind === "filtered-required-scenario" || + card.kind === "filtered-required-coverage") && + card.status === "MISSING", + "unexpected info gate card", + ); + } + if (severity === "warning") { + check( + card.kind === "missing-required-coverage" && card.status === "MISSING", + "unexpected warning gate card", + ); + } + severities[severity] += 1; + } + check( + count(gate.blockingCount, "blocking count") === severities.blocking, + "blocking count drift", + ); + check(count(gate.warningCount, "warning count") === severities.warning, "warning count drift"); + check(count(gate.infoCount, "info count") === severities.info, "info count drift"); + check( + count(gate.missingRequiredCount, "missing required count") === severities.info, + "missing count drift", + ); + return cards; +} + +function validateEnvelope(reportValue) { + const report = object(reportValue, "report"); + const gate = object(report.gate, "gate"); + const controls = object(report.controls, "controls"); + check( + report.schemaVersion === SCHEMA.report && report.mode === "execution", + "wrong report schema or mode", + ); + check( + gate.schemaVersion === SCHEMA.gate && gate.enabled === true, + "wrong or disabled gate schema", + ); + check(controls.gate === true, "gate controls were disabled"); + const filters = [ + ...array(controls.include, "include filters"), + ...array(controls.exclude, "exclude filters"), + ]; + check( + filters.length > 0 && + filters.every((filter) => typeof filter === "string" && filter.trim().length > 0), + "report filters were invalid", + ); + const repeat = count(controls.repeat, "repeat", { positive: true }); + check( + gate.partial === true && gate.complete === false && gate.ok === false, + "gate metadata was not partial", + ); + const records = array(report.records, "records").map(validateRecord); + check(records.length > 0, "report had no records"); + const cards = validateGateCards(gate); + validateBaselines(report, gate); + validateSummary(report, records); + validateTargetCleanup(report); + const groups = validatePerformance(report, records, repeat); + return { report, gate, records, cards, groups }; +} + +function deepProfiled(record) { + const profiling = record.profiling; + const measurements = record.measurements; + return [ + [profiling.enabled, true], + [profiling.deepProfile, true], + [profiling.nodeProfile, true], + [profiling.heapSnapshot, true], + [profiling.diagnosticReport, true], + [profiling.profileOnFailure, false], + [profiling.affectsResourceMeasurements, true], + [profiling.baselineEligible, false], + [profiling.interpretation, PROFILED_INTERPRETATION], + [measurements.profilingEnabled, true], + [measurements.profilingAffectsResourceMeasurements, true], + [measurements.profilingBaselineEligible, false], + [measurements.profilingResourceInterpretation, PROFILED_INTERPRETATION], + ].every(([actual, expected]) => actual === expected); +} + +function violationMeasurement(record, violation) { + const metric = text(violation.metric, "violation metric"); + if (violation.kind === "threshold" && DIRECT_VIOLATIONS.has(metric)) { + return finite(record.measurements[metric], `measurement ${metric}`); + } + if (violation.kind === "soak" && SOAK_VIOLATIONS.has(metric)) { + return finite(record.measurements[metric], `measurement ${metric}`); + } + const match = violation.kind === "resource" ? ROLE_VIOLATION.exec(metric) : null; + check(match !== null && violation.role === match[1], "role violation identity was invalid"); + const role = object( + object(record.measurements.resourceByRole, "role measurements")[match[1]], + "role measurement", + ); + return finite(role[match[2]], `role measurement ${metric}`); +} + +function validateProfiledFailure(record, card, group) { + check(deepProfiled(record), "failed record was not canonical deep profiling"); + check( + group.resourceInterpretation === "instrumented", + "failed record group was not instrumented", + ); + const violations = array(record.violations, "record violations"); + check(violations.length > 0, "failed record had no violations"); + for (const violationValue of violations) { + const violation = object(violationValue, "violation"); + const actual = finite(violation.actual, "violation actual"); + check(actual === violationMeasurement(record, violation), "violation evidence drift"); + if (violation.kind === "threshold") { + const metric = text(violation.metric, "violation metric"); + const samples = validateMetric( + object(group.metrics, "group metrics")[metric], + group.sampleCount, + `violation group metric ${metric}`, + ); + check(samples.includes(actual), "violation was not sampled in its performance group"); + } + text(violation.expected, "violation expectation"); + text(violation.message, "violation message"); + } + const messages = violations.map((violation) => violation.message); + check( + card.kind === "openclaw-failure" && + card.status === "FAIL" && + card.failedCommand === null && + card.scenario === record.scenario && + card.state === stateId(record), + "blocking card identity was invalid", + ); + check(card.summary === messages[0], "blocking card summary did not match"); + exactStrings(card.violations, messages, "blocking card violations"); + const cardMeasurements = object(card.measurements, "blocking card measurements"); + check( + cardMeasurements.peakRssMb === record.measurements.peakRssMb && + cardMeasurements.cpuPercentMax === record.measurements.cpuPercentMax, + "blocking card measurements did not match", + ); +} + +function evaluate(evaluator) { + try { + evaluator(); + return { ok: true }; + } catch (error) { + return { ok: false, reason: error instanceof Error ? error.message : String(error) }; + } } export function evaluateToleratedPartialKovaReport(report) { - const gate = report?.gate; - if (!gate || typeof gate !== "object" || Array.isArray(gate)) { - return { ok: false, reason: "missing gate metadata" }; - } - if (gate.verdict !== "PARTIAL") { - return { ok: false, reason: `gate verdict was ${JSON.stringify(gate.verdict)}` }; - } - - const blockingCount = numericCount(gate.blockingCount); - if (blockingCount === undefined) { - return { ok: false, reason: "missing blocking count" }; - } - if (blockingCount !== 0) { - return { ok: false, reason: `blocking count was ${JSON.stringify(gate.blockingCount)}` }; - } - - const reportBaseline = report?.baseline; - const gateBaseline = report?.gate?.baseline; - if ( - (reportBaseline !== null && reportBaseline !== undefined) || - (gateBaseline !== null && gateBaseline !== undefined) - ) { - const baselineRegressionCount = numericCount( - reportBaseline?.comparison?.regressionCount ?? gateBaseline?.regressionCount, + return evaluate(() => { + const { gate, records, cards } = validateEnvelope(report); + check(gate.verdict === "PARTIAL", "gate verdict was not PARTIAL"); + check(gate.blockingCount === 0, "PARTIAL gate had blocking cards"); + check( + cards.every((card) => card.severity !== "blocking"), + "PARTIAL gate had a blocking card", ); - if (baselineRegressionCount === undefined) { - return { ok: false, reason: "missing baseline regression count" }; + check( + records.every((record) => record.status === "PASS"), + "PARTIAL report had a non-PASS record", + ); + check( + records.every((record) => array(record.violations, "record violations").length === 0), + "PARTIAL report had violations", + ); + }); +} + +export function evaluateToleratedProfiledKovaReport(report) { + return evaluate(() => { + const { gate, records, cards, groups } = validateEnvelope(report); + check(gate.verdict === "DO_NOT_SHIP", "gate verdict was not DO_NOT_SHIP"); + check( + records.every((record) => record.status === "PASS" || record.status === "FAIL"), + "invalid record status", + ); + check( + records + .filter((record) => record.status === "PASS") + .every((record) => array(record.violations, "record violations").length === 0), + "PASS record had violations", + ); + const failed = records.filter((record) => record.status === "FAIL"); + const blocking = cards.filter((card) => card.severity === "blocking"); + check( + failed.length > 0 && blocking.length === failed.length, + "failure/card counts did not match", + ); + const remaining = [...blocking]; + for (const record of failed) { + const index = remaining.findIndex( + (card) => card.scenario === record.scenario && card.state === stateId(record), + ); + check(index >= 0, "blocking cards did not map one-to-one"); + const card = remaining.splice(index, 1)[0]; + validateProfiledFailure(record, card, groups.get(recordKey(record))); } - if (baselineRegressionCount !== 0) { - return { - ok: false, - reason: `baseline regression count was ${JSON.stringify(baselineRegressionCount)}`, - }; - } - } + check(remaining.length === 0, "blocking cards did not map one-to-one"); + }); +} - const statuses = report?.summary?.statuses; - if (!statuses || typeof statuses !== "object" || Array.isArray(statuses)) { - return { ok: false, reason: "missing status summary" }; +export function evaluateToleratedKovaReport(report) { + const partial = evaluateToleratedPartialKovaReport(report); + if (partial.ok) { + return { ok: true, classification: "filtered-partial" }; } - - const passCount = numericCount(statuses.PASS); - if (passCount === undefined || passCount <= 0) { - return { ok: false, reason: "status summary had no PASS records" }; + const profiled = evaluateToleratedProfiledKovaReport(report); + if (profiled.ok) { + return { ok: true, classification: "profiled-resource-only" }; } - - const nonPassStatuses = Object.entries(statuses).filter( - ([status, count]) => status !== "PASS" && (numericCount(count) ?? 0) > 0, - ); - if (nonPassStatuses.length > 0) { - return { - ok: false, - reason: `non-pass statuses present: ${nonPassStatuses - .map(([status, count]) => `${status}=${count}`) - .join(", ")}`, - }; - } - - const records = Array.isArray(report?.records) ? report.records : []; - if (records.length === 0) { - return { ok: false, reason: "missing selected scenario records" }; - } - - const groups = Array.isArray(report?.performance?.groups) ? report.performance.groups : []; - if (groups.length === 0) { - return { ok: false, reason: "missing performance groups" }; - } - - if (!groups.some((group) => hasSampledMetric(group, rssMetricIds))) { - return { ok: false, reason: "missing sampled RSS metric in performance groups" }; - } - - if (!groups.some((group) => hasSampledMetric(group, cpuMetricIds))) { - return { ok: false, reason: "missing sampled CPU metric in performance groups" }; - } - - return { ok: true }; + return { ok: false, reason: `partial: ${partial.reason}; profiled: ${profiled.reason}` }; } function readCliReportPath() { @@ -112,13 +552,13 @@ const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(proces if (modulePath === invokedPath) { try { - const reportPath = readCliReportPath(); - const report = JSON.parse(fs.readFileSync(reportPath, "utf8")); - const result = evaluateToleratedPartialKovaReport(report); + const report = JSON.parse(fs.readFileSync(readCliReportPath(), "utf8")); + const result = evaluateToleratedKovaReport(report); if (!result.ok) { - console.error(`Kova PARTIAL verdict is not tolerable: ${result.reason}`); + console.error(`Kova verdict is not tolerable: ${result.reason}`); process.exit(1); } + console.log(`Tolerated Kova verdict: ${result.classification}`); } catch (error) { console.error(error instanceof Error ? error.message : String(error)); process.exit(1); diff --git a/test/scripts/kova-report-gate.test.ts b/test/scripts/kova-report-gate.test.ts index 2bc80c405d6..a6f840a4053 100644 --- a/test/scripts/kova-report-gate.test.ts +++ b/test/scripts/kova-report-gate.test.ts @@ -1,36 +1,365 @@ -// Kova report gate tests cover tolerated PARTIAL performance verdict handling. +// Kova report gate tests use trimmed values from a real deep-profile release report. import { spawnSync } from "node:child_process"; import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { evaluateToleratedPartialKovaReport } from "../../scripts/lib/kova-report-gate.mjs"; +import { + evaluateToleratedKovaReport, + evaluateToleratedPartialKovaReport, + evaluateToleratedProfiledKovaReport, +} from "../../scripts/lib/kova-report-gate.mjs"; + +type JsonObject = Record; +type PathPart = number | string; +type ReportMutation = [string, (report: JsonObject) => void]; const tempRoots: string[] = []; const SCRIPT_PATH = "scripts/lib/kova-report-gate.mjs"; +const SCENARIO = "agent-cold-warm-message"; +const STATE = "mock-openai-provider"; +const SURFACE = "agent-cli-local-turn"; +const PROFILED_INTERPRETATION = + "instrumented run; CPU/RSS can include profiler and diagnostic overhead"; -function partialReport(overrides: Record = {}) { +function objectAt(value: unknown): JsonObject { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new TypeError("expected object fixture value"); + } + return value as JsonObject; +} + +function arrayAt(value: unknown): unknown[] { + if (!Array.isArray(value)) { + throw new TypeError("expected array fixture value"); + } + return value; +} + +function valueAt(root: unknown, path: PathPart[]): unknown { + let current = root; + for (const part of path) { + current = typeof part === "number" ? arrayAt(current)[part] : objectAt(current)[part]; + } + return current; +} + +function setAt(root: unknown, path: PathPart[], value: unknown): void { + const parent = valueAt(root, path.slice(0, -1)); + const key = path.at(-1); + if (typeof key === "number") { + arrayAt(parent)[key] = value; + } else if (typeof key === "string") { + objectAt(parent)[key] = value; + } else { + throw new TypeError("empty fixture path"); + } +} + +function deleteAt(root: unknown, path: PathPart[]): void { + const parent = valueAt(root, path.slice(0, -1)); + const key = path.at(-1); + if (typeof key !== "string") { + throw new TypeError("delete fixture path must end in a string"); + } + delete objectAt(parent)[key]; +} + +function metric(value: number) { return { - baseline: { comparison: { regressionCount: 0 } }, - gate: { verdict: "PARTIAL", blockingCount: 0 }, + classification: "stable", + count: 1, + max: value, + median: value, + min: value, + p95: value, + samples: [value], + }; +} + +function commandResult() { + return { + command: "ocm @env -- openclaw agent --local", + status: 0, + stderr: "", + stdout: "", + timedOut: false, + }; +} + +function cleanupResult() { + return { + command: "ocm env destroy env --json", + status: 0, + stderr: "", + stdout: "", + timedOut: false, + }; +} + +function targetCleanup() { + return { + command: "ocm runtime remove runtime --json", + result: cleanupResult(), + runtimeName: "runtime", + status: "removed", + }; +} + +function normalProfiling() { + return { + affectsResourceMeasurements: false, + baselineEligible: true, + deepProfile: false, + diagnosticReport: false, + enabled: false, + heapSnapshot: false, + interpretation: "normal user-path resource measurements", + nodeProfile: false, + profileOnFailure: false, + schemaVersion: "kova.profiling.v1", + }; +} + +function deepProfiling() { + return { + affectsResourceMeasurements: true, + baselineEligible: false, + deepProfile: true, + diagnosticReport: true, + enabled: true, + heapSnapshot: true, + interpretation: PROFILED_INTERPRETATION, + nodeProfile: true, + profileOnFailure: false, + schemaVersion: "kova.profiling.v1", + }; +} + +function infoCard() { + return { + kind: "filtered-required-scenario", + scenario: "fresh-install", + severity: "info", + state: "fresh", + status: "MISSING", + }; +} + +function partialReport(): JsonObject { + return { + baseline: null, + controls: { + exclude: [], + gate: true, + include: [`scenario:${SCENARIO}`], + repeat: 1, + }, + gate: { + baseline: null, + blockingCount: 0, + cards: [infoCard()], + complete: false, + enabled: true, + infoCount: 1, + missingRequiredCount: 1, + ok: false, + partial: true, + schemaVersion: "kova.gate.v1", + verdict: "PARTIAL", + warningCount: 0, + }, + mode: "execution", performance: { + groupCount: 1, groups: [ { + key: `${SCENARIO}|${SURFACE}|${STATE}`, metrics: { - cpuPercentMax: { count: 1 }, - resourcePeakGatewayRssMb: { count: 1 }, + cpuPercentMax: metric(80), + peakRssMb: metric(650), }, - scenario: "gateway", - state: "clean", + profiledRunCount: 0, + resourceInterpretation: "normal", + sampleCount: 1, + scenario: SCENARIO, + state: STATE, + statuses: { PASS: 1 }, + surface: SURFACE, }, ], + profiledRunCount: 0, + repeat: 1, + schemaVersion: "kova.performance.v1", + unstableGroupCount: 0, }, - records: [{ scenario: "gateway", state: "clean", status: "PASS" }], - summary: { statuses: { PASS: 3 } }, - ...overrides, + records: [ + { + cleanup: "destroyed", + cleanupResult: cleanupResult(), + measurements: { + cpuPercentMax: 80, + peakRssMb: 650, + }, + phases: [{ id: "agent-turn", results: [commandResult()] }], + profiling: normalProfiling(), + scenario: SCENARIO, + state: { id: STATE }, + status: "PASS", + surface: SURFACE, + violations: [], + }, + ], + schemaVersion: "kova.report.v1", + summary: { statuses: { PASS: 1 }, total: 1 }, + target: "local-build:/workspace/openclaw", + targetCleanup: targetCleanup(), }; } +function profiledResourceReport(): JsonObject { + const violationMessages = [ + "peak RSS 923.7 MB exceeded threshold 900 MB", + "agent-process peak RSS 923.7 MB exceeded threshold 900 MB", + ]; + return { + baseline: null, + controls: { + exclude: [], + gate: true, + include: [`scenario:${SCENARIO}`], + repeat: 1, + }, + gate: { + baseline: null, + blockingCount: 1, + cards: [ + infoCard(), + { + failedCommand: null, + kind: "openclaw-failure", + measurements: { cpuPercentMax: 156.2, peakRssMb: 923.7 }, + scenario: SCENARIO, + severity: "blocking", + state: STATE, + status: "FAIL", + summary: violationMessages[0], + violations: violationMessages, + }, + ], + complete: false, + enabled: true, + infoCount: 1, + missingRequiredCount: 1, + ok: false, + partial: true, + schemaVersion: "kova.gate.v1", + verdict: "DO_NOT_SHIP", + warningCount: 0, + }, + mode: "execution", + performance: { + groupCount: 1, + groups: [ + { + key: `${SCENARIO}|${SURFACE}|${STATE}`, + metrics: { + cpuPercentMax: metric(156.2), + peakRssMb: metric(923.7), + }, + profiledRunCount: 1, + resourceInterpretation: "instrumented", + sampleCount: 1, + scenario: SCENARIO, + state: STATE, + statuses: { FAIL: 1 }, + surface: SURFACE, + }, + ], + profiledRunCount: 1, + repeat: 1, + schemaVersion: "kova.performance.v1", + unstableGroupCount: 0, + }, + records: [ + { + cleanup: "destroyed", + cleanupResult: cleanupResult(), + measurements: { + cpuPercentMax: 156.2, + peakRssMb: 923.7, + profilingAffectsResourceMeasurements: true, + profilingBaselineEligible: false, + profilingEnabled: true, + profilingResourceInterpretation: PROFILED_INTERPRETATION, + resourceByRole: { + "agent-process": { maxCpuPercent: 156.2, peakRssMb: 923.7 }, + }, + }, + phases: [{ id: "agent-turn", results: [commandResult()] }], + profiling: deepProfiling(), + scenario: SCENARIO, + state: { id: STATE }, + status: "FAIL", + surface: SURFACE, + violations: [ + { + actual: 923.7, + expected: "<= 900", + kind: "threshold", + message: violationMessages[0], + metric: "peakRssMb", + }, + { + actual: 923.7, + expected: "<= 900", + kind: "resource", + message: violationMessages[1], + metric: "resourceByRole.agent-process.peakRssMb", + role: "agent-process", + }, + ], + }, + ], + schemaVersion: "kova.report.v1", + summary: { statuses: { FAIL: 1 }, total: 1 }, + target: "local-build:/workspace/openclaw", + targetCleanup: targetCleanup(), + }; +} + +function attachPassingBaseline(report: JsonObject): void { + report.baseline = { + comparison: { + baselineEntryCount: 1, + generatedAt: "2026-07-09T00:00:00.000Z", + groups: [], + missing: [], + missingBaselineCount: 0, + ok: true, + regressionCount: 0, + regressions: [], + schemaVersion: "kova.baselineComparison.v1", + }, + path: "/tmp/baseline.json", + }; + objectAt(report.gate).baseline = { + baselineEntryCount: 1, + missing: [], + missingBaselineCount: 0, + ok: true, + regressedGroups: [], + regressionCount: 0, + schemaVersion: "kova.gateBaselineSummary.v1", + }; +} + +function blockingCard(report: JsonObject): JsonObject { + const cards = arrayAt(objectAt(report.gate).cards); + const card = cards.find((candidate) => objectAt(candidate).severity === "blocking"); + return objectAt(card); +} + function writeReport(report: unknown): string { const root = mkdtempSync(join(tmpdir(), "openclaw-kova-report-")); tempRoots.push(root); @@ -39,6 +368,14 @@ function writeReport(report: unknown): string { return reportPath; } +function expectProfiledRejection(report: JsonObject): void { + expect(evaluateToleratedProfiledKovaReport(report).ok).toBe(false); +} + +function expectPartialRejection(report: JsonObject): void { + expect(evaluateToleratedPartialKovaReport(report).ok).toBe(false); +} + afterEach(() => { for (const root of tempRoots.splice(0)) { rmSync(root, { recursive: true, force: true }); @@ -46,158 +383,475 @@ afterEach(() => { }); describe("scripts/lib/kova-report-gate.mjs", () => { - it("accepts partial reports only when selected scenarios passed", () => { + it("accepts an exact filtered PARTIAL execution report", () => { expect(evaluateToleratedPartialKovaReport(partialReport())).toEqual({ ok: true }); + expect(evaluateToleratedKovaReport(partialReport())).toEqual({ + classification: "filtered-partial", + ok: true, + }); }); - it("rejects partial reports with missing status summaries", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - summary: {}, - }), - ), - ).toEqual({ ok: false, reason: "missing status summary" }); + it("accepts an exact deep-profile resource-only rejection", () => { + expect(evaluateToleratedProfiledKovaReport(profiledResourceReport())).toEqual({ + ok: true, + }); + expect(evaluateToleratedKovaReport(profiledResourceReport())).toEqual({ + classification: "profiled-resource-only", + ok: true, + }); }); - it("rejects partial reports without explicit blocking counts", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - gate: { verdict: "PARTIAL" }, - }), - ), - ).toEqual({ ok: false, reason: "missing blocking count" }); + it("accepts independently matching non-regressing baseline evidence", () => { + const partial = partialReport(); + const profiled = profiledResourceReport(); + attachPassingBaseline(partial); + attachPassingBaseline(profiled); + + expect(evaluateToleratedPartialKovaReport(partial)).toEqual({ ok: true }); + expect(evaluateToleratedProfiledKovaReport(profiled)).toEqual({ ok: true }); }); - it("rejects partial reports with malformed zero-like blocking counts", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - gate: { blockingCount: "", verdict: "PARTIAL" }, - }), - ), - ).toEqual({ ok: false, reason: "missing blocking count" }); + it("accepts proven already-absent cleanup results", () => { + const report = partialReport(); + const record = objectAt(valueAt(report, ["records", 0])); + record.cleanup = "already-absent"; + record.cleanupResult = { + ...cleanupResult(), + status: 1, + stderr: "environment not found", + }; + const cleanup = objectAt(report.targetCleanup); + cleanup.status = "already-absent"; + cleanup.result = { + ...cleanupResult(), + status: 1, + stderr: "runtime does not exist", + }; + + expect(evaluateToleratedPartialKovaReport(report)).toEqual({ ok: true }); }); - it("rejects partial reports without explicit baseline regression counts", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - baseline: {}, - }), - ), - ).toEqual({ ok: false, reason: "missing baseline regression count" }); + it("accepts profiled RSS growth emitted as a soak violation", () => { + const report = profiledResourceReport(); + const message = "resource-sampled RSS grew by 42 MB, over threshold 40 MB"; + setAt(report, ["records", 0, "measurements", "rssGrowthMb"], 42); + setAt( + report, + ["records", 0, "violations"], + [ + { + actual: 42, + expected: "<= 40", + kind: "soak", + message, + metric: "rssGrowthMb", + }, + ], + ); + blockingCard(report).summary = message; + blockingCard(report).violations = [message]; + + expect(evaluateToleratedProfiledKovaReport(report)).toEqual({ ok: true }); }); - it("accepts partial reports without a baseline comparison", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - baseline: null, - }), - ), - ).toEqual({ ok: true }); + it("rejects hidden violations on PASS records", () => { + const report = profiledResourceReport(); + const scenario = "passing-agent-message"; + const records = arrayAt(report.records); + const passRecord = objectAt(structuredClone(records[0])); + passRecord.scenario = scenario; + passRecord.status = "PASS"; + passRecord.violations = [{ message: "hidden violation" }]; + records.push(passRecord); + + const performance = objectAt(report.performance); + const groups = arrayAt(performance.groups); + const passGroup = objectAt(structuredClone(groups[0])); + passGroup.key = `${scenario}|${SURFACE}|${STATE}`; + passGroup.scenario = scenario; + passGroup.statuses = { PASS: 1 }; + groups.push(passGroup); + performance.groupCount = 2; + performance.profiledRunCount = 2; + report.summary = { statuses: { FAIL: 1, PASS: 1 }, total: 2 }; + + expectProfiledRejection(report); }); - it("rejects partial reports with malformed zero-like baseline regression counts", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - baseline: { comparison: { regressionCount: null } }, - }), - ), - ).toEqual({ ok: false, reason: "missing baseline regression count" }); - }); + const profiledMutations: ReportMutation[] = [ + [ + "rejects the wrong report schema", + (report) => setAt(report, ["schemaVersion"], "kova.report.v2"), + ], + ["rejects dry-run reports", (report) => setAt(report, ["mode"], "dry-run")], + [ + "rejects the wrong gate schema", + (report) => setAt(report, ["gate", "schemaVersion"], "kova.gate.v2"), + ], + ["rejects disabled gate controls", (report) => setAt(report, ["controls", "gate"], false)], + ["rejects unfiltered reports", (report) => setAt(report, ["controls", "include"], [])], + ["rejects malformed extra filters", (report) => setAt(report, ["controls", "exclude"], [" "])], + ["rejects non-partial gate metadata", (report) => setAt(report, ["gate", "partial"], false)], + ["rejects complete gate metadata", (report) => setAt(report, ["gate", "complete"], true)], + ["rejects ok gate metadata", (report) => setAt(report, ["gate", "ok"], true)], + ["rejects fractional gate counts", (report) => setAt(report, ["gate", "blockingCount"], 1.5)], + [ + "rejects noncanonical deep profiling", + (report) => setAt(report, ["records", 0, "profiling", "deepProfile"], false), + ], + [ + "rejects inconsistent derived profiling flags", + (report) => setAt(report, ["records", 0, "measurements", "profilingEnabled"], false), + ], + [ + "rejects failed phase commands", + (report) => setAt(report, ["records", 0, "phases", 0, "results", 0, "status"], 1), + ], + [ + "rejects timed-out phase commands", + (report) => setAt(report, ["records", 0, "phases", 0, "results", 0, "timedOut"], true), + ], + [ + "rejects retained record cleanup", + (report) => setAt(report, ["records", 0, "cleanup"], "retained"), + ], + [ + "rejects failed record cleanup", + (report) => setAt(report, ["records", 0, "cleanupResult", "status"], 1), + ], + [ + "rejects generic command-not-found record cleanup", + (report) => { + setAt(report, ["records", 0, "cleanup"], "already-absent"); + setAt(report, ["records", 0, "cleanupResult", "status"], 1); + setAt(report, ["records", 0, "cleanupResult", "stderr"], "ocm: command not found"); + }, + ], + [ + "rejects failed target cleanup", + (report) => setAt(report, ["targetCleanup", "status"], "remove-failed"), + ], + [ + "rejects timed-out target cleanup", + (report) => setAt(report, ["targetCleanup", "result", "timedOut"], true), + ], + [ + "rejects failed target removal", + (report) => setAt(report, ["targetCleanup", "result", "status"], 1), + ], + [ + "rejects generic command-not-found target cleanup", + (report) => { + setAt(report, ["targetCleanup", "status"], "already-absent"); + setAt(report, ["targetCleanup", "result", "status"], 1); + setAt(report, ["targetCleanup", "result", "stderr"], "ocm: command not found"); + }, + ], + ["rejects fractional summary totals", (report) => setAt(report, ["summary", "total"], 1.5)], + [ + "rejects summary totals that disagree with records", + (report) => setAt(report, ["summary", "total"], 2), + ], + [ + "rejects status counts that disagree with records", + (report) => setAt(report, ["summary", "statuses", "FAIL"], 2), + ], + ["rejects empty scenarios", (report) => setAt(report, ["records", 0, "scenario"], "")], + [ + "rejects whitespace-only scenarios", + (report) => setAt(report, ["records", 0, "scenario"], " "), + ], + ["rejects empty state ids", (report) => setAt(report, ["records", 0, "state", "id"], "")], + [ + "rejects wrong performance schemas", + (report) => setAt(report, ["performance", "schemaVersion"], "kova.performance.v2"), + ], + [ + "rejects wrong performance group counts", + (report) => setAt(report, ["performance", "groupCount"], 2), + ], + [ + "rejects wrong performance group keys", + (report) => setAt(report, ["performance", "groups", 0, "key"], "wrong"), + ], + [ + "rejects unstable group count drift", + (report) => setAt(report, ["performance", "unstableGroupCount"], 1), + ], + [ + "rejects fractional sample counts", + (report) => setAt(report, ["performance", "groups", 0, "sampleCount"], 1.5), + ], + [ + "rejects group statuses that disagree with records", + (report) => setAt(report, ["performance", "groups", 0, "statuses", "FAIL"], 2), + ], + [ + "rejects group profile counts that disagree with records", + (report) => setAt(report, ["performance", "groups", 0, "profiledRunCount"], 0), + ], + [ + "rejects metric counts without exact samples", + (report) => setAt(report, ["performance", "groups", 0, "metrics", "peakRssMb", "count"], 2), + ], + [ + "rejects metric counts above the group sample count", + (report) => { + setAt(report, ["performance", "groups", 0, "metrics", "peakRssMb", "count"], 2); + setAt( + report, + ["performance", "groups", 0, "metrics", "peakRssMb", "samples"], + [923.7, 923.7], + ); + }, + ], + [ + "rejects invalid metric classifications", + (report) => + setAt( + report, + ["performance", "groups", 0, "metrics", "peakRssMb", "classification"], + "unknown", + ), + ], + [ + "rejects violations not bound to direct measurements", + (report) => setAt(report, ["records", 0, "violations", 0, "actual"], 900), + ], + [ + "rejects violations without expectations", + (report) => setAt(report, ["records", 0, "violations", 0, "expected"], ""), + ], + [ + "rejects whitespace-only violation expectations", + (report) => setAt(report, ["records", 0, "violations", 0, "expected"], " "), + ], + [ + "rejects role names that disagree with role metrics", + (report) => setAt(report, ["records", 0, "violations", 1, "role"], "gateway"), + ], + [ + "rejects missing role measurements", + (report) => + deleteAt(report, [ + "records", + 0, + "measurements", + "resourceByRole", + "agent-process", + "peakRssMb", + ]), + ], + [ + "rejects non-resource profiling violations", + (report) => setAt(report, ["records", 0, "violations", 0, "metric"], "agentTurnMs"), + ], + [ + "rejects missing RSS samples in the matching group", + (report) => deleteAt(report, ["performance", "groups", 0, "metrics", "peakRssMb"]), + ], + [ + "rejects missing CPU samples in the matching group", + (report) => deleteAt(report, ["performance", "groups", 0, "metrics", "cpuPercentMax"]), + ], + [ + "rejects blocking cards with failed commands", + (report) => (blockingCard(report).failedCommand = "openclaw agent"), + ], + [ + "rejects blocking cards with rewritten violation messages", + (report) => (blockingCard(report).violations = ["different message"]), + ], + [ + "rejects blocking cards with mismatched measurements", + (report) => setAt(blockingCard(report), ["measurements", "peakRssMb"], 900), + ], + [ + "rejects blocking cards mapped to another state", + (report) => (blockingCard(report).state = "other-state"), + ], + [ + "rejects duplicate blocking cards", + (report) => { + const cards = arrayAt(objectAt(report.gate).cards); + cards.push(structuredClone(blockingCard(report))); + setAt(report, ["gate", "blockingCount"], 2); + }, + ], + [ + "rejects gate cards with inherited-property severities", + (report) => { + const cards = arrayAt(objectAt(report.gate).cards); + cards.push({ ...infoCard(), severity: "toString" }); + }, + ], + [ + "rejects unexpected info gate cards", + (report) => { + const cards = arrayAt(objectAt(report.gate).cards); + cards.push({ ...infoCard(), kind: "openclaw-failure", status: "FAIL" }); + setAt(report, ["gate", "infoCount"], 2); + setAt(report, ["gate", "missingRequiredCount"], 2); + }, + ], + [ + "rejects unexpected warning gate cards", + (report) => { + const cards = arrayAt(objectAt(report.gate).cards); + cards.push({ + ...infoCard(), + kind: "openclaw-failure", + severity: "warning", + status: "FAIL", + }); + setAt(report, ["gate", "warningCount"], 1); + }, + ], + [ + "rejects duplicate performance groups", + (report) => { + const groups = arrayAt(objectAt(report.performance).groups); + groups.push(structuredClone(groups[0])); + setAt(report, ["performance", "groupCount"], 2); + }, + ], + [ + "rejects report baseline regressions even when gate baseline is clean", + (report) => { + attachPassingBaseline(report); + setAt(report, ["baseline", "comparison", "regressionCount"], 1); + setAt(report, ["baseline", "comparison", "regressions"], [{}]); + }, + ], + [ + "rejects gate baseline regressions even when report baseline is clean", + (report) => { + attachPassingBaseline(report); + setAt(report, ["gate", "baseline", "regressionCount"], 1); + setAt(report, ["gate", "baseline", "regressedGroups"], [{}]); + }, + ], + [ + "rejects one-sided baseline evidence", + (report) => { + attachPassingBaseline(report); + setAt(report, ["gate", "baseline"], null); + }, + ], + ]; - it("rejects partial reports without PASS records", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - summary: { statuses: { PASS: 0 } }, - }), - ), - ).toEqual({ ok: false, reason: "status summary had no PASS records" }); - }); + for (const [name, mutate] of profiledMutations) { + it(name, () => { + const report = profiledResourceReport(); + mutate(report); + expectProfiledRejection(report); + }); + } - it("rejects partial reports with non-pass records", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - summary: { statuses: { PASS: 2, FAIL: 1 } }, - }), - ), - ).toEqual({ ok: false, reason: "non-pass statuses present: FAIL=1" }); - }); + const partialMutations: ReportMutation[] = [ + [ + "rejects PARTIAL gates without partial metadata", + (report) => setAt(report, ["gate", "partial"], false), + ], + [ + "rejects PARTIAL gates marked complete", + (report) => setAt(report, ["gate", "complete"], true), + ], + ["rejects PARTIAL gates marked ok", (report) => setAt(report, ["gate", "ok"], true)], + [ + "rejects PARTIAL gates without filters", + (report) => setAt(report, ["controls", "include"], []), + ], + [ + "rejects PARTIAL gates with blocking cards", + (report) => { + setAt(report, ["gate", "cards", 0, "severity"], "blocking"); + setAt(report, ["gate", "blockingCount"], 1); + setAt(report, ["gate", "infoCount"], 0); + }, + ], + [ + "rejects non-PASS PARTIAL records even with reconciled summaries", + (report) => { + setAt(report, ["records", 0, "status"], "FAIL"); + setAt(report, ["summary", "statuses"], { FAIL: 1 }); + setAt(report, ["performance", "groups", 0, "statuses"], { FAIL: 1 }); + }, + ], + [ + "rejects PARTIAL status summary drift", + (report) => setAt(report, ["summary", "statuses", "PASS"], 2), + ], + [ + "rejects PARTIAL phase failures", + (report) => setAt(report, ["records", 0, "phases", 0, "results", 0, "status"], 1), + ], + [ + "rejects PARTIAL cleanup failures", + (report) => setAt(report, ["records", 0, "cleanup"], "destroy-failed"), + ], + [ + "rejects PARTIAL target cleanup failures", + (report) => setAt(report, ["targetCleanup", "status"], "planned"), + ], + [ + "rejects PARTIAL group count drift", + (report) => setAt(report, ["performance", "groupCount"], 0), + ], + [ + "rejects PARTIAL fractional metric counts", + (report) => setAt(report, ["performance", "groups", 0, "metrics", "peakRssMb", "count"], 0.5), + ], + [ + "rejects PARTIAL records with violations", + (report) => setAt(report, ["records", 0, "violations"], [{}]), + ], + [ + "rejects PARTIAL reports without sampled RSS", + (report) => deleteAt(report, ["performance", "groups", 0, "metrics", "peakRssMb"]), + ], + [ + "rejects PARTIAL reports without sampled CPU", + (report) => deleteAt(report, ["performance", "groups", 0, "metrics", "cpuPercentMax"]), + ], + [ + "rejects PARTIAL one-sided baselines", + (report) => { + attachPassingBaseline(report); + setAt(report, ["gate", "baseline"], null); + }, + ], + ]; - it("rejects partial reports without selected scenario records", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - records: [], - }), - ), - ).toEqual({ ok: false, reason: "missing selected scenario records" }); - }); + for (const [name, mutate] of partialMutations) { + it(name, () => { + const report = partialReport(); + mutate(report); + expectPartialRejection(report); + }); + } - it("rejects partial reports without performance groups", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - performance: {}, - }), - ), - ).toEqual({ ok: false, reason: "missing performance groups" }); - }); - - it("rejects partial reports without sampled RSS metrics", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - performance: { - groups: [ - { - metrics: { cpuPercentMax: { count: 1 } }, - scenario: "gateway", - state: "clean", - }, - ], - }, - }), - ), - ).toEqual({ ok: false, reason: "missing sampled RSS metric in performance groups" }); - }); - - it("rejects partial reports without sampled CPU metrics", () => { - expect( - evaluateToleratedPartialKovaReport( - partialReport({ - performance: { - groups: [ - { - metrics: { resourcePeakGatewayRssMb: { count: 1 } }, - scenario: "gateway", - state: "clean", - }, - ], - }, - }), - ), - ).toEqual({ ok: false, reason: "missing sampled CPU metric in performance groups" }); - }); - - it("exits non-zero for malformed tolerated-partial candidates", () => { + it("exits zero for profiling-only resource failures", () => { const result = spawnSync( process.execPath, - [SCRIPT_PATH, writeReport(partialReport({ summary: {} }))], - { - cwd: process.cwd(), - encoding: "utf8", - }, + [SCRIPT_PATH, writeReport(profiledResourceReport())], + { cwd: process.cwd(), encoding: "utf8" }, ); + expect(result.status).toBe(0); + expect(result.stdout).toContain("profiled-resource-only"); + }); + + it("exits non-zero for malformed tolerated-report candidates", () => { + const report = partialReport(); + setAt(report, ["summary", "total"], 2); + const result = spawnSync(process.execPath, [SCRIPT_PATH, writeReport(report)], { + cwd: process.cwd(), + encoding: "utf8", + }); + expect(result.status).toBe(1); - expect(result.stderr).toContain("missing status summary"); + expect(result.stderr).toContain("Kova verdict is not tolerable"); }); it("runs the CLI guard from paths that need file URL escaping", () => { @@ -207,17 +861,15 @@ describe("scripts/lib/kova-report-gate.mjs", () => { mkdirSync(scriptDir); const scriptPath = join(scriptDir, "kova-report-gate.mjs"); copyFileSync(SCRIPT_PATH, scriptPath); + const report = partialReport(); + setAt(report, ["summary", "total"], 2); - const result = spawnSync( - process.execPath, - [scriptPath, writeReport(partialReport({ summary: {} }))], - { - cwd: process.cwd(), - encoding: "utf8", - }, - ); + const result = spawnSync(process.execPath, [scriptPath, writeReport(report)], { + cwd: process.cwd(), + encoding: "utf8", + }); expect(result.status).toBe(1); - expect(result.stderr).toContain("missing status summary"); + expect(result.stderr).toContain("Kova verdict is not tolerable"); }); }); diff --git a/test/scripts/openclaw-performance-workflow.test.ts b/test/scripts/openclaw-performance-workflow.test.ts index 73f6dc35956..859e6fd5045 100644 --- a/test/scripts/openclaw-performance-workflow.test.ts +++ b/test/scripts/openclaw-performance-workflow.test.ts @@ -102,6 +102,9 @@ describe("OpenClaw performance workflow", () => { 'node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json"', ); expect(runKova.run).not.toContain("report.summary?.statuses ?? {}"); + expect(runKova.run).toContain( + "profiling-affected resource thresholds with no baseline regression", + ); }); it("installs local workspace packages beside the OCM root tarball", () => {