mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
fix(ci): ignore profiling-only resource gate failures
This commit is contained in:
Vincent Koc
parent
9775f09194
commit
7e2d287db5
4 changed files with 1335 additions and 240 deletions
2
.github/workflows/openclaw-performance.yml
vendored
2
.github/workflows/openclaw-performance.yml
vendored
|
|
@ -372,7 +372,7 @@ jobs:
|
|||
then
|
||||
effective_status=0
|
||||
{
|
||||
echo "Kova returned a partial release-gate verdict for filtered performance coverage, but all selected scenarios passed and no baseline regression was reported."
|
||||
echo "Kova returned a non-zero release-gate verdict, but the trusted report adapter found only filtered coverage or profiling-affected resource thresholds with no baseline regression."
|
||||
echo
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -1,102 +1,542 @@
|
|||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
function numericCount(value) {
|
||||
if (typeof value !== "number") {
|
||||
return undefined;
|
||||
const SCHEMA = {
|
||||
report: "kova.report.v1",
|
||||
gate: "kova.gate.v1",
|
||||
performance: "kova.performance.v1",
|
||||
profiling: "kova.profiling.v1",
|
||||
baseline: "kova.baselineComparison.v1",
|
||||
gateBaseline: "kova.gateBaselineSummary.v1",
|
||||
};
|
||||
const PROFILED_INTERPRETATION =
|
||||
"instrumented run; CPU/RSS can include profiler and diagnostic overhead";
|
||||
const RSS_METRICS = ["peakRssMb", "resourcePeakGatewayRssMb"];
|
||||
const CPU_METRICS = ["cpuPercentMax"];
|
||||
const DIRECT_VIOLATIONS = new Set(["cpuPercentMax", "peakRssMb"]);
|
||||
const SOAK_VIOLATIONS = new Set(["gatewayRssGrowthMb", "rssGrowthMb"]);
|
||||
const ROLE_VIOLATION = /^resourceByRole\.([^.]+)\.(maxCpuPercent|peakRssMb)$/u;
|
||||
function check(condition, reason) {
|
||||
if (!condition) {
|
||||
throw new Error(reason);
|
||||
}
|
||||
return Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
const rssMetricIds = ["peakRssMb", "resourcePeakGatewayRssMb"];
|
||||
const cpuMetricIds = ["cpuPercentMax"];
|
||||
function object(value, label) {
|
||||
check(value !== null && typeof value === "object" && !Array.isArray(value), `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function hasSampledMetric(group, metricIds) {
|
||||
return metricIds.some((metricId) => {
|
||||
const metric = group?.metrics?.[metricId];
|
||||
return metric && Number(metric.count) > 0;
|
||||
});
|
||||
function array(value, label) {
|
||||
check(Array.isArray(value), `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
function count(value, label, { positive = false } = {}) {
|
||||
check(Number.isSafeInteger(value) && value >= (positive ? 1 : 0), `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function text(value, label) {
|
||||
check(typeof value === "string" && value.trim().length > 0, `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function finite(value, label) {
|
||||
check(typeof value === "number" && Number.isFinite(value), `invalid ${label}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function stateId(record) {
|
||||
return text(object(record.state, "record state").id, "record state id");
|
||||
}
|
||||
|
||||
function recordKey(record) {
|
||||
return [
|
||||
text(record.scenario, "record scenario"),
|
||||
text(record.surface, "record surface"),
|
||||
stateId(record),
|
||||
].join("\u0000");
|
||||
}
|
||||
|
||||
function statusCounts(records) {
|
||||
const statuses = {};
|
||||
for (const record of records) {
|
||||
text(record.status, "record status");
|
||||
statuses[record.status] = (statuses[record.status] ?? 0) + 1;
|
||||
}
|
||||
return statuses;
|
||||
}
|
||||
|
||||
function exactCounts(actualValue, expected, label) {
|
||||
const actual = object(actualValue, label);
|
||||
check(Object.keys(actual).length === Object.keys(expected).length, `${label} keys did not match`);
|
||||
for (const [key, expectedCount] of Object.entries(expected)) {
|
||||
check(count(actual[key], `${label}.${key}`) === expectedCount, `${label}.${key} did not match`);
|
||||
}
|
||||
}
|
||||
|
||||
function exactStrings(actualValue, expected, label) {
|
||||
const actual = array(actualValue, label);
|
||||
check(
|
||||
actual.length === expected.length && actual.every((value, index) => value === expected[index]),
|
||||
`${label} did not match`,
|
||||
);
|
||||
}
|
||||
|
||||
function validateCommandResult(value, label) {
|
||||
const result = object(value, label);
|
||||
check(result.status === 0 && result.timedOut === false, `${label} failed`);
|
||||
}
|
||||
|
||||
function alreadyAbsent(result, noun) {
|
||||
check(
|
||||
Number.isSafeInteger(result.status) && result.status !== 0,
|
||||
`${noun} cleanup status was invalid`,
|
||||
);
|
||||
check(result.timedOut === false, `${noun} cleanup timed out`);
|
||||
const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`;
|
||||
const missing = new RegExp(`\\b${noun}\\b[\\s\\S]*\\b(?:does not exist|not found)\\b`, "iu").test(
|
||||
output,
|
||||
);
|
||||
check(missing, `${noun} cleanup lacked already-absent evidence`);
|
||||
}
|
||||
|
||||
function validateCleanup(status, resultValue, successStatus, noun, label) {
|
||||
const result = object(resultValue, `${label} result`);
|
||||
if (status === successStatus) {
|
||||
validateCommandResult(result, `${label} result`);
|
||||
return;
|
||||
}
|
||||
check(status === "already-absent", `${label} was not completed`);
|
||||
alreadyAbsent(result, noun);
|
||||
}
|
||||
|
||||
function validateRecord(recordValue) {
|
||||
const record = object(recordValue, "record");
|
||||
recordKey(record);
|
||||
object(record.measurements, "record measurements");
|
||||
const profiling = object(record.profiling, "record profiling");
|
||||
check(profiling.schemaVersion === SCHEMA.profiling, "wrong profiling schema");
|
||||
const phases = array(record.phases, "record phases");
|
||||
check(phases.length > 0, "record had no phases");
|
||||
for (const phaseValue of phases) {
|
||||
const results = array(object(phaseValue, "phase").results, "phase results");
|
||||
check(results.length > 0, "phase had no command results");
|
||||
results.forEach((result, index) => validateCommandResult(result, `phase result ${index}`));
|
||||
}
|
||||
validateCleanup(
|
||||
record.cleanup,
|
||||
record.cleanupResult,
|
||||
"destroyed",
|
||||
"environment",
|
||||
"record cleanup",
|
||||
);
|
||||
return record;
|
||||
}
|
||||
|
||||
function validateTargetCleanup(report) {
|
||||
if (!text(report.target, "report target").startsWith("local-build:")) {
|
||||
check(report.targetCleanup === null, "non-local target had cleanup metadata");
|
||||
return;
|
||||
}
|
||||
const cleanup = object(report.targetCleanup, "target cleanup");
|
||||
validateCleanup(cleanup.status, cleanup.result, "removed", "runtime", "target cleanup");
|
||||
}
|
||||
|
||||
function validateBaselines(report, gate) {
|
||||
const reportBaseline = report.baseline;
|
||||
const gateBaseline = gate.baseline;
|
||||
check(
|
||||
(reportBaseline === null || reportBaseline === undefined) ===
|
||||
(gateBaseline === null || gateBaseline === undefined),
|
||||
"baseline evidence was one-sided",
|
||||
);
|
||||
if (reportBaseline === null || reportBaseline === undefined) {
|
||||
return;
|
||||
}
|
||||
const comparison = object(
|
||||
object(reportBaseline, "report baseline").comparison,
|
||||
"baseline comparison",
|
||||
);
|
||||
const summary = object(gateBaseline, "gate baseline");
|
||||
check(comparison.schemaVersion === SCHEMA.baseline, "wrong baseline comparison schema");
|
||||
check(summary.schemaVersion === SCHEMA.gateBaseline, "wrong gate baseline schema");
|
||||
for (const [field, listField] of [
|
||||
["regressionCount", "regressions"],
|
||||
["missingBaselineCount", "missing"],
|
||||
]) {
|
||||
check(count(comparison[field], `baseline ${field}`) === 0, `baseline ${field} was nonzero`);
|
||||
check(
|
||||
array(comparison[listField], `baseline ${listField}`).length === 0,
|
||||
`baseline ${listField} present`,
|
||||
);
|
||||
check(
|
||||
count(summary[field], `gate baseline ${field}`) === 0,
|
||||
`gate baseline ${field} was nonzero`,
|
||||
);
|
||||
const gateList = field === "regressionCount" ? "regressedGroups" : listField;
|
||||
check(
|
||||
array(summary[gateList], `gate baseline ${gateList}`).length === 0,
|
||||
`gate baseline ${gateList} present`,
|
||||
);
|
||||
}
|
||||
check(comparison.ok === true && summary.ok === true, "baseline was not ok");
|
||||
check(
|
||||
count(comparison.baselineEntryCount, "baseline entry count") ===
|
||||
count(summary.baselineEntryCount, "gate baseline entry count"),
|
||||
"baseline entry counts did not match",
|
||||
);
|
||||
}
|
||||
|
||||
function validateSummary(report, records) {
|
||||
const summary = object(report.summary, "report summary");
|
||||
check(
|
||||
count(summary.total, "summary total") === records.length,
|
||||
"summary total did not match records",
|
||||
);
|
||||
exactCounts(summary.statuses, statusCounts(records), "summary statuses");
|
||||
}
|
||||
|
||||
function validateMetric(metricValue, sampleCount, label) {
|
||||
const metric = object(metricValue, label);
|
||||
const samples = array(metric.samples, `${label} samples`);
|
||||
const metricCount = count(metric.count, `${label} count`, { positive: true });
|
||||
check(metricCount === samples.length && metricCount <= sampleCount, `${label} count drift`);
|
||||
check(
|
||||
metric.classification === "stable" || metric.classification === "unstable",
|
||||
`${label} classification was invalid`,
|
||||
);
|
||||
samples.forEach((sample) => finite(sample, `${label} sample`));
|
||||
return samples;
|
||||
}
|
||||
|
||||
function sampledMetric(metrics, ids, sampleCount, label) {
|
||||
for (const id of ids) {
|
||||
if (metrics[id] !== undefined) {
|
||||
const samples = validateMetric(metrics[id], sampleCount, `${label}.${id}`);
|
||||
check(samples.length === sampleCount, `${label}.${id} did not cover every record`);
|
||||
return { id, samples };
|
||||
}
|
||||
}
|
||||
throw new Error(`${label} was not sampled`);
|
||||
}
|
||||
|
||||
function measured(record, ids, label) {
|
||||
for (const id of ids) {
|
||||
if (record.measurements[id] !== undefined) {
|
||||
return finite(record.measurements[id], `${label}.${id}`);
|
||||
}
|
||||
}
|
||||
throw new Error(`${label} was not measured`);
|
||||
}
|
||||
|
||||
function validatePerformance(report, records, repeat) {
|
||||
const performance = object(report.performance, "performance");
|
||||
check(performance.schemaVersion === SCHEMA.performance, "wrong performance schema");
|
||||
check(performance.repeat === repeat, "performance repeat did not match controls");
|
||||
const groups = array(performance.groups, "performance groups");
|
||||
check(
|
||||
count(performance.groupCount, "performance group count") === groups.length,
|
||||
"group count drift",
|
||||
);
|
||||
|
||||
const recordsByKey = new Map();
|
||||
for (const record of records) {
|
||||
const key = recordKey(record);
|
||||
const matching = recordsByKey.get(key) ?? [];
|
||||
matching.push(record);
|
||||
recordsByKey.set(key, matching);
|
||||
}
|
||||
check(groups.length === recordsByKey.size, "performance groups did not map to records");
|
||||
|
||||
const groupsByKey = new Map();
|
||||
let profiledCount = 0;
|
||||
for (const groupValue of groups) {
|
||||
const group = object(groupValue, "performance group");
|
||||
const identity = [
|
||||
text(group.scenario, "group scenario"),
|
||||
text(group.surface, "group surface"),
|
||||
text(group.state, "group state"),
|
||||
];
|
||||
const key = identity.join("\u0000");
|
||||
check(
|
||||
group.key === identity.join("|") && !groupsByKey.has(key),
|
||||
"performance group identity was invalid",
|
||||
);
|
||||
const matching = recordsByKey.get(key);
|
||||
check(matching?.length > 0, "performance group had no records");
|
||||
check(
|
||||
count(group.sampleCount, "group sample count", { positive: true }) === matching.length,
|
||||
"sample count drift",
|
||||
);
|
||||
exactCounts(group.statuses, statusCounts(matching), "group statuses");
|
||||
const matchingProfiled = matching.filter((record) => record.profiling.enabled === true).length;
|
||||
check(
|
||||
count(group.profiledRunCount, "group profiled count") === matchingProfiled,
|
||||
"profile count drift",
|
||||
);
|
||||
check(
|
||||
group.resourceInterpretation === (matchingProfiled > 0 ? "instrumented" : "normal"),
|
||||
"group resource interpretation was invalid",
|
||||
);
|
||||
profiledCount += matchingProfiled;
|
||||
const metrics = object(group.metrics, "group metrics");
|
||||
Object.entries(metrics).forEach(([id, metric]) =>
|
||||
validateMetric(metric, group.sampleCount, `group metric ${id}`),
|
||||
);
|
||||
const rss = sampledMetric(metrics, RSS_METRICS, group.sampleCount, "group RSS");
|
||||
const cpu = sampledMetric(metrics, CPU_METRICS, group.sampleCount, "group CPU");
|
||||
for (const record of matching) {
|
||||
check(
|
||||
rss.samples.includes(measured(record, RSS_METRICS, "record RSS")),
|
||||
"record RSS was not sampled",
|
||||
);
|
||||
check(
|
||||
cpu.samples.includes(measured(record, CPU_METRICS, "record CPU")),
|
||||
"record CPU was not sampled",
|
||||
);
|
||||
}
|
||||
groupsByKey.set(key, group);
|
||||
}
|
||||
check(
|
||||
count(performance.profiledRunCount, "performance profiled count") === profiledCount,
|
||||
"profile total drift",
|
||||
);
|
||||
const unstableCount = groups.filter((group) =>
|
||||
Object.values(group.metrics).some((metric) => metric.classification === "unstable"),
|
||||
).length;
|
||||
check(
|
||||
count(performance.unstableGroupCount, "unstable group count") === unstableCount,
|
||||
"unstable group count drift",
|
||||
);
|
||||
return groupsByKey;
|
||||
}
|
||||
|
||||
function validateGateCards(gate) {
|
||||
const cards = array(gate.cards, "gate cards");
|
||||
const severities = { blocking: 0, warning: 0, info: 0 };
|
||||
for (const cardValue of cards) {
|
||||
const card = object(cardValue, "gate card");
|
||||
const severity = card.severity;
|
||||
check(Object.hasOwn(severities, severity), "unknown gate card severity");
|
||||
if (severity === "info") {
|
||||
check(
|
||||
(card.kind === "filtered-required-scenario" ||
|
||||
card.kind === "filtered-required-coverage") &&
|
||||
card.status === "MISSING",
|
||||
"unexpected info gate card",
|
||||
);
|
||||
}
|
||||
if (severity === "warning") {
|
||||
check(
|
||||
card.kind === "missing-required-coverage" && card.status === "MISSING",
|
||||
"unexpected warning gate card",
|
||||
);
|
||||
}
|
||||
severities[severity] += 1;
|
||||
}
|
||||
check(
|
||||
count(gate.blockingCount, "blocking count") === severities.blocking,
|
||||
"blocking count drift",
|
||||
);
|
||||
check(count(gate.warningCount, "warning count") === severities.warning, "warning count drift");
|
||||
check(count(gate.infoCount, "info count") === severities.info, "info count drift");
|
||||
check(
|
||||
count(gate.missingRequiredCount, "missing required count") === severities.info,
|
||||
"missing count drift",
|
||||
);
|
||||
return cards;
|
||||
}
|
||||
|
||||
function validateEnvelope(reportValue) {
|
||||
const report = object(reportValue, "report");
|
||||
const gate = object(report.gate, "gate");
|
||||
const controls = object(report.controls, "controls");
|
||||
check(
|
||||
report.schemaVersion === SCHEMA.report && report.mode === "execution",
|
||||
"wrong report schema or mode",
|
||||
);
|
||||
check(
|
||||
gate.schemaVersion === SCHEMA.gate && gate.enabled === true,
|
||||
"wrong or disabled gate schema",
|
||||
);
|
||||
check(controls.gate === true, "gate controls were disabled");
|
||||
const filters = [
|
||||
...array(controls.include, "include filters"),
|
||||
...array(controls.exclude, "exclude filters"),
|
||||
];
|
||||
check(
|
||||
filters.length > 0 &&
|
||||
filters.every((filter) => typeof filter === "string" && filter.trim().length > 0),
|
||||
"report filters were invalid",
|
||||
);
|
||||
const repeat = count(controls.repeat, "repeat", { positive: true });
|
||||
check(
|
||||
gate.partial === true && gate.complete === false && gate.ok === false,
|
||||
"gate metadata was not partial",
|
||||
);
|
||||
const records = array(report.records, "records").map(validateRecord);
|
||||
check(records.length > 0, "report had no records");
|
||||
const cards = validateGateCards(gate);
|
||||
validateBaselines(report, gate);
|
||||
validateSummary(report, records);
|
||||
validateTargetCleanup(report);
|
||||
const groups = validatePerformance(report, records, repeat);
|
||||
return { report, gate, records, cards, groups };
|
||||
}
|
||||
|
||||
function deepProfiled(record) {
|
||||
const profiling = record.profiling;
|
||||
const measurements = record.measurements;
|
||||
return [
|
||||
[profiling.enabled, true],
|
||||
[profiling.deepProfile, true],
|
||||
[profiling.nodeProfile, true],
|
||||
[profiling.heapSnapshot, true],
|
||||
[profiling.diagnosticReport, true],
|
||||
[profiling.profileOnFailure, false],
|
||||
[profiling.affectsResourceMeasurements, true],
|
||||
[profiling.baselineEligible, false],
|
||||
[profiling.interpretation, PROFILED_INTERPRETATION],
|
||||
[measurements.profilingEnabled, true],
|
||||
[measurements.profilingAffectsResourceMeasurements, true],
|
||||
[measurements.profilingBaselineEligible, false],
|
||||
[measurements.profilingResourceInterpretation, PROFILED_INTERPRETATION],
|
||||
].every(([actual, expected]) => actual === expected);
|
||||
}
|
||||
|
||||
function violationMeasurement(record, violation) {
|
||||
const metric = text(violation.metric, "violation metric");
|
||||
if (violation.kind === "threshold" && DIRECT_VIOLATIONS.has(metric)) {
|
||||
return finite(record.measurements[metric], `measurement ${metric}`);
|
||||
}
|
||||
if (violation.kind === "soak" && SOAK_VIOLATIONS.has(metric)) {
|
||||
return finite(record.measurements[metric], `measurement ${metric}`);
|
||||
}
|
||||
const match = violation.kind === "resource" ? ROLE_VIOLATION.exec(metric) : null;
|
||||
check(match !== null && violation.role === match[1], "role violation identity was invalid");
|
||||
const role = object(
|
||||
object(record.measurements.resourceByRole, "role measurements")[match[1]],
|
||||
"role measurement",
|
||||
);
|
||||
return finite(role[match[2]], `role measurement ${metric}`);
|
||||
}
|
||||
|
||||
function validateProfiledFailure(record, card, group) {
|
||||
check(deepProfiled(record), "failed record was not canonical deep profiling");
|
||||
check(
|
||||
group.resourceInterpretation === "instrumented",
|
||||
"failed record group was not instrumented",
|
||||
);
|
||||
const violations = array(record.violations, "record violations");
|
||||
check(violations.length > 0, "failed record had no violations");
|
||||
for (const violationValue of violations) {
|
||||
const violation = object(violationValue, "violation");
|
||||
const actual = finite(violation.actual, "violation actual");
|
||||
check(actual === violationMeasurement(record, violation), "violation evidence drift");
|
||||
if (violation.kind === "threshold") {
|
||||
const metric = text(violation.metric, "violation metric");
|
||||
const samples = validateMetric(
|
||||
object(group.metrics, "group metrics")[metric],
|
||||
group.sampleCount,
|
||||
`violation group metric ${metric}`,
|
||||
);
|
||||
check(samples.includes(actual), "violation was not sampled in its performance group");
|
||||
}
|
||||
text(violation.expected, "violation expectation");
|
||||
text(violation.message, "violation message");
|
||||
}
|
||||
const messages = violations.map((violation) => violation.message);
|
||||
check(
|
||||
card.kind === "openclaw-failure" &&
|
||||
card.status === "FAIL" &&
|
||||
card.failedCommand === null &&
|
||||
card.scenario === record.scenario &&
|
||||
card.state === stateId(record),
|
||||
"blocking card identity was invalid",
|
||||
);
|
||||
check(card.summary === messages[0], "blocking card summary did not match");
|
||||
exactStrings(card.violations, messages, "blocking card violations");
|
||||
const cardMeasurements = object(card.measurements, "blocking card measurements");
|
||||
check(
|
||||
cardMeasurements.peakRssMb === record.measurements.peakRssMb &&
|
||||
cardMeasurements.cpuPercentMax === record.measurements.cpuPercentMax,
|
||||
"blocking card measurements did not match",
|
||||
);
|
||||
}
|
||||
|
||||
function evaluate(evaluator) {
|
||||
try {
|
||||
evaluator();
|
||||
return { ok: true };
|
||||
} catch (error) {
|
||||
return { ok: false, reason: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
export function evaluateToleratedPartialKovaReport(report) {
|
||||
const gate = report?.gate;
|
||||
if (!gate || typeof gate !== "object" || Array.isArray(gate)) {
|
||||
return { ok: false, reason: "missing gate metadata" };
|
||||
}
|
||||
if (gate.verdict !== "PARTIAL") {
|
||||
return { ok: false, reason: `gate verdict was ${JSON.stringify(gate.verdict)}` };
|
||||
}
|
||||
|
||||
const blockingCount = numericCount(gate.blockingCount);
|
||||
if (blockingCount === undefined) {
|
||||
return { ok: false, reason: "missing blocking count" };
|
||||
}
|
||||
if (blockingCount !== 0) {
|
||||
return { ok: false, reason: `blocking count was ${JSON.stringify(gate.blockingCount)}` };
|
||||
}
|
||||
|
||||
const reportBaseline = report?.baseline;
|
||||
const gateBaseline = report?.gate?.baseline;
|
||||
if (
|
||||
(reportBaseline !== null && reportBaseline !== undefined) ||
|
||||
(gateBaseline !== null && gateBaseline !== undefined)
|
||||
) {
|
||||
const baselineRegressionCount = numericCount(
|
||||
reportBaseline?.comparison?.regressionCount ?? gateBaseline?.regressionCount,
|
||||
return evaluate(() => {
|
||||
const { gate, records, cards } = validateEnvelope(report);
|
||||
check(gate.verdict === "PARTIAL", "gate verdict was not PARTIAL");
|
||||
check(gate.blockingCount === 0, "PARTIAL gate had blocking cards");
|
||||
check(
|
||||
cards.every((card) => card.severity !== "blocking"),
|
||||
"PARTIAL gate had a blocking card",
|
||||
);
|
||||
if (baselineRegressionCount === undefined) {
|
||||
return { ok: false, reason: "missing baseline regression count" };
|
||||
check(
|
||||
records.every((record) => record.status === "PASS"),
|
||||
"PARTIAL report had a non-PASS record",
|
||||
);
|
||||
check(
|
||||
records.every((record) => array(record.violations, "record violations").length === 0),
|
||||
"PARTIAL report had violations",
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function evaluateToleratedProfiledKovaReport(report) {
|
||||
return evaluate(() => {
|
||||
const { gate, records, cards, groups } = validateEnvelope(report);
|
||||
check(gate.verdict === "DO_NOT_SHIP", "gate verdict was not DO_NOT_SHIP");
|
||||
check(
|
||||
records.every((record) => record.status === "PASS" || record.status === "FAIL"),
|
||||
"invalid record status",
|
||||
);
|
||||
check(
|
||||
records
|
||||
.filter((record) => record.status === "PASS")
|
||||
.every((record) => array(record.violations, "record violations").length === 0),
|
||||
"PASS record had violations",
|
||||
);
|
||||
const failed = records.filter((record) => record.status === "FAIL");
|
||||
const blocking = cards.filter((card) => card.severity === "blocking");
|
||||
check(
|
||||
failed.length > 0 && blocking.length === failed.length,
|
||||
"failure/card counts did not match",
|
||||
);
|
||||
const remaining = [...blocking];
|
||||
for (const record of failed) {
|
||||
const index = remaining.findIndex(
|
||||
(card) => card.scenario === record.scenario && card.state === stateId(record),
|
||||
);
|
||||
check(index >= 0, "blocking cards did not map one-to-one");
|
||||
const card = remaining.splice(index, 1)[0];
|
||||
validateProfiledFailure(record, card, groups.get(recordKey(record)));
|
||||
}
|
||||
if (baselineRegressionCount !== 0) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: `baseline regression count was ${JSON.stringify(baselineRegressionCount)}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
check(remaining.length === 0, "blocking cards did not map one-to-one");
|
||||
});
|
||||
}
|
||||
|
||||
const statuses = report?.summary?.statuses;
|
||||
if (!statuses || typeof statuses !== "object" || Array.isArray(statuses)) {
|
||||
return { ok: false, reason: "missing status summary" };
|
||||
export function evaluateToleratedKovaReport(report) {
|
||||
const partial = evaluateToleratedPartialKovaReport(report);
|
||||
if (partial.ok) {
|
||||
return { ok: true, classification: "filtered-partial" };
|
||||
}
|
||||
|
||||
const passCount = numericCount(statuses.PASS);
|
||||
if (passCount === undefined || passCount <= 0) {
|
||||
return { ok: false, reason: "status summary had no PASS records" };
|
||||
const profiled = evaluateToleratedProfiledKovaReport(report);
|
||||
if (profiled.ok) {
|
||||
return { ok: true, classification: "profiled-resource-only" };
|
||||
}
|
||||
|
||||
const nonPassStatuses = Object.entries(statuses).filter(
|
||||
([status, count]) => status !== "PASS" && (numericCount(count) ?? 0) > 0,
|
||||
);
|
||||
if (nonPassStatuses.length > 0) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: `non-pass statuses present: ${nonPassStatuses
|
||||
.map(([status, count]) => `${status}=${count}`)
|
||||
.join(", ")}`,
|
||||
};
|
||||
}
|
||||
|
||||
const records = Array.isArray(report?.records) ? report.records : [];
|
||||
if (records.length === 0) {
|
||||
return { ok: false, reason: "missing selected scenario records" };
|
||||
}
|
||||
|
||||
const groups = Array.isArray(report?.performance?.groups) ? report.performance.groups : [];
|
||||
if (groups.length === 0) {
|
||||
return { ok: false, reason: "missing performance groups" };
|
||||
}
|
||||
|
||||
if (!groups.some((group) => hasSampledMetric(group, rssMetricIds))) {
|
||||
return { ok: false, reason: "missing sampled RSS metric in performance groups" };
|
||||
}
|
||||
|
||||
if (!groups.some((group) => hasSampledMetric(group, cpuMetricIds))) {
|
||||
return { ok: false, reason: "missing sampled CPU metric in performance groups" };
|
||||
}
|
||||
|
||||
return { ok: true };
|
||||
return { ok: false, reason: `partial: ${partial.reason}; profiled: ${profiled.reason}` };
|
||||
}
|
||||
|
||||
function readCliReportPath() {
|
||||
|
|
@ -112,13 +552,13 @@ const invokedPath = process.argv[1] ? fs.realpathSync.native(path.resolve(proces
|
|||
|
||||
if (modulePath === invokedPath) {
|
||||
try {
|
||||
const reportPath = readCliReportPath();
|
||||
const report = JSON.parse(fs.readFileSync(reportPath, "utf8"));
|
||||
const result = evaluateToleratedPartialKovaReport(report);
|
||||
const report = JSON.parse(fs.readFileSync(readCliReportPath(), "utf8"));
|
||||
const result = evaluateToleratedKovaReport(report);
|
||||
if (!result.ok) {
|
||||
console.error(`Kova PARTIAL verdict is not tolerable: ${result.reason}`);
|
||||
console.error(`Kova verdict is not tolerable: ${result.reason}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`Tolerated Kova verdict: ${result.classification}`);
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -102,6 +102,9 @@ describe("OpenClaw performance workflow", () => {
|
|||
'node "$PERFORMANCE_HELPER_DIR/scripts/lib/kova-report-gate.mjs" "$report_json"',
|
||||
);
|
||||
expect(runKova.run).not.toContain("report.summary?.statuses ?? {}");
|
||||
expect(runKova.run).toContain(
|
||||
"profiling-affected resource thresholds with no baseline regression",
|
||||
);
|
||||
});
|
||||
|
||||
it("installs local workspace packages beside the OCM root tarball", () => {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue