From ad01434ca01b7ffd4bbac6dcedbcbf83976941ed Mon Sep 17 00:00:00 2001 From: Diego Rodrigues de Sa e Souza Date: Tue, 13 Oct 2026 14:43:21 -0300 Subject: [PATCH] fix(quality): reject incomplete complexity gate outcomes (#16533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cancelled, failed, or incomplete complexity measurements no longer show up as passing release-green metrics. Complete measured drift stays advisory. No baseline or threshold changes. Boarded on release/v3.8.52 at e01340c854aa. Focused tests: 317 passed. File-size kept only the inherited executeTargetAttempt.ts overage (1288 > 1284). This pull request does not touch that file. ⚠️ base-red inherited: #15306 --- .../release-green-complexity-outcomes.md | 1 + scripts/quality/validate-release-green.mjs | 94 +++++++----- .../release-green-complexity-outcomes.test.ts | 139 ++++++++++++++++++ 3 files changed, 198 insertions(+), 36 deletions(-) create mode 100644 changelog.d/fixes/release-green-complexity-outcomes.md create mode 100644 tests/unit/build/release-green-complexity-outcomes.test.ts diff --git a/changelog.d/fixes/release-green-complexity-outcomes.md b/changelog.d/fixes/release-green-complexity-outcomes.md new file mode 100644 index 0000000000..d67c5c1b5e --- /dev/null +++ b/changelog.d/fixes/release-green-complexity-outcomes.md @@ -0,0 +1 @@ +- **fix(quality):** prevent cancelled, failed, or incomplete complexity measurements from appearing as passing release-green metrics; retain advisory classification for complete, measured ratchet drift without changing baselines or CI workflows. diff --git a/scripts/quality/validate-release-green.mjs b/scripts/quality/validate-release-green.mjs index 4f22362339..f00cc5cdd3 100644 --- a/scripts/quality/validate-release-green.mjs +++ b/scripts/quality/validate-release-green.mjs @@ -255,6 +255,59 @@ export function parseCognitiveCount(out) { return m ? Number(m[1]) : null; } +/** Translate the shared complexity walk into the two metric records. */ +export function evaluateComplexityRun({ code, out }, cogBase) { + // The shared walk emits both machine counts followed by both ratchet verdicts. + // Do not use the legacy generic "N violações" fallback here: a partial walk + // containing only the cyclomatic summary would masquerade as cognitive data. + const cogMatch = /^cognitiveComplexity=(\d+)\r?$/m.exec(out); + const cycMatch = /^complexity=(\d+)\r?$/m.exec(out); + const cogSummary = /^\[cognitive-complexity\] (OK|REGRESSÃO) — (\d+) violações /m.exec(out); + const cycSummary = /^\[complexity\] (OK|REGRESSÃO) — (\d+) violações /m.exec(out); + const cogCurrent = cogMatch ? Number(cogMatch[1]) : null; + const cycCurrent = cycMatch ? Number(cycMatch[1]) : null; + const cogOver = isDrift(cogCurrent, cogBase); + const cycRegressed = cycSummary?.[1] === "REGRESSÃO"; + const measured = + [cogCurrent, cycCurrent, cogBase].every((count) => Number.isSafeInteger(count) && count >= 0) && + Number(cogSummary?.[2]) === cogCurrent && + Number(cycSummary?.[2]) === cycCurrent && + (cogSummary?.[1] === "REGRESSÃO") === cogOver; + // Exit 1 is expected only for a complete, measured ratchet regression. A + // cancelled/crashed tool or missing metrics is execution failure, not drift. + const measuredDrift = code === 1 && measured && (cogOver || cycRegressed); + if (!measured || (code !== 0 && !measuredDrift)) { + return [ + { + id: "complexity", + label: "Complexity measurement", + kind: "hard", + ok: false, + detail: + code !== 0 + ? `Complexity process exited ${code}: ${firstFailureLine(out)}` + : "Complexity process did not report complete finite metrics and baseline", + }, + ]; + } + return [ + { + id: "cognitive-complexity", + label: "Cognitive complexity (ratchet)", + kind: "drift", + ok: !cogOver, + detail: `${cogCurrent} vs baseline ${cogBase}${cogOver ? ` (+${cogCurrent - cogBase} drift → rebaseline at release)` : ""}`, + }, + { + id: "complexity", + label: "Cyclomatic complexity (ratchet)", + kind: "drift", + ok: !cycRegressed, + detail: `complexity=${cycCurrent} (shared walk with cognitive)${cycRegressed ? " REGRESSED" : ""}`, + }, + ]; +} + /** * Drift verdict for a ratchet: a metric that grew past its committed baseline is * "drift" (reported, never blocking). `direction:"down"` metrics (warnings, @@ -593,42 +646,11 @@ async function main() { // Complexity + cognitive (one ESLint walk; both still recorded as drift) { announce("Complexity + cognitive ratchets (shared ESLint walk)"); - const { out } = await run(npmCmd, ["run", "check:complexity-ratchets"]); - saveGateLog("complexity-ratchets", out); - const cogCurrent = parseCognitiveCount(out); - const cogBase = baselineValue("cognitiveComplexity"); - const cogOver = isDrift(cogCurrent, cogBase); - const cycMatch = /(?:^|\n)complexity=(\d+)/.exec(out); - const cycOkMatch = /\[complexity\] OK — (\d+)/.exec(out); - const cycRegMatch = /\[complexity\] REGRESSÃO — (\d+)/.exec(out); - const cycCurrent = cycMatch - ? Number(cycMatch[1]) - : cycOkMatch - ? Number(cycOkMatch[1]) - : cycRegMatch - ? Number(cycRegMatch[1]) - : null; - const cycRegressed = /\[complexity\] REGRESSÃO/.test(out); - record({ - id: "cognitive-complexity", - label: "Cognitive complexity (ratchet)", - kind: "drift", - ok: !cogOver, - detail: - cogCurrent == null - ? "could not parse count" - : `${cogCurrent} vs baseline ${cogBase}${cogOver ? ` (+${cogCurrent - cogBase} drift → rebaseline at release)` : ""}`, - }); - record({ - id: "complexity", - label: "Cyclomatic complexity (ratchet)", - kind: "drift", - ok: !cycRegressed, - detail: - cycCurrent == null - ? firstFailureLine(out) || "measured via check:complexity-ratchets" - : `complexity=${cycCurrent} (shared walk with cognitive)${cycRegressed ? " REGRESSED" : ""}`, - }); + const result = await run(npmCmd, ["run", "check:complexity-ratchets"]); + saveGateLog("complexity-ratchets", result.out); + for (const metric of evaluateComplexityRun(result, baselineValue("cognitiveComplexity"))) { + record(metric); + } } // file-size (drift) diff --git a/tests/unit/build/release-green-complexity-outcomes.test.ts b/tests/unit/build/release-green-complexity-outcomes.test.ts new file mode 100644 index 0000000000..491d5b947a --- /dev/null +++ b/tests/unit/build/release-green-complexity-outcomes.test.ts @@ -0,0 +1,139 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runGateProcess } from "../../../scripts/quality/gate-process.mjs"; +import { + computeVerdict, + evaluateComplexityRun, +} from "../../../scripts/quality/validate-release-green.mjs"; + +test("cancelled complexity measurement cannot emit passing metrics", () => { + const results = evaluateComplexityRun( + { code: 130, out: "Gate execution CANCELLED; see command receipt" }, + 100 + ); + assert.equal( + results.some((result) => result.ok), + false + ); + assert.equal(computeVerdict(results).releaseGreen, false); +}); + +function completeOutput(cognitive = 80, cyclomatic = 20, cyclomaticDrift = false) { + return [ + `complexity=${cyclomatic}`, + `cognitiveComplexity=${cognitive}`, + `[complexity] ${cyclomaticDrift ? "REGRESSÃO" : "OK"} — ${cyclomatic} violações ${cyclomaticDrift ? "> baseline 10" : "(baseline 20)"}`, + `[cognitive-complexity] ${cognitive > 100 ? "REGRESSÃO" : "OK"} — ${cognitive} violações ${cognitive > 100 ? "> baseline 100" : "(baseline 100)"}`, + ].join("\n"); +} + +for (const code of [1, 2, 42, 124, 130, 143]) { + test(`complete metrics cannot hide an unexpected process exit ${code}`, () => { + const results = evaluateComplexityRun({ code, out: completeOutput() }, 100); + assert.equal( + results.some((result) => result.ok), + false + ); + assert.equal(computeVerdict(results).releaseGreen, false); + assert.equal(results[0].kind, "hard"); + }); +} + +for (const out of [ + "", + "complexity=20\n[complexity] OK — 20 violações (baseline 20)", + "cognitiveComplexity=80\n[cognitive-complexity] OK — 80 violações (baseline 100)", + "complexity=20\ncognitiveComplexity=80", + completeOutput().replace("cognitiveComplexity=80", "cognitiveComplexity=NaN"), + completeOutput().replace("complexity=20", "complexity=20.5"), + completeOutput().replace("complexity=20", "complexity=20invalid"), + completeOutput().replace("complexity=20", "complexity=-20"), + completeOutput(Number.MAX_SAFE_INTEGER + 1), +]) { + test(`incomplete or invalid output fails closed: ${JSON.stringify(out)}`, () => { + const results = evaluateComplexityRun({ code: 0, out }, 100); + assert.equal( + results.some((result) => result.ok), + false + ); + assert.equal(computeVerdict(results).releaseGreen, false); + }); +} + +test("unknown baseline fails closed instead of assuming no drift", () => { + for (const baseline of [null, NaN, Infinity, -1]) { + const results = evaluateComplexityRun({ code: 0, out: completeOutput() }, baseline); + assert.equal(computeVerdict(results).releaseGreen, false); + } +}); + +test("completed metrics within baseline retain both passing records", () => { + const results = evaluateComplexityRun({ code: 0, out: completeOutput() }, 100); + assert.deepEqual( + results.map(({ id, kind, ok }) => ({ id, kind, ok })), + [ + { id: "cognitive-complexity", kind: "drift", ok: true }, + { id: "complexity", kind: "drift", ok: true }, + ] + ); + assert.equal(computeVerdict(results).releaseGreen, true); +}); + +for (const [cognitive, cyclomaticDrift] of [ + [101, false], + [80, true], + [101, true], +] as const) { + test(`measured drift remains advisory: cognitive=${cognitive}, cyclomatic=${cyclomaticDrift}`, () => { + const results = evaluateComplexityRun( + { code: 1, out: completeOutput(cognitive, 20, cyclomaticDrift) }, + 100 + ); + assert.equal(results.length, 2); + assert.ok(results.every((result) => result.kind === "drift")); + assert.equal(results[0].ok, cognitive <= 100); + assert.equal(results[1].ok, !cyclomaticDrift); + assert.equal(computeVerdict(results).releaseGreen, true); + }); +} + +test("partial regression output cannot disguise a crashed check as advisory drift", () => { + const results = evaluateComplexityRun( + { code: 1, out: "complexity=20\ncognitiveComplexity=101\nError: incomplete walk" }, + 100 + ); + assert.equal(computeVerdict(results).releaseGreen, false); +}); + +test("real subprocess cancellation preserves the receipt and cannot emit a metric PASS", async (t) => { + const directory = mkdtempSync(join(tmpdir(), "release-green-complexity-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const controller = new AbortController(); + const result = await runGateProcess( + process.execPath, + ["-e", "console.log('complexity=20'); setInterval(()=>{},1000)"], + { + cwd: directory, + logPath: join(directory, "complexity.log"), + timeout: 15000, + killGraceMs: 100, + signal: controller.signal, + heartbeatMs: 100, + onHeartbeat: ({ bytes }: { bytes: number }) => { + if (bytes > 0) controller.abort(); + }, + } + ); + assert.equal(result.receipt.outcome, "CANCELLED"); + assert.equal(JSON.parse(readFileSync(result.receiptPath, "utf8")).outcome, "CANCELLED"); + const checks = evaluateComplexityRun(result, 100); + assert.equal( + checks.some((check) => check.ok), + false + ); + const report = JSON.parse(JSON.stringify({ checks, ...computeVerdict(checks) })); + assert.equal(report.releaseGreen, false); +});