From ca0159f1c2ea5087df217430fc608815268ea180 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Fri, 2 Oct 2026 03:02:01 -0700 Subject: [PATCH] fix(release): remove unused flake classification bypass (#163412) * fix(release): remove FRV flake receipts * docs(release): require successful FRV CI gate --- .agents/skills/release-openclaw-ci/SKILL.md | 36 +- .../release-openclaw-maintainer/SKILL.md | 21 +- .../references/regular-release.md | 4 +- .../references/validation.md | 12 +- .../full-release-flake-classification.yml | 56 -- .../full-release-validation.md | 8 +- docs/reference/RELEASING.md | 19 +- .../full-release-validation/continuation.md | 42 +- .../full-release-validation/evidence.md | 7 +- .../full-release-validation/profiles.md | 3 +- scripts/frv.d.mts | 3 +- scripts/frv.mjs | 15 - .../full-release-flake-classification.d.mts | 68 --- scripts/full-release-flake-classification.mjs | 531 ------------------ scripts/full-release-validation-policy.d.mts | 14 +- scripts/full-release-validation-policy.mjs | 154 +---- scripts/full-release-validation-state.d.mts | 4 - scripts/full-release-validation-state.mjs | 16 - scripts/lib/release-publish-gates.mts | 2 +- scripts/release-ci-summary.mjs | 30 +- scripts/render-github-release-notes.mts | 9 +- test/scripts/frv.test.ts | 64 --- .../full-release-flake-classification.test.ts | 429 -------------- ...full-release-publication-admission.test.ts | 1 - .../full-release-validation-state.test.ts | 280 --------- .../package-acceptance-workflow.test.ts | 2 - test/scripts/release-ci-summary.test.ts | 206 ------- test/scripts/release-publish-gates.test.ts | 94 ---- .../render-github-release-notes.test.ts | 95 +--- ...e-full-release-validation-evidence.test.ts | 76 --- 30 files changed, 60 insertions(+), 2241 deletions(-) delete mode 100644 .github/workflows/full-release-flake-classification.yml delete mode 100644 scripts/full-release-flake-classification.d.mts delete mode 100644 scripts/full-release-flake-classification.mjs delete mode 100644 test/scripts/full-release-flake-classification.test.ts diff --git a/.agents/skills/release-openclaw-ci/SKILL.md b/.agents/skills/release-openclaw-ci/SKILL.md index de70a35a7568..e1108dac2c73 100644 --- a/.agents/skills/release-openclaw-ci/SKILL.md +++ b/.agents/skills/release-openclaw-ci/SKILL.md @@ -69,7 +69,7 @@ Use this with `$release-openclaw-maintainer` and `$openclaw-testing` when a rele - Validate provider secrets before dispatching expensive full release matrices. - Check the nightly parent for the Code SHA before dispatching a fresh main validation; it seals per-child receipts that exact-target dispatches adopt when inputs match. The nightly runs this helper route (`--sha --workflow-sha `), so its parent runs on a `release-ci/-` branch, not `main`. - Every selected validation lane must pass except the policy-owned - `windows-node-ci` and authenticated `recorded-flake` classes in FRV's `normalCi` child; see + `windows-node-ci` class in FRV's `normalCi` child; see [Publication requirements](#publication-requirements). Stable tags require stable/full evidence, soak, and blocking performance. Beta-profile evidence cannot qualify stable. No lane or soak waiver bypasses these requirements. All-group @@ -262,7 +262,7 @@ until their dependent enforcement changes land. release branch or beta tag records `coveragePolicy=npm-beta-v1`. It keeps Linux/macOS/Windows Node, Control UI, plugin, package, install/update, Linux/Windows/macOS cross-OS, QA parity, runtime-pair/restart, and tool coverage. - All selected tests except `windows-node-ci` and bound `recorded-flake` jobs gate npm/ClawHub. Native app + All selected tests except `windows-node-ci` jobs gate npm/ClawHub. Native app CI, performance, and published-package Telegram are deferred to confidence. Beta `all` without soak also defers Package Acceptance Telegram, including beta-profile checks of `main`. Record deferred checks as not run, @@ -299,7 +299,7 @@ until their dependent enforcement changes land. `$TMPDIR/openclaw-frv/--reruns.jsonl`. It refuses a passed child, an artifact producer (use `continue --failed`), and a child past its attempt budget: the default 2 allows one rerun; pass `--max-attempts 3` only - for a recorded flake. When a failed consumer binds a green producer's run + for a confirmed flake. When a failed consumer binds a green producer's run attempt (the install-smoke candidate payload, #161317), it reruns that producer job and its dependents instead. Reseal with `continue --failed`. - `pnpm frv continue --failed --run ` reruns each failed child @@ -590,23 +590,10 @@ This is policy-derived, never an operator input or waiver. Ordinary PR, push, scheduled, and main CI keep Windows blocking. Decide blocker or flake for every failed test. Rerun flakes on the same Release -SHA at most twice, file a fix-in-parallel issue/PR on `main`, and record eligible -still-failing `normalCi` jobs through `full-release-flake-classification.yml` on -trusted `main`. The `recorded-flake` receipt binds the parent, child, exact job -attempt, target SHA, actor, reason, and tracking link. Keep that failure visible; -never re-cut, change tooling, or start a new FRV for a flake. After the receipt -succeeds, `frv continue --failed` reseals only the parent when no blockers remain. -See [operator flow](../../../docs/reference/full-release-validation/continuation.md#record-a-flake). - -Other children stay strict in v1; extending classification is follow-up work. -Never classify CI coverage gates, seal/evidence, Build Artifacts, install smoke, -survivor lanes, `update-first-hop-compat*`, pack/npm -qualification, package integrity, Telegram, and Linux/Windows/macOS Gateway -checks, including Windows packaged install/upgrade checks in Release Checks. -A failed CI gate needs at least one recorded flake, every other failed job to be -advisory, and log proof that each non-passing entry is selected and failed. -Matrix display names may differ from gate keys. Skipped, cancelled, missing, -and unknown coverage blocks. No lane or soak waiver applies. +SHA at most twice and file a fix-in-parallel issue/PR on `main`. A selected job +that remains red still blocks publication; never re-cut, change tooling, or +start a new FRV solely to clear a flake. Skipped, cancelled, missing, and unknown +coverage also blocks. No lane or soak waiver applies. ### Publish children @@ -863,8 +850,7 @@ Interpret state precisely: remained active. Read every selected lane's actual conclusion. `passed` requires all selected -validation lanes outside `windows-node-ci` and authenticated `recorded-flake` -jobs to succeed and retains the advisory +validation lanes outside `windows-node-ci` jobs to succeed and retains the advisory failures; omitted coverage is not run, never passed. The `full-release-diagnostics--` artifact is the terminal @@ -885,9 +871,9 @@ run-ID-cached bytes first. them in a clean-home CLI probe, never as a substitute for a required Anthropic API-key lane. 5. For live-cache failures, inspect whether it is missing/invalid key, empty text, provider refusal, timeout, or baseline miss. Do not weaken release gates without clear provider evidence. -6. Decide blocker or flake for each failed test before editing. Flakes use - [recorded classification](#publication-requirements) and a fix on `main`; - classify blockers further: +6. Decide blocker or flake for each failed test before editing. Track a fix for + flakes on `main`; every selected job must still pass before publication. + Classify blockers further: - confirmed product/code failure: fix the release branch, freeze a new Code SHA, and invalidate product evidence - harness, tooling, or source mismatch: keep the Code SHA, fix the smallest diff --git a/.agents/skills/release-openclaw-maintainer/SKILL.md b/.agents/skills/release-openclaw-maintainer/SKILL.md index ca82f41785ec..f77c25b2b25c 100644 --- a/.agents/skills/release-openclaw-maintainer/SKILL.md +++ b/.agents/skills/release-openclaw-maintainer/SKILL.md @@ -33,8 +33,7 @@ failures remain recorded in the decision, GitHub step summary, and release evidence manifest. It is policy-derived, never an operator input or waiver. Ordinary PR, push, scheduled, and main CI keep Windows blocking. -Every other selected validation lane must succeed unless it is a recorded -flake (below): macOS Node and other normal CI jobs, install smoke, survivor +Every other selected validation lane must succeed: macOS Node and other normal CI jobs, install smoke, survivor lanes, `update-first-hop-compat*`, pack/npm qualification, package integrity, and Linux/Windows/macOS Gateway checks, including Windows packaged install/upgrade checks in Release Checks. A cancelled run still blocks. Preserve @@ -52,24 +51,13 @@ elsewhere or on rerun, no relation to the release delta, a runner or infra signature, a history of the same case flaking, or a pre-existing product bug that is not a regression (for example the 2026.9.6 "Assign to…" bug). A real blocker is a regression in shipped bytes or behavior, or an update/install/ -publish defect; fix it on the release branch. A flake never blocks: rerun it on +publish defect; fix it on the release branch. Rerun a flake on the same Release SHA with at most two recorded reruns by default, and file a fix-in-parallel issue or PR on `main` with the evidence. Never re-cut, change -tooling, or start a new FRV for a flake. Main-only failures and infrastructure +tooling, or start a new FRV for a flake. A flake that remains red blocks publication. Main-only failures and infrastructure failures (runner outages, GitHub ghost jobs, hosted-runner offload) count as flakes for the release. -A flake still red after its reruns in an eligible FRV `normalCi` job gets a -`recorded-flake` receipt from the trusted-main -`full-release-flake-classification.yml` workflow (exact job URL, tracking -issue/PR, reason), then the parent decision reruns per the -[CI skill](../release-openclaw-ci/SKILL.md#publication-requirements). The -receipt binds the exact parent run and attempt, job and attempt, and Release -SHA; the failure stays visible in the step summary, manifest, and release notes. -Other children stay strict for now. Never classifiable: the CI gate, -seal/evidence jobs, Build Artifacts, install smoke, survivor lanes, -`update-first-hop-compat*`, pack/npm qualification, and package integrity. - Dependency advisories never delay a release. A newly published advisory is never a reason to re-cut, change tooling, or rerun validation. Record it in the release evidence and handoff, then file or queue the dependency bump on `main` @@ -172,7 +160,6 @@ A passing sibling cannot replace missing required evidence. npm + ClawHub is the priority path. macOS, Windows, Linux, and Android native publication runs in parallel and never gates npm/ClawHub, GitHub release finalization, or main closeout. Selected Windows/macOS Gateway and native-app CI failures block release -validation unless the exact `normalCi` job has a valid `recorded-flake` receipt; -`windows-node-ci` remains policy-advisory. Platform +validation; `windows-node-ci` remains policy-advisory. Platform publishers retain their own artifact and updater contracts; report pending platforms and proof gaps accurately. diff --git a/.agents/skills/release-openclaw-maintainer/references/regular-release.md b/.agents/skills/release-openclaw-maintainer/references/regular-release.md index 03673ce66ad1..d5e1a7d61216 100644 --- a/.agents/skills/release-openclaw-maintainer/references/regular-release.md +++ b/.agents/skills/release-openclaw-maintainer/references/regular-release.md @@ -103,7 +103,7 @@ Record and reuse the full trusted Tooling SHA. Beta-publish uses canonical beta target). Stable-publish requires `release_profile=stable` or `full`, soak, and blocking performance. Beta-profile evidence cannot qualify stable. Every selected validation lane except policy-owned `windows-node-ci` -and authenticated `recorded-flake` jobs in `normalCi` must pass. +jobs in `normalCi` must pass. See [shared release boundaries](../SKILL.md#shared-release-boundaries), [validation](validation.md), and [publication recovery](publication-recovery.md). Diagnose @@ -374,7 +374,7 @@ Run [postpublish confidence](validation.md#postpublish-confidence) against the exact published package. For a beta-to-latest promotion, retain available deferred-lane results, including published-package Telegram, while enforcing the shared required publication proofs. All selected tests outside the -`windows-node-ci` and authenticated `recorded-flake` classes must pass before publication; retain advisory +`windows-node-ci` class must pass before publication; retain advisory failures in the release evidence. Run safe independent rosters concurrently while controlling local Docker/VM load. Classify failures before admitting a fix to the next beta; do not scan moving diff --git a/.agents/skills/release-openclaw-maintainer/references/validation.md b/.agents/skills/release-openclaw-maintainer/references/validation.md index 3f85f3c795ae..cd54d00cfe79 100644 --- a/.agents/skills/release-openclaw-maintainer/references/validation.md +++ b/.agents/skills/release-openclaw-maintainer/references/validation.md @@ -113,13 +113,11 @@ Package Telegram deferral applies to beta-profile `main` too, but it does not qualify for `npm-beta-v1`. FRV `normalCi` Windows Node shards are policy-advisory (`windows-node-ci`). -Eligible `normalCi` failures with authenticated `recorded-flake` receipts are -also advisory; all other selected failures block. Decide blocker or flake for -every failure, rerun flakes on the same Release SHA at most twice, and file a -fix-in-parallel issue/PR on `main`. Do not re-cut, change tooling, or start another -FRV for a flake. See the [CI skill](../../release-openclaw-ci/SKILL.md#publication-requirements). -Other children and package/install/update, artifact, and evidence gates stay -strict; extending classification beyond `normalCi` is follow-up work. +All other selected failures block. Decide blocker or flake for every failure, +rerun flakes on the same Release SHA at most twice, and file a fix-in-parallel +issue/PR on `main`. Do not re-cut, change tooling, or start another FRV solely +to clear a flake; the selected job must still pass before publication. See the +[CI skill](../../release-openclaw-ci/SKILL.md#publication-requirements). Native platform publication remains independent and follows its own gates. All-group cross-OS qualification requires all nine Linux/Windows/macOS install/upgrade pairs. Focused recovery may select individual lanes but does diff --git a/.github/workflows/full-release-flake-classification.yml b/.github/workflows/full-release-flake-classification.yml deleted file mode 100644 index 48fd94ee4595..000000000000 --- a/.github/workflows/full-release-flake-classification.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: FRV Flake Classification -# Actions cannot split the URL or use step outputs in run-name; the script validates it. -run-name: FRV flake classification ${{ inputs.job_url }} - -on: - workflow_dispatch: - inputs: - job_url: - description: Exact failed Normal CI job URL - required: true - type: string - tracking_url: - description: OpenClaw issue or PR tracking the fix on main - required: true - type: string - reason: - description: Single-line flake decision, 20–300 characters - required: true - type: string - -permissions: - contents: read - actions: read - issues: read - pull-requests: read - -jobs: - record: - if: github.ref == 'refs/heads/main' - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - name: Checkout trusted classification tooling - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ github.workflow_sha }} - persist-credentials: false - - - name: Setup Node - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - with: - node-version: "24" - - - name: Record validated flake classification - id: record - env: - GH_TOKEN: ${{ github.token }} - run: node scripts/full-release-flake-classification.mjs record - - - name: Upload classification receipt - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ${{ steps.record.outputs.artifact_name }} - path: ${{ steps.record.outputs.receipt_path }} - retention-days: 90 - if-no-files-found: error diff --git a/docs/ci/release-validation/full-release-validation.md b/docs/ci/release-validation/full-release-validation.md index 330dad689d53..a79703e00d28 100644 --- a/docs/ci/release-validation/full-release-validation.md +++ b/docs/ci/release-validation/full-release-validation.md @@ -67,12 +67,8 @@ stage matrix, exact workflow job names, profile differences, the `npm-beta-v1` and `npm-stable-v1` coverage policies, artifacts, and focused rerun handles. The `normal_ci` child dispatches `ci.yml` with the exact target and release scope, -without `release_gate`. In FRV only, `windows-node-ci` failures and eligible jobs -with authenticated `recorded-flake` receipts are advisory. Receipts bind exact -failed job attempts and retain the reason and fix-in-parallel issue/PR in the -manifest and release notes. Other children stay strict; coverage, package, -install/update, and artifact gates cannot be classified. See -[record a flake](/reference/full-release-validation/continuation#record-a-flake). +without `release_gate`. In FRV only, `windows-node-ci` failures are advisory. +Other failures stay blocking. Complete campaigns (`rerun_group=all`) retain QA Smoke's full scenario profile and Control UI performance independently of changed paths. Docker seed runs all six lanes in every ordinary manual/release scope: diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index 3ae61e49cdad..b092e8c2f735 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -76,22 +76,17 @@ still require Windows shards to pass. Every failed test needs an explicit release-lead decision: blocker or flake. Rerun a flake on the same Release SHA at most twice, file its fix-in-parallel -issue or PR on `main`, and retain the original failure. A still-failing eligible -`normalCi` job can use the authenticated `recorded-flake` classification workflow; -its receipt binds the parent, child run, exact job attempt, Release SHA, reason, -and tracking link. The decision, manifest, and release verification notes retain -the failure. Do not re-cut, change tooling, or start another Full Release -Validation for a flake. See [recorded flakes](/reference/full-release-validation/continuation#record-a-flake). +issue or PR on `main`, and retain the original failure. Do not re-cut, change +tooling, or start another Full Release Validation solely to clear a flake; a +selected job that remains red blocks publication. -Other children stay strict in v1; extending classification to them is follow-up -work. Non-classifiable jobs remain blocking: the CI coverage gate, seal/evidence, +Blocking jobs include the CI coverage gate, seal/evidence, Build Artifacts, install smoke, survivor lanes, `update-first-hop-compat*`, pack/npm qualification, package integrity, and all Linux/Windows/macOS Gateway checks, including Windows packaged install/upgrade checks in Release Checks. A cancelled -run still blocks. A failed CI gate is accepted only when its own log proves that -every non-passing entry selected a failed advisory job; missing, skipped, or -cancelled coverage blocks. Publication waivers cannot bypass failures or required -coverage. Validation covers source CI, packages, plugins, +run still blocks. The selected `openclaw/ci-gate` must succeed; failed, missing, +skipped, or cancelled coverage blocks. Publication waivers cannot bypass failures +or required coverage. Validation covers source CI, packages, plugins, Gateway installs and upgrades, and selected app, UI, Telegram, QA, and live-provider checks. All-group qualification includes all nine Gateway install/upgrade combinations across Linux, Windows, and macOS. Coverage otherwise diff --git a/docs/reference/full-release-validation/continuation.md b/docs/reference/full-release-validation/continuation.md index 9f0ab5d10cae..9fff06e6e3ee 100644 --- a/docs/reference/full-release-validation/continuation.md +++ b/docs/reference/full-release-validation/continuation.md @@ -143,46 +143,10 @@ not declare the current release-isolation contract or the `expected_sha` dispatch input; it never silently substitutes newer tooling. The workflow never creates or updates repository refs itself. -### Record a flake - -Decide explicitly whether each failed test blocks release or is a flake. Rerun -a flake on the same Release SHA at most twice with `frv rerun --job`, and file an -issue or PR tracking its fix on `main`. Do not re-cut the release, change tooling, -or start a new Full Release Validation for it. A flake never blocks publication -once its eligible failure is recorded. - -For a still-failing `normalCi` job, dispatch the classification workflow from -trusted `main`, using the exact job URL from its accepted attempt: - -```bash -gh workflow run full-release-flake-classification.yml --repo openclaw/openclaw --ref main \ - -f job_url='https://github.com/openclaw/openclaw/actions/runs//job/' \ - -f tracking_url='https://github.com/openclaw/openclaw/issues/' \ - -f reason='Describe the observed flake and why the release can proceed.' -``` - -Wait for that classification run to succeed. Its receipt binds the FRV parent, -CI child, Release SHA, accepted job ID/attempt, actor, reason, and tracking issue -or PR. Then run `pnpm frv continue --failed --run `: when all -remaining failures are advisory, it reruns only the parent collector and verifies -the sealed manifest. It does not rerun the classified child. Inspect `frv status` -first if other blockers remain, because `continue --failed` retries them. - -The `recorded-flake` class is limited to `normalCi` in v1. CI coverage gates, -seal/evidence jobs, Build Artifacts, install smoke, survivor, first-hop, pack/npm -qualification, Package Acceptance, and package integrity cannot be classified. -Other children remain strict; extending the scope is follow-up work. A failed -`openclaw/ci-gate` is accepted only when every other failed job is advisory, -at least one has a recorded classification, and its log proves every non-passing -entry is `selected=true` with result `failure`. Matrix job display names may -differ from gate keys. Skipped, cancelled, missing, or unrecognized entries block. -A later rerun creates a different job ID and invalidates the old classification -for that job. - ### Automatic retries for declared flakes -Automatic test retries are disabled. Unclassified failed or timed out jobs -outside `windows-node-ci` remain blockers; `known_flaky_jobs_json` is rejected +Automatic test retries are disabled. Failed or timed out jobs outside +`windows-node-ci` remain blockers; `known_flaky_jobs_json` is rejected on new dispatches. Inspect the original failure before requesting another execution. The explicit `frv rerun` and `frv continue --failed` commands remain operator recovery operations and never run as an automatic response to a test outcome. @@ -190,7 +154,7 @@ operations and never run as an automatic response to a test outcome. Published artifacts may contain empty `knownFlakyJobs` and `automaticRetries` fields. Readers retain their original plan digest and reject nonempty allowances or retry records. Current qualification requires successful selected results -or validated `windows-node-ci`/`recorded-flake` evidence. Retired waivers and +or validated `windows-node-ci` evidence. Retired waivers and pre-declared advisory failure allowances remain rejected and must be replaced with a fresh qualifying run; it cannot authorize publication. diff --git a/docs/reference/full-release-validation/evidence.md b/docs/reference/full-release-validation/evidence.md index b3135a3f67d8..fc8c81d96911 100644 --- a/docs/reference/full-release-validation/evidence.md +++ b/docs/reference/full-release-validation/evidence.md @@ -16,11 +16,7 @@ needed, and one narrow retry, then reassess; do not automatically rerun `all`. Narrow evidence is not publish authorization by itself. Decide blocker or flake for every failed test. Retain `windows-node-ci` advisory -failures and authenticated `recorded-flake` receipts for eligible `normalCi` jobs -in the manifest. Recorded flakes retain their exact job URL/attempt, reason, -tracking issue or PR, classifier run, and CI gate entries; step summaries and -release verification notes expose the decision. Other children stay strict. -See [record a flake](/reference/full-release-validation/continuation#record-a-flake). +failures in the manifest. Every other selected failure blocks publication. Linux, Windows, and macOS Gateway cross-OS install and upgrade lanes are required for beta, stable, and full validation. The manifest records their @@ -63,7 +59,6 @@ limit and fail sealing; evidence is not truncated to fit. ## Workflow files - `.github/workflows/full-release-validation.yml` -- `.github/workflows/full-release-flake-classification.yml` - `.github/workflows/full-release-candidate.yml` - `.github/workflows/openclaw-release-checks.yml` - `.github/workflows/openclaw-live-and-e2e-checks-reusable.yml` diff --git a/docs/reference/full-release-validation/profiles.md b/docs/reference/full-release-validation/profiles.md index 5702d879abce..f204109279d9 100644 --- a/docs/reference/full-release-validation/profiles.md +++ b/docs/reference/full-release-validation/profiles.md @@ -93,8 +93,7 @@ or `packaged-fresh,installer-fresh,packaged-upgrade` are accepted, while any all filter that omits one of the nine Linux/Windows/macOS install and upgrade pairs is rejected before scheduling. All selected cross-OS outcomes block on failure. Every all-group run must retain all nine install/upgrade combinations. Selected lanes must succeed except -`normalCi`'s policy-derived `windows-node-ci` and authenticated `recorded-flake` -jobs; see [record a flake](/reference/full-release-validation/continuation#record-a-flake). +`normalCi`'s policy-derived `windows-node-ci` jobs. Other children stay strict. No operator waiver can authorize publication with failed selected tests. Stable publication requires stable/full evidence, soak, and blocking performance. diff --git a/scripts/frv.d.mts b/scripts/frv.d.mts index 82f6dfa2b47e..20b6d7f1f35f 100644 --- a/scripts/frv.d.mts +++ b/scripts/frv.d.mts @@ -26,7 +26,6 @@ interface FrvReadOptions { export interface FrvClient { repository?: string; - loadFlakeClassifications: typeof import("./full-release-flake-classification.mjs").loadFlakeClassifications; getReleaseEvidenceClient: () => ReturnType< typeof import("./release-ci-summary.mjs").createReleaseEvidenceClient >; @@ -98,7 +97,7 @@ export function watchRelease( ): Promise<{ complete: boolean; statePath: string }>; export function inspectContinuation( plan: Record, - client: Pick, + client: Pick, options?: FrvReadOptions, ): Promise; export function createClient( diff --git a/scripts/frv.mjs b/scripts/frv.mjs index 5f304dee9e25..408aea8f2f30 100644 --- a/scripts/frv.mjs +++ b/scripts/frv.mjs @@ -16,7 +16,6 @@ import process from "node:process"; import { pathToFileURL } from "node:url"; import { promisify, stripVTControlCharacters } from "node:util"; import { validateArtifactProducerRun } from "./full-release-artifacts.mjs"; -import { loadFlakeClassifications } from "./full-release-flake-classification.mjs"; import { publicationAdmissionContract, publicationSourceContract, @@ -808,17 +807,6 @@ export async function inspectContinuation(plan, client, options = {}) { runId: child.runId, status: run.status, }; - if (!active && child.key === "normalCi" && run.conclusion !== "success") { - Object.assign( - policyChild, - await client.loadFlakeClassifications({ - child: policyChild, - parentRunId: plan.parentRunId, - parentRunAttempt: plan.parentRunAttempt, - targetSha: plan.targetSha, - }), - ); - } const passed = !active && terminalPolicyPass(policyChild); return { compositeJobsSha256: evidence.compositeJobsSha256, @@ -907,9 +895,6 @@ export function createClient(repository, dependencies = {}) { }; return { repository, - loadFlakeClassifications(request) { - return loadFlakeClassifications({ ...request, repo: repository }); - }, getReleaseEvidenceClient() { releaseEvidenceClient ??= createReleaseEvidenceClient(repository); return releaseEvidenceClient; diff --git a/scripts/full-release-flake-classification.d.mts b/scripts/full-release-flake-classification.d.mts deleted file mode 100644 index 5147ce0a413a..000000000000 --- a/scripts/full-release-flake-classification.d.mts +++ /dev/null @@ -1,68 +0,0 @@ -export type FlakeClassification = { - schema: "openclaw.frv-flake-classification.v1"; - parentRunId: string; - parentRunAttempt: number; - child: "normalCi"; - childRunId: string; - childRunAttempt: number; - targetSha: string; - jobId: string; - jobName: string; - jobUrl: string; - conclusion: "failure" | "timed_out"; - trackingUrl: string; - reason: string; - classifiedBy: string; - receiptRunId: string; - receiptRunAttempt: number; -}; -export type FlakeJob = { - name: string; - status: string; - conclusion: string; - id?: number | string; - html_url?: string; - url?: string; - acceptedRunAttempt?: number; - run_attempt?: number; -}; -export type FlakeChild = { key: string; runId: string; jobs: FlakeJob[] }; -export type FlakeBinding = { - child?: FlakeChild; - parentRunId?: string; - parentRunAttempt?: number; - targetSha?: string; -}; -export type FlakeGateEntry = { name: string; result: string; selected: boolean | string }; -export type FlakeEvidence = { - flakeClassifications?: FlakeClassification[]; - gateEntries?: FlakeGateEntry[]; -}; -export type FlakeApi = ( - path: string, - options?: { format?: "json" | "text" | "bytes"; maxBytes?: number; signal?: AbortSignal }, -) => Promise; -export const RECORDED_FLAKE_DENIED_JOB_PATTERNS: readonly RegExp[]; -export function isClassifiableFlakeJob(name: unknown): boolean; -export function validateFlakeClassification( - receipt: unknown, - expected?: FlakeBinding, -): FlakeClassification; -export function parseFlakeGateEntries(log: string): FlakeGateEntry[]; -export function validateFlakeGateEntries(entries: unknown): FlakeGateEntry[]; -export function recordFlakeClassification(options: { - inputs: Record; - env?: NodeJS.ProcessEnv; - api?: FlakeApi; -}): Promise; -export function loadFlakeClassifications( - options: FlakeBinding & { - repo?: string; - child: FlakeChild; - parentRunId: string; - parentRunAttempt: number; - targetSha: string; - api?: FlakeApi; - signal?: AbortSignal; - }, -): Promise; diff --git a/scripts/full-release-flake-classification.mjs b/scripts/full-release-flake-classification.mjs deleted file mode 100644 index 3cd930f384b3..000000000000 --- a/scripts/full-release-flake-classification.mjs +++ /dev/null @@ -1,531 +0,0 @@ -#!/usr/bin/env node -import { execFile } from "node:child_process"; -import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; -import { pathToFileURL } from "node:url"; -import { promisify } from "node:util"; -import { inspectActionsArtifactZip, sha256Digest } from "./lib/actions-artifact-archive.mjs"; - -const REPOSITORY = "openclaw/openclaw"; -const WORKFLOW = ".github/workflows/full-release-flake-classification.yml"; -const SCHEMA = "openclaw.frv-flake-classification.v1"; -const RECEIPT_FILE = "frv-flake-classification.json"; -const MAX_BYTES = 1024 * 1024; -const JOB_URL = - /^https:\/\/github\.com\/openclaw\/openclaw\/actions\/runs\/([1-9][0-9]*)\/job\/([1-9][0-9]*)$/u; -const TRACKING_URL = /^https:\/\/github\.com\/openclaw\/openclaw\/(issues|pull)\/([1-9][0-9]*)$/u; -const execFileAsync = promisify(execFile); - -export const RECORDED_FLAKE_DENIED_JOB_PATTERNS = Object.freeze([ - // Policy-advisory windows-node-ci shards need no receipt or receipt lookup. - /^checks-windows-node-/u, - /ci[- _/]gate/iu, - /seal|evidence/iu, - /build[- _]artifacts/iu, - /install[- _]smoke/iu, - /survivor/iu, - // This aggregate owns the published-upgrade-survivor lane. - /^docker-seed-e2e$/iu, - /update[- _]first[- _]hop[- _]compat|first[- _]hop/iu, - /pack[- _]budget|npm[- _]pack|qualify[- _]release[- _]npm/iu, - /package[- _]acceptance|package[- _]integrity/iu, -]); - -export function isClassifiableFlakeJob(name) { - return ( - typeof name === "string" && - name.trim() === name && - name.length > 0 && - !/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u.test(name) && - !RECORDED_FLAKE_DENIED_JOB_PATTERNS.some((pattern) => pattern.test(name)) - ); -} - -function requireValue(condition, message) { - if (!condition) { - throw new Error(`FRV flake classification: ${message}`); - } -} - -function id(value) { - return typeof value === "string" && /^[1-9][0-9]*$/u.test(value); -} - -function attempt(value) { - return Number.isSafeInteger(value) && value > 0; -} - -function reasonValid(value) { - return ( - typeof value === "string" && - value.trim() === value && - value.length >= 20 && - value.length <= 300 && - !/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u.test(value) - ); -} - -function receiptBinding(receipt, { child, parentRunId, parentRunAttempt, targetSha } = {}) { - requireValue( - receipt && typeof receipt === "object" && !Array.isArray(receipt), - "invalid receipt", - ); - const keys = [ - "schema", - "parentRunId", - "parentRunAttempt", - "child", - "childRunId", - "childRunAttempt", - "targetSha", - "jobId", - "jobName", - "jobUrl", - "conclusion", - "trackingUrl", - "reason", - "classifiedBy", - "receiptRunId", - "receiptRunAttempt", - ]; - requireValue( - Object.keys(receipt).length === keys.length && keys.every((key) => Object.hasOwn(receipt, key)), - "receipt schema keys differ", - ); - requireValue( - receipt.schema === SCHEMA && receipt.child === "normalCi", - "invalid receipt schema or child", - ); - requireValue( - [receipt.parentRunId, receipt.childRunId, receipt.jobId, receipt.receiptRunId].every(id) && - [receipt.parentRunAttempt, receipt.childRunAttempt, receipt.receiptRunAttempt].every(attempt), - "invalid receipt identity", - ); - const url = typeof receipt.jobUrl === "string" ? JOB_URL.exec(receipt.jobUrl) : null; - requireValue( - url?.[1] === receipt.childRunId && url?.[2] === receipt.jobId, - "receipt job URL differs", - ); - requireValue( - typeof receipt.targetSha === "string" && /^[a-f0-9]{40}$/u.test(receipt.targetSha), - "invalid target SHA", - ); - requireValue(isClassifiableFlakeJob(receipt.jobName), "job cannot be classified"); - requireValue(["failure", "timed_out"].includes(receipt.conclusion), "job is not failed"); - requireValue( - typeof receipt.trackingUrl === "string" && TRACKING_URL.test(receipt.trackingUrl), - "invalid tracking URL", - ); - requireValue(reasonValid(receipt.reason), "reason must be a single line of 20–300 characters"); - requireValue( - typeof receipt.classifiedBy === "string" && - /^[A-Za-z0-9][A-Za-z0-9-]*(?:\[bot\])?$/u.test(receipt.classifiedBy), - "invalid triggering actor", - ); - requireValue( - !child || (child.key === "normalCi" && String(child.runId) === receipt.childRunId), - "receipt child run differs", - ); - requireValue( - parentRunId === undefined || receipt.parentRunId === String(parentRunId), - "receipt parent run differs", - ); - requireValue( - parentRunAttempt === undefined || receipt.parentRunAttempt === parentRunAttempt, - "receipt parent attempt differs", - ); - requireValue( - targetSha === undefined || receipt.targetSha === targetSha, - "receipt target SHA differs", - ); - return receipt; -} - -export function validateFlakeClassification(receipt, expected = {}) { - receiptBinding(receipt, expected); - if (expected.child) { - const matches = expected.child.jobs.filter((job) => job.name === receipt.jobName); - const job = matches[0]; - requireValue( - matches.length === 1 && - job.status === "completed" && - job.conclusion === receipt.conclusion && - (job.html_url ?? job.url) === receipt.jobUrl && - (job.id === undefined || String(job.id) === receipt.jobId) && - (job.acceptedRunAttempt ?? job.run_attempt) === receipt.childRunAttempt, - "receipt does not match the accepted failed job", - ); - } - return receipt; -} - -function lines(log) { - requireValue( - typeof log === "string" && Buffer.byteLength(log) <= MAX_BYTES, - "job log exceeds its bound", - ); - return log - .split(/\r?\n/u) - .map((line) => line.replace(/^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z /u, "")); -} - -export function validateFlakeGateEntries(entries) { - requireValue( - Array.isArray(entries) && - entries.length <= 100 && - entries.every( - (entry) => - entry && - typeof entry === "object" && - Object.keys(entry).length === 3 && - typeof entry.name === "string" && - typeof entry.result === "string" && - /^[A-Za-z0-9_-]+$/u.test(entry.name) && - /^[A-Za-z0-9_-]*$/u.test(entry.result) && - (typeof entry.selected === "boolean" || - (typeof entry.selected === "string" && entry.selected.length <= 30)), - ) && - entries[0]?.name === "preflight" && - entries.at(-1)?.name === "pr-fail-fast" && - new Set(entries.map((entry) => entry.name)).size === entries.length, - "CI gate log entries are missing, duplicated, or incomplete", - ); - return entries; -} - -export function parseFlakeGateEntries(log) { - const entries = lines(log).flatMap((line) => { - if (!/^[A-Za-z0-9_-]+: .*\(selected\b/u.test(line)) { - return []; - } - const match = /^([A-Za-z0-9_-]+): (.*) \(selected=([^()]*)\)$/u.exec(line); - requireValue(match, "CI gate log entry is malformed"); - return [ - { - name: match[1], - result: match[2], - selected: match[3] === "true" ? true : match[3] === "false" ? false : match[3], - }, - ]; - }); - return validateFlakeGateEntries(entries); -} - -async function githubApi(path, { format = "json", maxBytes = MAX_BYTES, signal } = {}) { - const { stdout } = await execFileAsync("gh", ["api", `repos/${REPOSITORY}/${path}`], { - encoding: format === "bytes" ? null : "utf8", - maxBuffer: maxBytes, - timeout: 60_000, - killSignal: "SIGKILL", - signal, - }); - return format === "json" ? JSON.parse(stdout) : stdout; -} - -async function pages(api, path, key, signal) { - const values = []; - for (let page = 1; page <= 10; page++) { - const response = await api( - `${path}${path.includes("?") ? "&" : "?"}per_page=100&page=${page}`, - { signal }, - ); - const batch = response[key]; - requireValue( - Array.isArray(batch) && - batch.length <= 100 && - Number.isSafeInteger(response.total_count) && - response.total_count <= 1000, - "API enumeration exceeds its bound", - ); - values.push(...batch); - if (values.length === response.total_count) { - requireValue( - new Set(values.map((value) => value.id)).size === values.length, - "API enumeration is duplicated", - ); - return values; - } - requireValue( - batch.length === 100 && values.length < response.total_count, - "API enumeration is incomplete", - ); - } - throw new Error("FRV flake classification: API enumeration exceeds its bound"); -} - -function runIdentity(run, path) { - requireValue( - run?.repository?.full_name === REPOSITORY && - typeof run.path === "string" && - run.path.split("@", 1)[0] === path && - run.event === "workflow_dispatch", - "workflow identity differs", - ); -} - -export async function recordFlakeClassification({ inputs, env = process.env, api = githubApi }) { - requireValue( - env.GITHUB_REPOSITORY === REPOSITORY && - env.GITHUB_REF === "refs/heads/main" && - env.GITHUB_EVENT_NAME === "workflow_dispatch" && - env.GITHUB_WORKFLOW_REF === `${REPOSITORY}/${WORKFLOW}@refs/heads/main` && - /^[a-f0-9]{40}$/u.test(env.GITHUB_WORKFLOW_SHA) && - env.GITHUB_SHA === env.GITHUB_WORKFLOW_SHA, - "recording requires trusted main workflow", - ); - const match = JOB_URL.exec(inputs.job_url); - requireValue(match, "invalid job URL"); - const tracking = TRACKING_URL.exec(inputs.tracking_url); - requireValue(tracking, "invalid tracking URL"); - requireValue( - typeof inputs.reason === "string" && !/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u.test(inputs.reason), - "reason must be a single line of 20–300 characters", - ); - const reason = inputs.reason.trim(); - requireValue(reasonValid(reason), "reason must be a single line of 20–300 characters"); - const [, childRunId, jobId] = match; - const job = await api(`actions/jobs/${jobId}`); - requireValue( - String(job.id) === jobId && - String(job.run_id) === childRunId && - job.html_url === inputs.job_url && - job.status === "completed" && - ["failure", "timed_out"].includes(job.conclusion), - "job is not the requested completed failure", - ); - requireValue(isClassifiableFlakeJob(job.name), "job cannot be classified"); - const child = await api(`actions/runs/${childRunId}`); - runIdentity(child, ".github/workflows/ci.yml"); - const parentMatch = /^CI full-release-validation-([1-9][0-9]*)-([1-9][0-9]*)-ci$/u.exec( - child.display_title, - ); - requireValue(String(child.id) === childRunId && parentMatch, "CI dispatch title differs"); - const [, parentRunId, parentAttempt] = parentMatch; - const parentRunAttempt = Number(parentAttempt); - const parent = await api(`actions/runs/${parentRunId}/attempts/${parentRunAttempt}`); - runIdentity(parent, ".github/workflows/full-release-validation.yml"); - requireValue( - String(parent.id) === parentRunId && - parent.run_attempt === parentRunAttempt && - parent.head_sha === child.head_sha, - "parent run identity differs", - ); - const jobs = await pages( - api, - `actions/runs/${parentRunId}/attempts/${parentRunAttempt}/jobs`, - "jobs", - ); - const dispatchJobs = jobs.filter((entry) => entry.name === "Run normal full CI"); - const dispatch = dispatchJobs[0]; - requireValue( - dispatchJobs.length === 1 && - dispatch.status === "completed" && - dispatch.conclusion === "success" && - dispatch.run_attempt === parentRunAttempt, - "parent CI dispatch job is not uniquely successful", - ); - const dispatchLines = lines(await api(`actions/jobs/${dispatch.id}/logs`, { format: "text" })); - const targets = dispatchLines.flatMap( - (line) => /^\s+TARGET_SHA: ([a-f0-9]{40})$/u.exec(line)?.slice(1) ?? [], - ); - const witnesses = dispatchLines.filter((line) => line.startsWith("Dispatched ci.yml: ")); - requireValue( - targets.length === 1 && - witnesses.length === 1 && - new RegExp( - `^Dispatched ci\\.yml: https://github\\.com/openclaw/openclaw/actions/runs/${childRunId} \\(attempt [1-9][0-9]*\\)$`, - "u", - ).test(witnesses[0]), - "parent target SHA or dispatch witness differs", - ); - const tracked = await api(`issues/${tracking[2]}`); - requireValue( - String(tracked.number) === tracking[2] && - Boolean(tracked.pull_request) === (tracking[1] === "pull"), - "tracking issue or PR differs", - ); - const receipt = { - schema: SCHEMA, - parentRunId, - parentRunAttempt, - child: "normalCi", - childRunId, - childRunAttempt: job.run_attempt, - targetSha: targets[0], - jobId, - jobName: job.name, - jobUrl: job.html_url, - conclusion: job.conclusion, - trackingUrl: inputs.tracking_url, - reason, - classifiedBy: env.GITHUB_TRIGGERING_ACTOR, - receiptRunId: env.GITHUB_RUN_ID, - receiptRunAttempt: Number(env.GITHUB_RUN_ATTEMPT), - }; - const producer = await api(`actions/runs/${receipt.receiptRunId}`); - runIdentity(producer, WORKFLOW); - requireValue( - String(producer.id) === receipt.receiptRunId && - producer.run_attempt === receipt.receiptRunAttempt && - producer.head_branch === "main" && - producer.head_sha === env.GITHUB_WORKFLOW_SHA && - producer.triggering_actor?.login === receipt.classifiedBy && - producer.display_title === `FRV flake classification ${receipt.jobUrl}`, - "receipt producer identity differs", - ); - return validateFlakeClassification(receipt, { - child: { key: "normalCi", runId: childRunId, jobs: [job] }, - }); -} - -export async function loadFlakeClassifications({ - repo = REPOSITORY, - child, - parentRunId, - parentRunAttempt, - targetSha, - api = githubApi, - signal, -}) { - requireValue(repo === REPOSITORY, "repository differs"); - if ( - child.key !== "normalCi" || - !child.jobs.some( - (job) => - job.status === "completed" && - ["failure", "timed_out"].includes(job.conclusion) && - isClassifiableFlakeJob(job.name), - ) - ) { - return {}; - } - requireValue( - id(parentRunId) && attempt(parentRunAttempt) && /^[a-f0-9]{40}$/u.test(targetSha), - "loader requires exact parent and candidate bindings", - ); - // Receipts postdate their child run; scoping to its lifetime keeps unrelated history out of the bound. - const childRun = await api(`actions/runs/${child.runId}`, { signal }); - requireValue( - String(childRun.id) === String(child.runId) && - typeof childRun.created_at === "string" && - /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ$/u.test(childRun.created_at), - "CI child run identity differs", - ); - const runs = await pages( - api, - `actions/workflows/full-release-flake-classification.yml/runs?event=workflow_dispatch&branch=main&status=success&created=%3E%3D${childRun.created_at}`, - "workflow_runs", - signal, - ); - const prefix = `FRV flake classification https://github.com/${REPOSITORY}/actions/runs/${child.runId}/job/`; - const receipts = new Map(); - for (const listed of runs.filter((run) => String(run.display_title).startsWith(prefix))) { - const run = await api(`actions/runs/${listed.id}`, { signal }); - runIdentity(run, WORKFLOW); - requireValue( - run.id === listed.id && - run.head_branch === "main" && - run.status === "completed" && - run.conclusion === "success" && - /^\d+$/u.test(run.display_title.slice(prefix.length)) && - run.display_title.startsWith(prefix) && - typeof run.head_sha === "string" && - /^[a-f0-9]{40}$/u.test(run.head_sha), - "receipt workflow run differs", - ); - // head_branch cannot tell a main branch dispatch from a same-named tag; require main lineage. - const lineage = await api(`compare/${run.head_sha}...main?per_page=1`, { signal }); - requireValue( - ["ahead", "identical"].includes(lineage?.status) && - lineage.merge_base_commit?.sha === run.head_sha, - "receipt workflow revision is not a main ancestor", - ); - const artifacts = await api(`actions/runs/${run.id}/artifacts?per_page=100`, { signal }); - requireValue( - artifacts.total_count === 1 && artifacts.artifacts?.length === 1, - "receipt run must have one artifact", - ); - const artifact = artifacts.artifacts[0]; - const jobId = run.display_title.slice(prefix.length); - requireValue( - artifact.name === `frv-flake-classification-${child.runId}-${jobId}` && - artifact.expired === false && - String(artifact.workflow_run?.id) === String(run.id) && - Number.isSafeInteger(artifact.size_in_bytes) && - artifact.size_in_bytes > 0 && - artifact.size_in_bytes <= MAX_BYTES, - "receipt artifact identity differs", - ); - const bytes = await api(`actions/artifacts/${artifact.id}/zip`, { - format: "bytes", - maxBytes: MAX_BYTES, - signal, - }); - requireValue( - bytes.length === artifact.size_in_bytes && sha256Digest(bytes) === artifact.digest, - "receipt artifact digest differs", - ); - const files = inspectActionsArtifactZip(bytes, [RECEIPT_FILE], { - maxArchiveBytes: MAX_BYTES, - maxExpandedBytes: MAX_BYTES, - }); - const receipt = receiptBinding(JSON.parse(files.get(RECEIPT_FILE).toString("utf8")), { - child, - parentRunId, - parentRunAttempt, - targetSha, - }); - requireValue( - receipt.receiptRunId === String(run.id) && - receipt.receiptRunAttempt === run.run_attempt && - receipt.jobId === jobId && - receipt.classifiedBy === run.triggering_actor?.login, - "receipt producer differs", - ); - // A rerun executes a new job ID; its predecessor's authenticated receipt is historical only. - if (!child.jobs.some((job) => (job.html_url ?? job.url) === receipt.jobUrl)) { - continue; - } - validateFlakeClassification(receipt, { child, parentRunId, parentRunAttempt, targetSha }); - const previous = receipts.get(receipt.jobId); - if (!previous || BigInt(receipt.receiptRunId) > BigInt(previous.receiptRunId)) { - receipts.set(receipt.jobId, receipt); - } - } - if (receipts.size === 0) { - return {}; - } - const flakeClassifications = [...receipts.values()].toSorted((left, right) => - left.jobName === right.jobName ? 0 : left.jobName < right.jobName ? -1 : 1, - ); - const gates = child.jobs.filter((job) => job.name === "openclaw/ci-gate"); - let gateEntries; - if (gates.length === 1 && gates[0].status === "completed" && gates[0].conclusion === "failure") { - const gate = gates[0]; - const match = JOB_URL.exec(gate.html_url ?? gate.url); - requireValue(match?.[1] === String(child.runId), "CI gate URL differs"); - gateEntries = parseFlakeGateEntries( - await api(`actions/jobs/${match[2]}/logs`, { format: "text", maxBytes: MAX_BYTES, signal }), - ); - } - return { flakeClassifications, ...(gateEntries ? { gateEntries } : {}) }; -} - -if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { - try { - requireValue( - process.argv[2] === "record", - "usage: full-release-flake-classification.mjs record", - ); - const event = JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, "utf8")); - const receipt = await recordFlakeClassification({ inputs: event.inputs }); - const path = `${process.env.RUNNER_TEMP}/${RECEIPT_FILE}`; - writeFileSync(path, `${JSON.stringify(receipt, null, 2)}\n`); - appendFileSync( - process.env.GITHUB_OUTPUT, - `receipt_path=${path}\nartifact_name=frv-flake-classification-${receipt.childRunId}-${receipt.jobId}\n`, - ); - } catch (error) { - console.error(error instanceof Error ? error.message : String(error)); - process.exitCode = 1; - } -} diff --git a/scripts/full-release-validation-policy.d.mts b/scripts/full-release-validation-policy.d.mts index 54654eed0323..333cc1a5f0ed 100644 --- a/scripts/full-release-validation-policy.d.mts +++ b/scripts/full-release-validation-policy.d.mts @@ -12,18 +12,7 @@ interface ReleaseAdvisoryJobBase { runId: string; url: string; } -export type ReleaseAdvisoryJob = ReleaseAdvisoryJobBase & - ( - | { class: "windows-node-ci" } - | { - class: "recorded-flake"; - jobId: string; - trackingUrl: string; - reason: string; - receiptRunId: string; - } - ); -export function releaseChildClassificationEvidence(child: ReleaseRecord): ReleaseRecord; +export type ReleaseAdvisoryJob = ReleaseAdvisoryJobBase & { class: "windows-node-ci" }; export function releaseAdvisoryJobs(children: ReleaseRecord[]): ReleaseAdvisoryJob[]; export function validateReleaseManifestAdvisoryJobs(manifest: unknown): ReleaseAdvisoryJob[]; export const SPLIT_CHANGELOG_EVIDENCE_REUSE_POLICY: "split-changelog-release-v1"; @@ -183,7 +172,6 @@ export function selectReleaseStateArtifacts( }; }; export function formatReleaseStateOutcome(payload: ReleaseRecord): string; -export function releaseStateChildEvidence(child: ReleaseRecord): ReleaseRecord; export function affectedActiveRunIds( children: ReleaseRecord[], blockers: ReleaseRecord[], diff --git a/scripts/full-release-validation-policy.mjs b/scripts/full-release-validation-policy.mjs index f72b437588e6..b3293f065ed2 100644 --- a/scripts/full-release-validation-policy.mjs +++ b/scripts/full-release-validation-policy.mjs @@ -4,10 +4,6 @@ import { validateFullReleaseCandidateRequest, validateRecordedFullReleaseCandidateRequest, } from "./full-release-candidate-contract.mjs"; -import { - validateFlakeClassification, - validateFlakeGateEntries, -} from "./full-release-flake-classification.mjs"; import { FULL_RELEASE_PUBLICATION_ADMISSION_CONTRACT, FULL_RELEASE_SOURCE_ADMISSION_CONTRACT, @@ -44,46 +40,23 @@ function isWindowsNodeAdvisoryJob(child, job) { ); } -function recordedFlakeReceipt(child, job) { - return child.flakeClassifications?.find((receipt) => { - if (receipt.jobName !== job.name || receipt.jobUrl !== (job.html_url ?? job.url)) { - return false; - } - try { - validateFlakeClassification(receipt, { child: { ...child, jobs: [job] } }); - return true; - } catch { - return false; - } - }); -} - function isAdvisoryJob(child, job) { - return isWindowsNodeAdvisoryJob(child, job) || Boolean(recordedFlakeReceipt(child, job)); + return isWindowsNodeAdvisoryJob(child, job); } export function releaseAdvisoryJobs(children) { return children.flatMap((child) => child.jobs.flatMap((job) => { const windows = isWindowsNodeAdvisoryJob(child, job); - const receipt = windows ? undefined : recordedFlakeReceipt(child, job); - return windows || receipt + return windows ? [ { - class: windows ? WINDOWS_NODE_CI_ADVISORY.id : "recorded-flake", + class: WINDOWS_NODE_CI_ADVISORY.id, child: child.key, job: job.name, conclusion: job.conclusion, runId: child.runId, url: job.html_url ?? job.url ?? "", - ...(receipt - ? { - jobId: receipt.jobId, - trackingUrl: receipt.trackingUrl, - reason: receipt.reason, - receiptRunId: receipt.receiptRunId, - } - : {}), }, ] : []; @@ -91,50 +64,6 @@ export function releaseAdvisoryJobs(children) { ); } -export function releaseChildClassificationEvidence(child) { - return child.flakeClassifications === undefined && child.gateEntries === undefined - ? {} - : { - flakeClassifications: child.flakeClassifications ?? [], - gateEntries: child.gateEntries ?? [], - status: child.status, - conclusion: child.conclusion, - }; -} - -function validateChildClassificationEvidence(child, binding) { - if (child.flakeClassifications !== undefined) { - if (!Array.isArray(child.flakeClassifications)) { - throw new Error("Release flake classifications are invalid"); - } - if ( - child.flakeClassifications.length && - (!/^[1-9][0-9]*$/u.test(String(binding.parentRunId ?? "")) || - positiveInteger(binding.parentRunAttempt) === undefined || - !/^[a-f0-9]{40}$/u.test(String(binding.targetSha ?? ""))) - ) { - throw new Error("Release flake classification parent binding is invalid"); - } - const jobs = new Set(); - for (const receipt of child.flakeClassifications) { - validateFlakeClassification(receipt, { ...binding, child }); - if (jobs.has(receipt.jobId)) { - throw new Error("Release flake classifications repeat a job"); - } - jobs.add(receipt.jobId); - } - } - if (child.gateEntries !== undefined) { - if (!Array.isArray(child.gateEntries)) { - throw new Error("Release CI gate entries are invalid"); - } - if (child.gateEntries.length) { - validateFlakeGateEntries(child.gateEntries); - } - } - return releaseChildClassificationEvidence(child); -} - function validateReleaseAdvisoryJobs(value, children) { const expected = releaseAdvisoryJobs(children); const recorded = value === undefined ? [] : value; @@ -168,16 +97,7 @@ export function validateReleaseManifestAdvisoryJobs(manifest) { ) { throw new Error("Release advisory child run differs from the manifest"); } - const snapshot = Object.assign({}, child, { key }); - validateChildClassificationEvidence(snapshot, { - parentRunId: manifest.runId, - parentRunAttempt: manifest.sourceParentRunAttempt, - targetSha: manifest.targetSha, - }); - if (child.flakeClassifications?.length && !terminalPolicyPass(snapshot)) { - throw new Error("Release recorded flake evidence does not pass terminal policy"); - } - return snapshot; + return Object.assign({}, child, { key }); }); return validateReleaseAdvisoryJobs(manifest.advisoryJobs, children); } @@ -187,7 +107,6 @@ export function buildReleaseValidationManifest({ plan, drain, context }) { Object.entries(drain?.children ?? {}).map(([key, child]) => [ key, { - ...releaseChildClassificationEvidence(child), runId: child.runId, plannedRunAttempt: child.plannedRunAttempt, effectiveRunAttempt: child.runAttempt, @@ -1689,46 +1608,9 @@ function isFailedJob(job) { ); } -function recordedFlakeGatePass(child) { - const gates = child.jobs.filter((job) => job.name === "openclaw/ci-gate"); - const advisoryJobs = child.jobs.filter((job) => isAdvisoryJob(child, job)); - const entries = child.gateEntries; - if ( - gates.length !== 1 || - child.jobs.some((job) => job.status !== "completed") || - gates[0].conclusion !== "failure" || - !advisoryJobs.some((job) => recordedFlakeReceipt(child, job)) || - child.jobs.some((job) => isFailedJob(job) && job !== gates[0] && !isAdvisoryJob(child, job)) || - !Array.isArray(entries) - ) { - return false; - } - try { - validateFlakeGateEntries(entries); - } catch { - return false; - } - const nonPassing = entries.filter( - (entry) => - !((entry.selected === true || entry.selected === false) && entry.result === "success") && - !(entry.selected === false && entry.result === "skipped"), - ); - // Every failed job is advisory, so failure entries can only come from those jobs. - // Any other non-passing entry means lost coverage; matrix display names may differ. - return ( - nonPassing.length > 0 && - nonPassing.every((entry) => entry.selected === true && entry.result === "failure") - ); -} - export function terminalPolicyPass(child) { const failures = child.jobs.filter(isFailedJob); - const gatePass = recordedFlakeGatePass(child); - const advisoryOnly = - failures.length > 0 && - failures.every( - (job) => isAdvisoryJob(child, job) || (gatePass && job.name === "openclaw/ci-gate"), - ); + const advisoryOnly = failures.length > 0 && failures.every((job) => isAdvisoryJob(child, job)); const gates = child.jobs.filter((job) => job.name === "openclaw/ci-gate"); return ( child.status === "completed" && @@ -1736,7 +1618,6 @@ export function terminalPolicyPass(child) { (failures.length === 0 || advisoryOnly) && // Selected-vs-skipped coverage comes from the successful gate or its exact log. (!advisoryOnly || - gatePass || (gates.length === 1 && gates[0].status === "completed" && gates[0].conclusion === "success")) ); } @@ -1829,12 +1710,7 @@ export function classifyReleaseSnapshot({ ); const childJobBlockers = selected.flatMap((child) => child.jobs - .filter( - (job) => - isFailedJob(job) && - !isAdvisoryJob(child, job) && - !(job.name === "openclaw/ci-gate" && recordedFlakeGatePass(child)), - ) + .filter((job) => isFailedJob(job) && !isAdvisoryJob(child, job)) .map((job) => ({ child: child.key, conclusion: job.conclusion, @@ -2006,7 +1882,6 @@ export function buildReleaseStateArtifact({ .map((child) => [ child.key, { - ...releaseChildClassificationEvidence(child), compositeJobsSha256: boundedString(child.compositeJobsSha256, MAX_LABEL_LENGTH), conclusion: stringValue(child.conclusion), dispatchActor: boundedString(child.dispatchActor, MAX_LABEL_LENGTH), @@ -2354,14 +2229,6 @@ export function validateReleaseStateArtifact(payload, expected, expectedMode) { return [ key, { - ...validateChildClassificationEvidence( - { ...child, key, jobs: timingJobs }, - { - parentRunId: payload.parentRunId, - parentRunAttempt: payload.sourceParentRunAttempt, - targetSha: payload.targetSha, - }, - ), compositeJobsSha256: boundedString(child.compositeJobsSha256, MAX_LABEL_LENGTH), conclusion: stringValue(child.conclusion), dispatchActor: boundedString(child.dispatchActor, MAX_LABEL_LENGTH), @@ -2426,9 +2293,8 @@ export function releasePlanGateFailures(gates) { })); } -export function releaseStateChildEvidence(child) { +function releaseStateChildEvidence(child) { return canonicalValue({ - ...releaseChildClassificationEvidence(child), compositeJobsSha256: child.compositeJobsSha256, conclusion: child.conclusion, dispatchActor: child.dispatchActor, @@ -2753,11 +2619,7 @@ function releaseStateDetailLines(payload, maxItems = MAX_SUMMARY_ISSUES) { lines.push(issueSummary("Collector error", error)); } for (const advisory of payload.advisoryJobs ?? []) { - lines.push( - advisory.class === "recorded-flake" - ? `${issueSummary("Advisory [recorded-flake]", { ...advisory, job: `${advisory.child}/${advisory.job}` })} — ${advisory.reason} ${advisory.trackingUrl}` - : issueSummary(`Advisory [${advisory.class}]`, advisory), - ); + lines.push(issueSummary(`Advisory [${advisory.class}]`, advisory)); } const omitted = Math.max(0, payload.blockers.length - normalizedMax) + diff --git a/scripts/full-release-validation-state.d.mts b/scripts/full-release-validation-state.d.mts index ef03c2b67473..5a3094ace2f3 100644 --- a/scripts/full-release-validation-state.d.mts +++ b/scripts/full-release-validation-state.d.mts @@ -10,10 +10,6 @@ export function readChild( previous: ReleaseRecord | undefined, signal?: AbortSignal, options?: { - parentRunId?: string; - parentRunAttempt?: number; - targetSha?: string; - loadFlakeClassifications?: typeof import("./full-release-flake-classification.mjs").loadFlakeClassifications; reuseSelection?: { runAttempt: number }; readAttemptJobs?: ( runId: string, diff --git a/scripts/full-release-validation-state.mjs b/scripts/full-release-validation-state.mjs index abfd07a1cdd9..16e6111f4fd8 100644 --- a/scripts/full-release-validation-state.mjs +++ b/scripts/full-release-validation-state.mjs @@ -19,7 +19,6 @@ import { validateFullReleaseCandidateRequest, validateRecordedFullReleaseCandidateRequest, } from "./full-release-candidate-contract.mjs"; -import { loadFlakeClassifications } from "./full-release-flake-classification.mjs"; import { createPublicationAdmission, publicationObservationJson, @@ -39,7 +38,6 @@ import { composeReleaseChildAttemptEvidence, formatReleaseStateOutcome, releasePlanGateFailures, - releaseChildClassificationEvidence, MAX_RELEASE_ARTIFACT_BYTES, serializeReleaseArtifact, selectReleaseStateArtifacts, @@ -282,19 +280,6 @@ export async function readChild(child, previous, signal, options = {}) { observedRunAttempts: evidence.observedRunAttempts, sha256: evidence.compositeJobsSha256, }); - if (snapshot.status === "completed" && snapshot.errors.length === 0) { - Object.assign( - snapshot, - await (options.loadFlakeClassifications ?? loadFlakeClassifications)({ - repo: process.env.GITHUB_REPOSITORY, - child: snapshot, - parentRunId: options.parentRunId, - parentRunAttempt: options.parentRunAttempt, - targetSha: options.targetSha, - signal, - }), - ); - } return snapshot; } catch (error) { const degraded = classifyReleaseGhTransportError(error) === "transient"; @@ -1514,7 +1499,6 @@ async function validateManifestMode() { Object.entries(drain.children).map(([key, child]) => [ key, { - ...releaseChildClassificationEvidence(child), compositeJobsSha256: child.compositeJobsSha256, dispatchActor: child.dispatchActor, effectiveRunAttempt: child.runAttempt, diff --git a/scripts/lib/release-publish-gates.mts b/scripts/lib/release-publish-gates.mts index 73959f6600e7..ebe6ab205aae 100644 --- a/scripts/lib/release-publish-gates.mts +++ b/scripts/lib/release-publish-gates.mts @@ -148,7 +148,7 @@ export function evaluateReleasePublishGates(input: { "selected-lanes", selectedLanesError === "", selectedLanesError, - "Use authenticated Full Release Validation evidence with policy-derived Windows Node CI advisories or exact-job recorded flakes and no waivers.", + "Use authenticated Full Release Validation evidence with policy-derived Windows Node CI advisories and no waivers.", ); if (consumer === "stable-closeout") { for (const gate of gates) { diff --git a/scripts/release-ci-summary.mjs b/scripts/release-ci-summary.mjs index ad0d778b5ae8..92016a02adb3 100755 --- a/scripts/release-ci-summary.mjs +++ b/scripts/release-ci-summary.mjs @@ -12,7 +12,6 @@ import process from "node:process"; import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { validateFullReleaseCandidateBinding } from "./full-release-candidate-contract.mjs"; -import { loadFlakeClassifications } from "./full-release-flake-classification.mjs"; import { publicationAdmissionContract, publicationObservationJson, @@ -36,7 +35,6 @@ import { normalizeReleaseTelegramWaiver, releaseCompositeJobsSha256, releaseAdvisoryJobs, - releaseChildClassificationEvidence, terminalPolicyPass, validateReleaseManifestAdvisoryJobs, validateReleaseChildDispatchBinding, @@ -1160,7 +1158,6 @@ function normalizeManifestChildEvidence(value) { key, { ...composite, - ...releaseChildClassificationEvidence(child), compositeJobsSha256, dispatchActor, observedRunAttempts, @@ -2126,9 +2123,6 @@ function validateCompletedParentRun(parentView, parentRest, repository, runId) { export function createReleaseEvidenceClient(repository = DEFAULT_REPO) { const normalizedRepository = normalizeRepository(repository); return { - loadFlakeClassifications(request) { - return loadFlakeClassifications({ ...request, repo: normalizedRepository }); - }, validateChildReuse(selection, request) { return validateReusableReleaseChild(selection, request); }, @@ -2579,12 +2573,7 @@ async function validateStrictChildRun({ }); if ( JSON.stringify(sortReleaseJsonValueKeys(childEvidence)) !== - JSON.stringify( - sortReleaseJsonValueKeys({ - ...evidence, - ...releaseChildClassificationEvidence(childEvidence), - }), - ) + JSON.stringify(sortReleaseJsonValueKeys(evidence)) ) { throw new Error(`manifest child composite evidence mismatch: ${child.name}`); } @@ -2613,23 +2602,6 @@ async function validateStrictChildRun({ runId, status: run.status, }; - const classifications = - child.manifestKey === "normalCi" && run.conclusion !== "success" - ? await client.loadFlakeClassifications({ - child: policyChild, - parentRunId: parentEvidence.manifest.runId, - parentRunAttempt: originAttempt, - targetSha: parentEvidence.manifest.targetSha, - }) - : {}; - Object.assign(policyChild, classifications); - if ( - childEvidence && - JSON.stringify(sortReleaseJsonValueKeys(releaseChildClassificationEvidence(childEvidence))) !== - JSON.stringify(sortReleaseJsonValueKeys(releaseChildClassificationEvidence(policyChild))) - ) { - throw new Error(`manifest child classification evidence mismatch: ${child.name}`); - } if ( run.repository?.full_name !== repository || run.head_sha !== (plannedChild?.workflowSha ?? parentEvidence.manifest.workflowSha) || diff --git a/scripts/render-github-release-notes.mts b/scripts/render-github-release-notes.mts index b06327359447..8cfd4cb7dbda 100644 --- a/scripts/render-github-release-notes.mts +++ b/scripts/render-github-release-notes.mts @@ -88,11 +88,10 @@ function verificationWithAdvisories(verification: string, manifest: unknown) { return normalizeTail(verification); } const escape = (value: string) => value.replace(/[\\`*_{}[\]()<>!#|]/gu, "\\$&"); - const lines = validateReleaseManifestAdvisoryJobs(manifest).map((job) => { - const detail = - job.class === "recorded-flake" ? `; ${escape(job.reason)}; tracking: ${job.trackingUrl}` : ""; - return `${ADVISORY_LINE_PREFIX}${job.class}): ${escape(job.child)} / ${escape(job.job)} (${job.conclusion}): ${job.url}${detail}`; - }); + const lines = validateReleaseManifestAdvisoryJobs(manifest).map( + (job) => + `${ADVISORY_LINE_PREFIX}${job.class}): ${escape(job.child)} / ${escape(job.job)} (${job.conclusion}): ${job.url}`, + ); const proof = normalizeTail(verification) .split("\n") .filter((line) => !line.startsWith(ADVISORY_LINE_PREFIX)) diff --git a/test/scripts/frv.test.ts b/test/scripts/frv.test.ts index 3e05b67f7c32..98e29c1c7f7e 100644 --- a/test/scripts/frv.test.ts +++ b/test/scripts/frv.test.ts @@ -8,7 +8,6 @@ import { preflightContinuation, watchRelease, } from "../../scripts/frv.mjs"; -import type { FlakeClassification } from "../../scripts/full-release-flake-classification.mjs"; import { releaseChildSpec, releaseCompositeJobsSha256, @@ -57,7 +56,6 @@ function preflightMethods( })), ]; return { - loadFlakeClassifications: async () => ({}), getReleaseEvidenceClient: () => ({ ...createReleaseEvidenceClient(REPOSITORY), getWorkflowSource: () => "name: Full Release Validation\n", @@ -605,7 +603,6 @@ describe("FRV continuation preflight", () => { await expect( continueFailed(parentOwnedPlan, "77", { - loadFlakeClassifications: read, getReleaseEvidenceClient: () => { reads += 1; throw new Error("unexpected evidence client"); @@ -688,7 +685,6 @@ describe("FRV continuation preflight", () => { await expect( continueFailed(plan([first, second]), "77", { - loadFlakeClassifications: downstreamRead, getReleaseEvidenceClient: () => { downstreamReads += 1; throw new Error("unexpected evidence client"); @@ -752,7 +748,6 @@ describe("FRV same-parent recovery", () => { const runReads: string[] = []; const attemptReads: Array<[string, number]> = []; const result = await inspectContinuation(plan([selected, missing]), { - loadFlakeClassifications: async () => ({}), getAttemptJobs: async (runId: string, attempt: number) => { attemptReads.push([runId, attempt]); return [job("test")]; @@ -789,7 +784,6 @@ describe("FRV same-parent recovery", () => { it("reports the effective attempt and composite job evidence", async () => { const selected = child("normalCi", "101"); const result = await inspectContinuation(plan([selected]), { - loadFlakeClassifications: async () => ({}), getAttemptJobs: async (_runId: string, attempt: number) => [ job("test", attempt === 1 ? "failure" : "success"), ], @@ -1035,63 +1029,6 @@ describe("FRV same-parent recovery", () => { expect(client.verify).toHaveBeenCalledOnce(); }); - it("reseals the failed parent without rerunning a classified child or its failed gate", async () => { - const scenario = rerunScenario({ parentSource: [1, "failure"] }); - const receipt: FlakeClassification = { - schema: "openclaw.frv-flake-classification.v1", - parentRunId: "77", - parentRunAttempt: 1, - child: "normalCi", - childRunId: "101", - childRunAttempt: 1, - targetSha: TARGET_SHA, - jobId: "501", - jobName: "checks-node-test-2", - jobUrl: `https://github.com/${REPOSITORY}/actions/runs/101/job/501`, - conclusion: "failure", - trackingUrl: `https://github.com/${REPOSITORY}/issues/789`, - reason: "Shared test fixture races during cleanup; repair tracked on main.", - classifiedBy: "release-operator", - receiptRunId: "890", - receiptRunAttempt: 1, - }; - const client = { - ...scenario.client, - getAttemptJobs: async () => [ - { - ...job(receipt.jobName, "failure"), - id: 501, - run_id: 101, - run_attempt: 1, - html_url: receipt.jobUrl, - }, - { - ...job("openclaw/ci-gate", "failure"), - id: 502, - run_id: 101, - run_attempt: 1, - }, - ], - loadFlakeClassifications: vi.fn(async () => ({ - flakeClassifications: [receipt], - gateEntries: [ - { name: "preflight", result: "success", selected: true }, - { name: "checks-node", result: "failure", selected: true }, - { name: "pr-fail-fast", result: "skipped", selected: false }, - ], - })), - }; - await expect(continueFailed(plan([scenario.selected]), "77", client)).resolves.toMatchObject({ - action: "reran-parent", - reruns: [], - finalRunId: "77", - }); - expect(scenario.counters).toMatchObject({ posts: { child: 0, parent: 1 }, verifies: 1 }); - expect(client.loadFlakeClassifications).toHaveBeenCalledWith( - expect.objectContaining({ parentRunId: "77", parentRunAttempt: 1, targetSha: TARGET_SHA }), - ); - }); - it.each([false, true])( "reruns the exact carried failed job once, including ambiguous response=%s", async (ambiguous) => { @@ -1300,7 +1237,6 @@ describe("FRV same-parent recovery", () => { const selected = child(key, "101"); let latestReads = 0; const client = { - loadFlakeClassifications: async () => ({}), getRun: async () => runFor( selected, diff --git a/test/scripts/full-release-flake-classification.test.ts b/test/scripts/full-release-flake-classification.test.ts deleted file mode 100644 index 2c22a2998334..000000000000 --- a/test/scripts/full-release-flake-classification.test.ts +++ /dev/null @@ -1,429 +0,0 @@ -import { readFileSync } from "node:fs"; -import { describe, expect, it, vi } from "vitest"; -import { parse as parseYaml } from "yaml"; -import { - isClassifiableFlakeJob, - loadFlakeClassifications, - parseFlakeGateEntries, - recordFlakeClassification, - validateFlakeClassification, - type FlakeApi, -} from "../../scripts/full-release-flake-classification.mjs"; - -const sha = "a".repeat(40); -const targetSha = "b".repeat(40); -const jobUrl = "https://github.com/openclaw/openclaw/actions/runs/200/job/300"; -const trackingUrl = "https://github.com/openclaw/openclaw/issues/400"; -const workflow = ".github/workflows/full-release-flake-classification.yml"; -const reason = "Independent reproduction confirms a fixture scheduling race."; - -function fixture() { - const env = { - GITHUB_REPOSITORY: "openclaw/openclaw", - GITHUB_REF: "refs/heads/main", - GITHUB_EVENT_NAME: "workflow_dispatch", - GITHUB_WORKFLOW_REF: `openclaw/openclaw/${workflow}@refs/heads/main`, - GITHUB_WORKFLOW_SHA: sha, - GITHUB_SHA: sha, - GITHUB_TRIGGERING_ACTOR: "maintainer", - GITHUB_RUN_ID: "500", - GITHUB_RUN_ATTEMPT: "1", - }; - const run = { - repository: { full_name: "openclaw/openclaw" }, - event: "workflow_dispatch", - head_sha: sha, - }; - const job = { - id: 300, - run_id: 200, - run_attempt: 2, - name: "checks-fast-core", - html_url: jobUrl, - status: "completed", - conclusion: "failure", - }; - const child = { - ...run, - id: 200, - path: ".github/workflows/ci.yml", - display_title: "CI full-release-validation-100-1-ci", - }; - const parent = { - ...run, - id: 100, - path: ".github/workflows/full-release-validation.yml", - run_attempt: 1, - }; - const dispatch = { - id: 600, - name: "Run normal full CI", - run_attempt: 1, - status: "completed", - conclusion: "success", - }; - const producer = { - ...run, - id: 500, - path: workflow, - head_branch: "main", - run_attempt: 1, - triggering_actor: { login: "maintainer" }, - display_title: `FRV flake classification ${jobUrl}`, - }; - const responses: Record = { - "actions/jobs/300": job, - "actions/runs/200": child, - "actions/runs/100/attempts/1": parent, - "actions/runs/100/attempts/1/jobs?per_page=100&page=1": { total_count: 1, jobs: [dispatch] }, - "actions/jobs/600/logs": `2026-09-29T10:00:00.000Z TARGET_SHA: ${targetSha}\n2026-09-29T10:00:00.000Z Dispatched ci.yml: https://github.com/openclaw/openclaw/actions/runs/200 (attempt 1)`, - "issues/400": { number: 400 }, - "actions/runs/500": producer, - }; - const api: FlakeApi = vi.fn(async (path) => { - if (!(path in responses)) { - throw new Error(`Unexpected API route: ${path}`); - } - return responses[path]; - }); - return { - env, - job, - child, - parent, - dispatch, - producer, - responses, - api, - inputs: { job_url: jobUrl, tracking_url: trackingUrl, reason }, - }; -} - -describe("authenticated FRV flake classification", () => { - it("records the failed attempt and the parent's Release SHA instead of the CI tooling SHA", async () => { - const request = fixture(); - request.inputs.reason = ` ${reason} `; - const receipt = await recordFlakeClassification(request); - expect(receipt).toEqual({ - schema: "openclaw.frv-flake-classification.v1", - parentRunId: "100", - parentRunAttempt: 1, - child: "normalCi", - childRunId: "200", - childRunAttempt: 2, - targetSha, - jobId: "300", - jobName: "checks-fast-core", - jobUrl, - conclusion: "failure", - trackingUrl, - reason, - classifiedBy: "maintainer", - receiptRunId: "500", - receiptRunAttempt: 1, - }); - expect( - validateFlakeClassification(receipt, { - child: { key: "normalCi", runId: "200", jobs: [request.job] }, - parentRunId: "100", - parentRunAttempt: 1, - targetSha, - }), - ).toEqual(receipt); - }); - - it.each([ - [ - "nonfailed job", - (f: ReturnType) => { - f.job.conclusion = "success"; - }, - "completed failure", - ], - [ - "wrong child job", - (f: ReturnType) => { - f.job.run_id = 201; - }, - "completed failure", - ], - [ - "wrong workflow", - (f: ReturnType) => { - f.child.path = ".github/workflows/ci-lite.yml"; - }, - "workflow identity", - ], - [ - "wrong title", - (f: ReturnType) => { - f.child.display_title = "CI"; - }, - "dispatch title", - ], - [ - "wrong parent workflow", - (f: ReturnType) => { - f.parent.path = ".github/workflows/ci.yml"; - }, - "workflow identity", - ], - [ - "wrong parent attempt", - (f: ReturnType) => { - f.parent.run_attempt = 2; - }, - "parent run identity", - ], - [ - "failed dispatch", - (f: ReturnType) => { - f.dispatch.conclusion = "failure"; - }, - "uniquely successful", - ], - [ - "wrong receipt actor", - (f: ReturnType) => { - f.producer.triggering_actor.login = "someone-else"; - }, - "producer identity", - ], - [ - "untrusted branch", - (f: ReturnType) => { - f.env.GITHUB_REF = "refs/heads/other"; - }, - "trusted main", - ], - [ - "wrong target witness", - (f: ReturnType) => { - f.responses["actions/jobs/600/logs"] = - ` TARGET_SHA: ${targetSha}\nDispatched ci.yml: https://github.com/openclaw/openclaw/actions/runs/201 (attempt 1)`; - }, - "dispatch witness", - ], - [ - "duplicate target witness", - (f: ReturnType) => { - f.responses["actions/jobs/600/logs"] = - `${String(f.responses["actions/jobs/600/logs"])}\n TARGET_SHA: ${sha}`; - }, - "target SHA", - ], - [ - "foreign tracking URL", - (f: ReturnType) => { - f.inputs.tracking_url = "https://github.com/other/repo/issues/400"; - }, - "tracking URL", - ], - [ - "short reason", - (f: ReturnType) => { - f.inputs.reason = "flaky"; - }, - "20–300", - ], - [ - "long reason", - (f: ReturnType) => { - f.inputs.reason = "x".repeat(301); - }, - "20–300", - ], - [ - "multiline reason", - (f: ReturnType) => { - f.inputs.reason = `${reason}\n`; - }, - "single line", - ], - ])("rejects %s", async (_name, change, error) => { - const request = fixture(); - change(request); - await expect(recordFlakeClassification(request)).rejects.toThrow(error); - }); - - it.each([ - "openclaw/ci-gate", - "Seal release", - "ios-screenshot-evidence", - "build-artifacts", - "Build Artifacts", - "npm-install-smoke", - "install_smoke", - "Upgrade survivor", - "docker-seed-e2e", - "update-first-hop-compat", - "First-hop smoke", - "pack-budget", - "npm-pack", - "Qualify release npm", - "Package Acceptance", - "package-integrity", - "package_integrity", - ])("never records protected job %s", async (name) => { - const request = fixture(); - request.job.name = name; - expect(isClassifiableFlakeJob(name)).toBe(false); - await expect(recordFlakeClassification(request)).rejects.toThrow("cannot be classified"); - }); - - it("requires the accepted job identity and exact parent/candidate bindings", async () => { - const request = fixture(); - const receipt = await recordFlakeClassification(request); - const child = { key: "normalCi", runId: "200", jobs: [request.job] }; - for (const expected of [ - { child: { ...child, key: "releaseChecks" } }, - { - child: { - ...child, - jobs: [{ ...request.job, id: 301, html_url: jobUrl.replace("300", "301") }], - }, - }, - { child, parentRunId: "101" }, - { child, parentRunAttempt: 2 }, - { child, targetSha: sha }, - ]) { - expect(() => validateFlakeClassification(receipt, expected)).toThrow(); - } - }); - - it("rejects non-string receipt bindings instead of coercing JSON arrays", async () => { - const receipt = await recordFlakeClassification(fixture()); - for (const field of ["targetSha", "jobUrl", "trackingUrl"] as const) { - expect(() => - validateFlakeClassification({ ...receipt, [field]: [receipt[field]] }), - ).toThrow(); - } - }); - - it("fails closed on receipt API errors and never looks up unrelated children", async () => { - const api = vi.fn().mockRejectedValue(new Error("HTTP 503")); - const child = { key: "normalCi", runId: "200", jobs: [fixture().job] }; - await expect( - loadFlakeClassifications({ child, api, parentRunId: "100", parentRunAttempt: 1, targetSha }), - ).rejects.toThrow("HTTP 503"); - api.mockClear(); - await expect( - loadFlakeClassifications({ - child: { ...child, key: "releaseChecks" }, - api, - parentRunId: "100", - parentRunAttempt: 1, - targetSha, - }), - ).resolves.toEqual({}); - expect(api).not.toHaveBeenCalled(); - }); -}); - -describe("receipt discovery scope", () => { - it("skips lookups for policy-advisory Windows shards and scopes discovery to the child run", async () => { - const windowsJob = { ...fixture().job, name: "checks-windows-node-3" }; - const api = vi.fn(async (path) => { - if (path === "actions/runs/200") { - return { id: 200, created_at: "2026-09-29T10:00:00Z" }; - } - return { total_count: 0, workflow_runs: [] }; - }); - const request = { api, parentRunId: "100", parentRunAttempt: 1, targetSha }; - await expect( - loadFlakeClassifications({ - ...request, - child: { key: "normalCi", runId: "200", jobs: [windowsJob] }, - }), - ).resolves.toEqual({}); - expect(api).not.toHaveBeenCalled(); - await expect( - loadFlakeClassifications({ - ...request, - child: { key: "normalCi", runId: "200", jobs: [fixture().job] }, - }), - ).resolves.toEqual({}); - expect(api.mock.calls.map(([path]) => path)).toEqual([ - "actions/runs/200", - "actions/workflows/full-release-flake-classification.yml/runs?event=workflow_dispatch&branch=main&status=success&created=%3E%3D2026-09-29T10:00:00Z&per_page=100&page=1", - ]); - }); -}); - -describe("CI gate receipt log", () => { - it("parses only emitted entries and retains skipped, cancelled, and missing outcomes for policy rejection", () => { - const entries = parseFlakeGateEntries( - [ - '2026-09-29T10:00:00.000Z echo "${name}: ${result} (selected=${selected:-missing})"', - "2026-09-29T10:00:00.000Z preflight: success (selected=true)", - "2026-09-29T10:00:00.000Z checks-fast-core: failure (selected=true)", - "2026-09-29T10:00:00.000Z checks-ui: skipped (selected=true)", - "2026-09-29T10:00:00.000Z checks-ui-e2e: cancelled (selected=true)", - "2026-09-29T10:00:00.000Z checks-fast-plugin-contracts-shard: failure (selected=missing)", - "2026-09-29T10:00:00.000Z pr-fail-fast: skipped (selected=false)", - ].join("\n"), - ); - expect(entries).toEqual([ - { name: "preflight", result: "success", selected: true }, - { name: "checks-fast-core", result: "failure", selected: true }, - { name: "checks-ui", result: "skipped", selected: true }, - { name: "checks-ui-e2e", result: "cancelled", selected: true }, - { name: "checks-fast-plugin-contracts-shard", result: "failure", selected: "missing" }, - { name: "pr-fail-fast", result: "skipped", selected: false }, - ]); - }); - it.each(["", "preflight: success (selected=true)", "checks-fast-core: failure (selected=true)"])( - "rejects incomplete log %j", - (log) => { - expect(() => parseFlakeGateEntries(log)).toThrow("incomplete"); - }, - ); - it.each([ - "checks-ui: unknown.result (selected=true)", - "checks-ui: failure (selected=true", - "checks-ui: failure (selected =true)", - "checks-ui: failure (selected=true) unexpected", - ])("rejects malformed gate-shaped row %j instead of omitting it", (row) => { - const log = [ - "preflight: success (selected=true)", - row, - "pr-fail-fast: skipped (selected=false)", - ].join("\n"); - expect(() => parseFlakeGateEntries(log)).toThrow("CI gate log entr"); - }); -}); - -describe("classification workflow contract", () => { - it("uses trusted main and script-owned input parsing without write permissions", () => { - const parsed = parseYaml(readFileSync(new URL(`../../${workflow}`, import.meta.url), "utf8")); - expect(parsed.on).toEqual({ workflow_dispatch: { inputs: expect.any(Object) } }); - expect(parsed.permissions).toEqual({ - contents: "read", - actions: "read", - issues: "read", - "pull-requests": "read", - }); - expect(parsed.jobs.record.if).toBe("github.ref == 'refs/heads/main'"); - const steps = parsed.jobs.record.steps; - expect(steps[0].with).toMatchObject({ - ref: "${{ github.workflow_sha }}", - "persist-credentials": false, - }); - expect(steps.find((step: { run?: string }) => step.run)?.run).toBe( - "node scripts/full-release-flake-classification.mjs record", - ); - expect(steps.at(-1).with).toMatchObject({ "retention-days": 90, "if-no-files-found": "error" }); - }); - it("pins the CI gate's output grammar and completeness bookends", () => { - const ci = parseYaml( - readFileSync(new URL("../../.github/workflows/ci.yml", import.meta.url), "utf8"), - ); - const gate = ci.jobs["ci-gate"].steps.find( - (step: { name: string }) => step.name === "Verify selected CI lanes", - ); - expect(gate.run).toContain('echo "${name}: ${result} (selected=${selected:-missing})"'); - const rows = gate.env.JOB_RESULTS.trim().split("\n"); - expect(rows[0]).toMatch(/^preflight=/u); - expect(rows.at(-1)).toMatch(/^pr-fail-fast=/u); - }); -}); diff --git a/test/scripts/full-release-publication-admission.test.ts b/test/scripts/full-release-publication-admission.test.ts index 10a159ca9576..1358ae103656 100644 --- a/test/scripts/full-release-publication-admission.test.ts +++ b/test/scripts/full-release-publication-admission.test.ts @@ -112,7 +112,6 @@ const toolingPaths = [ "scripts/full-release-candidate-contract.mjs", "scripts/full-release-validation-state.mjs", "scripts/full-release-validation-policy.mjs", - "scripts/full-release-flake-classification.mjs", "scripts/release-ci-summary.mjs", "scripts/lib/full-release-candidate-reuse.mjs", "scripts/lib/full-release-child-request.mjs", diff --git a/test/scripts/full-release-validation-state.test.ts b/test/scripts/full-release-validation-state.test.ts index 2c416c4c8e46..6f3c07fe7b25 100644 --- a/test/scripts/full-release-validation-state.test.ts +++ b/test/scripts/full-release-validation-state.test.ts @@ -8,7 +8,6 @@ import { buildFullReleaseCandidateBinding, buildFullReleaseCandidateRequest, } from "../../scripts/full-release-candidate-contract.mjs"; -import type { FlakeClassification } from "../../scripts/full-release-flake-classification.mjs"; import { createPublicationAdmission, createPublicationObservations, @@ -27,7 +26,6 @@ import { terminalPolicyPass, validateReleaseChildDispatchBinding, validateReleaseCoveragePolicyBinding, - validateReleaseManifestAdvisoryJobs, } from "../../scripts/full-release-validation-policy.mjs"; import { affectedActiveRunIds, @@ -40,7 +38,6 @@ import { readChild, releaseGhRetryDelayMs, releasePlanGateFailures, - releaseStateChildEvidence, serializeReleaseArtifact, selectReleaseStateArtifacts, validateReleaseExecutionPlanArtifact, @@ -1334,283 +1331,6 @@ describe("release decision policy", () => { }, ); - function recordedFlakeChild() { - const job = { - name: "checks-node-compact-small-19-3", - conclusion: "failure", - status: "completed", - acceptedRunAttempt: 1, - url: "https://github.com/openclaw/openclaw/actions/runs/101/job/1001", - }; - const gateJob = { - ...job, - ...ciGate, - conclusion: "failure", - url: "https://github.com/openclaw/openclaw/actions/runs/101/job/1003", - }; - const receipt: FlakeClassification = { - schema: "openclaw.frv-flake-classification.v1", - parentRunId: "77", - parentRunAttempt: 1, - child: "normalCi", - childRunId: "101", - childRunAttempt: 1, - targetSha: TARGET_SHA, - jobId: "1001", - jobName: job.name, - jobUrl: job.url, - conclusion: "failure", - trackingUrl: "https://github.com/openclaw/openclaw/issues/42", - reason: "The shared fixture leaks state; repair is tracked on main.", - classifiedBy: "release-maintainer", - receiptRunId: "901", - receiptRunAttempt: 1, - }; - const preflight = { name: "preflight", result: "success", selected: true }; - const lastEntry = { name: "pr-fail-fast", result: "skipped", selected: false }; - const snapshot = { - ...child("normalCi", { conclusion: "failure", status: "completed" }), - jobs: [job, gateJob], - flakeClassifications: [receipt], - gateEntries: [ - preflight, - { name: "checks-node-core-test-nondist-shard", result: "failure", selected: true }, - lastEntry, - ], - }; - return { snapshot, job, gateJob, receipt, preflight, lastEntry }; - } - - it.each([ - { status: "completed", loaderFails: false }, - { status: "completed", loaderFails: true }, - { status: "in_progress", loaderFails: false }, - ])( - "hydrates receipts only after child completion: $status, loader error=$loaderFails", - async ({ status, loaderFails }) => { - const { - snapshot: { flakeClassifications, gateEntries, ...snapshot }, - } = recordedFlakeChild(); - let classificationReads = 0; - const observed = await readChild(snapshot, undefined, undefined, { - parentRunId: "77", - parentRunAttempt: 1, - targetSha: TARGET_SHA, - readRun: async () => ({ - actor: { login: "github-actions[bot]" }, - triggering_actor: { login: "github-actions[bot]" }, - conclusion: status === "completed" ? "failure" : null, - display_title: snapshot.displayTitle, - event: "workflow_dispatch", - head_branch: snapshot.workflowRef, - head_sha: SHA, - html_url: snapshot.url, - id: 101, - path: ".github/workflows/ci.yml", - repository: { full_name: "openclaw/openclaw" }, - run_attempt: 1, - status, - }), - readAttemptJobs: async () => snapshot.jobs, - loadFlakeClassifications: async (binding) => { - classificationReads += 1; - expect(binding).toMatchObject({ - parentRunId: "77", - parentRunAttempt: 1, - targetSha: TARGET_SHA, - }); - if (loaderFails) { - throw new Error("receipt artifact digest differs"); - } - return { flakeClassifications, gateEntries }; - }, - }); - if (status !== "completed") { - expect(observed).toMatchObject({ status, errors: [] }); - expect(classificationReads).toBe(0); - return; - } - expect(classificationReads).toBe(1); - const decision = classifyReleaseSnapshot({ children: [observed] }); - expect(decision.state).toBe(loaderFails ? "orchestration_error" : "passed"); - if (loaderFails) { - expect(decision.errors).toEqual([ - expect.objectContaining({ - kind: "api_error", - message: expect.stringContaining("receipt artifact digest differs"), - }), - ]); - } else { - expect(decision.advisoryJobs).toEqual([ - expect.objectContaining({ class: "recorded-flake", receiptRunId: "901" }), - ]); - } - }, - ); - - it("retains recorded matrix flakes with different gate names through state and manifest validation", () => { - const { snapshot, job, receipt } = recordedFlakeChild(); - const result = classifyReleaseSnapshot({ children: [snapshot] }); - expect(result).toMatchObject({ - state: "passed", - blockers: [], - advisoryJobs: [ - { - class: "recorded-flake", - child: "normalCi", - job: job.name, - conclusion: "failure", - runId: "101", - url: job.url, - jobId: "1001", - trackingUrl: receipt.trackingUrl, - reason: receipt.reason, - receiptRunId: "901", - }, - ], - }); - const artifact = buildReleaseStateArtifact({ - children: [snapshot], - decision: result, - executionPlan: { parentRunAttempt: 1, sha256: "a".repeat(64) }, - expected: { parentRunAttempt: 2, parentRunId: "77", targetSha: TARGET_SHA }, - mode: "decision", - releaseProfile: "stable", - rerunGroup: "all", - }); - const validated = validateReleaseStateArtifact(artifact); - assert(validated.children.normalCi, "normalCi evidence must survive validation"); - expect(releaseStateChildEvidence(validated.children.normalCi)).toMatchObject({ - flakeClassifications: snapshot.flakeClassifications, - gateEntries: snapshot.gateEntries, - }); - expect(formatReleaseStateOutcome(artifact)).toContain( - `normalCi/${job.name} (failure) ${job.url} — ${receipt.reason} ${receipt.trackingUrl}`, - ); - const manifest = buildReleaseValidationManifest({ - plan: executionPlan(), - drain: validated, - context: { runId: "77", runAttempt: 2, releaseProfile: "stable", validationInputs: {} }, - }); - expect(validateReleaseManifestAdvisoryJobs(manifest)).toEqual(result.advisoryJobs); - }); - - it.each(["new job ID", "new attempt", "new job name", "other child", "cancelled job"])( - "blocks a recorded flake after %s changes accepted evidence", - (scenario) => { - const { snapshot, job } = recordedFlakeChild(); - if (scenario === "new job ID") { - job.url = "https://github.com/openclaw/openclaw/actions/runs/101/job/1002"; - } - if (scenario === "new attempt") { - job.acceptedRunAttempt = 2; - } - if (scenario === "new job name") { - job.name = "checks-node-other"; - } - if (scenario === "other child") { - snapshot.key = "releaseChecksCandidate"; - } - if (scenario === "cancelled job") { - job.conclusion = "cancelled"; - } - expect(terminalPolicyPass(snapshot)).toBe(false); - expect(classifyReleaseSnapshot({ children: [snapshot] })).toMatchObject({ - state: "blocked_complete", - advisoryJobs: [], - }); - }, - ); - - it.each([ - { name: "checks-node-core-test-nondist-shard", result: "skipped", selected: true }, - { name: "checks-node-core-test-nondist-shard", result: "cancelled", selected: true }, - { name: "checks-node-core-test-nondist-shard", result: "missing", selected: true }, - { name: "checks-node-core-test-nondist-shard", result: "failure", selected: false }, - { name: "checks-node-core-test-nondist-shard", result: "failure", selected: "missing" }, - { name: "checks-node-core-test-nondist-shard", result: "neutral", selected: true }, - ])("blocks uncovered gate entry $name:$result:$selected", (entry) => { - const { snapshot: base, preflight, lastEntry } = recordedFlakeChild(); - const snapshot = { ...base, gateEntries: [preflight, entry, lastEntry] }; - expect(terminalPolicyPass(snapshot)).toBe(false); - expect(classifyReleaseSnapshot({ children: [snapshot] }).state).toBe("blocked_complete"); - }); - - it.each([ - "no entries", - "passing entries only", - "duplicate entries", - "missing gate", - "unclassified failure", - ])("requires complete gate coverage with %s", (scenario) => { - const { snapshot, preflight, gateJob } = recordedFlakeChild(); - if (scenario === "no entries") { - snapshot.gateEntries = []; - } - if (scenario === "passing entries only") { - snapshot.gateEntries = snapshot.gateEntries.slice(0, 1); - } - if (scenario === "duplicate entries") { - snapshot.gateEntries.push(preflight); - } - if (scenario === "missing gate") { - snapshot.jobs.pop(); - } - if (scenario === "unclassified failure") { - snapshot.jobs.push({ ...gateJob, name: "macos-node", conclusion: "failure" }); - } - expect(terminalPolicyPass(snapshot)).toBe(false); - }); - - it.each([ - "parent", - "parent attempt", - "child run", - "target", - "job success", - "denied job", - "advisory reason", - "gate coverage", - ])("rejects forged manifest %s evidence", (scenario) => { - const { snapshot, job, receipt } = recordedFlakeChild(); - const manifest = { - runId: "77", - sourceParentRunAttempt: 1, - targetSha: TARGET_SHA, - childRuns: { normalCi: "101" }, - childEvidence: { normalCi: snapshot }, - advisoryJobs: classifyReleaseSnapshot({ children: [snapshot] }).advisoryJobs, - }; - if (scenario === "parent") { - manifest.runId = "78"; - } - if (scenario === "parent attempt") { - manifest.sourceParentRunAttempt = 2; - } - if (scenario === "child run") { - receipt.childRunId = "102"; - } - if (scenario === "target") { - manifest.targetSha = SHA; - } - if (scenario === "job success") { - job.conclusion = "success"; - } - if (scenario === "denied job") { - receipt.jobName = "build-artifacts"; - job.name = receipt.jobName; - } - if (scenario === "advisory reason") { - receipt.reason = "A forged replacement reason is not the recorded advisory."; - } - if (scenario === "gate coverage") { - snapshot.gateEntries = []; - } - expect(() => validateReleaseManifestAdvisoryJobs(manifest)).toThrow( - scenario === "denied job" ? /job cannot be classified/u : undefined, - ); - }); - it.each([ ["normalCi", "macos-node"], ["normalCi", "macos-swift (tests)"], diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index d7cc8867a6d2..174778d50309 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -3254,7 +3254,6 @@ function runReleaseChecksInputValidation( const workdir = tempDirs.make("release-checks-input-validation-"); const fixture = frozenToolingFixture(workdir, [ "scripts/full-release-validation-policy.mjs", - "scripts/full-release-flake-classification.mjs", ...PUBLICATION_CONTRACT_FILES, "scripts/lib/release-changelog.mjs", "scripts/full-release-candidate-contract.mjs", @@ -14729,7 +14728,6 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`, for (const source of [ "scripts/release-ci-summary.mjs", "scripts/full-release-validation-policy.mjs", - "scripts/full-release-flake-classification.mjs", ...PUBLICATION_CONTRACT_FILES, "scripts/lib/release-changelog.mjs", "scripts/full-release-candidate-contract.mjs", diff --git a/test/scripts/release-ci-summary.test.ts b/test/scripts/release-ci-summary.test.ts index f748b66fbe63..36f6a2bfcc98 100644 --- a/test/scripts/release-ci-summary.test.ts +++ b/test/scripts/release-ci-summary.test.ts @@ -19,7 +19,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { parse } from "yaml"; import { continueFailed, preflightContinuation } from "../../scripts/frv.mjs"; import { buildFullReleaseCandidateRequest } from "../../scripts/full-release-candidate-contract.mjs"; -import { loadFlakeClassifications } from "../../scripts/full-release-flake-classification.mjs"; import { createPublicationAdmission, createPublicationObservations, @@ -403,7 +402,6 @@ describe("original publication admission reader", () => { observed.sha256 = releaseExecutionPlanSha256(observed); } const continuationClient = { - loadFlakeClassifications: async () => ({}), getReleaseEvidenceClient: () => client, getRun: client.getRun, getAttemptJobs: vi.fn(async () => { @@ -1777,7 +1775,6 @@ function trustedMainPackageFixture({ }; }; const client = { - loadFlakeClassifications: async () => ({}), getWorkflowSource: (_sha: string) => "name: Full Release Validation\n", compareCommitLineage: compareCommits, compareCommits, @@ -1893,22 +1890,6 @@ function trustedMainFullFixture() { }; const client = { ...fixture.client, - loadFlakeClassifications: (request: Parameters[0]) => - loadFlakeClassifications({ - ...request, - api: async (endpoint: string) => { - if (endpoint === `actions/runs/${request.child.runId}`) { - return { id: Number(request.child.runId), created_at: "2026-07-10T01:00:00Z" }; - } - if ( - endpoint === - "actions/workflows/full-release-flake-classification.yml/runs?event=workflow_dispatch&branch=main&status=success&created=%3E%3D2026-07-10T01:00:00Z&per_page=100&page=1" - ) { - return { total_count: 0, workflow_runs: [] }; - } - throw new Error(`unexpected classification lookup: ${endpoint}`); - }, - }), getJobLog: vi.fn((jobId: number) => { const index = jobs.findIndex((job) => job.id === jobId); const child = expectDefined(children[index], "dispatch child"); @@ -2543,7 +2524,6 @@ describe("release CI summary child correlation", () => { ); await expect( continueFailed(fixture.executionPlan, fixture.runId, { - loadFlakeClassifications: async () => ({}), repository, getRun, getRunAttempt: async (id: string) => (id === fixture.runId ? originalParent : getRun(id)), @@ -3404,192 +3384,6 @@ describe("release CI summary child correlation", () => { }, ); - it.each([ - "valid", - "changed-receipt", - "foreign-parent", - "failed-producer", - "tag-revision", - "changed-gate", - ])( - "rederives recorded flakes from authenticated receipt artifacts and the live CI gate: %s", - async (scenario) => { - const fixture = trustedMainNpmFixture(); - const selected = expectDefined( - fixture.executionPlan.children.find((child) => child.key === "normalCi"), - "normal CI child", - ); - const run = expectDefined( - fixture.runs.find((candidate) => String(candidate.id) === selected.runId), - "normal CI run", - ); - run.conclusion = "failure"; - const receipt = { - schema: "openclaw.frv-flake-classification.v1", - parentRunId: fixture.runId, - parentRunAttempt: 1, - child: "normalCi", - childRunId: selected.runId, - childRunAttempt: 1, - targetSha: fixture.targetSha, - jobId: "501", - jobName: "checks-node-test-2", - jobUrl: `https://github.com/openclaw/openclaw/actions/runs/${selected.runId}/job/501`, - conclusion: "failure", - trackingUrl: "https://github.com/openclaw/openclaw/issues/789", - reason: "Shared test fixture races during cleanup; repair tracked on main.", - classifiedBy: "release-operator", - receiptRunId: "890", - receiptRunAttempt: 1, - }; - const jobs = [ - { ...fixture.parentJob, id: 501, name: receipt.jobName, html_url: receipt.jobUrl }, - { - ...fixture.parentJob, - id: 502, - name: "openclaw/ci-gate", - html_url: `https://github.com/openclaw/openclaw/actions/runs/${selected.runId}/job/502`, - }, - ].map((job) => Object.assign(job, { conclusion: "failure" })); - const composite = composeReleaseAttemptJobs([{ jobs, runAttempt: 1 }], { - effectiveRunAttempt: 1, - plannedRunAttempt: 1, - }); - const gateEntries = [ - { name: "preflight", result: "success", selected: true }, - { name: "checks-node", result: "failure", selected: true }, - { name: "pr-fail-fast", result: "skipped", selected: false }, - ]; - Object.assign(expectDefined(fixture.manifest.childEvidence.normalCi, "CI evidence"), { - status: "completed", - conclusion: "failure", - jobs: composite.jobs, - compositeJobsSha256: composite.sha256, - flakeClassifications: [receipt], - gateEntries, - }); - const advisory = { - class: "recorded-flake", - child: "normalCi", - job: receipt.jobName, - conclusion: "failure", - runId: selected.runId, - url: receipt.jobUrl, - jobId: "501", - trackingUrl: receipt.trackingUrl, - reason: receipt.reason, - receiptRunId: "890", - }; - Object.assign(fixture.manifest, { advisoryJobs: [advisory] }); - const liveReceipt = { - ...receipt, - ...(scenario === "changed-receipt" - ? { reason: "A different classification was recorded." } - : {}), - ...(scenario === "foreign-parent" ? { parentRunId: "999" } : {}), - }; - const zip = makeStoredZip({ "frv-flake-classification.json": JSON.stringify(liveReceipt) }); - const producer = { - id: 890, - run_attempt: 1, - repository: { full_name: "openclaw/openclaw" }, - path: ".github/workflows/full-release-flake-classification.yml", - event: "workflow_dispatch", - head_branch: "main", - status: "completed", - conclusion: scenario === "failed-producer" ? "failure" : "success", - head_sha: "d".repeat(40), - display_title: `FRV flake classification ${receipt.jobUrl}`, - triggering_actor: { login: receipt.classifiedBy }, - }; - const api = vi.fn(async (path: string) => { - if (path === `actions/runs/${selected.runId}`) { - return { id: Number(selected.runId), created_at: "2026-09-29T10:00:00Z" }; - } - if ( - path.startsWith("actions/workflows/full-release-flake-classification.yml/runs?") && - path.includes("&created=%3E%3D2026-09-29T10:00:00Z&") - ) { - return { total_count: 1, workflow_runs: [producer] }; - } - if (path === "actions/runs/890") { - return producer; - } - if (path === `compare/${"d".repeat(40)}...main?per_page=1`) { - return scenario === "tag-revision" - ? { status: "diverged", merge_base_commit: { sha: "e".repeat(40) } } - : { status: "ahead", merge_base_commit: { sha: "d".repeat(40) } }; - } - if (path === "actions/runs/890/artifacts?per_page=100") { - return { - total_count: 1, - artifacts: [ - { - id: 891, - name: `frv-flake-classification-${selected.runId}-501`, - expired: false, - workflow_run: { id: 890 }, - size_in_bytes: zip.length, - digest: artifactDigest(zip), - }, - ], - }; - } - if (path === "actions/artifacts/891/zip") { - return zip; - } - if (path === "actions/jobs/502/logs") { - return gateEntries - .map( - (entry) => - `2026-09-29T12:00:00.000Z ${entry.name}: ${scenario === "changed-gate" && entry.name === "checks-node" ? "skipped" : entry.result} (selected=${entry.selected})`, - ) - .join("\n"); - } - throw new Error(`unexpected classification API request: ${path}`); - }); - const originalJobs = expectDefined( - fixture.client.getRunAttemptJobs.getMockImplementation(), - "job reader", - ); - const client = { - ...fixture.client, - getRunAttemptJobs: (runId: string) => - runId === selected.runId ? jobs : originalJobs(runId), - loadFlakeClassifications: (request: Parameters[0]) => - loadFlakeClassifications({ ...request, api }), - }; - const validation = validateReleaseRunEvidence( - { - repository: "openclaw/openclaw", - runId: fixture.runId, - verifierSourceContent: readFileSync(SCRIPT), - verifierSourceSha: "c".repeat(40), - }, - client, - ); - if (scenario === "valid") { - const evidence = await validation; - expect(evidence.valid).toBe(true); - expect(evidence.children).toContainEqual( - expect.objectContaining({ - role: "normalCi", - conclusion: "failure", - policyPassed: true, - advisoryJobs: [advisory], - }), - ); - expect(api.mock.calls.filter(([path]) => path === "actions/jobs/502/logs")).toHaveLength(1); - } else { - await expect(validation).rejects.toThrow( - ["foreign-parent", "failed-producer", "tag-revision"].includes(scenario) - ? /FRV flake classification/u - : /classification evidence mismatch/u, - ); - } - }, - ); - it.each(["", "ship"])( "reads published empty retry metadata without granting a waiver (%s)", async (laneWaiver) => { diff --git a/test/scripts/release-publish-gates.test.ts b/test/scripts/release-publish-gates.test.ts index 9a0694e8f728..ab872283c085 100644 --- a/test/scripts/release-publish-gates.test.ts +++ b/test/scripts/release-publish-gates.test.ts @@ -46,73 +46,6 @@ const windowsEvidence = { }, advisoryJobs: [windowsAdvisory], }; -const flakeReceipt = { - schema: "openclaw.frv-flake-classification.v1", - parentRunId: "123", - parentRunAttempt: 2, - child: "normalCi", - childRunId: "456", - childRunAttempt: 1, - targetSha, - jobId: "457", - jobName: "checks-node-test-2", - jobUrl: "https://github.com/openclaw/openclaw/actions/runs/456/job/457", - conclusion: "failure", - trackingUrl: "https://github.com/openclaw/openclaw/issues/789", - reason: "Shared test fixture races during cleanup; repair tracked on main.", - classifiedBy: "release-operator", - receiptRunId: "890", - receiptRunAttempt: 1, -}; -const flakeEvidence = { - ...manifest, - runId: "123", - runAttempt: "2", - sourceParentRunAttempt: 2, - childRuns: { normalCi: "456" }, - childEvidence: { - normalCi: { - runId: "456", - status: "completed", - conclusion: "failure", - jobs: [ - { - name: flakeReceipt.jobName, - status: "completed", - conclusion: "failure", - acceptedRunAttempt: 1, - url: flakeReceipt.jobUrl, - }, - { - name: "openclaw/ci-gate", - status: "completed", - conclusion: "failure", - url: "https://github.com/openclaw/openclaw/actions/runs/456/job/458", - }, - ], - flakeClassifications: [flakeReceipt], - gateEntries: [ - { name: "preflight", result: "success", selected: true }, - { name: "checks-node", result: "failure", selected: true }, - { name: "pr-fail-fast", result: "skipped", selected: false }, - ], - }, - }, - advisoryJobs: [ - { - class: "recorded-flake", - child: "normalCi", - job: flakeReceipt.jobName, - conclusion: "failure", - runId: "456", - url: flakeReceipt.jobUrl, - jobId: "457", - trackingUrl: flakeReceipt.trackingUrl, - reason: flakeReceipt.reason, - receiptRunId: "890", - }, - ], -}; it.each([ { releaseTag: "v2026.9.5-alpha.1", npmDistTag: "beta" }, @@ -362,33 +295,6 @@ describe("release publication control admission", () => { ); }); - it.each(["publisher", "core-npm", "stable-closeout"] as const)( - "admits recorded flakes only with their exact manifest proof at %s", - (consumer) => { - const selectedLaneGate = (evidence: unknown = flakeEvidence) => - evaluateReleasePublishGates({ - consumer, - releaseTag: "v2026.9.5", - npmDistTag: "latest", - manifest: evidence, - }).find((gate) => gate.id === `${consumer}.selected-lanes`); - expect(selectedLaneGate()).toMatchObject({ status: "PASS" }); - for (const overrides of [ - { runId: "124" }, - { targetSha: "b".repeat(40) }, - { advisoryJobs: [] }, - { childEvidence: {} }, - { validationInputs: { knownFlakyJobsJson: '["checks-node-test-2"]' } }, - { validationInputs: { laneWaiver: "approved" } }, - { publishInputs: { stableSoakWaiver: "approved" } }, - ]) { - expect(selectedLaneGate({ ...flakeEvidence, ...overrides })).toMatchObject({ - status: "FAIL", - }); - } - }, - ); - it.each([false, true])( "runs without installed dependencies and never emits waiver authority (waived=%s)", (waived) => { diff --git a/test/scripts/render-github-release-notes.test.ts b/test/scripts/render-github-release-notes.test.ts index 687a04d0d9b3..a84e3cb3cd49 100644 --- a/test/scripts/render-github-release-notes.test.ts +++ b/test/scripts/render-github-release-notes.test.ts @@ -579,37 +579,7 @@ describe("GitHub release-note rendering", () => { ).toBe(false); }); - it("renders and verifies advisory failures from bound release evidence", () => { - const receipt = { - schema: "openclaw.frv-flake-classification.v1", - parentRunId: "123", - parentRunAttempt: 2, - child: "normalCi", - childRunId: "456", - childRunAttempt: 1, - targetSha: "a".repeat(40), - jobId: "457", - jobName: "checks-node-test-2", - jobUrl: "https://github.com/openclaw/openclaw/actions/runs/456/job/457", - conclusion: "failure", - trackingUrl: "https://github.com/openclaw/openclaw/issues/789", - reason: "Shared fixture cleanup races; fixed in parallel on main.", - classifiedBy: "release-operator", - receiptRunId: "890", - receiptRunAttempt: 1, - }; - const advisory = { - class: "recorded-flake", - child: "normalCi", - job: receipt.jobName, - conclusion: receipt.conclusion, - runId: receipt.childRunId, - url: receipt.jobUrl, - jobId: receipt.jobId, - trackingUrl: receipt.trackingUrl, - reason: receipt.reason, - receiptRunId: receipt.receiptRunId, - }; + it("renders and verifies Windows advisory failures from bound release evidence", () => { const windows = { class: "windows-node-ci", child: "normalCi", @@ -619,42 +589,22 @@ describe("GitHub release-note rendering", () => { url: "https://github.com/openclaw/openclaw/actions/runs/456/job/459", }; const validationManifest = { - runId: "123", - sourceParentRunAttempt: 2, - targetSha: receipt.targetSha, childRuns: { normalCi: "456" }, childEvidence: { normalCi: { runId: "456", - status: "completed", - conclusion: "failure", - jobs: [advisory, windows] - .map((job) => ({ - name: job.job, + jobs: [ + { + name: windows.job, status: "completed", conclusion: "failure", acceptedRunAttempt: 1, - url: job.url, - })) - .concat([ - { - name: "openclaw/ci-gate", - status: "completed", - conclusion: "failure", - acceptedRunAttempt: 1, - url: "https://github.com/openclaw/openclaw/actions/runs/456/job/458", - }, - ]), - flakeClassifications: [receipt], - gateEntries: [ - { name: "preflight", result: "success", selected: true }, - { name: "checks-node", result: "failure", selected: true }, - { name: "checks-windows", result: "failure", selected: true }, - { name: "pr-fail-fast", result: "skipped", selected: false }, + url: windows.url, + }, ], }, }, - advisoryJobs: [advisory, windows], + advisoryJobs: [windows], }; const target = { changelog: changelogFor("- **PR #123** fix: example."), @@ -667,44 +617,15 @@ describe("GitHub release-note rendering", () => { ...target, verification: "### Release verification\n\n- release SHA: `abc123`", }); - expect(rendered.body).toContain( - `- Advisory job (recorded-flake): normalCi / checks-node-test-2 (failure): ${receipt.jobUrl}; ${receipt.reason}; tracking: ${receipt.trackingUrl}`, - ); expect(rendered.body).toContain( "- Advisory job (windows-node-ci): normalCi / checks-windows-node-test-2 (failure)", ); expect(verifyGithubReleaseNotes({ ...target, body: rendered.body }).matches).toBe(true); const advisoryOnly = renderGithubReleaseNotes(target); expect(advisoryOnly.body).toContain( - "### Release verification\n- Advisory job (recorded-flake)", + "### Release verification\n- Advisory job (windows-node-ci)", ); expect(verifyGithubReleaseNotes({ ...target, body: advisoryOnly.body }).matches).toBe(true); - for (const body of [ - rendered.body.replace(receipt.reason, "Unrecorded reason."), - rendered.body.replace(receipt.trackingUrl, "https://github.com/openclaw/openclaw/issues/999"), - rendered.body - .split("\n") - .filter((line) => !line.includes("Advisory job (recorded-flake)")) - .join("\n"), - ]) { - expect(verifyGithubReleaseNotes({ ...target, body }).matches).toBe(false); - } - expect(() => - renderGithubReleaseNotes({ - ...target, - validationManifest: { - ...validationManifest, - advisoryJobs: [{ ...advisory, reason: "Forged reason." }, windows], - }, - }), - ).toThrow("advisory jobs differ"); - const heading = `## ${version}\n\n`; - const nearLimitBody = heading + "x".repeat(GITHUB_RELEASE_BODY_MAX_BYTES - heading.length); - const nearLimitTarget = { ...target, changelog: nearLimitBody }; - expect(() => renderGithubReleaseNotes(nearLimitTarget)).toThrow("required advisory evidence"); - expect(() => verifyGithubReleaseNotes({ ...nearLimitTarget, body: nearLimitBody })).toThrow( - "required advisory evidence", - ); }); it("does not treat fenced verification headings as appended proof", () => { diff --git a/test/scripts/validate-full-release-validation-evidence.test.ts b/test/scripts/validate-full-release-validation-evidence.test.ts index 7adbce98c565..5bc6ec133240 100644 --- a/test/scripts/validate-full-release-validation-evidence.test.ts +++ b/test/scripts/validate-full-release-validation-evidence.test.ts @@ -203,82 +203,6 @@ describe("full release validation evidence", () => { expect(() => validate({}, inputs)).toThrow(/waivers|knownFlakyJobsJson/u); }); - it("binds a recorded flake to the original parent attempt and target during evidence admission", () => { - const receipt = { - schema: "openclaw.frv-flake-classification.v1", - parentRunId: "123", - parentRunAttempt: 1, - child: "normalCi", - childRunId: "456", - childRunAttempt: 1, - targetSha, - jobId: "457", - jobName: "checks-node-test-2", - jobUrl: "https://github.com/openclaw/openclaw/actions/runs/456/job/457", - conclusion: "failure", - trackingUrl: "https://github.com/openclaw/openclaw/issues/789", - reason: "Shared test fixture races during cleanup; repair tracked on main.", - classifiedBy: "release-operator", - receiptRunId: "890", - receiptRunAttempt: 1, - }; - const childEvidence = { - runId: "456", - status: "completed", - conclusion: "failure", - jobs: [ - { - name: receipt.jobName, - status: "completed", - conclusion: "failure", - acceptedRunAttempt: 1, - url: receipt.jobUrl, - }, - { name: "openclaw/ci-gate", status: "completed", conclusion: "success" }, - ], - flakeClassifications: [receipt], - }; - const manifest = { - sourceParentRunAttempt: 1, - childRuns: { normalCi: "456" }, - childEvidence: { normalCi: childEvidence }, - advisoryJobs: [ - { - class: "recorded-flake", - child: "normalCi", - job: receipt.jobName, - conclusion: "failure", - runId: "456", - url: receipt.jobUrl, - jobId: "457", - trackingUrl: receipt.trackingUrl, - reason: receipt.reason, - receiptRunId: "890", - }, - ], - }; - expect(validate({}, manifest).result.source).toBe("sha-pinned-main"); - for (const changed of [ - { parentRunId: "124" }, - { parentRunAttempt: 2 }, - { childRunId: "459" }, - { targetSha: workflowSha }, - { jobId: "458" }, - ]) { - expect(() => - validate( - {}, - { - ...manifest, - childEvidence: { - normalCi: { ...childEvidence, flakeClassifications: [{ ...receipt, ...changed }] }, - }, - }, - ), - ).toThrow(/recorded-flake|classification/u); - } - }); - it("keeps historical recovery outside new selection validation", () => { const expectedPublicationSelection = vi.fn(() => { throw new Error("new selection was evaluated");