From 887b85bc834599171bf6ea510c207f3e8acbb8e9 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 11 Aug 2026 23:18:14 +0100 Subject: [PATCH] Validate Core E2E tiering against run evidence (#1711) Contract-Neutral: E2E test tier metadata and validation only; no deployment runtime or public contract change. Co-authored-by: Pulse Autonomous Maintainer --- .github/workflows/test-e2e.yml | 33 +++- tests/integration/README.md | 14 +- tests/integration/e2e-tiering.mjs | 77 +++++++++ tests/integration/package.json | 1 + tests/integration/playwright.config.ts | 82 +--------- .../integration/scripts/check-e2e-tiering.mjs | 149 ++++++++++++++++++ 6 files changed, 275 insertions(+), 81 deletions(-) create mode 100644 tests/integration/e2e-tiering.mjs create mode 100644 tests/integration/scripts/check-e2e-tiering.mjs diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 60e8e16feb..39e9b7b882 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -35,8 +35,32 @@ permissions: contents: read jobs: + tier-selection: + name: Validate E2E tier selection + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: tests/integration/package-lock.json + + - name: Install integration dependencies + working-directory: tests/integration + run: npm ci + + - name: Validate stable/probation project accounting + working-directory: tests/integration + run: npm run check:e2e-tiers + e2e: name: Playwright Core E2E (shard ${{ matrix.shard }}/8) + needs: tier-selection runs-on: ubuntu-24.04 timeout-minutes: 60 strategy: @@ -82,7 +106,7 @@ jobs: PULSE_MULTI_TENANT_ENABLED: "true" run: node scripts/pretest.mjs - # Two-tier run (see PROBATION_SPECS in tests/integration/playwright.config.ts + # Two-tier run (see PROBATION_SPECS in tests/integration/e2e-tiering.mjs # for the tier mechanism and the promotion/demotion rule). The stable tier # is the gate; the probation tier reuses the same containers but reports # through continue-on-error so a probation flake cannot paint main red. @@ -252,6 +276,7 @@ jobs: name: E2E verdict runs-on: ubuntu-24.04 needs: + - tier-selection - e2e - agent-registration if: always() @@ -260,9 +285,13 @@ jobs: # runs behind continue-on-error inside each shard, so its failures are # reported (shard summaries + probation artifacts) without reaching # this verdict. Mechanism and promotion rule: PROBATION_SPECS in - # tests/integration/playwright.config.ts. + # tests/integration/e2e-tiering.mjs. - name: Check E2E results run: | + if [ "${{ needs.tier-selection.result }}" != "success" ]; then + echo "E2E tier selection is invalid (result: ${{ needs.tier-selection.result }})" + exit 1 + fi if [ "${{ needs.e2e.result }}" != "success" ]; then echo "Stable-tier E2E shards did not all pass (result: ${{ needs.e2e.result }})" exit 1 diff --git a/tests/integration/README.md b/tests/integration/README.md index 309ad60de9..b1065a33c2 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -80,10 +80,22 @@ tier lists, the classification data behind them, and the promotion/demotion rule (10 consecutive green main runs promotes a probation spec; one failure or retry-flake demotes it back) are documented next to `PROBATION_SPECS` and `QUARANTINED_SPECS` in -`playwright.config.ts`. Quarantined specs are rotted and do not run at +`e2e-tiering.mjs`. Quarantined specs are rotted and do not run at all; probation specs run on every push but cannot fail the `e2e-verdict` job in `.github/workflows/test-e2e.yml`. +Validate the ledger and Playwright's complete per-project split with: + +```bash +cd tests/integration +npm run check:e2e-tiers +``` + +The check uses Playwright's own `--list` output, verifies that stable and +probation are disjoint and exhaustive across every configured project, rejects +stale/duplicate/overlapping ledger entries, and protects a minimum stable-tier +signal floor. + ### Eval Packs (No Manual Steps) Run the curated scenario pack (multi-tenant, retired-trial-acquisition, cloud-hosting, cloud-billing-lifecycle) and emit a report: ```bash diff --git a/tests/integration/e2e-tiering.mjs b/tests/integration/e2e-tiering.mjs new file mode 100644 index 0000000000..1ea7e10de8 --- /dev/null +++ b/tests/integration/e2e-tiering.mjs @@ -0,0 +1,77 @@ +/** + * Specs that are deliberately not run. Quarantine is reserved for known, + * unresolved product/spec incompatibilities and is never an automatic result + * of an infrastructure failure. + */ +export const QUARANTINED_SPECS = [ + // Multi-tenant scenarios 6/7 have an unresolved org-resolution bug (see + // core-e2e-failure-taxonomy): after switchOrg+reload the UI still resolves + // the default org. CI runs a multi-tenant environment, so delisting this + // spec turns that open bug into a permanent red. + '**/03-multi-tenant.spec.ts', + '**/47-inline-selection-scroll-stability.spec.ts', + '**/48-summary-hover-selection.spec.ts', +]; + +/** + * Specs that run on every push but do not gate the Core E2E verdict. The + * stable tier is every non-quarantined spec not listed here. + * + * Audit (2026-08-11): the 10 most recent completed main workflow records at + * 2026-08-11T20:22:40Z contained six executed runs and four concurrency + * cancellations with no jobs. Only explicit Playwright `failed` or `flaky` + * summaries in the 48 executed shard logs count as incidents; progress lines + * and cancelled/non-executed runs do not. The audit found 13 stable-spec + * incidents and no shard-wide environment collapse. Existing probation specs + * remain because six evidence-bearing runs cannot prove the promotion rule. + * + * Promotion rule: a spec returns to stable after 10 consecutive executed Core + * E2E runs on main with neither a failure nor a retry-pass (`flaky`). Demotion + * rule: one genuine spec failure/flake on main returns a stable spec here. A + * shared environment or runner failure is recorded separately and never + * demotes every spec that happened to be scheduled on the affected shard. + */ +export const PROBATION_SPECS = [ + '**/01-core-e2e.spec.ts', + '**/02-navigation-perf.spec.ts', + '**/04-mobile.spec.ts', + '**/05-settings-mobile-audit.spec.ts', + '**/06-theme-visual.spec.ts', + '**/11-first-session.spec.ts', + '**/15-settings-shell-consistency.spec.ts', + '**/20-local-doc-links.spec.ts', + '**/21-truenas-connections-workspace.spec.ts', + '**/28-truenas-alert-resource-links.spec.ts', + '**/38-vmware-ai-chat-mentions.spec.ts', + '**/39-vmware-resource-detail-drawer.spec.ts', + '**/40-vmware-storage-source-filter.spec.ts', + '**/41-vmware-phase1-exclusion-integrity.spec.ts', + '**/42-vmware-ai-chat-read-recovery.spec.ts', + '**/43-platform-mock-runtime.spec.ts', + '**/44-workloads-chart-spacing.spec.ts', + '**/45-workloads-memory-tail.spec.ts', + '**/46-storage-summary-continuity.spec.ts', + '**/49-demo-scenario-curation.spec.ts', + '**/50-storage-physical-disk-io-history.spec.ts', + '**/52-ai-settings-provider-setup.spec.ts', + '**/53-demo-mode-commercial-boundary.spec.ts', + '**/55-self-hosted-upgrade-return.spec.ts', + '**/56-pulse-account-upgrade-bootstrap.spec.ts', + '**/59-self-hosted-plans-entitlement-summary.spec.ts', + '**/59-workloads-column-layout.spec.ts', + '**/62-storage-growth-column.spec.ts', + '**/64-workloads-proxmox-refresh-stability.spec.ts', + '**/68-infrastructure-onboarding.spec.ts', + '**/68-platform-pages-shell.spec.ts', + '**/70-self-hosted-manual-activation-success.spec.ts', + '**/75-agent-integrations-surface-contract.spec.ts', + '**/76-assistant-tool-output-preview.spec.ts', + '**/77-msp-isolation.spec.ts', + '**/78-monitor-first-patrol-workbench.spec.ts', + '**/79-update-flow.spec.ts', + '**/81-actions-inbox.spec.ts', + '**/90-operational-trust-protection-posture.spec.ts', + '**/91-operational-trust-attention-workbench.spec.ts', + '**/91-proxmox-node-display-names.spec.ts', + '**/92-operational-trust-availability-facet.spec.ts', +]; diff --git a/tests/integration/package.json b/tests/integration/package.json index b2ba727c45..f3eac27746 100644 --- a/tests/integration/package.json +++ b/tests/integration/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "test": "node ./scripts/run-playwright.mjs", + "check:e2e-tiers": "node ./scripts/check-e2e-tiering.mjs", "evals": "node ./scripts/run-evals.mjs", "test:visual": "npm test -- tests/06-theme-visual.spec.ts --project=chromium", "test:visual:update": "npm test -- tests/06-theme-visual.spec.ts --project=chromium --update-snapshots", diff --git a/tests/integration/playwright.config.ts b/tests/integration/playwright.config.ts index c8469a7834..e3dfe504c8 100644 --- a/tests/integration/playwright.config.ts +++ b/tests/integration/playwright.config.ts @@ -1,84 +1,10 @@ import { defineConfig, devices } from '@playwright/test'; +import { + PROBATION_SPECS, + QUARANTINED_SPECS, +} from './e2e-tiering.mjs'; import { preferredBrowserBaseURL } from './tests/runtime-defaults'; -/** - * QUARANTINE (2026-07-17): these specs assert the pre-platform-first UI - * (element locators, URLs, and subheader copy that the June IA rebuild - * replaced) and have failed on every push since 2026-06-29, drowning the - * signal from the healthy specs. Quarantining them keeps the rest of the - * suite as a live per-push gate while each file is re-pinned to the - * current UI and removed from this list. This is a stopgap, not a fix: - * the recovery work is tracked as its own effort, and a spec leaves this - * list only by being repaired, never by deletion. - */ -const QUARANTINED_SPECS = [ - // Multi-tenant scenarios 6/7 have an unresolved org-resolution bug (see - // core-e2e-failure-taxonomy): after switchOrg+reload the UI still resolves - // the default org. CI runs a multi-tenant environment, so delisting this - // spec turns that open bug into a permanent red. - '**/03-multi-tenant.spec.ts', - '**/47-inline-selection-scroll-stability.spec.ts', - '**/48-summary-hover-selection.spec.ts', -]; - -/** - * PROBATION (2026-07-20): specs that run on every push but do NOT gate the - * Core E2E verdict. CI runs the suite in two passes per shard, selected by - * PULSE_E2E_TIER: "stable" (everything not listed here — gating) and - * "probation" (this list — reported, continue-on-error). Locally, with - * PULSE_E2E_TIER unset, both tiers run together exactly as before. - * - * Seeded from per-spec failure data mined out of the "Core E2E Tests" - * workflow's failed-shard logs for the 31 completed main runs between the - * 2026-07-18 quarantine delist and 2026-07-20: every spec that failed or - * retry-flaked within the 10 most recent completed runs starts here; specs - * whose last incident is older already satisfy the promotion rule below. - * - * Promotion rule — a spec earns its way OUT of this list (into the gating - * stable tier) after 10 consecutive Core E2E runs on main in which it - * neither failed nor went flaky (a retry-pass reported as "flaky" counts as - * an incident — the retry hid it from the verdict, not from this ledger). - * Demotion rule — one flake on main demotes a stable spec back to this - * list, restarting its count. A spec leaves the suite only via QUARANTINE - * (still-rotted, doesn't run) or deliberate deletion; probation is for - * specs that run green sometimes but haven't yet proven they always do. - */ -const PROBATION_SPECS = [ - '**/01-core-e2e.spec.ts', - '**/02-navigation-perf.spec.ts', - '**/04-mobile.spec.ts', - '**/05-settings-mobile-audit.spec.ts', - '**/11-first-session.spec.ts', - '**/15-settings-shell-consistency.spec.ts', - '**/20-local-doc-links.spec.ts', - '**/21-truenas-connections-workspace.spec.ts', - '**/38-vmware-ai-chat-mentions.spec.ts', - '**/39-vmware-resource-detail-drawer.spec.ts', - '**/40-vmware-storage-source-filter.spec.ts', - '**/41-vmware-phase1-exclusion-integrity.spec.ts', - '**/42-vmware-ai-chat-read-recovery.spec.ts', - '**/43-platform-mock-runtime.spec.ts', - '**/44-workloads-chart-spacing.spec.ts', - '**/45-workloads-memory-tail.spec.ts', - '**/46-storage-summary-continuity.spec.ts', - '**/49-demo-scenario-curation.spec.ts', - '**/50-storage-physical-disk-io-history.spec.ts', - '**/56-pulse-account-upgrade-bootstrap.spec.ts', - '**/59-workloads-column-layout.spec.ts', - '**/62-storage-growth-column.spec.ts', - '**/64-workloads-proxmox-refresh-stability.spec.ts', - // Demoted 2026-08-05: rate-limit/shared-state retry flake on main in - // run 31033254657; it must earn 10 consecutive green runs to return. - '**/68-infrastructure-onboarding.spec.ts', - '**/68-platform-pages-shell.spec.ts', - '**/77-msp-isolation.spec.ts', - '**/79-update-flow.spec.ts', - // Demoted 2026-07-20: failed on main in run 29731882505, the first - // tiered run, one run after clearing the 10-green seeding window. - '**/90-operational-trust-protection-posture.spec.ts', - '**/91-proxmox-node-display-names.spec.ts', -]; - const E2E_TIER = String(process.env.PULSE_E2E_TIER || '') .trim() .toLowerCase(); diff --git a/tests/integration/scripts/check-e2e-tiering.mjs b/tests/integration/scripts/check-e2e-tiering.mjs new file mode 100644 index 0000000000..617b97183d --- /dev/null +++ b/tests/integration/scripts/check-e2e-tiering.mjs @@ -0,0 +1,149 @@ +#!/usr/bin/env node + +import fs from 'node:fs'; +import path from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; + +import { + PROBATION_SPECS, + QUARANTINED_SPECS, +} from '../e2e-tiering.mjs'; + +const integrationRoot = path.resolve(fileURLToPath(new URL('..', import.meta.url))); +const testsRoot = path.join(integrationRoot, 'tests'); +const playwrightBin = path.join(integrationRoot, 'node_modules', '.bin', 'playwright'); +const MIN_STABLE_SPEC_FILES = 10; + +function listSpecFiles() { + const specs = []; + function walk(directory, relativeDirectory = '') { + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + if (entry.isDirectory()) { + walk(path.join(directory, entry.name), `${relativeDirectory}${entry.name}/`); + } else if (entry.name.endsWith('.spec.ts')) { + specs.push(`${relativeDirectory}${entry.name}`); + } + } + } + walk(testsRoot); + return specs.sort(); +} + +function duplicates(values) { + const seen = new Set(); + return values.filter((value) => { + if (seen.has(value)) return true; + seen.add(value); + return false; + }); +} + +function ledgerFiles(values, label, allFiles, errors) { + const badFormat = values.filter((value) => !/^\*\*\/.+\.spec\.ts$/.test(value)); + if (badFormat.length) errors.push(`${label} has invalid entries: ${badFormat.join(', ')}`); + + const duplicateValues = duplicates(values); + if (duplicateValues.length) errors.push(`${label} has duplicates: ${duplicateValues.join(', ')}`); + + const sortedValues = [...values].sort(); + if (values.some((value, index) => value !== sortedValues[index])) { + errors.push(`${label} must be sorted`); + } + + const files = new Set(values.map((value) => value.replace(/^\*\*\//, ''))); + const unknown = [...files].filter((value) => !allFiles.has(value)); + if (unknown.length) errors.push(`${label} has unknown specs: ${unknown.join(', ')}`); + return files; +} + +function listPlaywrightTests(tier) { + if (!fs.existsSync(playwrightBin)) { + throw new Error('Playwright is not installed; run npm ci before this check'); + } + const env = { ...process.env }; + if (tier) env.PULSE_E2E_TIER = tier; + else delete env.PULSE_E2E_TIER; + + const result = spawnSync(playwrightBin, ['test', '--list'], { + cwd: integrationRoot, + env, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error( + `Playwright --list failed for ${tier || 'all'} tier:\n${result.stderr || result.stdout}`, + ); + } + + const tests = new Map(); + for (const rawLine of result.stdout.split('\n')) { + const line = rawLine.replace(/\x1b\[[0-9;]*m/g, ''); + const match = line.match(/^\s*\[([^\]]+)\] › ((.+?\.spec\.ts):\d+:\d+ › .+)$/); + if (!match) continue; + const key = `[${match[1]}] › ${match[2]}`; + if (tests.has(key)) throw new Error(`Playwright listed a duplicate test: ${key}`); + tests.set(key, match[3]); + } + return tests; +} + +function difference(left, right) { + return new Set([...left].filter((value) => !right.has(value))); +} + +function sameSet(actual, expected, label, errors) { + const missing = difference(expected, actual); + const extra = difference(actual, expected); + if (missing.size) errors.push(`${label} is missing: ${[...missing].join(', ')}`); + if (extra.size) errors.push(`${label} has unexpected entries: ${[...extra].join(', ')}`); +} + +const errors = []; +const allFiles = new Set(listSpecFiles()); +const probationFiles = ledgerFiles(PROBATION_SPECS, 'PROBATION_SPECS', allFiles, errors); +const quarantineFiles = ledgerFiles(QUARANTINED_SPECS, 'QUARANTINED_SPECS', allFiles, errors); +const overlap = [...probationFiles].filter((value) => quarantineFiles.has(value)); +if (overlap.length) errors.push(`Probation/quarantine overlap: ${overlap.join(', ')}`); + +const allTests = listPlaywrightTests(null); +const stableTests = listPlaywrightTests('stable'); +const probationTests = listPlaywrightTests('probation'); +const allKeys = new Set(allTests.keys()); +const stableKeys = new Set(stableTests.keys()); +const probationKeys = new Set(probationTests.keys()); + +const instanceOverlap = [...stableKeys].filter((key) => probationKeys.has(key)); +if (instanceOverlap.length) errors.push(`Stable/probation test overlap: ${instanceOverlap.join(', ')}`); +sameSet(new Set([...stableKeys, ...probationKeys]), allKeys, 'Stable + probation tests', errors); + +const listedAllFiles = new Set(allTests.values()); +const listedStableFiles = new Set(stableTests.values()); +const listedProbationFiles = new Set(probationTests.values()); +const expectedAllFiles = difference(allFiles, quarantineFiles); +const expectedStableFiles = difference(expectedAllFiles, probationFiles); +sameSet(listedAllFiles, expectedAllFiles, 'All-tier spec files', errors); +sameSet(listedStableFiles, expectedStableFiles, 'Stable-tier spec files', errors); +sameSet(listedProbationFiles, probationFiles, 'Probation-tier spec files', errors); + +if (listedStableFiles.size < MIN_STABLE_SPEC_FILES) { + errors.push( + `Stable tier has ${listedStableFiles.size} spec files; the signal floor is ${MIN_STABLE_SPEC_FILES}`, + ); +} + +if (errors.length) { + console.error('E2E tier selection validation failed:'); + for (const error of errors) console.error(`- ${error}`); + process.exit(1); +} + +console.log('E2E tier selection validation passed'); +console.log(` spec_files: ${allFiles.size}`); +console.log(` stable_spec_files: ${listedStableFiles.size}`); +console.log(` probation_spec_files: ${listedProbationFiles.size}`); +console.log(` quarantined_spec_files: ${quarantineFiles.size}`); +console.log(` test_project_instances: ${allTests.size}`); +console.log(` stable_test_project_instances: ${stableTests.size}`); +console.log(` probation_test_project_instances: ${probationTests.size}`);