mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 02:28:59 +00:00
Validate Core E2E tiering against run evidence (#1711)
Contract-Neutral: E2E test tier metadata and validation only; no deployment runtime or public contract change. Co-authored-by: Pulse Autonomous Maintainer <rcourtman@users.noreply.github.com>
This commit is contained in:
rcourtman
GitHub
parent
d9df2b9422
commit
887b85bc83
6 changed files with 275 additions and 81 deletions
|
|
@ -80,10 +80,22 @@ tier lists, the classification data behind them, and the
|
|||
promotion/demotion rule (10 consecutive green main runs promotes a
|
||||
probation spec; one failure or retry-flake demotes it back) are
|
||||
documented next to `PROBATION_SPECS` and `QUARANTINED_SPECS` in
|
||||
`playwright.config.ts`. Quarantined specs are rotted and do not run at
|
||||
`e2e-tiering.mjs`. Quarantined specs are rotted and do not run at
|
||||
all; probation specs run on every push but cannot fail the
|
||||
`e2e-verdict` job in `.github/workflows/test-e2e.yml`.
|
||||
|
||||
Validate the ledger and Playwright's complete per-project split with:
|
||||
|
||||
```bash
|
||||
cd tests/integration
|
||||
npm run check:e2e-tiers
|
||||
```
|
||||
|
||||
The check uses Playwright's own `--list` output, verifies that stable and
|
||||
probation are disjoint and exhaustive across every configured project, rejects
|
||||
stale/duplicate/overlapping ledger entries, and protects a minimum stable-tier
|
||||
signal floor.
|
||||
|
||||
### Eval Packs (No Manual Steps)
|
||||
Run the curated scenario pack (multi-tenant, retired-trial-acquisition, cloud-hosting, cloud-billing-lifecycle) and emit a report:
|
||||
```bash
|
||||
|
|
|
|||
77
tests/integration/e2e-tiering.mjs
Normal file
77
tests/integration/e2e-tiering.mjs
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
/**
|
||||
* Specs that are deliberately not run. Quarantine is reserved for known,
|
||||
* unresolved product/spec incompatibilities and is never an automatic result
|
||||
* of an infrastructure failure.
|
||||
*/
|
||||
export const QUARANTINED_SPECS = [
|
||||
// Multi-tenant scenarios 6/7 have an unresolved org-resolution bug (see
|
||||
// core-e2e-failure-taxonomy): after switchOrg+reload the UI still resolves
|
||||
// the default org. CI runs a multi-tenant environment, so delisting this
|
||||
// spec turns that open bug into a permanent red.
|
||||
'**/03-multi-tenant.spec.ts',
|
||||
'**/47-inline-selection-scroll-stability.spec.ts',
|
||||
'**/48-summary-hover-selection.spec.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
* Specs that run on every push but do not gate the Core E2E verdict. The
|
||||
* stable tier is every non-quarantined spec not listed here.
|
||||
*
|
||||
* Audit (2026-08-11): the 10 most recent completed main workflow records at
|
||||
* 2026-08-11T20:22:40Z contained six executed runs and four concurrency
|
||||
* cancellations with no jobs. Only explicit Playwright `failed` or `flaky`
|
||||
* summaries in the 48 executed shard logs count as incidents; progress lines
|
||||
* and cancelled/non-executed runs do not. The audit found 13 stable-spec
|
||||
* incidents and no shard-wide environment collapse. Existing probation specs
|
||||
* remain because six evidence-bearing runs cannot prove the promotion rule.
|
||||
*
|
||||
* Promotion rule: a spec returns to stable after 10 consecutive executed Core
|
||||
* E2E runs on main with neither a failure nor a retry-pass (`flaky`). Demotion
|
||||
* rule: one genuine spec failure/flake on main returns a stable spec here. A
|
||||
* shared environment or runner failure is recorded separately and never
|
||||
* demotes every spec that happened to be scheduled on the affected shard.
|
||||
*/
|
||||
export const PROBATION_SPECS = [
|
||||
'**/01-core-e2e.spec.ts',
|
||||
'**/02-navigation-perf.spec.ts',
|
||||
'**/04-mobile.spec.ts',
|
||||
'**/05-settings-mobile-audit.spec.ts',
|
||||
'**/06-theme-visual.spec.ts',
|
||||
'**/11-first-session.spec.ts',
|
||||
'**/15-settings-shell-consistency.spec.ts',
|
||||
'**/20-local-doc-links.spec.ts',
|
||||
'**/21-truenas-connections-workspace.spec.ts',
|
||||
'**/28-truenas-alert-resource-links.spec.ts',
|
||||
'**/38-vmware-ai-chat-mentions.spec.ts',
|
||||
'**/39-vmware-resource-detail-drawer.spec.ts',
|
||||
'**/40-vmware-storage-source-filter.spec.ts',
|
||||
'**/41-vmware-phase1-exclusion-integrity.spec.ts',
|
||||
'**/42-vmware-ai-chat-read-recovery.spec.ts',
|
||||
'**/43-platform-mock-runtime.spec.ts',
|
||||
'**/44-workloads-chart-spacing.spec.ts',
|
||||
'**/45-workloads-memory-tail.spec.ts',
|
||||
'**/46-storage-summary-continuity.spec.ts',
|
||||
'**/49-demo-scenario-curation.spec.ts',
|
||||
'**/50-storage-physical-disk-io-history.spec.ts',
|
||||
'**/52-ai-settings-provider-setup.spec.ts',
|
||||
'**/53-demo-mode-commercial-boundary.spec.ts',
|
||||
'**/55-self-hosted-upgrade-return.spec.ts',
|
||||
'**/56-pulse-account-upgrade-bootstrap.spec.ts',
|
||||
'**/59-self-hosted-plans-entitlement-summary.spec.ts',
|
||||
'**/59-workloads-column-layout.spec.ts',
|
||||
'**/62-storage-growth-column.spec.ts',
|
||||
'**/64-workloads-proxmox-refresh-stability.spec.ts',
|
||||
'**/68-infrastructure-onboarding.spec.ts',
|
||||
'**/68-platform-pages-shell.spec.ts',
|
||||
'**/70-self-hosted-manual-activation-success.spec.ts',
|
||||
'**/75-agent-integrations-surface-contract.spec.ts',
|
||||
'**/76-assistant-tool-output-preview.spec.ts',
|
||||
'**/77-msp-isolation.spec.ts',
|
||||
'**/78-monitor-first-patrol-workbench.spec.ts',
|
||||
'**/79-update-flow.spec.ts',
|
||||
'**/81-actions-inbox.spec.ts',
|
||||
'**/90-operational-trust-protection-posture.spec.ts',
|
||||
'**/91-operational-trust-attention-workbench.spec.ts',
|
||||
'**/91-proxmox-node-display-names.spec.ts',
|
||||
'**/92-operational-trust-availability-facet.spec.ts',
|
||||
];
|
||||
|
|
@ -5,6 +5,7 @@
|
|||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node ./scripts/run-playwright.mjs",
|
||||
"check:e2e-tiers": "node ./scripts/check-e2e-tiering.mjs",
|
||||
"evals": "node ./scripts/run-evals.mjs",
|
||||
"test:visual": "npm test -- tests/06-theme-visual.spec.ts --project=chromium",
|
||||
"test:visual:update": "npm test -- tests/06-theme-visual.spec.ts --project=chromium --update-snapshots",
|
||||
|
|
|
|||
|
|
@ -1,84 +1,10 @@
|
|||
import { defineConfig, devices } from '@playwright/test';
|
||||
import {
|
||||
PROBATION_SPECS,
|
||||
QUARANTINED_SPECS,
|
||||
} from './e2e-tiering.mjs';
|
||||
import { preferredBrowserBaseURL } from './tests/runtime-defaults';
|
||||
|
||||
/**
|
||||
* QUARANTINE (2026-07-17): these specs assert the pre-platform-first UI
|
||||
* (element locators, URLs, and subheader copy that the June IA rebuild
|
||||
* replaced) and have failed on every push since 2026-06-29, drowning the
|
||||
* signal from the healthy specs. Quarantining them keeps the rest of the
|
||||
* suite as a live per-push gate while each file is re-pinned to the
|
||||
* current UI and removed from this list. This is a stopgap, not a fix:
|
||||
* the recovery work is tracked as its own effort, and a spec leaves this
|
||||
* list only by being repaired, never by deletion.
|
||||
*/
|
||||
const QUARANTINED_SPECS = [
|
||||
// Multi-tenant scenarios 6/7 have an unresolved org-resolution bug (see
|
||||
// core-e2e-failure-taxonomy): after switchOrg+reload the UI still resolves
|
||||
// the default org. CI runs a multi-tenant environment, so delisting this
|
||||
// spec turns that open bug into a permanent red.
|
||||
'**/03-multi-tenant.spec.ts',
|
||||
'**/47-inline-selection-scroll-stability.spec.ts',
|
||||
'**/48-summary-hover-selection.spec.ts',
|
||||
];
|
||||
|
||||
/**
|
||||
* PROBATION (2026-07-20): specs that run on every push but do NOT gate the
|
||||
* Core E2E verdict. CI runs the suite in two passes per shard, selected by
|
||||
* PULSE_E2E_TIER: "stable" (everything not listed here — gating) and
|
||||
* "probation" (this list — reported, continue-on-error). Locally, with
|
||||
* PULSE_E2E_TIER unset, both tiers run together exactly as before.
|
||||
*
|
||||
* Seeded from per-spec failure data mined out of the "Core E2E Tests"
|
||||
* workflow's failed-shard logs for the 31 completed main runs between the
|
||||
* 2026-07-18 quarantine delist and 2026-07-20: every spec that failed or
|
||||
* retry-flaked within the 10 most recent completed runs starts here; specs
|
||||
* whose last incident is older already satisfy the promotion rule below.
|
||||
*
|
||||
* Promotion rule — a spec earns its way OUT of this list (into the gating
|
||||
* stable tier) after 10 consecutive Core E2E runs on main in which it
|
||||
* neither failed nor went flaky (a retry-pass reported as "flaky" counts as
|
||||
* an incident — the retry hid it from the verdict, not from this ledger).
|
||||
* Demotion rule — one flake on main demotes a stable spec back to this
|
||||
* list, restarting its count. A spec leaves the suite only via QUARANTINE
|
||||
* (still-rotted, doesn't run) or deliberate deletion; probation is for
|
||||
* specs that run green sometimes but haven't yet proven they always do.
|
||||
*/
|
||||
const PROBATION_SPECS = [
|
||||
'**/01-core-e2e.spec.ts',
|
||||
'**/02-navigation-perf.spec.ts',
|
||||
'**/04-mobile.spec.ts',
|
||||
'**/05-settings-mobile-audit.spec.ts',
|
||||
'**/11-first-session.spec.ts',
|
||||
'**/15-settings-shell-consistency.spec.ts',
|
||||
'**/20-local-doc-links.spec.ts',
|
||||
'**/21-truenas-connections-workspace.spec.ts',
|
||||
'**/38-vmware-ai-chat-mentions.spec.ts',
|
||||
'**/39-vmware-resource-detail-drawer.spec.ts',
|
||||
'**/40-vmware-storage-source-filter.spec.ts',
|
||||
'**/41-vmware-phase1-exclusion-integrity.spec.ts',
|
||||
'**/42-vmware-ai-chat-read-recovery.spec.ts',
|
||||
'**/43-platform-mock-runtime.spec.ts',
|
||||
'**/44-workloads-chart-spacing.spec.ts',
|
||||
'**/45-workloads-memory-tail.spec.ts',
|
||||
'**/46-storage-summary-continuity.spec.ts',
|
||||
'**/49-demo-scenario-curation.spec.ts',
|
||||
'**/50-storage-physical-disk-io-history.spec.ts',
|
||||
'**/56-pulse-account-upgrade-bootstrap.spec.ts',
|
||||
'**/59-workloads-column-layout.spec.ts',
|
||||
'**/62-storage-growth-column.spec.ts',
|
||||
'**/64-workloads-proxmox-refresh-stability.spec.ts',
|
||||
// Demoted 2026-08-05: rate-limit/shared-state retry flake on main in
|
||||
// run 31033254657; it must earn 10 consecutive green runs to return.
|
||||
'**/68-infrastructure-onboarding.spec.ts',
|
||||
'**/68-platform-pages-shell.spec.ts',
|
||||
'**/77-msp-isolation.spec.ts',
|
||||
'**/79-update-flow.spec.ts',
|
||||
// Demoted 2026-07-20: failed on main in run 29731882505, the first
|
||||
// tiered run, one run after clearing the 10-green seeding window.
|
||||
'**/90-operational-trust-protection-posture.spec.ts',
|
||||
'**/91-proxmox-node-display-names.spec.ts',
|
||||
];
|
||||
|
||||
const E2E_TIER = String(process.env.PULSE_E2E_TIER || '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
|
|
|
|||
149
tests/integration/scripts/check-e2e-tiering.mjs
Normal file
149
tests/integration/scripts/check-e2e-tiering.mjs
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import {
|
||||
PROBATION_SPECS,
|
||||
QUARANTINED_SPECS,
|
||||
} from '../e2e-tiering.mjs';
|
||||
|
||||
const integrationRoot = path.resolve(fileURLToPath(new URL('..', import.meta.url)));
|
||||
const testsRoot = path.join(integrationRoot, 'tests');
|
||||
const playwrightBin = path.join(integrationRoot, 'node_modules', '.bin', 'playwright');
|
||||
const MIN_STABLE_SPEC_FILES = 10;
|
||||
|
||||
function listSpecFiles() {
|
||||
const specs = [];
|
||||
function walk(directory, relativeDirectory = '') {
|
||||
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
||||
if (entry.isDirectory()) {
|
||||
walk(path.join(directory, entry.name), `${relativeDirectory}${entry.name}/`);
|
||||
} else if (entry.name.endsWith('.spec.ts')) {
|
||||
specs.push(`${relativeDirectory}${entry.name}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
walk(testsRoot);
|
||||
return specs.sort();
|
||||
}
|
||||
|
||||
function duplicates(values) {
|
||||
const seen = new Set();
|
||||
return values.filter((value) => {
|
||||
if (seen.has(value)) return true;
|
||||
seen.add(value);
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
function ledgerFiles(values, label, allFiles, errors) {
|
||||
const badFormat = values.filter((value) => !/^\*\*\/.+\.spec\.ts$/.test(value));
|
||||
if (badFormat.length) errors.push(`${label} has invalid entries: ${badFormat.join(', ')}`);
|
||||
|
||||
const duplicateValues = duplicates(values);
|
||||
if (duplicateValues.length) errors.push(`${label} has duplicates: ${duplicateValues.join(', ')}`);
|
||||
|
||||
const sortedValues = [...values].sort();
|
||||
if (values.some((value, index) => value !== sortedValues[index])) {
|
||||
errors.push(`${label} must be sorted`);
|
||||
}
|
||||
|
||||
const files = new Set(values.map((value) => value.replace(/^\*\*\//, '')));
|
||||
const unknown = [...files].filter((value) => !allFiles.has(value));
|
||||
if (unknown.length) errors.push(`${label} has unknown specs: ${unknown.join(', ')}`);
|
||||
return files;
|
||||
}
|
||||
|
||||
function listPlaywrightTests(tier) {
|
||||
if (!fs.existsSync(playwrightBin)) {
|
||||
throw new Error('Playwright is not installed; run npm ci before this check');
|
||||
}
|
||||
const env = { ...process.env };
|
||||
if (tier) env.PULSE_E2E_TIER = tier;
|
||||
else delete env.PULSE_E2E_TIER;
|
||||
|
||||
const result = spawnSync(playwrightBin, ['test', '--list'], {
|
||||
cwd: integrationRoot,
|
||||
env,
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
if (result.status !== 0) {
|
||||
throw new Error(
|
||||
`Playwright --list failed for ${tier || 'all'} tier:\n${result.stderr || result.stdout}`,
|
||||
);
|
||||
}
|
||||
|
||||
const tests = new Map();
|
||||
for (const rawLine of result.stdout.split('\n')) {
|
||||
const line = rawLine.replace(/\x1b\[[0-9;]*m/g, '');
|
||||
const match = line.match(/^\s*\[([^\]]+)\] › ((.+?\.spec\.ts):\d+:\d+ › .+)$/);
|
||||
if (!match) continue;
|
||||
const key = `[${match[1]}] › ${match[2]}`;
|
||||
if (tests.has(key)) throw new Error(`Playwright listed a duplicate test: ${key}`);
|
||||
tests.set(key, match[3]);
|
||||
}
|
||||
return tests;
|
||||
}
|
||||
|
||||
function difference(left, right) {
|
||||
return new Set([...left].filter((value) => !right.has(value)));
|
||||
}
|
||||
|
||||
function sameSet(actual, expected, label, errors) {
|
||||
const missing = difference(expected, actual);
|
||||
const extra = difference(actual, expected);
|
||||
if (missing.size) errors.push(`${label} is missing: ${[...missing].join(', ')}`);
|
||||
if (extra.size) errors.push(`${label} has unexpected entries: ${[...extra].join(', ')}`);
|
||||
}
|
||||
|
||||
const errors = [];
|
||||
const allFiles = new Set(listSpecFiles());
|
||||
const probationFiles = ledgerFiles(PROBATION_SPECS, 'PROBATION_SPECS', allFiles, errors);
|
||||
const quarantineFiles = ledgerFiles(QUARANTINED_SPECS, 'QUARANTINED_SPECS', allFiles, errors);
|
||||
const overlap = [...probationFiles].filter((value) => quarantineFiles.has(value));
|
||||
if (overlap.length) errors.push(`Probation/quarantine overlap: ${overlap.join(', ')}`);
|
||||
|
||||
const allTests = listPlaywrightTests(null);
|
||||
const stableTests = listPlaywrightTests('stable');
|
||||
const probationTests = listPlaywrightTests('probation');
|
||||
const allKeys = new Set(allTests.keys());
|
||||
const stableKeys = new Set(stableTests.keys());
|
||||
const probationKeys = new Set(probationTests.keys());
|
||||
|
||||
const instanceOverlap = [...stableKeys].filter((key) => probationKeys.has(key));
|
||||
if (instanceOverlap.length) errors.push(`Stable/probation test overlap: ${instanceOverlap.join(', ')}`);
|
||||
sameSet(new Set([...stableKeys, ...probationKeys]), allKeys, 'Stable + probation tests', errors);
|
||||
|
||||
const listedAllFiles = new Set(allTests.values());
|
||||
const listedStableFiles = new Set(stableTests.values());
|
||||
const listedProbationFiles = new Set(probationTests.values());
|
||||
const expectedAllFiles = difference(allFiles, quarantineFiles);
|
||||
const expectedStableFiles = difference(expectedAllFiles, probationFiles);
|
||||
sameSet(listedAllFiles, expectedAllFiles, 'All-tier spec files', errors);
|
||||
sameSet(listedStableFiles, expectedStableFiles, 'Stable-tier spec files', errors);
|
||||
sameSet(listedProbationFiles, probationFiles, 'Probation-tier spec files', errors);
|
||||
|
||||
if (listedStableFiles.size < MIN_STABLE_SPEC_FILES) {
|
||||
errors.push(
|
||||
`Stable tier has ${listedStableFiles.size} spec files; the signal floor is ${MIN_STABLE_SPEC_FILES}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (errors.length) {
|
||||
console.error('E2E tier selection validation failed:');
|
||||
for (const error of errors) console.error(`- ${error}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log('E2E tier selection validation passed');
|
||||
console.log(` spec_files: ${allFiles.size}`);
|
||||
console.log(` stable_spec_files: ${listedStableFiles.size}`);
|
||||
console.log(` probation_spec_files: ${listedProbationFiles.size}`);
|
||||
console.log(` quarantined_spec_files: ${quarantineFiles.size}`);
|
||||
console.log(` test_project_instances: ${allTests.size}`);
|
||||
console.log(` stable_test_project_instances: ${stableTests.size}`);
|
||||
console.log(` probation_test_project_instances: ${probationTests.size}`);
|
||||
Loading…
Add table
Add a link
Reference in a new issue