test(e2e): separate quarantined multi-tenant diagnostic from tier gates

The shell multi-tenant suite previously selected a spec ignored by every project, preventing real backend diagnosis. Add an explicit desktop-only configuration that rejects tier identity, while retaining the normal quarantine. Cover discovery and tier refusal and document the evidence boundaries. A local source-built diagnostic returned five passes, one failure at organisation-switch login, and one skip; this is not release qualification.

Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot] 2026-09-05 14:37:00 +01:00
parent 7caaa65ffa
commit 0b73d36011
5 changed files with 90 additions and 4 deletions

View file

@ -15,6 +15,18 @@
## Purpose
The shell-owned multi-tenant integration suite uses a dedicated desktop-only
Playwright configuration selecting the seven multi-tenant scenarios. It must
reject any non-empty E2E tier identity rather than impersonating stable or
probation CI. Normal tier selection retains the multi-tenant quarantine.
Diagnostic reports use invocation-specific roots under the shell runner;
direct npm, setup and helper paths do not inherit its isolation guarantees.
These scenarios do not establish delayed-admission, reconnect, interruption,
installed-customer or release qualification. The executable discovery and
tier-refusal checks in `tests/integration/scripts/managed-local-backend.test.mjs`
protect this boundary without launching a browser.
### Portable installer lifecycle ownership
The shared shell installer lifecycle directory (outside the least-privilege

View file

@ -289,3 +289,24 @@ Response behavior can be controlled via environment variables:
- ✅ Core E2E flows pass reliably in CI
- ✅ Update flow remains covered via API integration test + smoke UI check
### Quarantined multi-tenant diagnostic
From the repository root, run:
```sh
pulse-heavy-run -- bash tests/integration/scripts/run-tests.sh multi-tenant
```
This shell-owned run builds isolated images and allocates loopback ports. Its
dedicated `playwright.multi-tenant-diagnostic.config.ts` selects only the seven
desktop multi-tenant scenarios, including known failing scenarios. It refuses
a non-empty `PULSE_E2E_TIER`: unset that variable explicitly for diagnostics.
The normal stable/probation configuration and quarantine are unchanged.
A diagnostic pass is not a stable-gate, installed-customer or release receipt.
Reports remain under the invocation-specific report/result directories for
inspection; treat browser artifacts as potentially sensitive. Direct npm,
setup and helper invocations do not inherit the shell runner's isolation
guarantees. The existing seven scenarios do not establish the delayed admission
and reconnect recovery matrix.

View file

@ -0,0 +1,24 @@
import { defineConfig } from '@playwright/test';
import base from './playwright.config';
// Deliberately separate from stable/probation CI. A diagnostic result cannot
// promote this quarantined spec or be substituted for a stable-tier verdict.
if (process.env.PULSE_E2E_TIER?.trim()) {
throw new Error('Multi-tenant diagnostics cannot run as an E2E tier; unset PULSE_E2E_TIER');
}
export default defineConfig({
...base,
testMatch: ['**/03-multi-tenant.spec.ts'],
outputDir: process.env.PULSE_E2E_RESULTS_DIR || 'test-results/multi-tenant-diagnostic',
reporter: [
['list'],
['html', {
outputFolder: process.env.PULSE_E2E_REPORT_DIR || 'playwright-report/multi-tenant-diagnostic',
open: 'never',
}],
],
projects: base.projects!
.filter(project => project.name === 'chromium')
.map(project => ({ ...project, testIgnore: [] })),
});

View file

@ -29,7 +29,7 @@ test('integration setup pins the governed Node.js major', async () => {
test('buildManagedLocalBackendState uses deterministic defaults', () => {
const state = buildManagedLocalBackendState({});
assert.equal(state.repoRoot.endsWith('/repos/pulse'), true);
assert.equal(state.repoRoot, path.resolve(integrationRoot, '../..'));
assert.equal(state.backendVariant, 'core');
assert.equal(state.baseURL, 'http://127.0.0.1:8765');
assert.equal(state.metricsPort, '0');
@ -70,7 +70,7 @@ test('buildManagedLocalBackendState supports enterprise variant with sibling rep
);
assert.deepEqual(state.binaryBuildArgs, ['build', '-buildvcs=false', '-o', '__OUTPUT__', './cmd/pulse-enterprise']);
assert.ok(state.binarySourceRoots.some((root) => root.endsWith(path.join('pulse-enterprise', 'internal'))));
assert.ok(state.binarySourceRoots.some((root) => root.endsWith(path.join('repos', 'pulse', 'internal'))));
assert.ok(state.binarySourceRoots.includes(path.join(state.repoRoot, 'internal')));
});
test('buildManagedLocalBackendEnv seeds auth, bootstrap token, and billing path', () => {
@ -169,7 +169,8 @@ test('multi-tenant release auth reuses storage state and classifies login rate l
assert.match(multiTenantSpec, /createAuthenticatedStorageState/);
assert.match(multiTenantSpec, /storageState:\s*async\s*\(\{\s*authStorageStatePath\s*\},\s*use\)/);
assert.match(multiTenantSpec, /authStorageStatePath:\s*\[/);
assert.match(multiTenantSpec, /multi-tenant-\$\{workerInfo\.project\.name\}\.json/);
assert.match(multiTenantSpec, /fs\.mkdtempSync\(path\.join\(authRoot, 'multi-tenant-'\)\)/);
assert.match(multiTenantSpec, /finally\s*\{\s*fs\.rmSync\(authDir, \{ recursive: true, force: true \}\)/);
assert.match(multiTenantSpec, /\{\s*scope:\s*'worker'\s*\}/);
assert.match(helpers, /new URL\(response\.url\(\)\)\.pathname === "\/api\/login"/);
@ -386,3 +387,31 @@ test('shouldBuildManagedLocalBackendFrontend skips rebuild when embedded fronten
false,
);
});
// Diagnostic orchestration must not impersonate a stable runtime gate.
import { spawnSync } from 'node:child_process';
function list(config, tier = '') {
return spawnSync(process.execPath, [
'node_modules/@playwright/test/cli.js', 'test',
'--config', config, 'tests/03-multi-tenant.spec.ts', '--list',
], { cwd: integrationRoot, env: { ...process.env, PULSE_E2E_TIER: tier }, encoding: 'utf8' });
}
test('explicit diagnostic discovers only the seven desktop multi-tenant scenarios', () => {
const result = list('playwright.multi-tenant-diagnostic.config.ts');
assert.equal(result.status, 0, result.stdout + result.stderr);
assert.match(result.stdout, /Total: 7 tests in 1 file/);
assert.doesNotMatch(result.stdout, /mobile-chrome|mobile-safari/);
});
for (const tier of ['stable', 'probation']) {
test(`diagnostic refuses ${tier} gate identity`, () => {
const result = list('playwright.multi-tenant-diagnostic.config.ts', tier);
assert.notEqual(result.status, 0);
assert.match(result.stderr, /diagnostics cannot run as an E2E tier/);
});
}
test('normal stable configuration retains multi-tenant quarantine', () => {
const result = list('playwright.config.ts', 'stable');
assert.notEqual(result.status, 0);
assert.match(result.stdout, /Total: 0 tests/);
});

View file

@ -190,7 +190,7 @@ run_suite() {
npx playwright test "tests/06-theme-visual.spec.ts" --project=chromium --reporter=list
;;
multi-tenant)
npx playwright test "tests/03-multi-tenant.spec.ts" --project=chromium --reporter=list
npx playwright test --config=playwright.multi-tenant-diagnostic.config.ts --project=chromium
;;
retired-trial-acquisition)
npx playwright test "tests/07-retired-trial-acquisition.spec.ts" --project=chromium --reporter=list