mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-03 04:38:48 +00:00
46fd0b713e
Across the last 25 bot PRs the median open to merge time was about 33 minutes for one PR and about 45 when three opened together. The critical path was the required Backend tests (api) check, where one go test -race ./internal/api step took about 27 of its 31 minutes. Every other required check finishes within about 12 minutes. internal/api now runs as four Backend tests (api-N) shards. Each shard lists the package's tests with go test -race -list from the commit under test and runs one contiguous quarter of that list in go test's own order, so every test, including ones added later, runs in exactly one shard, and a shard that resolves no tests fails. The split is contiguous rather than interleaved by name because some internal/api tests depend on package state left by the tests just before them. A round-robin split by sorted name failed dozens of tests locally (admin bypass and session store globals), while the four contiguous slices and the full run all pass. A Backend tests (api) verdict job keeps the required check name and fails unless every shard succeeded. Backend tests (rest-0) and (rest-1) keep their names and package split. Local non-race timings put the heaviest slice at about 42 percent of the package, so the api check should drop from about 31 minutes to roughly 14. Core E2E no longer runs the non-gating probation tier on pull requests. It could not gate a PR and the promotion rule counts only main runs, so on a PR it only held each of the eight shard runners about eight minutes longer, which fed the runner queueing seen with concurrent PRs. Push and manual runs still execute it. Go test steps still run for frontend-only changes. Go tests read frontend sources directly (internal/api contract tests, internal/unifiedresources walking frontend-modern/src, and the internal/telemetry repository-wide wording scan), so skipping them by path would drop real coverage.
393 lines
16 KiB
YAML
393 lines
16 KiB
YAML
name: Core E2E Tests
|
|
|
|
on:
|
|
pull_request:
|
|
branches:
|
|
- main
|
|
- pulse/v6-release
|
|
- 'release/v*'
|
|
paths:
|
|
- 'frontend-modern/**'
|
|
- 'internal/**'
|
|
- 'tests/integration/**'
|
|
- 'Dockerfile'
|
|
- '.github/workflows/test-e2e.yml'
|
|
push:
|
|
branches:
|
|
- main
|
|
- master
|
|
- pulse/v6-release
|
|
- 'release/v*'
|
|
paths:
|
|
- 'frontend-modern/**'
|
|
- 'internal/**'
|
|
- 'tests/integration/**'
|
|
- 'Dockerfile'
|
|
- '.github/workflows/test-e2e.yml'
|
|
workflow_dispatch:
|
|
|
|
# Let branch runs finish so busy main and release branches retain verdicts.
|
|
# A newer PR commit supersedes its previous validation, so do not queue the
|
|
# replacement behind obsolete checks. Keep this aligned with Build and Test.
|
|
concurrency:
|
|
group: e2e-${{ github.workflow }}-${{ github.ref }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
tier-selection:
|
|
name: Validate E2E tier selection
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 10
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
with:
|
|
node-version: '24'
|
|
cache: 'npm'
|
|
cache-dependency-path: tests/integration/package-lock.json
|
|
|
|
- name: Validate telemetry schema version parity
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm run check:telemetry-schema-version
|
|
node --test scripts/check-telemetry-schema-version.test.mjs
|
|
|
|
- name: Install integration dependencies
|
|
working-directory: tests/integration
|
|
run: npm ci
|
|
|
|
- name: Validate stable/probation project accounting
|
|
working-directory: tests/integration
|
|
run: npm run check:e2e-tiers
|
|
|
|
- name: Validate stable failure reporting
|
|
working-directory: tests/integration
|
|
run: node --test scripts/report-stable-e2e-failures.test.mjs
|
|
|
|
offline-org-provisioning:
|
|
name: Offline Organization provisioning
|
|
# Fixture acceptance only, not quarantine promotion or private RBAC proof.
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 25
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
with:
|
|
node-version: '24'
|
|
cache: 'npm'
|
|
cache-dependency-path: |
|
|
tests/integration/package-lock.json
|
|
frontend-modern/package-lock.json
|
|
internal/cloudcp/portal/frontend/package-lock.json
|
|
- name: Set up Go
|
|
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
|
with:
|
|
go-version-file: go.mod
|
|
- name: Install locked dependencies
|
|
run: |
|
|
npm ci --prefix tests/integration
|
|
npm ci --prefix frontend-modern
|
|
npm ci --prefix internal/cloudcp/portal/frontend
|
|
cd tests/integration
|
|
npx playwright install --with-deps chromium
|
|
- name: Validate offline issuer boundaries
|
|
working-directory: tests/integration
|
|
run: >-
|
|
node --test scripts/offline-license-issuer.test.mjs
|
|
scripts/with-offline-entitlements.test.mjs
|
|
scripts/entitlement-bootstrap.test.mjs
|
|
- name: Prove authenticated default and created-org activation
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_USE_LOCAL_BACKEND: 'true'
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: 'true'
|
|
run: >-
|
|
node scripts/with-offline-entitlements.mjs node scripts/run-playwright.mjs
|
|
--config=playwright.multi-tenant-diagnostic.config.ts
|
|
--grep 'Scenario 1:|create, update, member manage' --workers=1 --retries=0
|
|
- name: Upload offline provisioning failure report
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: offline-org-provisioning-report
|
|
path: tests/integration/playwright-report/multi-tenant-diagnostic/
|
|
retention-days: 3
|
|
|
|
e2e:
|
|
name: Playwright Core E2E (shard ${{ matrix.shard }}/8)
|
|
needs: tier-selection
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
shard: [1, 2, 3, 4, 5, 6, 7, 8]
|
|
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
with:
|
|
node-version: '24'
|
|
cache: 'npm'
|
|
cache-dependency-path: tests/integration/package-lock.json
|
|
|
|
- name: Install Playwright dependencies
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm ci
|
|
npx playwright install --with-deps chromium webkit
|
|
|
|
- name: Install Pulse Account frontend dependencies
|
|
working-directory: internal/cloudcp/portal/frontend
|
|
run: npm ci
|
|
|
|
- name: Build Docker images for test environment
|
|
# GO_BUILD_TAGS="" drops the release build tag so the suite can enable
|
|
# mock fixtures; release-tag gating has its own -tags release Go tests.
|
|
run: |
|
|
docker build -t pulse:test --target e2e_runtime --build-arg GO_BUILD_TAGS="" .
|
|
docker build -t pulse-mock-github:test ./tests/integration/mock-github-server
|
|
|
|
- name: Start test containers
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: node scripts/pretest.mjs
|
|
|
|
# Two-tier run (see PROBATION_SPECS in tests/integration/e2e-tiering.mjs
|
|
# for the tier mechanism and the promotion/demotion rule). The stable tier
|
|
# is the gate; the probation tier reuses the same containers but reports
|
|
# through continue-on-error so a probation flake cannot paint main red.
|
|
- name: Run stable-tier E2E suite
|
|
id: stable
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_E2E_PERF: "1"
|
|
PULSE_E2E_REQUIRE_DEFAULT_MOCK_READY: "true"
|
|
PULSE_E2E_TIER: "stable"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: npm test -- --shard=${{ matrix.shard }}/8
|
|
|
|
# Playwright's report artifacts preserve the full forensic record, but
|
|
# artifact bodies and job logs require an authenticated GitHub session.
|
|
# Project the bounded failed testcase identity into ordinary check
|
|
# annotations so a red stable gate is diagnosable from the API without
|
|
# rerunning, demoting, or skipping the failure.
|
|
- name: Report stable-tier failures
|
|
if: ${{ !cancelled() && steps.stable.outcome == 'failure' }}
|
|
working-directory: tests/integration
|
|
run: node scripts/report-stable-e2e-failures.mjs test-results/junit.xml
|
|
|
|
# Runs even when the stable tier failed (its data still counts toward
|
|
# promotion), but not when the stable tier was skipped — that means the
|
|
# environment never came up and every probation spec would fail on it.
|
|
# Pull requests skip it. It cannot gate a PR, and the promotion rule in
|
|
# e2e-tiering.mjs counts only main runs, so on a PR it only held each
|
|
# of the eight shard runners for about eight more minutes.
|
|
- name: Run probation-tier E2E suite (non-gating)
|
|
id: probation
|
|
if: ${{ !cancelled() && github.event_name != 'pull_request' && steps.stable.conclusion != 'skipped' }}
|
|
continue-on-error: true
|
|
# A broken probation surface can otherwise consume the entire 60m job
|
|
# budget through retries and cancel the job before continue-on-error
|
|
# can make the observational result non-gating. Playwright's own
|
|
# global timeout preserves its reports; the step timeout is a backstop
|
|
# for runner/process failures outside Playwright.
|
|
timeout-minutes: 12
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_E2E_PERF: "1"
|
|
PULSE_E2E_REQUIRE_DEFAULT_MOCK_READY: "true"
|
|
PULSE_E2E_TIER: "probation"
|
|
PULSE_E2E_REPORT_DIR: playwright-report-probation
|
|
PULSE_E2E_RESULTS_DIR: test-results-probation
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: >-
|
|
npm test -- --shard=${{ matrix.shard }}/8 --pass-with-no-tests
|
|
--max-failures=5 --global-timeout=600000
|
|
|
|
- name: Report probation-tier outcome
|
|
if: ${{ !cancelled() && steps.probation.outcome != 'skipped' }}
|
|
run: |
|
|
if [ "${{ steps.probation.outcome }}" = "failure" ]; then
|
|
echo "⚠️ Probation tier FAILED on shard ${{ matrix.shard }} (non-gating; see the playwright-report-probation artifact). A probation spec's 10-consecutive-green promotion count restarts." >> "$GITHUB_STEP_SUMMARY"
|
|
else
|
|
echo "Probation tier passed on shard ${{ matrix.shard }} (non-gating)." >> "$GITHUB_STEP_SUMMARY"
|
|
fi
|
|
|
|
- name: Collect container logs
|
|
if: always()
|
|
working-directory: tests/integration
|
|
run: |
|
|
echo "=== Docker containers ==="
|
|
docker ps -a
|
|
echo "=== Pulse test server logs ==="
|
|
docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"
|
|
echo "=== Mock GitHub server logs ==="
|
|
docker logs pulse-mock-github 2>&1 || echo "No pulse-mock-github container"
|
|
|
|
- name: Upload Playwright report
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-shard-${{ matrix.shard }}
|
|
path: tests/integration/playwright-report/
|
|
retention-days: 3
|
|
|
|
# The HTML report already contains screenshots, videos and traces.
|
|
# Keep only the separate machine-readable result here.
|
|
- name: Upload JUnit results
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: junit-results-shard-${{ matrix.shard }}
|
|
path: tests/integration/test-results/junit.xml
|
|
retention-days: 3
|
|
|
|
# continue-on-error keeps the job green on a probation failure, so
|
|
# failure() never fires for these; key off the step outcome instead.
|
|
- name: Upload probation Playwright report
|
|
if: ${{ !cancelled() && steps.probation.outcome == 'failure' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-probation-shard-${{ matrix.shard }}
|
|
path: tests/integration/playwright-report-probation/
|
|
retention-days: 3
|
|
|
|
- name: Upload probation JUnit results
|
|
if: ${{ !cancelled() && steps.probation.outcome == 'failure' }}
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: junit-results-probation-shard-${{ matrix.shard }}
|
|
path: tests/integration/test-results-probation/junit.xml
|
|
retention-days: 3
|
|
|
|
agent-registration:
|
|
name: Agent registration lifecycle
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 30
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
|
|
with:
|
|
node-version: '24'
|
|
package-manager-cache: false
|
|
|
|
- name: Install Playwright dependencies
|
|
working-directory: tests/integration
|
|
run: |
|
|
npm ci
|
|
npx playwright install --with-deps chromium
|
|
|
|
- name: Install Pulse Account frontend dependencies
|
|
working-directory: internal/cloudcp/portal/frontend
|
|
run: npm ci
|
|
|
|
- name: Build test images
|
|
run: |
|
|
docker build -t pulse:test --target e2e_runtime --build-arg GO_BUILD_TAGS="" .
|
|
docker build -t pulse-mock-github:test ./tests/integration/mock-github-server
|
|
|
|
- name: Start non-mock test container
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_BASE_URL: http://localhost:7655
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MOCK_MODE: "false"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: node scripts/pretest.mjs
|
|
|
|
- name: Run agent registration lifecycle
|
|
working-directory: tests/integration
|
|
env:
|
|
PULSE_BASE_URL: http://localhost:7655
|
|
PULSE_E2E_AGENT_JOURNEY: "true"
|
|
PULSE_E2E_BOOTSTRAP_TOKEN: 0123456789abcdef0123456789abcdef0123456789abcdef
|
|
PULSE_E2E_SKIP_DOCKER: "true"
|
|
PULSE_E2E_SKIP_PLAYWRIGHT_INSTALL: "true"
|
|
PULSE_MOCK_MODE: "false"
|
|
PULSE_MULTI_TENANT_ENABLED: "true"
|
|
run: npx playwright test tests/journeys/04-agent-install-registration.spec.ts --project=chromium
|
|
|
|
- name: Collect container logs
|
|
if: always()
|
|
working-directory: tests/integration
|
|
run: docker logs pulse-test-server 2>&1 || echo "No pulse-test-server container"
|
|
|
|
- name: Upload Playwright report
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: playwright-report-agent-registration
|
|
path: tests/integration/playwright-report/
|
|
retention-days: 3
|
|
|
|
- name: Upload test videos and screenshots
|
|
if: failure()
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
with:
|
|
name: test-failures-agent-registration
|
|
path: tests/integration/test-results/
|
|
retention-days: 3
|
|
|
|
e2e-verdict:
|
|
name: E2E verdict
|
|
runs-on: ubuntu-24.04
|
|
timeout-minutes: 5
|
|
needs:
|
|
- tier-selection
|
|
- e2e
|
|
- agent-registration
|
|
if: always()
|
|
steps:
|
|
# The e2e shards fail only on stable-tier failures: the probation tier
|
|
# runs behind continue-on-error inside each shard, so its failures are
|
|
# reported (shard summaries + probation artifacts) without reaching
|
|
# this verdict. Mechanism and promotion rule: PROBATION_SPECS in
|
|
# tests/integration/e2e-tiering.mjs.
|
|
- name: Check E2E results
|
|
run: |
|
|
if [ "${{ needs.tier-selection.result }}" != "success" ]; then
|
|
echo "E2E tier selection is invalid (result: ${{ needs.tier-selection.result }})"
|
|
exit 1
|
|
fi
|
|
if [ "${{ needs.e2e.result }}" != "success" ]; then
|
|
echo "Stable-tier E2E shards did not all pass (result: ${{ needs.e2e.result }})"
|
|
exit 1
|
|
fi
|
|
if [ "${{ needs.agent-registration.result }}" != "success" ]; then
|
|
echo "Agent registration lifecycle did not pass (result: ${{ needs.agent-registration.result }})"
|
|
exit 1
|
|
fi
|
|
echo "All stable-tier E2E shards and the agent registration lifecycle passed (probation tier is non-gating)"
|