diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index f4da4f6fd2..07a6bbbedb 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -248,7 +248,41 @@ jobs: require_windows_signing: ${{ needs.prepare.outputs.require_windows_signing == 'true' }} windows_signing_backend: signpath - # Frontend checks run in parallel with backend tests + # Build the embed bundle independently so backend and smoke lanes can start + # without waiting for the full frontend quality suite. + frontend_bundle: + needs: prepare + if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' }} + runs-on: [self-hosted, Linux, X64, pulse-pve-build] + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: '20' + cache: 'npm' + cache-dependency-path: 'frontend-modern/package-lock.json' + + - name: Install dependencies + run: npm --prefix frontend-modern ci + + - name: Build frontend bundle + run: npm --prefix frontend-modern run build + + - name: Upload frontend bundle + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: release-frontend-${{ github.sha }} + path: frontend-modern/dist/ + if-no-files-found: error + retention-days: 1 + compression-level: 0 + overwrite: true + + # Frontend checks run independently from the bundle and backend lanes. frontend_checks: needs: prepare if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' }} @@ -283,19 +317,6 @@ jobs: - name: Test frontend run: npm --prefix frontend-modern test - - name: Build verified frontend bundle - run: npm --prefix frontend-modern run build - - - name: Upload verified frontend bundle - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: release-frontend-${{ github.sha }} - path: frontend-modern/dist/ - if-no-files-found: error - retention-days: 1 - compression-level: 0 - overwrite: true - windows_install_command_smoke: name: Windows PowerShell 5.1 Install Command Smoke needs: prepare @@ -321,18 +342,16 @@ jobs: working-directory: frontend-modern run: npm test -- --run src/utils/__tests__/agentInstallCommand.windows.test.ts - # Backend tests run in parallel with frontend checks + # The dedicated PVE test runner provides the memory needed to run two + # complete, disjoint internal/api shards while all other packages run in a + # third lane. It holds no signing or publication credentials. backend_tests: needs: - prepare - - frontend_checks + - frontend_bundle if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' }} - runs-on: ubuntu-24.04 - # The race-enabled internal/api package has measured at 24 minutes on the - # hosted runner. Keep the job ceiling above the Makefile's 30-minute - # per-package timeout so setup and result collection cannot become the - # effective release gate. - timeout-minutes: 40 + runs-on: [self-hosted, Linux, X64, pulse-pve-tests] + timeout-minutes: 20 env: FRONTEND_DIST: frontend-modern/dist steps: @@ -358,9 +377,7 @@ jobs: cache: true - name: Run backend tests - env: - PULSE_DATA_DIR: /tmp/pulse-test-data - run: make test + run: ./scripts/run-release-backend-tests.sh --data-root "$RUNNER_TEMP/pulse-test-data" # Docker build - amd64 only for prereleases, multi-arch for stable docker_build: @@ -558,7 +575,7 @@ jobs: integration_tests: needs: - prepare - - frontend_checks + - frontend_bundle if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' && needs.prepare.outputs.is_prerelease != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 45 @@ -721,7 +738,7 @@ jobs: release_smoke: needs: - prepare - - frontend_checks + - frontend_bundle if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' }} runs-on: ubuntu-24.04 timeout-minutes: 30 @@ -819,6 +836,7 @@ jobs: needs: - prepare - build_release_candidate + - frontend_bundle - frontend_checks - windows_install_command_smoke - backend_tests @@ -828,7 +846,7 @@ jobs: - release_smoke # Run if integration_tests passed OR was skipped (prereleases). The # release smoke has no skipped escape: it runs for prereleases too. - if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' && always() && needs.build_release_candidate.result == 'success' && needs.frontend_checks.result == 'success' && needs.windows_install_command_smoke.result == 'success' && needs.backend_tests.result == 'success' && needs.docker_build.result == 'success' && needs.helm_smoke.result == 'success' && needs.release_smoke.result == 'success' && (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') }} + if: ${{ needs.prepare.outputs.historical_asset_backfill_only != 'true' && always() && needs.build_release_candidate.result == 'success' && needs.frontend_bundle.result == 'success' && needs.frontend_checks.result == 'success' && needs.windows_install_command_smoke.result == 'success' && needs.backend_tests.result == 'success' && needs.docker_build.result == 'success' && needs.helm_smoke.result == 'success' && needs.release_smoke.result == 'success' && (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') }} runs-on: ubuntu-24.04 timeout-minutes: 30 permissions: diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 063fe42dcc..c67562c4e9 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -9644,7 +9644,21 @@ ], "coverage_gaps": [], "candidate_lanes": [], - "work_claims": [], + "work_claims": [ + { + "id": "release-acceleration-lane-l1", + "agent_id": "release-acceleration", + "summary": "Reduce warm RC dispatch-to-convergence time to 15 minutes with credential-isolated PVE runners, deterministic sharding, and cross-repo build parallelism.", + "target_id": "v6-product-lane-expansion", + "claimed_at": "2026-08-21T11:23:13Z", + "heartbeat_at": "2026-08-21T11:23:13Z", + "expires_at": "2026-08-21T19:23:13Z", + "work_item": { + "kind": "lane", + "id": "L1" + } + } + ], "open_decisions": [], "source_of_truth_file": "docs/release-control/v6/internal/SOURCE_OF_TRUTH.md", "resolved_decisions": [ diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index be17045b73..0738430047 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -92,6 +92,8 @@ discarding existing explicit disk exclusions. 34. `go.mod` 35. `go.sum` 36. `scripts/build-release.sh` +37. `scripts/run-release-backend-tests.sh` +38. `scripts/shard_go_tests.py` 37. `scripts/generate-release-notes.sh` 37. `scripts/check-workflow-dispatch-inputs.py` 38. `scripts/clean-mock-alerts.sh` @@ -485,7 +487,7 @@ upgrade, update, release, or artifact-selection behavior. ## Extension Points 1. Add or change deployment-type detection, update planning, or apply behavior through `internal/updates/` -2. Add or change release-build metadata injection, Docker build-context allowlists, release artifact assembly, governed promotion metadata resolution, artifact release-line validation, post-install live-runtime claim proof, the canonical version file, operator-facing release packet content, prerelease feedback intake wording, historical published-release integrity backfill, release asset validation status publication, download endpoint checksum/signature header proof, end-to-end install.sh smoke against staged or published release assets, or the canonical in-repo v6 upgrade guide through `scripts/build-release.sh`, `scripts/release_asset_common.sh`, `scripts/backfill-release-assets.sh`, `scripts/release_ldflags.sh`, `scripts/check-workflow-dispatch-inputs.py`, `scripts/release-preflight-worker.sh`, `scripts/run-release-preflight.sh`, `scripts/release_control/live_runtime_proof.py`, `scripts/release_control/live_runtime_proof_test.py`, `scripts/release_control/mobile_release_gate.py`, `scripts/release_control/render_release_body.py`, `scripts/release_control/resolve_release_promotion.py`, `scripts/release_control/validate_artifact_release_line.py`, `scripts/release_control/record_rc_to_ga_rehearsal.py`, `scripts/release_control/internal/record_rc_to_ga_rehearsal.py`, `scripts/release_control/release_promotion_policy_support.py`, `.dockerignore`, `Dockerfile`, `.github/ISSUE_TEMPLATE/v6_rc_feedback.yml`, `docs/RELEASE_NOTES.md`, `docs/releases/`, `docs/UPGRADE_v6.md`, `docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md`, `docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md`, `docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md`, `scripts/validate-release.sh`, `scripts/validate-published-release.sh`, the operator dispatch helpers `scripts/trigger-release.sh` and `scripts/trigger-release-dry-run.sh`, and the governed release workflows `.github/workflows/backfill-release-assets.yml`, `.github/workflows/create-release.yml`, `.github/workflows/deploy-demo-server.yml`, `.github/workflows/helm-pages.yml`, `.github/workflows/install-sh-smoke.yml`, `.github/workflows/promote-floating-tags.yml`, `.github/workflows/promote-private-pro-runtime.yml`, `.github/workflows/publish-docker.yml`, `.github/workflows/publish-helm-chart.yml`, `.github/workflows/release-convergence.yml`, `.github/workflows/release-dry-run.yml`, `.github/workflows/retry-release-convergence.yml`, `.github/workflows/update-demo-server.yml`, and `.github/workflows/validate-release-assets.yml` +2. Add or change release-build metadata injection, Docker build-context allowlists, release artifact assembly, governed promotion metadata resolution, artifact release-line validation, post-install live-runtime claim proof, the canonical version file, operator-facing release packet content, prerelease feedback intake wording, historical published-release integrity backfill, release asset validation status publication, download endpoint checksum/signature header proof, end-to-end install.sh smoke against staged or published release assets, or the canonical in-repo v6 upgrade guide through `scripts/build-release.sh`, `scripts/run-release-backend-tests.sh`, `scripts/shard_go_tests.py`, `scripts/release_asset_common.sh`, `scripts/backfill-release-assets.sh`, `scripts/release_ldflags.sh`, `scripts/check-workflow-dispatch-inputs.py`, `scripts/release-preflight-worker.sh`, `scripts/run-release-preflight.sh`, `scripts/release_control/live_runtime_proof.py`, `scripts/release_control/live_runtime_proof_test.py`, `scripts/release_control/mobile_release_gate.py`, `scripts/release_control/render_release_body.py`, `scripts/release_control/resolve_release_promotion.py`, `scripts/release_control/validate_artifact_release_line.py`, `scripts/release_control/record_rc_to_ga_rehearsal.py`, `scripts/release_control/internal/record_rc_to_ga_rehearsal.py`, `scripts/release_control/release_promotion_policy_support.py`, `.dockerignore`, `Dockerfile`, `.github/ISSUE_TEMPLATE/v6_rc_feedback.yml`, `docs/RELEASE_NOTES.md`, `docs/releases/`, `docs/UPGRADE_v6.md`, `docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md`, `docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md`, `docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md`, `scripts/validate-release.sh`, `scripts/validate-published-release.sh`, the operator dispatch helpers `scripts/trigger-release.sh` and `scripts/trigger-release-dry-run.sh`, and the governed release workflows `.github/workflows/backfill-release-assets.yml`, `.github/workflows/create-release.yml`, `.github/workflows/deploy-demo-server.yml`, `.github/workflows/helm-pages.yml`, `.github/workflows/install-sh-smoke.yml`, `.github/workflows/promote-floating-tags.yml`, `.github/workflows/promote-private-pro-runtime.yml`, `.github/workflows/publish-docker.yml`, `.github/workflows/publish-helm-chart.yml`, `.github/workflows/release-convergence.yml`, `.github/workflows/release-dry-run.yml`, `.github/workflows/retry-release-convergence.yml`, `.github/workflows/update-demo-server.yml`, and `.github/workflows/validate-release-assets.yml` Operators may configure an external Linux amd64 worker for the two trigger helpers. `scripts/run-release-preflight.sh` must resolve an immutable pushed commit, stream the worker implementation stored in that commit over SSH, @@ -505,12 +507,27 @@ upgrade, update, release, or artifact-selection behavior. persistent run directory and remove that bounded scratch directory on exit; a small WSL `/tmp` tmpfs must not turn release qualification into a false product failure. - The canonical backend race gate must likewise budget for hosted-runner - variance without weakening test coverage. `make test` owns a 30-minute - per-package timeout, and the release workflow's backend job owns a larger - 40-minute ceiling so checkout, toolchain setup, and result collection cannot - become a tighter implicit deadline. The release-promotion contract test must - pin both sides of that headroom relationship. + The exact-SHA worker must start frontend quality, backend qualification, and + integration-environment preparation as independent lanes after producing + the frontend embed bundle. Its receipt must distinguish elapsed wall time + from the sum of overlapping phase times. + The canonical public release workflow must use dedicated, credential-free + PVE runner identities for the frontend embed bundle and backend race gate. + Signing, package publication, and release mutation authority must remain on + hosted jobs. The bundle job must be independent from frontend quality so + backend and browser-smoke lanes can start as soon as the bundle is available. + The backend runner must compile the race-enabled `internal/api` test binary + once, enumerate every top-level test from that exact binary, and generate a + deterministic manifest proving a complete, disjoint partition. Two API + shards and the remaining Go packages may then execute concurrently with + isolated data directories. A failed shard must fail the complete backend + gate; sharding must never select a coverage subset. The backend job owns a + 20-minute ceiling, while each invocation retains the canonical 30-minute Go + timeout as protection against a stuck package. + The warm-path release-control performance objective is 15 minutes or less + from dispatch to definitive publication/convergence. This objective is an + optimization target, not permission to weaken exact-SHA qualification, + signing, artifact integrity, installer smoke, or convergence proof. The rehearsal diagnostic spec is opt-in by design, so both the hosted rehearsal and its worker profile must set `PULSE_E2E_DIAGNOSTIC=1`; invoking that spec while leaving it skipped is not browser proof. diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index dfddb1d7c7..a128a8116d 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4229,10 +4229,12 @@ "scripts/release_control/resolve_release_promotion.py", "scripts/release_control/validate_artifact_release_line.py", "scripts/release_ldflags.sh", + "scripts/run-release-backend-tests.sh", "scripts/run-release-preflight.sh", "scripts/run_cloud_public_signup_smoke.sh", "scripts/run_demo_public_browser_smoke.sh", "scripts/run_hosted_staging_smoke.sh", + "scripts/shard_go_tests.py", "scripts/toggle-mock.sh", "scripts/trigger-release-dry-run.sh", "scripts/trigger-release.sh", @@ -4374,7 +4376,9 @@ "scripts/release_control/render_release_body.py", "scripts/release_control/resolve_release_promotion.py", "scripts/release_control/validate_artifact_release_line.py", + "scripts/run-release-backend-tests.sh", "scripts/run-release-preflight.sh", + "scripts/shard_go_tests.py", "scripts/trigger-release-dry-run.sh", "scripts/trigger-release.sh", "scripts/trigger-stable-patch.sh", diff --git a/internal/api/security_regression_test.go b/internal/api/security_regression_test.go index 0ba0139040..90f23bcf3f 100644 --- a/internal/api/security_regression_test.go +++ b/internal/api/security_regression_test.go @@ -5228,7 +5228,15 @@ func TestSystemSettingsWebhookAllowlistPropagatesToAllTenantManagers(t *testing. } } - h := NewSystemSettingsHandler(cfg, persistence, nil, mtm, nil, func() {}, nil) + // Match the production router, which always supplies its already-created + // primary monitor. Passing nil here makes the constructor create and start a + // default monitor asynchronously against cfg just before this test mutates + // cfg, introducing a test-only race that production cannot take. + primaryMonitor, ok := mtm.PeekMonitor("org-a") + if !ok { + t.Fatal("org-a monitor was not initialized") + } + h := NewSystemSettingsHandler(cfg, persistence, nil, mtm, primaryMonitor, func() {}, nil) // Both org managers must reject the private target before the update. for _, orgID := range []string{"org-a", "org-b"} { diff --git a/scripts/release-preflight-worker.sh b/scripts/release-preflight-worker.sh index dbc1a57a49..3c91ccda88 100755 --- a/scripts/release-preflight-worker.sh +++ b/scripts/release-preflight-worker.sh @@ -39,7 +39,7 @@ GO_TMP_DIR="${RUN_DIR}/go-tmp" TIMINGS_FILE="${RUN_DIR}/timings.tsv" TEST_DATA_DIR="${WORKER_ROOT}/test-data/${PROFILE}" -for command_name in git go node npm docker curl flock timeout; do +for command_name in git go node npm docker curl flock timeout python3; do if ! command -v "$command_name" >/dev/null 2>&1; then echo "Error: required worker command is missing: ${command_name}" >&2 exit 3 @@ -68,6 +68,7 @@ if ! flock -n 9; then echo "Error: another release preflight is already using this worker." >&2 exit 5 fi +WALL_STARTED="$(date +%s)" # A preflight compiles and tests only. Keep publication and signing authority # out of the worker even if its login shell happens to define these names. @@ -129,29 +130,63 @@ if [ -n "$EXPECTED_GO" ] && [ "$ACTUAL_GO" != "$EXPECTED_GO" ]; then fi phase frontend-dependencies npm --prefix frontend-modern ci -phase frontend-lint npm --prefix frontend-modern run lint -phase frontend-headers npm --prefix frontend-modern run lint:headers -phase frontend-duplication npm --prefix frontend-modern run lint:cpd -phase frontend-types npm --prefix frontend-modern run type-check -phase frontend-tests npm --prefix frontend-modern test phase frontend-build npm --prefix frontend-modern run build rm -rf internal/api/frontend-modern mkdir -p internal/api/frontend-modern cp -R frontend-modern/dist internal/api/frontend-modern/ -rm -rf "$TEST_DATA_DIR" -mkdir -p "$TEST_DATA_DIR" +run_frontend_quality() { + phase frontend-lint npm --prefix frontend-modern run lint + phase frontend-headers npm --prefix frontend-modern run lint:headers + phase frontend-duplication npm --prefix frontend-modern run lint:cpd + phase frontend-types npm --prefix frontend-modern run type-check + phase frontend-tests npm --prefix frontend-modern test +} -if [ "$PROFILE" = "rehearsal" ]; then - phase backend-serial env PULSE_DATA_DIR="$TEST_DATA_DIR" go test -p 1 ./... -else - phase backend-race env PULSE_DATA_DIR="$TEST_DATA_DIR" make test -fi +run_backend() { + rm -rf "$TEST_DATA_DIR" + mkdir -p "$TEST_DATA_DIR" + if [ "$PROFILE" = "rehearsal" ]; then + phase backend-serial env PULSE_DATA_DIR="$TEST_DATA_DIR" go test -p 1 ./... + else + phase backend-race-sharded ./scripts/run-release-backend-tests.sh \ + --data-root "$TEST_DATA_DIR" \ + --api-shards auto + fi +} + +run_integration_prep() { + phase integration-dependencies npm --prefix tests/integration ci + PLAYWRIGHT_VERSION="$(node -p "require('./tests/integration/node_modules/@playwright/test/package.json').version")" + PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" + phase playwright-image docker pull "$PLAYWRIGHT_IMAGE" + phase mock-github-image docker build --tag pulse-mock-github:test tests/integration/mock-github-server +} + +# These lanes have no output dependency on one another. Running them together +# makes the critical path the slowest lane instead of their sum. +parallel_pids=() +run_frontend_quality & +parallel_pids+=("$!") +run_backend & +parallel_pids+=("$!") +run_integration_prep & +parallel_pids+=("$!") + +remaining="${#parallel_pids[@]}" +while [ "$remaining" -gt 0 ]; do + if wait -n; then + remaining=$((remaining - 1)) + else + status=$? + kill "${parallel_pids[@]}" >/dev/null 2>&1 || true + wait "${parallel_pids[@]}" >/dev/null 2>&1 || true + exit "$status" + fi +done -phase integration-dependencies npm --prefix tests/integration ci PLAYWRIGHT_VERSION="$(node -p "require('./tests/integration/node_modules/@playwright/test/package.json').version")" PLAYWRIGHT_IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" -phase playwright-image docker pull "$PLAYWRIGHT_IMAGE" VERSION="$(tr -d '\r\n' < VERSION)" if [ "$PROFILE" = "rehearsal" ]; then @@ -170,8 +205,6 @@ else --tag pulse:test \ . fi -phase mock-github-image docker build --tag pulse-mock-github:test tests/integration/mock-github-server - run_playwright() { docker run --rm \ --network host \ @@ -231,15 +264,17 @@ else fi FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" +WALL_SECONDS="$(($(date +%s) - WALL_STARTED))" TOTAL_SECONDS="$(awk -F '\t' '{ total += $2 } END { print total + 0 }' "$TIMINGS_FILE")" RECEIPT_PATH="${RECEIPT_DIR}/${RUN_ID}.json" { printf '{\n' - printf ' "schema_version": 1,\n' + printf ' "schema_version": 2,\n' printf ' "source_sha": "%s",\n' "$SOURCE_SHA" printf ' "profile": "%s",\n' "$PROFILE" printf ' "architecture": "%s",\n' "$(uname -m)" printf ' "finished_at": "%s",\n' "$FINISHED_AT" + printf ' "wall_seconds": %s,\n' "$WALL_SECONDS" printf ' "total_phase_seconds": %s,\n' "$TOTAL_SECONDS" printf ' "result": "success"\n' printf '}\n' @@ -249,5 +284,6 @@ echo echo "Exact-SHA release preflight passed." echo "Source SHA: ${SOURCE_SHA}" echo "Profile: ${PROFILE}" +echo "Wall time: ${WALL_SECONDS}s" echo "Phase time: ${TOTAL_SECONDS}s" echo "Receipt: ${RECEIPT_PATH}" diff --git a/scripts/release_control/internal/release_preflight_test.py b/scripts/release_control/internal/release_preflight_test.py index 6d8492b55f..d6e1c87a30 100755 --- a/scripts/release_control/internal/release_preflight_test.py +++ b/scripts/release_control/internal/release_preflight_test.py @@ -4,10 +4,15 @@ from __future__ import annotations import os import pathlib import subprocess +import sys +import tempfile import unittest ROOT = pathlib.Path(__file__).resolve().parents[3] +sys.path.insert(0, str(ROOT / "scripts")) + +from shard_go_tests import build_plan, write_plan class ReleasePreflightTest(unittest.TestCase): @@ -44,6 +49,15 @@ class ReleasePreflightTest(unittest.TestCase): self.assertIn("Runtime: WSL Ubuntu", result.stdout) def test_missing_optional_worker_is_a_non_gate(self) -> None: + configured_host = subprocess.run( + ["git", "config", "--get", "pulse.releasePreflightHost"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + if configured_host.returncode == 0: + self.skipTest("repository has a local release-preflight host configured") result = self.run_script( "--profile", "release", @@ -54,15 +68,6 @@ class ReleasePreflightTest(unittest.TestCase): "GIT_CONFIG_NOSYSTEM": "1", }, ) - configured_host = subprocess.run( - ["git", "config", "--get", "pulse.releasePreflightHost"], - cwd=ROOT, - check=False, - capture_output=True, - text=True, - ) - if configured_host.returncode == 0: - self.skipTest("repository has a local release-preflight host configured") self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("canonical hosted checks", result.stdout) @@ -104,6 +109,57 @@ class ReleasePreflightTest(unittest.TestCase): ) self.assertIn("is not reachable from a fetched origin branch", runner) + def test_api_shard_plan_is_deterministic_complete_and_disjoint(self) -> None: + test_names = [f"TestReleaseCase{index:04d}" for index in range(17)] + plan = build_plan(test_names, shard_count=2, batch_size=4) + reordered_plan = build_plan(list(reversed(test_names)), shard_count=2, batch_size=4) + self.assertEqual(plan, reordered_plan) + + assigned = [ + name + for shard in plan["shards"] + for batch in shard["batches"] + for name in batch + ] + self.assertCountEqual(assigned, test_names) + self.assertEqual(len(assigned), len(set(assigned))) + self.assertEqual( + [name for batch in plan["shards"][0]["batches"] for name in batch], + sorted(test_names)[:9], + ) + self.assertEqual( + [name for batch in plan["shards"][1]["batches"] for name in batch], + sorted(test_names)[9:], + ) + + with tempfile.TemporaryDirectory() as directory: + manifest_path = write_plan(plan, pathlib.Path(directory)) + manifest = manifest_path.read_text() + self.assertIn('"test_count": 17', manifest) + self.assertTrue((pathlib.Path(directory) / "shard-01-batch-01.regex").is_file()) + self.assertTrue((pathlib.Path(directory) / "shard-02-batch-01.regex").is_file()) + + def test_api_shard_plan_rejects_invalid_or_duplicate_names(self) -> None: + with self.assertRaisesRegex(ValueError, "invalid top-level"): + build_plan(["TestGood", "BenchmarkNotATest"], 2, 10) + with self.assertRaisesRegex(ValueError, "duplicate top-level"): + build_plan(["TestDuplicate", "TestDuplicate"], 2, 10) + with self.assertRaisesRegex(ValueError, "cannot exceed"): + build_plan(["TestOnly"], 2, 10) + + def test_release_workflow_uses_independent_pve_bundle_and_backend_lanes(self) -> None: + workflow = (ROOT / ".github/workflows/create-release.yml").read_text() + backend = (ROOT / "scripts/run-release-backend-tests.sh").read_text() + self.assertIn("frontend_bundle:", workflow) + self.assertIn("runs-on: [self-hosted, Linux, X64, pulse-pve-build]", workflow) + self.assertIn("runs-on: [self-hosted, Linux, X64, pulse-pve-tests]", workflow) + self.assertIn("./scripts/run-release-backend-tests.sh", workflow) + self.assertNotIn(" - frontend_checks\n if:", workflow[workflow.index(" backend_tests:"):workflow.index(" docker_build:")]) + self.assertIn("go test -c -race", backend) + self.assertIn("python3 scripts/shard_go_tests.py", backend) + self.assertIn("export GITHUB_ACTIONS=true", backend) + self.assertIn("export CI=true", backend) + if __name__ == "__main__": unittest.main() diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.sh new file mode 100755 index 0000000000..08af0fbe9c --- /dev/null +++ b/scripts/run-release-backend-tests.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: run-release-backend-tests.sh [options] + +Options: + --data-root DIR Isolated test-data root (required) + --api-shards VALUE auto or a positive integer (default: auto) + --batch-size VALUE Top-level tests per API test-binary invocation (default: 10000) +EOF +} + +DATA_ROOT="" +API_SHARDS="${PULSE_BACKEND_TEST_SHARDS:-auto}" +BATCH_SIZE=10000 + +while [ "$#" -gt 0 ]; do + case "$1" in + --data-root) + DATA_ROOT="${2:-}" + shift 2 + ;; + --api-shards) + API_SHARDS="${2:-}" + shift 2 + ;; + --batch-size) + BATCH_SIZE="${2:-}" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Error: unknown option: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +if [ -z "$DATA_ROOT" ]; then + echo "Error: --data-root is required." >&2 + exit 2 +fi +if [[ ! "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: --batch-size must be a positive integer." >&2 + exit 2 +fi + +for command_name in go python3 getconf awk; do + if ! command -v "$command_name" >/dev/null 2>&1; then + echo "Error: required backend-test command is missing: $command_name" >&2 + exit 3 + fi +done + +REPOSITORY_ROOT="$(git rev-parse --show-toplevel)" +cd "$REPOSITORY_ROOT" +# This is the canonical release gate even when exercised directly on the +# worker. Preserve hosted-CI security and isolation semantics in both paths. +export GITHUB_ACTIONS=true +export CI=true +RUN_ROOT="$DATA_ROOT/release-backend-run" +PLAN_DIR="$RUN_ROOT/api-plan" +API_BINARY="$RUN_ROOT/internal-api.test" +TEST_NAMES_FILE="$RUN_ROOT/internal-api-tests.txt" +rm -rf "$RUN_ROOT" +mkdir -p "$PLAN_DIR" "$RUN_ROOT/data/other" + +VCPUS="$(getconf _NPROCESSORS_ONLN)" +AVAILABLE_KIB="$(awk '/^MemAvailable:/ { print $2; exit }' /proc/meminfo)" +if [ -z "$AVAILABLE_KIB" ]; then + AVAILABLE_KIB=0 +fi + +if [ "$API_SHARDS" = auto ]; then + cpu_shards=$((VCPUS / 4)) + memory_shards=$(((AVAILABLE_KIB - 2097152) / 4194304)) + if [ "$cpu_shards" -lt 1 ]; then cpu_shards=1; fi + if [ "$memory_shards" -lt 1 ]; then memory_shards=1; fi + API_SHARDS="$cpu_shards" + if [ "$memory_shards" -lt "$API_SHARDS" ]; then API_SHARDS="$memory_shards"; fi + if [ "$API_SHARDS" -gt 2 ]; then API_SHARDS=2; fi +elif [[ ! "$API_SHARDS" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: --api-shards must be auto or a positive integer." >&2 + exit 2 +fi + +echo "Release backend test plan" +echo " vCPUs: $VCPUS" +echo " Available MiB: $((AVAILABLE_KIB / 1024))" +echo " API shards: $API_SHARDS" +echo " Batch size: $BATCH_SIZE" + +./scripts/ensure_test_assets.sh + +echo "Compiling the race-enabled internal/api test binary once..." +go test -c -race -o "$API_BINARY" ./internal/api +( + cd internal/api + "$API_BINARY" -test.list '^Test' +) | awk '/^Test[A-Za-z0-9_]+$/ { print }' > "$TEST_NAMES_FILE" + +python3 scripts/shard_go_tests.py \ + --tests-file "$TEST_NAMES_FILE" \ + --shards "$API_SHARDS" \ + --batch-size "$BATCH_SIZE" \ + --output-dir "$PLAN_DIR" + +API_IMPORT_PATH="$(go list -f '{{.ImportPath}}' ./internal/api)" +mapfile -t OTHER_PACKAGES < <( + go list ./cmd/... ./internal/... ./pkg/... ./scripts/... ./tests/... \ + | awk -v api="$API_IMPORT_PATH" '$0 != api' +) +if [ "${#OTHER_PACKAGES[@]}" -eq 0 ]; then + echo "Error: no non-API backend test packages were discovered." >&2 + exit 4 +fi + +run_other_packages() { + echo "Running ${#OTHER_PACKAGES[@]} non-API packages..." + env PULSE_DATA_DIR="$RUN_ROOT/data/other" \ + go test -race -timeout 30m "${OTHER_PACKAGES[@]}" +} + +run_api_shard() { + local shard_index="$1" + local shard_dir="$RUN_ROOT/data/api-shard-$shard_index" + local regex_file regex batch_index=0 + mkdir -p "$shard_dir" + shopt -s nullglob + local regex_files=("$PLAN_DIR"/shard-"$shard_index"-batch-*.regex) + if [ "${#regex_files[@]}" -eq 0 ]; then + echo "Error: API shard $shard_index has no batches." >&2 + return 4 + fi + for regex_file in "${regex_files[@]}"; do + batch_index=$((batch_index + 1)) + mkdir -p "$shard_dir/batch-$batch_index" + regex="$(tr -d '\r\n' < "$regex_file")" + echo "Running internal/api shard $shard_index batch $batch_index/${#regex_files[@]}..." + ( + cd internal/api + env \ + GOMAXPROCS="$((VCPUS / API_SHARDS))" \ + PULSE_DATA_DIR="$shard_dir/batch-$batch_index" \ + "$API_BINARY" -test.run "$regex" -test.timeout 30m + ) + done +} + +pids=() +run_other_packages & +pids+=("$!") +for shard_number in $(seq 1 "$API_SHARDS"); do + shard_index="$(printf '%02d' "$shard_number")" + run_api_shard "$shard_index" & + pids+=("$!") +done + +remaining="${#pids[@]}" +while [ "$remaining" -gt 0 ]; do + if wait -n; then + remaining=$((remaining - 1)) + else + status=$? + kill "${pids[@]}" >/dev/null 2>&1 || true + wait "${pids[@]}" >/dev/null 2>&1 || true + exit "$status" + fi +done + +echo "Race-enabled backend release tests passed with $API_SHARDS API shard(s)." diff --git a/scripts/shard_go_tests.py b/scripts/shard_go_tests.py new file mode 100755 index 0000000000..c17746c67a --- /dev/null +++ b/scripts/shard_go_tests.py @@ -0,0 +1,135 @@ +#!/usr/bin/env python3 +"""Create deterministic, coverage-complete batches for a Go test package.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path +from typing import Sequence + + +TEST_NAME_PATTERN = re.compile(r"^Test[A-Za-z0-9_]+$") + + +def _digest(names: Sequence[str]) -> str: + payload = "".join(f"{name}\n" for name in names).encode() + return hashlib.sha256(payload).hexdigest() + + +def build_plan( + test_names: Sequence[str], shard_count: int, batch_size: int +) -> dict[str, object]: + if shard_count < 1: + raise ValueError("shard_count must be at least 1") + if batch_size < 1: + raise ValueError("batch_size must be at least 1") + if not test_names: + raise ValueError("the Go package did not expose any top-level tests") + if shard_count > len(test_names): + raise ValueError("shard_count cannot exceed the number of top-level tests") + + invalid = sorted({name for name in test_names if not TEST_NAME_PATTERN.fullmatch(name)}) + if invalid: + raise ValueError(f"invalid top-level Go test name(s): {', '.join(invalid)}") + if len(set(test_names)) != len(test_names): + duplicates = sorted( + name for name in set(test_names) if test_names.count(name) > 1 + ) + raise ValueError(f"duplicate top-level Go test name(s): {', '.join(duplicates)}") + + canonical = sorted(test_names) + # Preserve the suite's canonical ordering inside each shard. Some legacy + # API tests still exercise package-global state, so hash distribution or + # extra process boundaries can change their historical neighbours. + base_size, remainder = divmod(len(canonical), shard_count) + shards: list[list[str]] = [] + offset = 0 + for shard_index in range(shard_count): + size = base_size + (1 if shard_index < remainder else 0) + shards.append(canonical[offset : offset + size]) + offset += size + + shard_records: list[dict[str, object]] = [] + assigned: list[str] = [] + for shard_index, names in enumerate(shards, start=1): + ordered_names = sorted(names) + assigned.extend(ordered_names) + batches = [ + ordered_names[offset : offset + batch_size] + for offset in range(0, len(ordered_names), batch_size) + ] + shard_records.append( + { + "index": shard_index, + "test_count": len(ordered_names), + "test_names_sha256": _digest(ordered_names), + "batches": batches, + } + ) + + if sorted(assigned) != canonical or len(assigned) != len(set(assigned)): + raise RuntimeError("generated shard plan is not a complete, disjoint partition") + + return { + "schema_version": 1, + "test_count": len(canonical), + "test_names_sha256": _digest(canonical), + "shard_count": shard_count, + "batch_size": batch_size, + "shards": shard_records, + } + + +def write_plan(plan: dict[str, object], output_dir: Path) -> Path: + output_dir.mkdir(parents=True, exist_ok=True) + manifest_shards: list[dict[str, object]] = [] + for shard in plan["shards"]: # type: ignore[index] + shard_record = dict(shard) + batches = shard_record.pop("batches") + batch_records: list[dict[str, object]] = [] + for batch_index, names in enumerate(batches, start=1): + filename = f"shard-{shard_record['index']:02d}-batch-{batch_index:02d}.regex" + regex = "^(?:" + "|".join(re.escape(name) for name in names) + ")$" + (output_dir / filename).write_text(f"{regex}\n") + batch_records.append( + { + "index": batch_index, + "test_count": len(names), + "regex_file": filename, + "test_names_sha256": _digest(names), + } + ) + shard_record["batches"] = batch_records + manifest_shards.append(shard_record) + + manifest = dict(plan) + manifest["shards"] = manifest_shards + manifest_path = output_dir / "manifest.json" + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n") + return manifest_path + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--tests-file", type=Path, required=True) + parser.add_argument("--shards", type=int, required=True) + parser.add_argument("--batch-size", type=int, default=10000) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + + test_names = [ + line.strip() + for line in args.tests_file.read_text().splitlines() + if line.strip() + ] + plan = build_plan(test_names, args.shards, args.batch_size) + manifest_path = write_plan(plan, args.output_dir) + print(manifest_path.read_text(), end="") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())