From 0293f67635a2b69cea049ab3a91f590fc84b6450 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 21 Aug 2026 12:46:50 +0100 Subject: [PATCH] Preserve API test order in release shards --- .../subsystems/deployment-installability.md | 16 +++++++---- .../installtests/build_release_assets_test.go | 28 +++++++++++++++---- .../internal/release_preflight_test.py | 13 +++++---- scripts/run-release-backend-tests.sh | 17 +++++++++-- scripts/shard_go_tests.py | 13 +++++---- 5 files changed, 62 insertions(+), 25 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 0738430047..264c91a08e 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -518,12 +518,16 @@ upgrade, update, release, or artifact-selection behavior. backend and browser-smoke lanes can start as soon as the bundle is available. The backend runner must compile the race-enabled `internal/api` test binary once, enumerate every top-level test from that exact binary, and generate a - deterministic manifest proving a complete, disjoint partition. Two API - shards and the remaining Go packages may then execute concurrently with - isolated data directories. A failed shard must fail the complete backend - gate; sharding must never select a coverage subset. The backend job owns a - 20-minute ceiling, while each invocation retains the canonical 30-minute Go - timeout as protection against a stuck package. + deterministic manifest proving a complete, disjoint partition. Each + partition must preserve the exact order emitted by that binary; sorting or + hash distribution is not equivalent while legacy tests still exercise + package-global state. Two API shards and the remaining Go packages may then + execute concurrently with isolated data directories. A failed shard must + fail the complete backend gate and terminate every descendant test process; + sharding must never select a coverage subset or leave orphan race binaries + consuming the worker. The backend job owns a 20-minute ceiling, while each + invocation retains the canonical 30-minute Go timeout as protection against + a stuck package. The warm-path release-control performance objective is 15 minutes or less from dispatch to definitive publication/convergence. This objective is an optimization target, not permission to weaken exact-SHA qualification, diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index dc31db1f1a..8e293607d8 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -2044,7 +2044,7 @@ func TestReleasePipelinePromotesOneImmutableCandidate(t *testing.T) { t.Fatal("create_release must promote the verified candidate instead of rebuilding release assets") } - if !strings.Contains(backendJob, "- frontend_checks") || !strings.Contains(integrationJob, "- frontend_checks") { + if !strings.Contains(backendJob, "- frontend_bundle") || !strings.Contains(integrationJob, "- frontend_bundle") { t.Fatal("backend and integration jobs must consume the shared verified frontend bundle") } if strings.Contains(integrationJob, "- backend_tests") { @@ -2341,7 +2341,7 @@ func TestCreateReleasePublishesPrivateProRuntime(t *testing.T) { } } -func TestReleaseBackendRaceGateHasHostedRunnerHeadroom(t *testing.T) { +func TestReleaseBackendRaceGateUsesCompletePVEPartition(t *testing.T) { makefileBytes, err := os.ReadFile(repoFile("Makefile")) if err != nil { t.Fatalf("read Makefile: %v", err) @@ -2358,11 +2358,27 @@ func TestReleaseBackendRaceGateHasHostedRunnerHeadroom(t *testing.T) { t.Fatalf("read create-release.yml: %v", err) } backendJob := workflowJobBlock(t, string(workflowBytes), "backend_tests") - if !strings.Contains(backendJob, "timeout-minutes: 40") { - t.Fatal("release backend job must leave setup and result-collection headroom above the package timeout") + if !strings.Contains(backendJob, "timeout-minutes: 20") { + t.Fatal("release backend job must retain the measured PVE execution ceiling") } - if !strings.Contains(backendJob, "run: make test") { - t.Fatal("release backend job must consume the governed Makefile race-suite timeout") + if !strings.Contains(backendJob, "pulse-pve-tests") || !strings.Contains(backendJob, "run-release-backend-tests.sh") { + t.Fatal("release backend job must use the dedicated PVE partition runner") + } + + backendScriptBytes, err := os.ReadFile(repoFile("scripts", "run-release-backend-tests.sh")) + if err != nil { + t.Fatalf("read run-release-backend-tests.sh: %v", err) + } + backendScript := string(backendScriptBytes) + for _, needle := range []string{ + "go test -c -race", + "python3 scripts/shard_go_tests.py", + "go test -race -timeout 30m", + `-test.timeout 30m`, + } { + if !strings.Contains(backendScript, needle) { + t.Fatalf("PVE backend partition missing coverage contract: %s", needle) + } } } diff --git a/scripts/release_control/internal/release_preflight_test.py b/scripts/release_control/internal/release_preflight_test.py index d6e1c87a30..3842c44ae7 100755 --- a/scripts/release_control/internal/release_preflight_test.py +++ b/scripts/release_control/internal/release_preflight_test.py @@ -110,10 +110,13 @@ class ReleasePreflightTest(unittest.TestCase): self.assertIn("is not reachable from a fetched origin branch", runner) def test_api_shard_plan_is_deterministic_complete_and_disjoint(self) -> None: - test_names = [f"TestReleaseCase{index:04d}" for index in range(17)] + test_names = [ + f"TestReleaseCase{index:04d}" + for index in [8, 1, 14, 3, 16, 0, 7, 11, 2, 15, 5, 13, 4, 10, 6, 12, 9] + ] plan = build_plan(test_names, shard_count=2, batch_size=4) reordered_plan = build_plan(list(reversed(test_names)), shard_count=2, batch_size=4) - self.assertEqual(plan, reordered_plan) + self.assertNotEqual(plan, reordered_plan) assigned = [ name @@ -121,15 +124,15 @@ class ReleasePreflightTest(unittest.TestCase): for batch in shard["batches"] for name in batch ] - self.assertCountEqual(assigned, test_names) + self.assertEqual(assigned, test_names) self.assertEqual(len(assigned), len(set(assigned))) self.assertEqual( [name for batch in plan["shards"][0]["batches"] for name in batch], - sorted(test_names)[:9], + test_names[:9], ) self.assertEqual( [name for batch in plan["shards"][1]["batches"] for name in batch], - sorted(test_names)[9:], + test_names[9:], ) with tempfile.TemporaryDirectory() as directory: diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.sh index 08af0fbe9c..2706b2f636 100755 --- a/scripts/run-release-backend-tests.sh +++ b/scripts/run-release-backend-tests.sh @@ -51,7 +51,7 @@ if [[ ! "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]]; then exit 2 fi -for command_name in go python3 getconf awk; do +for command_name in go python3 getconf awk pgrep; do if ! command -v "$command_name" >/dev/null 2>&1; then echo "Error: required backend-test command is missing: $command_name" >&2 exit 3 @@ -163,12 +163,25 @@ for shard_number in $(seq 1 "$API_SHARDS"); do done remaining="${#pids[@]}" +terminate_tree() { + local parent_pid="$1" + local child_pid + while IFS= read -r child_pid; do + if [ -n "$child_pid" ]; then + terminate_tree "$child_pid" + fi + done < <(pgrep -P "$parent_pid" || true) + kill -TERM "$parent_pid" >/dev/null 2>&1 || true +} + while [ "$remaining" -gt 0 ]; do if wait -n; then remaining=$((remaining - 1)) else status=$? - kill "${pids[@]}" >/dev/null 2>&1 || true + for pid in "${pids[@]}"; do + terminate_tree "$pid" + done wait "${pids[@]}" >/dev/null 2>&1 || true exit "$status" fi diff --git a/scripts/shard_go_tests.py b/scripts/shard_go_tests.py index c17746c67a..9171d8f3c5 100755 --- a/scripts/shard_go_tests.py +++ b/scripts/shard_go_tests.py @@ -40,10 +40,11 @@ def build_plan( ) raise ValueError(f"duplicate top-level Go test name(s): {', '.join(duplicates)}") - canonical = sorted(test_names) - # Preserve the suite's canonical ordering inside each shard. Some legacy - # API tests still exercise package-global state, so hash distribution or - # extra process boundaries can change their historical neighbours. + canonical = list(test_names) + # Preserve the exact order emitted by the compiled test binary. Some + # legacy API tests still exercise package-global state, so sorting, hash + # distribution, or extra process boundaries can change their historical + # neighbours. base_size, remainder = divmod(len(canonical), shard_count) shards: list[list[str]] = [] offset = 0 @@ -55,7 +56,7 @@ def build_plan( shard_records: list[dict[str, object]] = [] assigned: list[str] = [] for shard_index, names in enumerate(shards, start=1): - ordered_names = sorted(names) + ordered_names = list(names) assigned.extend(ordered_names) batches = [ ordered_names[offset : offset + batch_size] @@ -70,7 +71,7 @@ def build_plan( } ) - if sorted(assigned) != canonical or len(assigned) != len(set(assigned)): + if assigned != canonical or len(assigned) != len(set(assigned)): raise RuntimeError("generated shard plan is not a complete, disjoint partition") return {