Preserve API test order in release shards

docs/release-control/v6/internal/subsystems/deployment-installability.md

@@ -518,12 +518,16 @@ upgrade, update, release, or artifact-selection behavior.
    backend and browser-smoke lanes can start as soon as the bundle is available.
    The backend runner must compile the race-enabled `internal/api` test binary
    once, enumerate every top-level test from that exact binary, and generate a
-   deterministic manifest proving a complete, disjoint partition. Two API
+   deterministic manifest proving a complete, disjoint partition. Each
-   shards and the remaining Go packages may then execute concurrently with
+   partition must preserve the exact order emitted by that binary; sorting or
-   isolated data directories. A failed shard must fail the complete backend
+   hash distribution is not equivalent while legacy tests still exercise
-   gate; sharding must never select a coverage subset. The backend job owns a
+   package-global state. Two API shards and the remaining Go packages may then
-   20-minute ceiling, while each invocation retains the canonical 30-minute Go
+   execute concurrently with isolated data directories. A failed shard must
-   timeout as protection against a stuck package.
+   fail the complete backend gate and terminate every descendant test process;
+   sharding must never select a coverage subset or leave orphan race binaries
+   consuming the worker. The backend job owns a 20-minute ceiling, while each
+   invocation retains the canonical 30-minute Go timeout as protection against
+   a stuck package.
    The warm-path release-control performance objective is 15 minutes or less
    from dispatch to definitive publication/convergence. This objective is an
    optimization target, not permission to weaken exact-SHA qualification,

scripts/installtests/build_release_assets_test.go

@@ -2044,7 +2044,7 @@ func TestReleasePipelinePromotesOneImmutableCandidate(t *testing.T) {
 		t.Fatal("create_release must promote the verified candidate instead of rebuilding release assets")
 	}
 
-	if !strings.Contains(backendJob, "- frontend_checks") || !strings.Contains(integrationJob, "- frontend_checks") {
+	if !strings.Contains(backendJob, "- frontend_bundle") || !strings.Contains(integrationJob, "- frontend_bundle") {
 		t.Fatal("backend and integration jobs must consume the shared verified frontend bundle")
 	}
 	if strings.Contains(integrationJob, "- backend_tests") {
@@ -2341,7 +2341,7 @@ func TestCreateReleasePublishesPrivateProRuntime(t *testing.T) {
 	}
 }
 
-func TestReleaseBackendRaceGateHasHostedRunnerHeadroom(t *testing.T) {
+func TestReleaseBackendRaceGateUsesCompletePVEPartition(t *testing.T) {
 	makefileBytes, err := os.ReadFile(repoFile("Makefile"))
 	if err != nil {
 		t.Fatalf("read Makefile: %v", err)
@@ -2358,11 +2358,27 @@ func TestReleaseBackendRaceGateHasHostedRunnerHeadroom(t *testing.T) {
 		t.Fatalf("read create-release.yml: %v", err)
 	}
 	backendJob := workflowJobBlock(t, string(workflowBytes), "backend_tests")
-	if !strings.Contains(backendJob, "timeout-minutes: 40") {
+	if !strings.Contains(backendJob, "timeout-minutes: 20") {
-		t.Fatal("release backend job must leave setup and result-collection headroom above the package timeout")
+		t.Fatal("release backend job must retain the measured PVE execution ceiling")
 	}
-	if !strings.Contains(backendJob, "run: make test") {
+	if !strings.Contains(backendJob, "pulse-pve-tests") || !strings.Contains(backendJob, "run-release-backend-tests.sh") {
-		t.Fatal("release backend job must consume the governed Makefile race-suite timeout")
+		t.Fatal("release backend job must use the dedicated PVE partition runner")
+	}
+
+	backendScriptBytes, err := os.ReadFile(repoFile("scripts", "run-release-backend-tests.sh"))
+	if err != nil {
+		t.Fatalf("read run-release-backend-tests.sh: %v", err)
+	}
+	backendScript := string(backendScriptBytes)
+	for _, needle := range []string{
+		"go test -c -race",
+		"python3 scripts/shard_go_tests.py",
+		"go test -race -timeout 30m",
+		`-test.timeout 30m`,
+	} {
+		if !strings.Contains(backendScript, needle) {
+			t.Fatalf("PVE backend partition missing coverage contract: %s", needle)
+		}
 	}
 }
 

scripts/release_control/internal/release_preflight_test.py

@@ -110,10 +110,13 @@ class ReleasePreflightTest(unittest.TestCase):
         self.assertIn("is not reachable from a fetched origin branch", runner)
 
     def test_api_shard_plan_is_deterministic_complete_and_disjoint(self) -> None:
-        test_names = [f"TestReleaseCase{index:04d}" for index in range(17)]
+        test_names = [
+            f"TestReleaseCase{index:04d}"
+            for index in [8, 1, 14, 3, 16, 0, 7, 11, 2, 15, 5, 13, 4, 10, 6, 12, 9]
+        ]
         plan = build_plan(test_names, shard_count=2, batch_size=4)
         reordered_plan = build_plan(list(reversed(test_names)), shard_count=2, batch_size=4)
-        self.assertEqual(plan, reordered_plan)
+        self.assertNotEqual(plan, reordered_plan)
 
         assigned = [
             name
@@ -121,15 +124,15 @@ class ReleasePreflightTest(unittest.TestCase):
             for batch in shard["batches"]
             for name in batch
         ]
-        self.assertCountEqual(assigned, test_names)
+        self.assertEqual(assigned, test_names)
         self.assertEqual(len(assigned), len(set(assigned)))
         self.assertEqual(
             [name for batch in plan["shards"][0]["batches"] for name in batch],
-            sorted(test_names)[:9],
+            test_names[:9],
         )
         self.assertEqual(
             [name for batch in plan["shards"][1]["batches"] for name in batch],
-            sorted(test_names)[9:],
+            test_names[9:],
         )
 
         with tempfile.TemporaryDirectory() as directory:

scripts/run-release-backend-tests.sh

@@ -51,7 +51,7 @@ if [[ ! "$BATCH_SIZE" =~ ^[1-9][0-9]*$ ]]; then
   exit 2
 fi
 
-for command_name in go python3 getconf awk; do
+for command_name in go python3 getconf awk pgrep; do
   if ! command -v "$command_name" >/dev/null 2>&1; then
     echo "Error: required backend-test command is missing: $command_name" >&2
     exit 3
@@ -163,12 +163,25 @@ for shard_number in $(seq 1 "$API_SHARDS"); do
 done
 
 remaining="${#pids[@]}"
+terminate_tree() {
+  local parent_pid="$1"
+  local child_pid
+  while IFS= read -r child_pid; do
+    if [ -n "$child_pid" ]; then
+      terminate_tree "$child_pid"
+    fi
+  done < <(pgrep -P "$parent_pid" || true)
+  kill -TERM "$parent_pid" >/dev/null 2>&1 || true
+}
+
 while [ "$remaining" -gt 0 ]; do
   if wait -n; then
     remaining=$((remaining - 1))
   else
     status=$?
-    kill "${pids[@]}" >/dev/null 2>&1 || true
+    for pid in "${pids[@]}"; do
+      terminate_tree "$pid"
+    done
     wait "${pids[@]}" >/dev/null 2>&1 || true
     exit "$status"
   fi

scripts/shard_go_tests.py

@@ -40,10 +40,11 @@ def build_plan(
         )
         raise ValueError(f"duplicate top-level Go test name(s): {', '.join(duplicates)}")
 
-    canonical = sorted(test_names)
+    canonical = list(test_names)
-    # Preserve the suite's canonical ordering inside each shard. Some legacy
+    # Preserve the exact order emitted by the compiled test binary. Some
-    # API tests still exercise package-global state, so hash distribution or
+    # legacy API tests still exercise package-global state, so sorting, hash
-    # extra process boundaries can change their historical neighbours.
+    # distribution, or extra process boundaries can change their historical
+    # neighbours.
     base_size, remainder = divmod(len(canonical), shard_count)
     shards: list[list[str]] = []
     offset = 0
@@ -55,7 +56,7 @@ def build_plan(
     shard_records: list[dict[str, object]] = []
     assigned: list[str] = []
     for shard_index, names in enumerate(shards, start=1):
-        ordered_names = sorted(names)
+        ordered_names = list(names)
         assigned.extend(ordered_names)
         batches = [
             ordered_names[offset : offset + batch_size]
@@ -70,7 +71,7 @@ def build_plan(
             }
         )
 
-    if sorted(assigned) != canonical or len(assigned) != len(set(assigned)):
+    if assigned != canonical or len(assigned) != len(set(assigned)):
         raise RuntimeError("generated shard plan is not a complete, disjoint partition")
 
     return {