fix(ci): retry Gateway-first packing only when the hybrid plan exceeds its cap

aa4bbb40d5 sorted serial Gateway groups first for every hybrid plan, which changed initial row membership before runtime placement and broke the ci-test-timings runtime placement observations (no serial Gateway recipient). Hybrid packing now keeps cost-first placement when the completed plan fits its row cap and rebuilds once with Gateway-first ordering only when it would exceed the cap; both attempts run the same admission and runtime-placement checks. The hourly plan stays at 77 rows and the runtime placement tests pass again.

(cherry picked from commit 51973b2ed5af62d03c0120e699063787435663df)

docs/ci/capacity.md

@@ -517,9 +517,12 @@ Oversized CLI and Blacksmith agent-support families use the existing file
 splitter, preserving complete inventories and their serial resource policy.
 Ordinary self-hosted groups can share the existing promoted 32-class capacity
 across logical classes, and the existing group exchange fills stranded capacity.
-Hybrid initial packing places serial Gateway groups before flexible parallel
-groups, preserving space for repeated stripe families before the final parallel
-repack. Admission still enforces the same family, time, and worker constraints.
+Hybrid packing keeps descending-cost initial placement when the completed plan
+fits its row cap, preserving existing serial Gateway recipients for runtime work.
+Only an over-cap plan is rebuilt once with serial Gateway groups before flexible
+parallel groups, preserving space for repeated stripe families before the final
+parallel repack. The alternate reruns all admission and runtime-placement checks
+with the same family, time, and worker constraints; it still fails if over cap.
 The final 70/130 push/PR caps remain unchanged. Native compact admission allows 90 rows, while hosted admission allows 96. Native tooling bins retain separate
 two-worker child processes and a 300-second test budget. Rows containing the
 partitioned changed-Node planner proof use 150 seconds in both initial packing

scripts/lib/ci-node-test-plan.mts

@@ -4528,6 +4528,7 @@ function createCompactNodeTestShardBundles(
   splitHostedToolingTails = false,
   hostedToolingTailBudgets?: ReadonlyMap<string, number>,
   hostedToolingTailDonation?: HostedToolingTailDonation,
+  prioritizeSerialGateway = false,
 ): CompactNodeTestShard[] {
   if (options.runnerBackend === "runson") {
     // Hybrid owns placement and measured serial packing; RunsOn only extracts cron.
@@ -4540,6 +4541,7 @@ function createCompactNodeTestShardBundles(
         splitHostedToolingTails,
         hostedToolingTailBudgets,
         hostedToolingTailDonation,
+        prioritizeSerialGateway,
       ),
       options.compactNodeJobCap ?? COMPACT_NODE_TEST_JOB_CAP,
     );
@@ -4824,13 +4826,13 @@ function createCompactNodeTestShardBundles(
     const usesBlacksmithRunner = usesBlacksmithCapacity(groups[0].runner);
     // Admit the final groups with their shared prerequisite. Rebalancing after
     // this check can break build sharing and exceed a bin's admitted cap.
-    // Place serial Gateway work first: flexible parallel groups can fill its
-    // remaining space without stranding same-family stripes in the final repack.
+    // Retry over-cap hybrid plans with Gateway first; leave successful cost-first
+    // placement and its runtime recipients intact.
     const sortedGroups = groups
       .flatMap((group) => expandCompactGroup(group, options.runnerBackend))
       .toSorted(
         (a, b) =>
-          (options.runnerBackend === "hybrid"
+          (prioritizeSerialGateway
             ? Number(b.configs.some(isExclusiveCiTestConfig)) -
               Number(a.configs.some(isExclusiveCiTestConfig))
             : 0) ||
@@ -5346,6 +5348,20 @@ function createCompactNodeTestShardBundles(
         )
       : finalJobs;
   if (measuredJobs.length > compactJobCap) {
+    if (options.runnerBackend === "hybrid" && !prioritizeSerialGateway) {
+      // Rebuild from source so the alternate order passes every admission and
+      // runtime-placement check before any rows are published.
+      return createCompactNodeTestShardBundles(
+        sourceShards,
+        options,
+        compactMode,
+        selectedToolingFiles,
+        splitHostedToolingTails,
+        hostedToolingTailBudgets,
+        hostedToolingTailDonation,
+        true,
+      );
+    }
     throw new Error(
       `compact ${options.runnerBackend ?? "blacksmith"} node test plan exceeds ${compactJobCap} jobs (${measuredJobs.length} planned)`,
     );

test/scripts/ci-node-test-plan.test.ts

@@ -451,7 +451,7 @@ function listAllToolingTestFiles(): string[] {
 }
 
 describe("scripts/lib/ci-node-test-plan.mts", () => {
-  it("reserves hybrid serial Gateway capacity before packing flexible ordinary groups", () => {
+  it("retries hybrid Gateway-first packing only when the completed plan exceeds its cap", () => {
     const entries = [
       ["ordinary-a", "hooks", 180],
       ["ordinary-b", "hooks", 180],
@@ -476,11 +476,19 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
           projects: [`test/vitest/vitest.${config}.config.ts`],
         })),
       );
-      const jobs = createNodeTestShardBundles({
+      const options = {
         compactMode: "push",
         runnerBackend: "hybrid",
         includeReleaseOnlyPluginShards: false,
-      });
+      } as const;
+      const originalJobs = createNodeTestShardBundles(options);
+      expect(originalJobs).toHaveLength(3);
+      expect(
+        originalJobs.find((job) =>
+          job.groups.some((group) => group.shard_name === "gateway-constrained"),
+        ),
+      ).toMatchObject({ planConcurrency: 1, predictedSeconds: 280 });
+      const jobs = createNodeTestShardBundles({ ...options, compactNodeJobCap: 2 });
       // Cost-first packing spends 280s on Gateway and strands 510s of ordinary
       // work across two rows; admitting Gateway first leaves one 490s parallel row.
       expect(jobs).toHaveLength(2);
@@ -507,6 +515,9 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
       expect(jobs.flatMap((job) => job.groups.map((group) => group.shard_name)).toSorted()).toEqual(
         entries.map(([name]) => name).toSorted(),
       );
+      expect(() => createNodeTestShardBundles({ ...options, compactNodeJobCap: 1 })).toThrow(
+        "compact hybrid node test plan exceeds 1 jobs (2 planned)",
+      );
     } finally {
       fullSuiteVitestShards.splice(0, fullSuiteVitestShards.length, ...original);
     }