From aa4bbb40d504e0dfeed2a82962010e67288518ea Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 29 Sep 2026 02:02:11 -0700 Subject: [PATCH] fix(ci): pack serial Gateway groups first so the hybrid hourly plan fits its row cap Since 32d364240be (#160909) the hybrid hourly Node plan needed 80 rows against its 79-row cap, failing keeps the hybrid hourly plan within the main-tier cap on main and in every PR's ci-gate. The new Gateway auth-policy test reshuffled four agent-chat stripes, and greedy initial packing then stranded three 140-second storage groups in separate rows. Hybrid initial packing now places exclusive serial Gateway groups before flexible parallel groups so parallel work fills the remaining space. The plan returns to 77 rows (75 Node + 2 dist); caps, budgets, timing baselines, and worker policies are unchanged. --- docs/ci/capacity.md | 3 ++ scripts/lib/ci-node-test-plan.mts | 6 +++ test/scripts/ci-node-test-plan.test.ts | 61 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) diff --git a/docs/ci/capacity.md b/docs/ci/capacity.md index c9f3ee9d1152..7657c700f86c 100644 --- a/docs/ci/capacity.md +++ b/docs/ci/capacity.md @@ -517,6 +517,9 @@ Oversized CLI and Blacksmith agent-support families use the existing file splitter, preserving complete inventories and their serial resource policy. Ordinary self-hosted groups can share the existing promoted 32-class capacity across logical classes, and the existing group exchange fills stranded capacity. +Hybrid initial packing places serial Gateway groups before flexible parallel +groups, preserving space for repeated stripe families before the final parallel +repack. Admission still enforces the same family, time, and worker constraints. The final 70/130 push/PR caps remain unchanged. Native compact admission allows 90 rows, while hosted admission allows 96. Native tooling bins retain separate two-worker child processes and a 300-second test budget. Rows containing the partitioned changed-Node planner proof use 150 seconds in both initial packing diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index d85d97881a0a..bc4ea19dd6ab 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -4824,10 +4824,16 @@ function createCompactNodeTestShardBundles( const usesBlacksmithRunner = usesBlacksmithCapacity(groups[0].runner); // Admit the final groups with their shared prerequisite. Rebalancing after // this check can break build sharing and exceed a bin's admitted cap. + // Place serial Gateway work first: flexible parallel groups can fill its + // remaining space without stranding same-family stripes in the final repack. const sortedGroups = groups .flatMap((group) => expandCompactGroup(group, options.runnerBackend)) .toSorted( (a, b) => + (options.runnerBackend === "hybrid" + ? Number(b.configs.some(isExclusiveCiTestConfig)) - + Number(a.configs.some(isExclusiveCiTestConfig)) + : 0) || estimateBinSeconds([b]) - estimateBinSeconds([a]) || runnerRank(b) - runnerRank(a) || a.shard_name.localeCompare(b.shard_name), diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index 7692aec80cdd..01c6d7add8af 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -451,6 +451,67 @@ function listAllToolingTestFiles(): string[] { } describe("scripts/lib/ci-node-test-plan.mts", () => { + it("reserves hybrid serial Gateway capacity before packing flexible ordinary groups", () => { + const entries = [ + ["ordinary-a", "hooks", 180], + ["ordinary-b", "hooks", 180], + ["ordinary-c", "hooks", 160], + ["gateway-constrained", "gateway-core", 120], + ["ordinary-d", "hooks", 120], + ["ordinary-e", "hooks", 30], + ] as const; + vi.spyOn(testTimings, "readCompactGroupTimings").mockReturnValue( + Object.fromEntries(entries.map(([name, , seconds]) => [name, seconds])), + ); + vi.spyOn(testTimings, "readRuntimePlacementTimings").mockReturnValue([]); + vi.spyOn(buildPrerequisites, "resolveVitestPretestBuildMode").mockReturnValue(undefined); + const original = fullSuiteVitestShards.slice(); + try { + fullSuiteVitestShards.splice( + 0, + fullSuiteVitestShards.length, + ...entries.map(([name, config]) => ({ + name, + config: `fixture-${name}.config.ts`, + projects: [`test/vitest/vitest.${config}.config.ts`], + })), + ); + const jobs = createNodeTestShardBundles({ + compactMode: "push", + runnerBackend: "hybrid", + includeReleaseOnlyPluginShards: false, + }); + // Cost-first packing spends 280s on Gateway and strands 510s of ordinary + // work across two rows; admitting Gateway first leaves one 490s parallel row. + expect(jobs).toHaveLength(2); + const gateway = expectDefined( + jobs.find((job) => job.groups.some((group) => group.shard_name === "gateway-constrained")), + "Gateway job", + ); + expect(gateway).toMatchObject({ + planConcurrency: 1, + predictedSeconds: 300, + predictedTestSeconds: 300, + env: { OPENCLAW_VITEST_MAX_WORKERS: "2" }, + }); + const ordinary = expectDefined( + jobs.find((job) => job !== gateway), + "ordinary job", + ); + expect(ordinary).toMatchObject({ + planConcurrency: 2, + predictedSeconds: 490, + predictedTestSeconds: 280, + }); + expect(jobs.every((job) => job.runner === EXTRA_LARGE_NODE_TEST_RUNNER)).toBe(true); + expect(jobs.flatMap((job) => job.groups.map((group) => group.shard_name)).toSorted()).toEqual( + entries.map(([name]) => name).toSorted(), + ); + } finally { + fullSuiteVitestShards.splice(0, fullSuiteVitestShards.length, ...original); + } + }); + it("packs ordinary work more densely while retaining the serial Gateway budget", () => { vi.spyOn(testTimings, "readCompactGroupTimings").mockReturnValue({ "agentic-gateway-server-isolated": 200,