mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 17:53:39 +00:00
30e80e790d
Keep the Gateway isolated/database-worker cohort at its existing two-worker budget so cold startup has room within unchanged test deadlines. Preserve former eight-worker complete generations as conservative timing floors without relabeling them as current measurements. Validation: 325 owning tests, selected changed checks, and fresh managed review. Generated job counts and coverage are unchanged. Exact changed CI remains required.
5427 lines
206 KiB
TypeScript
5427 lines
206 KiB
TypeScript
// Builds CI node/Vitest shard plans from the full suite configuration.
|
|
import { statSync } from "node:fs";
|
|
import { matchesGlob, relative, resolve } from "node:path";
|
|
import {
|
|
agentVitestProjectOwners,
|
|
embeddedAgentVitestProjectOwners,
|
|
} from "../../test/vitest/vitest.agents-paths.mjs";
|
|
import {
|
|
databaseWorkerCoreTestFiles,
|
|
isDatabaseWorkerCoreTestFile,
|
|
} from "../../test/vitest/vitest.database-worker-core-paths.mjs";
|
|
import {
|
|
gatewayDatabaseWorkerTestFiles,
|
|
gatewayPluginTestFiles,
|
|
gatewayServerExcludedTestFiles,
|
|
gatewayServerIsolatedTestFiles,
|
|
gatewayServerSerialTestFiles,
|
|
isGatewayServerBackedHttpTestFile,
|
|
isGatewayServerTestFile,
|
|
} from "../../test/vitest/vitest.gateway-server-paths.mjs";
|
|
import { startupCorpusTestFiles } from "../../test/vitest/vitest.startup-corpus-paths.mjs";
|
|
import { fullSuiteVitestShards } from "../../test/vitest/vitest.test-shards.mjs";
|
|
import { toolingIsolatedTestFiles } from "../../test/vitest/vitest.tooling-isolated-paths.mjs";
|
|
import { uiIsolatedTestFiles } from "../../test/vitest/vitest.ui-isolated-paths.mjs";
|
|
import {
|
|
controlUiE2eTestGlobs,
|
|
isPluginControlUiPath,
|
|
isUiBrowserTestFile,
|
|
isUiTestTarget,
|
|
uiTimingTestFiles,
|
|
uiE2ePrebuiltParallelTestFiles,
|
|
uiE2eRealGatewayTestFiles,
|
|
} from "../../test/vitest/vitest.ui-paths.mjs";
|
|
import {
|
|
getUnitFastIsolatedTestFiles,
|
|
getUnitFastTestFiles,
|
|
getUnitFastTestFilesForIncludePatterns,
|
|
getUnitFastTimerTestFiles,
|
|
} from "../../test/vitest/vitest.unit-fast-paths.mjs";
|
|
import {
|
|
boundaryTestFiles,
|
|
filterUnitConfigTestFiles,
|
|
} from "../../test/vitest/vitest.unit-paths.mjs";
|
|
import {
|
|
buildVitestRunPlans,
|
|
isTestFileTarget,
|
|
isToolingTestOwnerPath,
|
|
resolveAffectedTestsFromImportGraph,
|
|
resolveChangedTestTargetPlan,
|
|
} from "../test-projects.test-support.mts";
|
|
import {
|
|
COMMANDS_PARALLEL_TIMING_SUFFIX,
|
|
COMMANDS_RUNTIME_GROUP,
|
|
estimateSerialCommandSeconds,
|
|
estimateLegacyCommandStripeSeconds,
|
|
commandFileSecondsFloor,
|
|
createAgenticCommandSplitShards,
|
|
isParallelCommandsGroup,
|
|
estimateCommandWorkerSeconds,
|
|
} from "./ci-command-test-plan.mts";
|
|
import { rebalanceMeasuredSerialJobs } from "./ci-measured-compact-packing.mts";
|
|
import {
|
|
COMPACT_EMBEDDED_BASE_GROUP_NAME,
|
|
canSplitWholeConfigGroup,
|
|
listScopedOwnerTestFiles,
|
|
listNodeTestConfigFiles,
|
|
listWholeConfigFiles,
|
|
listWholeConfigSplitFiles,
|
|
} from "./ci-node-test-inventory.mts";
|
|
import { resolvePolicyTestTargets } from "./ci-policy-test-watch.mts";
|
|
import {
|
|
isCiProofTestFile,
|
|
isPrExemptRuntimeTestFile,
|
|
isReleaseOnlyRuntimeTestFile,
|
|
} from "./ci-proof-test-inventory.mts";
|
|
import { rebalanceRuntimeTestJobs } from "./ci-runtime-test-placement.mts";
|
|
import {
|
|
NATIVE_SOLO_TIMING_PROFILE,
|
|
createNativeSoloTimingKey,
|
|
isRuntimePlacementIncludePatterns,
|
|
} from "./ci-test-timings-schema.mts";
|
|
import {
|
|
readCompactGroupTimings,
|
|
readCompleteSplitGenerationSeconds,
|
|
readRuntimePlacementTimings,
|
|
readToolingFileTimings,
|
|
resolveRuntimePlacementSeconds,
|
|
} from "./ci-test-timings.mts";
|
|
import { isStripeEligibleTestFile, listTrackedTestFiles } from "./list-test-files.mts";
|
|
import { isExclusiveCiTestConfig } from "./local-check-runtime.mts";
|
|
import { readPositiveEnvInt } from "./numeric-options.mjs";
|
|
import {
|
|
listVitestRuntimeConsumerFiles,
|
|
mergeVitestPretestBuildModes,
|
|
resolveVitestPretestBuildMode,
|
|
type VitestPretestBuildMode as NodeTestPretestBuildMode,
|
|
} from "./vitest-build-prerequisites.mts";
|
|
import {
|
|
COMPACT_GITHUB_GROUP_SECONDS_SCALE,
|
|
COMPACT_HYBRID_GROUP_SECONDS_SCALE,
|
|
VITEST_PRETEST_BUILD_SECONDS,
|
|
createCompactSplitTimingGeneration,
|
|
estimateVitestTestFileSeconds as stripeFileWeight,
|
|
estimateVitestToolingFileSeconds as toolingFileWeight,
|
|
parseCompactSplitTimingKey,
|
|
} from "./vitest-shard-metadata.mts";
|
|
|
|
export { isToolingTestOwnerPath } from "../test-projects.test-support.mts";
|
|
|
|
export type NodeTestShardGroup = {
|
|
shard_name: string;
|
|
timing_key?: string;
|
|
configs: string[];
|
|
includePatterns?: string[];
|
|
pretestBuildMode?: NodeTestPretestBuildMode;
|
|
requiresDist: boolean;
|
|
runner: string;
|
|
env?: Record<string, string>;
|
|
fallbackMaxWorkers?: number;
|
|
minTotalMemoryBytes?: number;
|
|
};
|
|
|
|
function compactGroupTimingKey(group: NodeTestShardGroup): string {
|
|
return group.timing_key ?? group.shard_name;
|
|
}
|
|
|
|
export type NodeTestShard = {
|
|
checkName: string;
|
|
shardName: string;
|
|
timing_key?: string;
|
|
configs: string[];
|
|
runner: string;
|
|
requiresDist: boolean;
|
|
pretestBuildMode?: NodeTestPretestBuildMode;
|
|
includePatterns?: string[];
|
|
env?: Record<string, string>;
|
|
groups?: NodeTestShardGroup[];
|
|
timeoutMinutes?: number;
|
|
planConcurrency?: number;
|
|
predictedSeconds?: number;
|
|
predictedTestSeconds?: number;
|
|
};
|
|
|
|
type NodeTestPlanOptions = {
|
|
changedPaths?: readonly string[];
|
|
includeReleaseOnlyPluginShards?: boolean;
|
|
includeProofTests?: boolean;
|
|
includeReleaseOnlyToolingShards?: boolean;
|
|
includeReleaseOnlyRuntimeTests?: boolean;
|
|
includePrExemptRuntimeTests?: boolean;
|
|
compact?: boolean;
|
|
compactMode?: CompactNodeTestPlanMode;
|
|
compactGroupCount?: number;
|
|
compactWholeGroupCount?: number;
|
|
compactNodeJobCap?: number;
|
|
runnerBackend?: string;
|
|
};
|
|
|
|
export type RuntimeTestSelection = Pick<
|
|
NodeTestPlanOptions,
|
|
"changedPaths" | "includeReleaseOnlyRuntimeTests" | "includePrExemptRuntimeTests"
|
|
>;
|
|
|
|
const prExemptChangedTargets = new WeakMap<readonly string[], Map<string, ReadonlySet<string>>>();
|
|
|
|
function resolvePrExemptChangedTargets(changedPaths: readonly string[], cwd: string) {
|
|
let byCwd = prExemptChangedTargets.get(changedPaths);
|
|
const cached = byCwd?.get(cwd);
|
|
if (cached) {
|
|
return cached;
|
|
}
|
|
const paths = [...changedPaths];
|
|
// Explicit source owners cover spawned processes and filename readers;
|
|
// transitive imports cover subjects reached through shared integration fixtures.
|
|
const targets = new Set(
|
|
[
|
|
...resolvePolicyTestTargets(paths),
|
|
...resolveChangedTestTargetPlan(paths, {
|
|
broad: false,
|
|
combineSiblingWithImportGraph: true,
|
|
cwd,
|
|
forceFullImportGraph: true,
|
|
includeExtensionImpact: false,
|
|
resolveAliases: true,
|
|
runtimeOnly: true,
|
|
}).targets,
|
|
...resolveAffectedTestsFromImportGraph(paths, cwd, {
|
|
tooling: true,
|
|
forceFull: true,
|
|
resolveAliases: true,
|
|
runtimeOnly: true,
|
|
}),
|
|
].filter(isPrExemptRuntimeTestFile),
|
|
);
|
|
if (!byCwd) {
|
|
byCwd = new Map();
|
|
prExemptChangedTargets.set(changedPaths, byCwd);
|
|
}
|
|
byCwd.set(cwd, targets);
|
|
return targets;
|
|
}
|
|
|
|
export function isRuntimeTestFileIncluded(
|
|
file: string,
|
|
options: RuntimeTestSelection,
|
|
cwd = process.cwd(),
|
|
): boolean {
|
|
const releaseOnly =
|
|
options.includeReleaseOnlyRuntimeTests === false && isReleaseOnlyRuntimeTestFile(file);
|
|
const prExempt = options.includePrExemptRuntimeTests === false && isPrExemptRuntimeTestFile(file);
|
|
if (!releaseOnly && !prExempt) {
|
|
return true;
|
|
}
|
|
if (statSync(resolve(cwd, file), { throwIfNoEntry: false })?.isFile() !== true) {
|
|
return false;
|
|
}
|
|
const changedPaths = options.changedPaths ?? [];
|
|
if (changedPaths.includes(file)) {
|
|
return true;
|
|
}
|
|
return (
|
|
!releaseOnly &&
|
|
changedPaths.length > 0 &&
|
|
resolvePrExemptChangedTargets(changedPaths, cwd).has(file)
|
|
);
|
|
}
|
|
|
|
export function resolveStartupCorpusTestFiles(options: RuntimeTestSelection = {}): string[] {
|
|
return startupCorpusTestFiles.filter((file) => isRuntimeTestFileIncluded(file, options));
|
|
}
|
|
|
|
export function hasCompleteStartupCorpusCoverage(
|
|
shards: readonly {
|
|
requiresDist: boolean;
|
|
targets?: readonly string[];
|
|
groups?: readonly NodeTestShardGroup[];
|
|
}[],
|
|
expectedFiles: readonly string[] = startupCorpusTestFiles,
|
|
): boolean {
|
|
// Only explicit, unfiltered file owners prove the corpus is complete. A
|
|
// config name or native shard can still execute just part of the matrix.
|
|
const groups = shards.flatMap((shard) =>
|
|
!shard.requiresDist && !shard.targets?.length ? (shard.groups ?? []) : [],
|
|
);
|
|
return (
|
|
expectedFiles.length > 0 &&
|
|
expectedFiles.every((file) =>
|
|
groups.some(
|
|
(group) =>
|
|
group.configs.length === 1 &&
|
|
group.configs[0] === "test/vitest/vitest.runtime-config.config.ts" &&
|
|
Object.keys(group.env ?? {}).every((key) => key === "OPENCLAW_VITEST_MAX_WORKERS") &&
|
|
group.includePatterns?.includes(file),
|
|
),
|
|
)
|
|
);
|
|
}
|
|
|
|
type CompactNodeTestPlanMode = "pull-request" | "push";
|
|
|
|
function includesReleaseOnlyTooling(options: NodeTestPlanOptions): boolean {
|
|
return (
|
|
options.includeReleaseOnlyToolingShards !== false ||
|
|
(options.changedPaths ?? []).some(isToolingTestOwnerPath)
|
|
);
|
|
}
|
|
|
|
export type CompactNodeTestShard = Omit<NodeTestShard, "configs" | "groups"> & {
|
|
groups: NodeTestShardGroup[];
|
|
};
|
|
|
|
type NodeTestSplitShard = Omit<NodeTestShard, "checkName" | "runner" | "pretestBuildMode"> & {
|
|
includeExternalConfigs?: boolean;
|
|
runner?: string;
|
|
};
|
|
|
|
const EXCLUDED_FULL_SUITE_SHARDS = new Set([
|
|
"test/vitest/vitest.full-core-contracts.config.ts",
|
|
"test/vitest/vitest.full-core-bundled.config.ts",
|
|
"test/vitest/vitest.full-extensions.config.ts",
|
|
]);
|
|
|
|
export const SOURCE_CHANNEL_TEST_POLICY = {
|
|
config: "test/vitest/vitest.channels.config.ts",
|
|
env: {
|
|
NODE_OPTIONS: "--max-old-space-size=8192",
|
|
OPENCLAW_VITEST_MAX_WORKERS: "1",
|
|
},
|
|
} as const;
|
|
|
|
const EXCLUDED_PROJECT_CONFIGS = new Set([
|
|
SOURCE_CHANNEL_TEST_POLICY.config,
|
|
// checks-ui owns the Chromium project; Node stripes retain Node-driven Playwright tests.
|
|
"test/vitest/vitest.ui-browser.config.ts",
|
|
]);
|
|
const DEFAULT_NODE_TEST_RUNNER = "blacksmith-8vcpu-ubuntu-2404";
|
|
const BUNDLED_NODE_TEST_RUNNER = "blacksmith-4vcpu-ubuntu-2404";
|
|
const EXTRA_LARGE_NODE_TEST_RUNNER = "blacksmith-32vcpu-ubuntu-2404";
|
|
// Startup-core transforms the broad gateway graph before its assertions run.
|
|
// Keep enough CPU here to avoid spending minutes in Vitest imports on 4 vCPU.
|
|
const GATEWAY_STARTUP_CORE_RUNNER = DEFAULT_NODE_TEST_RUNNER;
|
|
// Fail the known warm-cache startup stall at its existing scoped deadline.
|
|
const GATEWAY_STARTUP_HEALTH_RUNTIME_ENV = {
|
|
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "60000",
|
|
};
|
|
// The first embedded-agent file owns 157 serial tests and can stay quiet for
|
|
// more than five minutes on a cold GitHub-hosted fork runner. Keep the outer
|
|
// watchdog above the scoped 600-second hook budget so it cannot preempt Vitest.
|
|
const AGENTS_EMBEDDED_AGENT_ENV = {
|
|
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "660000",
|
|
};
|
|
const COMPACT_EMBEDDED_GROUP_NAMES = [
|
|
COMPACT_EMBEDDED_BASE_GROUP_NAME,
|
|
"agentic-agents-embedded-incomplete-turn",
|
|
"agentic-agents-embedded-overflow-compaction",
|
|
"agentic-agents-embedded-run",
|
|
];
|
|
const MAX_BUNDLED_NODE_TEST_PATTERNS = 64;
|
|
// Compact bundles trade a little serial work for fewer ephemeral runner registrations.
|
|
// Keep runner classes and subprocess isolation intact while bounding each combined job.
|
|
// Settle the original 360s placement before compacting only its parallel rows.
|
|
// Serial jobs retain 200s/276s caps; expanded serial jobs retain 210s.
|
|
const COMPACT_LARGE_NODE_TEST_JOB_SECONDS = 200;
|
|
const COMPACT_SMALL_NODE_TEST_JOB_SECONDS = 276;
|
|
const COMPACT_PARALLEL_NODE_TEST_JOB_SECONDS = 360;
|
|
const COMPACT_SERIAL_NODE_TEST_JOB_SECONDS = 300;
|
|
const COMPACT_FINAL_PARALLEL_NODE_TEST_JOB_SECONDS = 500;
|
|
const COMPACT_EXPANDED_NODE_TEST_JOB_SECONDS = 210;
|
|
// Runtime preparation is charged once beside the same serial test budget.
|
|
// This is admission, never a test deadline.
|
|
const COMPACT_HYBRID_RUNTIME_JOB_SECONDS =
|
|
COMPACT_SERIAL_NODE_TEST_JOB_SECONDS + VITEST_PRETEST_BUILD_SECONDS.runtime;
|
|
// Split groups above this hosted prediction before packing. Hybrid reuses the
|
|
// hosted-derived splits so retries cannot reunite an oversized hosted group.
|
|
const COMPACT_GITHUB_MAX_PREDICTED_SECONDS = 150;
|
|
// Hosted run 35477045216 timed out after an hour on a 203-file serial stripe;
|
|
// its 196-file sibling took 2867s. Bound admission independently of stale costs.
|
|
const COMPACT_HOSTED_STORAGE_STATE_MAX_FILES = 64;
|
|
// Hourly hosted run 35983526919 spent ~25 minutes in each of these owners.
|
|
// Bound the main-tier work independently of stale whole-owner timing estimates.
|
|
const COMPACT_HOSTED_MAIN_MAX_FILES = new Map([
|
|
["agentic-control-plane-agent-chat", 30],
|
|
["agentic-gateway-methods", 96],
|
|
]);
|
|
const HOSTED_MAIN_UPDATE_TEST = "src/cli/update-cli.test.ts";
|
|
// Trusted forks can use the GitHub profile on Blacksmith. Every compact
|
|
// profile must fit the same runner-registration allowance.
|
|
const COMPACT_NODE_TEST_JOB_CAP = 90;
|
|
const COMPACT_GITHUB_NODE_TEST_JOB_CAP = 96;
|
|
const COMPACT_NODE_TEST_JOB_GROUPS = 10;
|
|
const COMPACT_TOOLING_NODE_TEST_GROUPS = 16;
|
|
const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120;
|
|
// Keep capacity with the workload when packing changes; hosted stripes follow
|
|
// their parent's profile policy without changing test or worker boundaries.
|
|
const COMPACT_NODE_TEST_OWNER_RUNNERS = new Map([
|
|
[
|
|
"blacksmith",
|
|
new Map([
|
|
["agentic-agents-core-isolated", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-agents-support", EXTRA_LARGE_NODE_TEST_RUNNER],
|
|
["agentic-cli", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-doctor", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-doctor-sessions-cron", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-doctor-sessions-cron-memory", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-doctor-sessions-cron-sqlite", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-doctor-sessions-cron-sqlite-recovery", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-doctor-platform", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-status-tools", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-control-plane-auth-node", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-control-plane-http-plugin-ws", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-control-plane-runtime-ui-tools", DEFAULT_NODE_TEST_RUNNER],
|
|
["auto-reply-reply-agent-runner", DEFAULT_NODE_TEST_RUNNER],
|
|
["core-runtime-infra-heartbeat-runner", DEFAULT_NODE_TEST_RUNNER],
|
|
["core-runtime-infra-storage-state", DEFAULT_NODE_TEST_RUNNER],
|
|
["core-runtime-infra-system-runtime", DEFAULT_NODE_TEST_RUNNER],
|
|
]),
|
|
],
|
|
[
|
|
"hybrid",
|
|
new Map([
|
|
["agentic-commands-doctor-config-state", DEFAULT_NODE_TEST_RUNNER],
|
|
[COMMANDS_RUNTIME_GROUP, DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-commands-doctor-platform", DEFAULT_NODE_TEST_RUNNER],
|
|
["agentic-control-plane-runtime-shared-token", DEFAULT_NODE_TEST_RUNNER],
|
|
["core-runtime-cron-parallel-service", DEFAULT_NODE_TEST_RUNNER],
|
|
]),
|
|
],
|
|
[
|
|
"github",
|
|
new Map([
|
|
["agentic-agents-tools", DEFAULT_NODE_TEST_RUNNER],
|
|
["core-runtime-cron-parallel-service", DEFAULT_NODE_TEST_RUNNER],
|
|
["core-runtime-infra-storage-state", DEFAULT_NODE_TEST_RUNNER],
|
|
]),
|
|
],
|
|
]);
|
|
const AUTO_REPLY_COMMANDS_STRIPES = 3;
|
|
const AGENTS_CORE_RUNNER_CLI_STRIPES = 3;
|
|
const AGENTIC_GATEWAY_CORE_STRIPES = 3;
|
|
const CORE_RUNTIME_MEDIA_UI_STRIPES = 3;
|
|
const CORE_UNIT_SRC_SECURITY_STRIPES = 3;
|
|
const UNIT_FAST_NODE_TEST_STRIPES = 2;
|
|
// Preserve three balanced base stripes: the largest individual harness files
|
|
// remain indivisible even after files within each stripe run in parallel.
|
|
const EMBEDDED_BASE_NODE_TEST_STRIPES = 3;
|
|
// Cold-start fallback when committed CI measurements are missing. Refresh
|
|
// config/ci-test-timings.json with pnpm ci:timings:refit, not these literals.
|
|
const COMPACT_GROUP_SECONDS_HINTS = new Map<string, number>([
|
|
["agentic-agents-core-auth", 30],
|
|
["agentic-agents-core-isolated", 18],
|
|
["agentic-agents-core-models", 41],
|
|
["agentic-agents-core-runner-cli-1", 6],
|
|
["agentic-agents-core-runner-cli-2", 13],
|
|
["agentic-agents-core-runner-cli-3", 7],
|
|
["agentic-agents-core-runner-commands", 28],
|
|
["agentic-agents-core-runner-embedded", 17],
|
|
["agentic-agents-core-runner-sessions", 14],
|
|
["agentic-agents-core-runtime", 106],
|
|
["agentic-agents-core-subagents", 20],
|
|
["agentic-agents-core-tools", 39],
|
|
["agentic-agents-embedded-base-1", 86],
|
|
["agentic-agents-embedded-base-2", 86],
|
|
["agentic-agents-embedded-base-3", 86],
|
|
["agentic-agents-embedded-incomplete-turn", 19],
|
|
["agentic-agents-embedded-overflow-compaction", 20],
|
|
["agentic-agents-embedded-run", 46],
|
|
// Main runs 33537556582/33537739443/33543106647 totaled 478.25s/450.21s/418.13s
|
|
// across the complete support inventory. Keep its upper bound as the fallback
|
|
// when membership changes and exact generation timings no longer match.
|
|
["agentic-agents-support", 479],
|
|
["agentic-agents-tools", 69],
|
|
// The measured 131s pair split per config; apportioned by the hosted
|
|
// per-config walls (139s/67s) until direct Blacksmith samples exist.
|
|
["agentic-cli", 88],
|
|
["agentic-cli-process", 43],
|
|
["agentic-command-support", 49],
|
|
["agentic-commands-agent-channel", 76],
|
|
["agentic-commands-doctor", 23],
|
|
["agentic-commands-doctor-auth", 19],
|
|
["agentic-commands-doctor-config-state", 67],
|
|
["agentic-commands-doctor-device", 2],
|
|
["agentic-commands-doctor-gateway", 3],
|
|
["agentic-commands-doctor-platform", 5],
|
|
["agentic-commands-doctor-plugins-tools", 13],
|
|
// Job 99770912022 measured 116.6s/112.3s of test bodies and 53.8s of
|
|
// group overhead. Charge that full overhead to each new process until
|
|
// the canonical refit has two main-run samples for the new owners.
|
|
["agentic-commands-doctor-sessions-cron", 31],
|
|
["agentic-commands-doctor-sessions-cron-memory", 167],
|
|
["agentic-commands-doctor-sessions-cron-sqlite", 171],
|
|
// Job 106098306092 measured 11.27s including setup for both recovery files.
|
|
["agentic-commands-doctor-sessions-cron-sqlite-recovery", 15],
|
|
["agentic-commands-doctor-shared", 37],
|
|
["agentic-commands-doctor-whatsapp", 1],
|
|
["agentic-commands-doctor-workspace", 1],
|
|
["agentic-commands-models", 32],
|
|
["agentic-commands-onboard-config", 49],
|
|
["agentic-commands-status-tools", 35],
|
|
["agentic-control-plane-agent-chat", 167],
|
|
["agentic-control-plane-auth-node", 166],
|
|
["agentic-control-plane-http-models", 41],
|
|
["agentic-control-plane-http-plugin-ws", 52],
|
|
["agentic-control-plane-runtime", 19],
|
|
["agentic-control-plane-runtime-config", 20],
|
|
["agentic-control-plane-runtime-cron", 22],
|
|
["agentic-control-plane-runtime-server", 23],
|
|
["agentic-control-plane-runtime-shared-token", 9],
|
|
["agentic-control-plane-runtime-state", 33],
|
|
["agentic-control-plane-runtime-ui-tools", 9],
|
|
["agentic-control-plane-startup-config", 5],
|
|
["agentic-control-plane-startup-core", 31],
|
|
["agentic-control-plane-startup-health-runtime", 11],
|
|
["agentic-control-plane-startup-restart-close", 10],
|
|
// Run 33364935118 measured 21s of tests; the build belongs to bin admission.
|
|
["agentic-gateway-core-runtime", 21],
|
|
// Two-CPU Linux proof: 40s including preparation, 19s for the 13,000-file case.
|
|
["agentic-gateway-core-inventory", 40],
|
|
["agentic-gateway-core-1", 99],
|
|
["agentic-gateway-core-2", 99],
|
|
["agentic-gateway-core-3", 99],
|
|
// Five successful main runs through 35607421993 measured a 1,043s median
|
|
// across both configs and complete child spans; direct refits remain authoritative.
|
|
["agentic-gateway-server-isolated", 1043],
|
|
["agentic-gateway-methods", 157],
|
|
["agentic-plugin-sdk", 45],
|
|
["auto-reply-core-top-level", 27],
|
|
["auto-reply-reply-agent-runner", 60],
|
|
["auto-reply-reply-commands-1", 28],
|
|
["auto-reply-reply-commands-2", 9],
|
|
["auto-reply-reply-commands-3", 24],
|
|
["auto-reply-reply-dispatch", 15],
|
|
["auto-reply-reply-dispatch-core", 35],
|
|
["auto-reply-reply-dispatch-delivery", 32],
|
|
["auto-reply-reply-dispatch-lifecycle", 8],
|
|
["auto-reply-reply-session", 34],
|
|
["auto-reply-reply-state-routing", 63],
|
|
// Apportioned from the split infra-process trio (see below).
|
|
["core-runtime-config", 113],
|
|
["core-runtime-cron-parallel-core", 13],
|
|
["core-runtime-cron-parallel-isolated-agent", 53],
|
|
["core-runtime-cron-parallel-service", 29],
|
|
["core-runtime-hooks", 19],
|
|
["core-runtime-infra-approval-exec", 28],
|
|
["core-runtime-infra-channel-plugin", 19],
|
|
["core-runtime-infra-cli-ui", 2],
|
|
["core-runtime-infra-core-utils", 3],
|
|
["core-runtime-infra-device", 8],
|
|
["core-runtime-infra-diagnostics-state", 24],
|
|
["core-runtime-infra-env-auth", 6],
|
|
["core-runtime-infra-events-runtime", 8],
|
|
["core-runtime-infra-file-safety", 2],
|
|
["core-runtime-infra-files-commands", 5],
|
|
["core-runtime-infra-gateway-lock-argv", 3],
|
|
["core-runtime-infra-gateway-processes", 1],
|
|
["core-runtime-infra-gateway-watch", 1],
|
|
["core-runtime-infra-heartbeat-core", 7],
|
|
["core-runtime-infra-heartbeat-runner", 59],
|
|
["core-runtime-infra-misc", 14],
|
|
["core-runtime-infra-misc-dedupe-disk", 1],
|
|
["core-runtime-infra-misc-os", 1],
|
|
["core-runtime-infra-misc-values", 2],
|
|
["core-runtime-infra-net-install", 11],
|
|
["core-runtime-infra-network-node", 3],
|
|
["core-runtime-infra-network-platform", 5],
|
|
["core-runtime-infra-outbound-actions", 37],
|
|
["core-runtime-infra-outbound-core", 59],
|
|
// The measured 126s trio split; apportioned by the hosted per-config walls
|
|
// (17s/157s) until direct Blacksmith samples exist.
|
|
["core-runtime-infra-process", 13],
|
|
["core-runtime-infra-provider-push", 13],
|
|
["core-runtime-infra-repo-tooling", 4],
|
|
["core-runtime-infra-storage-state", 104],
|
|
["core-runtime-infra-system-runtime", 36],
|
|
["core-runtime-media-ui-1", 93],
|
|
["core-runtime-media-ui-2", 93],
|
|
["core-runtime-media-ui-3", 93],
|
|
["core-runtime-media-ui-support", 100],
|
|
["core-runtime-secrets", 61],
|
|
["core-runtime-shared", 67],
|
|
// This dist-only group is outside the sampled nondist logs and retains its
|
|
// prior measured hint. The exclusive-bin cap keeps its lane lightly packed.
|
|
["core-runtime-tui-pty", 116],
|
|
// This PR-only owner is excluded from sampled push plans. Retained exact runs
|
|
// measured 108.79s/130.83s, so use the conservative wall until its owner can
|
|
// supply canonical samples through another path.
|
|
["core-tooling-isolated", 131],
|
|
["core-unit-fast-1", 66],
|
|
["core-unit-fast-2", 64],
|
|
// The measured 116s pair split per config; apportioned by the hosted
|
|
// per-config walls (158s/32s) until direct Blacksmith samples exist.
|
|
["core-unit-fast-fake-timers", 20],
|
|
["core-unit-fast-isolated", 96],
|
|
["core-unit-src-security-1", 101],
|
|
["core-unit-src-security-2", 101],
|
|
["core-unit-src-security-3", 101],
|
|
["core-unit-src-security-support", 12],
|
|
["core-unit-support", 20],
|
|
]);
|
|
|
|
// Rounded mean of the same 8-vCPU groups across successful canonical-main
|
|
// compact runs 31684307744, 31683213137, 31682494259, 31682258389,
|
|
// 31681118857, 31680010311, 31678309660, 31678086868, and 31677305067.
|
|
// Means expose recurrent slow tails hidden by medians; resource ownership
|
|
// remains with resolveCiNodeTestRunner.
|
|
const COMPACT_LARGE_GROUP_STRIPE_SECONDS_HINTS = new Map<string, number>([
|
|
["agentic-agents-core-auth", 33],
|
|
["agentic-agents-core-models", 41],
|
|
["agentic-agents-core-runner-cli-1", 7],
|
|
["agentic-agents-core-runner-cli-2", 14],
|
|
["agentic-agents-core-runner-cli-3", 7],
|
|
["agentic-agents-core-runner-commands", 28],
|
|
["agentic-agents-core-runner-embedded", 20],
|
|
["agentic-agents-core-runner-sessions", 16],
|
|
["agentic-agents-core-runtime", 119],
|
|
["agentic-agents-core-subagents", 21],
|
|
["agentic-agents-core-tools", 47],
|
|
["agentic-agents-embedded-base-1", 86],
|
|
["agentic-agents-embedded-base-2", 86],
|
|
["agentic-agents-embedded-base-3", 86],
|
|
["agentic-agents-embedded-incomplete-turn", 20],
|
|
["agentic-agents-embedded-overflow-compaction", 21],
|
|
["agentic-agents-embedded-run", 47],
|
|
["agentic-agents-support", 479],
|
|
["agentic-control-plane-startup-core", 33],
|
|
// Run 31691151297 measured 296.68s for gateway-core and 303.93s for unit-src.
|
|
// Run 31694057974 measured the two isolated UI envelopes at 159.50s and
|
|
// 120.55s. Rebalance those walls over the three-way LPT weights: 457/455/455,
|
|
// 633/634/633, and 393/393/393 respectively.
|
|
["agentic-gateway-core-1", 99],
|
|
["agentic-gateway-core-2", 99],
|
|
["agentic-gateway-core-3", 99],
|
|
["agentic-gateway-methods", 153],
|
|
["auto-reply-reply-commands-1", 34],
|
|
["auto-reply-reply-commands-2", 11],
|
|
["auto-reply-reply-commands-3", 28],
|
|
["auto-reply-reply-dispatch", 18],
|
|
["auto-reply-reply-dispatch-core", 42],
|
|
["auto-reply-reply-dispatch-delivery", 38],
|
|
["auto-reply-reply-dispatch-lifecycle", 10],
|
|
["core-runtime-media-ui-1", 93],
|
|
["core-runtime-media-ui-2", 93],
|
|
["core-runtime-media-ui-3", 93],
|
|
["core-runtime-media-ui-support", 100],
|
|
["core-unit-fast-1", 68],
|
|
["core-unit-fast-2", 67],
|
|
["core-unit-fast-fake-timers", 21],
|
|
["core-unit-fast-isolated", 96],
|
|
["core-unit-src-security-1", 101],
|
|
["core-unit-src-security-2", 101],
|
|
["core-unit-src-security-3", 101],
|
|
["core-unit-src-security-support", 12],
|
|
]);
|
|
|
|
// Rounded medians from standard 4-core GitHub-hosted runs 31737316152,
|
|
// 31742781948, 31749838728, 31754493208, 31776290645, 31784022043, and
|
|
// 31784883914. Exclude failed samples and reject media-ui-3's 444s compact
|
|
// retry sample because its log records a 300s no-output timeout; its three
|
|
// healthy samples are 52-63s. Unmeasured groups use the scale above.
|
|
const COMPACT_GITHUB_GROUP_SECONDS_HINTS = new Map<string, number>([
|
|
["agentic-agents-core-auth", 50],
|
|
["agentic-agents-core-isolated", 23],
|
|
["agentic-agents-core-models", 198],
|
|
["agentic-agents-core-runner-cli-1", 16],
|
|
["agentic-agents-core-runner-cli-2", 25],
|
|
["agentic-agents-core-runner-cli-3", 23],
|
|
["agentic-agents-core-runner-commands", 55],
|
|
["agentic-agents-core-runner-embedded", 30],
|
|
["agentic-agents-core-runner-sessions", 23],
|
|
["agentic-agents-core-runtime", 185],
|
|
["agentic-agents-core-subagents", 29],
|
|
["agentic-agents-core-tools", 83],
|
|
["agentic-agents-embedded-base-1", 138],
|
|
["agentic-agents-embedded-base-2", 138],
|
|
["agentic-agents-embedded-base-3", 138],
|
|
["agentic-agents-embedded-incomplete-turn", 3],
|
|
["agentic-agents-embedded-overflow-compaction", 31],
|
|
["agentic-agents-embedded-run", 62],
|
|
["agentic-agents-support", 253],
|
|
["agentic-agents-tools", 124],
|
|
// Measured per config inside run 31814517685's combined 206s wall.
|
|
["agentic-cli", 139],
|
|
["agentic-cli-process", 67],
|
|
["agentic-command-support", 67],
|
|
["agentic-commands-agent-channel", 121],
|
|
["agentic-commands-doctor", 33],
|
|
["agentic-commands-doctor-auth", 32],
|
|
["agentic-commands-doctor-config-state", 124],
|
|
["agentic-commands-doctor-device", 5],
|
|
["agentic-commands-doctor-gateway", 8],
|
|
["agentic-commands-doctor-platform", 7],
|
|
["agentic-commands-doctor-plugins-tools", 21],
|
|
// Conservative native fallbacks above, scaled by 1.6 until hosted samples exist.
|
|
["agentic-commands-doctor-sessions-cron", 87],
|
|
["agentic-commands-doctor-sessions-cron-memory", 268],
|
|
["agentic-commands-doctor-sessions-cron-sqlite", 274],
|
|
["agentic-commands-doctor-shared", 61],
|
|
["agentic-commands-doctor-whatsapp", 2],
|
|
["agentic-commands-doctor-workspace", 3],
|
|
["agentic-commands-models", 64],
|
|
["agentic-commands-onboard-config", 76],
|
|
["agentic-commands-status-tools", 57],
|
|
["agentic-control-plane-agent-chat", 232],
|
|
["agentic-control-plane-auth-node", 254],
|
|
["agentic-control-plane-http-models", 59],
|
|
["agentic-control-plane-http-plugin-ws", 86],
|
|
["agentic-control-plane-runtime", 31],
|
|
["agentic-control-plane-runtime-config", 31],
|
|
["agentic-control-plane-runtime-cron", 52],
|
|
["agentic-control-plane-runtime-server", 54],
|
|
["agentic-control-plane-runtime-shared-token", 28],
|
|
["agentic-control-plane-runtime-state", 55],
|
|
["agentic-control-plane-runtime-ui-tools", 31],
|
|
["agentic-control-plane-startup-config", 15],
|
|
["agentic-control-plane-startup-core", 51],
|
|
["agentic-control-plane-startup-health-runtime", 31],
|
|
["agentic-control-plane-startup-restart-close", 28],
|
|
["agentic-gateway-core-1", 176],
|
|
["agentic-gateway-core-2", 149],
|
|
["agentic-gateway-core-3", 141],
|
|
["agentic-gateway-core-inventory", 40],
|
|
["agentic-gateway-methods", 169],
|
|
// Full Release Validation job 106995310855 (4-core ubuntu-24.04, two workers)
|
|
// ran the whole cohort in 2914s wall; 1.6x the Blacksmith median predicted 1669s.
|
|
["agentic-gateway-server-isolated", 2914],
|
|
["agentic-plugin-sdk", 70],
|
|
["auto-reply-core-top-level", 43],
|
|
["auto-reply-reply-agent-runner", 169],
|
|
["auto-reply-reply-commands-1", 53],
|
|
["auto-reply-reply-commands-2", 26],
|
|
["auto-reply-reply-commands-3", 48],
|
|
["auto-reply-reply-dispatch", 55],
|
|
["auto-reply-reply-dispatch-core", 75],
|
|
["auto-reply-reply-dispatch-delivery", 70],
|
|
["auto-reply-reply-dispatch-lifecycle", 20],
|
|
["auto-reply-reply-session", 79],
|
|
["auto-reply-reply-state-routing", 34],
|
|
// Measured per config inside run 31814517685's combined 175s infra wall.
|
|
["core-runtime-config", 157],
|
|
["core-runtime-cron-parallel-core", 22],
|
|
["core-runtime-cron-parallel-isolated-agent", 77],
|
|
["core-runtime-cron-parallel-service", 66],
|
|
["core-runtime-hooks", 31],
|
|
["core-runtime-infra-approval-exec", 45],
|
|
["core-runtime-infra-channel-plugin", 30],
|
|
["core-runtime-infra-cli-ui", 3],
|
|
["core-runtime-infra-core-utils", 7],
|
|
["core-runtime-infra-device", 13],
|
|
["core-runtime-infra-diagnostics-state", 34],
|
|
["core-runtime-infra-env-auth", 10],
|
|
["core-runtime-infra-events-runtime", 11],
|
|
["core-runtime-infra-file-safety", 4],
|
|
["core-runtime-infra-files-commands", 7],
|
|
["core-runtime-infra-gateway-lock-argv", 3],
|
|
["core-runtime-infra-gateway-processes", 1],
|
|
["core-runtime-infra-gateway-watch", 1],
|
|
["core-runtime-infra-heartbeat-core", 10],
|
|
["core-runtime-infra-heartbeat-runner", 106],
|
|
["core-runtime-infra-misc", 33],
|
|
["core-runtime-infra-misc-dedupe-disk", 1],
|
|
["core-runtime-infra-misc-os", 1],
|
|
["core-runtime-infra-misc-values", 2],
|
|
["core-runtime-infra-net-install", 17],
|
|
["core-runtime-infra-network-node", 5],
|
|
["core-runtime-infra-network-platform", 8],
|
|
["core-runtime-infra-outbound-actions", 53],
|
|
["core-runtime-infra-outbound-core", 112],
|
|
// Measured per config inside run 31814517685's combined 175s wall.
|
|
["core-runtime-infra-process", 17],
|
|
["core-runtime-infra-provider-push", 29],
|
|
["core-runtime-infra-repo-tooling", 6],
|
|
["core-runtime-infra-storage-state", 235],
|
|
["core-runtime-infra-system-runtime", 69],
|
|
["core-runtime-media-ui-1", 97],
|
|
["core-runtime-media-ui-2", 78],
|
|
["core-runtime-media-ui-3", 71],
|
|
["core-runtime-media-ui-support", 101],
|
|
["core-runtime-secrets", 73],
|
|
["core-runtime-shared", 92],
|
|
["core-tooling-isolated", 41],
|
|
["core-unit-fast-1", 85],
|
|
["core-unit-fast-2", 84],
|
|
// Measured per config inside run 31814517685's combined 190s wall.
|
|
["core-unit-fast-fake-timers", 32],
|
|
["core-unit-fast-isolated", 158],
|
|
["core-unit-src-security-1", 132],
|
|
["core-unit-src-security-2", 131],
|
|
["core-unit-src-security-3", 132],
|
|
["core-unit-src-security-support", 20],
|
|
["core-unit-support", 32],
|
|
]);
|
|
|
|
// Hybrid-specific Blacksmith observations, plus the gateway-core-3 139.5s spike
|
|
// in 31938297538 that must stay singleton.
|
|
// Sum a shard's per-config Duration lines before taking a median; pooling them
|
|
// reads as a large over-prediction that is not there. Normalize each run by its
|
|
// own VM speed (median of every shard's duration over that shard's cross-run
|
|
// median) before comparing, or a slow draw looks like a hint miss.
|
|
// Values below are VM-normalized medians over runs 32316204633, 32317242374,
|
|
// 32318250756, and 32320063231 (2026-08-20). Across 100 groups the GitHub hints
|
|
// run 0.64x on Blacksmith, so only the ones that overshoot are pinned here:
|
|
// leaving these low packs partners onto the tallest bins, which set the wall.
|
|
const COMPACT_HYBRID_GROUP_SECONDS_HINTS = new Map<string, number>([
|
|
// Preserve the observed support budget through inventory growth without
|
|
// scaling the current Blacksmith evidence by an older whole-suite ratio.
|
|
["agentic-agents-support", 479],
|
|
["agentic-agents-core-models", 81],
|
|
["agentic-cli-process", 110],
|
|
["agentic-commands-doctor", 83],
|
|
["agentic-gateway-core-3", 140],
|
|
["core-runtime-cron-parallel-service", 54],
|
|
["core-runtime-infra-process", 35],
|
|
]);
|
|
|
|
// Exact 6a0571 storage-child replay on a four-CPU Testbox: native module
|
|
// setup + collection + suite elapsed time, not case sums or eight-CPU CI walls.
|
|
// Keep these config-scoped work hints through file regrouping. The two-slot
|
|
// work estimate was 369.995s; its 389.731s wrapper included 12.745s preparation.
|
|
const STORAGE_MODULE_WORK_SECONDS = new Map<string, number>([
|
|
["src/infra/sqlite-readonly-location.copy.test.ts", 0.852],
|
|
["src/infra/sqlite-snapshot-staging-cancelled-owner.test.ts", 0.368],
|
|
["src/infra/sqlite-worker-client.test.ts", 0.379],
|
|
["src/infra/sqlite-worker-store.generation.test.ts", 2.427],
|
|
["src/infra/state-migrations.caller-mode.refusal.test.ts", 20.635],
|
|
["src/infra/state-migrations.media-persistence.canonical-archives.test.ts", 9.166],
|
|
["src/infra/state-migrations.retained-deleted-agent.test.ts", 36.507],
|
|
["src/channels/message/ingress-drain.async-work.test.ts", 2.656],
|
|
["src/tui/tui-last-session.test.ts", 5.299],
|
|
["src/worker/worker.chat-abort.test.ts", 15.867],
|
|
["src/agents/bash-tools.exec-background-followup.test.ts", 2.969],
|
|
["src/channels/message-access/operator-authority.test.ts", 31.775],
|
|
["src/agents/subagents/registry/subagent-registry.persistence.test.ts", 23.639],
|
|
["src/auto-reply/reply/session.acp-reset-routing.test.ts", 8.938],
|
|
["src/agents/tools/skill-workshop-tool.support-paths.test.ts", 1.686],
|
|
["src/claws/package-update.test.ts", 4.164],
|
|
["src/cli/update-cli.git-service.test.ts", 24.578],
|
|
["src/flows/doctor-health.fleet-preflight.test.ts", 8.648],
|
|
["src/agents/embedded-agent-runner/run/attempt-prompt-submit.retention.test.ts", 6.452],
|
|
["src/agents/embedded-agent-runner/compact.delegate.test.ts", 16.151],
|
|
["src/cli/plugins-cli.update.test.ts", 9.511],
|
|
["src/state/openclaw-agent-participants-migration.test.ts", 9.279],
|
|
["src/agents/tools/transcripts-tool.selection.test.ts", 14.535],
|
|
["src/agents/main-session-recovery/main-session-restart-recovery-admission.test.ts", 16.187],
|
|
["src/agents/main-session-recovery/main-session-restart-recovery.test.ts", 279.963],
|
|
["test/runtime-agent.codex-initialization.integration.test.ts", 45.303],
|
|
["src/tasks/task-executor.test.ts", 7.528],
|
|
["src/agents/agent-harness-native-custody.test.ts", 27.986],
|
|
["src/system-agent/setup-inference-activate.test.ts", 69.039],
|
|
["src/plugins/install-record-commit.retention.test.ts", 7.563],
|
|
["src/infra/device-pairing-node.lifecycle.test.ts", 4.165],
|
|
["src/agents/tools-effective-inventory.cold-provider.test.ts", 9.37],
|
|
["src/meeting-bot/participation.capacity-race.test.ts", 0.909],
|
|
["test/imessage-reply-alias.integration.test.ts", 6.901],
|
|
["test/plugins/memory-dreaming-cron.test.ts", 2.991],
|
|
["src/config/sessions/session-accessor.sqlite-history-query-plan.test.ts", 5.374],
|
|
]);
|
|
const STORAGE_WRAPPER_SECONDS = 20;
|
|
|
|
const DEFAULT_WHOLE_GROUP_SECONDS = 25;
|
|
const DEFAULT_SECONDS_PER_TEST_FILE = 0.5;
|
|
const COMPACT_PUSH_EXCLUDED_SHARDS = new Set([
|
|
"core-runtime-tui-pty",
|
|
...Array.from(
|
|
{ length: COMPACT_TOOLING_NODE_TEST_GROUPS },
|
|
(_, index) => `core-tooling-${index + 1}`,
|
|
),
|
|
"core-tooling-isolated",
|
|
]);
|
|
const COMPACT_BLACKSMITH_SPLIT_OWNERS = new Set([
|
|
"agentic-cli",
|
|
"agentic-agents-support",
|
|
"agentic-control-plane-agent-chat",
|
|
"agentic-gateway-core-1",
|
|
"agentic-gateway-core-2",
|
|
"agentic-gateway-core-3",
|
|
"core-runtime-infra-storage-state",
|
|
]);
|
|
// Spawn/signal-timing suites (process-group waits, PTY smoke) flake when a
|
|
// concurrent sibling Vitest run competes for the 4 vCPU runner. Pack them
|
|
// into bins the shard runner executes at concurrency 1.
|
|
const EXCLUSIVE_COMPACT_GROUP_RE =
|
|
/^core-tooling(?:-\d+(?:-hosted-\d+)?|-isolated)$|^core-runtime-tui-pty$|^agentic-gateway-core-(?:runtime|inventory)$|^agentic-cli(?:-hosted-\d+|-process(?:-hosted-\d+)?)?$/u;
|
|
// Exclusive bins run serially, so their packed estimate is their wall clock.
|
|
// An indivisible file above this budget must not acquire additional work.
|
|
const COMPACT_EXCLUSIVE_JOB_SECONDS = 150;
|
|
const COMPACT_HYBRID_SERIAL_CLI_JOB_SECONDS = 250;
|
|
const COMPACT_SERIAL_CLI_GROUP_NAME_RE = /^agentic-cli(?:(?:-process)?-hosted-\d+)?$/u;
|
|
|
|
export function isExclusiveCompactShardName(shardName: string): boolean {
|
|
return EXCLUSIVE_COMPACT_GROUP_RE.test(shardName);
|
|
}
|
|
|
|
function isExclusiveCompactGroup(group: NodeTestShardGroup): boolean {
|
|
return (
|
|
isExclusiveCompactShardName(group.shard_name) ||
|
|
// Blacksmith's whole support owner used its host alone. File partitions
|
|
// retain that allocation instead of inheriting overlapping worker caps.
|
|
(group.runner === EXTRA_LARGE_NODE_TEST_RUNNER &&
|
|
/^agentic-agents-support-hosted-\d+$/u.test(group.shard_name))
|
|
);
|
|
}
|
|
|
|
function isParallelCompactGroup(group: NodeTestShardGroup): boolean {
|
|
return !isExclusiveCompactGroup(group) && !group.requiresDist && !group.pretestBuildMode;
|
|
}
|
|
|
|
// Spawn/signal/PTY-timing suites also flake under high in-process worker
|
|
// counts; pin them to the proven 2-worker budget while the job-level default
|
|
// scales with the runner class. infra-process spawns child processes per test
|
|
// and hit worker-startup timeouts under contention before serialization.
|
|
const PINNED_WORKER_COMPACT_GROUP_RE =
|
|
/^core-tooling(?:-\d+(?:-hosted-\d+)?|-isolated)$|^core-runtime-tui-pty$|^core-runtime-infra-process$|^core-runtime-config$|^core-runtime-media-ui-(?:\d+|support)$|^agentic-cli(?:-process)?$|^agentic-gateway-(?:core-\d+|methods)$/u;
|
|
const PINNED_COMPACT_GROUP_ENV = { OPENCLAW_VITEST_MAX_WORKERS: "2" };
|
|
const MEASURED_GATEWAY_ISOLATED_GROUP_RE = /^agentic-gateway-server-isolated(?:-hosted-\d+)?$/u;
|
|
const MEASURED_GATEWAY_METHODS_GROUP_RE = /^agentic-gateway-methods(?:-hosted-\d+)?$/u;
|
|
const MEASURED_CLI_GROUP_RE = /^agentic-cli(?:-hosted-\d+)?$/u;
|
|
const FILE_PARALLEL_AGENT_GROUP_RE = /^agentic-agents-(?:embedded-base-\d+|embedded-run|tools)$/u;
|
|
const FILE_PARALLEL_AGENT_MIN_WORKERS = 2;
|
|
|
|
// Existing samples describe serial file execution. Keep new parallel wall
|
|
// samples under a separate identity so refitting never discounts them twice.
|
|
function fileParallelAgentFallbackSeconds(
|
|
group: NodeTestShardGroup,
|
|
seconds: number | undefined,
|
|
): number | undefined {
|
|
return seconds !== undefined && FILE_PARALLEL_AGENT_GROUP_RE.test(group.shard_name)
|
|
? Math.max(
|
|
seconds / FILE_PARALLEL_AGENT_MIN_WORKERS,
|
|
...(group.includePatterns ?? []).map(stripeFileWeight),
|
|
)
|
|
: seconds;
|
|
}
|
|
|
|
function isAutoReplyReplyGroup(group: NodeTestShardGroup): boolean {
|
|
return (
|
|
group.configs.length === 1 &&
|
|
group.configs[0] === "test/vitest/vitest.auto-reply-reply.config.ts"
|
|
);
|
|
}
|
|
|
|
function compactEffectiveFileWorkers(group: NodeTestShardGroup, fileCount: number): number {
|
|
return isAutoReplyReplyGroup(group) ? Math.max(1, Math.min(2, fileCount)) : 1;
|
|
}
|
|
|
|
function readSerialAutoReplySplitSeconds(
|
|
group: NodeTestShardGroup,
|
|
profile: "blacksmith" | "github",
|
|
): number | undefined {
|
|
if (!isAutoReplyReplyGroup(group) || !group.includePatterns?.length) {
|
|
return undefined;
|
|
}
|
|
const { OPENCLAW_VITEST_MAX_WORKERS: _workers, ...env } = group.env ?? {};
|
|
const legacy = createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env,
|
|
parentShardName: group.shard_name,
|
|
stripes: [group.includePatterns],
|
|
});
|
|
return readCompleteSplitGenerationSeconds(readCompactGroupTimings(profile), legacy.selectorKey);
|
|
}
|
|
|
|
function parallelCompactFallbackSeconds(
|
|
group: NodeTestShardGroup,
|
|
seconds: number,
|
|
profile: "blacksmith" | "github" = "blacksmith",
|
|
): number {
|
|
// A complete legacy generation can be newer than its parent; partial or changed inventories cannot.
|
|
return (
|
|
Math.max(seconds, readSerialAutoReplySplitSeconds(group, profile) ?? 0) /
|
|
compactEffectiveFileWorkers(group, group.includePatterns?.length ?? 1)
|
|
);
|
|
}
|
|
|
|
function isParallelAgentsCoreGroup(group: NodeTestShardGroup): boolean {
|
|
return group.configs.length === 1 && group.configs[0] === agentVitestProjectOwners.core.config;
|
|
}
|
|
|
|
function agentsCoreWorkFiles(group: NodeTestShardGroup): string[] {
|
|
const files = group.includePatterns ?? [];
|
|
const unitFastFiles = new Set(getUnitFastTestFilesForIncludePatterns(files));
|
|
return files.filter((file) => isStripeEligibleTestFile(file, unitFastFiles));
|
|
}
|
|
|
|
function effectiveAgentsCoreWorkers(group: NodeTestShardGroup): number {
|
|
if (!isParallelAgentsCoreGroup(group) || !group.includePatterns?.length) {
|
|
return 1;
|
|
}
|
|
const weights = agentsCoreWorkFiles(group).map(stripeFileWeight);
|
|
if (weights.length === 0) {
|
|
return 1;
|
|
}
|
|
const total = weights.reduce((sum, weight) => sum + weight, 0);
|
|
// Two workers share the file work; an indivisible file keeps its serial cost.
|
|
return Math.min(2, weights.length, total / Math.max(...weights));
|
|
}
|
|
|
|
function readSerialAgentsCoreSeconds(
|
|
group: NodeTestShardGroup,
|
|
profile: "blacksmith" | "github",
|
|
): number | undefined {
|
|
if (!isParallelAgentsCoreGroup(group)) {
|
|
return undefined;
|
|
}
|
|
const { OPENCLAW_VITEST_MAX_WORKERS: _workers, ...env } = group.env ?? {};
|
|
const complete = group.includePatterns?.length
|
|
? readCompleteSplitGenerationSeconds(
|
|
readCompactGroupTimings(profile),
|
|
createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env,
|
|
parentShardName: group.shard_name,
|
|
stripes: [group.includePatterns],
|
|
}).selectorKey,
|
|
)
|
|
: undefined;
|
|
const parent = readCompactGroupTimings(profile)[group.shard_name];
|
|
return parent === undefined && complete === undefined
|
|
? undefined
|
|
: Math.max(parent ?? 0, complete ?? 0);
|
|
}
|
|
|
|
function usesMeasuredCompactWorkers(group: NodeTestShardGroup, runnerBackend: string | undefined) {
|
|
return (
|
|
(runnerBackend === undefined || runnerBackend === "blacksmith" || runnerBackend === "hybrid") &&
|
|
(MEASURED_CLI_GROUP_RE.test(group.shard_name) ||
|
|
isParallelCommandsGroup(group) ||
|
|
/^agentic-gateway-core-2(?:-hosted-\d+)?$/u.test(group.shard_name) ||
|
|
MEASURED_GATEWAY_ISOLATED_GROUP_RE.test(group.shard_name) ||
|
|
MEASURED_GATEWAY_METHODS_GROUP_RE.test(group.shard_name) ||
|
|
FILE_PARALLEL_AGENT_GROUP_RE.test(group.shard_name.replace(/-hosted-\d+$/u, "")))
|
|
);
|
|
}
|
|
|
|
function isParallelGatewayServerGroup(group: NodeTestShardGroup): boolean {
|
|
return (
|
|
group.configs.length === 1 && group.configs[0] === "test/vitest/vitest.gateway-server.config.ts"
|
|
);
|
|
}
|
|
|
|
function gatewayServerEffectiveWorkers(group: NodeTestShardGroup): number {
|
|
return Math.max(
|
|
1,
|
|
Math.min(
|
|
2,
|
|
group.includePatterns?.filter((file) => !gatewayServerSerialTestFiles.includes(file))
|
|
.length ?? 2,
|
|
),
|
|
);
|
|
}
|
|
|
|
function gatewayServerSerialSeconds(
|
|
files: readonly string[] | undefined,
|
|
runnerBackend: string | undefined,
|
|
): number {
|
|
const scale =
|
|
runnerBackend === "github"
|
|
? COMPACT_GITHUB_GROUP_SECONDS_SCALE
|
|
: runnerBackend === "hybrid"
|
|
? COMPACT_HYBRID_GROUP_SECONDS_SCALE
|
|
: 1;
|
|
return (files ?? []).reduce(
|
|
(seconds, file) =>
|
|
seconds + (gatewayServerSerialTestFiles.includes(file) ? stripeFileWeight(file) * scale : 0),
|
|
0,
|
|
);
|
|
}
|
|
|
|
function estimateLegacyGatewayServerSeconds(
|
|
group: NodeTestShardGroup,
|
|
profile: "blacksmith" | "github",
|
|
hint: number | undefined,
|
|
): number | undefined {
|
|
const { OPENCLAW_VITEST_MAX_WORKERS: _workers, ...env } = group.env ?? {};
|
|
const generation = createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env,
|
|
parentShardName: group.shard_name,
|
|
stripes: [group.includePatterns ?? []],
|
|
});
|
|
const measured = readCompactGroupTimings(profile)[group.shard_name];
|
|
const complete = readCompleteSplitGenerationSeconds(
|
|
readCompactGroupTimings(profile),
|
|
generation.selectorKey,
|
|
);
|
|
const seconds = measured ?? hint;
|
|
const serialSeconds = gatewayServerSerialSeconds(group.includePatterns, profile);
|
|
return seconds === undefined && complete === undefined
|
|
? undefined
|
|
: serialSeconds +
|
|
Math.max(0, Math.max(seconds ?? 0, complete ?? 0) - serialSeconds) /
|
|
gatewayServerEffectiveWorkers(group);
|
|
}
|
|
|
|
function applyCompactGroupWorkerPins(
|
|
group: NodeTestShardGroup,
|
|
runnerBackend: string | undefined,
|
|
): NodeTestShardGroup {
|
|
if (isAutoReplyReplyGroup(group)) {
|
|
// Keep serial observations separate so refitted parallel walls are never divided again.
|
|
return {
|
|
...group,
|
|
env: { ...group.env, ...PINNED_COMPACT_GROUP_ENV },
|
|
timing_key: `${compactGroupTimingKey(group)}-parallel-2`,
|
|
};
|
|
}
|
|
if (isParallelGatewayServerGroup(group)) {
|
|
// New measurements must not be divided again after serial files become parallel.
|
|
return {
|
|
...group,
|
|
timing_key: `${compactGroupTimingKey(group)}-parallel${gatewayServerSerialSeconds(group.includePatterns, runnerBackend) > 0 ? "-native-serial" : ""}`,
|
|
env: { ...group.env, ...PINNED_COMPACT_GROUP_ENV },
|
|
};
|
|
}
|
|
if (isParallelAgentsCoreGroup(group)) {
|
|
// Parallel samples must not overwrite serial history and be divided again.
|
|
return {
|
|
...group,
|
|
env: { ...group.env, ...PINNED_COMPACT_GROUP_ENV },
|
|
timing_key: `${compactGroupTimingKey(group)}-parallel`,
|
|
};
|
|
}
|
|
if (isParallelCommandsGroup(group)) {
|
|
return {
|
|
...group,
|
|
fallbackMaxWorkers: 2,
|
|
// Preserve the coverage tier while separating parallel and serial samples.
|
|
timing_key: `${compactGroupTimingKey(group)}${COMMANDS_PARALLEL_TIMING_SUFFIX}`,
|
|
};
|
|
}
|
|
const timedGroup =
|
|
FILE_PARALLEL_AGENT_GROUP_RE.test(group.shard_name) && group.timing_key === undefined
|
|
? { ...group, timing_key: `${group.shard_name}#file-parallel` }
|
|
: group;
|
|
if (
|
|
MEASURED_GATEWAY_METHODS_GROUP_RE.test(timedGroup.shard_name) &&
|
|
usesMeasuredCompactWorkers(timedGroup, runnerBackend)
|
|
) {
|
|
return {
|
|
...timedGroup,
|
|
timing_key: `${compactGroupTimingKey(timedGroup)}#workers-4`,
|
|
env: { ...timedGroup.env, OPENCLAW_VITEST_MAX_WORKERS: "4" },
|
|
minTotalMemoryBytes: 28 * 1024 ** 3,
|
|
fallbackMaxWorkers: 2,
|
|
};
|
|
}
|
|
if (usesMeasuredCompactWorkers(timedGroup, runnerBackend)) {
|
|
return {
|
|
...timedGroup,
|
|
...(MEASURED_GATEWAY_ISOLATED_GROUP_RE.test(timedGroup.shard_name)
|
|
? {
|
|
// Leave cold Gateway startup room within the existing test deadlines.
|
|
env: { ...timedGroup.env, OPENCLAW_VITEST_MAX_WORKERS: "2" },
|
|
minTotalMemoryBytes: 28 * 1024 ** 3,
|
|
}
|
|
: {}),
|
|
fallbackMaxWorkers: 2,
|
|
};
|
|
}
|
|
if (!PINNED_WORKER_COMPACT_GROUP_RE.test(timedGroup.shard_name)) {
|
|
return timedGroup;
|
|
}
|
|
return { ...timedGroup, env: { ...timedGroup.env, ...PINNED_COMPACT_GROUP_ENV } };
|
|
}
|
|
|
|
function readCompactGroupSeconds(
|
|
group: NodeTestShardGroup,
|
|
profile: "blacksmith" | "github",
|
|
): number | undefined {
|
|
const timings = readCompactGroupTimings(profile);
|
|
const key = compactGroupTimingKey(group);
|
|
const fullOwnerKey = key.startsWith("changed-") ? key.slice("changed-".length) : undefined;
|
|
// A reduced parallel sample retires the full-owner fallback for both timing shapes.
|
|
const hasReducedParallelSample =
|
|
fullOwnerKey !== undefined &&
|
|
isParallelGatewayServerGroup(group) &&
|
|
timings[key.endsWith("-stripes") ? key.slice(0, -"-stripes".length) : `${key}-stripes`] !==
|
|
undefined;
|
|
// Keep the full owner's admission estimate until the reduced tier has samples.
|
|
const measured =
|
|
timings[key] ??
|
|
(fullOwnerKey !== undefined && !hasReducedParallelSample ? timings[fullOwnerKey] : undefined);
|
|
if (measured !== undefined) {
|
|
return measured;
|
|
}
|
|
if (key.endsWith("#workers-4") && MEASURED_GATEWAY_METHODS_GROUP_RE.test(group.shard_name)) {
|
|
return (
|
|
timings[key.slice(0, -"#workers-4".length)] ??
|
|
(fullOwnerKey === undefined
|
|
? undefined
|
|
: timings[fullOwnerKey.slice(0, -"#workers-4".length)])
|
|
);
|
|
}
|
|
if (FILE_PARALLEL_AGENT_GROUP_RE.test(group.shard_name)) {
|
|
return fileParallelAgentFallbackSeconds(group, timings[group.shard_name]);
|
|
}
|
|
const cronOwner = /^core-runtime-cron-parallel-(core|isolated-agent|service)$/u.exec(
|
|
compactGroupTimingKey(group),
|
|
)?.[1];
|
|
const serialSeconds =
|
|
cronOwner === undefined ? undefined : timings[`core-runtime-cron-${cronOwner}`];
|
|
// Compact overlap and inherited Gateway caps allow two workers; hosted serial
|
|
// rows allow at least three. New names let parallel samples replace this
|
|
// conservative serial-cost fallback without dividing those samples again.
|
|
return serialSeconds === undefined ? undefined : serialSeconds / 2;
|
|
}
|
|
|
|
function isParallelToolingGroup(group: NodeTestShardGroup): boolean {
|
|
return /^core-tooling-\d+(?:-hosted-\d+)?$/u.test(group.shard_name);
|
|
}
|
|
|
|
function compactToolingTestSecondsLimit(
|
|
groups: readonly NodeTestShardGroup[],
|
|
runnerBackend: string | undefined,
|
|
): number {
|
|
// Keep the measured planner-proof tail bounded without fragmenting unrelated tooling.
|
|
const hasPlannerProof = groups.some((group) =>
|
|
group.includePatterns?.some((file) =>
|
|
/^test\/scripts\/ci-changed-node-test-plan(?:\.[^/]+)?\.test\.ts$/u.test(file),
|
|
),
|
|
);
|
|
return runnerBackend !== "github" && hasPlannerProof
|
|
? COMPACT_EXCLUSIVE_JOB_SECONDS
|
|
: COMPACT_SERIAL_NODE_TEST_JOB_SECONDS;
|
|
}
|
|
|
|
function estimateParallelToolingSeconds(
|
|
group: Pick<NodeTestShardGroup, "env">,
|
|
files: readonly string[],
|
|
runnerBackend: string | undefined,
|
|
fileTimings = readToolingFileTimings(runnerBackend === "github" ? "github" : "blacksmith"),
|
|
): number {
|
|
const workers = Math.min(
|
|
files.length,
|
|
Number(
|
|
group.env?.OPENCLAW_VITEST_MAX_WORKERS ??
|
|
PINNED_COMPACT_GROUP_ENV.OPENCLAW_VITEST_MAX_WORKERS,
|
|
),
|
|
);
|
|
const fallbackTimings =
|
|
runnerBackend === "github" ? readToolingFileTimings("blacksmith") : fileTimings;
|
|
const weights = files.map(
|
|
(file) =>
|
|
(fileTimings[file] ?? toolingFileWeight(file, fallbackTimings)) *
|
|
(runnerBackend === "github" && fileTimings[file] === undefined
|
|
? COMPACT_GITHUB_GROUP_SECONDS_SCALE
|
|
: 1),
|
|
);
|
|
// File observations retain their elapsed cost under parallel execution. Old
|
|
// numbered parent/child spans describe serial files and cannot price this lane.
|
|
return Math.max(
|
|
0,
|
|
...weights,
|
|
weights.reduce((sum, seconds) => sum + seconds, 0) / Math.max(1, workers),
|
|
);
|
|
}
|
|
|
|
function estimateDefaultCompactGroupSeconds(group: NodeTestShardGroup): number {
|
|
if (isParallelToolingGroup(group) && group.includePatterns) {
|
|
return estimateParallelToolingSeconds(group, group.includePatterns, "blacksmith");
|
|
}
|
|
const measured = readCompactGroupSeconds(group, "blacksmith");
|
|
if (measured !== undefined) {
|
|
return measured;
|
|
}
|
|
const hint = isParallelGatewayServerGroup(group)
|
|
? estimateLegacyGatewayServerSeconds(
|
|
group,
|
|
"blacksmith",
|
|
COMPACT_GROUP_SECONDS_HINTS.get(group.shard_name),
|
|
)
|
|
: ((isAutoReplyReplyGroup(group)
|
|
? readCompactGroupTimings("blacksmith")[group.shard_name]
|
|
: undefined) ??
|
|
readSerialAgentsCoreSeconds(group, "blacksmith") ??
|
|
(isParallelAgentsCoreGroup(group)
|
|
? COMPACT_LARGE_GROUP_STRIPE_SECONDS_HINTS.get(group.shard_name)
|
|
: undefined) ??
|
|
fileParallelAgentFallbackSeconds(group, COMPACT_GROUP_SECONDS_HINTS.get(group.shard_name)));
|
|
if (hint !== undefined) {
|
|
return parallelCompactFallbackSeconds(group, hint) / effectiveAgentsCoreWorkers(group);
|
|
}
|
|
if (Array.isArray(group.includePatterns)) {
|
|
return (
|
|
parallelCompactFallbackSeconds(
|
|
group,
|
|
Math.max(3, Math.round(group.includePatterns.length * DEFAULT_SECONDS_PER_TEST_FILE)),
|
|
) / effectiveAgentsCoreWorkers(group)
|
|
);
|
|
}
|
|
return DEFAULT_WHOLE_GROUP_SECONDS;
|
|
}
|
|
|
|
function usesExpandedRunnerProfile(runnerBackend: string | undefined): boolean {
|
|
return runnerBackend === "github" || runnerBackend === "hybrid";
|
|
}
|
|
|
|
// Hand-fitted tables stand in only until a group has direct Blacksmith samples,
|
|
// so a committed measurement owns the weight and the table covers the rest.
|
|
// Reading the tables first made every pinned group immune to the nightly refit:
|
|
// pins stale-low kept packing partners onto the tallest bins, and pins
|
|
// stale-high kept splitting groups that had since become cheap.
|
|
function readUnmeasuredCompactHint(
|
|
group: NodeTestShardGroup,
|
|
hints: ReadonlyMap<string, number>,
|
|
): number | undefined {
|
|
const timings = readCompactGroupTimings("blacksmith");
|
|
if (
|
|
readCompactGroupSeconds(group, "blacksmith") !== undefined ||
|
|
readSerialAgentsCoreSeconds(group, "blacksmith") !== undefined ||
|
|
(isAutoReplyReplyGroup(group) && timings[group.shard_name] !== undefined) ||
|
|
readSerialAutoReplySplitSeconds(group, "blacksmith") !== undefined
|
|
) {
|
|
return undefined;
|
|
}
|
|
const hint = fileParallelAgentFallbackSeconds(group, hints.get(group.shard_name));
|
|
return hint === undefined ? undefined : parallelCompactFallbackSeconds(group, hint);
|
|
}
|
|
|
|
function estimateHybridCompactGroupSeconds(group: NodeTestShardGroup, seconds: number): number {
|
|
// Hybrid first attempts use the same Blacksmith allocation as these samples.
|
|
// A historical hosted-to-Blacksmith factor must not discount a direct wall.
|
|
if (readCompactGroupSeconds(group, "blacksmith") !== undefined) {
|
|
return seconds;
|
|
}
|
|
// The 4,723s Blacksmith push hint sum measured 3,742.046s/3,756.674s
|
|
// (79.230%/79.540%) in runs 31945998653/31949756966. A 0.87 scale keeps
|
|
// 9.379% headroom above the higher ratio. With direct outlier hints, it sits
|
|
// one point above the 0.86 packing cliff.
|
|
const hint = readUnmeasuredCompactHint(group, COMPACT_HYBRID_GROUP_SECONDS_HINTS);
|
|
return hint === undefined
|
|
? Math.round(seconds * COMPACT_HYBRID_GROUP_SECONDS_SCALE)
|
|
: hint / effectiveAgentsCoreWorkers(group);
|
|
}
|
|
|
|
function estimateCompactGroupSeconds(
|
|
group: NodeTestShardGroup,
|
|
runnerBackend: string | undefined,
|
|
): number {
|
|
if (isParallelToolingGroup(group) && group.includePatterns) {
|
|
return estimateParallelToolingSeconds(group, group.includePatterns, runnerBackend);
|
|
}
|
|
if (
|
|
isParallelCommandsGroup(group) &&
|
|
group.timing_key?.endsWith(COMMANDS_PARALLEL_TIMING_SUFFIX)
|
|
) {
|
|
const profile = runnerBackend === "github" ? "github" : "blacksmith";
|
|
const measured = readCompactGroupTimings(profile)[group.timing_key];
|
|
if (measured !== undefined) {
|
|
return runnerBackend === "hybrid"
|
|
? estimateHybridCompactGroupSeconds(group, measured)
|
|
: measured;
|
|
}
|
|
return estimateSerialCommandSeconds(group, runnerBackend, (source) =>
|
|
estimateCompactGroupSeconds({ ...group, ...source, timing_key: undefined }, runnerBackend),
|
|
);
|
|
}
|
|
const defaultSeconds = estimateDefaultCompactGroupSeconds(group);
|
|
// Hybrid attempt 1 runs on Blacksmith. It keeps the expanded topology for
|
|
// hosted retries, but its packing weights must describe the runner that
|
|
// normally executes the plan.
|
|
if (runnerBackend === "hybrid") {
|
|
return Math.max(
|
|
isParallelGatewayServerGroup(group)
|
|
? gatewayServerSerialSeconds(group.includePatterns, runnerBackend)
|
|
: 0,
|
|
estimateHybridCompactGroupSeconds(group, defaultSeconds),
|
|
);
|
|
}
|
|
if (runnerBackend !== "github") {
|
|
return Math.max(
|
|
defaultSeconds,
|
|
isParallelGatewayServerGroup(group)
|
|
? gatewayServerSerialSeconds(group.includePatterns, runnerBackend)
|
|
: 0,
|
|
);
|
|
}
|
|
const serialFloor = isParallelGatewayServerGroup(group)
|
|
? gatewayServerSerialSeconds(group.includePatterns, runnerBackend)
|
|
: 0;
|
|
const measured = readCompactGroupSeconds(group, "github");
|
|
if (measured !== undefined) {
|
|
return Math.max(serialFloor, measured);
|
|
}
|
|
const hint = isParallelGatewayServerGroup(group)
|
|
? estimateLegacyGatewayServerSeconds(
|
|
group,
|
|
"github",
|
|
COMPACT_GITHUB_GROUP_SECONDS_HINTS.get(group.shard_name),
|
|
)
|
|
: ((isAutoReplyReplyGroup(group)
|
|
? readCompactGroupTimings("github")[group.shard_name]
|
|
: undefined) ??
|
|
readSerialAgentsCoreSeconds(group, "github") ??
|
|
fileParallelAgentFallbackSeconds(
|
|
group,
|
|
COMPACT_GITHUB_GROUP_SECONDS_HINTS.get(group.shard_name),
|
|
));
|
|
return Math.max(
|
|
serialFloor,
|
|
hint === undefined
|
|
? Math.round(defaultSeconds * COMPACT_GITHUB_GROUP_SECONDS_SCALE)
|
|
: parallelCompactFallbackSeconds(group, hint, "github") / effectiveAgentsCoreWorkers(group),
|
|
parallelCompactFallbackSeconds(group, 0, "github"),
|
|
);
|
|
}
|
|
|
|
function estimateCompactStripeSeconds(
|
|
group: NodeTestShardGroup,
|
|
runnerBackend: string | undefined,
|
|
): number {
|
|
if (isParallelToolingGroup(group) && group.includePatterns) {
|
|
return estimateParallelToolingSeconds(group, group.includePatterns, runnerBackend);
|
|
}
|
|
if (
|
|
(isParallelAgentsCoreGroup(group) && group.timing_key === `${group.shard_name}-parallel`) ||
|
|
group.timing_key?.endsWith(COMMANDS_PARALLEL_TIMING_SUFFIX)
|
|
) {
|
|
return estimateCompactGroupSeconds(group, runnerBackend);
|
|
}
|
|
const split = group.timing_key && parseCompactSplitTimingKey(group.timing_key);
|
|
if (split && group.timing_key) {
|
|
// The parent-derived floor owns a new split until its exact child has samples.
|
|
// File-count fallbacks would price the same work again after partitioning.
|
|
const timings = readCompactGroupTimings(runnerBackend === "github" ? "github" : "blacksmith");
|
|
const exact = timings[group.timing_key];
|
|
if (exact !== undefined) {
|
|
return exact;
|
|
}
|
|
// Repacking siblings changes the generation, not an unchanged child's
|
|
// config, worker environment, or file membership within this selector.
|
|
const membership = group.timing_key.slice(group.timing_key.lastIndexOf("#include-"));
|
|
return Math.max(
|
|
0,
|
|
...Object.entries(timings)
|
|
.filter(
|
|
([key]) =>
|
|
key.endsWith(membership) &&
|
|
parseCompactSplitTimingKey(key)?.selectorKey === split.selectorKey,
|
|
)
|
|
.map(([, seconds]) => seconds),
|
|
);
|
|
}
|
|
if (
|
|
runnerBackend === "github" ||
|
|
isParallelGatewayServerGroup(group) ||
|
|
isAutoReplyReplyGroup(group)
|
|
) {
|
|
return estimateCompactGroupSeconds(group, runnerBackend);
|
|
}
|
|
const blacksmithSeconds =
|
|
readUnmeasuredCompactHint(group, COMPACT_LARGE_GROUP_STRIPE_SECONDS_HINTS) ??
|
|
estimateDefaultCompactGroupSeconds(group);
|
|
return runnerBackend === "hybrid"
|
|
? estimateHybridCompactGroupSeconds(group, blacksmithSeconds)
|
|
: blacksmithSeconds;
|
|
}
|
|
|
|
// Identify split siblings, including nested children of deliberately separated
|
|
// fixed stripes.
|
|
function compactStripeFamily(group: NodeTestShardGroup): string | undefined {
|
|
if (
|
|
/^agentic-commands-doctor-sessions-cron(?:-(?:memory|sqlite(?:-recovery)?))?(?:-hosted-\d+)*$/u.test(
|
|
group.shard_name,
|
|
)
|
|
) {
|
|
return "agentic-commands-doctor-sessions-cron";
|
|
}
|
|
return (
|
|
/^(agentic-agents-embedded-base|agentic-gateway-core|core-runtime-media-ui|core-unit-src-security)-\d+(?:-hosted-\d+)*$/u.exec(
|
|
group.shard_name,
|
|
)?.[1] ??
|
|
(group.timing_key ? parseCompactSplitTimingKey(group.timing_key)?.selectorKey : undefined)
|
|
);
|
|
}
|
|
|
|
function expandCompactGroup(
|
|
group: NodeTestShardGroup,
|
|
runnerBackend: string | undefined,
|
|
): NodeTestShardGroup[] {
|
|
if (group.shard_name !== "agentic-agents-embedded") {
|
|
return [group];
|
|
}
|
|
if (group.configs.length !== COMPACT_EMBEDDED_GROUP_NAMES.length) {
|
|
throw new Error("embedded compact group names must cover every config");
|
|
}
|
|
|
|
const expandedGroups: NodeTestShardGroup[] = [];
|
|
const selectedFiles = group.includePatterns ? new Set(group.includePatterns) : undefined;
|
|
for (const [index, config] of group.configs.entries()) {
|
|
const shardName = COMPACT_EMBEDDED_GROUP_NAMES[index];
|
|
if (!shardName) {
|
|
throw new Error("embedded compact group name is missing");
|
|
}
|
|
const configFiles = listNodeTestConfigFiles(config);
|
|
if (!configFiles) {
|
|
throw new Error(`embedded compact config lacks its file owner: ${config}`);
|
|
}
|
|
const selectedConfigFiles = selectedFiles
|
|
? configFiles.filter((file) => selectedFiles.has(file))
|
|
: undefined;
|
|
if (selectedConfigFiles?.length === 0) {
|
|
continue;
|
|
}
|
|
if (shardName !== COMPACT_EMBEDDED_BASE_GROUP_NAME) {
|
|
expandedGroups.push({
|
|
...group,
|
|
configs: [config],
|
|
...(selectedConfigFiles
|
|
? { includePatterns: selectedConfigFiles, timing_key: `changed-${shardName}` }
|
|
: {}),
|
|
shard_name: shardName,
|
|
});
|
|
continue;
|
|
}
|
|
const stripes = createStripedBatches(
|
|
configFiles,
|
|
EMBEDDED_BASE_NODE_TEST_STRIPES,
|
|
stripeFileWeight,
|
|
);
|
|
for (const [stripeIndex, stripe] of stripes.entries()) {
|
|
const includePatterns = selectedFiles
|
|
? stripe.filter((file) => selectedFiles.has(file))
|
|
: stripe;
|
|
// An empty include list makes the shard runner drop the include file and
|
|
// run the whole config, so every stripe would replay the entire suite.
|
|
if (includePatterns.length === 0) {
|
|
if (selectedFiles) {
|
|
continue;
|
|
}
|
|
throw new Error("embedded base stripe cannot be empty");
|
|
}
|
|
expandedGroups.push({
|
|
...group,
|
|
configs: [config],
|
|
includePatterns,
|
|
...(selectedFiles ? { timing_key: `changed-${shardName}-${stripeIndex + 1}` } : {}),
|
|
shard_name: `${shardName}-${stripeIndex + 1}`,
|
|
});
|
|
}
|
|
}
|
|
return expandedGroups.map((expandedGroup) =>
|
|
applyCompactGroupWorkerPins(expandedGroup, runnerBackend),
|
|
);
|
|
}
|
|
const TOOLING_CONFIG = "test/vitest/vitest.tooling.config.ts";
|
|
const TOOLING_DOCKER_TEST_FILE = "test/scripts/docker-build-helper.test.ts";
|
|
const TOOLING_UNIFIED_DECLARATIONS_TEST_FILE = "test/scripts/write-unified-entry-dts.test.ts";
|
|
const TOOLING_LARGE_CAPACITY_TEST_FILES = new Set([
|
|
// Generation-retention cases require eight CPUs / 24 GiB; the two-CPU
|
|
// screen also peaked at 6.05 GiB before those gated cases could run.
|
|
"test/scripts/vitest-worker-artifacts.ci.test.ts",
|
|
"test/scripts/write-unified-entry-dts.test.ts",
|
|
"test/scripts/write-plugin-sdk-entry-dts.test.ts",
|
|
]);
|
|
const TOOLING_ISOLATED_CONFIG = "test/vitest/vitest.tooling-isolated.config.ts";
|
|
// The full matrix is capped at 28 jobs. Admit the consistently slow serial
|
|
// shards first so short alphabetical groups cannot leave them on the tail.
|
|
const FULL_NODE_TEST_ADMISSION_PRIORITY = new Map([
|
|
// Admit the broad unit-fast graphs before short alphabetical groups.
|
|
["core-unit-fast-1", 0],
|
|
["core-unit-fast-2", 0],
|
|
...Array.from(
|
|
{ length: COMPACT_TOOLING_NODE_TEST_GROUPS },
|
|
(_, index) => [`core-tooling-${index + 1}`, 1] as const,
|
|
),
|
|
]);
|
|
// Commands and cron run non-isolated, so keep their split shards as separate
|
|
// processes. Combining their include lists can retain test state across groups.
|
|
const BUNDLEABLE_NODE_TEST_CONFIGS = new Set(["test/vitest/vitest.infra.config.ts"]);
|
|
const KEEP_LARGE_NODE_TEST_RUNNER = new Set([
|
|
"agentic-agents-core-auth",
|
|
"agentic-agents-core-models",
|
|
"agentic-agents-core-runtime",
|
|
"agentic-agents-core-subagents",
|
|
"agentic-agents-embedded",
|
|
"agentic-agents-support",
|
|
"agentic-agents-core-runner-cli-1",
|
|
"agentic-agents-core-runner-cli-2",
|
|
"agentic-agents-core-runner-cli-3",
|
|
"agentic-agents-core-runner-commands",
|
|
"agentic-agents-core-runner-embedded",
|
|
"agentic-agents-core-runner-sessions",
|
|
"agentic-agents-core-tools",
|
|
"agentic-control-plane-startup-core",
|
|
"agentic-gateway-core-1",
|
|
"agentic-gateway-core-2",
|
|
"agentic-gateway-core-3",
|
|
"agentic-gateway-methods",
|
|
"agentic-gateway-server-isolated",
|
|
"auto-reply-reply-dispatch",
|
|
"auto-reply-reply-dispatch-core",
|
|
"auto-reply-reply-dispatch-delivery",
|
|
"auto-reply-reply-dispatch-lifecycle",
|
|
// The commands stripes and security suite are import-bound (30-45s of
|
|
// module-graph import per file); the 8 vCPU class with a higher Vitest
|
|
// worker budget cuts their wall clock roughly linearly.
|
|
"auto-reply-reply-commands-1",
|
|
"auto-reply-reply-commands-2",
|
|
"auto-reply-reply-commands-3",
|
|
"core-runtime-media-ui-1",
|
|
"core-runtime-media-ui-2",
|
|
"core-runtime-media-ui-3",
|
|
"core-runtime-media-ui-support",
|
|
"core-unit-fast-1",
|
|
"core-unit-fast-2",
|
|
"core-unit-fast-isolated",
|
|
"core-unit-src-security-1",
|
|
"core-unit-src-security-2",
|
|
"core-unit-src-security-3",
|
|
"core-unit-src-security-support",
|
|
]);
|
|
const RELEASE_ONLY_PLUGIN_SHARDS = new Set(["agentic-plugins"]);
|
|
const RELEASE_ONLY_TOOLING_SHARDS = new Set(["core-tooling"]);
|
|
const RELEASE_ONLY_UI_TEST_FILES = new Set([
|
|
"ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/board-fixture.e2e.test.ts",
|
|
"ui/src/e2e/chat-attachment-menu.e2e.test.ts",
|
|
"ui/src/e2e/chat-mobile-bubble-margin.e2e.test.ts",
|
|
"ui/src/e2e/github-link-hovercard.e2e.test.ts",
|
|
"ui/src/e2e/settings-layout.e2e.test.ts",
|
|
"ui/src/e2e/theme-muted-contrast.e2e.test.ts",
|
|
"ui/src/e2e/native-embed-settings.e2e.test.ts",
|
|
"ui/src/e2e/chat-session-entry.e2e.test.ts",
|
|
"ui/src/components/app-sidebar.stress.browser.test.ts",
|
|
"ui/src/e2e/cron-duration-save.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/desktop-resize.real-gateway.e2e.test.ts",
|
|
"extensions/qa-lab/src/control-ui-automation-management.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/quota-reset-status.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/session-pr-reader-lifetime.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/chat-collaborator-scroll.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/mcp-app-conformance.e2e.test.ts",
|
|
"ui/src/e2e/usage-sessions-owner-attribution.e2e.test.ts",
|
|
"extensions/qa-lab/src/control-ui-openclaw-delegation.real-gateway.e2e.test.ts",
|
|
"extensions/qa-lab/src/control-ui-media-transcript.real-gateway.e2e.test.ts",
|
|
"extensions/qa-lab/src/session-host-command-state.real-gateway.e2e.test.ts",
|
|
]);
|
|
|
|
export function createUiTestShardGroups(
|
|
options: RuntimeTestSelection & { includeReleaseOnlyTests?: boolean } = {},
|
|
) {
|
|
const includeReleaseOnlyTests = options.includeReleaseOnlyTests ?? true;
|
|
const changedPaths = new Set(options.changedPaths ?? []);
|
|
const files =
|
|
includeReleaseOnlyTests && options.includePrExemptRuntimeTests !== false
|
|
? undefined
|
|
: listTrackedTestFiles(".").filter(
|
|
(file) =>
|
|
(includeReleaseOnlyTests ||
|
|
!RELEASE_ONLY_UI_TEST_FILES.has(file) ||
|
|
changedPaths.has(file)) &&
|
|
isRuntimeTestFileIncluded(file, options),
|
|
);
|
|
const group = (config: string, ownsFile: (file: string) => boolean) => [
|
|
{
|
|
configs: [config],
|
|
shard_name: config,
|
|
...(files ? { includePatterns: files.filter(ownsFile) } : {}),
|
|
},
|
|
];
|
|
return {
|
|
ui: group("ui/vitest.config.ts", isUiTestTarget),
|
|
e2e: group(
|
|
"test/vitest/vitest.ui-e2e.config.ts",
|
|
(file) =>
|
|
controlUiE2eTestGlobs.some((pattern) => matchesGlob(file, pattern)) ||
|
|
uiE2eRealGatewayTestFiles.includes(file),
|
|
),
|
|
};
|
|
}
|
|
|
|
export function createUiRealGatewayTestShards(
|
|
e2eGroups: ReturnType<typeof createUiTestShardGroups>["e2e"],
|
|
) {
|
|
const selected = new Set(
|
|
e2eGroups.flatMap((group) => group.includePatterns ?? uiE2eRealGatewayTestFiles),
|
|
);
|
|
const parallelFiles = new Set(uiE2ePrebuiltParallelTestFiles);
|
|
// Balance the serial phase with standalone fixtures that need no preview build.
|
|
const standaloneCompanions = new Set([
|
|
"ui/src/e2e/chat-loading-performance.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/chat-project-media.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/chat-widget-sandbox.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/command-palette-catalog.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/model-api-keys.real-gateway.e2e.test.ts",
|
|
"ui/src/e2e/model-catalog-partial-refresh.real-gateway.e2e.test.ts",
|
|
]);
|
|
const desktop = "ui/src/e2e/desktop-resize.real-gateway.e2e.test.ts";
|
|
const files = uiE2eRealGatewayTestFiles.filter((file) => selected.has(file) && file !== desktop);
|
|
// Desktop transport proof owns its file separately, alongside the serial phase.
|
|
return ([1, 2] as const).map((shard) => ({
|
|
shard,
|
|
shard_count: 2 as const,
|
|
run_desktop: shard === 1 && selected.has(desktop),
|
|
groups: [
|
|
{
|
|
configs: ["test/vitest/vitest.ui-e2e-prebuilt.config.ts"],
|
|
shard_name: `ui-e2e-real-gateway-${shard === 1 ? "desktop" : "parallel"}`,
|
|
includePatterns: files.filter(
|
|
(file) => (parallelFiles.has(file) && !standaloneCompanions.has(file)) === (shard === 2),
|
|
),
|
|
},
|
|
],
|
|
}));
|
|
}
|
|
|
|
export const RELEASE_ONLY_TOOLING_CONFIGS = new Set(
|
|
fullSuiteVitestShards
|
|
.filter((shard) => RELEASE_ONLY_TOOLING_SHARDS.has(shard.name))
|
|
.flatMap((shard) => shard.projects),
|
|
);
|
|
|
|
export function isReleaseOnlyToolingTestFile(file: string): boolean {
|
|
return (
|
|
!file.endsWith(".e2e.test.ts") &&
|
|
!file.endsWith(".live.test.ts") &&
|
|
(file.startsWith("test/scripts/") || file.startsWith("src/scripts/"))
|
|
);
|
|
}
|
|
|
|
function listToolingGroupFiles(configs: readonly string[]): string[] | undefined {
|
|
const files = [
|
|
...(configs.includes(TOOLING_CONFIG) ? listCompactToolingTestFiles() : []),
|
|
...(configs.includes(TOOLING_ISOLATED_CONFIG) ? toolingIsolatedTestFiles : []),
|
|
...(configs.includes("test/vitest/vitest.tooling-docker.config.ts")
|
|
? [TOOLING_DOCKER_TEST_FILE]
|
|
: []),
|
|
...(configs.includes("test/vitest/vitest.unit-fast-isolated.config.ts")
|
|
? getUnitFastIsolatedTestFiles()
|
|
: []),
|
|
...(configs.includes("test/vitest/vitest.unit-fast-fake-timers.config.ts")
|
|
? getUnitFastTimerTestFiles()
|
|
: []),
|
|
];
|
|
return files.length > 0 ? files : undefined;
|
|
}
|
|
|
|
function resolveTestFilesBuildMode(files: readonly string[]): NodeTestPretestBuildMode | undefined {
|
|
// Planner inventories contain resolved paths. Preserve their exact membership
|
|
// instead of reparsing every file as a glob against the runtime consumers.
|
|
const selectedFiles = new Set(files);
|
|
return resolveVitestPretestBuildMode([{ matchesFile: (file) => selectedFiles.has(file) }]);
|
|
}
|
|
|
|
function createAutoReplyReplySplitShards(): NodeTestSplitShard[] {
|
|
const files = listTrackedTestFiles("src/auto-reply/reply");
|
|
const groups = {
|
|
"auto-reply-reply-agent-runner": [] as string[],
|
|
"auto-reply-reply-commands": [] as string[],
|
|
"auto-reply-reply-dispatch": [] as string[],
|
|
"auto-reply-reply-dispatch-core": [] as string[],
|
|
"auto-reply-reply-dispatch-delivery": [] as string[],
|
|
"auto-reply-reply-dispatch-lifecycle": [] as string[],
|
|
"auto-reply-reply-session": [] as string[],
|
|
"auto-reply-reply-state-routing": [] as string[],
|
|
};
|
|
const dispatchEntrypoints = new Map<string, keyof typeof groups>([
|
|
["dispatch-from-config.test.ts", "auto-reply-reply-dispatch-core"],
|
|
["dispatch-from-config.delivery.test.ts", "auto-reply-reply-dispatch-delivery"],
|
|
["dispatch-from-config.lifecycle.test.ts", "auto-reply-reply-dispatch-lifecycle"],
|
|
]);
|
|
|
|
for (const file of files) {
|
|
const name = relative("src/auto-reply/reply", file).replaceAll("\\", "/");
|
|
const dispatchEntrypointGroup = dispatchEntrypoints.get(name);
|
|
if (dispatchEntrypointGroup) {
|
|
groups[dispatchEntrypointGroup].push(file);
|
|
continue;
|
|
}
|
|
if (
|
|
name.startsWith("agent-runner") ||
|
|
name.startsWith("acp-") ||
|
|
name === "abort.test.ts" ||
|
|
name === "bash-command.stop.test.ts" ||
|
|
name.startsWith("block-")
|
|
) {
|
|
groups["auto-reply-reply-agent-runner"].push(file);
|
|
} else if (name.startsWith("commands")) {
|
|
groups["auto-reply-reply-commands"].push(file);
|
|
} else if (
|
|
name.startsWith("directive-") ||
|
|
name.startsWith("dispatch") ||
|
|
name.startsWith("followup-") ||
|
|
name.startsWith("get-reply")
|
|
) {
|
|
groups["auto-reply-reply-dispatch"].push(file);
|
|
} else if (name.startsWith("session")) {
|
|
groups["auto-reply-reply-session"].push(file);
|
|
} else {
|
|
groups["auto-reply-reply-state-routing"].push(file);
|
|
}
|
|
}
|
|
|
|
return Object.entries(groups)
|
|
.flatMap(([groupName, includePatterns]) => {
|
|
// Retain separate command stripes so packing can spread their import cost across jobs.
|
|
if (groupName === "auto-reply-reply-commands") {
|
|
return createStripedBatches(
|
|
includePatterns,
|
|
AUTO_REPLY_COMMANDS_STRIPES,
|
|
stripeFileWeight,
|
|
).map((batch, index) => ({
|
|
configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
|
|
includePatterns: batch,
|
|
requiresDist: false,
|
|
shardName: `${groupName}-${index + 1}`,
|
|
}));
|
|
}
|
|
return [
|
|
{
|
|
configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
|
|
includePatterns,
|
|
requiresDist: false,
|
|
shardName: groupName,
|
|
},
|
|
];
|
|
})
|
|
.filter((shard) => shard.includePatterns.length > 0);
|
|
}
|
|
|
|
function resolveAgentCoreShardName(file: string): string {
|
|
const name = relative("src/agents", file).replaceAll("\\", "/");
|
|
if (
|
|
name.includes("auth") ||
|
|
name.includes("credential") ||
|
|
name.includes("api-key") ||
|
|
name.includes("token")
|
|
) {
|
|
return "agentic-agents-core-auth";
|
|
}
|
|
if (
|
|
name.startsWith("model") ||
|
|
name.includes("provider") ||
|
|
name.includes("openai") ||
|
|
name.includes("anthropic") ||
|
|
name.includes("gemini") ||
|
|
name.includes("moonshot") ||
|
|
name.includes("minimax") ||
|
|
name.includes("xai") ||
|
|
name.includes("zai") ||
|
|
name.includes("chutes") ||
|
|
name.includes("catalog")
|
|
) {
|
|
return "agentic-agents-core-models";
|
|
}
|
|
if (
|
|
name.startsWith("agent-tools") ||
|
|
name.startsWith("openclaw-tools") ||
|
|
name.startsWith("bash-tools") ||
|
|
name.startsWith("tool") ||
|
|
name.startsWith("apply-patch") ||
|
|
name.startsWith("exec") ||
|
|
name.startsWith("sandbox")
|
|
) {
|
|
return "agentic-agents-core-tools";
|
|
}
|
|
if (
|
|
name.startsWith("subagent") ||
|
|
name.startsWith("spawn") ||
|
|
name.startsWith("embedded-agent-subscribe")
|
|
) {
|
|
return "agentic-agents-core-subagents";
|
|
}
|
|
// The former single "core-runner" bucket serialized ~3 minutes of tests in
|
|
// one group; keep these three slices separate so packing can balance them.
|
|
if (name.startsWith("embedded-agent-runner")) {
|
|
return "agentic-agents-core-runner-embedded";
|
|
}
|
|
if (
|
|
name.startsWith("agent-command") ||
|
|
name.startsWith("command") ||
|
|
name.includes("compaction")
|
|
) {
|
|
return "agentic-agents-core-runner-commands";
|
|
}
|
|
if (name.startsWith("cli-runner")) {
|
|
return "agentic-agents-core-runner-cli";
|
|
}
|
|
if (name.includes("session")) {
|
|
return "agentic-agents-core-runner-sessions";
|
|
}
|
|
return "agentic-agents-core-runtime";
|
|
}
|
|
|
|
function createAgentCoreSplitShards(): NodeTestSplitShard[] {
|
|
const excludedTests = new Set(agentVitestProjectOwners.core.exclude);
|
|
const groups = new Map<string, string[]>();
|
|
for (const file of listTrackedTestFiles("src/agents")) {
|
|
const name = relative("src/agents", file).replaceAll("\\", "/");
|
|
if (name.includes("/") || excludedTests.has(file)) {
|
|
continue;
|
|
}
|
|
const shardName = resolveAgentCoreShardName(file);
|
|
groups.set(shardName, [...(groups.get(shardName) ?? []), file]);
|
|
}
|
|
|
|
const sharedShards = [
|
|
"agentic-agents-core-auth",
|
|
"agentic-agents-core-models",
|
|
"agentic-agents-core-tools",
|
|
"agentic-agents-core-subagents",
|
|
"agentic-agents-core-runner-cli",
|
|
"agentic-agents-core-runner-commands",
|
|
"agentic-agents-core-runner-embedded",
|
|
"agentic-agents-core-runner-sessions",
|
|
"agentic-agents-core-runtime",
|
|
]
|
|
.flatMap((shardName) => {
|
|
const includePatterns = groups.get(shardName) ?? [];
|
|
// Retain the import-heavy CLI stripes; their files also share the bounded
|
|
// worker pool, while timing estimates account for the effective file workers.
|
|
if (shardName === "agentic-agents-core-runner-cli") {
|
|
return createStripedBatches(
|
|
includePatterns,
|
|
AGENTS_CORE_RUNNER_CLI_STRIPES,
|
|
stripeFileWeight,
|
|
).map((batch, index) => ({
|
|
configs: [agentVitestProjectOwners.core.config],
|
|
includePatterns: batch,
|
|
requiresDist: false,
|
|
shardName: `${shardName}-${index + 1}`,
|
|
}));
|
|
}
|
|
return [
|
|
{
|
|
configs: [agentVitestProjectOwners.core.config],
|
|
includePatterns,
|
|
requiresDist: false,
|
|
shardName,
|
|
},
|
|
];
|
|
})
|
|
.filter((shard) => shard.includePatterns.length > 0);
|
|
|
|
return [
|
|
...sharedShards,
|
|
{
|
|
configs: [agentVitestProjectOwners.spawnProductionBoundary.config],
|
|
includePatterns: agentVitestProjectOwners.spawnProductionBoundary.include,
|
|
requiresDist: false,
|
|
shardName: "agentic-agents-core-spawn-production-boundary",
|
|
},
|
|
{
|
|
configs: [agentVitestProjectOwners.coreIsolated.config],
|
|
includePatterns: agentVitestProjectOwners.coreIsolated.include,
|
|
requiresDist: false,
|
|
shardName: "agentic-agents-core-isolated",
|
|
},
|
|
];
|
|
}
|
|
|
|
function resolveGatewayStartupShardName(file: string): string {
|
|
const name = relative("src/gateway", file).replaceAll("\\", "/");
|
|
if (name.startsWith("server-startup-config") || name.startsWith("server-startup-early")) {
|
|
return "agentic-control-plane-startup-config";
|
|
}
|
|
if (
|
|
name.startsWith("server-runtime") ||
|
|
name.startsWith("server.health") ||
|
|
name.startsWith("server.lazy")
|
|
) {
|
|
return "agentic-control-plane-startup-health-runtime";
|
|
}
|
|
if (name.startsWith("server-restart") || name === "server-close.test.ts") {
|
|
return "agentic-control-plane-startup-restart-close";
|
|
}
|
|
return "agentic-control-plane-startup-core";
|
|
}
|
|
|
|
function resolveGatewayServerShardName(file: string): string {
|
|
const name = relative("src/gateway", file).replaceAll("\\", "/");
|
|
if (
|
|
isGatewayServerBackedHttpTestFile(file) ||
|
|
name.startsWith("server.models") ||
|
|
name.startsWith("server.talk")
|
|
) {
|
|
return "agentic-control-plane-http-models";
|
|
}
|
|
if (
|
|
name.startsWith("server.agent") ||
|
|
name.startsWith("server.chat") ||
|
|
name.startsWith("server.sessions")
|
|
) {
|
|
return "agentic-control-plane-agent-chat";
|
|
}
|
|
if (
|
|
name.includes("auth") ||
|
|
name.includes("device") ||
|
|
name.includes("node") ||
|
|
name.includes("roles") ||
|
|
name.includes("silent") ||
|
|
name.includes("preauth") ||
|
|
name.includes("control-plane-rate-limit")
|
|
) {
|
|
return "agentic-control-plane-auth-node";
|
|
}
|
|
if (
|
|
name.startsWith("server-startup") ||
|
|
name.startsWith("server-restart") ||
|
|
name.startsWith("server-runtime") ||
|
|
name.startsWith("server.lazy") ||
|
|
name.startsWith("server.health") ||
|
|
name === "server-close.test.ts"
|
|
) {
|
|
return resolveGatewayStartupShardName(file);
|
|
}
|
|
if (name.includes("cron")) {
|
|
return "agentic-control-plane-runtime-cron";
|
|
}
|
|
if (
|
|
name.includes("plugin") ||
|
|
name.includes("hooks") ||
|
|
name.includes("http") ||
|
|
name.includes("ws-connection")
|
|
) {
|
|
return "agentic-control-plane-http-plugin-ws";
|
|
}
|
|
if (name.startsWith("server-")) {
|
|
return "agentic-control-plane-runtime-server";
|
|
}
|
|
if (name.startsWith("server.config-patch")) {
|
|
return "agentic-control-plane-runtime-config";
|
|
}
|
|
if (name.startsWith("server.shared-token")) {
|
|
return "agentic-control-plane-runtime-shared-token";
|
|
}
|
|
if (
|
|
name.startsWith("server.control-ui-root") ||
|
|
name.startsWith("server.ios-client-id") ||
|
|
name.startsWith("server.tools-catalog")
|
|
) {
|
|
return "agentic-control-plane-runtime-ui-tools";
|
|
}
|
|
if (name.startsWith("server.")) {
|
|
return "agentic-control-plane-runtime-state";
|
|
}
|
|
return "agentic-control-plane-runtime";
|
|
}
|
|
|
|
function createGatewayServerSplitShards(): NodeTestSplitShard[] {
|
|
const groups = new Map<string, string[]>();
|
|
for (const file of listTrackedTestFiles("src/gateway").filter(isGatewayServerTestFile)) {
|
|
const shardName = resolveGatewayServerShardName(file);
|
|
groups.set(shardName, [...(groups.get(shardName) ?? []), file]);
|
|
}
|
|
return [
|
|
"agentic-control-plane-agent-chat",
|
|
"agentic-control-plane-auth-node",
|
|
"agentic-control-plane-http-models",
|
|
"agentic-control-plane-http-plugin-ws",
|
|
"agentic-control-plane-runtime",
|
|
"agentic-control-plane-runtime-config",
|
|
"agentic-control-plane-runtime-cron",
|
|
"agentic-control-plane-runtime-server",
|
|
"agentic-control-plane-runtime-shared-token",
|
|
"agentic-control-plane-runtime-state",
|
|
"agentic-control-plane-runtime-ui-tools",
|
|
"agentic-control-plane-startup-config",
|
|
"agentic-control-plane-startup-core",
|
|
"agentic-control-plane-startup-health-runtime",
|
|
"agentic-control-plane-startup-restart-close",
|
|
]
|
|
.map((shardName) => ({
|
|
configs: ["test/vitest/vitest.gateway-server.config.ts"],
|
|
env:
|
|
shardName === "agentic-control-plane-startup-health-runtime"
|
|
? GATEWAY_STARTUP_HEALTH_RUNTIME_ENV
|
|
: undefined,
|
|
includePatterns: groups.get(shardName) ?? [],
|
|
requiresDist: false,
|
|
runner:
|
|
shardName === "agentic-control-plane-startup-core"
|
|
? GATEWAY_STARTUP_CORE_RUNNER
|
|
: BUNDLED_NODE_TEST_RUNNER,
|
|
shardName,
|
|
}))
|
|
.filter((shard) => shard.includePatterns.length > 0);
|
|
}
|
|
|
|
function resolveCronShardName(file: string): string {
|
|
const name = relative("src/cron", file).replaceAll("\\", "/");
|
|
if (name.startsWith("isolated-agent")) {
|
|
return "core-runtime-cron-parallel-isolated-agent";
|
|
}
|
|
if (name.startsWith("service")) {
|
|
return "core-runtime-cron-parallel-service";
|
|
}
|
|
return "core-runtime-cron-parallel-core";
|
|
}
|
|
|
|
function createCronSplitShards(): NodeTestSplitShard[] {
|
|
const groups = new Map<string, string[]>();
|
|
for (const file of listTrackedTestFiles("src/cron")) {
|
|
const shardName = resolveCronShardName(file);
|
|
groups.set(shardName, [...(groups.get(shardName) ?? []), file]);
|
|
}
|
|
|
|
return [
|
|
"core-runtime-cron-parallel-core",
|
|
"core-runtime-cron-parallel-isolated-agent",
|
|
"core-runtime-cron-parallel-service",
|
|
]
|
|
.map((shardName) => ({
|
|
configs: ["test/vitest/vitest.cron.config.ts"],
|
|
includePatterns: groups.get(shardName) ?? [],
|
|
requiresDist: false,
|
|
shardName,
|
|
}))
|
|
.filter((shard) => shard.includePatterns.length > 0);
|
|
}
|
|
|
|
// Keep precedence: specific owners such as heartbeat-runner precede their broad prefixes.
|
|
const INFRA_SHARD_PREFIXES = [
|
|
["approval-exec", ["approval", "exec"]],
|
|
["heartbeat-runner", ["heartbeat-runner"]],
|
|
["heartbeat-core", ["heartbeat"]],
|
|
["outbound-actions", ["outbound/message-action"]],
|
|
["outbound-core", ["outbound/"]],
|
|
["net-install", ["net/", "install", "npm", "brew", "binaries"]],
|
|
["device", ["device"]],
|
|
["gateway-lock-argv", ["gateway-lock", "gateway-process-argv"]],
|
|
["gateway-processes", ["gateway-processes"]],
|
|
["gateway-watch", ["gateway-watch"]],
|
|
["network-node", ["node", "bonjour", "network"]],
|
|
["diagnostics-state", ["archive", "backup", "diagnostic"]],
|
|
[
|
|
"files-commands",
|
|
[
|
|
"command-analysis/",
|
|
"command-explainer/",
|
|
"file-",
|
|
"fs-",
|
|
"json",
|
|
"path",
|
|
"shell",
|
|
"tmp-openclaw-dir",
|
|
],
|
|
],
|
|
["provider-push", ["provider-usage", "push-"]],
|
|
["storage-state", ["kysely", "session", "sqlite", "stale-lock", "state-migrations"]],
|
|
["channel-plugin", ["channel", "plugin", "pairing", "voicewake"]],
|
|
[
|
|
"system-runtime",
|
|
["package", "ports", "process", "restart", "runtime", "run-node", "system", "update"],
|
|
],
|
|
[
|
|
"env-auth",
|
|
[
|
|
"dotenv",
|
|
"env",
|
|
"gemini-auth",
|
|
"google-api",
|
|
"home-dir",
|
|
"host-env",
|
|
"openclaw-exec-env",
|
|
"secret",
|
|
"secure-random",
|
|
],
|
|
],
|
|
[
|
|
"repo-tooling",
|
|
[
|
|
"build-stamp",
|
|
"changelog",
|
|
"clawhub",
|
|
"detect-package-manager",
|
|
"git-",
|
|
"openclaw-root",
|
|
"tsdown",
|
|
"vitest",
|
|
],
|
|
],
|
|
[
|
|
"network-platform",
|
|
["scp", "ssh", "tailnet", "tailscale", "tcp", "tls/", "transport", "widearea", "windows", "ws"],
|
|
],
|
|
[
|
|
"core-utils",
|
|
[
|
|
"abort",
|
|
"backoff",
|
|
"errors",
|
|
"fatal-error",
|
|
"fetch",
|
|
"fixed-window",
|
|
"format-time/",
|
|
"http-body",
|
|
"plain-object",
|
|
"prototype-keys",
|
|
"retry",
|
|
"warning-filter",
|
|
],
|
|
],
|
|
["cli-ui", ["browser", "cli-", "clipboard", "control-ui", "embedded", "is-main"]],
|
|
[
|
|
"events-runtime",
|
|
["agent-events", "event-session", "infra-", "non-fatal", "supervisor", "unhandled"],
|
|
],
|
|
[
|
|
"file-safety",
|
|
[
|
|
"boundary",
|
|
"hardlink",
|
|
"replace-file",
|
|
"resolve-system-bin",
|
|
"safe-package-install",
|
|
"stable-node-path",
|
|
"watch-node",
|
|
],
|
|
],
|
|
["misc-dedupe-disk", ["dedupe", "disk-space"]],
|
|
["misc-values", ["inline-option-token", "map-size", "machine-name"]],
|
|
["misc-os", ["os-summary"]],
|
|
] as const;
|
|
|
|
function resolveInfraShardName(file: string): string {
|
|
const name = relative("src/infra", file).replaceAll("\\", "/");
|
|
const owner = INFRA_SHARD_PREFIXES.find(([, prefixes]) =>
|
|
prefixes.some((prefix) => name.startsWith(prefix)),
|
|
);
|
|
return `core-runtime-infra-${owner?.[0] ?? "misc"}`;
|
|
}
|
|
|
|
function createInfraSplitShards(): NodeTestSplitShard[] {
|
|
const groups = new Map<string, string[]>();
|
|
for (const file of listTrackedTestFiles("src/infra")) {
|
|
if (isDatabaseWorkerCoreTestFile(file)) {
|
|
continue;
|
|
}
|
|
const shardName = resolveInfraShardName(file);
|
|
groups.set(shardName, [...(groups.get(shardName) ?? []), file]);
|
|
}
|
|
groups.set("core-runtime-infra-storage-state", [
|
|
...(groups.get("core-runtime-infra-storage-state") ?? []),
|
|
...databaseWorkerCoreTestFiles,
|
|
]);
|
|
|
|
return [
|
|
"core-runtime-infra-approval-exec",
|
|
"core-runtime-infra-channel-plugin",
|
|
"core-runtime-infra-cli-ui",
|
|
"core-runtime-infra-device",
|
|
"core-runtime-infra-diagnostics-state",
|
|
"core-runtime-infra-core-utils",
|
|
"core-runtime-infra-env-auth",
|
|
"core-runtime-infra-events-runtime",
|
|
"core-runtime-infra-file-safety",
|
|
"core-runtime-infra-files-commands",
|
|
"core-runtime-infra-gateway-lock-argv",
|
|
"core-runtime-infra-gateway-processes",
|
|
"core-runtime-infra-gateway-watch",
|
|
"core-runtime-infra-heartbeat-core",
|
|
"core-runtime-infra-heartbeat-runner",
|
|
"core-runtime-infra-misc",
|
|
"core-runtime-infra-misc-dedupe-disk",
|
|
"core-runtime-infra-misc-os",
|
|
"core-runtime-infra-misc-values",
|
|
"core-runtime-infra-net-install",
|
|
"core-runtime-infra-network-node",
|
|
"core-runtime-infra-network-platform",
|
|
"core-runtime-infra-outbound-actions",
|
|
"core-runtime-infra-outbound-core",
|
|
"core-runtime-infra-provider-push",
|
|
"core-runtime-infra-repo-tooling",
|
|
"core-runtime-infra-storage-state",
|
|
"core-runtime-infra-system-runtime",
|
|
]
|
|
.map((shardName) => ({
|
|
configs: ["test/vitest/vitest.infra.config.ts"],
|
|
includePatterns: groups.get(shardName) ?? [],
|
|
requiresDist: false,
|
|
runner: "blacksmith-4vcpu-ubuntu-2404",
|
|
shardName,
|
|
}))
|
|
.filter((shard) => shard.includePatterns.length > 0);
|
|
}
|
|
|
|
// The broad unit-fast graph is import-bound (~180s of module evaluation on an
|
|
// 8 vCPU runner as one job); striping the file list halves the wall clock.
|
|
// Isolated and fake-timer projects stay whole: they are small and own
|
|
// worker-isolation semantics that include lists must not slice.
|
|
function createUnitFastSplitShards(): NodeTestSplitShard[] {
|
|
const timerTestFiles = new Set(getUnitFastTimerTestFiles());
|
|
const isolatedTestFiles = new Set(getUnitFastIsolatedTestFiles());
|
|
const stripeFiles = getUnitFastTestFiles().filter(
|
|
(file) => !timerTestFiles.has(file) && !isolatedTestFiles.has(file),
|
|
);
|
|
return [
|
|
...createStripedBatches(stripeFiles, UNIT_FAST_NODE_TEST_STRIPES, stripeFileWeight).map(
|
|
(includePatterns, index) => ({
|
|
shardName: `core-unit-fast-${index + 1}`,
|
|
configs: ["test/vitest/vitest.unit-fast.config.ts"],
|
|
includePatterns,
|
|
requiresDist: false,
|
|
}),
|
|
),
|
|
// Split per config: the combined pair owned a ~190s hosted wall that no
|
|
// bin packing could shorten, while the halves fit normal lanes.
|
|
{
|
|
shardName: "core-unit-fast-isolated",
|
|
configs: ["test/vitest/vitest.unit-fast-isolated.config.ts"],
|
|
requiresDist: false,
|
|
},
|
|
{
|
|
shardName: "core-unit-fast-fake-timers",
|
|
configs: ["test/vitest/vitest.unit-fast-fake-timers.config.ts"],
|
|
requiresDist: false,
|
|
},
|
|
];
|
|
}
|
|
|
|
// Price the sixteen parent stripes with the same file workers as their compact
|
|
// children, so a long file can share spare worker capacity with shorter files.
|
|
// Push compacts omit tooling; retained compact groups stay exclusive.
|
|
function createToolingSplitShards(): NodeTestSplitShard[] {
|
|
const files = listCompactToolingTestFiles();
|
|
// Resolve file ownership once; batch scoring reuses those prepared facts.
|
|
const buildModes = new Map(files.map((file) => [file, resolveTestFilesBuildMode([file])]));
|
|
const toolingBatchWeight = (batch: string[]) => {
|
|
const mode = mergeVitestPretestBuildModes(batch.map((file) => buildModes.get(file)));
|
|
return (
|
|
estimateParallelToolingSeconds({ env: PINNED_COMPACT_GROUP_ENV }, batch, "blacksmith") +
|
|
(mode ? VITEST_PRETEST_BUILD_SECONDS[mode] : 0)
|
|
);
|
|
};
|
|
return [
|
|
...createStripedBatches(
|
|
files,
|
|
COMPACT_TOOLING_NODE_TEST_GROUPS,
|
|
(file) => toolingBatchWeight([file]),
|
|
toolingBatchWeight,
|
|
).map((includePatterns, index) => ({
|
|
shardName: `core-tooling-${index + 1}`,
|
|
configs: [TOOLING_CONFIG],
|
|
includePatterns,
|
|
requiresDist: false,
|
|
})),
|
|
{
|
|
shardName: "core-tooling-isolated",
|
|
configs: ["test/vitest/vitest.tooling-docker.config.ts", TOOLING_ISOLATED_CONFIG],
|
|
requiresDist: false,
|
|
},
|
|
];
|
|
}
|
|
|
|
function createStripedSplitShards(params: {
|
|
configs: string[];
|
|
files: string[];
|
|
includeExternalConfigs?: boolean;
|
|
shardName: string;
|
|
stripeCount: number;
|
|
}): NodeTestSplitShard[] {
|
|
return createStripedBatches(params.files, params.stripeCount, stripeFileWeight).map(
|
|
(includePatterns, index) => ({
|
|
configs: params.configs,
|
|
includeExternalConfigs: params.includeExternalConfigs,
|
|
includePatterns,
|
|
requiresDist: false,
|
|
shardName: `${params.shardName}-${index + 1}`,
|
|
}),
|
|
);
|
|
}
|
|
|
|
function createCoreUnitSrcSecuritySplitShards(): NodeTestSplitShard[] {
|
|
const unitFastFiles = new Set(getUnitFastTestFiles());
|
|
const files = filterUnitConfigTestFiles(
|
|
listTrackedTestFiles("src").filter(
|
|
(file) =>
|
|
isStripeEligibleTestFile(file, unitFastFiles) &&
|
|
!file.startsWith("src/acp/") &&
|
|
!file.startsWith("src/security/"),
|
|
),
|
|
);
|
|
return [
|
|
...createStripedSplitShards({
|
|
configs: ["test/vitest/vitest.unit-src.config.ts"],
|
|
files,
|
|
shardName: "core-unit-src-security",
|
|
stripeCount: CORE_UNIT_SRC_SECURITY_STRIPES,
|
|
}),
|
|
{
|
|
configs: ["test/vitest/vitest.unit-security.config.ts"],
|
|
includeExternalConfigs: true,
|
|
requiresDist: false,
|
|
shardName: "core-unit-src-security-support",
|
|
},
|
|
];
|
|
}
|
|
|
|
function createCoreRuntimeMediaUiSplitShards(): NodeTestSplitShard[] {
|
|
const unitFastFiles = new Set(getUnitFastTestFiles());
|
|
const separateUiFiles = new Set([...uiIsolatedTestFiles, ...uiTimingTestFiles]);
|
|
const files = [
|
|
...listTrackedTestFiles("ui/src"),
|
|
...listTrackedTestFiles("extensions").filter(isPluginControlUiPath),
|
|
].filter(
|
|
(file) =>
|
|
isStripeEligibleTestFile(file, unitFastFiles) &&
|
|
!separateUiFiles.has(file) &&
|
|
!isUiBrowserTestFile(file),
|
|
);
|
|
return [
|
|
...createStripedSplitShards({
|
|
configs: ["test/vitest/vitest.ui.config.ts"],
|
|
files,
|
|
shardName: "core-runtime-media-ui",
|
|
stripeCount: CORE_RUNTIME_MEDIA_UI_STRIPES,
|
|
}),
|
|
{
|
|
configs: [
|
|
"test/vitest/vitest.media.config.ts",
|
|
"test/vitest/vitest.media-understanding.config.ts",
|
|
"test/vitest/vitest.tui.config.ts",
|
|
"test/vitest/vitest.ui-isolated.config.ts",
|
|
"test/vitest/vitest.ui-timing.config.ts",
|
|
"test/vitest/vitest.wizard.config.ts",
|
|
],
|
|
requiresDist: false,
|
|
shardName: "core-runtime-media-ui-support",
|
|
},
|
|
];
|
|
}
|
|
|
|
function partitionRuntimeTestFiles(configs: string[], files: string[]) {
|
|
const runtimeFiles = new Set(listVitestRuntimeConsumerFiles(configs));
|
|
return {
|
|
runtimeFiles: files.filter((file) => runtimeFiles.has(file)),
|
|
otherFiles: files.filter((file) => !runtimeFiles.has(file)),
|
|
};
|
|
}
|
|
|
|
const GATEWAY_CORE_NODE_TEST_CONFIGS = [
|
|
"test/vitest/vitest.gateway-core.config.ts",
|
|
"test/vitest/vitest.gateway-client.config.ts",
|
|
];
|
|
|
|
function createAgenticGatewayCoreSplitShards(): NodeTestSplitShard[] {
|
|
const unitFastFiles = new Set(getUnitFastTestFiles());
|
|
const excludedGatewayFiles = new Set([
|
|
...databaseWorkerCoreTestFiles,
|
|
...gatewayDatabaseWorkerTestFiles,
|
|
...gatewayServerExcludedTestFiles,
|
|
...gatewayServerIsolatedTestFiles,
|
|
]);
|
|
const gatewayFiles = listTrackedTestFiles("src/gateway").filter(
|
|
(file) =>
|
|
isStripeEligibleTestFile(file, unitFastFiles) &&
|
|
!file.startsWith("src/gateway/server-methods/") &&
|
|
!isGatewayServerTestFile(file) &&
|
|
!excludedGatewayFiles.has(file),
|
|
);
|
|
const packageFiles = ["packages/gateway-client/src", "packages/gateway-protocol/src"]
|
|
.flatMap((rootDir) => listTrackedTestFiles(rootDir))
|
|
.filter((file) => isStripeEligibleTestFile(file, unitFastFiles));
|
|
const configs = GATEWAY_CORE_NODE_TEST_CONFIGS;
|
|
// The pretest runtime build is charged per job, so a stripe holding one of
|
|
// these files pays it for the whole stripe. Striping spreads them, which made
|
|
// every gateway-core job pay a 275s build to run ~120s of tests. Keep them in
|
|
// one shard so exactly one job builds.
|
|
const { runtimeFiles, otherFiles } = partitionRuntimeTestFiles(configs, [
|
|
...gatewayFiles,
|
|
...packageFiles,
|
|
]);
|
|
const inventoryFile = "src/gateway/worker-environments/workspace-large-inventory.test.ts";
|
|
return [
|
|
...createStripedSplitShards({
|
|
configs,
|
|
files: otherFiles.filter((file) => file !== inventoryFile),
|
|
shardName: "agentic-gateway-core",
|
|
stripeCount: AGENTIC_GATEWAY_CORE_STRIPES,
|
|
}),
|
|
// Keep the full journal/apply/recovery proof off the shared Vitest worker pool.
|
|
{
|
|
configs: ["test/vitest/vitest.gateway-core.config.ts"],
|
|
includePatterns: [inventoryFile],
|
|
requiresDist: false,
|
|
shardName: "agentic-gateway-core-inventory",
|
|
},
|
|
...(runtimeFiles.length > 0
|
|
? [
|
|
{
|
|
configs,
|
|
includePatterns: runtimeFiles,
|
|
requiresDist: false,
|
|
shardName: "agentic-gateway-core-runtime",
|
|
},
|
|
]
|
|
: []),
|
|
];
|
|
}
|
|
|
|
const TUI_PTY_NODE_TEST_SHARD: NodeTestSplitShard = {
|
|
shardName: "core-runtime-tui-pty",
|
|
configs: ["test/vitest/vitest.tui-pty.config.ts"],
|
|
env: {
|
|
OPENCLAW_TUI_PTY_INCLUDE_LOCAL: "1",
|
|
OPENCLAW_TUI_PTY_USE_BUILT_CLI: "1",
|
|
},
|
|
requiresDist: true,
|
|
runner: "blacksmith-4vcpu-ubuntu-2404",
|
|
};
|
|
const AGENTS_EMBEDDED_NODE_TEST_SHARD: NodeTestSplitShard = {
|
|
shardName: "agentic-agents-embedded",
|
|
configs: embeddedAgentVitestProjectOwners.map((owner) => owner.config),
|
|
env: AGENTS_EMBEDDED_AGENT_ENV,
|
|
requiresDist: false,
|
|
};
|
|
|
|
const SPLIT_NODE_SHARDS = new Map<string, NodeTestSplitShard[] | (() => NodeTestSplitShard[])>([
|
|
["core-unit-fast", createUnitFastSplitShards],
|
|
["core-tooling", createToolingSplitShards],
|
|
["core-unit-src", createCoreUnitSrcSecuritySplitShards],
|
|
["core-unit-security", []],
|
|
[
|
|
"core-unit-support",
|
|
[
|
|
{
|
|
shardName: "core-unit-support",
|
|
configs: ["test/vitest/vitest.unit-support.config.ts"],
|
|
requiresDist: false,
|
|
},
|
|
],
|
|
],
|
|
[
|
|
"core-runtime",
|
|
() => [
|
|
{
|
|
shardName: "core-runtime-hooks",
|
|
configs: ["test/vitest/vitest.hooks.config.ts"],
|
|
requiresDist: false,
|
|
runner: "blacksmith-4vcpu-ubuntu-2404",
|
|
},
|
|
...createInfraSplitShards(),
|
|
{
|
|
shardName: "core-runtime-secrets",
|
|
configs: ["test/vitest/vitest.secrets.config.ts"],
|
|
requiresDist: false,
|
|
runner: "blacksmith-4vcpu-ubuntu-2404",
|
|
},
|
|
// runtime-config owns ~90% of the former three-config wall; keeping it
|
|
// separate lets the hosted splitter stripe it while logging/process
|
|
// stay a cheap pair.
|
|
{
|
|
shardName: "core-runtime-infra-process",
|
|
configs: ["test/vitest/vitest.logging.config.ts", "test/vitest/vitest.process.config.ts"],
|
|
includePatterns: ["src/logging", "src/process"].flatMap((root) =>
|
|
listScopedOwnerTestFiles({
|
|
root,
|
|
include: [`${root}/**/*.test.ts`],
|
|
exclude: [...databaseWorkerCoreTestFiles],
|
|
}),
|
|
),
|
|
requiresDist: false,
|
|
runner: "blacksmith-4vcpu-ubuntu-2404",
|
|
},
|
|
{
|
|
shardName: "core-runtime-config",
|
|
configs: ["test/vitest/vitest.runtime-config.config.ts"],
|
|
requiresDist: false,
|
|
runner: "blacksmith-4vcpu-ubuntu-2404",
|
|
},
|
|
TUI_PTY_NODE_TEST_SHARD,
|
|
...createCoreRuntimeMediaUiSplitShards(),
|
|
{
|
|
shardName: "core-runtime-shared",
|
|
configs: [
|
|
"test/vitest/vitest.acp.config.ts",
|
|
"test/vitest/vitest.shared-core.config.ts",
|
|
"test/vitest/vitest.utils.config.ts",
|
|
],
|
|
requiresDist: false,
|
|
},
|
|
...createCronSplitShards(),
|
|
],
|
|
],
|
|
[
|
|
"auto-reply",
|
|
() => [
|
|
{
|
|
shardName: "auto-reply-core-top-level",
|
|
configs: [
|
|
"test/vitest/vitest.auto-reply-core.config.ts",
|
|
"test/vitest/vitest.auto-reply-top-level.config.ts",
|
|
],
|
|
requiresDist: false,
|
|
},
|
|
...createAutoReplyReplySplitShards(),
|
|
],
|
|
],
|
|
[
|
|
"agentic",
|
|
() => [
|
|
...createGatewayServerSplitShards(),
|
|
{
|
|
shardName: "agentic-gateway-server-isolated",
|
|
configs: [
|
|
"test/vitest/vitest.gateway-server-isolated.config.ts",
|
|
"test/vitest/vitest.gateway-database-workers.config.ts",
|
|
],
|
|
requiresDist: false,
|
|
},
|
|
// Split per config: the combined pair owned a ~206s hosted wall that no
|
|
// bin packing could shorten, while the halves fit normal lanes.
|
|
{
|
|
shardName: "agentic-cli",
|
|
configs: ["test/vitest/vitest.cli.config.ts"],
|
|
requiresDist: false,
|
|
},
|
|
{
|
|
shardName: "agentic-cli-process",
|
|
configs: ["test/vitest/vitest.cli-process.config.ts"],
|
|
requiresDist: false,
|
|
},
|
|
{
|
|
shardName: "agentic-command-support",
|
|
configs: [
|
|
"test/vitest/vitest.commands-light.config.ts",
|
|
"test/vitest/vitest.daemon.config.ts",
|
|
],
|
|
requiresDist: false,
|
|
},
|
|
...createAgenticCommandSplitShards(resolveTestFilesBuildMode),
|
|
...createAgentCoreSplitShards(),
|
|
AGENTS_EMBEDDED_NODE_TEST_SHARD,
|
|
{
|
|
shardName: "agentic-agents-support",
|
|
configs: [agentVitestProjectOwners.support.config],
|
|
requiresDist: false,
|
|
},
|
|
{
|
|
shardName: "agentic-agents-tools",
|
|
configs: [agentVitestProjectOwners.tools.config],
|
|
requiresDist: false,
|
|
},
|
|
...createAgenticGatewayCoreSplitShards(),
|
|
{
|
|
shardName: "agentic-gateway-methods",
|
|
configs: [
|
|
"test/vitest/vitest.gateway-methods.config.ts",
|
|
"test/vitest/vitest.gateway-methods-isolated.config.ts",
|
|
],
|
|
requiresDist: false,
|
|
},
|
|
{
|
|
shardName: "agentic-plugin-sdk",
|
|
configs: [
|
|
"test/vitest/vitest.plugin-sdk-light.config.ts",
|
|
"test/vitest/vitest.plugin-sdk.config.ts",
|
|
],
|
|
requiresDist: false,
|
|
},
|
|
{
|
|
shardName: "agentic-plugins",
|
|
configs: ["test/vitest/vitest.plugins.config.ts"],
|
|
requiresDist: false,
|
|
},
|
|
],
|
|
],
|
|
]);
|
|
const DIST_DEPENDENT_NODE_SHARD_NAMES = new Set(["core-support-boundary"]);
|
|
// Definitions are immutable for a planner process; defer file discovery until an
|
|
// owner is requested while retaining the original canonical metadata snapshot.
|
|
const canonicalNodeTestOwners = fullSuiteVitestShards.map((shard) => ({
|
|
...shard,
|
|
projects: [...shard.projects],
|
|
}));
|
|
let canonicalMetadataConfigs: ReadonlySet<string> | undefined;
|
|
type CanonicalTargetInventory = {
|
|
configsByFile: Map<string, Set<string>>;
|
|
configs: Set<string>;
|
|
completeConfigs: Set<string>;
|
|
};
|
|
const canonicalTargetInventories = new Map<
|
|
(typeof canonicalNodeTestOwners)[number] | undefined,
|
|
CanonicalTargetInventory
|
|
>();
|
|
|
|
function resolveCanonicalTargetInventory(requestedConfig?: string) {
|
|
const complete = canonicalTargetInventories.get(undefined);
|
|
if (complete) {
|
|
return complete;
|
|
}
|
|
const matchingOwners = requestedConfig
|
|
? canonicalNodeTestOwners.filter((owner) => owner.projects.includes(requestedConfig))
|
|
: [];
|
|
const owner = matchingOwners.length === 1 ? matchingOwners[0] : undefined;
|
|
const cached = canonicalTargetInventories.get(owner);
|
|
if (cached) {
|
|
return cached;
|
|
}
|
|
const configsByFile = new Map<string, Set<string>>();
|
|
const configs = new Set<string>();
|
|
const incompleteConfigs = new Set<string>();
|
|
for (const shard of createNodeTestShardsForOwners(owner ? [owner] : canonicalNodeTestOwners, {
|
|
includeReleaseOnlyPluginShards: true,
|
|
includeReleaseOnlyToolingShards: true,
|
|
includeProofTests: false,
|
|
})) {
|
|
const envelope = shard.includePatterns ?? listWholeConfigFiles(shard.shardName);
|
|
const included = envelope ? new Set(envelope) : undefined;
|
|
for (const config of shard.configs) {
|
|
configs.add(config);
|
|
const files =
|
|
listNodeTestConfigFiles(config) ?? (shard.configs.length === 1 ? envelope : undefined);
|
|
if (!files) {
|
|
incompleteConfigs.add(config);
|
|
continue;
|
|
}
|
|
for (const file of files) {
|
|
if (included && !included.has(file)) {
|
|
continue;
|
|
}
|
|
const owners = configsByFile.get(file) ?? new Set<string>();
|
|
owners.add(config);
|
|
configsByFile.set(file, owners);
|
|
}
|
|
}
|
|
}
|
|
const inventory = {
|
|
configsByFile,
|
|
configs,
|
|
completeConfigs: new Set([...configs].filter((config) => !incompleteConfigs.has(config))),
|
|
};
|
|
canonicalTargetInventories.set(owner, inventory);
|
|
return inventory;
|
|
}
|
|
|
|
export function isCanonicalNodeTestConfig(config: string): boolean {
|
|
return resolveCanonicalTargetInventory(config).configs.has(config);
|
|
}
|
|
|
|
/** Null excludes a file from a complete inventory; undefined leaves ownership unresolved. */
|
|
export function resolveCanonicalNodeTestConfig(
|
|
target: string,
|
|
config: string,
|
|
): string | null | undefined {
|
|
let inventory = resolveCanonicalTargetInventory(config);
|
|
let owners = inventory.configsByFile.get(target);
|
|
if (
|
|
owners?.has(config) ||
|
|
(inventory.configs.has(config) && !inventory.completeConfigs.has(config))
|
|
) {
|
|
return config;
|
|
}
|
|
// A direct owner should not discover unrelated suites. Aggregate or moved
|
|
// targets still need the complete inventory to prove a unique replacement.
|
|
inventory = resolveCanonicalTargetInventory();
|
|
owners = inventory.configsByFile.get(target);
|
|
if (
|
|
owners?.has(config) ||
|
|
(inventory.configs.has(config) && !inventory.completeConfigs.has(config))
|
|
) {
|
|
return config;
|
|
}
|
|
if (owners?.size === 1) {
|
|
return owners.values().next().value;
|
|
}
|
|
if (
|
|
EXCLUDED_PROJECT_CONFIGS.has(config) ||
|
|
canonicalNodeTestOwners.some(
|
|
(owner) => EXCLUDED_FULL_SUITE_SHARDS.has(owner.config) && owner.projects.includes(config),
|
|
)
|
|
) {
|
|
return null;
|
|
}
|
|
if (!owners && inventory.configs.has(config)) {
|
|
return inventory.completeConfigs.has(config) ? null : config;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function resolveSplitNodeShards(name: string): NodeTestSplitShard[] | undefined {
|
|
const entry = SPLIT_NODE_SHARDS.get(name);
|
|
if (typeof entry !== "function") {
|
|
return entry;
|
|
}
|
|
const shards = entry();
|
|
SPLIT_NODE_SHARDS.set(name, shards);
|
|
return shards;
|
|
}
|
|
|
|
export function nodeTestConfigRequiresCanonicalMetadata(config: string): boolean {
|
|
// Aggregate configs can be exclusive without owning canonical shard metadata.
|
|
// Check immutable project descriptors before discovering unrelated test files.
|
|
if (!canonicalNodeTestOwners.some((owner) => owner.projects.includes(config))) {
|
|
return false;
|
|
}
|
|
const toolingOwner = canonicalNodeTestOwners.some(
|
|
(owner) => owner.name === "core-tooling" && owner.projects.includes(config),
|
|
);
|
|
if (config === TOOLING_CONFIG) {
|
|
return toolingOwner;
|
|
}
|
|
// Gateway environment overrides belong to exclusive configs. The remaining
|
|
// environment owners share these descriptors with shard construction.
|
|
if (
|
|
!toolingOwner &&
|
|
!isExclusiveCiTestConfig(config) &&
|
|
!(
|
|
GATEWAY_CORE_NODE_TEST_CONFIGS.includes(config) &&
|
|
GATEWAY_CORE_NODE_TEST_CONFIGS.some(isExclusiveCiTestConfig)
|
|
) &&
|
|
![TUI_PTY_NODE_TEST_SHARD, AGENTS_EMBEDDED_NODE_TEST_SHARD].some((shard) =>
|
|
shard.configs.includes(config),
|
|
)
|
|
) {
|
|
return false;
|
|
}
|
|
canonicalMetadataConfigs ??= new Set(
|
|
createNodeTestShardsForOwners(canonicalNodeTestOwners, {
|
|
includeReleaseOnlyPluginShards: false,
|
|
})
|
|
.filter(
|
|
(shard) =>
|
|
shard.env ||
|
|
shard.shardName.startsWith("core-tooling") ||
|
|
shard.configs.some(isExclusiveCiTestConfig),
|
|
)
|
|
.flatMap((shard) => shard.configs),
|
|
);
|
|
return canonicalMetadataConfigs.has(config);
|
|
}
|
|
|
|
function formatNodeTestShardCheckName(shardName: string): string {
|
|
const normalizedShardName = shardName.startsWith("core-unit-")
|
|
? `core-${shardName.slice("core-unit-".length)}`
|
|
: shardName;
|
|
return `checks-node-${normalizedShardName}`;
|
|
}
|
|
|
|
/** Create node test shard descriptors for CI with explicit release-tier selection. */
|
|
export function createNodeTestShards(options: NodeTestPlanOptions = {}): NodeTestShard[] {
|
|
return createNodeTestShardsForOwners(fullSuiteVitestShards, options);
|
|
}
|
|
|
|
function createNodeTestShardsForOwners(
|
|
owners: readonly (typeof fullSuiteVitestShards)[number][],
|
|
options: NodeTestPlanOptions,
|
|
toolingOnly = false,
|
|
preparedChangedTestPlans?: ReturnType<typeof buildVitestRunPlans>,
|
|
): NodeTestShard[] {
|
|
const includeReleaseOnlyPluginShards = options.includeReleaseOnlyPluginShards ?? true;
|
|
const includeProofTests =
|
|
options.includeProofTests ??
|
|
(options.compactMode ?? (options.compact ? "pull-request" : undefined)) !== "pull-request";
|
|
const includeTooling = includesReleaseOnlyTooling(options);
|
|
const keepPrExemptRuntimeFile = (file: string) =>
|
|
options.includePrExemptRuntimeTests !== undefined &&
|
|
isPrExemptRuntimeTestFile(file) &&
|
|
isRuntimeTestFileIncluded(file, options);
|
|
const changedTestPlans = includeReleaseOnlyPluginShards
|
|
? []
|
|
: (preparedChangedTestPlans ??
|
|
(options.changedPaths ?? [])
|
|
.filter(
|
|
(file) =>
|
|
isTestFileTarget(file) &&
|
|
!file.endsWith(".live.test.ts") &&
|
|
statSync(file, { throwIfNoEntry: false })?.isFile(),
|
|
)
|
|
.flatMap((file) => buildVitestRunPlans([file])));
|
|
|
|
return owners.flatMap((shard) => {
|
|
if (
|
|
EXCLUDED_FULL_SUITE_SHARDS.has(shard.config) ||
|
|
(toolingOnly &&
|
|
shard.name !== "core-tooling" &&
|
|
shard.name !== "core-runtime" &&
|
|
!DIST_DEPENDENT_NODE_SHARD_NAMES.has(shard.name))
|
|
) {
|
|
return [];
|
|
}
|
|
|
|
const configs = shard.projects.filter((config) => !EXCLUDED_PROJECT_CONFIGS.has(config));
|
|
if (configs.length === 0) {
|
|
return [];
|
|
}
|
|
|
|
const splitShards =
|
|
toolingOnly && shard.name === "core-runtime"
|
|
? [TUI_PTY_NODE_TEST_SHARD]
|
|
: resolveSplitNodeShards(shard.name);
|
|
if (splitShards) {
|
|
return splitShards.flatMap((splitShard) => {
|
|
const splitConfigs = splitShard.includeExternalConfigs
|
|
? splitShard.configs
|
|
: splitShard.configs.filter((config) => configs.includes(config));
|
|
if (splitConfigs.length === 0) {
|
|
return [];
|
|
}
|
|
|
|
let includePatterns =
|
|
splitShard.includePatterns ??
|
|
(!includeTooling ? listToolingGroupFiles(splitConfigs) : undefined);
|
|
let timingKey: string | undefined;
|
|
if (!includeProofTests) {
|
|
const files = includePatterns ?? listWholeConfigSplitFiles(splitShard.shardName);
|
|
if (files?.some(isCiProofTestFile)) {
|
|
includePatterns = files.filter((file) => !isCiProofTestFile(file));
|
|
if (includePatterns.length === 0) {
|
|
return [];
|
|
}
|
|
}
|
|
}
|
|
if (
|
|
options.includeReleaseOnlyRuntimeTests === false ||
|
|
options.includePrExemptRuntimeTests === false
|
|
) {
|
|
const files =
|
|
includePatterns ??
|
|
listWholeConfigFiles(splitShard.shardName) ??
|
|
listToolingGroupFiles(splitConfigs);
|
|
if (
|
|
files?.some(
|
|
(file) => isReleaseOnlyRuntimeTestFile(file) || isPrExemptRuntimeTestFile(file),
|
|
)
|
|
) {
|
|
const selectedFiles = files.filter((file) => isRuntimeTestFileIncluded(file, options));
|
|
if (selectedFiles.length === 0) {
|
|
return [];
|
|
}
|
|
if (selectedFiles.length < files.length) {
|
|
includePatterns = selectedFiles;
|
|
// Refit folds complete split generations into their timing parent.
|
|
// Reduced automatic coverage must never reprice the full release owner.
|
|
timingKey = `changed-${splitShard.shardName}`;
|
|
}
|
|
}
|
|
}
|
|
if (includePatterns && !includeTooling) {
|
|
const selectedFiles = includePatterns.filter(
|
|
(file) =>
|
|
(!RELEASE_ONLY_TOOLING_SHARDS.has(shard.name) &&
|
|
!isReleaseOnlyToolingTestFile(file)) ||
|
|
keepPrExemptRuntimeFile(file),
|
|
);
|
|
if (selectedFiles.length < includePatterns.length) {
|
|
timingKey = `changed-${splitShard.shardName}`;
|
|
}
|
|
includePatterns = selectedFiles;
|
|
if (includePatterns.length === 0) {
|
|
return [];
|
|
}
|
|
}
|
|
if (
|
|
RELEASE_ONLY_PLUGIN_SHARDS.has(splitShard.shardName) &&
|
|
!includeReleaseOnlyPluginShards
|
|
) {
|
|
// Keep direct edits and the selected integration tier without enabling
|
|
// the unrelated plugin sweep or changing canonical Vitest ownership.
|
|
const files = includePatterns ?? listWholeConfigFiles(splitShard.shardName) ?? [];
|
|
includePatterns = [
|
|
...new Set([
|
|
...files.filter(keepPrExemptRuntimeFile),
|
|
...changedTestPlans
|
|
.filter((plan) => splitConfigs.includes(plan.config))
|
|
.flatMap((plan) => plan.includePatterns ?? []),
|
|
]),
|
|
]
|
|
.filter((file) => includeProofTests || !isCiProofTestFile(file))
|
|
.toSorted();
|
|
if (includePatterns.length === 0) {
|
|
return [];
|
|
}
|
|
if (options.includePrExemptRuntimeTests !== undefined) {
|
|
timingKey = `changed-${splitShard.shardName}`;
|
|
}
|
|
}
|
|
|
|
const pretestBuildMode = includePatterns
|
|
? resolveTestFilesBuildMode(includePatterns)
|
|
: resolveVitestPretestBuildMode([{ configs: splitConfigs }]);
|
|
|
|
return [
|
|
{
|
|
checkName: formatNodeTestShardCheckName(splitShard.shardName),
|
|
shardName: splitShard.shardName,
|
|
...(timingKey ? { timing_key: timingKey } : {}),
|
|
configs: splitConfigs,
|
|
...(splitShard.env ? { env: splitShard.env } : {}),
|
|
...(includePatterns ? { includePatterns } : {}),
|
|
...(pretestBuildMode ? { pretestBuildMode } : {}),
|
|
runner: splitShard.runner ?? DEFAULT_NODE_TEST_RUNNER,
|
|
requiresDist: splitShard.requiresDist,
|
|
},
|
|
];
|
|
});
|
|
}
|
|
|
|
const pretestBuildMode = resolveVitestPretestBuildMode([{ configs }]);
|
|
return [
|
|
{
|
|
checkName: formatNodeTestShardCheckName(shard.name),
|
|
shardName: shard.name,
|
|
configs,
|
|
...(pretestBuildMode ? { pretestBuildMode } : {}),
|
|
runner: DEFAULT_NODE_TEST_RUNNER,
|
|
requiresDist: DIST_DEPENDENT_NODE_SHARD_NAMES.has(shard.name),
|
|
},
|
|
];
|
|
});
|
|
}
|
|
|
|
/** Select planner envelopes that produce the protected Vitest transform-cache seed. */
|
|
export function createVitestCacheWarmGroups(profile: "full" | "hybrid-hosted" = "full"): Array<{
|
|
configs: string[];
|
|
env?: Record<string, string>;
|
|
includePatterns?: string[];
|
|
shard_name: string;
|
|
}> {
|
|
// Preserve the package root and aliases used by checks-ui in either backend.
|
|
const uiGroup = {
|
|
configs: ["ui/vitest.config.ts"],
|
|
env: { OPENCLAW_VITEST_MAX_WORKERS: "1" },
|
|
includePatterns: [
|
|
"ui/src/components/app-sidebar.catalog.test.ts",
|
|
"ui/src/components/app-sidebar.interactions.test.ts",
|
|
"ui/src/components/app-sidebar.people.test.ts",
|
|
"ui/src/components/app-sidebar.sessions.test.ts",
|
|
"ui/src/pages/chat/chat-view.test.ts",
|
|
"ui/src/pages/chat/chat-pane-lifecycle.test.ts",
|
|
"ui/src/pages/usage/metrics.node.test.ts",
|
|
],
|
|
shard_name: "cache-warm:ui-package",
|
|
};
|
|
if (profile === "hybrid-hosted") {
|
|
// Seed the hosted CI-routing and contract closures without collecting all
|
|
// tooling tests or building the runtime. Ordinary CI still runs every test.
|
|
return [
|
|
{
|
|
configs: ["test/vitest/vitest.tooling.config.ts"],
|
|
includePatterns: [
|
|
"test/scripts/ci-workflow-guards.test.ts",
|
|
"test/scripts/ci-workflow-planning.test.ts",
|
|
"test/scripts/ci-workflow-evidence.test.ts",
|
|
"test/scripts/ci-run-node-test-shard.test.ts",
|
|
],
|
|
shard_name: "cache-warm:hosted-tooling",
|
|
},
|
|
...(
|
|
[
|
|
["plugin", "src/plugins/contracts/registry.contract.test.ts"],
|
|
["channel-surface", "src/channels/plugins/contracts/channel-catalog.contract.test.ts"],
|
|
[
|
|
"channel-config",
|
|
"src/channels/plugins/contracts/gateway-auth-artifact.contract.test.ts",
|
|
],
|
|
[
|
|
"channel-registry",
|
|
"src/channels/plugins/contracts/plugins-core.loader.contract.test.ts",
|
|
],
|
|
[
|
|
"channel-session",
|
|
"src/channels/plugins/contracts/session-key-artifact.contract.test.ts",
|
|
],
|
|
] as const
|
|
).map(([name, file]) => ({
|
|
configs: [`test/vitest/vitest.contracts-${name}.config.ts`],
|
|
includePatterns: [file],
|
|
shard_name: `cache-warm:hosted-contracts-${name}`,
|
|
})),
|
|
uiGroup,
|
|
];
|
|
}
|
|
const additionalShardNames = new Set([
|
|
"agentic-agents-embedded",
|
|
"agentic-gateway-methods",
|
|
"auto-reply-reply-commands-3",
|
|
]);
|
|
const allShards = createNodeTestShards();
|
|
const coreShards = allShards.filter((candidate) =>
|
|
candidate.shardName.startsWith("core-unit-fast"),
|
|
);
|
|
if (coreShards.length === 0) {
|
|
throw new Error("core-unit-fast cache seed shards are missing");
|
|
}
|
|
const additionalShards = allShards.filter((candidate) =>
|
|
additionalShardNames.has(candidate.shardName),
|
|
);
|
|
const foundAdditionalShardNames = new Set(additionalShards.map((shard) => shard.shardName));
|
|
const missingShardNames = [...additionalShardNames].filter(
|
|
(name) => !foundAdditionalShardNames.has(name),
|
|
);
|
|
if (missingShardNames.length > 0) {
|
|
throw new Error(`cache seed shards are missing: ${missingShardNames.join(", ")}`);
|
|
}
|
|
return [
|
|
...[...coreShards, ...additionalShards].flatMap((shard) =>
|
|
shard.configs.map((config) => ({
|
|
configs: [config],
|
|
...(shard.env ? { env: shard.env } : {}),
|
|
...(shard.includePatterns ? { includePatterns: shard.includePatterns } : {}),
|
|
shard_name: `cache-warm:${shard.shardName}:${config}`,
|
|
})),
|
|
),
|
|
uiGroup,
|
|
];
|
|
}
|
|
|
|
function resolveCiNodeTestRunner(shard: NodeTestShard, compactProfile?: string): string {
|
|
const ownerRunner = COMPACT_NODE_TEST_OWNER_RUNNERS.get(compactProfile ?? "")?.get(
|
|
shard.shardName,
|
|
);
|
|
if (ownerRunner) {
|
|
return ownerRunner;
|
|
}
|
|
if (
|
|
(compactProfile === "blacksmith" &&
|
|
shard.includePatterns?.includes("src/cli/update-dry-run-state.process.test.ts")) ||
|
|
(compactProfile === "hybrid" &&
|
|
(shard.includePatterns?.includes("src/cli/gateway-backed-exit.process.test.ts") ||
|
|
shard.includePatterns?.includes("src/cli/gateway-backed-exit-health.process.test.ts")))
|
|
) {
|
|
return DEFAULT_NODE_TEST_RUNNER;
|
|
}
|
|
if (shard.runner !== DEFAULT_NODE_TEST_RUNNER) {
|
|
return shard.runner;
|
|
}
|
|
// The full-build compiler fixture must pass the real 4352MB heap guard even
|
|
// after earlier tooling tests have retained their module graphs.
|
|
return KEEP_LARGE_NODE_TEST_RUNNER.has(shard.shardName) ||
|
|
shard.includePatterns?.includes(TOOLING_UNIFIED_DECLARATIONS_TEST_FILE)
|
|
? DEFAULT_NODE_TEST_RUNNER
|
|
: BUNDLED_NODE_TEST_RUNNER;
|
|
}
|
|
|
|
function resolveCiNodeTestRunnerClass(runner: string) {
|
|
const name =
|
|
runner === EXTRA_LARGE_NODE_TEST_RUNNER
|
|
? "large32"
|
|
: runner.includes("-8vcpu-")
|
|
? "large"
|
|
: "small";
|
|
// Each runner bucket starts numbering at one; distinct classes need distinct
|
|
// names while larger requests retain the conservative large-job budget.
|
|
return {
|
|
name,
|
|
secondsCap:
|
|
name === "small" ? COMPACT_SMALL_NODE_TEST_JOB_SECONDS : COMPACT_LARGE_NODE_TEST_JOB_SECONDS,
|
|
};
|
|
}
|
|
|
|
function bundleNameForConfigs(configs: string[]): string {
|
|
const config = configs[0] ?? "node";
|
|
return config
|
|
.replace(/^test\/vitest\/vitest\./u, "")
|
|
.replace(/\.config\.ts$/u, "")
|
|
.replace(/[^a-z0-9-]+/giu, "-");
|
|
}
|
|
|
|
function compareFullNodeTestAdmissionOrder(a: NodeTestShard, b: NodeTestShard): number {
|
|
const fallbackPriority = FULL_NODE_TEST_ADMISSION_PRIORITY.size;
|
|
return (
|
|
(FULL_NODE_TEST_ADMISSION_PRIORITY.get(a.shardName) ?? fallbackPriority) -
|
|
(FULL_NODE_TEST_ADMISSION_PRIORITY.get(b.shardName) ?? fallbackPriority) ||
|
|
a.checkName.localeCompare(b.checkName)
|
|
);
|
|
}
|
|
|
|
// Deterministic cost-aware batching (greedy LPT): heaviest values first, each
|
|
// into the currently lightest batch. Round-robin by discovery order can pack
|
|
// one whale next to another and leave sibling batches much lighter.
|
|
function createStripedBatches<T>(
|
|
values: T[],
|
|
batchCount: number,
|
|
weightForValue: (value: T) => number,
|
|
weightForBatch?: (values: T[]) => number,
|
|
): T[][] {
|
|
if (batchCount < 1) {
|
|
throw new Error("striped batch count must be positive");
|
|
}
|
|
const entries = values.map((value, index) => ({
|
|
index,
|
|
value,
|
|
weight: weightForValue(value),
|
|
}));
|
|
entries.sort((a, b) => b.weight - a.weight || a.index - b.index);
|
|
const batches: Array<{
|
|
totalWeight: number;
|
|
entries: Array<{ index: number; value: T; weight: number }>;
|
|
}> = Array.from({ length: batchCount }, () => ({ totalWeight: 0, entries: [] }));
|
|
const firstBatch = batches[0];
|
|
if (!firstBatch) {
|
|
throw new Error("striped batch allocation failed");
|
|
}
|
|
for (const entry of entries) {
|
|
const nextWeight = (batch: (typeof batches)[number]) =>
|
|
weightForBatch
|
|
? weightForBatch([...batch.entries.map(({ value }) => value), entry.value])
|
|
: batch.totalWeight + entry.weight;
|
|
let target = firstBatch;
|
|
for (const batch of batches) {
|
|
if (nextWeight(batch) < nextWeight(target)) {
|
|
target = batch;
|
|
}
|
|
}
|
|
target.totalWeight = nextWeight(target);
|
|
target.entries.push(entry);
|
|
}
|
|
// Keep discovery order inside each batch so include lists stay stable.
|
|
return batches.map((batch) =>
|
|
batch.entries.toSorted((a, b) => a.index - b.index).map((entry) => entry.value),
|
|
);
|
|
}
|
|
|
|
function listCompactToolingTestFiles(): string[] {
|
|
const unitFastFiles = getUnitFastTestFilesForIncludePatterns([
|
|
"test/**/*.test.ts",
|
|
"src/scripts/**/*.test.ts",
|
|
]);
|
|
const excludedFiles = new Set([
|
|
...boundaryTestFiles,
|
|
...gatewayPluginTestFiles,
|
|
...gatewayDatabaseWorkerTestFiles,
|
|
...unitFastFiles,
|
|
TOOLING_DOCKER_TEST_FILE,
|
|
...toolingIsolatedTestFiles,
|
|
...databaseWorkerCoreTestFiles,
|
|
]);
|
|
return [...listTrackedTestFiles("test"), ...listTrackedTestFiles("src/scripts")].filter(
|
|
(file) =>
|
|
!file.startsWith("test/fixtures/") &&
|
|
!file.endsWith(".e2e.test.ts") &&
|
|
!file.endsWith(".live.test.ts") &&
|
|
!excludedFiles.has(file),
|
|
);
|
|
}
|
|
|
|
function applyNativeSoloTimings(
|
|
jobs: CompactNodeTestShard[],
|
|
runnerBackend: string | undefined,
|
|
): CompactNodeTestShard[] {
|
|
if (runnerBackend !== undefined && !["blacksmith", "hybrid", "runson"].includes(runnerBackend)) {
|
|
return jobs;
|
|
}
|
|
const timings = readCompactGroupTimings("blacksmith");
|
|
return jobs.map((job) => {
|
|
if (
|
|
job.runner !== NATIVE_SOLO_TIMING_PROFILE.runner ||
|
|
job.planConcurrency !== 1 ||
|
|
job.groups.length !== 1 ||
|
|
job.requiresDist ||
|
|
job.pretestBuildMode !== undefined ||
|
|
Object.entries(job.env ?? {}).some(
|
|
([key, value]) =>
|
|
key !== "OPENCLAW_VITEST_MAX_WORKERS" ||
|
|
value !== String(NATIVE_SOLO_TIMING_PROFILE.maxWorkers),
|
|
)
|
|
) {
|
|
return job;
|
|
}
|
|
const key = createNativeSoloTimingKey(job.groups[0]!);
|
|
const seconds = key === undefined ? undefined : timings[key];
|
|
// Solo measurements price only the final allocation. Feeding them into
|
|
// packing would let an unqualified shared row spend the solo discount.
|
|
return seconds === undefined
|
|
? job
|
|
: { ...job, predictedSeconds: seconds, predictedTestSeconds: seconds };
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Collapse split include-pattern shards into bounded jobs for normal CI.
|
|
* The base plan remains unchanged for release and coverage consumers.
|
|
*/
|
|
export function createNodeTestShardBundles(
|
|
options: NodeTestPlanOptions & { compactMode: CompactNodeTestPlanMode },
|
|
): CompactNodeTestShard[];
|
|
/** @deprecated Use compactMode so push and pull-request coverage stay explicit. */
|
|
export function createNodeTestShardBundles(
|
|
options: NodeTestPlanOptions & { compact: true },
|
|
): CompactNodeTestShard[];
|
|
export function createNodeTestShardBundles(options?: NodeTestPlanOptions): NodeTestShard[];
|
|
export function createNodeTestShardBundles(
|
|
options: NodeTestPlanOptions = {},
|
|
): NodeTestShard[] | CompactNodeTestShard[] {
|
|
const compactMode =
|
|
options.compactMode ?? (options.compact === true ? "pull-request" : undefined);
|
|
if (compactMode !== undefined) {
|
|
return applyNativeSoloTimings(
|
|
createCompactNodeTestShardBundles(
|
|
// Keep complete owners for cost admission; compact projection below gives
|
|
// a reduced tooling selection its own timing identity.
|
|
createNodeTestShards({ ...options, includeReleaseOnlyToolingShards: true }),
|
|
{ ...options, compactMode },
|
|
compactMode,
|
|
),
|
|
options.runnerBackend,
|
|
);
|
|
}
|
|
|
|
const shards = createNodeTestShards(options);
|
|
const unbundled: NodeTestShard[] = [];
|
|
const groups = new Map<
|
|
string,
|
|
{
|
|
configs: string[];
|
|
pretestBuildMode?: NodeTestPretestBuildMode;
|
|
requiresDist: boolean;
|
|
runner: string;
|
|
shards: NodeTestShard[];
|
|
}
|
|
>();
|
|
|
|
for (const shard of shards) {
|
|
const runner = resolveCiNodeTestRunner(shard);
|
|
if (shard.shardName === "agentic-gateway-server-isolated") {
|
|
// Full release validation retains two workers. Its whole native cohort
|
|
// exhausted the job deadline, so reuse the existing file envelope here.
|
|
const files = shard.includePatterns ?? [
|
|
...gatewayServerIsolatedTestFiles,
|
|
...gatewayDatabaseWorkerTestFiles,
|
|
];
|
|
const stripes = createStripedBatches(
|
|
files,
|
|
Math.ceil(files.length / MAX_BUNDLED_NODE_TEST_PATTERNS),
|
|
stripeFileWeight,
|
|
).flatMap((stripe) =>
|
|
Array.from(
|
|
{ length: Math.ceil(stripe.length / MAX_BUNDLED_NODE_TEST_PATTERNS) },
|
|
(_, index) =>
|
|
stripe.slice(
|
|
index * MAX_BUNDLED_NODE_TEST_PATTERNS,
|
|
(index + 1) * MAX_BUNDLED_NODE_TEST_PATTERNS,
|
|
),
|
|
),
|
|
);
|
|
const timingKeys = shard.timing_key
|
|
? createCompactSplitTimingGeneration({
|
|
configs: shard.configs,
|
|
env: shard.env,
|
|
parentShardName: shard.timing_key,
|
|
stripes,
|
|
}).timingKeys
|
|
: undefined;
|
|
for (const [index, includePatterns] of stripes.entries()) {
|
|
const shardName = `${shard.shardName}-${index + 1}`;
|
|
unbundled.push({
|
|
...shard,
|
|
checkName: formatNodeTestShardCheckName(shardName),
|
|
shardName,
|
|
...(timingKeys ? { timing_key: timingKeys[index]! } : {}),
|
|
includePatterns,
|
|
runner,
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
const [config] = shard.configs;
|
|
if (
|
|
shard.requiresDist ||
|
|
shard.configs.length !== 1 ||
|
|
config === undefined ||
|
|
!BUNDLEABLE_NODE_TEST_CONFIGS.has(config) ||
|
|
!Array.isArray(shard.includePatterns) ||
|
|
shard.includePatterns.length === 0
|
|
) {
|
|
unbundled.push({ ...shard, runner });
|
|
continue;
|
|
}
|
|
|
|
const key = JSON.stringify([shard.configs, shard.pretestBuildMode, shard.requiresDist, runner]);
|
|
const group = groups.get(key) ?? {
|
|
configs: shard.configs,
|
|
...(shard.pretestBuildMode ? { pretestBuildMode: shard.pretestBuildMode } : {}),
|
|
requiresDist: shard.requiresDist,
|
|
runner,
|
|
shards: [],
|
|
};
|
|
group.shards.push(shard);
|
|
groups.set(key, group);
|
|
}
|
|
|
|
const bundled: NodeTestShard[] = [];
|
|
for (const group of groups.values()) {
|
|
const bins: Array<{ includePatterns: string[] }> = [];
|
|
const sortedShards = group.shards.toSorted(
|
|
(a, b) =>
|
|
(b.includePatterns?.length ?? 0) - (a.includePatterns?.length ?? 0) ||
|
|
a.shardName.localeCompare(b.shardName),
|
|
);
|
|
for (const shard of sortedShards) {
|
|
const patterns = shard.includePatterns ?? [];
|
|
for (let offset = 0; offset < patterns.length; offset += MAX_BUNDLED_NODE_TEST_PATTERNS) {
|
|
const chunk = patterns.slice(offset, offset + MAX_BUNDLED_NODE_TEST_PATTERNS);
|
|
const bin = bins.find(
|
|
(candidate) =>
|
|
candidate.includePatterns.length + chunk.length <= MAX_BUNDLED_NODE_TEST_PATTERNS,
|
|
);
|
|
if (bin) {
|
|
bin.includePatterns.push(...chunk);
|
|
} else {
|
|
bins.push({ includePatterns: [...chunk] });
|
|
}
|
|
}
|
|
}
|
|
|
|
const { name: runnerClass } = resolveCiNodeTestRunnerClass(group.runner);
|
|
const buildModeSuffix = group.pretestBuildMode ? `-${group.pretestBuildMode}` : "";
|
|
const bundleName = `${bundleNameForConfigs(group.configs)}-${runnerClass}${buildModeSuffix}`;
|
|
for (const [index, bin] of bins.entries()) {
|
|
const shardName = `bundle-${bundleName}-${index + 1}`;
|
|
bundled.push({
|
|
checkName: formatNodeTestShardCheckName(shardName),
|
|
shardName,
|
|
...(group.shards.some((shard) => shard.timing_key)
|
|
? {
|
|
timing_key: createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
parentShardName: `changed-${shardName}`,
|
|
stripes: [bin.includePatterns],
|
|
}).timingKeys[0]!,
|
|
}
|
|
: {}),
|
|
configs: group.configs,
|
|
includePatterns: bin.includePatterns.toSorted((a, b) => a.localeCompare(b)),
|
|
...(group.pretestBuildMode ? { pretestBuildMode: group.pretestBuildMode } : {}),
|
|
runner: group.runner,
|
|
requiresDist: group.requiresDist,
|
|
});
|
|
}
|
|
}
|
|
|
|
const full = [...unbundled, ...bundled];
|
|
return (
|
|
options.runnerBackend === "github" ? full.flatMap(splitHostedReleaseShard) : full
|
|
).toSorted(compareFullNodeTestAdmissionOrder);
|
|
}
|
|
|
|
// Full release jobs include setup and can execute both runtimes. Keep their
|
|
// measured walls separate from compact test-group spans and reserve eight minutes
|
|
// of the 20-minute objective for changes in setup and cold-run overhead.
|
|
function splitHostedReleaseShard(shard: NodeTestShard): NodeTestShard[] {
|
|
const budget = 720;
|
|
const parentShardName = `release-full-${shard.shardName}`;
|
|
const timings = readCompactGroupTimings("github");
|
|
const files = canSplitWholeConfigGroup(shard.shardName)
|
|
? (shard.includePatterns ?? listWholeConfigSplitFiles(shard.shardName))
|
|
: undefined;
|
|
if (!isRuntimePlacementIncludePatterns(files)) {
|
|
const seconds = timings[parentShardName];
|
|
if (seconds !== undefined && seconds > budget) {
|
|
throw new Error(
|
|
`Release shard ${shard.shardName} cannot fit the hosted budget; split its test owner before release`,
|
|
);
|
|
}
|
|
return [
|
|
{
|
|
...shard,
|
|
timing_key: parentShardName,
|
|
...(seconds === undefined ? {} : { predictedSeconds: seconds }),
|
|
},
|
|
];
|
|
}
|
|
const generation = (stripes: string[][]) =>
|
|
createCompactSplitTimingGeneration({
|
|
configs: shard.configs,
|
|
env: shard.env,
|
|
parentShardName,
|
|
stripes,
|
|
});
|
|
const original = generation([files]);
|
|
const singletonCosts = new Map<string, number>();
|
|
for (const [key, cost] of Object.entries(timings)) {
|
|
const singleton = key.match(/#include-1-[a-f0-9]{12}$/u)?.[0];
|
|
if (singleton && key.startsWith(`${original.selectorKey}#generation-`)) {
|
|
singletonCosts.set(singleton, Math.max(singletonCosts.get(singleton) ?? 0, cost));
|
|
}
|
|
}
|
|
if ([...singletonCosts.values()].some((cost) => cost > budget)) {
|
|
throw new Error(
|
|
`Release shard ${shard.shardName} contains an indivisible test above the hosted budget; split that test before release`,
|
|
);
|
|
}
|
|
const seconds = Math.max(
|
|
timings[parentShardName] ?? 0,
|
|
timings[original.timingKeys[0]!] ?? 0,
|
|
readCompleteSplitGenerationSeconds(timings, original.selectorKey) ?? 0,
|
|
);
|
|
if (seconds <= budget) {
|
|
return [
|
|
{
|
|
...shard,
|
|
timing_key: original.timingKeys[0]!,
|
|
...(seconds === 0 ? {} : { predictedSeconds: seconds }),
|
|
},
|
|
];
|
|
}
|
|
const weight = (entries: readonly string[]) =>
|
|
entries.reduce((sum, file) => sum + stripeFileWeight(file), 0);
|
|
const totalWeight = weight(files);
|
|
let count = Math.min(
|
|
files.length,
|
|
Math.max(
|
|
2,
|
|
Math.ceil(seconds / budget),
|
|
Math.ceil(files.length / MAX_BUNDLED_NODE_TEST_PATTERNS),
|
|
),
|
|
);
|
|
for (;;) {
|
|
const stripes = createStripedBatches(files, count, stripeFileWeight);
|
|
const keys = generation(stripes).timingKeys;
|
|
const predicted = stripes.map((stripe, index) =>
|
|
Math.ceil(timings[keys[index]!] ?? (seconds * weight(stripe)) / totalWeight),
|
|
);
|
|
if (
|
|
predicted.every((cost) => cost <= budget) &&
|
|
stripes.every((stripe) => stripe.length <= MAX_BUNDLED_NODE_TEST_PATTERNS)
|
|
) {
|
|
const result: NodeTestShard[] = [];
|
|
for (const [index, includePatterns] of stripes.entries()) {
|
|
const shardName = `${shard.shardName}-hosted-${index + 1}`;
|
|
result.push({
|
|
...shard,
|
|
shardName,
|
|
checkName: formatNodeTestShardCheckName(shardName),
|
|
timing_key: keys[index]!,
|
|
includePatterns,
|
|
predictedSeconds: predicted[index]!,
|
|
});
|
|
}
|
|
return result;
|
|
}
|
|
if (count === files.length) {
|
|
throw new Error(
|
|
`Release shard ${shard.shardName} contains an indivisible test above the hosted budget; split that test before release`,
|
|
);
|
|
}
|
|
count += 1;
|
|
}
|
|
}
|
|
|
|
type HostedToolingTailDonation = {
|
|
parentShardName: string;
|
|
file: string;
|
|
freedSeconds: number;
|
|
};
|
|
|
|
function selectHostedToolingTailDonation(
|
|
stripes: readonly string[][],
|
|
secondsForFiles: (files: readonly string[]) => number,
|
|
selectedFile?: string,
|
|
selectedToolingFiles?: ReadonlySet<string>,
|
|
): { file: string; donorIndex: number; freedSeconds: number } | undefined {
|
|
const tail = stripes.at(-1);
|
|
if (!tail) {
|
|
return undefined;
|
|
}
|
|
let best: { file: string; donorIndex: number; freedSeconds: number } | undefined;
|
|
for (const [donorIndex, donor] of stripes.slice(0, -1).entries()) {
|
|
const donorSeconds = secondsForFiles(donor);
|
|
if (donor.length < 2 || donorSeconds <= COMPACT_EXCLUSIVE_JOB_SECONDS / 2) {
|
|
continue;
|
|
}
|
|
for (const file of donor) {
|
|
const freedSeconds = donorSeconds - secondsForFiles(donor.filter((entry) => entry !== file));
|
|
if (
|
|
(selectedFile !== undefined && file !== selectedFile) ||
|
|
(selectedToolingFiles !== undefined && !selectedToolingFiles.has(file)) ||
|
|
freedSeconds <= 0 ||
|
|
secondsForFiles([...tail, file]) > COMPACT_EXCLUSIVE_JOB_SECONDS / 2
|
|
) {
|
|
continue;
|
|
}
|
|
if (
|
|
!best ||
|
|
freedSeconds > best.freedSeconds ||
|
|
(freedSeconds === best.freedSeconds && file.localeCompare(best.file) < 0)
|
|
) {
|
|
best = { file, donorIndex, freedSeconds };
|
|
}
|
|
}
|
|
}
|
|
return best;
|
|
}
|
|
|
|
function hasStorageModuleWork(group: NodeTestShardGroup): boolean {
|
|
return group.configs.length === 1 && group.configs[0] === "test/vitest/vitest.infra.config.ts";
|
|
}
|
|
|
|
function storageModuleWorkFloor(
|
|
group: NodeTestShardGroup,
|
|
runnerBackend: string | undefined,
|
|
): number {
|
|
if (runnerBackend === "github" || !hasStorageModuleWork(group)) {
|
|
return 0;
|
|
}
|
|
const measured = (group.includePatterns ?? []).flatMap((file) => {
|
|
const seconds = STORAGE_MODULE_WORK_SECONDS.get(file);
|
|
return seconds === undefined ? [] : [seconds];
|
|
});
|
|
if (measured.length === 0) {
|
|
return 0;
|
|
}
|
|
// Packed children keep two workers. Additional solo capacity cannot divide
|
|
// the longest file; compiler and wrapper work are charged once per child.
|
|
return (
|
|
STORAGE_WRAPPER_SECONDS +
|
|
Math.max(...measured, measured.reduce((sum, seconds) => sum + seconds, 0) / 2)
|
|
);
|
|
}
|
|
|
|
function splitOversizedCompactGroup(
|
|
group: NodeTestShardGroup,
|
|
runnerBackend: string | undefined,
|
|
observedSeconds = 0,
|
|
runtimePartition?: ReturnType<typeof partitionRuntimeTestFiles>,
|
|
splitHostedToolingTails = false,
|
|
hostedToolingTailBudgets?: ReadonlyMap<string, number>,
|
|
hostedToolingTailDonation?: HostedToolingTailDonation,
|
|
onHostedToolingTailDonation?: (donation: HostedToolingTailDonation) => void,
|
|
selectedToolingFiles?: ReadonlySet<string>,
|
|
hostedMain = false,
|
|
): Array<{ group: NodeTestShardGroup; seconds: number }> {
|
|
// Hybrid groups must fit both the first-attempt runner and hosted retries;
|
|
// a faster retry estimate must not leave a slow first attempt unsplit.
|
|
const isCliProcess = group.shard_name === "agentic-cli-process";
|
|
const isTooling = isParallelToolingGroup(group);
|
|
const hostedFileLimit =
|
|
(hostedMain ? COMPACT_HOSTED_MAIN_MAX_FILES.get(group.shard_name) : undefined) ??
|
|
(runnerBackend === "github" && group.shard_name === "core-runtime-infra-storage-state"
|
|
? COMPACT_HOSTED_STORAGE_STATE_MAX_FILES
|
|
: undefined);
|
|
const exceedsHostedFileLimit =
|
|
hostedFileLimit !== undefined && (group.includePatterns?.length ?? 0) > hostedFileLimit;
|
|
const splitHostedUpdate =
|
|
hostedMain &&
|
|
(group.includePatterns?.length ?? 0) > 1 &&
|
|
group.includePatterns!.includes(HOSTED_MAIN_UPDATE_TEST);
|
|
const measuredProfileSeconds = Math.max(
|
|
estimateCompactGroupSeconds(group, runnerBackend),
|
|
observedSeconds,
|
|
storageModuleWorkFloor(group, runnerBackend),
|
|
);
|
|
const measuredHostedSeconds = Math.max(
|
|
estimateCompactGroupSeconds(group, "github"),
|
|
observedSeconds > 0
|
|
? runnerBackend === "github"
|
|
? observedSeconds
|
|
: estimateCompactStripeSeconds(group, "github") ||
|
|
observedSeconds * COMPACT_GITHUB_GROUP_SECONDS_SCALE
|
|
: 0,
|
|
);
|
|
if (!canSplitWholeConfigGroup(group.shard_name)) {
|
|
return [{ group, seconds: measuredProfileSeconds }];
|
|
}
|
|
// These consumers share one prepared runtime; admission retains the retry budget.
|
|
if (group.shard_name === COMMANDS_RUNTIME_GROUP && isParallelCommandsGroup(group)) {
|
|
return [{ group, seconds: measuredProfileSeconds }];
|
|
}
|
|
const parallelCommands = isParallelCommandsGroup(group);
|
|
const parallelGateway = isParallelGatewayServerGroup(group);
|
|
const parentWorkers = isAutoReplyReplyGroup(group)
|
|
? compactEffectiveFileWorkers(group, group.includePatterns?.length ?? 1)
|
|
: effectiveAgentsCoreWorkers(group);
|
|
// Whole parallel walls and sums of split invocations have different worker costs.
|
|
const splitTimingParent = `${compactGroupTimingKey(group)}${parallelGateway ? "-stripes" : ""}`;
|
|
const splitTimingPrefixes = [
|
|
splitTimingParent,
|
|
...(parallelCommands ? [group.shard_name] : []),
|
|
].map((key) => `${key}#selector-`);
|
|
const splitParentSeconds = {
|
|
blacksmith: parallelGateway
|
|
? (readCompactGroupSeconds({ ...group, timing_key: splitTimingParent }, "blacksmith") ?? 0)
|
|
: 0,
|
|
github: parallelGateway
|
|
? (readCompactGroupSeconds({ ...group, timing_key: splitTimingParent }, "github") ?? 0)
|
|
: 0,
|
|
};
|
|
const hasSplitTimingHistory =
|
|
!isTooling &&
|
|
(["blacksmith", "github"] as const).some(
|
|
(profile) =>
|
|
splitParentSeconds[profile] > 0 ||
|
|
Object.keys(readCompactGroupTimings(profile)).some((key) =>
|
|
splitTimingPrefixes.some((prefix) => key.startsWith(prefix)),
|
|
),
|
|
);
|
|
if (
|
|
!isCliProcess &&
|
|
!exceedsHostedFileLimit &&
|
|
!splitHostedUpdate &&
|
|
!runtimePartition &&
|
|
!hasSplitTimingHistory &&
|
|
Math.max(measuredProfileSeconds, measuredHostedSeconds) <= COMPACT_GITHUB_MAX_PREDICTED_SECONDS
|
|
) {
|
|
return [{ group, seconds: measuredProfileSeconds }];
|
|
}
|
|
const includePatterns = group.includePatterns ?? listWholeConfigSplitFiles(group.shard_name);
|
|
const buildModes = new Map(
|
|
includePatterns?.map((file) => [file, resolveTestFilesBuildMode([file])]) ?? [],
|
|
);
|
|
// Hybrid also reserves hosted retry costs; use the same spare-worker packing.
|
|
const packTooling = isTooling && (runnerBackend === "github" || runnerBackend === "hybrid");
|
|
const agentsCoreFiles = isParallelAgentsCoreGroup(group)
|
|
? new Set(agentsCoreWorkFiles(group))
|
|
: undefined;
|
|
const weightForFile = isTooling
|
|
? toolingFileWeight
|
|
: (file: string) =>
|
|
!agentsCoreFiles || agentsCoreFiles.has(file)
|
|
? Math.max(
|
|
stripeFileWeight(file),
|
|
hasStorageModuleWork(group) ? (STORAGE_MODULE_WORK_SECONDS.get(file) ?? 0) : 0,
|
|
)
|
|
: 0;
|
|
const totalWeight =
|
|
includePatterns?.reduce((seconds, file) => seconds + weightForFile(file), 0) ?? 0;
|
|
// A measured whole-config parent can lag newly cataloged files. Its old
|
|
// aggregate must not hide the complete process owner's file costs.
|
|
const profileSeconds = Math.max(measuredProfileSeconds, isCliProcess ? totalWeight : 0);
|
|
const splitBuildMode =
|
|
isCliProcess && !runtimePartition
|
|
? mergeVitestPretestBuildModes([...buildModes.values()])
|
|
: undefined;
|
|
const splitBuildSeconds = splitBuildMode ? VITEST_PRETEST_BUILD_SECONDS[splitBuildMode] : 0;
|
|
const hostedProfileSeconds = Math.max(measuredHostedSeconds, isCliProcess ? totalWeight : 0);
|
|
const splitSeconds = Math.max(
|
|
profileSeconds + splitBuildSeconds,
|
|
hostedProfileSeconds + Math.round(splitBuildSeconds * COMPACT_GITHUB_GROUP_SECONDS_SCALE),
|
|
);
|
|
if (!includePatterns || includePatterns.length < 2 || totalWeight === 0) {
|
|
return [{ group, seconds: profileSeconds }];
|
|
}
|
|
|
|
// The prerequisite is charged once per emitted job. Include it in placement
|
|
// so a balanced test stripe still leaves room for its runtime build.
|
|
let tailDonation: HostedToolingTailDonation | undefined;
|
|
const createStripes = (seconds: number) => {
|
|
tailDonation = undefined;
|
|
const files = runtimePartition?.otherFiles ?? includePatterns;
|
|
const batchWeight = (patterns: readonly string[]) => {
|
|
const mode = mergeVitestPretestBuildModes(patterns.map((file) => buildModes.get(file)));
|
|
const weight = patterns.reduce((sum, file) => sum + weightForFile(file), 0);
|
|
return (
|
|
(isTooling
|
|
? estimateParallelToolingSeconds(group, patterns, packTooling ? "github" : runnerBackend)
|
|
: weight) +
|
|
Math.round(
|
|
(mode ? VITEST_PRETEST_BUILD_SECONDS[mode] : 0) *
|
|
(packTooling ? COMPACT_GITHUB_GROUP_SECONDS_SCALE : 1),
|
|
)
|
|
);
|
|
};
|
|
const weightForValue =
|
|
isCliProcess || isTooling ? (file: string) => batchWeight([file]) : weightForFile;
|
|
let stripes: string[][];
|
|
if (packTooling) {
|
|
// Retain the normal file partition; unrelated families share its tails.
|
|
const discoveryOrder = (a: string, b: string) => files.indexOf(a) - files.indexOf(b);
|
|
const packFiles = (patterns: string[], secondsCap: number) =>
|
|
packNodeTestGroups(
|
|
patterns.toSorted(
|
|
(a, b) => weightForValue(b) - weightForValue(a) || discoveryOrder(a, b),
|
|
),
|
|
(bin, file) => {
|
|
const combinedSeconds = batchWeight([...bin, file]);
|
|
// Fill an indivisible file's spare worker without increasing its
|
|
// cost, but keep files above the whole-job budget alone.
|
|
return (
|
|
combinedSeconds <= secondsCap ||
|
|
(bin.length === 1 &&
|
|
combinedSeconds <= COMPACT_SERIAL_NODE_TEST_JOB_SECONDS &&
|
|
combinedSeconds <= batchWeight(bin))
|
|
);
|
|
},
|
|
).map((batch) => batch.toSorted(discoveryOrder));
|
|
// Full children plus small tails can strand a whole row even when the
|
|
// files fit. On overflow, expose smaller file envelopes for placement.
|
|
const refineTooling =
|
|
splitHostedToolingTails && !hostedToolingTailBudgets && !hostedToolingTailDonation;
|
|
stripes = refineTooling
|
|
? packFiles(files, COMPACT_EXCLUSIVE_JOB_SECONDS / 2)
|
|
: packFiles(files, COMPACT_EXCLUSIVE_JOB_SECONDS);
|
|
const tail = stripes.at(-1);
|
|
if (
|
|
splitHostedToolingTails &&
|
|
!refineTooling &&
|
|
tail &&
|
|
tail.length > 1 &&
|
|
batchWeight(tail) <= COMPACT_EXCLUSIVE_JOB_SECONDS
|
|
) {
|
|
// Keep runtime generations whole; ordinary tails use available capacity
|
|
// only after half-budget packing failed. Indivisible files keep their cost.
|
|
const tailBudget =
|
|
runtimePartition === undefined &&
|
|
group.pretestBuildMode === undefined &&
|
|
!group.requiresDist
|
|
? hostedToolingTailBudgets?.get(group.shard_name)
|
|
: undefined;
|
|
stripes.splice(-1, 1, ...packFiles(tail, tailBudget ?? COMPACT_EXCLUSIVE_JOB_SECONDS / 2));
|
|
}
|
|
const selectedDonation =
|
|
hostedToolingTailDonation?.parentShardName === group.shard_name
|
|
? hostedToolingTailDonation
|
|
: undefined;
|
|
if (
|
|
(onHostedToolingTailDonation || selectedDonation) &&
|
|
runtimePartition === undefined &&
|
|
group.pretestBuildMode === undefined &&
|
|
!group.requiresDist
|
|
) {
|
|
const donation = selectHostedToolingTailDonation(
|
|
stripes,
|
|
batchWeight,
|
|
selectedDonation?.file,
|
|
selectedToolingFiles,
|
|
);
|
|
if (donation) {
|
|
if (selectedDonation) {
|
|
stripes[donation.donorIndex] = stripes[donation.donorIndex]!.filter(
|
|
(file) => file !== donation.file,
|
|
);
|
|
stripes[stripes.length - 1] = [...stripes.at(-1)!, donation.file].toSorted(
|
|
discoveryOrder,
|
|
);
|
|
} else {
|
|
tailDonation = {
|
|
parentShardName: group.shard_name,
|
|
file: donation.file,
|
|
freedSeconds: donation.freedSeconds,
|
|
};
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
// The fixed build stays with its runtime child; only remaining test
|
|
// work benefits from more stripes. Empty include lists run the whole config.
|
|
const remainingSeconds = isTooling
|
|
? estimateParallelToolingSeconds(group, files, "github")
|
|
: runtimePartition
|
|
? (seconds * files.reduce((sum, file) => sum + weightForFile(file), 0)) / totalWeight
|
|
: seconds;
|
|
stripes = createStripedBatches(
|
|
files,
|
|
Math.min(
|
|
agentsCoreFiles
|
|
? Math.max(1, files.filter((file) => agentsCoreFiles.has(file)).length)
|
|
: files.length,
|
|
Math.max(1, Math.ceil(remainingSeconds / COMPACT_GITHUB_MAX_PREDICTED_SECONDS)),
|
|
),
|
|
weightForValue,
|
|
isCliProcess || isTooling ? batchWeight : undefined,
|
|
);
|
|
}
|
|
const partitioned = runtimePartition ? [runtimePartition.runtimeFiles, ...stripes] : stripes;
|
|
// Preserve prerequisite ownership and the existing weighted stripes. The
|
|
// family guard prevents cost packing from joining these serial chunks again.
|
|
const bounded =
|
|
hostedFileLimit === undefined
|
|
? partitioned
|
|
: partitioned.flatMap((stripeFiles) =>
|
|
Array.from({ length: Math.ceil(stripeFiles.length / hostedFileLimit) }, (_, index) =>
|
|
stripeFiles.slice(index * hostedFileLimit, (index + 1) * hostedFileLimit),
|
|
),
|
|
);
|
|
// The update file alone contributed 827 seconds of serial case time.
|
|
// Preserve the other storage files' existing order, workers and ownership.
|
|
return splitHostedUpdate
|
|
? bounded.flatMap((stripeFiles) =>
|
|
stripeFiles.includes(HOSTED_MAIN_UPDATE_TEST) && stripeFiles.length > 1
|
|
? [
|
|
[HOSTED_MAIN_UPDATE_TEST],
|
|
stripeFiles.filter((file) => file !== HOSTED_MAIN_UPDATE_TEST),
|
|
]
|
|
: [stripeFiles],
|
|
)
|
|
: bounded;
|
|
};
|
|
const timingEnv = parallelCommands ? { ...group.env, ...PINNED_COMPACT_GROUP_ENV } : group.env;
|
|
let stripes = createStripes(splitSeconds);
|
|
let timingGeneration = createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env: timingEnv,
|
|
parentShardName: splitTimingParent,
|
|
stripes,
|
|
});
|
|
// The measured worker cutover changes timing identity, not admission budgets.
|
|
// Retain the previous two-worker observations as floors until timing refits retire them.
|
|
const previousWorkerEnv: Record<string, string> | undefined =
|
|
!parallelCommands && usesMeasuredCompactWorkers(group, runnerBackend)
|
|
? { ...group.env, ...PINNED_COMPACT_GROUP_ENV }
|
|
: undefined;
|
|
// The isolated cohort inherited its old two-worker cap from the job, not its descriptor.
|
|
if (previousWorkerEnv && MEASURED_GATEWAY_ISOLATED_GROUP_RE.test(group.shard_name)) {
|
|
delete previousWorkerEnv.OPENCLAW_VITEST_MAX_WORKERS;
|
|
}
|
|
const previousWorkerGeneration = previousWorkerEnv
|
|
? createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env: previousWorkerEnv,
|
|
parentShardName: splitTimingParent,
|
|
stripes,
|
|
})
|
|
: undefined;
|
|
// Former eight-worker Gateway measurements remain floors, not two-worker samples.
|
|
const previousGatewayWorkerGeneration =
|
|
previousWorkerEnv && MEASURED_GATEWAY_ISOLATED_GROUP_RE.test(group.shard_name)
|
|
? createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env: { ...group.env, OPENCLAW_VITEST_MAX_WORKERS: "8" },
|
|
parentShardName: splitTimingParent,
|
|
stripes,
|
|
})
|
|
: undefined;
|
|
const selectors = (
|
|
isTooling
|
|
? []
|
|
: [
|
|
timingGeneration.selectorKey,
|
|
previousWorkerGeneration?.selectorKey,
|
|
previousGatewayWorkerGeneration?.selectorKey,
|
|
]
|
|
).filter((selector): selector is string => selector !== undefined);
|
|
const completeBlacksmithSeconds = Math.max(
|
|
splitParentSeconds.blacksmith,
|
|
...selectors.map(
|
|
(selector) =>
|
|
readCompleteSplitGenerationSeconds(readCompactGroupTimings("blacksmith"), selector) ?? 0,
|
|
),
|
|
);
|
|
const completeHostedSeconds = Math.max(
|
|
splitParentSeconds.github,
|
|
...selectors.map(
|
|
(selector) =>
|
|
readCompleteSplitGenerationSeconds(readCompactGroupTimings("github"), selector) ?? 0,
|
|
),
|
|
);
|
|
const completeMeasuredSeconds =
|
|
runnerBackend === "github"
|
|
? completeHostedSeconds
|
|
: runnerBackend === "hybrid"
|
|
? Math.max(completeBlacksmithSeconds, completeHostedSeconds)
|
|
: completeBlacksmithSeconds;
|
|
// A measured two-worker envelope must keep that ceiling after its children move to solo rows.
|
|
const measuredGroup =
|
|
parallelCommands && completeMeasuredSeconds > 0 ? { ...group, env: timingEnv } : group;
|
|
if (
|
|
!runtimePartition &&
|
|
!exceedsHostedFileLimit &&
|
|
!splitHostedUpdate &&
|
|
(!parallelGateway || completeMeasuredSeconds === 0) &&
|
|
Math.max(splitSeconds, completeMeasuredSeconds) <= COMPACT_GITHUB_MAX_PREDICTED_SECONDS
|
|
) {
|
|
return [
|
|
{
|
|
group: measuredGroup,
|
|
seconds: parallelCommands
|
|
? Math.max(
|
|
profileSeconds,
|
|
runnerBackend === "github" ? completeHostedSeconds : completeBlacksmithSeconds,
|
|
)
|
|
: profileSeconds,
|
|
},
|
|
];
|
|
}
|
|
if (completeMeasuredSeconds > splitSeconds) {
|
|
stripes = createStripes(completeMeasuredSeconds);
|
|
timingGeneration = createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env: timingEnv,
|
|
parentShardName: splitTimingParent,
|
|
stripes,
|
|
});
|
|
}
|
|
const completeProfileSeconds =
|
|
runnerBackend === "github" ? completeHostedSeconds : completeBlacksmithSeconds;
|
|
const distributedProfileSeconds = Math.max(profileSeconds, completeProfileSeconds);
|
|
if (tailDonation) {
|
|
onHostedToolingTailDonation?.(tailDonation);
|
|
}
|
|
const legacyCommandTimingKeys = parallelCommands
|
|
? createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
parentShardName: group.shard_name,
|
|
stripes,
|
|
}).timingKeys
|
|
: [];
|
|
const previousWorkerTimingKeys = previousWorkerGeneration
|
|
? createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env: previousWorkerEnv,
|
|
parentShardName: splitTimingParent,
|
|
stripes,
|
|
}).timingKeys
|
|
: [];
|
|
const gatewaySingletonWorkers = parallelGateway ? gatewayServerEffectiveWorkers(group) : 1;
|
|
const serialSeconds = parallelGateway
|
|
? gatewayServerSerialSeconds(includePatterns, runnerBackend)
|
|
: 0;
|
|
const parallelWeight = includePatterns.reduce(
|
|
(sum, file) => sum + (gatewayServerSerialTestFiles.includes(file) ? 0 : weightForFile(file)),
|
|
0,
|
|
);
|
|
const mixedPhaseSeconds = (patterns: string[]) => {
|
|
const childSerialSeconds = gatewayServerSerialSeconds(patterns, runnerBackend);
|
|
const childParallelWeight = patterns.reduce(
|
|
(sum, file) => sum + (gatewayServerSerialTestFiles.includes(file) ? 0 : weightForFile(file)),
|
|
0,
|
|
);
|
|
if (parallelWeight === 0) {
|
|
return (
|
|
(Math.max(profileSeconds, distributedProfileSeconds) * childSerialSeconds) / serialSeconds
|
|
);
|
|
}
|
|
const fraction = childParallelWeight / parallelWeight;
|
|
const childWorkers = gatewayServerEffectiveWorkers({ ...group, includePatterns: patterns });
|
|
return (
|
|
childSerialSeconds +
|
|
Math.max(
|
|
(Math.max(0, profileSeconds - serialSeconds) * gatewaySingletonWorkers) / childWorkers,
|
|
Math.max(0, distributedProfileSeconds - serialSeconds),
|
|
) *
|
|
fraction
|
|
);
|
|
};
|
|
const serialChildSeconds = new Map<string, number>();
|
|
const childTimings = readCompactGroupTimings(
|
|
runnerBackend === "github" ? "github" : "blacksmith",
|
|
);
|
|
if (isParallelAgentsCoreGroup(group)) {
|
|
const { OPENCLAW_VITEST_MAX_WORKERS: _workers, ...env } = group.env ?? {};
|
|
const legacy = createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env,
|
|
parentShardName: group.shard_name,
|
|
stripes,
|
|
});
|
|
for (const [key, seconds] of Object.entries(childTimings)) {
|
|
if (parseCompactSplitTimingKey(key)?.selectorKey !== legacy.selectorKey) {
|
|
continue;
|
|
}
|
|
// Unchanged child membership retains its sample when sibling stripes merge.
|
|
const membership = key.slice(key.lastIndexOf("#include-"));
|
|
serialChildSeconds.set(
|
|
membership,
|
|
Math.max(serialChildSeconds.get(membership) ?? 0, seconds),
|
|
);
|
|
}
|
|
}
|
|
return stripes.flatMap((patterns, index) => {
|
|
const timingKey = timingGeneration.timingKeys[index]!;
|
|
const child: NodeTestShardGroup = {
|
|
...measuredGroup,
|
|
includePatterns: patterns,
|
|
pretestBuildMode: mergeVitestPretestBuildModes(patterns.map((file) => buildModes.get(file))),
|
|
shard_name: `${group.shard_name}-hosted-${index + 1}`,
|
|
timing_key: timingKey,
|
|
};
|
|
if (isTooling) {
|
|
return [
|
|
{ group: child, seconds: estimateParallelToolingSeconds(child, patterns, runnerBackend) },
|
|
];
|
|
}
|
|
const measuredChild = estimateCompactStripeSeconds(child, runnerBackend);
|
|
const fileWorkFloor = storageModuleWorkFloor(child, runnerBackend);
|
|
const childFloor = Math.max(measuredChild, fileWorkFloor);
|
|
if (childFloor > 0) {
|
|
if (
|
|
childFloor > COMPACT_SERIAL_NODE_TEST_JOB_SECONDS &&
|
|
!child.requiresDist &&
|
|
!child.pretestBuildMode &&
|
|
patterns.length > 1 &&
|
|
isRuntimePlacementIncludePatterns(patterns) &&
|
|
(child.env?.OPENCLAW_VITEST_MAX_WORKERS === "2" ||
|
|
(child.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined &&
|
|
patterns.every((file) => STORAGE_MODULE_WORK_SECONDS.has(file)) &&
|
|
fileWorkFloor >= childFloor))
|
|
) {
|
|
// Two-worker observations or the two-slot storage floor can fund this cap.
|
|
// An unpinned measured wall can describe eight workers; preserve its policy.
|
|
return splitOversizedCompactGroup(
|
|
{ ...child, env: { ...child.env, ...PINNED_COMPACT_GROUP_ENV } },
|
|
runnerBackend,
|
|
childFloor,
|
|
);
|
|
}
|
|
if (measuredChild > 0) {
|
|
return [{ group: child, seconds: childFloor }];
|
|
}
|
|
}
|
|
const childWorkers = isAutoReplyReplyGroup(child)
|
|
? compactEffectiveFileWorkers(child, patterns.length)
|
|
: effectiveAgentsCoreWorkers(child);
|
|
const weight = patterns.reduce((seconds, file) => seconds + weightForFile(file), 0);
|
|
const projectedSeconds = parallelGateway
|
|
? serialSeconds > 0
|
|
? mixedPhaseSeconds(patterns)
|
|
: (Math.max(
|
|
distributedProfileSeconds,
|
|
patterns.length === 1 ? profileSeconds * gatewaySingletonWorkers : 0,
|
|
) *
|
|
weight) /
|
|
totalWeight
|
|
: Math.max(
|
|
// Convert parent wall back to work before projecting a child's worker budget.
|
|
// Complete-generation totals already sum child walls and must not be multiplied.
|
|
((profileSeconds * weight) / totalWeight) * (parentWorkers / childWorkers),
|
|
(completeProfileSeconds * weight) / totalWeight,
|
|
((serialChildSeconds.get(timingKey.slice(timingKey.lastIndexOf("#include-"))) ?? 0) *
|
|
(runnerBackend === "hybrid" ? COMPACT_HYBRID_GROUP_SECONDS_SCALE : 1)) /
|
|
childWorkers,
|
|
);
|
|
return [
|
|
{
|
|
group: child,
|
|
// Round once at the job boundary; per-child rounding can duplicate a build
|
|
// when many small consumers together still fit one preparation budget.
|
|
seconds: Math.max(
|
|
parallelCommands ? commandFileSecondsFloor(patterns, runnerBackend) : 0,
|
|
legacyCommandTimingKeys[index]
|
|
? estimateLegacyCommandStripeSeconds(
|
|
patterns,
|
|
legacyCommandTimingKeys[index],
|
|
runnerBackend,
|
|
)
|
|
: 0,
|
|
projectedSeconds,
|
|
fileWorkFloor,
|
|
previousWorkerTimingKeys[index]
|
|
? estimateCompactStripeSeconds(
|
|
{ ...group, timing_key: previousWorkerTimingKeys[index] },
|
|
runnerBackend,
|
|
)
|
|
: 0,
|
|
),
|
|
},
|
|
];
|
|
});
|
|
}
|
|
|
|
// Owners supply admission order and compatibility; placement retains the original
|
|
// descriptors and never emits an empty job.
|
|
export function packNodeTestGroups<Group>(
|
|
orderedGroups: readonly Group[],
|
|
canShareJob: (bin: readonly [Group, ...Group[]], group: Group) => boolean,
|
|
allowGroupExchange = false,
|
|
): Array<[Group, ...Group[]]> {
|
|
const bins: Array<[Group, ...Group[]]> = [];
|
|
const admits = ([first, ...rest]: [Group, ...Group[]]) => {
|
|
const admitted: [Group, ...Group[]] = [first];
|
|
for (const entry of rest) {
|
|
if (!canShareJob(admitted, entry)) {
|
|
return false;
|
|
}
|
|
admitted.push(entry);
|
|
}
|
|
return true;
|
|
};
|
|
// A single exchange can free both time and group slots without adding a job.
|
|
// Validate complete replacement bins before changing either existing bin.
|
|
const exchange = (group: Group) => {
|
|
for (const [leftIndex, left] of bins.entries()) {
|
|
for (const right of bins.slice(leftIndex + 1)) {
|
|
for (const [leftSlot, leftGroup] of left.entries()) {
|
|
for (const [rightSlot, rightGroup] of right.entries()) {
|
|
const nextLeft: [Group, ...Group[]] = [...left];
|
|
const nextRight: [Group, ...Group[]] = [...right];
|
|
nextLeft[leftSlot] = rightGroup;
|
|
nextRight[rightSlot] = leftGroup;
|
|
const target = canShareJob(nextLeft, group)
|
|
? nextLeft
|
|
: canShareJob(nextRight, group)
|
|
? nextRight
|
|
: undefined;
|
|
if (target && admits(nextLeft) && admits(nextRight)) {
|
|
target.push(group);
|
|
left.splice(0, left.length, ...nextLeft);
|
|
right.splice(0, right.length, ...nextRight);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
for (const group of orderedGroups) {
|
|
const bin = bins.find((candidate) => canShareJob(candidate, group));
|
|
if (bin) {
|
|
bin.push(group);
|
|
} else if (!allowGroupExchange || !exchange(group)) {
|
|
bins.push([group]);
|
|
}
|
|
}
|
|
return bins;
|
|
}
|
|
|
|
/** Select exact files without losing their canonical process and artifact owners. */
|
|
export function createSelectedNodeTestShardBundles(
|
|
targets: readonly string[],
|
|
options: Pick<NodeTestPlanOptions, "runnerBackend"> &
|
|
RuntimeTestSelection & {
|
|
onFallback?: (reason: string) => void;
|
|
preparedTestPlans?: ReadonlyMap<string, ReturnType<typeof buildVitestRunPlans>>;
|
|
} = {},
|
|
): CompactNodeTestShard[] | null {
|
|
if (options.runnerBackend === "runson") {
|
|
const selected = createSelectedNodeTestShardBundles(targets, {
|
|
...options,
|
|
runnerBackend: "hybrid",
|
|
});
|
|
return selected && routeRunsOnJobs(selected, COMPACT_NODE_TEST_JOB_CAP);
|
|
}
|
|
const selected = new Set(
|
|
targets.filter((file) => !isCiProofTestFile(file) && isRuntimeTestFileIncluded(file, options)),
|
|
);
|
|
const configs = new Map<string, string>();
|
|
const changedTestPlans: ReturnType<typeof buildVitestRunPlans> = [];
|
|
for (const target of selected) {
|
|
const plans = options.preparedTestPlans?.get(target) ?? buildVitestRunPlans([target]);
|
|
const exactFilter =
|
|
plans.length === 1 &&
|
|
plans[0]!.forwardedArgs.length === 1 &&
|
|
plans[0]!.forwardedArgs[0] === target;
|
|
if (
|
|
!isTestFileTarget(target) ||
|
|
!statSync(target, { throwIfNoEntry: false })?.isFile() ||
|
|
plans.length !== 1 ||
|
|
plans[0]!.watchMode ||
|
|
(plans[0]!.forwardedArgs.length > 0 && !exactFilter) ||
|
|
(plans[0]!.includePatterns
|
|
? plans[0]!.includePatterns.length !== 1 || plans[0]!.includePatterns[0] !== target
|
|
: !exactFilter)
|
|
) {
|
|
options.onFallback?.(
|
|
`unsupported canonical target: ${target} (${plans.map((plan) => plan.config).join(", ")})`,
|
|
);
|
|
return null;
|
|
}
|
|
const config = resolveCanonicalNodeTestConfig(target, plans[0]!.config);
|
|
if (!config) {
|
|
options.onFallback?.(`missing canonical config: ${target} (${plans[0]!.config})`);
|
|
return null;
|
|
}
|
|
configs.set(target, config);
|
|
if (!target.endsWith(".live.test.ts")) {
|
|
changedTestPlans.push(...plans);
|
|
}
|
|
}
|
|
if (selected.size === 0) {
|
|
return targets.length > 0 ? [] : null;
|
|
}
|
|
const tooling = new Set([...selected].filter((target) => configs.get(target) === TOOLING_CONFIG));
|
|
const shards = createNodeTestShardsForOwners(
|
|
fullSuiteVitestShards,
|
|
{
|
|
...options,
|
|
// Exact selection opts into its existing owner, not the unrelated plugin sweep.
|
|
changedPaths: [...selected],
|
|
includeReleaseOnlyPluginShards: false,
|
|
includeProofTests: false,
|
|
},
|
|
tooling.size === selected.size,
|
|
changedTestPlans,
|
|
);
|
|
const owners = new Set<NodeTestShard>();
|
|
for (const target of tooling) {
|
|
const matches = shards.filter(
|
|
(shard) =>
|
|
shard.configs.length === 1 &&
|
|
shard.configs[0] === TOOLING_CONFIG &&
|
|
shard.includePatterns?.includes(target),
|
|
);
|
|
if (matches.length !== 1) {
|
|
options.onFallback?.(`canonical tooling owner count ${matches.length}: ${target}`);
|
|
return null;
|
|
}
|
|
owners.add(matches[0]!);
|
|
}
|
|
// The complete plan owns process boundaries and resource policies. Its prices
|
|
// are planning facts; a selected subset must not reserve every original row.
|
|
const groupSeconds = new Map<NodeTestShardGroup, number>();
|
|
const full =
|
|
selected.size > tooling.size
|
|
? createCompactNodeTestShardBundles(shards, { ...options, groupSeconds }, "pull-request")
|
|
: [];
|
|
const canonicalGroups = full.flatMap((shard) => shard.groups);
|
|
const isWholeToolingPair = (group: NodeTestShardGroup) =>
|
|
group.shard_name === "core-tooling-isolated" &&
|
|
group.configs.length === 2 &&
|
|
group.configs[0] === "test/vitest/vitest.tooling-docker.config.ts" &&
|
|
group.configs[1] === TOOLING_ISOLATED_CONFIG;
|
|
const selectedGroups = new Map<NodeTestShardGroup, string[]>();
|
|
for (const target of selected) {
|
|
if (tooling.has(target)) {
|
|
continue;
|
|
}
|
|
const matches = canonicalGroups.filter(
|
|
(group) =>
|
|
group.configs.includes(configs.get(target)!) &&
|
|
group.env?.OPENCLAW_NODE_TEST_VITEST_ARGS_JSON === undefined &&
|
|
(!group.includePatterns || group.includePatterns.includes(target)),
|
|
);
|
|
if (matches.length !== 1) {
|
|
options.onFallback?.(
|
|
`canonical owner count ${matches.length}: ${target} (${configs.get(target)})`,
|
|
);
|
|
return null;
|
|
}
|
|
const owner = matches[0]!;
|
|
selectedGroups.set(owner, [...(selectedGroups.get(owner) ?? []), target]);
|
|
}
|
|
const canonicalFamilies = new Map<NodeTestShardGroup, string | undefined>();
|
|
const selectedTimings = readCompactGroupTimings(
|
|
options.runnerBackend === "github" ? "github" : "blacksmith",
|
|
);
|
|
const projected = full.flatMap((shard) => {
|
|
let retainedSeconds = 0;
|
|
const groups = shard.groups.flatMap((group) => {
|
|
const seconds = groupSeconds.get(group)!;
|
|
const files = selectedGroups.get(group);
|
|
if (!files?.length) {
|
|
return [];
|
|
}
|
|
// Artifact descriptors and opaque paired configs retain their complete owner.
|
|
if (group.requiresDist || isWholeToolingPair(group)) {
|
|
retainedSeconds += seconds;
|
|
canonicalFamilies.set(group, compactStripeFamily(group));
|
|
return [group];
|
|
}
|
|
const includePatterns =
|
|
group.includePatterns?.filter((file) => files.includes(file)) ?? files;
|
|
const selectedConfigs = group.configs.filter((config) =>
|
|
files.some((file) => configs.get(file) === config),
|
|
);
|
|
const inventory =
|
|
group.includePatterns ??
|
|
listWholeConfigFiles(group.shard_name) ??
|
|
(group.configs.length === 1 ? listNodeTestConfigFiles(group.configs[0]!) : undefined);
|
|
// Preserve opaque prices. Known inventories use the existing relative file
|
|
// weights, retaining an indivisible-file floor and separate build admission.
|
|
const weight = (paths: readonly string[]) =>
|
|
paths.reduce((total, file) => total + stripeFileWeight(file), 0);
|
|
const fallbackSeconds = inventory?.length
|
|
? Math.min(
|
|
seconds,
|
|
Math.max(
|
|
(seconds * weight(includePatterns)) / weight(inventory),
|
|
DEFAULT_WHOLE_GROUP_SECONDS *
|
|
(options.runnerBackend === "github" ? COMPACT_GITHUB_GROUP_SECONDS_SCALE : 1),
|
|
...includePatterns.map(stripeFileWeight),
|
|
),
|
|
)
|
|
: seconds;
|
|
const canonicalTimingKey = compactGroupTimingKey(group);
|
|
const timingParent =
|
|
parseCompactSplitTimingKey(canonicalTimingKey)?.parentShardName ?? canonicalTimingKey;
|
|
const { timingKeys } = createCompactSplitTimingGeneration({
|
|
configs: selectedConfigs,
|
|
env: group.env,
|
|
// Retain worker/parallel timing policy without borrowing the full owner's inventory.
|
|
parentShardName: timingParent.startsWith("changed-")
|
|
? timingParent
|
|
: `changed-${timingParent}`,
|
|
stripes: [includePatterns],
|
|
});
|
|
const selectedSeconds = Math.max(fallbackSeconds, selectedTimings[timingKeys[0]!] ?? 0);
|
|
retainedSeconds += selectedSeconds;
|
|
const projectedGroup = {
|
|
...group,
|
|
configs: selectedConfigs,
|
|
includePatterns,
|
|
timing_key: timingKeys[0]!,
|
|
};
|
|
groupSeconds.set(projectedGroup, selectedSeconds);
|
|
canonicalFamilies.set(projectedGroup, compactStripeFamily(group));
|
|
return [projectedGroup];
|
|
});
|
|
return groups.length
|
|
? [
|
|
{
|
|
...shard,
|
|
checkName: `checks-node-changed-${shard.shardName}`,
|
|
shardName: `changed-${shard.shardName}`,
|
|
groups,
|
|
predictedSeconds: Math.ceil(
|
|
retainedSeconds +
|
|
compactPreparationSeconds(shard.pretestBuildMode, options.runnerBackend),
|
|
),
|
|
},
|
|
]
|
|
: [];
|
|
});
|
|
return applyNativeSoloTimings(
|
|
[
|
|
...(tooling.size
|
|
? createCompactNodeTestShardBundles(
|
|
shards.filter((shard) => owners.has(shard)),
|
|
options,
|
|
"pull-request",
|
|
tooling,
|
|
)
|
|
: []),
|
|
...packSelectedNodeTestJobs(
|
|
projected,
|
|
options.runnerBackend,
|
|
canonicalFamilies,
|
|
groupSeconds,
|
|
),
|
|
],
|
|
options.runnerBackend,
|
|
);
|
|
}
|
|
|
|
function compactPreparationSeconds(
|
|
mode: NodeTestPretestBuildMode | undefined,
|
|
runnerBackend: string | undefined,
|
|
) {
|
|
return mode
|
|
? Math.round(
|
|
VITEST_PRETEST_BUILD_SECONDS[mode] *
|
|
(runnerBackend === "github" ? COMPACT_GITHUB_GROUP_SECONDS_SCALE : 1),
|
|
)
|
|
: 0;
|
|
}
|
|
|
|
function estimateParallelTestSeconds(
|
|
groups: readonly NodeTestShardGroup[],
|
|
estimateGroupSeconds: (group: NodeTestShardGroup) => number,
|
|
) {
|
|
// The executor takes the next group as soon as either of its two slots is free.
|
|
const slots = [0, 0];
|
|
for (const group of groups) {
|
|
const slot = slots[0]! <= slots[1]! ? 0 : 1;
|
|
slots[slot]! += estimateGroupSeconds(group);
|
|
}
|
|
return Math.ceil(Math.max(...slots));
|
|
}
|
|
|
|
function packSelectedNodeTestJobs(
|
|
jobs: CompactNodeTestShard[],
|
|
runnerBackend: string | undefined,
|
|
canonicalFamilies: ReadonlyMap<NodeTestShardGroup, string | undefined>,
|
|
groupSeconds: ReadonlyMap<NodeTestShardGroup, number>,
|
|
) {
|
|
const policy = (job: CompactNodeTestShard) =>
|
|
JSON.stringify([
|
|
job.runner,
|
|
job.requiresDist,
|
|
job.pretestBuildMode,
|
|
job.planConcurrency,
|
|
job.groups.some(isExclusiveCompactGroup),
|
|
Object.entries(job.env ?? {}).toSorted(([a], [b]) => a.localeCompare(b)),
|
|
job.timeoutMinutes,
|
|
]);
|
|
const seconds = (bin: readonly CompactNodeTestShard[]) =>
|
|
bin.reduce(
|
|
(total, job) =>
|
|
total +
|
|
job.predictedSeconds! -
|
|
compactPreparationSeconds(job.pretestBuildMode, runnerBackend),
|
|
0,
|
|
) + compactPreparationSeconds(bin[0]!.pretestBuildMode, runnerBackend);
|
|
const testSeconds = (bin: readonly CompactNodeTestShard[]) =>
|
|
bin[0]!.planConcurrency === 2
|
|
? estimateParallelTestSeconds(
|
|
bin.flatMap((job) => job.groups),
|
|
(group) => groupSeconds.get(group)!,
|
|
)
|
|
: Math.max(
|
|
0,
|
|
Math.ceil(seconds(bin)) -
|
|
compactPreparationSeconds(bin[0]!.pretestBuildMode, runnerBackend),
|
|
);
|
|
const cap = (bin: readonly CompactNodeTestShard[]) => {
|
|
const first = bin[0]!;
|
|
const groups = bin.flatMap((job) => job.groups);
|
|
const hostedBuild = runnerBackend === "github" && first.pretestBuildMode;
|
|
if (groups.some(isExclusiveCompactGroup) && !hostedBuild) {
|
|
return runnerBackend === "hybrid" &&
|
|
!first.pretestBuildMode &&
|
|
bin.every((job) => job.predictedSeconds! <= COMPACT_EXCLUSIVE_JOB_SECONDS) &&
|
|
groups.every((group) => COMPACT_SERIAL_CLI_GROUP_NAME_RE.test(group.shard_name))
|
|
? COMPACT_HYBRID_SERIAL_CLI_JOB_SECONDS
|
|
: COMPACT_EXCLUSIVE_JOB_SECONDS;
|
|
}
|
|
if (first.pretestBuildMode === "runtime" && runnerBackend === "hybrid") {
|
|
return COMPACT_HYBRID_RUNTIME_JOB_SECONDS;
|
|
}
|
|
if (first.planConcurrency === 2) {
|
|
return COMPACT_FINAL_PARALLEL_NODE_TEST_JOB_SECONDS;
|
|
}
|
|
return usesExpandedRunnerProfile(runnerBackend)
|
|
? COMPACT_EXPANDED_NODE_TEST_JOB_SECONDS
|
|
: resolveCiNodeTestRunnerClass(first.runner).secondsCap;
|
|
};
|
|
// Whole projected jobs are indivisible: regrouping cannot reduce a child's
|
|
// prior concurrency envelope and accidentally unlock more workers.
|
|
return packNodeTestGroups(
|
|
jobs.toSorted(
|
|
(a, b) => b.predictedSeconds! - a.predictedSeconds! || a.checkName.localeCompare(b.checkName),
|
|
),
|
|
(bin, job) => {
|
|
if (job.requiresDist || policy(bin[0]) !== policy(job)) {
|
|
return false;
|
|
}
|
|
const combined = [...bin, job];
|
|
const groups = combined.flatMap((entry) => entry.groups);
|
|
const families = groups
|
|
.map((group) => canonicalFamilies.get(group))
|
|
.filter((family) => family !== undefined);
|
|
return (
|
|
groups.length <= COMPACT_NODE_TEST_JOB_GROUPS &&
|
|
(job.planConcurrency === 1 || new Set(families).size === families.length) &&
|
|
seconds(combined) <= cap(combined) &&
|
|
(job.planConcurrency !== 2 || testSeconds(combined) <= COMPACT_SERIAL_NODE_TEST_JOB_SECONDS)
|
|
);
|
|
},
|
|
)
|
|
.map((bin) =>
|
|
Object.assign(bin[0], {
|
|
groups: bin.flatMap((job) => job.groups),
|
|
predictedSeconds: Math.ceil(seconds(bin)),
|
|
predictedTestSeconds: testSeconds(bin),
|
|
}),
|
|
)
|
|
.toSorted((a, b) => a.checkName.localeCompare(b.checkName));
|
|
}
|
|
|
|
function routeRunsOnJobs(
|
|
jobs: CompactNodeTestShard[],
|
|
compactNodeJobCap: number,
|
|
): CompactNodeTestShard[] {
|
|
const cronGroups: NodeTestShardGroup[] = [];
|
|
const cronTimeouts: number[] = [];
|
|
const routed = jobs.flatMap((job) => {
|
|
if (
|
|
job.requiresDist ||
|
|
job.pretestBuildMode ||
|
|
(job.env?.OPENCLAW_VITEST_MAX_WORKERS !== undefined &&
|
|
job.env.OPENCLAW_VITEST_MAX_WORKERS !== "2") ||
|
|
Object.keys(job.env ?? {}).some((key) => key !== "OPENCLAW_VITEST_MAX_WORKERS")
|
|
) {
|
|
return [job];
|
|
}
|
|
const retained = job.groups.filter((group) => {
|
|
if (
|
|
!/^core-runtime-cron-parallel-(?:core|isolated-agent|service)(?:-hosted-\d+)?$/u.test(
|
|
group.shard_name,
|
|
) ||
|
|
group.requiresDist ||
|
|
group.pretestBuildMode ||
|
|
group.configs.length !== 1 ||
|
|
group.configs[0] !== "test/vitest/vitest.cron.config.ts" ||
|
|
!group.includePatterns?.length ||
|
|
group.includePatterns.some((file) => !file.startsWith("src/cron/")) ||
|
|
(group.env?.OPENCLAW_VITEST_MAX_WORKERS !== undefined &&
|
|
group.env.OPENCLAW_VITEST_MAX_WORKERS !== "2") ||
|
|
(group.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined &&
|
|
job.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined &&
|
|
job.planConcurrency !== 2)
|
|
) {
|
|
return true;
|
|
}
|
|
cronGroups.push(group);
|
|
// The matrix's existing default is 60 minutes; extraction cannot extend
|
|
// any contributing job's execution deadline.
|
|
cronTimeouts.push(job.timeoutMinutes ?? 60);
|
|
return false;
|
|
});
|
|
return retained.length === job.groups.length
|
|
? [job]
|
|
: retained.length
|
|
? [{ ...job, groups: retained }]
|
|
: [];
|
|
});
|
|
if (cronGroups.length > 0) {
|
|
routed.push({
|
|
checkName: "checks-node-runson-cron",
|
|
shardName: "runson-cron",
|
|
runner: "runson-c8i-8xlarge",
|
|
groups: cronGroups,
|
|
requiresDist: false,
|
|
planConcurrency: 1,
|
|
timeoutMinutes: Math.min(...cronTimeouts),
|
|
// Extraction must not turn a packed child's two-worker allowance into
|
|
// the larger host default. Keep its selectors and group policy intact.
|
|
env: { ...PINNED_COMPACT_GROUP_ENV },
|
|
});
|
|
}
|
|
const jobCap = Math.min(
|
|
COMPACT_NODE_TEST_JOB_CAP,
|
|
compactNodeJobCap + routed.filter((job) => job.requiresDist).length,
|
|
);
|
|
if (routed.length > jobCap) {
|
|
throw new Error(
|
|
`compact runson node test plan exceeds ${jobCap} jobs (${routed.length} planned)`,
|
|
);
|
|
}
|
|
return routed.toSorted((a, b) => a.checkName.localeCompare(b.checkName));
|
|
}
|
|
|
|
function createCompactNodeTestShardBundles(
|
|
sourceShards: readonly NodeTestShard[],
|
|
options: NodeTestPlanOptions & { groupSeconds?: Map<NodeTestShardGroup, number> },
|
|
compactMode: CompactNodeTestPlanMode,
|
|
selectedToolingFiles?: ReadonlySet<string>,
|
|
splitHostedToolingTails = false,
|
|
hostedToolingTailBudgets?: ReadonlyMap<string, number>,
|
|
hostedToolingTailDonation?: HostedToolingTailDonation,
|
|
prioritizeSerialGateway = false,
|
|
): CompactNodeTestShard[] {
|
|
if (options.runnerBackend === "runson") {
|
|
// Hybrid owns placement and measured serial packing; RunsOn only extracts cron.
|
|
return routeRunsOnJobs(
|
|
createCompactNodeTestShardBundles(
|
|
sourceShards,
|
|
{ ...options, runnerBackend: "hybrid" },
|
|
compactMode,
|
|
selectedToolingFiles,
|
|
splitHostedToolingTails,
|
|
hostedToolingTailBudgets,
|
|
hostedToolingTailDonation,
|
|
prioritizeSerialGateway,
|
|
),
|
|
options.compactNodeJobCap ?? COMPACT_NODE_TEST_JOB_CAP,
|
|
);
|
|
}
|
|
const profileJobCap =
|
|
options.runnerBackend === "github"
|
|
? COMPACT_GITHUB_NODE_TEST_JOB_CAP
|
|
: COMPACT_NODE_TEST_JOB_CAP;
|
|
const compactNodeJobCap = options.compactNodeJobCap ?? profileJobCap;
|
|
if (!Number.isSafeInteger(compactNodeJobCap) || compactNodeJobCap < 1) {
|
|
throw new Error("compact Node job cap must be a positive integer");
|
|
}
|
|
const effectiveJobCap = (bins: readonly (readonly NodeTestShardGroup[])[]) =>
|
|
Math.min(profileJobCap, compactNodeJobCap + bins.filter((bin) => bin[0]?.requiresDist).length);
|
|
const includeTooling = compactMode !== "push" && includesReleaseOnlyTooling(options);
|
|
const keepPrExemptRuntimeFile = (file: string) =>
|
|
options.includePrExemptRuntimeTests !== undefined &&
|
|
isPrExemptRuntimeTestFile(file) &&
|
|
isRuntimeTestFileIncluded(file, options);
|
|
const isBlacksmithProfile = (options.runnerBackend ?? "blacksmith") === "blacksmith";
|
|
const packsHostedTooling = compactMode === "pull-request" && options.runnerBackend === "github";
|
|
let bestTailDonation: HostedToolingTailDonation | undefined;
|
|
const collectTailDonation =
|
|
packsHostedTooling && splitHostedToolingTails && !hostedToolingTailDonation
|
|
? (donation: HostedToolingTailDonation) => {
|
|
if (
|
|
!bestTailDonation ||
|
|
donation.freedSeconds > bestTailDonation.freedSeconds ||
|
|
(donation.freedSeconds === bestTailDonation.freedSeconds &&
|
|
`${donation.parentShardName}/${donation.file}`.localeCompare(
|
|
`${bestTailDonation.parentShardName}/${bestTailDonation.file}`,
|
|
) < 0)
|
|
) {
|
|
bestTailDonation = donation;
|
|
}
|
|
}
|
|
: undefined;
|
|
const shards = sourceShards.filter(
|
|
(shard) =>
|
|
((compactMode !== "push" || !COMPACT_PUSH_EXCLUDED_SHARDS.has(shard.shardName)) &&
|
|
(includeTooling ||
|
|
!shard.configs.every((config) => RELEASE_ONLY_TOOLING_CONFIGS.has(config)))) ||
|
|
(shard.includePatterns ?? listToolingGroupFiles(shard.configs))?.some(
|
|
keepPrExemptRuntimeFile,
|
|
),
|
|
);
|
|
const groupsByRunner = new Map<string, [NodeTestShardGroup, ...NodeTestShardGroup[]]>();
|
|
const synthesizedSplitSeconds = new Map<string, number>();
|
|
const runnerRank = (group: Pick<NodeTestShardGroup, "runner">) =>
|
|
[BUNDLED_NODE_TEST_RUNNER, DEFAULT_NODE_TEST_RUNNER, EXTRA_LARGE_NODE_TEST_RUNNER].indexOf(
|
|
group.runner,
|
|
);
|
|
|
|
for (const shard of shards) {
|
|
const runner = resolveCiNodeTestRunner(shard);
|
|
const group = applyCompactGroupWorkerPins(
|
|
{
|
|
configs: shard.configs,
|
|
...(shard.env ? { env: shard.env } : {}),
|
|
...(shard.includePatterns ? { includePatterns: shard.includePatterns } : {}),
|
|
...(shard.pretestBuildMode ? { pretestBuildMode: shard.pretestBuildMode } : {}),
|
|
requiresDist: shard.requiresDist,
|
|
runner,
|
|
shard_name: shard.shardName,
|
|
...(shard.timing_key ? { timing_key: shard.timing_key } : {}),
|
|
},
|
|
options.runnerBackend,
|
|
);
|
|
const partitionFiles = group.pretestBuildMode
|
|
? (group.includePatterns ?? listWholeConfigSplitFiles(group.shard_name))
|
|
: undefined;
|
|
const partition = partitionFiles
|
|
? partitionRuntimeTestFiles(group.configs, partitionFiles)
|
|
: undefined;
|
|
const runtimePartition =
|
|
partition?.runtimeFiles.length && partition.otherFiles.length ? partition : undefined;
|
|
// Resolve whole-config ownership before splitting so ordinary files do not
|
|
// inherit a runtime build. Keep consumers together and split the remaining work.
|
|
let plannedGroups =
|
|
usesExpandedRunnerProfile(options.runnerBackend) ||
|
|
COMPACT_BLACKSMITH_SPLIT_OWNERS.has(group.shard_name) ||
|
|
(isParallelToolingGroup(group) &&
|
|
compactToolingTestSecondsLimit([group], options.runnerBackend) ===
|
|
COMPACT_EXCLUSIVE_JOB_SECONDS) ||
|
|
isParallelGatewayServerGroup(group) ||
|
|
isParallelCommandsGroup(group) ||
|
|
runtimePartition !== undefined ||
|
|
(group.pretestBuildMode !== undefined && group.includePatterns === undefined)
|
|
? splitOversizedCompactGroup(
|
|
group,
|
|
options.runnerBackend,
|
|
0,
|
|
runtimePartition,
|
|
splitHostedToolingTails,
|
|
hostedToolingTailBudgets,
|
|
hostedToolingTailDonation,
|
|
collectTailDonation,
|
|
selectedToolingFiles,
|
|
compactMode === "push" && options.runnerBackend === "github",
|
|
)
|
|
: [{ group, seconds: estimateCompactGroupSeconds(group, options.runnerBackend) }];
|
|
const reducedToolingTier =
|
|
!includeTooling &&
|
|
(shard.configs.every((config) => RELEASE_ONLY_TOOLING_CONFIGS.has(config)) ||
|
|
plannedGroups.some((planned) =>
|
|
(planned.group.includePatterns ?? listToolingGroupFiles(planned.group.configs))?.some(
|
|
isReleaseOnlyToolingTestFile,
|
|
),
|
|
));
|
|
const selectedTooling =
|
|
(selectedToolingFiles && group.configs.includes(TOOLING_CONFIG)) || reducedToolingTier;
|
|
if (selectedTooling) {
|
|
// Keep the parent's admission cost and partition policy, but never report
|
|
// a precise subset as a sample of the complete canonical stripe.
|
|
plannedGroups = plannedGroups.flatMap((planned) => {
|
|
const includePatterns = (
|
|
planned.group.includePatterns ?? listToolingGroupFiles(planned.group.configs)
|
|
)?.filter(
|
|
(file) =>
|
|
(!selectedToolingFiles || selectedToolingFiles.has(file)) &&
|
|
(includeTooling ||
|
|
(!planned.group.configs.every((config) => RELEASE_ONLY_TOOLING_CONFIGS.has(config)) &&
|
|
!isReleaseOnlyToolingTestFile(file)) ||
|
|
keepPrExemptRuntimeFile(file)),
|
|
);
|
|
return includePatterns?.length
|
|
? [{ ...planned, group: { ...planned.group, includePatterns } }]
|
|
: [];
|
|
});
|
|
const generation = createCompactSplitTimingGeneration({
|
|
configs: group.configs,
|
|
env: group.env,
|
|
parentShardName: reducedToolingTier ? `changed-${group.shard_name}` : group.shard_name,
|
|
stripes: plannedGroups.map((planned) => planned.group.includePatterns!),
|
|
});
|
|
plannedGroups.forEach((planned, index) => {
|
|
planned.group.timing_key = generation.timingKeys[index]!;
|
|
});
|
|
}
|
|
for (const planned of plannedGroups) {
|
|
planned.group.runner = resolveCiNodeTestRunner(
|
|
{
|
|
...shard,
|
|
includePatterns: planned.group.includePatterns,
|
|
},
|
|
options.runnerBackend ?? "blacksmith",
|
|
);
|
|
if (
|
|
storageModuleWorkFloor(planned.group, options.runnerBackend) > COMPACT_EXCLUSIVE_JOB_SECONDS
|
|
) {
|
|
// Retain the measured storage allocation when splitting leaves an outlier alone.
|
|
planned.group.runner = EXTRA_LARGE_NODE_TEST_RUNNER;
|
|
}
|
|
// Ordinary packed self-hosted jobs already receive the 32-class. Share
|
|
// those slots across logical classes while keeping runtime owners separate.
|
|
const sharesOrdinaryCapacity =
|
|
(options.runnerBackend === "github" ||
|
|
((isBlacksmithProfile || options.runnerBackend === "hybrid") &&
|
|
!planned.group.pretestBuildMode)) &&
|
|
!planned.group.requiresDist &&
|
|
!isExclusiveCompactGroup(planned.group) &&
|
|
runnerRank(planned.group) >= 0;
|
|
const dedicatedHostedMainGroup =
|
|
compactMode === "push" &&
|
|
options.runnerBackend === "github" &&
|
|
(/^agentic-control-plane-agent-chat(?:-hosted-\d+)?$/u.test(planned.group.shard_name) ||
|
|
(/^agentic-gateway-methods(?:-hosted-\d+)?$/u.test(planned.group.shard_name) &&
|
|
(planned.group.includePatterns?.length ?? 0) > 32) ||
|
|
planned.group.includePatterns?.includes(HOSTED_MAIN_UPDATE_TEST));
|
|
const key = JSON.stringify([
|
|
sharesOrdinaryCapacity ? "ordinary" : planned.group.runner,
|
|
shard.requiresDist,
|
|
// Keep measured heavy children alone despite their stale packing estimates.
|
|
// Small Gateway tails and its runtime prerequisite keep ordinary packing.
|
|
dedicatedHostedMainGroup ? planned.group.shard_name : undefined,
|
|
]);
|
|
const groups = groupsByRunner.get(key);
|
|
if (groups) {
|
|
groups.push(planned.group);
|
|
} else {
|
|
groupsByRunner.set(key, [planned.group]);
|
|
}
|
|
// Exact child measurements replace parent projections. Changed membership
|
|
// keeps its parent floor until the new workload has an observation.
|
|
if (
|
|
selectedTooling ||
|
|
isParallelCommandsGroup(group) ||
|
|
planned.group.shard_name !== group.shard_name
|
|
) {
|
|
synthesizedSplitSeconds.set(compactGroupTimingKey(planned.group), planned.seconds);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Packing revisits immutable groups; prepare their cost and family once.
|
|
// Keep facts within this plan's timing inputs and partitions.
|
|
const stripeFacts = new Map<
|
|
NodeTestShardGroup,
|
|
{ seconds: number; family: string | undefined }
|
|
>();
|
|
const prepareStripe = (group: NodeTestShardGroup) => {
|
|
let facts = stripeFacts.get(group);
|
|
if (!facts) {
|
|
const seconds = Math.max(
|
|
synthesizedSplitSeconds.get(compactGroupTimingKey(group)) ?? 0,
|
|
estimateCompactStripeSeconds(group, options.runnerBackend),
|
|
storageModuleWorkFloor(group, options.runnerBackend),
|
|
);
|
|
facts = {
|
|
// Worker selection after packing may lower a price, never introduce a
|
|
// previously uncharged file floor or measured worker-specific wall.
|
|
seconds: isParallelCommandsGroup(group)
|
|
? Math.max(
|
|
seconds,
|
|
estimateCommandWorkerSeconds(
|
|
group,
|
|
seconds,
|
|
Math.min(
|
|
options.runnerBackend === "github" ? 2 : 8,
|
|
readPositiveEnvInt("OPENCLAW_VITEST_MAX_WORKERS", group.env ?? {}, 8),
|
|
),
|
|
options.runnerBackend,
|
|
).seconds,
|
|
)
|
|
: seconds,
|
|
family: compactStripeFamily(group),
|
|
};
|
|
stripeFacts.set(group, facts);
|
|
}
|
|
return facts;
|
|
};
|
|
const estimateStripeSeconds = (group: NodeTestShardGroup) => prepareStripe(group).seconds;
|
|
const estimateBinSeconds = (groups: NodeTestShardGroup[]) => {
|
|
const mode = mergeVitestPretestBuildModes(groups.map((group) => group.pretestBuildMode));
|
|
return (
|
|
groups.reduce((seconds, group) => seconds + estimateStripeSeconds(group), 0) +
|
|
compactPreparationSeconds(mode, options.runnerBackend)
|
|
);
|
|
};
|
|
const isHostedToolingGroup = (group: NodeTestShardGroup) =>
|
|
!group.requiresDist &&
|
|
group.pretestBuildMode === undefined &&
|
|
/^core-tooling-\d+-hosted-\d+$/u.test(group.shard_name) &&
|
|
group.configs.includes(TOOLING_CONFIG) &&
|
|
runnerRank(group) >= 0;
|
|
const hasDistinctStripeFamilies = (groups: NodeTestShardGroup[]) => {
|
|
const families = groups
|
|
.map((group) => prepareStripe(group).family)
|
|
.filter((family): family is string => family !== undefined);
|
|
return new Set(families).size === families.length;
|
|
};
|
|
const admitsCompactBin = (
|
|
groups: NodeTestShardGroup[],
|
|
secondsCap: number,
|
|
seconds = estimateBinSeconds,
|
|
{ sharedFamily = false, parallel = false } = {},
|
|
) =>
|
|
groups.length > 0 &&
|
|
(sharedFamily || hasDistinctStripeFamilies(groups)) &&
|
|
(parallel || groups.length <= COMPACT_NODE_TEST_JOB_GROUPS) &&
|
|
seconds(groups) <= secondsCap &&
|
|
(!parallel ||
|
|
estimateParallelTestSeconds(groups, estimateStripeSeconds) <=
|
|
COMPACT_SERIAL_NODE_TEST_JOB_SECONDS);
|
|
const usesBlacksmithCapacity = (runner: string) =>
|
|
isBlacksmithProfile ||
|
|
(options.runnerBackend === "hybrid" &&
|
|
[DEFAULT_NODE_TEST_RUNNER, BUNDLED_NODE_TEST_RUNNER, EXTRA_LARGE_NODE_TEST_RUNNER].includes(
|
|
runner,
|
|
));
|
|
const hostedToolingGroups: NodeTestShardGroup[] = [];
|
|
const maximumInitialBinSeconds = Math.max(
|
|
COMPACT_PARALLEL_NODE_TEST_JOB_SECONDS,
|
|
COMPACT_SERIAL_NODE_TEST_JOB_SECONDS,
|
|
COMPACT_HYBRID_SERIAL_CLI_JOB_SECONDS,
|
|
COMPACT_EXCLUSIVE_JOB_SECONDS,
|
|
COMPACT_EXPANDED_NODE_TEST_JOB_SECONDS,
|
|
COMPACT_LARGE_NODE_TEST_JOB_SECONDS,
|
|
COMPACT_SMALL_NODE_TEST_JOB_SECONDS,
|
|
);
|
|
let packedBins = [...groupsByRunner.values()].flatMap((groups) => {
|
|
const usesBlacksmithRunner = usesBlacksmithCapacity(groups[0].runner);
|
|
// Admit the final groups with their shared prerequisite. Rebalancing after
|
|
// this check can break build sharing and exceed a bin's admitted cap.
|
|
// Retry over-cap hybrid plans with Gateway first; leave successful cost-first
|
|
// placement and its runtime recipients intact.
|
|
const sortedGroups = groups
|
|
.flatMap((group) => expandCompactGroup(group, options.runnerBackend))
|
|
.toSorted(
|
|
(a, b) =>
|
|
(prioritizeSerialGateway
|
|
? Number(b.configs.some(isExclusiveCiTestConfig)) -
|
|
Number(a.configs.some(isExclusiveCiTestConfig))
|
|
: 0) ||
|
|
estimateBinSeconds([b]) - estimateBinSeconds([a]) ||
|
|
runnerRank(b) - runnerRank(a) ||
|
|
a.shard_name.localeCompare(b.shard_name),
|
|
);
|
|
// Hosted inventory must not influence non-hosted anchor membership.
|
|
const anchorGroups = packsHostedTooling
|
|
? sortedGroups.filter((group) => !isHostedToolingGroup(group))
|
|
: sortedGroups;
|
|
if (packsHostedTooling) {
|
|
hostedToolingGroups.push(...sortedGroups.filter(isHostedToolingGroup));
|
|
}
|
|
const canShareCompactJob = (
|
|
candidate: readonly [NodeTestShardGroup, ...NodeTestShardGroup[]],
|
|
group: NodeTestShardGroup,
|
|
) => {
|
|
const exclusive = isExclusiveCompactGroup(group);
|
|
if (isExclusiveCompactGroup(candidate[0]) !== exclusive) {
|
|
return false;
|
|
}
|
|
// Keep ordinary work off serial runtime hosts. Hybrid exclusive/dist bins
|
|
// retain their existing prerequisite sharing and admission policy.
|
|
if (
|
|
(isBlacksmithProfile || (usesBlacksmithRunner && !exclusive && !group.requiresDist)) &&
|
|
Boolean(candidate[0].pretestBuildMode) !== Boolean(group.pretestBuildMode)
|
|
) {
|
|
return false;
|
|
}
|
|
const combined = [...candidate, group];
|
|
const combinedSeconds = estimateBinSeconds(combined);
|
|
// Exchange candidates usually overflow before their finer policy matters.
|
|
// Every multi-group admission below has one of these bounded budgets.
|
|
if (combinedSeconds > maximumInitialBinSeconds) {
|
|
return false;
|
|
}
|
|
// Spend the larger budget only on a complete no-build CLI bin. Each child
|
|
// keeps its 150s admission limit, worker budget and separate process.
|
|
const sharesSerialCliBudget =
|
|
options.runnerBackend === "hybrid" &&
|
|
combined.every(
|
|
(entry) =>
|
|
!entry.requiresDist &&
|
|
!entry.pretestBuildMode &&
|
|
COMPACT_SERIAL_CLI_GROUP_NAME_RE.test(entry.shard_name) &&
|
|
estimateBinSeconds([entry]) <= COMPACT_EXCLUSIVE_JOB_SECONDS,
|
|
);
|
|
// Hosted preparation exceeds the exclusive test budget by itself. Share
|
|
// that build within the normal serial budget while keeping no-build work separate.
|
|
const sharesHostedBuild =
|
|
options.runnerBackend === "github" &&
|
|
combined.every((entry) => entry.pretestBuildMode !== undefined && !entry.requiresDist);
|
|
const serialSecondsCap = sharesSerialCliBudget
|
|
? COMPACT_HYBRID_SERIAL_CLI_JOB_SECONDS
|
|
: combined.every(isParallelToolingGroup)
|
|
? compactToolingTestSecondsLimit(combined, options.runnerBackend)
|
|
: exclusive && !sharesHostedBuild
|
|
? COMPACT_EXCLUSIVE_JOB_SECONDS
|
|
: usesExpandedRunnerProfile(options.runnerBackend)
|
|
? COMPACT_EXPANDED_NODE_TEST_JOB_SECONDS
|
|
: resolveCiNodeTestRunnerClass(group.runner).secondsCap;
|
|
const parallel =
|
|
usesBlacksmithRunner &&
|
|
combined.every(isParallelCompactGroup) &&
|
|
combined.every((entry) => estimateBinSeconds([entry]) <= serialSecondsCap);
|
|
const secondsCap = parallel
|
|
? combined.some((entry) => entry.configs.some(isExclusiveCiTestConfig))
|
|
? COMPACT_SERIAL_NODE_TEST_JOB_SECONDS
|
|
: COMPACT_PARALLEL_NODE_TEST_JOB_SECONDS
|
|
: serialSecondsCap;
|
|
return admitsCompactBin(combined, secondsCap, () => combinedSeconds, {
|
|
sharedFamily: sharesSerialCliBudget,
|
|
parallel,
|
|
});
|
|
};
|
|
const bins = packNodeTestGroups(anchorGroups, canShareCompactJob, true);
|
|
if (options.runnerBackend === "github") {
|
|
for (const bin of bins) {
|
|
bin.sort((a, b) => runnerRank(b) - runnerRank(a));
|
|
}
|
|
}
|
|
bins.sort(
|
|
(a, b) => Number(isExclusiveCompactGroup(a[0])) - Number(isExclusiveCompactGroup(b[0])),
|
|
);
|
|
return bins;
|
|
});
|
|
const canShareHostedGroup = (candidate: NodeTestShardGroup[], group: NodeTestShardGroup) => {
|
|
const owner = candidate[0];
|
|
const combined = [...candidate, group];
|
|
return (
|
|
owner !== undefined &&
|
|
((owner.runner === group.runner && owner.requiresDist === group.requiresDist) ||
|
|
(combined.every(isHostedToolingGroup) && runnerRank(owner) > runnerRank(group))) &&
|
|
isExclusiveCompactGroup(owner) === isExclusiveCompactGroup(group) &&
|
|
candidate.length < COMPACT_NODE_TEST_JOB_GROUPS &&
|
|
hasDistinctStripeFamilies(combined)
|
|
);
|
|
};
|
|
const hostedBinBudget = (groups: NodeTestShardGroup[]) =>
|
|
groups.every(isParallelToolingGroup)
|
|
? COMPACT_SERIAL_NODE_TEST_JOB_SECONDS
|
|
: COMPACT_EXCLUSIVE_JOB_SECONDS;
|
|
if (packsHostedTooling) {
|
|
const anchors = packedBins;
|
|
const hostedGroups = hostedToolingGroups.toSorted(
|
|
(a, b) =>
|
|
runnerRank(b) - runnerRank(a) ||
|
|
estimateBinSeconds([b]) - estimateBinSeconds([a]) ||
|
|
a.shard_name.localeCompare(b.shard_name),
|
|
);
|
|
const strongestGroupCount =
|
|
hostedGroups.findLastIndex((group) => group.runner === hostedGroups[0]!.runner) + 1;
|
|
type HostedUnit = [NodeTestShardGroup, ...NodeTestShardGroup[]];
|
|
const units = [
|
|
...hostedGroups.slice(0, strongestGroupCount).map((group) => [group]),
|
|
...anchors,
|
|
...hostedGroups.slice(strongestGroupCount).map((group) => [group]),
|
|
] as HostedUnit[];
|
|
const canShareHostedUnit = (
|
|
candidate: readonly [HostedUnit, ...HostedUnit[]],
|
|
unit: HostedUnit,
|
|
) => {
|
|
if (!isHostedToolingGroup(unit[0])) {
|
|
return false;
|
|
}
|
|
const groups = candidate.flat();
|
|
if (!canShareHostedGroup(groups, unit[0])) {
|
|
return false;
|
|
}
|
|
const combined = [...groups, ...unit];
|
|
return estimateBinSeconds(combined) <= hostedBinBudget(combined);
|
|
};
|
|
packedBins = packNodeTestGroups(units, canShareHostedUnit, true).map(
|
|
(bin) => bin.flat() as HostedUnit,
|
|
);
|
|
// Preserve successful plans; compare one alternate only after tail splitting still overflows.
|
|
if (splitHostedToolingTails && packedBins.length > effectiveJobCap(packedBins)) {
|
|
const alternateUnits = [
|
|
...anchors,
|
|
...hostedGroups
|
|
.toSorted(
|
|
(a, b) =>
|
|
estimateBinSeconds([b]) - estimateBinSeconds([a]) ||
|
|
runnerRank(b) - runnerRank(a) ||
|
|
a.shard_name.localeCompare(b.shard_name),
|
|
)
|
|
.map((group) => [group]),
|
|
] as HostedUnit[];
|
|
const alternateBins = packNodeTestGroups(alternateUnits, canShareHostedUnit, true).map(
|
|
(bin) => bin.flat() as HostedUnit,
|
|
);
|
|
if (alternateBins.length < packedBins.length) {
|
|
packedBins = alternateBins;
|
|
}
|
|
}
|
|
}
|
|
|
|
const compactJobs: CompactNodeTestShard[] = [];
|
|
const nextJobIndexByClass = new Map<string, number>();
|
|
for (const bin of packedBins) {
|
|
const [firstGroup] = bin;
|
|
const { name: runnerClass } = resolveCiNodeTestRunnerClass(firstGroup.runner);
|
|
const distSuffix = firstGroup.requiresDist ? "-dist" : "";
|
|
const jobClass = `${runnerClass}${distSuffix}`;
|
|
const jobIndex = (nextJobIndexByClass.get(jobClass) ?? 0) + 1;
|
|
nextJobIndexByClass.set(jobClass, jobIndex);
|
|
const checkName = `checks-node-compact-${jobClass}-${jobIndex}`;
|
|
const runner = firstGroup.runner;
|
|
const pretestBuildMode = mergeVitestPretestBuildModes(
|
|
bin.map((group) => group.pretestBuildMode),
|
|
);
|
|
// The runner admits overlap only after measuring capacity; exclusive and
|
|
// runtime-building jobs stay serial regardless of the requested class.
|
|
const planConcurrency =
|
|
usesBlacksmithCapacity(firstGroup.runner) &&
|
|
bin.length > 1 &&
|
|
bin.every(isParallelCompactGroup)
|
|
? 2
|
|
: 1;
|
|
// Tooling and the full CLI need host capacity while keeping serial isolation.
|
|
// Promote only the emitted runner so packing, names and timing keys stay stable.
|
|
const capacityRunner =
|
|
runner === EXTRA_LARGE_NODE_TEST_RUNNER ||
|
|
planConcurrency === 2 ||
|
|
(isBlacksmithProfile && bin.some((group) => group.configs.includes(TOOLING_CONFIG))) ||
|
|
(options.runnerBackend === "hybrid" &&
|
|
usesBlacksmithCapacity(runner) &&
|
|
bin.some((group) =>
|
|
group.includePatterns?.some((file) => TOOLING_LARGE_CAPACITY_TEST_FILES.has(file)),
|
|
)) ||
|
|
(usesBlacksmithCapacity(runner) &&
|
|
bin.some((group) => MEASURED_CLI_GROUP_RE.test(group.shard_name)))
|
|
? EXTRA_LARGE_NODE_TEST_RUNNER
|
|
: runner;
|
|
compactJobs.push({
|
|
checkName,
|
|
groups: bin,
|
|
...(pretestBuildMode ? { pretestBuildMode } : {}),
|
|
requiresDist: firstGroup.requiresDist,
|
|
runner: capacityRunner,
|
|
shardName: `compact-${jobClass}-${jobIndex}`,
|
|
// Whole-config groups run entire suites; keep their generous timeout.
|
|
...(bin.some((group) => !group.includePatterns)
|
|
? { timeoutMinutes: COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES }
|
|
: {}),
|
|
planConcurrency,
|
|
predictedSeconds: Math.ceil(estimateBinSeconds(bin)),
|
|
});
|
|
}
|
|
|
|
const compactJobCap = effectiveJobCap(packedBins);
|
|
if (packsHostedTooling && compactJobs.length > compactJobCap) {
|
|
if (!splitHostedToolingTails) {
|
|
// Repartition once at the file owner so timing identities and build costs
|
|
// describe the smaller tails before the same admission checks pack them.
|
|
return createCompactNodeTestShardBundles(
|
|
sourceShards,
|
|
options,
|
|
compactMode,
|
|
selectedToolingFiles,
|
|
true,
|
|
);
|
|
}
|
|
if (hostedToolingTailBudgets === undefined && hostedToolingTailDonation === undefined) {
|
|
// Repartition only stranded tails to fit capacity left by compatible owners.
|
|
// File ownership and timing identities are rebuilt before normal admission.
|
|
const tailBudgets = new Map<string, number>();
|
|
for (const group of packedBins.slice(compactJobCap).flat()) {
|
|
if (!isHostedToolingGroup(group) || (group.includePatterns?.length ?? 0) < 2) {
|
|
continue;
|
|
}
|
|
const available = Math.max(
|
|
0,
|
|
...packedBins
|
|
.filter((candidate) => canShareHostedGroup(candidate, group))
|
|
.map(
|
|
(candidate) => hostedBinBudget([...candidate, group]) - estimateBinSeconds(candidate),
|
|
),
|
|
);
|
|
if (available > 0) {
|
|
tailBudgets.set(group.shard_name.replace(/-hosted-\d+$/u, ""), available);
|
|
}
|
|
}
|
|
if (tailBudgets.size > 0) {
|
|
return createCompactNodeTestShardBundles(
|
|
sourceShards,
|
|
options,
|
|
compactMode,
|
|
selectedToolingFiles,
|
|
true,
|
|
tailBudgets,
|
|
);
|
|
}
|
|
}
|
|
if (bestTailDonation) {
|
|
// One largest safe donation opens donor capacity without changing the family selector.
|
|
return createCompactNodeTestShardBundles(
|
|
sourceShards,
|
|
options,
|
|
compactMode,
|
|
selectedToolingFiles,
|
|
true,
|
|
hostedToolingTailBudgets,
|
|
bestTailDonation,
|
|
);
|
|
}
|
|
}
|
|
|
|
// Settle Gateway admission before runtime placement reads the recipient's policy.
|
|
for (const job of compactJobs) {
|
|
if (
|
|
job.planConcurrency !== 2 ||
|
|
!job.groups.some((group) => group.configs.some(isExclusiveCiTestConfig))
|
|
) {
|
|
continue;
|
|
}
|
|
// Keep packed jobs and their summed time budgets; Gateway boots own the host
|
|
// serially. Preserve the previous two-worker ceiling during runtime placement.
|
|
job.planConcurrency = 1;
|
|
job.env = { ...job.env, ...PINNED_COMPACT_GROUP_ENV };
|
|
}
|
|
|
|
// Only the public complete-plan entry normalizes this option. Precise plans
|
|
// retain their original template capacity before projecting selected files.
|
|
if (options.runnerBackend === "hybrid" && options.compactMode !== undefined) {
|
|
const timings = readRuntimePlacementTimings("blacksmith");
|
|
const placementJobs = compactJobs.filter(
|
|
(job) =>
|
|
job.pretestBuildMode !== "private-qa" &&
|
|
!job.requiresDist &&
|
|
runnerRank(job) >= 0 &&
|
|
job.groups.every(
|
|
(group) => group.pretestBuildMode !== "private-qa" && !isExclusiveCompactGroup(group),
|
|
),
|
|
);
|
|
const measured = (group: NodeTestShardGroup) => resolveRuntimePlacementSeconds(group, timings);
|
|
if (placementJobs.some((job) => job.groups.some((group) => measured(group) !== undefined))) {
|
|
// Observe complete existing envelopes only after splitting/packing. These
|
|
// floors cannot feed a runtime cost back into ordinary stripe generation.
|
|
const cost = (groups: NodeTestShardGroup[]) =>
|
|
VITEST_PRETEST_BUILD_SECONDS.runtime +
|
|
groups.reduce(
|
|
(total, group) =>
|
|
total +
|
|
Math.max(
|
|
estimateStripeSeconds(group),
|
|
isParallelCommandsGroup(group) || measured(group) === undefined
|
|
? estimateCompactGroupSeconds(group, "hybrid")
|
|
: Math.round(measured(group)! * COMPACT_HYBRID_GROUP_SECONDS_SCALE),
|
|
),
|
|
0,
|
|
);
|
|
const admits = (groups: NodeTestShardGroup[]) =>
|
|
admitsCompactBin(groups, COMPACT_HYBRID_RUNTIME_JOB_SECONDS, cost);
|
|
const prepareRecipient = (job: CompactNodeTestShard) => {
|
|
if (job.planConcurrency !== 2) {
|
|
return job.groups;
|
|
}
|
|
if (
|
|
job.groups.some(
|
|
(group) =>
|
|
group.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined &&
|
|
!isParallelCommandsGroup(group) &&
|
|
!isRuntimePlacementIncludePatterns(group.includePatterns),
|
|
)
|
|
) {
|
|
return undefined;
|
|
}
|
|
// Preserve the executed allowance and its prepared timing identity.
|
|
// Re-keying an equivalent cap would break a complete parent generation.
|
|
return job.groups.map((group) =>
|
|
group.env?.OPENCLAW_VITEST_MAX_WORKERS !== undefined || isParallelCommandsGroup(group)
|
|
? group
|
|
: Object.assign({}, group, { env: { ...group.env, ...PINNED_COMPACT_GROUP_ENV } }),
|
|
);
|
|
};
|
|
rebalanceRuntimeTestJobs(placementJobs, { cost, admits, runnerRank, prepareRecipient });
|
|
}
|
|
}
|
|
|
|
for (const job of compactJobs) {
|
|
// Memory-gated plans need the measured 8-CPU/30.95-GiB allocation, even alone.
|
|
// Normalize the previous two-worker allowance onto their unmeasured siblings below.
|
|
if (
|
|
usesBlacksmithCapacity(job.runner) &&
|
|
job.runner !== EXTRA_LARGE_NODE_TEST_RUNNER &&
|
|
job.groups.some((group) => group.minTotalMemoryBytes !== undefined)
|
|
) {
|
|
job.runner = EXTRA_LARGE_NODE_TEST_RUNNER;
|
|
job.env = { ...job.env, ...PINNED_COMPACT_GROUP_ENV };
|
|
}
|
|
if (
|
|
job.env?.OPENCLAW_VITEST_MAX_WORKERS !== "2" ||
|
|
!job.groups.some((group) => usesMeasuredCompactWorkers(group, options.runnerBackend))
|
|
) {
|
|
continue;
|
|
}
|
|
// Finish placement before moving the job cap onto every unproven sibling,
|
|
// including donated runtime groups. Measured groups retain their own worker limits.
|
|
const { OPENCLAW_VITEST_MAX_WORKERS: _workers, ...env } = job.env;
|
|
job.env = Object.keys(env).length > 0 ? env : undefined;
|
|
job.groups = job.groups.map((group) =>
|
|
usesMeasuredCompactWorkers(group, options.runnerBackend)
|
|
? group
|
|
: Object.assign({}, group, {
|
|
env: {
|
|
...group.env,
|
|
OPENCLAW_VITEST_MAX_WORKERS: String(
|
|
Math.min(2, Number(group.env?.OPENCLAW_VITEST_MAX_WORKERS ?? 2)),
|
|
),
|
|
},
|
|
}),
|
|
);
|
|
}
|
|
|
|
// Larger initial bins change which groups reach serial Gateway/runtime rows.
|
|
// Freeze those settled rows; only already-overlapping jobs can share more work.
|
|
const parallelJobs: CompactNodeTestShard[] = [];
|
|
for (const job of compactJobs) {
|
|
if (
|
|
job.planConcurrency !== 2 ||
|
|
job.runner !== EXTRA_LARGE_NODE_TEST_RUNNER ||
|
|
!usesBlacksmithCapacity(job.runner) ||
|
|
job.pretestBuildMode ||
|
|
job.requiresDist ||
|
|
!job.groups.every(isParallelCompactGroup) ||
|
|
job.groups.some((group) => group.configs.some(isExclusiveCiTestConfig))
|
|
) {
|
|
continue;
|
|
}
|
|
parallelJobs.push(job);
|
|
}
|
|
const retiredJobs = new Set<CompactNodeTestShard>();
|
|
if (parallelJobs.length > 1) {
|
|
const groups = parallelJobs
|
|
.flatMap((job) => job.groups)
|
|
.toSorted(
|
|
(a, b) =>
|
|
estimateStripeSeconds(b) - estimateStripeSeconds(a) ||
|
|
a.shard_name.localeCompare(b.shard_name),
|
|
);
|
|
const bins = packNodeTestGroups(
|
|
groups,
|
|
(candidate, group) =>
|
|
admitsCompactBin(
|
|
[...candidate, group],
|
|
COMPACT_FINAL_PARALLEL_NODE_TEST_JOB_SECONDS,
|
|
estimateBinSeconds,
|
|
{ parallel: true },
|
|
),
|
|
options.runnerBackend === "hybrid",
|
|
);
|
|
if (bins.length < parallelJobs.length) {
|
|
parallelJobs.forEach((job, index) => {
|
|
const bin = bins[index];
|
|
if (!bin) {
|
|
retiredJobs.add(job);
|
|
return;
|
|
}
|
|
job.groups = bin;
|
|
job.predictedSeconds = Math.ceil(estimateBinSeconds(bin));
|
|
job.planConcurrency = bin.length > 1 ? 2 : 1;
|
|
job.timeoutMinutes = bin.some((group) => !group.includePatterns)
|
|
? COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES
|
|
: undefined;
|
|
if (bin.length === 1) {
|
|
// Losing a sibling must not increase this child's previous worker allowance.
|
|
job.env = { ...job.env, ...PINNED_COMPACT_GROUP_ENV };
|
|
}
|
|
});
|
|
}
|
|
}
|
|
const finalJobs = compactJobs.filter((job) => !retiredJobs.has(job));
|
|
for (const job of finalJobs) {
|
|
// The 4/8 classes both deliver two CPUs. Routing must not alter placement anchors.
|
|
if (usesBlacksmithCapacity(job.runner) && job.runner === BUNDLED_NODE_TEST_RUNNER) {
|
|
job.runner = DEFAULT_NODE_TEST_RUNNER;
|
|
}
|
|
}
|
|
|
|
// Split/packing admission retains the two-worker retry budget. Once placement
|
|
// settles, price commands at the allocation that the executor can actually use.
|
|
// The observed 32-class supplies 8 CPUs / 30.95 GiB; smaller classes and
|
|
// overlapping plans take H2's two-worker fallback. Live pressure can reduce it.
|
|
for (const job of finalJobs) {
|
|
const workers =
|
|
job.runner === EXTRA_LARGE_NODE_TEST_RUNNER &&
|
|
job.planConcurrency === 1 &&
|
|
job.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined &&
|
|
options.runnerBackend !== "github"
|
|
? 8
|
|
: 2;
|
|
let savedSeconds = 0;
|
|
job.groups = job.groups.map((group) => {
|
|
if (!isParallelCommandsGroup(group)) {
|
|
return group;
|
|
}
|
|
const previousSeconds = estimateStripeSeconds(group);
|
|
const adjusted = estimateCommandWorkerSeconds(
|
|
group,
|
|
previousSeconds,
|
|
Math.min(
|
|
workers,
|
|
readPositiveEnvInt("OPENCLAW_VITEST_MAX_WORKERS", group.env ?? {}, workers),
|
|
),
|
|
options.runnerBackend,
|
|
);
|
|
savedSeconds += previousSeconds - adjusted.seconds;
|
|
const prepared = { ...group, timing_key: adjusted.timingKey };
|
|
stripeFacts.set(prepared, { ...prepareStripe(group), seconds: adjusted.seconds });
|
|
return prepared;
|
|
});
|
|
job.predictedSeconds = Math.ceil(job.predictedSeconds! - savedSeconds);
|
|
}
|
|
|
|
const hostedHourly = options.runnerBackend === "github" && compactMode === "push";
|
|
const toolingFileTimings =
|
|
options.runnerBackend === "hybrid" || hostedHourly
|
|
? readToolingFileTimings("blacksmith")
|
|
: undefined;
|
|
const measuredJobs =
|
|
(options.runnerBackend === "hybrid" || hostedHourly) && options.compactMode !== undefined
|
|
? (hostedHourly
|
|
? [DEFAULT_NODE_TEST_RUNNER, BUNDLED_NODE_TEST_RUNNER]
|
|
: [DEFAULT_NODE_TEST_RUNNER]
|
|
).reduce(
|
|
(jobs, runner) =>
|
|
rebalanceMeasuredSerialJobs(jobs, {
|
|
runner,
|
|
profile: hostedHourly ? "hosted-hourly" : "native",
|
|
estimateGroup: (group) => ({
|
|
seconds: estimateParallelToolingSeconds(
|
|
group,
|
|
group.includePatterns ?? [],
|
|
hostedHourly ? "github" : "blacksmith",
|
|
toolingFileTimings,
|
|
),
|
|
complete: Boolean(
|
|
group.includePatterns?.every((file) => toolingFileTimings?.[file] !== undefined),
|
|
),
|
|
}),
|
|
...(hostedHourly
|
|
? {}
|
|
: {
|
|
testSecondsLimit: (groups: NodeTestShardGroup[]) =>
|
|
compactToolingTestSecondsLimit(groups, options.runnerBackend),
|
|
}),
|
|
canShare: (groups) =>
|
|
groups.length <= COMPACT_NODE_TEST_JOB_GROUPS && hasDistinctStripeFamilies(groups),
|
|
}),
|
|
finalJobs,
|
|
)
|
|
: finalJobs;
|
|
if (measuredJobs.length > compactJobCap) {
|
|
if (options.runnerBackend === "hybrid" && !prioritizeSerialGateway) {
|
|
// Rebuild from source so the alternate order passes every admission and
|
|
// runtime-placement check before any rows are published.
|
|
return createCompactNodeTestShardBundles(
|
|
sourceShards,
|
|
options,
|
|
compactMode,
|
|
selectedToolingFiles,
|
|
splitHostedToolingTails,
|
|
hostedToolingTailBudgets,
|
|
hostedToolingTailDonation,
|
|
true,
|
|
);
|
|
}
|
|
throw new Error(
|
|
`compact ${options.runnerBackend ?? "blacksmith"} node test plan exceeds ${compactJobCap} jobs (${measuredJobs.length} planned)`,
|
|
);
|
|
}
|
|
for (const job of measuredJobs) {
|
|
if (job.planConcurrency !== 2) {
|
|
job.predictedTestSeconds = Math.max(
|
|
0,
|
|
(job.predictedSeconds ?? 0) -
|
|
compactPreparationSeconds(job.pretestBuildMode, options.runnerBackend),
|
|
);
|
|
continue;
|
|
}
|
|
job.predictedTestSeconds = estimateParallelTestSeconds(job.groups, estimateStripeSeconds);
|
|
}
|
|
if (options.groupSeconds) {
|
|
for (const job of measuredJobs) {
|
|
const seconds = job.groups.map(estimateStripeSeconds);
|
|
const total = seconds.reduce((sum, value) => sum + value, 0);
|
|
const testSeconds = Math.max(
|
|
0,
|
|
job.predictedSeconds! -
|
|
compactPreparationSeconds(job.pretestBuildMode, options.runnerBackend),
|
|
);
|
|
job.groups.forEach((group, index) =>
|
|
options.groupSeconds!.set(
|
|
group,
|
|
total > 0 ? (testSeconds * seconds[index]!) / total : testSeconds / job.groups.length,
|
|
),
|
|
);
|
|
}
|
|
}
|
|
return measuredJobs.toSorted((a, b) => a.checkName.localeCompare(b.checkName));
|
|
}
|