vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-10-03 01:29:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 29

fix(ci): avoid overloaded jobs when runtime test groups grow (#147095)

Browse source 
* fix(ci): avoid overloaded jobs when runtime test groups grow

Retain measured workload descriptors across file additions and reuse existing ordinary capacity without adding jobs. Preserve timing identities and worker ceilings while moving complete runtime groups.

* test(ci): compare worker policies from original admission
This commit is contained in:
Peter Steinberger 2026-09-13 09:09:28 -07:00 • committed by GitHub
parent ca0746ad8c
commit 4b47d65d29
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 1010 additions and 202 deletions
Download patch file Download diff file Expand all files Collapse all files

29
.agents/skills/openclaw-ci-limits/SKILL.md
View file

@ -271,17 +271,26 @@ These are intentionally guarded by `test/scripts/ci-workflow-guards.test.ts`:
Complete ordinary hybrid bins containing only non-build CLI groups may use
250s and co-locate split siblings, provided each original child still fits
150s. Keep file splits, workers, process isolation and other profiles unchanged.
Runtime consumers in ordinary bins share preparation only with other consumers;
complete hybrid main and PR runtime-placement observations apply only after file
splitting. Precise changed-file templates retain their original capacity and floors.
Whole pinned groups may move between existing compatible serial runtime bins
under a 440s budget including the existing 100s build reserve. Preserve runner
anchors, all descriptors and invocation/generation counts; no additional jobs,
builds, worker limits or test deadlines. Reapply shared admission to both bins;
do not bypass a failed budget or count a runtime subset as a complete parent.
An unfit optimization keeps the complete runnable plan and its truthful estimate.
Initial packing separates runtime consumers from ordinary groups. Complete
hybrid main and PR runtime-placement observations apply only after file splitting;
precise changed-file templates retain their original capacity and floors.
Typed observations preserve configs, environment, complete files and build mode.
Prefer an exact measurement; otherwise use the maximum compatible contained
workload as an advisory floor, never sum overlaps or treat globs as whole files.
Whole pinned runtime groups may move to existing compatible ordinary jobs under
a 440s budget including the existing 100s build reserve. Keep runner anchors,
test partitions, invocation counts and worker limits. An ordinary recipient
becomes serial, explicitly retaining its old parallel groups' two-worker budget
while preserving their prepared timing identities and complete parent
generations. The CI executor applies the smaller of the
supplied job ceiling and group cap. This may add one runtime preparation while
reducing requested process slots; measure the tradeoff without adding jobs or
registrations. Equal maximum estimates prefer more recipient headroom.
Reapply shared family, group-count and budget admission to both replacements;
never suppress coverage or count a runtime subset as a complete parent.
An unfit optimization retains the runnable plan and its truthful estimate.
Compare recipients with the donor job's fixed anchor, not only its group class.
Other serial, exclusive, private-QA, dist and hosted policies stay unchanged.
Exclusive, private-QA, dist and hosted policies stay unchanged.
Affordable generated CLI runtime children may share one preparation in an
exclusive serial bin within the same 150s budget; fixed stripe families remain
separate. Other hybrid exclusive/dist sharing is unchanged. Complete inventories

148
config/ci-test-timings.json
View file

@ -144,21 +144,145 @@
"core-unit-src-security-2": 116,
"core-unit-src-security-3": 158,
"core-unit-src-security-support": 24,
"core-unit-support": 52,
"runtime-placement#2b1c0b8d48d8b50414fd067c349b2f47d081f400": 13,
"runtime-placement#2c4f0f682816a7ffb3ee8014fa8e16aa03a06ab4": 14,
"runtime-placement#5356616168e63778076bc126ea579625afa09dc8": 201,
"runtime-placement#75f68d733e4d254c846ecc193d2b1d729b44511b": 41,
"runtime-placement#7cc54c89f2c576111cfe9f8937c1e9f3c8876a7d": 12,
"runtime-placement#8096f17bd1f37f16ce0520c79b25fd55adb24bb4": 10,
"runtime-placement#964c8a634d056041fe627995ffe302457739a77d": 54,
"runtime-placement#9dc07ef8e9c6e5f9b68095a12c495d8ff00d7fd0": 265,
"runtime-placement#ac5a9db3ba50321154aeb563325047ee1fc5e62c": 79,
"runtime-placement#d8077c092e8ae6a94b867b0a5aec3ffe38662939": 43,
"runtime-placement#dedf4dae6cad3774bfcc936841002adf0fa2e5aa": 12
"core-unit-support": 52
},
"github": {}
},
"runtimePlacementTimings": {
"blacksmith": [
{
"configs": [
"test/vitest/vitest.agents-core.config.ts"
],
"env": {},
"includePatterns": [
"src/agents/prepared-model-catalog-worker.integration.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 41
},
{
"configs": [
"test/vitest/vitest.agents-core.config.ts"
],
"env": {},
"includePatterns": [
"src/agents/runtime-plugins.context-engine.integration.test.ts",
"src/agents/simple-completion-runtime.plugin-scope.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 12
},
{
"configs": [
"test/vitest/vitest.commands.config.ts"
],
"env": {},
"includePatterns": [
"src/commands/doctor-config-preflight.process.test.ts",
"src/commands/doctor-config-preflight.refusal.process.test.ts",
"src/commands/doctor-config-preflight.v17-atomicity.process.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 79
},
{
"configs": [
"test/vitest/vitest.commands.config.ts"
],
"env": {},
"includePatterns": [
"src/commands/doctor-plugin-install-config.process.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 43
},
{
"configs": [
"test/vitest/vitest.gateway-server.config.ts"
],
"env": {},
"includePatterns": [
"src/gateway/server-sidecar-retention.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 14
},
{
"configs": [
"test/vitest/vitest.gateway-server.config.ts"
],
"env": {},
"includePatterns": [
"src/gateway/server.config-patch.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 54
},
{
"configs": [
"test/vitest/vitest.infra.config.ts"
],
"env": {},
"includePatterns": [
"src/infra/update-candidate-canary.integration.test.ts",
"src/infra/update-managed-service-handoff-lifecycle.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 265
},
{
"configs": [
"test/vitest/vitest.plugin-sdk-light.config.ts",
"test/vitest/vitest.plugin-sdk.config.ts"
],
"env": {},
"includePatterns": [
"src/plugin-sdk/channel-entry-contract.lifecycle.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 10
},
{
"configs": [
"test/vitest/vitest.runtime-config.config.ts"
],
"env": {
"OPENCLAW_VITEST_MAX_WORKERS": "2"
},
"includePatterns": [
"src/config/config-startup-corpus.test.ts",
"src/config/state-startup-corpus.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 201
},
{
"configs": [
"test/vitest/vitest.unit-fast.config.ts"
],
"env": {},
"includePatterns": [
"src/plugins/setup-registry.migrations.test.ts",
"src/plugins/source-checkout-runtime.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 12
},
{
"configs": [
"test/vitest/vitest.unit-src.config.ts"
],
"env": {},
"includePatterns": [
"src/node-host/linux-node-plugin.integration.test.ts"
],
"pretestBuildMode": "runtime",
"seconds": 13
}
],
"github": []
},
"repoE2eFileSeconds": {
"extensions/browser/src/browser/client-fetch.attach-only.e2e.test.ts": 1,
"extensions/browser/src/browser/extension-install.native-host.e2e.test.ts": 2,

68
docs/ci/capacity.md
View file

@ -56,7 +56,7 @@ The shared plugin catch-all, QA and provider suites use native Vitest sharding,
Precise and fallback plugin envelopes share the same packing owner and a 240-second aggregate estimated budget per job, including multiple envelopes of the same config. Members retain compatible runner/dist requirements and run one at a time; total cost bounds packing rather than a pair limit. Each envelope retains its original child process, environment, native shard arguments and include scope, including process-bounded Codex, Matrix and Telegram work. Runtime-preparing envelopes remain separate. Co-location preserves each original file/process bound and native shard partition; a physical job may contain several such envelopes. Workers, timeouts and serial stop-on-failure behavior stay unchanged. Costs retain the larger complete-family rate from [run 33676780376](https://github.com/openclaw/openclaw/actions/runs/33676780376) and [run 33747183683](https://github.com/openclaw/openclaw/actions/runs/33747183683), rounded up per counting file without lowering prior floors. Both cohorts used two CPUs and two workers; counting inputs include the config-owned exclusions, and runtime preparation is charged separately. Repacking the retained 78 envelopes with these rates projects 30 jobs instead of 32. The largest sum of matching observed child spans is 340.128 seconds. This is a forecast across different source revisions, not measured combined-job latency; native CI must verify elapsed time and cleanup within the eight-minute end-to-end objective.
Eligible Blacksmith and hybrid compact bins with multiple ordinary groups request the existing 32-vCPU runner and two child-process slots. They admit 360 predicted aggregate seconds; compatible small groups can fill that budget without the ten-group cutoff retained by serial jobs. Runtime consumers in ordinary bins share preparation only with other consumers, keeping no-build groups on their own capacity. Blacksmith serial jobs retain their 200/276-second budgets; hybrid serial jobs retain 210 seconds. Exclusive jobs retain 150 seconds by default. Only complete ordinary hybrid bins of non-build CLI groups may use 250 seconds and share split siblings; every child must still fit 150 seconds. Groups above their existing serial cap stay alone. Exclusive groups, single groups, dist descriptors and jobs with runtime preparation remain serial. Hybrid exclusive and dist bins retain their existing prerequisite sharing. The shard executor admits at most two processes only when the actual host has at least eight available CPUs and 24 GiB of memory; smaller capacity admits one. Each overlapping child keeps two Vitest workers, inner project parallelism remains one, and commands retain their serial file policy. The primary `github` profile stays serial at 210 seconds. Preflight records the actual row count for each source revision; canonical inventory comparisons must preserve every original child plan and test input. Native elapsed-time, memory and cleanup evidence must establish the actual effect.
Eligible Blacksmith and hybrid compact bins with multiple ordinary groups request the existing 32-vCPU runner and two child-process slots. They admit 360 predicted aggregate seconds; compatible small groups can fill that budget without the ten-group cutoff retained by serial jobs. Initial packing separates runtime consumers from groups that need no build; the measured hybrid placement pass below can use spare ordinary capacity. Blacksmith serial jobs retain their 200/276-second budgets; hybrid serial jobs retain 210 seconds. Exclusive jobs retain 150 seconds by default. Only complete ordinary hybrid bins of non-build CLI groups may use 250 seconds and share split siblings; every child must still fit 150 seconds. Groups above their existing serial cap stay alone. Exclusive groups, single groups, dist descriptors and jobs with runtime preparation remain serial. Hybrid exclusive and dist bins retain their existing prerequisite sharing. The shard executor admits at most two processes only when the actual host has at least eight available CPUs and 24 GiB of memory; smaller capacity admits one. Each overlapping child keeps two Vitest workers, inner project parallelism remains one, and commands retain their serial file policy. The primary `github` profile stays serial at 210 seconds. Preflight records the actual row count for each source revision; canonical inventory comparisons must preserve every original child plan and test input. Native elapsed-time, memory and cleanup evidence must establish the actual effect.
The `github` compact planner can place smaller ordinary groups on an already-required stronger logical runner. Each emitted job retains its strongest capacity owner, serial execution, original child processes and worker limits, the 210-second budget, and the ten-group limit. SDK and plugin runtime consumers are separated from ordinary files before packing; the SDK light project intersects shared include lists with its own inventory. Parent-derived fractional costs are rounded only at the emitted job boundary, so rounding small consumers cannot create a redundant runtime build. The shared packer can exchange groups to fill compatible capacity while rechecking complete replacements. Runtime-only hosted groups may share their strongest prerequisite within the existing 210-second serial budget; the build itself costs 160–166 seconds and is charged once per job. No-build exclusive groups keep their 150-second limit. Stranded tooling tails are sized against remaining compatible capacity before normal admission is reapplied. Exclusive, dist, oversized, and runtime-preparing bins retain their separate constraints.
@ -86,31 +86,49 @@ Canonical-repo CI keeps Blacksmith as the default runner path for pushes and fir
## Measured shard weights
Complete hybrid main and pull-request runtime plans retain their existing jobs and
runner allocations while admitting measured runtime groups within 440 seconds, including the
existing 100-second build allowance. This reserves 40 seconds of the eight-minute
objective for checkout/setup; it does not change test deadlines or guarantee
elapsed time. Only non-exclusive ordinary-runtime bins participate. A group can
move to an already-required equal-or-stronger runner only with its own explicit
worker limit. Both replacement bins must pass the shared family, group-count and
budget checks. If no transfer fits, CI retains the complete existing plan and
reports its over-budget estimate; an optimization cannot suppress test coverage.
Recipient capacity must preserve the donor job's fixed runner anchor, including
any earlier promotion of that group. Private-QA, dist, exclusive, hosted,
and ordinary two-slot policies remain unchanged.
Precise changed-file plans retain their original placement templates and admission
floors; a whole-group measurement cannot move a subset onto a different runner.
Complete hybrid main and pull-request plans retain their existing jobs and runner
allocations while admitting measured runtime groups within 440 seconds, including
the existing 100-second build allowance. This reserves 40 seconds of the
eight-minute objective for checkout/setup; it does not change test deadlines or
guarantee elapsed time. Only non-exclusive jobs without dist or private-QA
preparation participate. A whole runtime group can move to an already-required
equal-or-stronger runner only with its own explicit worker limit.
The refit records separate runtime-placement observations from the emitted group
The recipient can already prepare a runtime or have spare ordinary capacity. An
ordinary recipient becomes serial and prepares one runtime; its previously
parallel groups retain their two-worker allowance through explicit pins. Their
prepared timing identities remain unchanged, preserving complete parent
generations. The CI executor applies the smaller of the supplied
job ceiling and group cap. When transfers have the same maximum estimate, the
planner prefers the recipient with more remaining time. This can add a runtime
build while reducing requested process concurrency, without adding a job or
runner registration. Count that preparation cost when verifying the result.
Both replacement bins must pass the shared family, group-count and budget
checks. If no transfer fits, CI retains the complete existing plan and reports
its over-budget estimate; an optimization cannot suppress test coverage.
Recipient capacity must preserve the donor job's fixed runner anchor, including
any earlier promotion of that group. Exclusive, private-QA, dist and hosted
policies remain unchanged. Precise changed-file plans retain their original
placement templates and admission floors.
The refit stores typed runtime-placement observations from the emitted group
descriptor, successful complete envelope and runtime-readiness marker. Configs,
group environment, exact include set and prebuild mode define their identity;
unrelated sibling repartitioning does not erase them. These observations use the
same independent-run sampling rules but are consumed only after file splitting.
They cannot reconstruct a parent or create more worker generations. Existing
parent and exact-child timing keys keep their original meaning. Preparation is
already included in the envelope; each job's shared runtime build is charged once.
Unknown groups retain positive fallback costs, and native same-inventory evidence
must verify latency, actual resources and cleanup before claiming improvement.
group environment, complete file membership and prebuild mode accompany each
measurement. An exact matching observation takes precedence. When it is absent,
the largest compatible observation whose files are all still present supplies an
advisory floor. This retains known work after a file is added without summing
overlapping observations, borrowing another environment's cost, or interpreting
directory/glob selectors as complete files. A later exact measurement can lower
the estimate after an optimization.
These observations use the same independent-run sampling rules and apply only
after file splitting. They do not reconstruct a full parent or change the test
partition. Existing parent and exact-child timing keys retain their meaning.
Wrapper preparation is already included in each observation; the separate shared
runtime build is charged once per job. Unknown groups retain positive fallback
costs. Native evidence with the same inventory must verify latency, actual
resources and cleanup before claiming improvement.
`config/ci-test-timings.json` records CI measurements for UI and Gateway E2E files
and compact Node groups. UI and compact packers prefer these weights over their in-source cold-start
@ -173,7 +191,7 @@ release observations, including run IDs, attempts, workflow SHAs, creation dates
parsed profiles and timing-job counts.
Fewer than two independent main compact contributors fails the invocation.
It also fails if no compact key meets the existing independent-run sampling rules.
It also fails if neither a compact key nor a runtime-placement observation meets the existing independent-run sampling rules.
Retained baseline weights and release measurements cannot satisfy these checks.
Both failures leave the timing file unchanged.
Measurements come only from successful UI E2E, Gateway E2E, and compact jobs; compact groups

5
scripts/ci-refit-test-timings.mts
View file

@ -265,9 +265,10 @@ async function main() {
const mainContributors = new Set([...blacksmith, ...github]);
if (
mainContributors.size < 2 ||
Object.values(fresh.timings.compactGroupSeconds).every(
(Object.values(fresh.timings.compactGroupSeconds).every(
(profile) => Object.keys(profile).length === 0,
)
) &&
Object.values(fresh.timings.runtimePlacementTimings).every((profile) => profile.length === 0))
) {
throw new Error(
`Found ${mainContributors.size} independent main compact contributors. Need at least two and a newly eligible compact measurement in the frozen UTC window; retry after successful main CI. No timing file written.`,

13
scripts/ci-run-node-test-shard.mts
View file

@ -129,7 +129,18 @@ function prepareChildEnv(entry: ShardPlan, baseEnv: NodeJS.ProcessEnv): NodeJS.P
}
for (const [key, value] of Object.entries(entry.plan.env ?? {})) {
if (typeof value === "string") {
childEnv[key] = value;
const inherited = baseEnv[key]?.trim();
// Pins may lower the admitted job budget, never raise it. Compiler
// preparation and test children must inherit the same intersection.
childEnv[key] =
key === "OPENCLAW_VITEST_MAX_WORKERS" && inherited
? String(
Math.min(
parsePositiveInt(inherited, key),
parsePositiveInt(value.trim() || inherited, key),
),
)
: value;
}
}
}

47
scripts/lib/ci-node-test-plan.mts
View file

@ -32,7 +32,12 @@ import {
} from "../../test/vitest/vitest.unit-paths.mjs";
import { buildVitestRunPlans, isTestFileTarget } from "../test-projects.test-support.mts";
import { rebalanceRuntimeTestJobs } from "./ci-runtime-test-placement.mts";
import { readCompactGroupTimings } from "./ci-test-timings.mts";
import { isRuntimePlacementIncludePatterns } from "./ci-test-timings-schema.mts";
import {
readCompactGroupTimings,
readRuntimePlacementTimings,
resolveRuntimePlacementSeconds,
} from "./ci-test-timings.mts";
import { listTrackedTestFiles } from "./list-test-files.mts";
import {
listVitestRuntimeConsumerFiles,
@ -46,7 +51,6 @@ import {
estimateVitestTestFileSeconds as stripeFileWeight,
estimateVitestToolingFileSeconds as toolingFileWeight,
parseCompactSplitTimingKey,
runtimePlacementTimingKey,
} from "./vitest-shard-metadata.mts";
export type NodeTestShardGroup = {
@ -3376,22 +3380,18 @@ function createCompactNodeTestShardBundles(
// Only the public complete-plan entry normalizes this option. Precise plans
// retain their original template capacity before projecting selected files.
if (options.runnerBackend === "hybrid" && options.compactMode !== undefined) {
const timings = readCompactGroupTimings("blacksmith");
const runtimeJobs = compactJobs.filter(
const timings = readRuntimePlacementTimings("blacksmith");
const placementJobs = compactJobs.filter(
(job) =>
job.pretestBuildMode === "runtime" &&
job.pretestBuildMode !== "private-qa" &&
!job.requiresDist &&
job.planConcurrency === 1 &&
runnerRank(job) >= 0 &&
job.groups.every(
(group) => group.pretestBuildMode === "runtime" && !isExclusiveCompactGroup(group),
(group) => group.pretestBuildMode !== "private-qa" && !isExclusiveCompactGroup(group),
),
);
const measured = (group: NodeTestShardGroup) => {
const key = runtimePlacementTimingKey(group);
return key === undefined ? undefined : timings[key];
};
if (runtimeJobs.some((job) => job.groups.some((group) => measured(group) !== undefined))) {
const measured = (group: NodeTestShardGroup) => resolveRuntimePlacementSeconds(group, timings);
if (placementJobs.some((job) => job.groups.some((group) => measured(group) !== undefined))) {
// Observe complete existing envelopes only after splitting/packing. These
// floors cannot feed a runtime cost back into ordinary stripe generation.
const cost = (groups: NodeTestShardGroup[]) =>
@ -3409,7 +3409,28 @@ function createCompactNodeTestShardBundles(
);
const admits = (groups: NodeTestShardGroup[]) =>
admitsCompactBin(groups, COMPACT_HYBRID_RUNTIME_JOB_SECONDS, cost);
rebalanceRuntimeTestJobs(runtimeJobs, { cost, admits, runnerRank });
const prepareRecipient = (job: CompactNodeTestShard) => {
if (job.planConcurrency !== 2) {
return job.groups;
}
if (
job.groups.some(
(group) =>
group.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined &&
!isRuntimePlacementIncludePatterns(group.includePatterns),
)
) {
return undefined;
}
// Preserve the executed allowance and its prepared timing identity.
// Re-keying an equivalent cap would break a complete parent generation.
return job.groups.map((group) =>
group.env?.OPENCLAW_VITEST_MAX_WORKERS !== undefined
? group
: Object.assign({}, group, { env: { ...group.env, ...PINNED_COMPACT_GROUP_ENV } }),
);
};
rebalanceRuntimeTestJobs(placementJobs, { cost, admits, runnerRank, prepareRecipient });
}
}

51
scripts/lib/ci-runtime-test-placement.mts
View file

@ -8,22 +8,34 @@ export function rebalanceRuntimeTestJobs(
cost,
admits,
runnerRank,
prepareRecipient,
}: {
cost: (groups: NodeTestShardGroup[]) => number;
admits: (groups: NodeTestShardGroup[]) => boolean;
runnerRank: (job: Pick<CompactNodeTestShard, "runner">) => number;
prepareRecipient: (job: CompactNodeTestShard) => NodeTestShardGroup[] | undefined;
},
) {
for (const donor of jobs.toSorted((a, b) => cost(b.groups) - cost(a.groups))) {
const donors = jobs.filter((job) => job.pretestBuildMode === "runtime");
for (const donor of donors.toSorted((a, b) => cost(b.groups) - cost(a.groups))) {
if (admits(donor.groups)) {
continue;
}
let best:
| { recipient: CompactNodeTestShard; group: NodeTestShardGroup; maximum: number }
| {
recipient: CompactNodeTestShard;
group: NodeTestShardGroup;
groups: NodeTestShardGroup[];
maximum: number;
recipientSeconds: number;
}
| undefined;
for (const group of donor.groups) {
// An inherited job allowance could increase on a different host.
if (group.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined) {
if (
group.pretestBuildMode !== "runtime" ||
group.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined
) {
continue;
}
const remaining = donor.groups.filter((entry) => entry !== group);
@ -35,22 +47,41 @@ export function rebalanceRuntimeTestJobs(
if (recipient === donor || runnerRank(recipient) < runnerRank(donor)) {
continue;
}
const combined = [...recipient.groups, group];
const maximum = Math.max(cost(remaining), cost(combined));
if (admits(combined) && (!best || maximum < best.maximum)) {
best = { recipient, group, maximum };
const prepared = prepareRecipient(recipient);
if (!prepared) {
continue;
}
const combined = [...prepared, group];
const recipientSeconds = cost(combined);
const maximum = Math.max(cost(remaining), recipientSeconds);
// When the retained donor dominates both choices, keep more receiver
// headroom instead of selecting whichever job happened to appear first.
if (
admits(combined) &&
(!best ||
maximum < best.maximum ||
(maximum === best.maximum &&
(recipientSeconds < best.recipientSeconds ||
(recipientSeconds === best.recipientSeconds &&
recipient.checkName.localeCompare(best.recipient.checkName) < 0))))
) {
best = { recipient, group, groups: combined, maximum, recipientSeconds };
}
}
}
if (best) {
const { recipient, group } = best;
const { recipient, group, groups } = best;
donor.groups = donor.groups.filter((entry) => entry !== group);
recipient.groups = [...recipient.groups, group];
recipient.groups = groups;
recipient.pretestBuildMode = "runtime";
recipient.planConcurrency = 1;
}
}
for (const job of jobs) {
// An over-budget unchanged plan is still runnable; estimates are not gates
// for test coverage. Only proposed replacements must satisfy admission.
job.predictedSeconds = Math.ceil(cost(job.groups));
if (job.pretestBuildMode === "runtime") {
job.predictedSeconds = Math.ceil(cost(job.groups));
}
}
}

91
scripts/lib/ci-test-timings-refit.mts
View file

@ -1,7 +1,12 @@
import { stripVTControlCharacters } from "node:util";
import { decodeNodeTestGroups } from "./ci-node-test-groups-codec.mts";
import type { CiTestTimings } from "./ci-test-timings-schema.mts";
import { parseCompactSplitTimingKey, runtimePlacementTimingKey } from "./vitest-shard-metadata.mts";
import {
isRuntimePlacementTiming,
runtimePlacementTimingIdentity,
type CiTestTimings,
type RuntimePlacementTiming,
} from "./ci-test-timings-schema.mts";
import { parseCompactSplitTimingKey } from "./vitest-shard-metadata.mts";
export type CiTimingRun = {
id: number;
@ -126,6 +131,8 @@ function readCompactLog(
text: string,
labels: string[],
samples: { blacksmith: Samples; github: Samples },
runtimeSamples: { blacksmith: Samples; github: Samples },
runtimeDescriptors: Map<string, RuntimePlacementTiming>,
) {
const profile = labels.some((label) => label.startsWith("blacksmith-")) ? "blacksmith" : "github";
const starts = new Map<string, number>();
@ -166,18 +173,37 @@ function readCompactLog(
// concurrent, and admission must use the wrapper span it actually ran.
recordSample(samples[profile], key, (Date.parse(timestamp) - started) / 1000);
const matches = descriptors.filter((group) => (group.timing_key ?? group.shard_name) === key);
const placementKey =
matches.length === 1
? runtimePlacementTimingKey({ ...matches[0]!, pretestBuildMode: runtimeModes.get(key) })
: undefined;
if (placementKey) {
recordSample(samples[profile], placementKey, (Date.parse(timestamp) - started) / 1000);
if (matches.length === 1) {
const group = matches[0]!;
const observation = {
configs: group.configs,
env: Object.fromEntries(
Object.entries(group.env ?? {}).toSorted(([a], [b]) => a.localeCompare(b)),
),
includePatterns: group.includePatterns.toSorted(),
pretestBuildMode: runtimeModes.get(key),
seconds: Math.max(1, Math.round((Date.parse(timestamp) - started) / 1000)),
};
if (isRuntimePlacementTiming(observation)) {
const identity = runtimePlacementTimingIdentity(observation);
runtimeDescriptors.set(identity, observation);
recordSample(runtimeSamples[profile], identity, (Date.parse(timestamp) - started) / 1000);
}
}
}
starts.delete(key);
}
}
function runtimePlacementSecondsMap(observations: readonly RuntimePlacementTiming[] = []) {
return Object.fromEntries(
observations.map((observation) => [
runtimePlacementTimingIdentity(observation),
observation.seconds,
]),
);
}
function recordCompleteParentSamples(samples: Samples, observedParents: Set<string>) {
const generations = new Map<
string,
@ -254,23 +280,48 @@ export function refitTestTimings(runs: CiTimingRun[], previous?: CiTestTimings)
};
const overhead: number[] = [];
const observedParents = { blacksmith: new Set<string>(), github: new Set<string>() };
const runtimeSamples = {
blacksmith: new Map<string, number[]>(),
github: new Map<string, number[]>(),
};
const runtimeDescriptors = new Map<string, RuntimePlacementTiming>(
Object.values(previous?.runtimePlacementTimings ?? {})
.flat()
.map((observation) => [runtimePlacementTimingIdentity(observation), observation]),
);
const uniqueRuns = new Map<number, CiTimingRun>();
for (const run of runs) {
const retained = uniqueRuns.get(run.id);
if (retained) {
retained.logs.push(...run.logs);
} else {
uniqueRuns.set(run.id, { ...run, logs: [...run.logs] });
}
}
for (const run of uniqueRuns.values()) {
const current = {
uiE2e: new Map<string, number[]>(),
repoE2e: new Map<string, number[]>(),
blacksmith: new Map<string, number[]>(),
github: new Map<string, number[]>(),
};
const currentRuntime = {
blacksmith: new Map<string, number[]>(),
github: new Map<string, number[]>(),
};
for (const log of run.logs) {
const text = stripVTControlCharacters(log.text);
if (log.kind === "compact") {
readCompactLog(text, log.labels, current);
readCompactLog(text, log.labels, current, currentRuntime, runtimeDescriptors);
} else {
readE2eLog(text, current[log.kind], log.kind === "uiE2e" ? overhead : undefined);
}
}
for (const profile of ["blacksmith", "github"] as const) {
recordCompleteParentSamples(current[profile], observedParents[profile]);
for (const [identity, values] of currentRuntime[profile]) {
recordSample(runtimeSamples[profile], identity, median(values));
}
}
// Retries or duplicate reporter lines in one run must not satisfy the two-run minimum.
for (const profile of ["uiE2e", "repoE2e", "blacksmith", "github"] as const) {
@ -291,6 +342,17 @@ export function refitTestTimings(runs: CiTimingRun[], previous?: CiTestTimings)
measuredOverhead === undefined ||
(oldOverhead !== undefined && Math.abs(measuredOverhead - oldOverhead) <= oldOverhead * 0.15);
const runIds = [...new Set(runs.map((run) => run.id))].toSorted((a, b) => a - b);
function refitRuntime(profile: "blacksmith" | "github"): RuntimePlacementTiming[] {
return Object.entries(
refitMap(
runtimeSamples[profile],
runtimePlacementSecondsMap(previous?.runtimePlacementTimings[profile]),
contributingRuns[profile].size,
),
).map(([identity, measuredSeconds]) =>
Object.assign({}, runtimeDescriptors.get(identity)!, { seconds: measuredSeconds }),
);
}
const timings: CiTestTimings = {
compactGroupSeconds: {
blacksmith: refitMap(
@ -311,6 +373,10 @@ export function refitTestTimings(runs: CiTimingRun[], previous?: CiTestTimings)
previous?.repoE2eFileSeconds,
contributingRuns.repoE2e.size,
),
runtimePlacementTimings: {
blacksmith: refitRuntime("blacksmith"),
github: refitRuntime("github"),
},
source: `median of ${runIds.length} successful CI and release-check runs: ${runIds.join(", ")}`,
uiE2e: {
fileSeconds: refitMap(
@ -333,6 +399,13 @@ export function refitTestTimings(runs: CiTimingRun[], previous?: CiTestTimings)
};
const changes: { key: string; old: number | undefined; next: number | undefined }[] = [];
const comparedMaps: [string, Record<string, number>, Record<string, number> | undefined][] = [
...(["blacksmith", "github"] as const).map(
(profile): [string, Record<string, number>, Record<string, number>] => [
`runtimePlacementTimings.${profile}`,
runtimePlacementSecondsMap(timings.runtimePlacementTimings[profile]),
runtimePlacementSecondsMap(previous?.runtimePlacementTimings[profile]),
],
),
[
"compactGroupSeconds.blacksmith",
timings.compactGroupSeconds.blacksmith,

84
scripts/lib/ci-test-timings-schema.mts
View file

@ -1,5 +1,28 @@
export type RuntimePlacementTiming = {
configs: string[];
env: Record<string, string>;
includePatterns: string[];
pretestBuildMode: "runtime" | "private-qa";
seconds: number;
};
export function runtimePlacementTimingIdentity(
group: Omit<RuntimePlacementTiming, "seconds">,
): string {
return JSON.stringify({
configs: group.configs,
env: Object.entries(group.env).toSorted(([a], [b]) => a.localeCompare(b)),
includePatterns: group.includePatterns.toSorted(),
pretestBuildMode: group.pretestBuildMode,
});
}
export type CiTestTimings = {
compactGroupSeconds: { blacksmith: Record<string, number>; github: Record<string, number> };
runtimePlacementTimings: {
blacksmith: RuntimePlacementTiming[];
github: RuntimePlacementTiming[];
};
repoE2eFileSeconds: Record<string, number>;
source: string;
uiE2e: { fileSeconds: Record<string, number>; perFileOverheadSeconds: number };
@ -36,11 +59,56 @@ function isSecondsMap(value: unknown): value is Record<string, number> {
);
}
function isNonemptyStrings(value: unknown): value is string[] {
return (
Array.isArray(value) &&
value.length > 0 &&
value.every((entry) => typeof entry === "string" && entry.length > 0)
);
}
export function isRuntimePlacementIncludePatterns(value: unknown): value is string[] {
return (
isNonemptyStrings(value) &&
// Match explicit test-target syntax without importing the test-project planner.
value.every(
(file) => /\.(?:test|spec)\.[cm]?[jt]sx?$/u.test(file) && !/[*?[\]{}]|[@+!]\(/u.test(file),
) &&
new Set(value).size === value.length
);
}
export function isRuntimePlacementTiming(value: unknown): value is RuntimePlacementTiming {
return (
isRecord(value) &&
hasExactKeys(value, ["configs", "env", "includePatterns", "pretestBuildMode", "seconds"]) &&
isNonemptyStrings(value.configs) &&
isRecord(value.env) &&
Object.entries(value.env).every(
([key, entry]) => key.length > 0 && typeof entry === "string",
) &&
isRuntimePlacementIncludePatterns(value.includePatterns) &&
(value.pretestBuildMode === "runtime" || value.pretestBuildMode === "private-qa") &&
typeof value.seconds === "number" &&
Number.isSafeInteger(value.seconds) &&
value.seconds > 0
);
}
function isRuntimePlacementTimings(value: unknown): value is RuntimePlacementTiming[] {
return (
Array.isArray(value) &&
value.every(isRuntimePlacementTiming) &&
new Set(value.map(runtimePlacementTimingIdentity)).size === value.length
);
}
function isCiTestTimings(value: unknown): value is CiTestTimings {
if (
!isRecord(value) ||
!hasExactKeys(value, [
"compactGroupSeconds",
"runtimePlacementTimings",
"repoE2eFileSeconds",
"source",
"uiE2e",
@ -50,7 +118,15 @@ function isCiTestTimings(value: unknown): value is CiTestTimings {
) {
return false;
}
const { compactGroupSeconds, repoE2eFileSeconds, source, uiE2e, updatedAt, version } = value;
const {
compactGroupSeconds,
runtimePlacementTimings,
repoE2eFileSeconds,
source,
uiE2e,
updatedAt,
version,
} = value;
return (
version === 1 &&
typeof source === "string" &&
@ -71,7 +147,11 @@ function isCiTestTimings(value: unknown): value is CiTestTimings {
isRecord(compactGroupSeconds) &&
hasExactKeys(compactGroupSeconds, ["blacksmith", "github"]) &&
isSecondsMap(compactGroupSeconds.blacksmith) &&
isSecondsMap(compactGroupSeconds.github)
isSecondsMap(compactGroupSeconds.github) &&
isRecord(runtimePlacementTimings) &&
hasExactKeys(runtimePlacementTimings, ["blacksmith", "github"]) &&
isRuntimePlacementTimings(runtimePlacementTimings.blacksmith) &&
isRuntimePlacementTimings(runtimePlacementTimings.github)
);
}

49
scripts/lib/ci-test-timings.mts
View file

@ -1,9 +1,15 @@
// Committed CI measurements are advisory weights, never a test inventory.
import fs from "node:fs";
import { ciTestTimingsSchema, type CiTestTimings } from "./ci-test-timings-schema.mts";
import {
ciTestTimingsSchema,
isRuntimePlacementIncludePatterns,
type CiTestTimings,
type RuntimePlacementTiming,
} from "./ci-test-timings-schema.mts";
const emptyUiTimings = { fileSeconds: {}, perFileOverheadSeconds: 0 };
const emptyGroupTimings: Readonly<Record<string, number>> = {};
const emptyRuntimeTimings: readonly RuntimePlacementTiming[] = [];
let cachedTimings: CiTestTimings | null | undefined;
let cachedReadFileSync: typeof fs.readFileSync | undefined;
@ -46,3 +52,44 @@ export function readCompactGroupTimings(
export function readRepoE2eFileTimings(): Readonly<Record<string, number>> {
return readTestTimings()?.repoE2eFileSeconds ?? emptyGroupTimings;
}
export function readRuntimePlacementTimings(
profile: "blacksmith" | "github",
): readonly RuntimePlacementTiming[] {
return readTestTimings()?.runtimePlacementTimings[profile] ?? emptyRuntimeTimings;
}
export function resolveRuntimePlacementSeconds(
group: {
configs: readonly string[];
env?: Readonly<Record<string, string>>;
includePatterns?: readonly string[];
pretestBuildMode?: "runtime" | "private-qa";
},
observations: readonly RuntimePlacementTiming[],
): number | undefined {
if (!group.pretestBuildMode || !isRuntimePlacementIncludePatterns(group.includePatterns)) {
return undefined;
}
const files = new Set(group.includePatterns);
const env = group.env ?? {};
let contained: number | undefined;
for (const observation of observations) {
if (
observation.pretestBuildMode !== group.pretestBuildMode ||
observation.configs.length !== group.configs.length ||
!observation.configs.every((config, index) => config === group.configs[index]) ||
Object.keys(observation.env).length !== Object.keys(env).length ||
!Object.entries(observation.env).every(([key, value]) => env[key] === value) ||
!observation.includePatterns.every((file) => files.has(file))
) {
continue;
}
// A newer exact measurement can be faster than an older contained workload.
if (observation.includePatterns.length === files.size) {
return observation.seconds;
}
contained = Math.max(contained ?? 0, observation.seconds);
}
return contained;
}

20
scripts/lib/vitest-shard-metadata.mts
View file

@ -9,26 +9,6 @@ export const VITEST_PRETEST_BUILD_SECONDS: Record<VitestPretestBuildMode, number
"private-qa": 104,
};
// Placement observations cannot become parent samples or influence file splitting.
// Configs own this identity: generated stripe names can change around the same readers.
export function runtimePlacementTimingKey(group: {
configs: readonly string[];
env?: Readonly<Record<string, string>>;
includePatterns?: readonly string[];
pretestBuildMode?: VitestPretestBuildMode;
}): string | undefined {
if (!group.pretestBuildMode || !group.includePatterns?.length) {
return undefined;
}
const identity = JSON.stringify({
configs: group.configs,
env: Object.entries(group.env ?? {}).toSorted(([a], [b]) => a.localeCompare(b)),
includePatterns: group.includePatterns.toSorted(),
pretestBuildMode: group.pretestBuildMode,
});
return `runtime-placement#${createHash("sha1").update(identity).digest("hex")}`;
}
export type VitestShardTimingSpec = {
config: string;
env?: NodeJS.ProcessEnv;

208
test/scripts/ci-node-test-plan.test.ts
View file

@ -19,6 +19,7 @@ import {
packNodeTestGroups,
resolvePolicyTestTargets,
} from "../../scripts/lib/ci-node-test-plan.mts";
import { isRuntimePlacementIncludePatterns } from "../../scripts/lib/ci-test-timings-schema.mts";
import * as testTimings from "../../scripts/lib/ci-test-timings.mts";
import { listVitestRuntimeConsumerFiles } from "../../scripts/lib/vitest-build-prerequisites.mts";
import {
@ -770,6 +771,7 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
);
it("keeps hybrid fallback bounds and unmeasured stripes when other measurements change", () => {
vi.spyOn(testTimings, "readRuntimePlacementTimings").mockReturnValue([]);
const timings = vi.spyOn(testTimings, "readCompactGroupTimings").mockReturnValue({});
const options = {
includeReleaseOnlyPluginShards: false,
@ -1211,6 +1213,15 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
compactMode: "pull-request",
runnerBackend: "hybrid",
});
const placementTimings = vi
.spyOn(testTimings, "readRuntimePlacementTimings")
.mockReturnValue([]);
const hybridBeforePlacement = createNodeTestShardBundles({
includeReleaseOnlyPluginShards: false,
compactMode: "pull-request",
runnerBackend: "hybrid",
});
placementTimings.mockRestore();
const expectedToolingOwnerNames = Array.from(
{ length: 16 },
(_, index) => `core-tooling-${index + 1}`,
@ -1458,7 +1469,34 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
expect(exclusiveCount).toBe(shard.groups.length);
expect(shard.planConcurrency).toBe(1);
}
if (!githubPullRequestCompact.includes(shard) && !exclusiveCount && !shard.requiresDist) {
const originalHybridJob = hybridPullRequestCompact.includes(shard)
? expectDefined(
hybridBeforePlacement.find((job) => job.checkName === shard.checkName),
"original hybrid runner anchor",
)
: undefined;
const promoted = originalHybridJob?.planConcurrency === 2 && shard.planConcurrency === 1;
if (promoted) {
expect(shard.pretestBuildMode).toBe("runtime");
expect(exclusiveCount).toBe(0);
expect(shard.requiresDist).toBe(false);
for (const original of originalHybridJob.groups) {
const retained = expectDefined(
shard.groups.find((group) => group.shard_name === original.shard_name),
"retained ordinary group",
);
expect(retained).toEqual({
...original,
env: { OPENCLAW_VITEST_MAX_WORKERS: "2", ...original.env },
});
}
}
if (
!githubPullRequestCompact.includes(shard) &&
!exclusiveCount &&
!shard.requiresDist &&
!promoted
) {
expect(
shard.groups.every(
(group) => Boolean(group.pretestBuildMode) === Boolean(shard.pretestBuildMode),
@ -1482,11 +1520,13 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
!githubPullRequestCompact.includes(shard) &&
shard.groups.some((group) => group.shard_name === "agentic-cli");
expect(shard.runner).toBe(
blacksmithTooling || shard.groups[0]?.runner === EXTRA_LARGE_NODE_TEST_RUNNER
? EXTRA_LARGE_NODE_TEST_RUNNER
: nativeFullCli
? "blacksmith-16vcpu-ubuntu-2404"
: shard.groups[0]?.runner,
originalHybridJob
? originalHybridJob.runner
: blacksmithTooling || shard.groups[0]?.runner === EXTRA_LARGE_NODE_TEST_RUNNER
? EXTRA_LARGE_NODE_TEST_RUNNER
: nativeFullCli
? "blacksmith-16vcpu-ubuntu-2404"
: shard.groups[0]?.runner,
);
}
}
@ -3574,11 +3614,14 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
includeReleaseOnlyPluginShards: false,
runnerBackend,
};
const changedOptions = { ...options, changedPaths: STORE_ALIAS_CHANGED_PATHS };
// Each input has its own admission before runtime placement materializes caps.
const observations = vi.spyOn(testTimings, "readRuntimePlacementTimings").mockReturnValue([]);
const beforeAdmission = createNodeTestShardBundles(options);
const afterAdmission = createNodeTestShardBundles(changedOptions);
observations.mockRestore();
const before = createNodeTestShardBundles(options);
const after = createNodeTestShardBundles({
...options,
changedPaths: STORE_ALIAS_CHANGED_PATHS,
});
const after = createNodeTestShardBundles(changedOptions);
const groups = after.flatMap((shard) => shard.groups);
expect(groups.filter((group) => group.shard_name === "agentic-plugins")).toEqual([
{
@ -3615,17 +3658,15 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
parseCompactSplitTimingKey(group.timing_key),
"parsed compact split timing key",
);
const name = expectDefined(
/^(.+)-hosted-([1-9]\d*)$/u.exec(group.shard_name),
"numbered hosted group name",
const parent = key.parentShardName;
const part = key.part;
const hostedName = /^(.+)-hosted-([1-9]\d*)$/u.exec(group.shard_name);
expect(parent, "timing key parent must match hosted group name").toBe(
hostedName?.[1] ?? group.shard_name,
);
expect(part, "timing key part must match hosted group ordinal").toBe(
hostedName ? Number(hostedName[2]) : 1,
);
const parent = expectDefined(name[1], "hosted group parent");
const part = Number(name[2]);
expect(
key.selectorKey.split("#selector-")[0],
"timing key parent must match hosted group name",
).toBe(parent);
expect(key.part, "timing key part must match hosted group ordinal").toBe(part);
const family = families.get(parent) ?? [];
family.push({ group, part });
families.set(parent, family);
@ -3635,42 +3676,83 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
}
return families;
};
const inheritedGroupsFor = (admission: typeof before) =>
new Map(
admission.flatMap((job) =>
job.planConcurrency === 2
? job.groups
.filter((group) => group.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined)
.map((group): [string, Group] => [group.shard_name, group])
: [],
),
);
const beforeInherited = inheritedGroupsFor(beforeAdmission);
const afterInherited = inheritedGroupsFor(afterAdmission);
const declarationEnv = (
group: Pick<Group, "shard_name" | "env">,
plan: typeof before,
originals: Map<string, Group>,
) => {
const original = originals.get(group.shard_name);
const job = plan.find((entry) =>
entry.groups.some((candidate) => candidate.shard_name === group.shard_name),
);
if (original && job?.planConcurrency === 1) {
const env = expectDefined(group.env, "materialized serial worker cap");
expect(env.OPENCLAW_VITEST_MAX_WORKERS).toBe("2");
const { OPENCLAW_VITEST_MAX_WORKERS: _workers, ...otherEnv } = env;
expect(otherEnv).toEqual(original.env ?? {});
return original.env;
}
return group.env;
};
const expectedTimingKeys = (
parent: string,
family: Array<{ group: Group; part: number }>,
plan: typeof before,
originals: Map<string, Group>,
) => {
const first = expectDefined(family[0], "first timing family entry").group;
const env = declarationEnv(first, plan, originals);
return createCompactSplitTimingGeneration({
parentShardName: parent,
configs: first.configs,
env: first.env,
env,
stripes: family.map(({ group }) => {
expect(group.configs).toEqual(first.configs);
expect(group.env).toEqual(first.env);
expect(declarationEnv(group, plan, originals)).toEqual(env);
const files = expectDefined(group.includePatterns, "timing family group membership");
expect(files.length).toBeGreaterThan(0);
return files;
}),
}).timingKeys;
};
const expectTimingFamilies = (plan: typeof before) => {
const expectTimingFamilies = (plan: typeof before, originals: Map<string, Group>) => {
for (const [parent, family] of timingFamilies(plan)) {
expect(family.map(({ part }) => part)).toEqual(
Array.from({ length: family.length }, (_, index) => index + 1),
);
expect(family.map(({ group }) => group.timing_key)).toEqual(
expectedTimingKeys(parent, family),
expectedTimingKeys(parent, family, plan, originals),
);
}
};
const policies = (plan: typeof before) => {
const policies = (plan: typeof before, originals: Map<string, Group>) => {
const nonPlugin = plan
.flatMap((shard) => shard.groups)
.filter((group) => group.shard_name !== "agentic-plugins");
return {
descriptors: nonPlugin
.filter((group) => !isRepartitionableTooling(group))
.map(({ runner: _runner, ...group }) => group)
.map(({ runner: _runner, ...group }) => {
const env = declarationEnv(group, plan, originals);
if (env === undefined) {
delete group.env;
} else {
group.env = env;
}
return group;
})
.toSorted((a, b) => a.shard_name.localeCompare(b.shard_name)),
// Allocation may change, but every file must retain its complete execution policy.
tooling: nonPlugin
@ -3699,13 +3781,61 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
.toSorted((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
};
};
expectTimingFamilies(before);
expectTimingFamilies(after);
expect(policies(after)).toEqual(policies(before));
expectTimingFamilies(before, beforeInherited);
expectTimingFamilies(after, afterInherited);
expect(policies(after, afterInherited)).toEqual(policies(before, beforeInherited));
if (runnerBackend === "hybrid") {
const promoted = structuredClone(before);
const recipient = expectDefined(
promoted.find(
(job) =>
job.planConcurrency === 2 &&
job.groups.some(
(group) => group.timing_key && group.env?.OPENCLAW_VITEST_MAX_WORKERS === undefined,
) &&
job.groups.every(
(group) =>
group.env?.OPENCLAW_VITEST_MAX_WORKERS !== undefined ||
isRuntimePlacementIncludePatterns(group.includePatterns),
),
),
"existing parallel recipient with hosted timing parts",
);
recipient.planConcurrency = 1;
recipient.pretestBuildMode = "runtime";
const keys = recipient.groups.map((group) => group.timing_key);
for (const group of recipient.groups) {
group.env = { OPENCLAW_VITEST_MAX_WORKERS: "2", ...group.env };
}
expectTimingFamilies(promoted, beforeInherited);
expect(policies(promoted, beforeInherited)).toEqual(policies(before, beforeInherited));
expect(recipient.groups.map((group) => group.timing_key)).toEqual(keys);
const hosted = expectDefined(
recipient.groups.find(
(group) => group.timing_key && beforeInherited.has(group.shard_name),
),
"hosted recipient group",
);
const original = expectDefined(
beforeInherited.get(hosted.shard_name),
"original parallel group",
);
for (const env of [original.env, { ...original.env, OPENCLAW_VITEST_MAX_WORKERS: "3" }]) {
if (env === undefined) {
delete hosted.env;
} else {
hosted.env = env;
}
expect(() => expectTimingFamilies(promoted, beforeInherited)).toThrow();
expect(() =>
expect(policies(promoted, beforeInherited)).toEqual(policies(before, beforeInherited)),
).toThrow();
}
}
if (runnerBackend === "github") {
const regenerateTimingKeys = (plan: typeof before) => {
for (const [parent, family] of timingFamilies(plan)) {
const keys = expectedTimingKeys(parent, family);
const keys = expectedTimingKeys(parent, family, plan, afterInherited);
family.forEach(({ group }, index) => {
group.timing_key = expectDefined(keys[index], "regenerated family timing key");
});
@ -3749,9 +3879,10 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
: BUNDLED_NODE_TEST_RUNNER;
}
regenerateTimingKeys(mutated);
expectTimingFamilies(mutated);
expectTimingFamilies(mutated, afterInherited);
expect(
() => expect(policies(mutated)).toEqual(policies(before)),
() =>
expect(policies(mutated, afterInherited)).toEqual(policies(before, beforeInherited)),
`${mutation} must fail policy validation even with valid timing keys`,
).toThrow();
}
@ -3765,11 +3896,16 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
const wrongParent = `${parent}-forged`;
expect(families.has(wrongParent)).toBe(false);
const ordered = identity === "part" ? family.toReversed() : family;
const keys = expectedTimingKeys(identity === "parent" ? wrongParent : parent, ordered);
const keys = expectedTimingKeys(
identity === "parent" ? wrongParent : parent,
ordered,
forged,
afterInherited,
);
ordered.forEach(({ group }, index) => {
group.timing_key = expectDefined(keys[index], "forged family timing key");
});
expect(() => expectTimingFamilies(forged)).toThrow(
expect(() => expectTimingFamilies(forged, afterInherited)).toThrow(
identity === "parent"
? "timing key parent must match hosted group name"
: "timing key part must match hosted group ordinal",
@ -3789,10 +3925,10 @@ describe("scripts/lib/ci-node-test-plan.mts", () => {
const savedKey = expectDefined(staleGroup.timing_key, "original stale-control timing key");
expectDefined(staleGroup.includePatterns, "stale-control group membership").pop();
regenerateTimingKeys(stale);
expectTimingFamilies(stale);
expectTimingFamilies(stale, afterInherited);
staleGroup.timing_key = savedKey;
expect(
() => expectTimingFamilies(stale),
() => expectTimingFamilies(stale, afterInherited),
"stale generation key must fail identity",
).toThrow();
}

41
test/scripts/ci-run-node-test-shard.test.ts
View file

@ -25,6 +25,7 @@ import {
} from "../../scripts/ci-run-node-test-shard.mts";
import { encodeNodeTestGroups } from "../../scripts/lib/ci-node-test-groups-codec.mts";
import { refitTestTimings } from "../../scripts/lib/ci-test-timings-refit.mts";
import { resolveLocalVitestScheduling } from "../../scripts/lib/vitest-local-scheduling.mts";
import * as groupOwner from "../../scripts/vitest-process-group.mts";
import { createDeferred } from "../helpers/promise.js";
@ -166,6 +167,46 @@ describe("scripts/ci-run-node-test-shard.mts", () => {
expect(bare.OPENCLAW_VITEST_INCLUDE_FILE).toBeUndefined();
});
it.each([
{ job: "1", group: "2", expected: 1 },
{ job: "1", group: "", expected: 1 },
{ job: "1", group: " ", expected: 1 },
{ job: "2", group: "2", expected: 2 },
{ job: "3", group: "2", expected: 2 },
{ job: "4", group: "2", expected: 2 },
{ job: "6", group: "2", expected: 2 },
{ job: "6", group: "1", expected: 1 },
{ job: undefined, group: "2", expected: 2 },
{ job: "6", group: undefined, expected: 6 },
{ job: "1", group: undefined, expected: 1, target: true },
])(
"intersects inherited worker ceiling $job with group cap $group (target=$target)",
({ job, group, expected, target }) => {
const childEnv = buildChildEnv(
target
? { kind: "target", name: "one", target: "one.test.ts" }
: {
kind: "group",
name: "one",
plan: {
configs: ["one.config.ts"],
env: { OPENCLAW_VITEST_MAX_WORKERS: group, EXTRA: "group" },
},
},
{ CI: "true", OPENCLAW_VITEST_MAX_WORKERS: job, EXTRA: "job" },
makeScratchDir(),
0,
);
expect(childEnv.OPENCLAW_VITEST_MAX_WORKERS).toBe(String(expected));
expect(childEnv.EXTRA).toBe(target ? "job" : "group");
expect(resolveLocalVitestScheduling(childEnv, { cpuCount: Number(job) || 8 })).toEqual({
maxWorkers: expected,
fileParallelism: expected > 1,
throttledBySystem: false,
});
},
);
it.each([
{ key: "NODE_OPTIONS", shared: true },
{ key: "NODE_OPTIONS", shared: false },

357
test/scripts/ci-test-timings.test.ts
View file

@ -26,12 +26,10 @@ import { refitTestTimings, type CiTimingRun } from "../../scripts/lib/ci-test-ti
import {
ciTestTimingsSchema,
type CiTestTimings,
type RuntimePlacementTiming,
} from "../../scripts/lib/ci-test-timings-schema.mts";
import * as testTimings from "../../scripts/lib/ci-test-timings.mts";
import {
createCompactSplitTimingGeneration,
runtimePlacementTimingKey,
} from "../../scripts/lib/vitest-shard-metadata.mts";
import { createCompactSplitTimingGeneration } from "../../scripts/lib/vitest-shard-metadata.mts";
import { fullSuiteVitestShards } from "../vitest/vitest.test-shards.mjs";
function uiLog(files: Record<string, number>, overhead = 0.6) {
@ -64,6 +62,7 @@ function compactLog(seconds: number, key = "core-unit-src-security-2") {
const measuredFile = "ui/src/e2e/measured.e2e.test.ts";
const baseline: CiTestTimings = {
compactGroupSeconds: { blacksmith: {}, github: {} },
runtimePlacementTimings: { blacksmith: [], github: [] },
repoE2eFileSeconds: {},
source: "median of 2 successful main CI runs: 1, 2",
uiE2e: { fileSeconds: { [measuredFile]: 100 }, perFileOverheadSeconds: 0.6 },
@ -74,13 +73,121 @@ const baseline: CiTestTimings = {
const sampleNow = "2026-08-28T12:00:00.000Z";
describe("runtime placement observations", () => {
it("retains recorded runtime work when its current group gains a file", () => {
const options = {
compactMode: "push" as const,
runnerBackend: "hybrid",
includeReleaseOnlyPluginShards: false,
};
const groups = createNodeTestShardBundles(options).flatMap((job) => job.groups);
const corpusFile = "src/config/state-startup-corpus.test.ts";
const handoffFile = "src/infra/update-managed-service-handoff-lifecycle.test.ts";
const corpus = groups.find((group) => group.includePatterns?.includes(corpusFile))!;
const handoff = groups.find((group) => group.includePatterns?.includes(handoffFile))!;
expect(corpus.includePatterns!.length).toBeGreaterThan(1);
const observations = [
{ ...corpus, includePatterns: [corpusFile], seconds: 200 },
{ ...handoff, seconds: 300 },
];
const runs = [1, 2].map((id) =>
timingRun(
id,
observations.map(({ seconds, ...group }, index) => {
const descriptor = {
...group,
shard_name: `recorded-runtime-${index}`,
timing_key: undefined,
};
const [begin, end] = compactLog(seconds, descriptor.shard_name).split("\n");
return {
kind: "compact" as const,
labels: ["blacksmith-4vcpu-ubuntu-2404"],
text: [
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor])}`,
begin,
`2026-08-27T23:00:01Z [shard:${descriptor.shard_name}] [test] preparing runtime runtime before Vitest workers`,
end,
].join("\n"),
};
}),
),
);
const data = refitTestTimings(runs).timings;
const original = fs.readFileSync;
const timingPath = fileURLToPath(new URL("../../config/ci-test-timings.json", import.meta.url));
const read = vi
.spyOn(fs, "readFileSync")
.mockImplementation((file, readOptions) =>
(file instanceof URL ? fileURLToPath(file) : file) === timingPath
? JSON.stringify(data)
: original(file, readOptions),
);
syncBuiltinESMExports();
try {
const plan = createNodeTestShardBundles(options);
const corpusJob = plan.find((job) =>
job.groups.some((group) => group.includePatterns?.includes(corpusFile)),
);
const handoffJob = plan.find((job) =>
job.groups.some((group) => group.includePatterns?.includes(handoffFile)),
);
expect(corpusJob).toBeDefined();
expect(handoffJob).toBeDefined();
// These recorded workloads exceed the shared 440s budget, including
// preparation. Added files must not make the known reader appear cheap.
expect(corpusJob).not.toBe(handoffJob);
} finally {
read.mockRestore();
syncBuiltinESMExports();
}
});
const reader = {
configs: ["test/vitest/reader.config.ts"],
env: { OPENCLAW_VITEST_MAX_WORKERS: "2" },
includePatterns: ["src/reader.test.ts"],
pretestBuildMode: "runtime" as const,
};
const key = runtimePlacementTimingKey(reader)!;
it.each([
{ label: "added files", files: ["a.test.ts", "b.test.ts", "new.test.ts"], expected: 201 },
{ label: "same files reordered", files: ["b.test.ts", "a.test.ts"], expected: 201 },
{ label: "current subset", files: ["a.test.ts"], expected: undefined },
{ label: "different files", files: ["a.test.ts", "other.test.ts"], expected: undefined },
{ label: "glob selection", files: ["*.test.ts"], expected: undefined },
])("matches complete recorded workloads with $label", ({ files, expected }) => {
expect(
testTimings.resolveRuntimePlacementSeconds({ ...reader, includePatterns: files }, [
{ ...reader, includePatterns: ["a.test.ts", "b.test.ts"], seconds: 201 },
]),
).toBe(expected);
});
it("uses one contained estimate and lets a later exact measurement replace it", () => {
const current = { ...reader, includePatterns: ["a.test.ts", "b.test.ts", "c.test.ts"] };
const prior = [
{ ...reader, includePatterns: ["a.test.ts", "b.test.ts"], seconds: 201 },
{ ...reader, includePatterns: ["b.test.ts", "c.test.ts"], seconds: 180 },
];
expect(testTimings.resolveRuntimePlacementSeconds(current, prior)).toBe(201);
for (const observations of [
[...prior, { ...current, seconds: 100 }],
[{ ...current, seconds: 100 }, ...prior],
]) {
expect(testTimings.resolveRuntimePlacementSeconds(current, observations)).toBe(100);
}
});
it.each<Partial<Parameters<typeof testTimings.resolveRuntimePlacementSeconds>[0]>>([
{ configs: ["other.config.ts"] },
{ env: {} },
{ env: { OPENCLAW_VITEST_MAX_WORKERS: "3" } },
{ pretestBuildMode: "private-qa" as const },
])("does not borrow a contained observation across ownership %j", (change) => {
expect(
testTimings.resolveRuntimePlacementSeconds(
{ ...reader, ...change, includePatterns: [...reader.includePatterns, "new.test.ts"] },
[{ ...reader, seconds: 201 }],
),
).toBeUndefined();
});
function runtimeLog(
id: number,
overrides: Partial<Omit<typeof reader, "pretestBuildMode">> & {
@ -125,13 +232,14 @@ describe("runtime placement observations", () => {
.map((line) => `job\tstep\t${line}`)
.join("\n"),
);
expect(refitTestTimings([first]).timings.compactGroupSeconds.blacksmith[key]).toBeUndefined();
const result = refitTestTimings([first, second]).timings.compactGroupSeconds.blacksmith;
expect(result).toEqual({ "fixture-parent": 25, [key]: 20 });
expect(refitTestTimings([first]).timings.runtimePlacementTimings.blacksmith).toEqual([]);
const result = refitTestTimings([first, second]).timings;
expect(result.compactGroupSeconds.blacksmith).toEqual({ "fixture-parent": 25 });
expect(result.runtimePlacementTimings.blacksmith).toEqual([{ ...reader, seconds: 20 }]);
expect(
refitTestTimings([{ ...first, logs: [...first.logs, ...first.logs] }]).timings
.compactGroupSeconds.blacksmith[key],
).toBeUndefined();
.runtimePlacementTimings.blacksmith,
).toEqual([]);
});
it.each([
@ -141,11 +249,7 @@ describe("runtime placement observations", () => {
{ pretestBuildMode: "private-qa" as const },
])("does not merge runtime placement with changed ownership %j", (change) => {
const result = refitTestTimings([sample(1, runtimeLog(1)), sample(2, runtimeLog(2, change))]);
expect(
Object.keys(result.timings.compactGroupSeconds.blacksmith).filter((entry) =>
entry.startsWith("runtime-placement#"),
),
).toEqual([]);
expect(result.timings.runtimePlacementTimings.blacksmith).toEqual([]);
});
it.each(["failed", "missing readiness", "malformed descriptor"])(
@ -167,10 +271,30 @@ describe("runtime placement observations", () => {
}
return sample(id, text);
});
expect(refitTestTimings(runs).timings.compactGroupSeconds.blacksmith[key]).toBeUndefined();
expect(refitTestTimings(runs).timings.runtimePlacementTimings.blacksmith).toEqual([]);
},
);
it("publishes fresh runtime observations when no split generation repeats", () => {
withSamplerFixture(
{
runs: [samplerRun(1), samplerRun(2)],
jobs: [1, 2].map((id) =>
samplerJob(id * 10, id, {
log: runtimeLog(id).split("\n").slice(0, 4).join("\n"),
}),
),
},
(fixture) => {
const result = fixture.invoke();
expect(result.status, result.stderr).toBe(0);
const timings = ciTestTimingsSchema.parse(JSON.parse(fixture.contents()));
expect(timings.compactGroupSeconds.blacksmith).toEqual({});
expect(timings.runtimePlacementTimings.blacksmith).toEqual([{ ...reader, seconds: 20 }]);
},
);
});
it.each(["push", "pull-request"] as const)(
"admits complete %s runtime placement without changing inventories or precise capacity",
(compactMode) => {
@ -182,7 +306,8 @@ describe("runtime placement observations", () => {
infrastructure,
"test/vitest/vitest.gateway-methods.config.ts",
]);
const spy = vi.spyOn(testTimings, "readCompactGroupTimings").mockReturnValue({});
const compactSpy = vi.spyOn(testTimings, "readCompactGroupTimings").mockReturnValue({});
const spy = vi.spyOn(testTimings, "readRuntimePlacementTimings").mockReturnValue([]);
const options = {
compactMode,
runnerBackend: "hybrid",
@ -208,19 +333,18 @@ describe("runtime placement observations", () => {
const preciseBefore = createSelectedNodeTestShardBundles(selected, {
runnerBackend: "hybrid",
});
// Synthetic costs follow the emitted readers; recorded measurements belong
// to their exact historical membership and must not be reused after growth.
const blacksmith = Object.fromEntries(
runtimeGroups.map((group) => [
runtimePlacementTimingKey(group)!,
group.configs.includes(runtimeConfig)
? 200
: group.configs.includes(infrastructure)
? 300
: 20,
]),
);
spy.mockImplementation((profile) => (profile === "blacksmith" ? blacksmith : {}));
const blacksmith: RuntimePlacementTiming[] = runtimeGroups.map((group) => ({
configs: group.configs,
env: group.env ?? {},
includePatterns: group.includePatterns!,
pretestBuildMode: "runtime",
seconds: group.configs.includes(runtimeConfig)
? 200
: group.configs.includes(infrastructure)
? 300
: 20,
}));
spy.mockImplementation((profile) => (profile === "blacksmith" ? blacksmith : []));
const after = createNodeTestShardBundles(options);
if (compactMode === "pull-request") {
expect(
@ -232,24 +356,32 @@ describe("runtime placement observations", () => {
);
const groups = (jobs: typeof before) =>
jobs
.flatMap((job) => job.groups)
.map((group) => JSON.stringify(group))
.flatMap((job) =>
job.groups.map((group) =>
JSON.stringify({
...group,
timing_key: undefined,
env: {
...group.env,
OPENCLAW_VITEST_MAX_WORKERS:
group.env?.OPENCLAW_VITEST_MAX_WORKERS ??
(job.planConcurrency === 2 ? "2" : undefined),
},
}),
),
)
.toSorted();
expect(groups(after)).toEqual(groups(before));
expect(after.map((job) => [job.checkName, job.runner])).toEqual(
before.map((job) => [job.checkName, job.runner]),
);
expect(after.reduce((sum, job) => sum + (job.planConcurrency ?? 1), 0)).toBeLessThanOrEqual(
before.reduce((sum, job) => sum + (job.planConcurrency ?? 1), 0),
);
expect(
after.map((job) => [
job.checkName,
job.runner,
job.planConcurrency,
job.pretestBuildMode,
]),
).toEqual(
before.map((job) => [
job.checkName,
job.runner,
job.planConcurrency,
job.pretestBuildMode,
]),
after.filter((job) => job.pretestBuildMode === "runtime").length,
).toBeLessThanOrEqual(
before.filter((job) => job.pretestBuildMode === "runtime").length + 1,
);
const changed = after.filter(
(job, index) => JSON.stringify(job.groups) !== JSON.stringify(before[index]!.groups),
@ -269,15 +401,10 @@ describe("runtime placement observations", () => {
expect(crossing.every((group) => group.env?.OPENCLAW_VITEST_MAX_WORKERS === "2")).toBe(
true,
);
const configReader = runtimeGroups.find((group) => group.configs.includes(runtimeConfig))!;
spy.mockImplementation((profile) =>
profile === "blacksmith"
? Object.fromEntries(
Object.entries(blacksmith).filter(
([entry]) => entry !== runtimePlacementTimingKey(configReader),
),
)
: {},
? blacksmith.filter((entry) => !entry.configs.includes(runtimeConfig))
: [],
);
const unmeasured = createNodeTestShardBundles(options);
expect(unmeasured.map((job) => [job.checkName, job.runner, job.groups])).toEqual(
@ -291,13 +418,8 @@ describe("runtime placement observations", () => {
expect(readerJob.predictedSeconds).toBeGreaterThan(361);
spy.mockImplementation((profile) =>
profile === "blacksmith"
? Object.fromEntries(
Object.entries(blacksmith).map(([entry, value]) => [
entry,
entry.startsWith("runtime-placement#") ? 1_000 : value,
]),
)
: {},
? blacksmith.map((entry) => Object.assign({}, entry, { seconds: 1_000 }))
: [],
);
const unfit = createNodeTestShardBundles(options);
expect(groups(unfit)).toEqual(groups(before));
@ -307,6 +429,7 @@ describe("runtime placement observations", () => {
expect(unfit.some((job) => (job.predictedSeconds ?? 0) > 440)).toBe(true);
} finally {
spy.mockRestore();
compactSpy.mockRestore();
fullSuiteVitestShards.splice(0, fullSuiteVitestShards.length, ...originalShards);
}
},
@ -351,6 +474,7 @@ describe("runtime placement observations", () => {
cost,
admits: (groups) => groups.length > 0 && cost(groups) <= 440,
runnerRank: ({ runner }) => ["small", "medium", "strong"].indexOf(runner),
prepareRecipient: (recipient) => recipient.groups,
});
expect(jobs.map((entry) => entry.runner)).toEqual(["strong", recipientRunner]);
expect(jobs.map((entry) => entry.groups)).toEqual(
@ -361,6 +485,69 @@ describe("runtime placement observations", () => {
);
},
);
it("uses spare ordinary capacity when the retained donor makes both maxima equal", () => {
const group = (name: string, runtime = true): NodeTestShardGroup => ({
shard_name: name,
configs: ["reader.config.ts"],
includePatterns: [`src/${name}.test.ts`],
requiresDist: false,
runner: "same",
...(runtime
? { pretestBuildMode: "runtime", env: { OPENCLAW_VITEST_MAX_WORKERS: "2" } }
: {}),
});
const moved = group("moved");
const retained = group("retained");
const spare = group("spare");
const ordinary = group("ordinary", false);
const job = (
name: string,
groups: NodeTestShardGroup[],
runtime = true,
): CompactNodeTestShard => ({
checkName: name,
shardName: name,
runner: "same",
groups,
requiresDist: false,
planConcurrency: runtime ? 1 : 2,
...(runtime ? { pretestBuildMode: "runtime" } : {}),
});
const jobs = [
job("donor", [moved, retained]),
job("first-runtime", [spare]),
job("ordinary", [ordinary], false),
];
const weights: Record<string, number> = { moved: 100, retained: 330, spare: 80, ordinary: 50 };
const cost = (groups: NodeTestShardGroup[]) =>
100 + groups.reduce((sum, entry) => sum + weights[entry.shard_name]!, 0);
rebalanceRuntimeTestJobs(jobs, {
cost,
admits: (groups) => groups.length > 0 && cost(groups) <= 440,
runnerRank: () => 0,
prepareRecipient: (recipient) =>
recipient.planConcurrency === 2
? recipient.groups.map((entry) => ({
...entry,
env: { OPENCLAW_VITEST_MAX_WORKERS: "2" },
}))
: recipient.groups,
});
expect(jobs.map((entry) => entry.groups.map((value) => value.shard_name))).toEqual([
["retained"],
["spare"],
["ordinary", "moved"],
]);
expect(jobs[2]).toMatchObject({
pretestBuildMode: "runtime",
planConcurrency: 1,
predictedSeconds: 250,
});
expect(jobs[2]!.groups.every((entry) => entry.env?.OPENCLAW_VITEST_MAX_WORKERS === "2")).toBe(
true,
);
});
});
function samplerRun(id: number, overrides: Record<string, unknown> = {}) {
@ -1058,9 +1245,28 @@ it.todo("retains todo coverage");
it("generates identical sorted data when equivalent runs, logs, and file rows arrive in different orders", () => {
const files = { "ui/src/e2e/z.e2e.test.ts": 5, "ui/src/e2e/a.e2e.test.ts": 4 };
const runtimeSpan = (id: number) => {
const descriptor = {
shard_name: "runtime-order",
configs: ["test/vitest/vitest.runtime-config.config.ts"],
env: id === 1 ? { FIXTURE_A: "1", FIXTURE_B: "2" } : { FIXTURE_B: "2", FIXTURE_A: "1" },
includePatterns: id === 1 ? ["a.test.ts", "b.test.ts"] : ["b.test.ts", "a.test.ts"],
};
const [begin, end] = compactLog(20, descriptor.shard_name).split("\n");
return [
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor])}`,
begin,
"2026-08-27T23:00:01Z [shard:runtime-order] [test] preparing runtime runtime before Vitest workers",
end,
].join("\n");
};
const runs = [2, 1].map((id) =>
timingRun(id, [
{ kind: "compact", labels: ["ubuntu-24.04"], text: compactLog(20) },
{
kind: "compact",
labels: ["ubuntu-24.04"],
text: `${compactLog(20)}\n${runtimeSpan(id)}`,
},
{ kind: "uiE2e", text: uiLog(files) },
]),
);
@ -1071,7 +1277,11 @@ it.todo("retains todo coverage");
kind: "uiE2e",
text: uiLog(Object.fromEntries(Object.entries(files).toReversed())),
},
{ kind: "compact", labels: ["ubuntu-24.04"], text: compactLog(20) },
{
kind: "compact",
labels: ["ubuntu-24.04"],
text: `${compactLog(20)}\n${runtimeSpan(run.id)}`,
},
]),
);
@ -1365,6 +1575,31 @@ describe("CI timing sampler provenance", () => {
});
describe("CI timing schema", () => {
const observation: RuntimePlacementTiming = {
configs: ["reader.config.ts"],
env: {},
includePatterns: ["src/reader.test.ts"],
pretestBuildMode: "runtime",
seconds: 20,
};
it.each([
[{ ...observation, includePatterns: [] }],
[{ ...observation, includePatterns: ["src/*.test.ts"] }],
[{ ...observation, includePatterns: ["src/reader.test.ts", "src/reader.test.ts"] }],
[{ ...observation, env: { workers: 2 } }],
[{ ...observation, configs: [] }],
[{ ...observation, seconds: 0 }],
[{ ...observation, pretestBuildMode: "unknown" }],
[observation, { ...observation }],
])("rejects malformed or duplicate runtime observations %j", (...observations) => {
expect(() =>
ciTestTimingsSchema.parse({
...baseline,
runtimePlacementTimings: { blacksmith: observations, github: [] },
}),
).toThrow("Invalid CI test timings");
});
const invalidTimings: Array<[string, string]> = [
["non-object root", "null"],
["unknown root key", JSON.stringify({ ...baseline, extra: 1 })],

1
test/vitest-ui-e2e-config.test.ts
View file

@ -49,6 +49,7 @@ function timingFile(fileSeconds: Record<string, number>, perFileOverheadSeconds
source: "fixture measurements",
uiE2e: { fileSeconds, perFileOverheadSeconds },
compactGroupSeconds: { blacksmith: {}, github: {} },
runtimePlacementTimings: { blacksmith: [], github: [] },
repoE2eFileSeconds: {},
});
}