mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
ci: collect exact pull request timing samples
Use complete config, file, and worker identities when refitting compact PR workloads. Preserve independent-run admission and partial-inventory retention, and keep main-run tooling logs out of the PR timing cohort. Publish the complete refit report as an artifact with bounded job outputs. Linux Testbox proof: 508 owner cases, workflow validation, extracted-wrapper inventory, types, boundary lint, three Knip scans, lint, and formatting. The 174-case timing file took 45.452s with one worker. The mixed-source regression failed before the fix and passed afterward. P2 review is clean. Broad PR plans are byte-identical against the base before timing refreshes.
This commit is contained in:
Peter Steinberger
parent
45e7c8e380
commit
ef681882ff
8 changed files with 1510 additions and 76 deletions
35
.github/workflows/ci-test-timings-refit.yml
vendored
35
.github/workflows/ci-test-timings-refit.yml
vendored
|
|
@ -48,16 +48,30 @@ jobs:
|
|||
run: |
|
||||
set -euo pipefail
|
||||
pnpm --silent ci:timings:refit > "$RUNNER_TEMP/ci-test-timings-report.md"
|
||||
node --input-type=module <<'NODE'
|
||||
import { appendFileSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
const report = readFileSync(path.join(process.env.RUNNER_TEMP, "ci-test-timings-report.md"), "utf8");
|
||||
const runIds = report.match(/^Sampled CI and release-check runs with successful timing jobs: ([0-9, ]+)$/mu)?.[1];
|
||||
if (!runIds || runIds.length > 2048 || !/^\d+(?:, \d+)*$/u.test(runIds)) {
|
||||
throw new Error("Missing or oversized sampled-run summary; the full report is retained as an artifact.");
|
||||
}
|
||||
appendFileSync(process.env.GITHUB_OUTPUT, `run_ids=${runIds}\n`);
|
||||
NODE
|
||||
if ! git diff --quiet -- config/ci-test-timings.json; then
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
delimiter="$(node -p 'crypto.randomUUID()')"
|
||||
{
|
||||
echo "report<<${delimiter}"
|
||||
cat "$RUNNER_TEMP/ci-test-timings-report.md"
|
||||
echo "${delimiter}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Upload full timing refit report
|
||||
id: report
|
||||
if: ${{ !cancelled() && steps.refit.outcome != 'skipped' }}
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: ci-test-timings-report-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/ci-test-timings-report.md
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
- name: Update standing refit pull request
|
||||
if: steps.refit.outputs.changed == 'true'
|
||||
uses: ./.github/actions/publish-generated-pr
|
||||
|
|
@ -78,6 +92,7 @@ jobs:
|
|||
scripts/lib/ci-test-timings-refit.mts
|
||||
scripts/lib/ci-node-test-groups-codec.mts
|
||||
scripts/lib/ci-test-timings-schema.mts
|
||||
scripts/lib/local-check-runtime.mts
|
||||
scripts/lib/vitest-shard-metadata.mts
|
||||
scripts/lib/direct-run.mjs
|
||||
scripts/lib/numeric-options.mjs
|
||||
|
|
@ -92,11 +107,11 @@ jobs:
|
|||
pr-body: |
|
||||
## What Problem This Solves
|
||||
|
||||
Refresh stale CI shard weights from successful jobs in completed main-push workflows and successful release-check runs, plus tooling file weights from successful pull-request CI runs.
|
||||
Refresh stale CI shard weights from successful jobs in completed main-push workflows, successful release-check runs, and exact compact selectors and tooling files from successful pull-request CI runs.
|
||||
|
||||
## Why This Change Was Made
|
||||
|
||||
The daily refit uses medians, rejects outliers and single samples, and keeps existing weights when the new median is within 15% of them. Completed failed main workflows contribute only their successful timing jobs; their missing jobs never supply pruning evidence. Pull-request samples measure their merge-ref and update only PR-only tooling files. Main compact keys absent from every contributing run are removed only when their profile has observations from at least three successful workflows; partial PR inventories never prune tooling files. The table below lists each removal explicitly.
|
||||
The daily refit uses independent-run medians, rejects outliers and single samples, and keeps existing weights within the 15% replacement band. Failed main workflows contribute successful timing jobs without supplying pruning evidence. Pull-request measurements describe the merge-ref: exact compact selector keys, runtime inventory observations, and native tooling file costs can update, but partial plans never reprice unqualified compact parents or prune absent measurements. Worker timing identities must agree with observed admission. Main compact keys absent from every contributing run are removed only after three successful contributing workflows for that profile. The complete artifact report lists every change and removal.
|
||||
|
||||
## User Impact
|
||||
|
||||
|
|
@ -104,6 +119,6 @@ jobs:
|
|||
|
||||
## Evidence
|
||||
|
||||
Generated with `pnpm ci:timings:refit`:
|
||||
Generated with `pnpm ci:timings:refit` from runs: ${{ steps.refit.outputs.run_ids }}.
|
||||
|
||||
${{ steps.refit.outputs.report }}
|
||||
[Download the complete timing report](${{ steps.report.outputs.artifact-url }}).
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ async function main() {
|
|||
console.log(`Frozen UTC window: ${lower} through ${upper} (7 days).\n`);
|
||||
console.log("Release workflow SHAs identify tooling, not the measured target.\n");
|
||||
console.log(
|
||||
"PR tooling measurements execute the merge-ref; workflow/job SHAs identify the PR head.\n",
|
||||
"PR compact and tooling measurements execute the merge-ref; workflow/job SHAs identify the PR head.\n",
|
||||
);
|
||||
console.log(
|
||||
"| Source | Run | Attempt | Workflow SHA | Created (UTC) | Workflow result | Parsed profiles | Timing jobs |\n| --- | ---: | ---: | --- | --- | --- | --- | ---: |",
|
||||
|
|
@ -107,10 +107,12 @@ async function main() {
|
|||
const runs: CiTimingRun[] = [];
|
||||
const seenRuns = new Map<number, string>();
|
||||
const seenJobs = new Map<number, string>();
|
||||
type TimingSource = "main" | "release" | "tooling";
|
||||
type TimingSource = "main" | "release" | "pull-request" | "tooling";
|
||||
async function readRun(run: z.infer<typeof runSchema>, source: TimingSource) {
|
||||
const logs: CiTimingRun["logs"] = [];
|
||||
const completeInventory = run.conclusion === "success";
|
||||
// Successful PRs can select a strict subset of the suite.
|
||||
const completeInventory =
|
||||
source !== "pull-request" && source !== "tooling" && run.conclusion === "success";
|
||||
const jobsByAttempt: TimingJob[][] = [];
|
||||
let afterCutoff = false;
|
||||
let pages = 0;
|
||||
|
|
@ -170,18 +172,21 @@ async function main() {
|
|||
);
|
||||
}
|
||||
afterCutoff ||= Date.parse(job.completed_at) > Date.parse(upper);
|
||||
const compactJob = /^checks-node-(?:changed-(?:config-)?)?compact-/u.test(job.name);
|
||||
const extensionJob =
|
||||
/^checks-node-changed-extensions-(?:bundle-\d+|config(?:-\d+)?)$/u.test(job.name);
|
||||
const kind =
|
||||
source === "release"
|
||||
? /(?:^| \/ )Repo E2E \(Gateway \d+\/\d+\)$/u.test(job.name)
|
||||
? "repoE2e"
|
||||
: undefined
|
||||
: source === "tooling"
|
||||
? job.name.startsWith("checks-node-compact-")
|
||||
? compactJob
|
||||
? "tooling"
|
||||
: undefined
|
||||
: job.name.startsWith("checks-ui-e2e (")
|
||||
: source === "main" && job.name.startsWith("checks-ui-e2e (")
|
||||
? "uiE2e"
|
||||
: job.name.startsWith("checks-node-compact-")
|
||||
: compactJob || extensionJob
|
||||
? "compact"
|
||||
: undefined;
|
||||
if (kind) {
|
||||
|
|
@ -229,18 +234,19 @@ async function main() {
|
|||
createdAt: run.created_at,
|
||||
logs,
|
||||
completeInventory,
|
||||
...(source === "pull-request" ? { pullRequestMergeRef: true } : {}),
|
||||
};
|
||||
}
|
||||
async function sampleWorkflow(workflow: string, source: TimingSource) {
|
||||
const event =
|
||||
source === "main" ? "push" : source === "tooling" ? "pull_request" : "workflow_dispatch";
|
||||
source === "main" ? "push" : source === "pull-request" ? "pull_request" : "workflow_dispatch";
|
||||
const pageSchema = z.object({
|
||||
total_count: z.number().int().nonnegative(),
|
||||
workflow_runs: z.array(
|
||||
runSchema.extend({
|
||||
conclusion: source === "main" ? z.string().nullable() : z.literal("success"),
|
||||
// PRs measure their merge-ref, which is appropriate only for PR-only
|
||||
// tooling. Their workflow/job head_sha identifies the PR head, not that merge.
|
||||
// PR measurements retain their merge-ref provenance and partial inventory.
|
||||
// Their workflow/job head_sha identifies the PR head, not that merge.
|
||||
// A release dispatch can check out target_ref; push alone proves main.
|
||||
event: z.literal(event),
|
||||
head_branch: source === "main" ? z.literal("main") : z.string().min(1),
|
||||
|
|
@ -301,15 +307,16 @@ async function main() {
|
|||
const contributes =
|
||||
source === "main"
|
||||
? compact
|
||||
: source === "tooling"
|
||||
? contributingRunIds.toolingBlacksmith.length +
|
||||
: source === "pull-request"
|
||||
? compact ||
|
||||
contributingRunIds.toolingBlacksmith.length +
|
||||
contributingRunIds.toolingGithub.length >
|
||||
0
|
||||
0
|
||||
: contributingRunIds.repoE2e.length > 0;
|
||||
if (contributes) {
|
||||
runs.push(timingRun);
|
||||
}
|
||||
if (source !== "main" || compact) {
|
||||
if (source === "release" || contributes) {
|
||||
sampled += 1;
|
||||
}
|
||||
if (sampled === count) {
|
||||
|
|
@ -381,7 +388,7 @@ async function main() {
|
|||
]) {
|
||||
await sampleWorkflow(workflow, "release");
|
||||
}
|
||||
await sampleWorkflow("ci.yml", "tooling");
|
||||
await sampleWorkflow("ci.yml", "pull-request");
|
||||
}
|
||||
let previous;
|
||||
try {
|
||||
|
|
@ -389,18 +396,33 @@ async function main() {
|
|||
} catch {
|
||||
// A missing or invalid baseline has no measurements worth preserving.
|
||||
}
|
||||
const { timings, changes, runIds, contributingRunIds } = refitTestTimings(runs, previous, {
|
||||
seedTooling,
|
||||
});
|
||||
const { timings, changes, runIds, contributingRunIds, rejectedWorkerKeys } = refitTestTimings(
|
||||
runs,
|
||||
previous,
|
||||
{
|
||||
seedTooling,
|
||||
},
|
||||
);
|
||||
const { blacksmith, github, toolingBlacksmith, toolingGithub } = contributingRunIds;
|
||||
const mainContributors = new Set([...blacksmith, ...github]);
|
||||
const prRunIds = new Set(runs.filter((run) => run.pullRequestMergeRef).map((run) => run.id));
|
||||
const mainBlacksmith = blacksmith.filter((id) => !prRunIds.has(id));
|
||||
const mainGithub = github.filter((id) => !prRunIds.has(id));
|
||||
const mainContributors = new Set([...mainBlacksmith, ...mainGithub]);
|
||||
console.log(
|
||||
`\nIndependent main compact contributors: ${mainContributors.size} (Blacksmith: ${blacksmith.length}; GitHub: ${github.length}). Release Gateway contributors: ${contributingRunIds.repoE2e.length}.\n`,
|
||||
`\nIndependent main compact contributors: ${mainContributors.size} (Blacksmith: ${mainBlacksmith.length}; GitHub: ${mainGithub.length}). Release Gateway contributors: ${contributingRunIds.repoE2e.length}.\n`,
|
||||
);
|
||||
console.log(
|
||||
`Independent PR compact contributors: ${new Set([...blacksmith, ...github].filter((id) => prRunIds.has(id))).size}; merge-ref samples never prune absent timings.\n`,
|
||||
);
|
||||
console.log(
|
||||
`Independent PR tooling contributors: ${new Set([...toolingBlacksmith, ...toolingGithub]).size} (Blacksmith: ${toolingBlacksmith.length}; GitHub: ${toolingGithub.length}).${seedTooling ? " Explicit tooling seed; single-run measurements allowed." : ""}\n`,
|
||||
);
|
||||
ciTestTimingsSchema.parse(timings);
|
||||
for (const profile of ["blacksmith", "github"] as const) {
|
||||
for (const key of rejectedWorkerKeys[profile]) {
|
||||
console.error(`[ci-timings] retained ${profile} ${key}: ambiguous or mixed worker ceilings`);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`Sampled CI and release-check runs with successful timing jobs: ${runIds.join(", ")}\n`,
|
||||
);
|
||||
|
|
|
|||
|
|
@ -2,17 +2,28 @@ import { stripVTControlCharacters } from "node:util";
|
|||
import { decodeNodeTestGroups } from "./ci-node-test-groups-codec.mts";
|
||||
import {
|
||||
isRuntimePlacementTiming,
|
||||
isRuntimePlacementIncludePatterns,
|
||||
runtimePlacementTimingIdentity,
|
||||
type CiTestTimings,
|
||||
type RuntimePlacementTiming,
|
||||
} from "./ci-test-timings-schema.mts";
|
||||
import { parseCompactSplitTimingKey } from "./vitest-shard-metadata.mts";
|
||||
import {
|
||||
createExtensionTestTimingKey,
|
||||
splitExtensionTestProcessTargets,
|
||||
} from "./extension-test-plan.mts";
|
||||
import { isConstrainedCiCheckHost } from "./local-check-runtime.mts";
|
||||
import {
|
||||
createCompactSplitTimingGeneration,
|
||||
parseCompactSplitTimingKey,
|
||||
} from "./vitest-shard-metadata.mts";
|
||||
|
||||
export type CiTimingRun = {
|
||||
id: number;
|
||||
createdAt: string;
|
||||
/** Failed workflows supply positive samples, never evidence that absent keys disappeared. */
|
||||
/** Failed workflows and selected PR plans never prove absent keys disappeared. */
|
||||
completeInventory: boolean;
|
||||
/** The workflow/job SHA identifies the PR head, while tests run its merge-ref. */
|
||||
pullRequestMergeRef?: boolean;
|
||||
logs: (
|
||||
| { kind: "uiE2e" | "repoE2e"; text: string }
|
||||
| { kind: "compact" | "tooling"; text: string; labels: string[] }
|
||||
|
|
@ -20,31 +31,52 @@ export type CiTimingRun = {
|
|||
};
|
||||
|
||||
type Samples = Map<string, number[]>;
|
||||
type WorkerCeilings = Map<string, Set<number | "unspecified" | "ambiguous">>;
|
||||
|
||||
type RuntimeTimingGroup = {
|
||||
shard_name: string;
|
||||
timing_key?: string;
|
||||
configs: string[];
|
||||
includePatterns: string[];
|
||||
includePatterns?: string[] | null;
|
||||
env?: Record<string, string>;
|
||||
fallbackMaxWorkers?: number;
|
||||
minTotalMemoryBytes?: number;
|
||||
};
|
||||
|
||||
function readRuntimeTimingGroups(text: string): RuntimeTimingGroup[] {
|
||||
const encoded = new Set(
|
||||
[
|
||||
...text.matchAll(
|
||||
/\d{4}-\d\d-\d\dT[\d:.]+Z\s+OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: (\S+)$/gmu,
|
||||
/^\d{4}-\d\d-\d\dT[\d:.]+Z\s+OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: (\S+)$/gmu,
|
||||
),
|
||||
].map((match) => match[1]!),
|
||||
);
|
||||
if (encoded.size !== 1) {
|
||||
if (encoded.size > 1) {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const groups = decodeNodeTestGroups([...encoded][0]!);
|
||||
const jsonEnv = (key: string): unknown => {
|
||||
const value = readLogEnv(text, key);
|
||||
if (value === null) {
|
||||
throw new Error(`Ambiguous ${key}`);
|
||||
}
|
||||
return value ? JSON.parse(value) : undefined;
|
||||
};
|
||||
// Singleton matrix rows use the same executor without a packed group descriptor.
|
||||
const groups: unknown[] =
|
||||
encoded.size === 1
|
||||
? decodeNodeTestGroups([...encoded][0]!)
|
||||
: [
|
||||
{
|
||||
shard_name: readLogEnv(text, "OPENCLAW_VITEST_SHARD_NAME"),
|
||||
configs: jsonEnv("OPENCLAW_NODE_TEST_CONFIGS_JSON"),
|
||||
includePatterns: jsonEnv("OPENCLAW_NODE_TEST_INCLUDE_PATTERNS_JSON"),
|
||||
env: jsonEnv("OPENCLAW_NODE_TEST_ENV_JSON") ?? undefined,
|
||||
},
|
||||
];
|
||||
const strings = (value: unknown): value is string[] =>
|
||||
Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
||||
return groups.filter((group): group is RuntimeTimingGroup => {
|
||||
const validGroups = groups.filter((group): group is RuntimeTimingGroup => {
|
||||
if (typeof group !== "object" || group === null) {
|
||||
return false;
|
||||
}
|
||||
|
|
@ -55,16 +87,26 @@ function readRuntimeTimingGroups(text: string): RuntimeTimingGroup[] {
|
|||
"configs" in group &&
|
||||
strings(group.configs) &&
|
||||
group.configs.length > 0 &&
|
||||
"includePatterns" in group &&
|
||||
strings(group.includePatterns) &&
|
||||
group.includePatterns.length > 0 &&
|
||||
(!("includePatterns" in group) ||
|
||||
group.includePatterns === null ||
|
||||
strings(group.includePatterns)) &&
|
||||
(!("fallbackMaxWorkers" in group) ||
|
||||
(typeof group.fallbackMaxWorkers === "number" &&
|
||||
Number.isSafeInteger(group.fallbackMaxWorkers) &&
|
||||
group.fallbackMaxWorkers > 0)) &&
|
||||
(!("minTotalMemoryBytes" in group) ||
|
||||
(typeof group.minTotalMemoryBytes === "number" &&
|
||||
Number.isSafeInteger(group.minTotalMemoryBytes) &&
|
||||
group.minTotalMemoryBytes > 0)) &&
|
||||
(!("env" in group) ||
|
||||
group.env === undefined ||
|
||||
(typeof group.env === "object" &&
|
||||
group.env !== null &&
|
||||
!Array.isArray(group.env) &&
|
||||
Object.values(group.env).every((value) => typeof value === "string")))
|
||||
);
|
||||
});
|
||||
return validGroups.length === groups.length ? validGroups : [];
|
||||
} catch {
|
||||
// Historical/malformed descriptors cannot supply a placement identity.
|
||||
return [];
|
||||
|
|
@ -90,6 +132,85 @@ function seconds(value: string, unit: string): number {
|
|||
return Number(value) / (unit === "ms" ? 1000 : 1);
|
||||
}
|
||||
|
||||
function parseWorkerCeiling(value: unknown): number | null | undefined {
|
||||
if (value === undefined || value === "") {
|
||||
return undefined;
|
||||
}
|
||||
if (typeof value !== "string" || !/^[1-9]\d*$/u.test(value)) {
|
||||
return null;
|
||||
}
|
||||
const workers = Number(value);
|
||||
return Number.isSafeInteger(workers) ? workers : null;
|
||||
}
|
||||
|
||||
function intersectWorkerCeilings(...values: (number | null | undefined)[]) {
|
||||
if (values.includes(null)) {
|
||||
return null;
|
||||
}
|
||||
const ceilings = values.filter((value): value is number => typeof value === "number");
|
||||
return ceilings.length > 0 ? Math.min(...ceilings) : undefined;
|
||||
}
|
||||
|
||||
function readLogEnv(text: string, key: string): string | null | undefined {
|
||||
const values = new Set(
|
||||
[
|
||||
...text.matchAll(
|
||||
new RegExp(
|
||||
`^\\d{4}-\\d\\d-\\d\\dT[\\d:.]+Z\\s+${key}: ([\\[{]\\n[\\s\\S]*?\\n[\\]}]|[^\\n]*)$`,
|
||||
"gmu",
|
||||
),
|
||||
),
|
||||
].map((match) => match[1]!.trim()),
|
||||
);
|
||||
return values.size > 1 ? null : [...values][0];
|
||||
}
|
||||
|
||||
function readJobWorkerCeiling(text: string) {
|
||||
const inherited = parseWorkerCeiling(readLogEnv(text, "OPENCLAW_VITEST_MAX_WORKERS"));
|
||||
const encoded = readLogEnv(text, "OPENCLAW_NODE_TEST_ENV_JSON");
|
||||
if (encoded === null) {
|
||||
return null;
|
||||
}
|
||||
let override: number | null | undefined;
|
||||
if (encoded) {
|
||||
try {
|
||||
const value: unknown = JSON.parse(encoded);
|
||||
if (value !== null) {
|
||||
override =
|
||||
typeof value === "object" && !Array.isArray(value)
|
||||
? parseWorkerCeiling(
|
||||
"OPENCLAW_VITEST_MAX_WORKERS" in value
|
||||
? value.OPENCLAW_VITEST_MAX_WORKERS
|
||||
: undefined,
|
||||
)
|
||||
: null;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return intersectWorkerCeilings(inherited, override);
|
||||
}
|
||||
|
||||
function readWorkerResources(text: string) {
|
||||
const matches = [
|
||||
...text.matchAll(
|
||||
/^\d{4}-\d\d-\d\dT[\d:.]+Z\s+\[shard:resources\] logicalCpuCount=(\d+) totalMemoryBytes=(\d+) requested plans=(\d+) admitted plans=(\d+)$/gmu,
|
||||
),
|
||||
];
|
||||
if (matches.length !== 1) {
|
||||
return undefined;
|
||||
}
|
||||
const values = matches[0]!.slice(1).map(Number);
|
||||
if (
|
||||
!values.every((value) => Number.isSafeInteger(value) && value > 0) ||
|
||||
values[3]! > values[2]!
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return { logicalCpuCount: values[0]!, totalMemoryBytes: values[1]!, admittedPlans: values[3]! };
|
||||
}
|
||||
|
||||
function readE2eLog(text: string, samples: Samples, overhead?: number[]) {
|
||||
const files = new Map<string, number>();
|
||||
let hasParallelFiles = false;
|
||||
|
|
@ -129,18 +250,140 @@ function readE2eLog(text: string, samples: Samples, overhead?: number[]) {
|
|||
}
|
||||
}
|
||||
|
||||
function readSingletonExtensionInvocations(
|
||||
lines: readonly string[],
|
||||
config: string,
|
||||
files: readonly string[],
|
||||
): Map<string, number> | undefined {
|
||||
const declared = new Set(files);
|
||||
const measured = new Map<string, number>();
|
||||
let active:
|
||||
| {
|
||||
started: number;
|
||||
files: Set<string>;
|
||||
runs: number;
|
||||
summaries: number;
|
||||
durations: number;
|
||||
duration?: number;
|
||||
}
|
||||
| undefined;
|
||||
let verified = false;
|
||||
let passed = false;
|
||||
const finish = (ended: number) => {
|
||||
if (!active) {
|
||||
return false;
|
||||
}
|
||||
const [file] = active.files;
|
||||
const elapsed = (ended - active.started) / 1000;
|
||||
if (
|
||||
active.files.size !== 1 ||
|
||||
file === undefined ||
|
||||
!declared.has(file) ||
|
||||
measured.has(file) ||
|
||||
active.runs !== 1 ||
|
||||
active.summaries !== 1 ||
|
||||
active.durations !== 1 ||
|
||||
active.duration === undefined ||
|
||||
!Number.isFinite(active.duration) ||
|
||||
!Number.isFinite(elapsed) ||
|
||||
active.duration > elapsed ||
|
||||
elapsed <= 0
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
measured.set(file, elapsed);
|
||||
active = undefined;
|
||||
return true;
|
||||
};
|
||||
for (const line of lines) {
|
||||
const record = /^(\d{4}-\d\d-\d\dT[\d:.]+Z)\s+\[shard:[^\]]+\]\s+(.*)$/u.exec(line);
|
||||
if (!record) {
|
||||
continue;
|
||||
}
|
||||
const timestamp = Date.parse(record[1]!);
|
||||
if (!Number.isFinite(timestamp)) {
|
||||
return undefined;
|
||||
}
|
||||
const body = record[2]!.trim();
|
||||
const start = /^\[test\] starting (\S+)$/u.exec(body);
|
||||
if (start) {
|
||||
if (verified || start[1] !== config || (active && !finish(timestamp))) {
|
||||
return undefined;
|
||||
}
|
||||
active = { started: timestamp, files: new Set(), runs: 0, summaries: 0, durations: 0 };
|
||||
continue;
|
||||
}
|
||||
if (body === "[vitest-workers] verifying completed generation before cleanup") {
|
||||
if (verified || !finish(timestamp)) {
|
||||
return undefined;
|
||||
}
|
||||
verified = true;
|
||||
continue;
|
||||
}
|
||||
const completion = /^\[test\] passed (\d+) Vitest shards? in [\d.]+s$/u.exec(body);
|
||||
if (completion) {
|
||||
if (!verified || passed || Number(completion[1]) !== files.length) {
|
||||
return undefined;
|
||||
}
|
||||
passed = true;
|
||||
continue;
|
||||
}
|
||||
if (/^RUN\s+v\S+/u.test(body)) {
|
||||
if (!active) {
|
||||
return undefined;
|
||||
}
|
||||
active.runs += 1;
|
||||
}
|
||||
const file =
|
||||
/^✓\s+(?:\|[^|]+\||\S+)\s+(\S+\.(?:test|spec)\.[cm]?[jt]sx?)(?:\s+>|\s+\(\d+ tests?)/u.exec(
|
||||
body,
|
||||
);
|
||||
if (file && active) {
|
||||
active.files.add(file[1]!);
|
||||
}
|
||||
if (/^Test Files\b/u.test(body)) {
|
||||
if (!active || !/^Test Files\s+1 passed\s+\(1\)$/u.test(body)) {
|
||||
return undefined;
|
||||
}
|
||||
active.summaries += 1;
|
||||
}
|
||||
if (/^Duration\b/u.test(body)) {
|
||||
const duration = /^Duration\s+([\d.]+)(m?s)(?:\s|$)/u.exec(body);
|
||||
if (!active || !duration) {
|
||||
return undefined;
|
||||
}
|
||||
active.durations += 1;
|
||||
active.duration = seconds(duration[1]!, duration[2]!);
|
||||
}
|
||||
}
|
||||
return verified && passed && measured.size === declared.size ? measured : undefined;
|
||||
}
|
||||
|
||||
function readCompactLog(
|
||||
text: string,
|
||||
labels: string[],
|
||||
samples: { blacksmith: Samples; github: Samples },
|
||||
runtimeSamples: { blacksmith: Samples; github: Samples },
|
||||
runtimeDescriptors: Map<string, RuntimePlacementTiming>,
|
||||
workerCeilings: { blacksmith: WorkerCeilings; github: WorkerCeilings },
|
||||
exactInventoryOnly: boolean,
|
||||
) {
|
||||
const profile = labels.some((label) => label.startsWith("blacksmith-")) ? "blacksmith" : "github";
|
||||
const starts = new Map<string, number>();
|
||||
const ambiguousStarts = new Set<string>();
|
||||
const descriptors = readRuntimeTimingGroups(text);
|
||||
const runtimeModes = new Map<string, "runtime" | "private-qa">();
|
||||
const singletonLogs = new Map<string, { key: string; lines: string[] }>();
|
||||
const jobWorkerCeiling = readJobWorkerCeiling(text);
|
||||
const resources = readWorkerResources(text);
|
||||
const runnerEnvironment = readLogEnv(text, "RUNNER_ENVIRONMENT");
|
||||
const frozenTarget = readLogEnv(text, "FROZEN_TARGET");
|
||||
const jobExtraArgs = readLogEnv(text, "OPENCLAW_NODE_TEST_VITEST_ARGS_JSON");
|
||||
for (const line of text.split("\n")) {
|
||||
const output = /^\d{4}-\d\d-\d\dT[\d:.]+Z\s+\[shard:([^\]]+)\]/u.exec(line);
|
||||
if (output) {
|
||||
singletonLogs.get(output[1]!)?.lines.push(line);
|
||||
}
|
||||
const readiness =
|
||||
/\[shard:([^\]]+)\] \[test\] preparing (runtime|private-qa) runtime before Vitest workers/u.exec(
|
||||
line,
|
||||
|
|
@ -156,7 +399,7 @@ function readCompactLog(
|
|||
}
|
||||
}
|
||||
const event =
|
||||
/(\d{4}-\d\d-\d\dT[\d:.]+Z)\s+.*?\[shard:([^\]]+)\] (begin|end \(exit (\d+)\))/u.exec(line);
|
||||
/^(\d{4}-\d\d-\d\dT[\d:.]+Z)\s+\[shard:([^\]]+)\] (begin|end \(exit (\d+)\))$/u.exec(line);
|
||||
if (!event) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -165,24 +408,184 @@ function readCompactLog(
|
|||
const action = event[3]!;
|
||||
const exitCode = event[4];
|
||||
if (action === "begin") {
|
||||
if (starts.has(key)) {
|
||||
ambiguousStarts.add(key);
|
||||
}
|
||||
starts.set(key, Date.parse(timestamp));
|
||||
runtimeModes.delete(key);
|
||||
const matches = descriptors.filter((group) => (group.timing_key ?? group.shard_name) === key);
|
||||
const descriptor = matches.length === 1 ? matches[0] : undefined;
|
||||
if (
|
||||
descriptor?.shard_name.startsWith("changed-extensions-config") &&
|
||||
descriptor.configs.length === 1 &&
|
||||
isRuntimePlacementIncludePatterns(descriptor.includePatterns)
|
||||
) {
|
||||
const processes = splitExtensionTestProcessTargets(
|
||||
descriptor.configs[0]!,
|
||||
descriptor.includePatterns,
|
||||
);
|
||||
if (
|
||||
processes.length === descriptor.includePatterns.length &&
|
||||
processes.every((files) => files.length === 1)
|
||||
) {
|
||||
singletonLogs.set(descriptor.shard_name, { key, lines: [] });
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const started = starts.get(key);
|
||||
if (exitCode === "0" && started !== undefined) {
|
||||
if (exitCode === "0" && started !== undefined && !ambiguousStarts.has(key)) {
|
||||
// Preserve the workload as executed. Packed plans may be serial or
|
||||
// concurrent, and admission must use the wrapper span it actually ran.
|
||||
recordSample(samples[profile], key, (Date.parse(timestamp) - started) / 1000);
|
||||
const matches = descriptors.filter((group) => (group.timing_key ?? group.shard_name) === key);
|
||||
if (matches.length === 1) {
|
||||
const group = matches[0]!;
|
||||
const group = matches.length === 1 ? matches[0] : undefined;
|
||||
// Runtime subsets inherit the envelope's worker pin, but their files
|
||||
// cannot supply a full-envelope runtime placement observation.
|
||||
const workerMatches = descriptors.filter(
|
||||
(entry) =>
|
||||
(entry.timing_key ?? entry.shard_name) === key.replace(/^(?:bun|node-subset):/u, ""),
|
||||
);
|
||||
const workerGroup = workerMatches.length === 1 ? workerMatches[0] : undefined;
|
||||
let workerCeiling = intersectWorkerCeilings(
|
||||
jobWorkerCeiling,
|
||||
parseWorkerCeiling(workerGroup?.env?.OPENCLAW_VITEST_MAX_WORKERS),
|
||||
);
|
||||
const fallback = workerGroup?.fallbackMaxWorkers;
|
||||
if (
|
||||
workerGroup !== undefined &&
|
||||
fallback !== undefined &&
|
||||
(typeof workerCeiling !== "number" || workerCeiling > fallback)
|
||||
) {
|
||||
const eligibleResources =
|
||||
resources &&
|
||||
!isConstrainedCiCheckHost(resources) &&
|
||||
resources.admittedPlans === 1 &&
|
||||
resources.totalMemoryBytes >= (workerGroup.minTotalMemoryBytes ?? 0);
|
||||
const fallbackApplies =
|
||||
(resources && !eligibleResources) ||
|
||||
runnerEnvironment === "github-hosted" ||
|
||||
frozenTarget === "true";
|
||||
const measuredHost =
|
||||
eligibleResources && runnerEnvironment === "self-hosted" && frozenTarget === "false";
|
||||
workerCeiling = fallbackApplies
|
||||
? intersectWorkerCeilings(workerCeiling, fallback)
|
||||
: measuredHost
|
||||
? workerCeiling
|
||||
: null;
|
||||
}
|
||||
const namedWorkers = [...key.matchAll(/#(?:workers|file-parallel)-([1-9]\d*)(?=#|$)/gu)];
|
||||
const splitTiming = parseCompactSplitTimingKey(key);
|
||||
const requiresWorkerCeiling =
|
||||
namedWorkers.length > 0 || (exactInventoryOnly && splitTiming !== undefined);
|
||||
if (
|
||||
(requiresWorkerCeiling && typeof workerCeiling !== "number") ||
|
||||
(typeof workerCeiling === "number" &&
|
||||
namedWorkers.some((match) => Number(match[1]) !== workerCeiling))
|
||||
) {
|
||||
// A fallback or incomplete receipt cannot refresh the requested worker identity.
|
||||
workerCeiling = null;
|
||||
}
|
||||
const extensionGroup = group?.shard_name.startsWith("changed-extensions-config") === true;
|
||||
const selectedFiles = group?.includePatterns;
|
||||
const hasExactSelection =
|
||||
isRuntimePlacementIncludePatterns(selectedFiles) &&
|
||||
(jobExtraArgs === undefined || jobExtraArgs === "[]") &&
|
||||
group?.env?.OPENCLAW_NODE_TEST_VITEST_ARGS_JSON === undefined;
|
||||
let matchesSplitSelection = false;
|
||||
if (group && splitTiming && hasExactSelection) {
|
||||
const selectedKey = createCompactSplitTimingGeneration({
|
||||
configs: group.configs,
|
||||
env: group.env,
|
||||
parentShardName: key,
|
||||
stripes: [selectedFiles],
|
||||
}).timingKeys[0]!;
|
||||
matchesSplitSelection = key.endsWith(
|
||||
selectedKey.slice(selectedKey.lastIndexOf("#include-")),
|
||||
);
|
||||
}
|
||||
let exactKey: string | undefined;
|
||||
if (group && !splitTiming && typeof workerCeiling === "number" && hasExactSelection) {
|
||||
const env = { ...group.env, OPENCLAW_VITEST_MAX_WORKERS: String(workerCeiling) };
|
||||
if (extensionGroup && group.configs.length === 1) {
|
||||
exactKey = createExtensionTestTimingKey(group.configs[0]!, selectedFiles, env);
|
||||
} else if (workerCeiling === 2 && key.endsWith("#file-parallel-2")) {
|
||||
// PR descriptors prove this selected inventory, never an unsplit family total.
|
||||
// Eight-worker command placement rewrites the name after its selector is made.
|
||||
exactKey = createCompactSplitTimingGeneration({
|
||||
configs: group.configs,
|
||||
env,
|
||||
parentShardName: key,
|
||||
stripes: [selectedFiles],
|
||||
}).timingKeys[0];
|
||||
}
|
||||
}
|
||||
const singletonLog = group && singletonLogs.get(group.shard_name);
|
||||
if (
|
||||
extensionGroup &&
|
||||
exactKey &&
|
||||
group &&
|
||||
singletonLog?.key === key &&
|
||||
!runtimeModes.has(key)
|
||||
) {
|
||||
const invocations = readSingletonExtensionInvocations(
|
||||
singletonLog.lines,
|
||||
group.configs[0]!,
|
||||
group.includePatterns!,
|
||||
);
|
||||
if (invocations) {
|
||||
const total = [...invocations.values()].reduce((sum, value) => sum + value, 0);
|
||||
const overhead = (Date.parse(timestamp) - started) / 1000 - total;
|
||||
if (overhead >= 0) {
|
||||
const env = { ...group.env, OPENCLAW_VITEST_MAX_WORKERS: String(workerCeiling) };
|
||||
for (const [file, duration] of invocations) {
|
||||
recordSample(
|
||||
samples[profile],
|
||||
createExtensionTestTimingKey(
|
||||
group.configs[0]!,
|
||||
[file],
|
||||
env,
|
||||
"singleton-invocation",
|
||||
)!,
|
||||
duration,
|
||||
);
|
||||
}
|
||||
recordSample(
|
||||
samples[profile],
|
||||
createExtensionTestTimingKey(group.configs[0]!, [], env, "wrapper-overhead")!,
|
||||
overhead,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const measuredKeys = [
|
||||
...(!extensionGroup && (!exactInventoryOnly || matchesSplitSelection) ? [key] : []),
|
||||
...(exactKey ? [exactKey] : []),
|
||||
];
|
||||
for (const measuredKey of measuredKeys) {
|
||||
const measuredSplit = parseCompactSplitTimingKey(measuredKey);
|
||||
for (const identity of [measuredKey, measuredSplit?.parentShardName]) {
|
||||
if (identity === undefined) {
|
||||
continue;
|
||||
}
|
||||
const observed = workerCeilings[profile].get(identity) ?? new Set();
|
||||
observed.add(workerCeiling === null ? "ambiguous" : (workerCeiling ?? "unspecified"));
|
||||
workerCeilings[profile].set(identity, observed);
|
||||
}
|
||||
// An unsplit PR key does not prove that its full owner inventory ran.
|
||||
recordSample(samples[profile], measuredKey, (Date.parse(timestamp) - started) / 1000);
|
||||
}
|
||||
if (group && workerCeiling !== null) {
|
||||
const observation = {
|
||||
configs: group.configs,
|
||||
env: Object.fromEntries(
|
||||
Object.entries(group.env ?? {}).toSorted(([a], [b]) => a.localeCompare(b)),
|
||||
Object.entries({
|
||||
...group.env,
|
||||
...(workerCeiling === undefined
|
||||
? {}
|
||||
: { OPENCLAW_VITEST_MAX_WORKERS: String(workerCeiling) }),
|
||||
}).toSorted(([a], [b]) => a.localeCompare(b)),
|
||||
),
|
||||
includePatterns: group.includePatterns.toSorted(),
|
||||
includePatterns: group.includePatterns?.toSorted(),
|
||||
pretestBuildMode: runtimeModes.get(key),
|
||||
seconds: Math.max(1, Math.round((Date.parse(timestamp) - started) / 1000)),
|
||||
};
|
||||
|
|
@ -194,6 +597,11 @@ function readCompactLog(
|
|||
}
|
||||
}
|
||||
starts.delete(key);
|
||||
for (const [shard, log] of singletonLogs) {
|
||||
if (log.key === key) {
|
||||
singletonLogs.delete(shard);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -229,12 +637,12 @@ function readToolingLog(text: string, samples: Samples) {
|
|||
cases: new Map(),
|
||||
files: new Map(),
|
||||
complete: false,
|
||||
declaredFiles: new Set(descriptor.includePatterns),
|
||||
declaredFiles: new Set(descriptor.includePatterns ?? []),
|
||||
singletonFile:
|
||||
!active.has(shard) &&
|
||||
descriptor.configs.length === 1 &&
|
||||
descriptor.configs[0] === "test/vitest/vitest.tooling.config.ts" &&
|
||||
descriptor.includePatterns.length === 1
|
||||
descriptor.includePatterns?.length === 1
|
||||
? descriptor.includePatterns[0]
|
||||
: undefined,
|
||||
fileSummaryCount: 0,
|
||||
|
|
@ -322,7 +730,11 @@ function runtimePlacementSecondsMap(observations: readonly RuntimePlacementTimin
|
|||
);
|
||||
}
|
||||
|
||||
function recordCompleteParentSamples(samples: Samples, observedParents: Set<string>) {
|
||||
function recordCompleteParentSamples(
|
||||
samples: Samples,
|
||||
observedParents: Set<string>,
|
||||
foldParents: boolean,
|
||||
) {
|
||||
const generations = new Map<
|
||||
string,
|
||||
{ parent: string; expected: number; parts: Map<number, number> }
|
||||
|
|
@ -333,6 +745,11 @@ function recordCompleteParentSamples(samples: Samples, observedParents: Set<stri
|
|||
continue;
|
||||
}
|
||||
observedParents.add(parsed.parentShardName);
|
||||
// A selected PR subset can share the reduced full inventory's parent name.
|
||||
// Its exact child key is evidence; completeness of that subset is not.
|
||||
if (!foldParents || parsed.parentShardName.startsWith("extension-test:")) {
|
||||
continue;
|
||||
}
|
||||
const generation = generations.get(parsed.generationKey) ?? {
|
||||
parent: parsed.parentShardName,
|
||||
expected: parsed.expectedParts,
|
||||
|
|
@ -416,6 +833,10 @@ export function refitTestTimings(
|
|||
blacksmith: new Map<string, number[]>(),
|
||||
github: new Map<string, number[]>(),
|
||||
};
|
||||
const workerCeilings = {
|
||||
blacksmith: new Map<string, Set<number | "unspecified" | "ambiguous">>(),
|
||||
github: new Map<string, Set<number | "unspecified" | "ambiguous">>(),
|
||||
};
|
||||
const runtimeDescriptors = new Map<string, RuntimePlacementTiming>(
|
||||
Object.values(previous?.runtimePlacementTimings ?? {})
|
||||
.flat()
|
||||
|
|
@ -427,6 +848,7 @@ export function refitTestTimings(
|
|||
if (retained) {
|
||||
retained.logs.push(...run.logs);
|
||||
retained.completeInventory = retained.completeInventory && run.completeInventory;
|
||||
retained.pullRequestMergeRef ||= run.pullRequestMergeRef;
|
||||
} else {
|
||||
uniqueRuns.set(run.id, { ...run, logs: [...run.logs] });
|
||||
}
|
||||
|
|
@ -445,20 +867,43 @@ export function refitTestTimings(
|
|||
github: new Map<string, number[]>(),
|
||||
};
|
||||
for (const log of run.logs) {
|
||||
const text = stripVTControlCharacters(log.text);
|
||||
// `gh run view --log` adds job/step columns outside the timestamped record.
|
||||
// A timestamped child line may quote that format; its contents stay nested.
|
||||
const text = stripVTControlCharacters(log.text).replace(
|
||||
/^(?!\d{4}-\d\d-\d\dT[\d:.]+Z(?:\s|$))[^\t\r\n]+\t[^\t\r\n]+\t(?=\d{4}-\d\d-\d\dT[\d:.]+Z(?:\s|$))/gmu,
|
||||
"",
|
||||
);
|
||||
if (log.kind === "tooling") {
|
||||
const profile = log.labels.some((label) => label.startsWith("blacksmith-"))
|
||||
? "toolingBlacksmith"
|
||||
: "toolingGithub";
|
||||
readToolingLog(text, current[profile]);
|
||||
} else if (log.kind === "compact") {
|
||||
readCompactLog(text, log.labels, current, currentRuntime, runtimeDescriptors);
|
||||
readCompactLog(
|
||||
text,
|
||||
log.labels,
|
||||
current,
|
||||
currentRuntime,
|
||||
runtimeDescriptors,
|
||||
workerCeilings,
|
||||
run.pullRequestMergeRef === true,
|
||||
);
|
||||
if (run.pullRequestMergeRef) {
|
||||
const profile = log.labels.some((label) => label.startsWith("blacksmith-"))
|
||||
? "toolingBlacksmith"
|
||||
: "toolingGithub";
|
||||
readToolingLog(text, current[profile]);
|
||||
}
|
||||
} else {
|
||||
readE2eLog(text, current[log.kind], log.kind === "uiE2e" ? overhead : undefined);
|
||||
}
|
||||
}
|
||||
for (const profile of ["blacksmith", "github"] as const) {
|
||||
recordCompleteParentSamples(current[profile], observedParents[profile]);
|
||||
recordCompleteParentSamples(
|
||||
current[profile],
|
||||
observedParents[profile],
|
||||
!run.pullRequestMergeRef,
|
||||
);
|
||||
for (const [identity, values] of currentRuntime[profile]) {
|
||||
recordSample(runtimeSamples[profile], identity, median(values));
|
||||
}
|
||||
|
|
@ -482,6 +927,20 @@ export function refitTestTimings(
|
|||
}
|
||||
}
|
||||
|
||||
const rejectedWorkerKeys = { blacksmith: [] as string[], github: [] as string[] };
|
||||
for (const profile of ["blacksmith", "github"] as const) {
|
||||
for (const [key, ceilings] of workerCeilings[profile]) {
|
||||
if (ceilings.size > 1 || ceilings.has("ambiguous")) {
|
||||
// Historical keys can omit inherited job caps. Do not average unlike
|
||||
// execution policies or turn rejected evidence into a pruning signal.
|
||||
samples[profile].delete(key);
|
||||
observedParents[profile].add(key);
|
||||
rejectedWorkerKeys[profile].push(key);
|
||||
}
|
||||
}
|
||||
rejectedWorkerKeys[profile].sort();
|
||||
}
|
||||
|
||||
const completeInventoryRuns = new Set(
|
||||
[...uniqueRuns.values()].filter((run) => run.completeInventory).map((run) => run.id),
|
||||
);
|
||||
|
|
@ -495,6 +954,7 @@ export function refitTestTimings(
|
|||
measuredOverhead === undefined ||
|
||||
(oldOverhead !== undefined && Math.abs(measuredOverhead - oldOverhead) <= oldOverhead * 0.15);
|
||||
const runIds = [...new Set(runs.map((run) => run.id))].toSorted((a, b) => a - b);
|
||||
const pullRequestRunIds = runs.filter((run) => run.pullRequestMergeRef).map((run) => run.id);
|
||||
function refitRuntime(profile: "blacksmith" | "github"): RuntimePlacementTiming[] {
|
||||
return Object.entries(
|
||||
refitMap(
|
||||
|
|
@ -534,7 +994,7 @@ export function refitTestTimings(
|
|||
},
|
||||
source: options.seedTooling
|
||||
? `tooling seed from successful pull_request CI merge-ref runs: ${runIds.join(", ")}; retained other timings: ${previous?.source ?? "none"}`
|
||||
: `median of successful timing jobs from ${runIds.length} CI and release-check runs: ${runIds.join(", ")}`,
|
||||
: `median of successful timing jobs from ${runIds.length} CI and release-check runs: ${runIds.join(", ")}${pullRequestRunIds.length > 0 ? `; pull_request merge-ref runs: ${[...new Set(pullRequestRunIds)].toSorted((a, b) => a - b).join(", ")}` : ""}`,
|
||||
// PR plans may select only part of tooling. Absence is not evidence that
|
||||
// a file disappeared; preserve unobserved measurements across those windows.
|
||||
toolingFileSeconds: {
|
||||
|
|
@ -622,6 +1082,7 @@ export function refitTestTimings(
|
|||
timings,
|
||||
changes: changes.toSorted((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)),
|
||||
runIds,
|
||||
rejectedWorkerKeys,
|
||||
contributingRunIds: {
|
||||
blacksmith: [...contributingRuns.blacksmith].toSorted((a, b) => a - b),
|
||||
github: [...contributingRuns.github].toSorted((a, b) => a - b),
|
||||
|
|
|
|||
|
|
@ -34,8 +34,11 @@ import { isSharedVitestExcludedPath } from "../../test/vitest/vitest.pattern-fil
|
|||
import { isPluginControlUiPath } from "../../test/vitest/vitest.ui-paths.mjs";
|
||||
import { BUNDLED_PLUGIN_PATH_PREFIX, BUNDLED_PLUGIN_ROOT_DIR } from "./bundled-plugin-paths.mjs";
|
||||
import { listAvailableExtensionIds } from "./changed-extensions.mts";
|
||||
import { isRuntimePlacementIncludePatterns } from "./ci-test-timings-schema.mts";
|
||||
import { readCompactGroupTimings } from "./ci-test-timings.mts";
|
||||
import { GIT_LS_FILES_MAX_BUFFER_BYTES } from "./list-test-files.mts";
|
||||
import { parsePositiveInt } from "./numeric-options.mjs";
|
||||
import { createCompactSplitTimingGeneration } from "./vitest-shard-metadata.mts";
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, "..", "..");
|
||||
const TRACKED_EXTENSION_TEST_PATHSPECS = [
|
||||
|
|
@ -418,6 +421,31 @@ function countTestFiles(rootPath: string) {
|
|||
return listFilesystemTestFiles(rootPath).length;
|
||||
}
|
||||
|
||||
/** Exact envelope identity survives ordinal renumbering without crossing worker policies. */
|
||||
export function createExtensionTestTimingKey(
|
||||
config: string,
|
||||
files: readonly string[],
|
||||
env: Readonly<Record<string, string>> = { OPENCLAW_VITEST_MAX_WORKERS: "2" },
|
||||
kind: "envelope" | "singleton-invocation" | "wrapper-overhead" = "envelope",
|
||||
): string | undefined {
|
||||
if (
|
||||
!/^test\/vitest\/vitest\.extensions?(?:-[^/]+)?\.config\.ts$/u.test(config) ||
|
||||
(kind === "wrapper-overhead"
|
||||
? files.length !== 0
|
||||
: !isRuntimePlacementIncludePatterns(files) ||
|
||||
(kind === "singleton-invocation" && files.length !== 1)) ||
|
||||
!/^[1-9]\d*$/u.test(env.OPENCLAW_VITEST_MAX_WORKERS ?? "")
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return createCompactSplitTimingGeneration({
|
||||
configs: [config],
|
||||
env,
|
||||
parentShardName: `extension-test:${config}#workers-${env.OPENCLAW_VITEST_MAX_WORKERS}${kind === "envelope" ? "" : `#${kind}`}`,
|
||||
stripes: [files],
|
||||
}).timingKeys[0];
|
||||
}
|
||||
|
||||
export function estimateExtensionTestCost(
|
||||
config: string,
|
||||
testFileCount: number,
|
||||
|
|
@ -438,7 +466,41 @@ export function estimateExtensionTestCost(
|
|||
config === DATABASE_WORKER_CONFIG
|
||||
? files.filter((file) => file.startsWith("extensions/codex/src/app-server/")).length
|
||||
: 0;
|
||||
return Math.max(1, Math.ceil(testFileCount * multiplier + appServerFiles * (17.31 - multiplier)));
|
||||
const key =
|
||||
files.length === testFileCount ? createExtensionTestTimingKey(config, files) : undefined;
|
||||
const timings = readCompactGroupTimings("blacksmith");
|
||||
const measured = key ? timings[key] : undefined;
|
||||
const processes = key ? splitExtensionTestProcessTargets(config, [...files]) : [];
|
||||
let singletonSeconds = 0;
|
||||
if (processes.length === files.length && processes.every((process) => process.length === 1)) {
|
||||
let observed = false;
|
||||
for (const file of files) {
|
||||
const fileKey = createExtensionTestTimingKey(
|
||||
config,
|
||||
[file],
|
||||
undefined,
|
||||
"singleton-invocation",
|
||||
)!;
|
||||
const fileSeconds = timings[fileKey];
|
||||
observed ||= fileSeconds !== undefined;
|
||||
singletonSeconds += Math.max(
|
||||
fileSeconds ?? 0,
|
||||
config === DATABASE_WORKER_CONFIG && file.startsWith("extensions/codex/src/app-server/")
|
||||
? 17.31
|
||||
: multiplier,
|
||||
);
|
||||
}
|
||||
const overheadKey = createExtensionTestTimingKey(config, [], undefined, "wrapper-overhead");
|
||||
singletonSeconds = observed
|
||||
? singletonSeconds + (overheadKey ? (timings[overheadKey] ?? 0) : 0)
|
||||
: 0;
|
||||
}
|
||||
return Math.max(
|
||||
1,
|
||||
measured ?? 0,
|
||||
Math.ceil(singletonSeconds),
|
||||
Math.ceil(testFileCount * multiplier + appServerFiles * (17.31 - multiplier)),
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolve the dedicated Vitest config for an extension root or test file. */
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
.github/workflows/pr-crabbox-gate-publisher.yml
|
||||
cli-root-options.mjs
|
||||
config/ci-test-timings.json
|
||||
node-compile-cache.mjs
|
||||
node-sqlite.mjs
|
||||
node-version.mjs
|
||||
|
|
@ -118,6 +119,8 @@ scripts/lib/arg-utils.runtime.mjs
|
|||
scripts/lib/bundled-plugin-paths.mjs
|
||||
scripts/lib/changed-extensions.mts
|
||||
scripts/lib/changed-path-facts.mjs
|
||||
scripts/lib/ci-test-timings-schema.mts
|
||||
scripts/lib/ci-test-timings.mts
|
||||
scripts/lib/direct-run.mjs
|
||||
scripts/lib/error-format.mts
|
||||
scripts/lib/extension-test-plan.mts
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import {
|
|||
type RuntimePlacementTiming,
|
||||
} from "../../scripts/lib/ci-test-timings-schema.mts";
|
||||
import * as testTimings from "../../scripts/lib/ci-test-timings.mts";
|
||||
import { createExtensionTestTimingKey } from "../../scripts/lib/extension-test-plan.mts";
|
||||
import * as localCheckRuntime from "../../scripts/lib/local-check-runtime.mts";
|
||||
import { createCompactSplitTimingGeneration } from "../../scripts/lib/vitest-shard-metadata.mts";
|
||||
import {
|
||||
|
|
@ -84,6 +85,148 @@ const baseline: CiTestTimings = {
|
|||
|
||||
const sampleNow = "2026-08-28T12:00:00.000Z";
|
||||
|
||||
describe("native singleton invocation timings", () => {
|
||||
const config = "test/vitest/vitest.extension-database-workers.config.ts";
|
||||
const files = ["extensions/telegram/src/one.test.ts", "extensions/telegram/src/two.test.ts"];
|
||||
const shard = "changed-extensions-config-1";
|
||||
const descriptor = { configs: [config], includePatterns: files, shard_name: shard };
|
||||
const line = (second: number, body: string) =>
|
||||
`${new Date(Date.parse("2026-09-26T00:00:00Z") + second * 1000).toISOString()} [shard:${shard}] ${body}`;
|
||||
const invocationLog = () =>
|
||||
[
|
||||
"2026-09-26T00:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2",
|
||||
`2026-09-26T00:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor])}`,
|
||||
line(0, "begin"),
|
||||
line(1, `[test] starting ${config}`),
|
||||
line(2, "RUN v5.0.1 /checkout"),
|
||||
line(3, `✓ extension-database-workers ${files[0]} > first case 1ms`),
|
||||
line(9, "Test Files 1 passed (1)"),
|
||||
line(9, "Duration 8s (tests 60%, import 40%)"),
|
||||
line(11, `[test] starting ${config}`),
|
||||
line(12, "RUN v5.0.1 /checkout"),
|
||||
line(14, `✓ extension-database-workers ${files[1]} > second case 2ms`),
|
||||
line(30, "Test Files 1 passed (1)"),
|
||||
line(30, "Duration 18s (tests 70%, import 30%)"),
|
||||
line(32, "[vitest-workers] verifying completed generation before cleanup"),
|
||||
line(33, "[test] passed 2 Vitest shards in 32s"),
|
||||
line(34, "end (exit 0)"),
|
||||
].join("\n");
|
||||
const refit = (text: string, pullRequestMergeRef = true) =>
|
||||
refitTestTimings(
|
||||
[1, 2].map((id) =>
|
||||
Object.assign(
|
||||
timingRun(id, [{ kind: "compact", labels: ["blacksmith-8vcpu-ubuntu-2404"], text }]),
|
||||
{ pullRequestMergeRef, completeInventory: !pullRequestMergeRef },
|
||||
),
|
||||
),
|
||||
).timings.compactGroupSeconds.blacksmith;
|
||||
|
||||
it.each([false, true])(
|
||||
"retains invocation walls and shared overhead without parent folding (PR: %s)",
|
||||
(pullRequestMergeRef) => {
|
||||
expect(refit(invocationLog(), pullRequestMergeRef)).toEqual({
|
||||
[createExtensionTestTimingKey(config, files)!]: 34,
|
||||
[createExtensionTestTimingKey(config, [files[0]!], undefined, "singleton-invocation")!]: 10,
|
||||
[createExtensionTestTimingKey(config, [files[1]!], undefined, "singleton-invocation")!]: 21,
|
||||
[createExtensionTestTimingKey(config, [], undefined, "wrapper-overhead")!]: 3,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps an outer singleton wall distinct from its invocation and shared overhead", () => {
|
||||
const only = [files[0]!];
|
||||
const text = [
|
||||
"2026-09-26T00:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2",
|
||||
`2026-09-26T00:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([{ ...descriptor, includePatterns: only }])}`,
|
||||
line(0, "begin"),
|
||||
line(1, `[test] starting ${config}`),
|
||||
line(2, "RUN v5.0.1 /checkout"),
|
||||
line(19, `✓ extension-database-workers ${only[0]} > case 1ms`),
|
||||
line(20, "Test Files 1 passed (1)"),
|
||||
line(20, "Duration 18s"),
|
||||
line(21, "[vitest-workers] verifying completed generation before cleanup"),
|
||||
line(21, "[test] passed 1 Vitest shard in 21s"),
|
||||
line(22, "end (exit 0)"),
|
||||
].join("\n");
|
||||
expect(refit(text)).toEqual({
|
||||
[createExtensionTestTimingKey(config, only)!]: 22,
|
||||
[createExtensionTestTimingKey(config, only, undefined, "singleton-invocation")!]: 20,
|
||||
[createExtensionTestTimingKey(config, [], undefined, "wrapper-overhead")!]: 2,
|
||||
});
|
||||
const noOverhead = text
|
||||
.replace(line(0, "begin"), line(1, "begin"))
|
||||
.replace(line(22, "end (exit 0)"), line(21, "end (exit 0)"));
|
||||
expect(
|
||||
refit(noOverhead)[createExtensionTestTimingKey(config, [], undefined, "wrapper-overhead")!],
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ name: "failed outer group", replace: "end (exit 0)", with: "end (exit 1)" },
|
||||
{
|
||||
name: "invalid timestamp",
|
||||
replace: line(11, `[test] starting ${config}`),
|
||||
with: `2026-99-26T00:00:11Z [shard:${shard}] [test] starting ${config}`,
|
||||
},
|
||||
{ name: "missing completion", replace: "[test] passed 2 Vitest shards in 32s", with: "" },
|
||||
{ name: "duplicate file", replace: files[1]!, with: files[0]! },
|
||||
{ name: "foreign file", replace: files[1]!, with: "extensions/telegram/src/foreign.test.ts" },
|
||||
{
|
||||
name: "wrong config",
|
||||
replace: `[test] starting ${config}`,
|
||||
with: "[test] starting test/vitest/vitest.extension-codex.config.ts",
|
||||
},
|
||||
{
|
||||
name: "multi-file invocation",
|
||||
replace: "Test Files 1 passed (1)",
|
||||
with: "Test Files 2 passed (2)",
|
||||
},
|
||||
{
|
||||
name: "duplicate duration",
|
||||
replace: line(9, "Duration 8s (tests 60%, import 40%)"),
|
||||
with: [line(9, "Duration 8s"), line(9, "Duration 8s")].join("\n"),
|
||||
},
|
||||
{
|
||||
name: "invalid duration",
|
||||
replace: "Duration 8s (tests 60%, import 40%)",
|
||||
with: "Duration 1.2.3s",
|
||||
},
|
||||
{
|
||||
name: "duplicate start",
|
||||
replace: line(1, `[test] starting ${config}`),
|
||||
with: [line(1, `[test] starting ${config}`), line(1, `[test] starting ${config}`)].join("\n"),
|
||||
},
|
||||
{
|
||||
name: "ambiguous worker environment",
|
||||
replace: "OPENCLAW_VITEST_MAX_WORKERS: 2",
|
||||
with: "OPENCLAW_VITEST_MAX_WORKERS: 2\n2026-09-26T00:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 8",
|
||||
},
|
||||
{
|
||||
name: "unsupported arguments",
|
||||
replace: "OPENCLAW_VITEST_MAX_WORKERS: 2",
|
||||
with: 'OPENCLAW_VITEST_MAX_WORKERS: 2\n2026-09-26T00:00:00Z OPENCLAW_NODE_TEST_VITEST_ARGS_JSON: ["--exclude=**/two.test.ts"]',
|
||||
},
|
||||
])("rejects derived singleton prices atomically for $name", (mutation) => {
|
||||
const result = refit(invocationLog().replace(mutation.replace, mutation.with));
|
||||
expect(
|
||||
Object.keys(result).filter(
|
||||
(key) => key.includes("#include-1-") || key.includes("#wrapper-overhead"),
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not infer singleton prices for a multi-file process owner", () => {
|
||||
const ordinaryConfig = "test/vitest/vitest.extension-telegram.config.ts";
|
||||
const text = invocationLog()
|
||||
.replace(
|
||||
encodeNodeTestGroups([descriptor]),
|
||||
encodeNodeTestGroups([{ ...descriptor, configs: [ordinaryConfig] }]),
|
||||
)
|
||||
.replaceAll(`[test] starting ${config}`, `[test] starting ${ordinaryConfig}`);
|
||||
expect(refit(text)).toEqual({ [createExtensionTestTimingKey(ordinaryConfig, files)!]: 34 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime placement observations", () => {
|
||||
it("retains recorded runtime work when its current group gains a file", () => {
|
||||
const options = {
|
||||
|
|
@ -217,9 +360,15 @@ describe("runtime placement observations", () => {
|
|||
shard_name: `reader-${id}`,
|
||||
timing_key: generation.timingKeys[0]!,
|
||||
};
|
||||
const sibling = {
|
||||
...descriptor,
|
||||
shard_name: `ordinary-${id}`,
|
||||
includePatterns: [`src/ordinary-${id}.test.ts`],
|
||||
timing_key: generation.timingKeys[1]!,
|
||||
};
|
||||
const [begin, end] = compactLog(20, descriptor.timing_key).split("\n");
|
||||
return [
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor])}`,
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor, sibling])}`,
|
||||
begin,
|
||||
`2026-08-27T23:00:01Z [shard:${descriptor.shard_name}] [test] preparing ${group.pretestBuildMode} runtime before Vitest workers`,
|
||||
end,
|
||||
|
|
@ -684,11 +833,14 @@ function samplerJob(id: number, runId: number, overrides: Record<string, unknown
|
|||
|
||||
const toolingFile = "test/scripts/measured.test.ts";
|
||||
|
||||
function samplerToolingLog(seconds: number) {
|
||||
function samplerToolingLog(
|
||||
seconds: number,
|
||||
additionalGroups: Parameters<typeof encodeNodeTestGroups>[0] = [],
|
||||
) {
|
||||
const shard = "core-tooling-1-hosted-1";
|
||||
const [begin, end] = compactLog(seconds + 1, shard).split("\n");
|
||||
return [
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([{ shard_name: shard, configs: ["test/vitest/vitest.tooling.config.ts"], includePatterns: [toolingFile] }])}`,
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([{ shard_name: shard, configs: ["test/vitest/vitest.tooling.config.ts"], includePatterns: [toolingFile] }, ...additionalGroups])}`,
|
||||
begin,
|
||||
`2026-08-27T23:00:01Z [shard:${shard}] ✓ tooling ${toolingFile} (1 test) ${seconds * 1000}ms`,
|
||||
`2026-08-27T23:00:01Z [shard:${shard}] Duration ${seconds + 1}s`,
|
||||
|
|
@ -1062,6 +1214,413 @@ it.todo("retains todo coverage");
|
|||
});
|
||||
});
|
||||
|
||||
it.each(["plugin bundle", "plugin singleton", "Doctor"] as const)(
|
||||
"refits complete explicit %s PR work without trusting ordinal or family names",
|
||||
(kind) => {
|
||||
const plugin = kind !== "Doctor";
|
||||
const configs = [
|
||||
plugin
|
||||
? "test/vitest/vitest.extension-database-workers.config.ts"
|
||||
: "test/vitest/vitest.commands.config.ts",
|
||||
];
|
||||
const files = plugin
|
||||
? ["extensions/telegram/src/native-a.test.ts", "extensions/telegram/src/native-b.test.ts"]
|
||||
: ["src/commands/doctor-a.test.ts", "src/commands/doctor-b.test.ts"];
|
||||
const env = { OPENCLAW_VITEST_MAX_WORKERS: "2" };
|
||||
const parent = "changed-agentic-commands-doctor#file-parallel-2";
|
||||
const key = plugin
|
||||
? createExtensionTestTimingKey(configs[0]!, files, env)!
|
||||
: createCompactSplitTimingGeneration({
|
||||
configs,
|
||||
env,
|
||||
parentShardName: parent,
|
||||
stripes: [files],
|
||||
}).timingKeys[0]!;
|
||||
const runs = [1, 2].map((id) => {
|
||||
const name = plugin ? `changed-extensions-config-${id * 20}` : parent;
|
||||
const descriptor = { configs, includePatterns: files.toReversed(), shard_name: name };
|
||||
const text = [
|
||||
"2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2",
|
||||
...(kind === "plugin singleton"
|
||||
? [
|
||||
`2026-08-27T23:00:00Z OPENCLAW_VITEST_SHARD_NAME: ${name}`,
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_CONFIGS_JSON: ${JSON.stringify(configs, null, 2)}`,
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_INCLUDE_PATTERNS_JSON: ${JSON.stringify(files, null, 2)}`,
|
||||
"2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_ENV_JSON: null",
|
||||
]
|
||||
: [
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor])}`,
|
||||
]),
|
||||
compactLog(id === 1 ? 180 : 220, name),
|
||||
// Nested tests cannot overwrite their containing job's resource/worker provenance.
|
||||
"2026-08-27T23:04:00Z [shard:fixture] OPENCLAW_VITEST_MAX_WORKERS: 8",
|
||||
`2026-08-27T23:04:01Z [shard:fixture] [shard:${name}] begin`,
|
||||
`2026-08-27T23:59:00Z [shard:fixture] [shard:${name}] end (exit 0)`,
|
||||
`2026-08-27T23:04:01Z [shard:fixture] job\tstep\t2026-08-27T23:04:01Z [shard:${name}] begin`,
|
||||
`2026-08-27T23:59:00Z [shard:fixture] job\tstep\t2026-08-27T23:59:00Z [shard:${name}] end (exit 0)`,
|
||||
].join("\n");
|
||||
return Object.assign(
|
||||
timingRun(id, [
|
||||
{ kind: "compact" as const, labels: ["blacksmith-8vcpu-ubuntu-2404"], text },
|
||||
]),
|
||||
{ completeInventory: false, pullRequestMergeRef: true },
|
||||
);
|
||||
});
|
||||
expect(refitTestTimings([runs[0]!, runs[0]!]).timings.compactGroupSeconds.blacksmith).toEqual(
|
||||
{},
|
||||
);
|
||||
const result = refitTestTimings(runs);
|
||||
expect(result.timings.compactGroupSeconds.blacksmith).toEqual({ [key]: 200 });
|
||||
expect(result.timings.runtimePlacementTimings.blacksmith).toEqual([]);
|
||||
if (plugin) {
|
||||
for (const run of runs) {
|
||||
run.completeInventory = true;
|
||||
run.pullRequestMergeRef = false;
|
||||
}
|
||||
expect(refitTestTimings(runs).timings.compactGroupSeconds.blacksmith).toEqual({
|
||||
[key]: 200,
|
||||
});
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
"missing workers",
|
||||
"duplicate descriptor",
|
||||
"glob",
|
||||
"duplicate file",
|
||||
"native shard",
|
||||
"partial runtime",
|
||||
"mismatched workers",
|
||||
])("rejects unsplit PR timing with %s evidence", (reason) => {
|
||||
const key =
|
||||
reason === "mismatched workers" ? "doctor#file-parallel-2" : "changed-extensions-config-1";
|
||||
const group = {
|
||||
configs: ["test/vitest/vitest.extension-database-workers.config.ts"],
|
||||
includePatterns:
|
||||
reason === "glob"
|
||||
? ["extensions/telegram/*.test.ts"]
|
||||
: ["extensions/telegram/native.test.ts"],
|
||||
shard_name: key,
|
||||
...(reason === "native shard"
|
||||
? { env: { OPENCLAW_NODE_TEST_VITEST_ARGS_JSON: '["--shard=1/2"]' } }
|
||||
: {}),
|
||||
};
|
||||
if (reason === "duplicate file") {
|
||||
group.includePatterns.push(group.includePatterns[0]!);
|
||||
}
|
||||
const text = [
|
||||
...(reason === "missing workers"
|
||||
? []
|
||||
: [
|
||||
`2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: ${reason === "mismatched workers" ? 8 : 2}`,
|
||||
]),
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups(reason === "duplicate descriptor" ? [group, group] : [group])}`,
|
||||
compactLog(200, reason === "partial runtime" ? `node-subset:${key}` : key),
|
||||
].join("\n");
|
||||
const runs = [1, 2].map((id) =>
|
||||
Object.assign(
|
||||
timingRun(id, [
|
||||
{ kind: "compact" as const, labels: ["blacksmith-8vcpu-ubuntu-2404"], text },
|
||||
]),
|
||||
{ completeInventory: false, pullRequestMergeRef: true },
|
||||
),
|
||||
);
|
||||
expect(refitTestTimings(runs).timings.compactGroupSeconds.blacksmith).toEqual({});
|
||||
});
|
||||
|
||||
it("keeps mixed worker ceilings out of compact weights and keys runtime observations by their inherited ceiling", () => {
|
||||
const descriptor = {
|
||||
configs: ["test/vitest/reader.config.ts"],
|
||||
includePatterns: ["src/reader.test.ts"],
|
||||
shard_name: "reader",
|
||||
timing_key: createCompactSplitTimingGeneration({
|
||||
configs: ["test/vitest/reader.config.ts"],
|
||||
parentShardName: "reader-parent",
|
||||
stripes: [["src/reader.test.ts"]],
|
||||
}).timingKeys[0]!,
|
||||
};
|
||||
const previous: CiTestTimings = {
|
||||
...baseline,
|
||||
compactGroupSeconds: {
|
||||
blacksmith: { [descriptor.timing_key]: 100, "reader-parent": 100 },
|
||||
github: {},
|
||||
},
|
||||
};
|
||||
const log = (workers: number, groupWorkers?: number, jobWorkers?: number) => {
|
||||
const [begin, end] = compactLog(40, descriptor.timing_key).split("\n");
|
||||
return [
|
||||
`2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: ${workers}`,
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_ENV_JSON: ${JSON.stringify(jobWorkers ? { OPENCLAW_VITEST_MAX_WORKERS: String(jobWorkers) } : null, null, 2)}`,
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([{ ...descriptor, ...(groupWorkers ? { env: { OPENCLAW_VITEST_MAX_WORKERS: String(groupWorkers) } } : {}) }])}`,
|
||||
begin,
|
||||
"2026-08-27T23:00:01Z [shard:reader] [test] preparing runtime runtime before Vitest workers",
|
||||
end,
|
||||
].join("\n");
|
||||
};
|
||||
const runs = [2, 8, 8].map((workers, index) =>
|
||||
timingRun(index + 1, [
|
||||
{ kind: "compact", labels: ["blacksmith-32vcpu-ubuntu-2404"], text: log(workers) },
|
||||
{ kind: "compact", labels: ["ubuntu-24.04"], text: log(4) },
|
||||
]),
|
||||
);
|
||||
const mixed = refitTestTimings(runs, previous);
|
||||
expect(mixed.timings.compactGroupSeconds).toEqual({
|
||||
blacksmith: previous.compactGroupSeconds.blacksmith,
|
||||
github: { [descriptor.timing_key]: 40, "reader-parent": 40 },
|
||||
});
|
||||
expect(mixed.rejectedWorkerKeys).toEqual({
|
||||
blacksmith: ["reader-parent", descriptor.timing_key].toSorted(),
|
||||
github: [],
|
||||
});
|
||||
expect(mixed.timings.runtimePlacementTimings.blacksmith).toEqual([
|
||||
{
|
||||
configs: descriptor.configs,
|
||||
includePatterns: descriptor.includePatterns,
|
||||
env: { OPENCLAW_VITEST_MAX_WORKERS: "8" },
|
||||
pretestBuildMode: "runtime",
|
||||
seconds: 40,
|
||||
},
|
||||
]);
|
||||
// Both the job JSON and group pins can lower, but never raise, an inherited ceiling.
|
||||
const pinned = refitTestTimings(
|
||||
[log(8, 2), log(2, 8), log(8, 8, 2)].map((text, index) =>
|
||||
timingRun(index + 1, [
|
||||
{ kind: "compact", labels: ["blacksmith-32vcpu-ubuntu-2404"], text },
|
||||
]),
|
||||
),
|
||||
);
|
||||
expect(pinned.rejectedWorkerKeys.blacksmith).toEqual([]);
|
||||
expect(pinned.timings.compactGroupSeconds.blacksmith).toEqual({
|
||||
[descriptor.timing_key]: 40,
|
||||
"reader-parent": 40,
|
||||
});
|
||||
expect(pinned.timings.runtimePlacementTimings.blacksmith[0]?.env).toEqual({
|
||||
OPENCLAW_VITEST_MAX_WORKERS: "2",
|
||||
});
|
||||
const subset = refitTestTimings(
|
||||
[log(8, 2), log(2, 8)].map((text, index) =>
|
||||
timingRun(index + 1, [
|
||||
{
|
||||
kind: "compact",
|
||||
labels: ["blacksmith-32vcpu-ubuntu-2404"],
|
||||
text: text.replaceAll(
|
||||
`[shard:${descriptor.timing_key}]`,
|
||||
`[shard:bun:${descriptor.timing_key}]`,
|
||||
),
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
expect(subset.rejectedWorkerKeys.blacksmith).toEqual([]);
|
||||
expect(subset.timings.compactGroupSeconds.blacksmith).toEqual({
|
||||
[`bun:${descriptor.timing_key}`]: 40,
|
||||
"bun:reader-parent": 40,
|
||||
});
|
||||
expect(subset.timings.runtimePlacementTimings.blacksmith).toEqual([]);
|
||||
const admissionLog = log(8).replace(
|
||||
encodeNodeTestGroups([descriptor]),
|
||||
encodeNodeTestGroups([
|
||||
{
|
||||
...descriptor,
|
||||
fallbackMaxWorkers: 2,
|
||||
minTotalMemoryBytes: 28 * 1024 ** 3,
|
||||
},
|
||||
]),
|
||||
);
|
||||
for (const { label, gib, plans, cpus, runner, frozen, expected } of [
|
||||
{
|
||||
label: "qualified",
|
||||
gib: 32,
|
||||
plans: 1,
|
||||
cpus: 8,
|
||||
runner: "self-hosted",
|
||||
frozen: "false",
|
||||
expected: 8,
|
||||
},
|
||||
{
|
||||
label: "memory floor",
|
||||
gib: 27,
|
||||
plans: 1,
|
||||
cpus: 8,
|
||||
runner: "self-hosted",
|
||||
frozen: "false",
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
label: "overlapping plans",
|
||||
gib: 32,
|
||||
plans: 2,
|
||||
cpus: 8,
|
||||
runner: "self-hosted",
|
||||
frozen: "false",
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
label: "constrained CPU",
|
||||
gib: 32,
|
||||
plans: 1,
|
||||
cpus: 4,
|
||||
runner: "self-hosted",
|
||||
frozen: "false",
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
label: "hosted",
|
||||
gib: 32,
|
||||
plans: 1,
|
||||
cpus: 8,
|
||||
runner: "github-hosted",
|
||||
frozen: "false",
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
label: "frozen target",
|
||||
gib: 32,
|
||||
plans: 1,
|
||||
cpus: 8,
|
||||
runner: "self-hosted",
|
||||
frozen: "true",
|
||||
expected: 2,
|
||||
},
|
||||
{
|
||||
label: "missing resources",
|
||||
gib: undefined,
|
||||
plans: 1,
|
||||
cpus: 8,
|
||||
runner: "self-hosted",
|
||||
frozen: "false",
|
||||
expected: undefined,
|
||||
},
|
||||
{
|
||||
label: "missing runner",
|
||||
gib: 32,
|
||||
plans: 1,
|
||||
cpus: 8,
|
||||
runner: undefined,
|
||||
frozen: "false",
|
||||
expected: undefined,
|
||||
},
|
||||
]) {
|
||||
const text = [
|
||||
...(runner ? [`2026-08-27T23:00:00Z RUNNER_ENVIRONMENT: ${runner}`] : []),
|
||||
`2026-08-27T23:00:00Z FROZEN_TARGET: ${frozen}`,
|
||||
...(gib
|
||||
? [
|
||||
`2026-08-27T23:00:00Z [shard:resources] logicalCpuCount=${cpus} totalMemoryBytes=${gib * 1024 ** 3} requested plans=${plans} admitted plans=${plans}`,
|
||||
]
|
||||
: []),
|
||||
admissionLog,
|
||||
].join("\n");
|
||||
const result = refitTestTimings(
|
||||
[1, 2].map((id) =>
|
||||
timingRun(id, [{ kind: "compact", labels: ["blacksmith-32vcpu-ubuntu-2404"], text }]),
|
||||
),
|
||||
previous,
|
||||
);
|
||||
expect(result.timings.compactGroupSeconds.blacksmith[descriptor.timing_key], label).toBe(
|
||||
expected === undefined ? 100 : 40,
|
||||
);
|
||||
expect(
|
||||
result.timings.runtimePlacementTimings.blacksmith[0]?.env.OPENCLAW_VITEST_MAX_WORKERS,
|
||||
label,
|
||||
).toBe(expected?.toString());
|
||||
expect(result.rejectedWorkerKeys.blacksmith, label).toEqual(
|
||||
expected === undefined ? ["reader-parent", descriptor.timing_key].toSorted() : [],
|
||||
);
|
||||
}
|
||||
const wholeConfigLog = log(8).replace(
|
||||
encodeNodeTestGroups([descriptor]),
|
||||
encodeNodeTestGroups([
|
||||
{
|
||||
...descriptor,
|
||||
includePatterns: undefined,
|
||||
fallbackMaxWorkers: 2,
|
||||
minTotalMemoryBytes: 28 * 1024 ** 3,
|
||||
},
|
||||
]),
|
||||
);
|
||||
const wholeConfigRuns = [27, 32].map((gib, index) =>
|
||||
timingRun(index + 1, [
|
||||
{
|
||||
kind: "compact",
|
||||
labels: ["blacksmith-32vcpu-ubuntu-2404"],
|
||||
text: [
|
||||
"2026-08-27T23:00:00Z RUNNER_ENVIRONMENT: self-hosted",
|
||||
"2026-08-27T23:00:00Z FROZEN_TARGET: false",
|
||||
`2026-08-27T23:00:00Z [shard:resources] logicalCpuCount=8 totalMemoryBytes=${gib * 1024 ** 3} requested plans=1 admitted plans=1`,
|
||||
wholeConfigLog,
|
||||
].join("\n"),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const whole = refitTestTimings(wholeConfigRuns, previous);
|
||||
expect(whole.timings.compactGroupSeconds.blacksmith).toEqual(
|
||||
previous.compactGroupSeconds.blacksmith,
|
||||
);
|
||||
expect(whole.rejectedWorkerKeys.blacksmith).toEqual(
|
||||
["reader-parent", descriptor.timing_key].toSorted(),
|
||||
);
|
||||
expect(whole.timings.runtimePlacementTimings.blacksmith).toEqual([]);
|
||||
for (const suffix of ["#workers-4", "#file-parallel-4"]) {
|
||||
const namedKey = createCompactSplitTimingGeneration({
|
||||
configs: descriptor.configs,
|
||||
parentShardName: `reader${suffix}`,
|
||||
stripes: [descriptor.includePatterns],
|
||||
}).timingKeys[0]!;
|
||||
const namedLog = admissionLog
|
||||
.replace(
|
||||
encodeNodeTestGroups([
|
||||
{ ...descriptor, fallbackMaxWorkers: 2, minTotalMemoryBytes: 28 * 1024 ** 3 },
|
||||
]),
|
||||
encodeNodeTestGroups([
|
||||
{
|
||||
...descriptor,
|
||||
timing_key: namedKey,
|
||||
env: { OPENCLAW_VITEST_MAX_WORKERS: "4" },
|
||||
fallbackMaxWorkers: 2,
|
||||
minTotalMemoryBytes: 28 * 1024 ** 3,
|
||||
},
|
||||
]),
|
||||
)
|
||||
.replaceAll(`[shard:${descriptor.timing_key}]`, `[shard:${namedKey}]`);
|
||||
const previousNamed: CiTestTimings = {
|
||||
...baseline,
|
||||
compactGroupSeconds: { blacksmith: { [namedKey]: 100 }, github: {} },
|
||||
};
|
||||
for (const gib of [27, 32]) {
|
||||
const text = [
|
||||
"2026-08-27T23:00:00Z RUNNER_ENVIRONMENT: self-hosted",
|
||||
"2026-08-27T23:00:00Z FROZEN_TARGET: false",
|
||||
`2026-08-27T23:00:00Z [shard:resources] logicalCpuCount=8 totalMemoryBytes=${gib * 1024 ** 3} requested plans=1 admitted plans=1`,
|
||||
namedLog,
|
||||
].join("\n");
|
||||
const samples = [1, 2].map((id) =>
|
||||
timingRun(id, [{ kind: "compact", labels: ["blacksmith-32vcpu-ubuntu-2404"], text }]),
|
||||
);
|
||||
const result = refitTestTimings(samples, previousNamed);
|
||||
expect(result.timings.compactGroupSeconds.blacksmith[namedKey], `${suffix} ${gib}GiB`).toBe(
|
||||
gib === 27 ? 100 : 40,
|
||||
);
|
||||
expect(result.rejectedWorkerKeys.blacksmith.includes(namedKey)).toBe(gib === 27);
|
||||
}
|
||||
const missingWorkers = refitTestTimings(
|
||||
[1, 2].map((id) =>
|
||||
timingRun(id, [
|
||||
{
|
||||
kind: "compact",
|
||||
labels: ["blacksmith-32vcpu-ubuntu-2404"],
|
||||
text: compactLog(40, namedKey),
|
||||
},
|
||||
]),
|
||||
),
|
||||
previousNamed,
|
||||
);
|
||||
expect(missingWorkers.timings.compactGroupSeconds.blacksmith[namedKey]).toBe(100);
|
||||
expect(missingWorkers.rejectedWorkerKeys.blacksmith).toContain(namedKey);
|
||||
}
|
||||
});
|
||||
|
||||
it("retains measured parent costs across split inventory changes without double-counting retries", () => {
|
||||
const parentShardName = "agentic-control-plane-agent-chat";
|
||||
const generations = [0, 1, 2].map((index) =>
|
||||
|
|
@ -1109,6 +1668,100 @@ it.todo("retains todo coverage");
|
|||
});
|
||||
});
|
||||
|
||||
it("does not promote selected PR generations into full-parent samples", () => {
|
||||
const parent = "changed-agentic-cli";
|
||||
const configs = ["test/vitest/vitest.cli.config.ts"];
|
||||
const files = ["src/cli/selected.test.ts"];
|
||||
const child = createCompactSplitTimingGeneration({
|
||||
configs,
|
||||
parentShardName: parent,
|
||||
stripes: [files],
|
||||
}).timingKeys[0]!;
|
||||
const descriptor = {
|
||||
shard_name: "selected-cli",
|
||||
timing_key: child,
|
||||
configs,
|
||||
includePatterns: files,
|
||||
};
|
||||
const selectionLine = `2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor])}`;
|
||||
const previous: CiTestTimings = {
|
||||
...baseline,
|
||||
compactGroupSeconds: { blacksmith: { [parent]: 1_000 }, github: {} },
|
||||
};
|
||||
const runs = [20, 40, 60].map((seconds, index) =>
|
||||
Object.assign(
|
||||
timingRun(index + 1, [
|
||||
{
|
||||
kind: "compact",
|
||||
labels: ["blacksmith-8vcpu-ubuntu-2404"],
|
||||
text: `${selectionLine}\n2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2\n${compactLog(seconds, child)}\n${compactLog(600, parent)}`,
|
||||
},
|
||||
]),
|
||||
{ pullRequestMergeRef: true, completeInventory: false },
|
||||
),
|
||||
);
|
||||
// Neither selected generations nor repeated unsplit PR spans prove the
|
||||
// full owner inventory. Only the exact selected child can refit.
|
||||
const expected = {
|
||||
[parent]: 1_000,
|
||||
[child]: 40,
|
||||
};
|
||||
expect(refitTestTimings(runs, previous).timings.compactGroupSeconds.blacksmith).toEqual(
|
||||
expected,
|
||||
);
|
||||
const fragments = runs.flatMap((run) => [{ ...run, pullRequestMergeRef: false }, run]);
|
||||
expect(refitTestTimings(fragments, previous).timings.compactGroupSeconds.blacksmith).toEqual(
|
||||
expected,
|
||||
);
|
||||
const missingWorkers = runs.map((run) => ({
|
||||
...run,
|
||||
logs: run.logs.map((log) => ({
|
||||
...log,
|
||||
text: log.text.replace("2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2\n", ""),
|
||||
})),
|
||||
}));
|
||||
expect(
|
||||
refitTestTimings(missingWorkers, previous).timings.compactGroupSeconds.blacksmith,
|
||||
).toEqual(previous.compactGroupSeconds.blacksmith);
|
||||
for (const [reason, selection] of [
|
||||
["missing descriptor", ""],
|
||||
[
|
||||
"malformed descriptor",
|
||||
"2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: invalid",
|
||||
],
|
||||
[
|
||||
"duplicate descriptor",
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([descriptor, descriptor])}`,
|
||||
],
|
||||
[
|
||||
"different files",
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([{ ...descriptor, includePatterns: ["src/cli/other.test.ts"] }])}`,
|
||||
],
|
||||
[
|
||||
"glob files",
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([{ ...descriptor, includePatterns: ["src/cli/*.test.ts"] }])}`,
|
||||
],
|
||||
[
|
||||
"filtered invocation",
|
||||
`${selectionLine}\n2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_VITEST_ARGS_JSON: ["--shard=1/2"]`,
|
||||
],
|
||||
] as const) {
|
||||
const incomplete = runs.map((run) => ({
|
||||
...run,
|
||||
logs: run.logs.map((log) => ({ ...log, text: log.text.replace(selectionLine, selection) })),
|
||||
}));
|
||||
expect(
|
||||
refitTestTimings(incomplete, previous).timings.compactGroupSeconds.blacksmith,
|
||||
reason,
|
||||
).toEqual(previous.compactGroupSeconds.blacksmith);
|
||||
}
|
||||
const main = runs.map((run) => Object.assign({}, run, { pullRequestMergeRef: false }));
|
||||
expect(refitTestTimings(main, previous).timings.compactGroupSeconds.blacksmith).toEqual({
|
||||
[parent]: 600,
|
||||
[child]: 40,
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["missing part", "different generation", "different profile", "different run"])(
|
||||
"does not invent a complete parent measurement from %s",
|
||||
(condition) => {
|
||||
|
|
@ -1516,7 +2169,7 @@ describe("CI timing sampler provenance", () => {
|
|||
: timings.repoE2eFileSeconds["test/release.e2e.test.ts"],
|
||||
).toBe(40);
|
||||
expect(result.stderr).toContain(
|
||||
`Skipped ${source} run 1: jobs completed after frozen UTC cutoff ${sampleNow}.`,
|
||||
`Skipped ${source === "tooling" ? "pull-request" : source} run 1: jobs completed after frozen UTC cutoff ${sampleNow}.`,
|
||||
);
|
||||
const requests = fixture.requests();
|
||||
expect(requests.some((args) => args[1]?.includes("/runs/1/attempts/2/jobs?"))).toBe(true);
|
||||
|
|
@ -1638,22 +2291,64 @@ describe("CI timing sampler provenance", () => {
|
|||
);
|
||||
});
|
||||
|
||||
it("samples PR merge-ref tooling without replacing main, release, or UI measurements", () => {
|
||||
it("requires two PR runs before repricing tooling from compact logs", () => {
|
||||
const runs = [1, 2, 3, 4].map((id) =>
|
||||
Object.assign(
|
||||
timingRun(id, [
|
||||
{
|
||||
kind: "compact",
|
||||
labels: ["blacksmith-8vcpu-ubuntu-2404"],
|
||||
text: samplerToolingLog(id * 10),
|
||||
},
|
||||
]),
|
||||
{ pullRequestMergeRef: id > 2, completeInventory: id <= 2 },
|
||||
),
|
||||
);
|
||||
expect(refitTestTimings(runs.slice(0, 3)).timings.toolingFileSeconds).toEqual({
|
||||
blacksmith: {},
|
||||
github: {},
|
||||
});
|
||||
const result = refitTestTimings(runs);
|
||||
expect(result.timings.toolingFileSeconds).toEqual({
|
||||
blacksmith: { [toolingFile]: 35 },
|
||||
github: {},
|
||||
});
|
||||
expect(result.contributingRunIds.toolingBlacksmith).toEqual([3, 4]);
|
||||
});
|
||||
|
||||
it("samples contributing PR compact and tooling jobs while retaining partial-plan history and runner identity", () => {
|
||||
const gatewayGroup = createCompactSplitTimingGeneration({
|
||||
configs: ["test/vitest/vitest.gateway-methods.config.ts"],
|
||||
parentShardName: "agentic-gateway-methods-hosted-2",
|
||||
stripes: [["src/gateway/selected.test.ts"]],
|
||||
}).timingKeys[0]!;
|
||||
const gatewayDescriptor = {
|
||||
shard_name: "agentic-gateway-methods-hosted-2",
|
||||
timing_key: gatewayGroup,
|
||||
configs: ["test/vitest/vitest.gateway-methods.config.ts"],
|
||||
includePatterns: ["src/gateway/selected.test.ts"],
|
||||
};
|
||||
const extensionConfig = "test/vitest/vitest.extension-database-workers.config.ts";
|
||||
const extensionFiles = ["extensions/telegram/src/native.test.ts"];
|
||||
const extensionKey = createExtensionTestTimingKey(extensionConfig, extensionFiles)!;
|
||||
withSamplerFixture(
|
||||
{
|
||||
baseline: retained,
|
||||
runs: [samplerRun(1), samplerRun(2)],
|
||||
toolingRuns: [3, 4].map((id) =>
|
||||
toolingRuns: [7, 8, 3, 4, 5].map((id) =>
|
||||
samplerRun(id, { event: "pull_request", head_branch: "feature" }),
|
||||
),
|
||||
jobs: [
|
||||
samplerJob(11, 1),
|
||||
samplerJob(21, 2),
|
||||
...[3, 4].flatMap((id) => [
|
||||
samplerJob(81, 8, { log: "No complete shard spans" }),
|
||||
...[3, 4, 5].flatMap((id) => [
|
||||
samplerJob(id * 10 + 1, id, {
|
||||
name: `checks-node-${id === 3 ? "changed-" : id === 4 ? "changed-config-" : ""}compact-large-1`,
|
||||
log: [
|
||||
samplerToolingLog(id === 3 ? 30 : 50),
|
||||
compactLog(900),
|
||||
"2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2",
|
||||
samplerToolingLog((id - 2) * 20, [gatewayDescriptor]),
|
||||
compactLog((id - 2) * 100 + 700, gatewayGroup),
|
||||
uiLog({ [measuredFile]: 900 }),
|
||||
].join("\n"),
|
||||
}),
|
||||
|
|
@ -1661,25 +2356,59 @@ describe("CI timing sampler provenance", () => {
|
|||
name: "checks-ui-e2e (1/6)",
|
||||
log: uiLog({ [measuredFile]: 900 }),
|
||||
}),
|
||||
samplerJob(id * 10 + 3, id, {
|
||||
name: "checks-node-changed-config-compact-small-1",
|
||||
labels: ["ubuntu-24.04"],
|
||||
log: `2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2\n2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups([gatewayDescriptor])}\n${compactLog(600, gatewayGroup)}`,
|
||||
}),
|
||||
samplerJob(id * 10 + 4, id, {
|
||||
name:
|
||||
id === 4
|
||||
? "checks-node-changed-extensions-config"
|
||||
: `checks-node-changed-extensions-bundle-${id}`,
|
||||
log: [
|
||||
"2026-08-27T23:00:00Z OPENCLAW_VITEST_MAX_WORKERS: 2",
|
||||
`2026-08-27T23:00:00Z OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64: ${encodeNodeTestGroups(
|
||||
[
|
||||
{
|
||||
configs: [extensionConfig],
|
||||
includePatterns: extensionFiles,
|
||||
shard_name: `changed-extensions-config-${id}`,
|
||||
},
|
||||
],
|
||||
)}`,
|
||||
compactLog(160 + id * 10, `changed-extensions-config-${id}`),
|
||||
].join("\n"),
|
||||
}),
|
||||
]),
|
||||
],
|
||||
},
|
||||
(fixture) => {
|
||||
const result = fixture.invoke();
|
||||
const result = fixture.invoke(false, 3);
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const timings = ciTestTimingsSchema.parse(JSON.parse(fixture.contents()));
|
||||
expect(timings).toMatchObject({
|
||||
compactGroupSeconds: retained.compactGroupSeconds,
|
||||
runtimePlacementTimings: retained.runtimePlacementTimings,
|
||||
repoE2eFileSeconds: retained.repoE2eFileSeconds,
|
||||
uiE2e: retained.uiE2e,
|
||||
});
|
||||
expect(timings.compactGroupSeconds).toEqual(retained.compactGroupSeconds);
|
||||
expect(timings.compactGroupSeconds).toEqual({
|
||||
blacksmith: {
|
||||
...retained.compactGroupSeconds.blacksmith,
|
||||
[gatewayGroup]: 900,
|
||||
[extensionKey]: 200,
|
||||
},
|
||||
github: { ...retained.compactGroupSeconds.github, [gatewayGroup]: 600 },
|
||||
});
|
||||
expect(timings.toolingFileSeconds).toEqual({
|
||||
blacksmith: { ...retained.toolingFileSeconds.blacksmith, [toolingFile]: 40 },
|
||||
github: retained.toolingFileSeconds.github,
|
||||
});
|
||||
expect(result.stdout).toContain("PR tooling measurements execute the merge-ref");
|
||||
expect(result.stdout).toContain(
|
||||
"PR compact and tooling measurements execute the merge-ref",
|
||||
);
|
||||
expect(result.stdout).toContain("Independent PR compact contributors: 3");
|
||||
expect(timings.source).toContain("pull_request merge-ref runs: 3, 4, 5");
|
||||
const request = fixture.requests().find((args) => args[1]?.includes("event=pull_request"));
|
||||
expect(request).toBeDefined();
|
||||
const params = new URL(request![1]!, "https://api.github.com").searchParams;
|
||||
|
|
@ -1687,8 +2416,14 @@ describe("CI timing sampler provenance", () => {
|
|||
expect(params.get("created")).toBe(`2026-08-21T12:00:00.000Z..${sampleNow}`);
|
||||
expect(params.has("branch")).toBe(false);
|
||||
expect(
|
||||
fixture.requests().some((args) => /\/jobs\/(?:32|42)\/logs$/u.test(args[1] ?? "")),
|
||||
fixture.requests().some((args) => /\/jobs\/(?:32|42|52)\/logs$/u.test(args[1] ?? "")),
|
||||
).toBe(false);
|
||||
expect(
|
||||
fixture.requests().filter((args) => /\/jobs\/(?:31|41|51)\/logs$/u.test(args[1] ?? "")),
|
||||
).toHaveLength(3);
|
||||
expect(
|
||||
fixture.requests().filter((args) => /\/jobs\/(?:34|44|54)\/logs$/u.test(args[1] ?? "")),
|
||||
).toHaveLength(3);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
|
@ -1723,7 +2458,12 @@ describe("CI timing sampler provenance", () => {
|
|||
baseline: retained,
|
||||
runs: [],
|
||||
toolingRuns: [samplerRun(3, { event: "pull_request", head_branch: "feature" })],
|
||||
jobs: [samplerJob(31, 3, { log: samplerToolingLog(40) })],
|
||||
jobs: [
|
||||
samplerJob(31, 3, {
|
||||
name: "checks-node-changed-config-compact-large-1",
|
||||
log: `${samplerToolingLog(40)}\n${compactLog(900)}`,
|
||||
}),
|
||||
],
|
||||
},
|
||||
(fixture) => {
|
||||
const dryRun = fixture.invoke(true, 2, [3]);
|
||||
|
|
@ -1776,7 +2516,7 @@ describe("CI timing sampler provenance", () => {
|
|||
},
|
||||
);
|
||||
|
||||
it("does not let a retried PR satisfy the ordinary two-run tooling minimum", () => {
|
||||
it("does not let a retried PR satisfy the ordinary two-run compact or tooling minimum", () => {
|
||||
withSamplerFixture(
|
||||
{
|
||||
baseline: retained,
|
||||
|
|
@ -1785,8 +2525,11 @@ describe("CI timing sampler provenance", () => {
|
|||
jobs: [
|
||||
samplerJob(11, 1),
|
||||
samplerJob(21, 2),
|
||||
samplerJob(31, 3, { log: samplerToolingLog(30) }),
|
||||
samplerJob(32, 3, { run_attempt: 2, log: samplerToolingLog(50) }),
|
||||
samplerJob(31, 3, { log: `${samplerToolingLog(30)}\n${compactLog(900, "pr-only")}` }),
|
||||
samplerJob(32, 3, {
|
||||
run_attempt: 2,
|
||||
log: `${samplerToolingLog(50)}\n${compactLog(900, "pr-only")}`,
|
||||
}),
|
||||
],
|
||||
},
|
||||
(fixture) => {
|
||||
|
|
|
|||
|
|
@ -2374,9 +2374,11 @@ AFTER_CD
|
|||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"defers timing refits when only the runtime group codec changes on main",
|
||||
() => {
|
||||
it
|
||||
.skipIf(process.platform === "win32")
|
||||
.each(["scripts/lib/ci-node-test-groups-codec.mts", "scripts/lib/local-check-runtime.mts"])(
|
||||
"defers timing refits when only %s changes on main",
|
||||
(sourcePath) => {
|
||||
const workflow = readWorkflow(".github/workflows/ci-test-timings-refit.yml");
|
||||
const publisher = expectDefined(
|
||||
workflow.jobs.refit.steps.find(
|
||||
|
|
@ -2386,7 +2388,7 @@ AFTER_CD
|
|||
);
|
||||
const result = runGeneratedPublisherScenario(null, {
|
||||
invalidationPaths: publisher.with["invalidation-paths"],
|
||||
updateSource: "scripts/lib/ci-node-test-groups-codec.mts",
|
||||
updateSource: sourcePath,
|
||||
});
|
||||
|
||||
expect(result.branchExists).toBe(false);
|
||||
|
|
@ -2398,6 +2400,79 @@ AFTER_CD
|
|||
},
|
||||
);
|
||||
|
||||
it("preserves large timing refit reports outside generated PR environments", () => {
|
||||
const workflow = readWorkflow(".github/workflows/ci-test-timings-refit.yml");
|
||||
const steps: WorkflowStep[] = workflow.jobs.refit.steps;
|
||||
const refit = expectDefined(
|
||||
steps.find((step) => step.id === "refit"),
|
||||
"refit step",
|
||||
);
|
||||
const root = tempDirs.make("openclaw-refit-report-");
|
||||
const bin = path.join(root, "bin");
|
||||
const source = path.join(root, "source.md");
|
||||
const output = path.join(root, "github-output");
|
||||
mkdirSync(bin);
|
||||
writeExecutable(path.join(bin, "pnpm"), [
|
||||
"#!/bin/sh",
|
||||
'test "$*" = "--silent ci:timings:refit" || exit 64',
|
||||
'cat "$REFIT_FIXTURE_REPORT"',
|
||||
]);
|
||||
writeExecutable(path.join(bin, "git"), [
|
||||
"#!/bin/sh",
|
||||
'test "$*" = "diff --quiet -- config/ci-test-timings.json" || exit 64',
|
||||
"exit 1",
|
||||
]);
|
||||
const validIds = "36204091214, 36208949888";
|
||||
const largeTable = "| fixture-test | 100 | 50 | -50% |\n".repeat(12_000);
|
||||
for (const runIds of [
|
||||
validIds,
|
||||
"invalid",
|
||||
Array.from({ length: 200 }, (_, index) => String(36_200_000_000 + index)).join(", "),
|
||||
]) {
|
||||
const report = `Sampled CI and release-check runs with successful timing jobs: ${runIds}\n\n${largeTable}`;
|
||||
writeFileSync(source, report);
|
||||
writeFileSync(output, "");
|
||||
const result = runWorkflowShellScript(expectDefined(refit.run, "refit body"), {
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${bin}${path.delimiter}${process.env.PATH ?? ""}`,
|
||||
RUNNER_TEMP: root,
|
||||
GITHUB_OUTPUT: output,
|
||||
REFIT_FIXTURE_REPORT: source,
|
||||
},
|
||||
});
|
||||
expect(readFileSync(path.join(root, "ci-test-timings-report.md"), "utf8")).toBe(report);
|
||||
expect(Buffer.byteLength(readFileSync(output, "utf8"))).toBeLessThan(4096);
|
||||
if (runIds === validIds) {
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
expect(readWorkflowOutputs(output)).toEqual({ run_ids: validIds, changed: "true" });
|
||||
} else {
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.stderr).toContain("Missing or oversized sampled-run summary");
|
||||
expect(readWorkflowOutputs(output)).toEqual({});
|
||||
}
|
||||
}
|
||||
const upload = expectDefined(
|
||||
steps.find((step) => step.id === "report"),
|
||||
"report upload",
|
||||
);
|
||||
const publisher = expectDefined(
|
||||
steps.find((step) => step.uses === "./.github/actions/publish-generated-pr"),
|
||||
"refit publisher",
|
||||
);
|
||||
const uploadInputs = expectDefined(upload.with, "report upload inputs");
|
||||
const publisherInputs = expectDefined(publisher.with, "refit publisher inputs");
|
||||
expect(upload.uses).toBe(UPLOAD_ARTIFACT_V7);
|
||||
expect(uploadInputs.path).toBe("${{ runner.temp }}/ci-test-timings-report.md");
|
||||
expect(uploadInputs["if-no-files-found"]).toBe("error");
|
||||
expect(upload.if).toBe("${{ !cancelled() && steps.refit.outcome != 'skipped' }}");
|
||||
expect(steps.indexOf(upload)).toBeLessThan(steps.indexOf(publisher));
|
||||
expect(publisherInputs["pr-body"]).toContain("${{ steps.refit.outputs.run_ids }}");
|
||||
expect(publisherInputs["pr-body"]).toContain("${{ steps.report.outputs.artifact-url }}");
|
||||
expect(publisherInputs["pr-body"]).not.toContain("${{ steps.refit.outputs.report }}");
|
||||
});
|
||||
|
||||
it
|
||||
.skipIf(process.platform === "win32")
|
||||
.each(["src/config/schema.help.runtime.ts", "src/config/zod-schema.agent-runtime.ts"])(
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { mkdirSync, writeFileSync } from "node:fs";
|
|||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { listAvailableExtensionIds } from "../../scripts/lib/changed-extensions.mts";
|
||||
import * as testTimings from "../../scripts/lib/ci-test-timings.mts";
|
||||
import * as extensionTestPlan from "../../scripts/lib/extension-test-plan.mts";
|
||||
import { resolveBoundedVitestInvocations } from "../../scripts/run-vitest.mts";
|
||||
import {
|
||||
|
|
@ -20,6 +21,58 @@ const workerConfig = "test/vitest/vitest.extension-database-workers.config.ts";
|
|||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
describe("extension executable test plans", () => {
|
||||
it("reuses singleton invocation costs across envelopes without pricing multi-file pools as serial", () => {
|
||||
const files = ["extensions/telegram/src/one.test.ts", "extensions/telegram/src/two.test.ts"];
|
||||
const added = "extensions/telegram/src/three.test.ts";
|
||||
const key = extensionTestPlan.createExtensionTestTimingKey;
|
||||
const timings: Record<string, number> = {
|
||||
[key(workerConfig, [files[0]!], undefined, "singleton-invocation")!]: 10,
|
||||
[key(workerConfig, [files[1]!], undefined, "singleton-invocation")!]: 21,
|
||||
[key(workerConfig, [], undefined, "wrapper-overhead")!]: 3,
|
||||
};
|
||||
const samples = vi.spyOn(testTimings, "readCompactGroupTimings").mockReturnValue(timings);
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(workerConfig, 2, files)).toBe(34);
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(workerConfig, 3, [...files, added])).toBe(
|
||||
42,
|
||||
);
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(workerConfig, 2, [files[1]!, added])).toBe(
|
||||
32,
|
||||
);
|
||||
timings[key(workerConfig, files)!] = 60;
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(workerConfig, 2, files)).toBe(60);
|
||||
|
||||
samples.mockReturnValue({
|
||||
[key(telegramConfig, [files[0]!], undefined, "singleton-invocation")!]: 100,
|
||||
[key(telegramConfig, [files[1]!], undefined, "singleton-invocation")!]: 100,
|
||||
[key(telegramConfig, [], undefined, "wrapper-overhead")!]: 20,
|
||||
});
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(telegramConfig, 2, files)).toBe(9);
|
||||
samples.mockReturnValue({
|
||||
[key(
|
||||
workerConfig,
|
||||
[files[0]!],
|
||||
{ OPENCLAW_VITEST_MAX_WORKERS: "8" },
|
||||
"singleton-invocation",
|
||||
)!]: 100,
|
||||
[key(
|
||||
workerConfig,
|
||||
[files[1]!],
|
||||
{ OPENCLAW_VITEST_MAX_WORKERS: "2", MODE: "other" },
|
||||
"singleton-invocation",
|
||||
)!]: 100,
|
||||
});
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(workerConfig, 2, files)).toBe(16);
|
||||
const only = [files[0]!];
|
||||
samples.mockReturnValue({ [key(workerConfig, only)!]: 22 });
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(workerConfig, 1, only)).toBe(22);
|
||||
samples.mockReturnValue({
|
||||
[key(workerConfig, only)!]: 22,
|
||||
[key(workerConfig, only, undefined, "singleton-invocation")!]: 20,
|
||||
[key(workerConfig, [], undefined, "wrapper-overhead")!]: 2,
|
||||
});
|
||||
expect(extensionTestPlan.estimateExtensionTestCost(workerConfig, 1, only)).toBe(22);
|
||||
});
|
||||
|
||||
it.each(["git", "filesystem"])("reads each candidate checkout's %s plugin inventory", (kind) => {
|
||||
const root = "extensions/fixture";
|
||||
for (const snapshot of ["before", "after"]) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue