fix(test): preserve the first Vitest timeout

The project runner replaced timed-out configurations with another attempt
and could turn their failed hooks into a green job. CI run 35928401414,
job 107409413294, masked seven settlement-hook failures this way.

Remove replacement dispatch and its expanded retry deadline. Keep the first
watchdog failure even when a child exits zero, retain incomplete reports,
and preserve process/cache cleanup.

Proof: the temporary fail-first test failed the real CI shard entry point
with exit 143 and one attempt. The runner owner passed 20 standalone runs
and three CI-config replays on Testbox; shared-helper importers passed.
This commit is contained in:
Peter Steinberger 2026-09-23 18:15:35 -07:00
parent f8db002494
commit 36baa2cd98
No known key found for this signature in database
12 changed files with 93 additions and 137 deletions

View file

@ -121,7 +121,7 @@ Native dependency policy:
behavior.
- `scripts/run-vitest.mjs` terminates explicit non-watch Vitest runs
when their configured no-output deadline expires. Expiry fails the run
even when the child shuts down with exit code zero. Set
without retrying the shard, even when the child shuts down with exit code zero. Set
`OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS=0` to disable the watchdog for
an intentionally silent investigation.
- `scripts/run-tsgo.mjs` leaves tsgo unbounded by default, preserving the

View file

@ -245,7 +245,8 @@ and blob files, then publish the requested JSON from Vitest's native report merg
They print a companion `<output>.reports-<unique>` directory. Keep that directory:
it contains original reports, per-attempt coverage files when coverage is enabled,
and an `index.json` with child exit codes, signals, timeouts and unstarted work.
Only the accepted retry attempt contributes to the aggregate.
Each invocation runs once. A no-output timeout fails the command and leaves the
report set incomplete; it never starts a replacement attempt.
Blob reports are exact-version artifacts. Rerun child reports with the current
Vitest version before merging artifacts produced by another version.

View file

@ -334,8 +334,7 @@ const EXTRA_LARGE_NODE_TEST_RUNNER = "blacksmith-32vcpu-ubuntu-2404";
// Startup-core transforms the broad gateway graph before its assertions run.
// Keep enough CPU here to avoid spending minutes in Vitest imports on 4 vCPU.
const GATEWAY_STARTUP_CORE_RUNNER = DEFAULT_NODE_TEST_RUNNER;
// This cold gateway graph can stall after warming Vitest's module cache; its
// retry completes in seconds, so do not spend the global five-minute timeout.
// Fail the known warm-cache startup stall at its existing scoped deadline.
const GATEWAY_STARTUP_HEALTH_RUNTIME_ENV = {
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "60000",
};

View file

@ -36,7 +36,7 @@ export function resolveVitestCacheSlotPath(
return path.join(root, "slots", configCacheKey(config, cwd), String(slot));
}
/** A slot remains borrowed through retries and the process owner's final join. */
/** A slot remains borrowed through preflight and the process owner's final join. */
export function createVitestCacheSlots(platform = process.platform) {
const idleSlots = new Map<string, number[]>();
const nextSlots = new Map<string, number>();

View file

@ -57,7 +57,7 @@ function caseInventory(reports: JsonTestResults[]) {
.toSorted();
}
/** Own file artifacts only; callers retain admission, retry, environment and process ownership. */
/** Own file artifacts only; callers retain admission, environment and process ownership. */
export async function createVitestReportOwner(invocations: Invocation[], cwd: string) {
if (
invocations.length < 2 ||

View file

@ -51,11 +51,9 @@ import {
resolveParallelFullSuiteConcurrency,
resolveChangedTestTargetPlanForArgs,
resolveChangedTargetArgs,
shouldRetryVitestNoOutputTimeout,
type FailedVitestShard,
type VitestRunSpec as BaseVitestRunSpec,
type VitestCacheAssignment,
withRetryNoOutputTimeout,
writeVitestIncludeFile,
} from "./test-projects.test-support.mts";
@ -196,16 +194,9 @@ function applyDefaultParallelVitestWorkerBudget(specs: VitestRunSpec[], env: Nod
async function runLoggedVitestSpec(spec: VitestRunSpec, reports: VitestReportOwner) {
console.error(`[test] starting ${spec.config}`);
const startedAt = performance.now();
let result = await runVitestSpec(spec, reports);
if (result.noOutputTimedOut && !spec.watchMode && shouldRetryVitestNoOutputTimeout(spec.env)) {
assertCacheLeaseJoined(spec, result);
console.error(`[test] retrying ${spec.config} after no-output timeout`);
const firstJoined = result.groupJoined;
result = await runVitestSpec(withRetryNoOutputTimeout(spec), reports);
result = { ...result, groupJoined: firstJoined && result.groupJoined };
}
const result = await runVitestSpec(spec, reports);
const durationMs = performance.now() - startedAt;
if (result.noOutputTimedOut && result.signal) {
if (result.noOutputTimedOut) {
console.error(`[test] ${spec.config} exceeded no-output timeout`);
return {
...result,

View file

@ -944,7 +944,6 @@ const TOOLING_IMPORT_GRAPH_GREP_PATHS = importGraphPathspecs(
const BROAD_CHANGED_ENV_KEY = "OPENCLAW_TEST_CHANGED_BROAD";
const VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS";
const VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS";
const VITEST_NO_OUTPUT_RETRY_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_RETRY";
/** Default no-output timeout applied to test-projects Vitest children. */
const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS = String(900_000);
/** Default heartbeat interval applied to test-projects Vitest children. */
@ -4812,35 +4811,6 @@ export function applyDefaultVitestNoOutputTimeout<T extends WatchableVitestSpecS
});
}
export function shouldRetryVitestNoOutputTimeout(env = process.env) {
const value = env[VITEST_NO_OUTPUT_RETRY_ENV_KEY]?.trim().toLowerCase();
if (value === undefined && isCiLikeEnv(env)) {
return false;
}
return !["0", "false", "no", "off"].includes(value ?? "");
}
// Shards may pin a short no-output window so the known warm-cache stall dies
// fast (see AGENTS_CORE_RUNTIME_ENV in ci-node-test-plan.mts). A cold Vitest
// module cache makes those same imports legitimately silent for minutes, so
// the retry attempt must get the full watchdog window or it re-dies at the
// short limit and the job fails without ever running a test.
const RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS = 300_000;
export function withRetryNoOutputTimeout<T extends { env?: NodeJS.ProcessEnv }>(spec: T): T {
const current = Number(spec.env?.[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]);
if (!Number.isFinite(current) || current <= 0 || current >= RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS) {
return spec;
}
return {
...spec,
env: {
...spec.env,
[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]: String(RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS),
},
};
}
export function createVitestRunSpecs(
args: string[],
params: { baseEnv?: NodeJS.ProcessEnv; cwd?: string } = {},

View file

@ -574,36 +574,22 @@ describe("cache lease completion", () => {
vi.stubEnv("OPENCLAW_VITEST_FS_MODULE_CACHE_PATH", "");
});
it.each([
{ platform: "linux", phase: "preflight" },
{ platform: "linux", phase: "retry" },
{ platform: "win32", phase: "preflight" },
{ platform: "win32", phase: "retry" },
] as const)(
"preserves $platform policy after an unverified $phase completion",
async ({ platform, phase }) => {
it.each(["linux", "win32"] as const)(
"preserves %s policy after an unverified preflight completion",
async (platform) => {
vi.spyOn(process, "platform", "get").mockReturnValue(platform);
vi.stubEnv("OPENCLAW_TEST_PROJECTS_PARALLEL", "2");
vi.stubEnv("OPENCLAW_VITEST_NO_OUTPUT_RETRY", "1");
const { runTestProjects } = await import("../../scripts/test-projects-run.mts");
let preflights = 0;
let attempts = 0;
commands.reader.mockImplementation(({ pnpmArgs, onNoOutputTimeout }) => {
let groupJoined = platform !== "win32";
let timedOut = false;
commands.reader.mockImplementation(({ pnpmArgs }) => {
if (pnpmArgs.includes("scripts/ensure-playwright-chromium.mts")) {
preflights += 1;
groupJoined = platform !== "win32" && phase !== "preflight";
} else if (pnpmArgs.includes("test/vitest/vitest.ui-e2e.config.ts")) {
attempts += 1;
groupJoined = false;
if (attempts === 1) {
timedOut = true;
onNoOutputTimeout();
}
}
return {
completion: Promise.resolve({ code: timedOut ? 143 : 0, signal: null, groupJoined }),
completion: Promise.resolve({ code: 0, signal: null, groupJoined: false }),
getForwardedSignal: () => undefined,
};
});
@ -613,8 +599,7 @@ describe("cache lease completion", () => {
]);
if (platform === "win32") {
await expect(running).resolves.toBeUndefined();
expect(preflights).toBe(2);
expect(attempts).toBe(2);
expect(attempts).toBe(1);
} else {
await expect(running).rejects.toMatchObject({
errors: [
@ -623,9 +608,43 @@ describe("cache lease completion", () => {
}),
],
});
expect(preflights).toBe(1);
expect(attempts).toBe(phase === "preflight" ? 0 : 1);
expect(attempts).toBe(0);
}
expect(preflights).toBe(1);
},
);
it.each([
{ signal: "SIGTERM", code: 143 },
{ signal: null, code: 143 },
{ signal: null, code: 0 },
] as const)(
"fails on the first no-output timeout without retrying (signal=$signal, code=$code)",
async ({ signal, code }) => {
vi.stubEnv("CI", "true");
vi.stubEnv("OPENCLAW_VITEST_NO_OUTPUT_RETRY", "1");
const { runTestProjects } = await import("../../scripts/test-projects-run.mts");
const exitBySignal = vi.fn(async () => {});
commands.reader
.mockImplementationOnce(({ onNoOutputTimeout }) => {
onNoOutputTimeout();
return {
completion: Promise.resolve({ code, signal, groupJoined: true }),
getForwardedSignal: () => undefined,
};
})
.mockImplementation(() => ({
completion: Promise.resolve({ code: 0, signal: null, groupJoined: true }),
getForwardedSignal: () => undefined,
}));
await runTestProjects(exitBySignal, ["test/vitest/vitest.cli.config.ts"]);
expect(process.exitCode).toBe(143);
expect(commands.reader).toHaveBeenCalledTimes(1);
expect(exitBySignal).not.toHaveBeenCalled();
expect(console.error).toHaveBeenCalledWith(expect.stringMatching(/^\[test\] failed /u));
expect(console.error).not.toHaveBeenCalledWith(expect.stringMatching(/^\[test\] passed /u));
},
);
@ -635,13 +654,12 @@ describe("cache lease completion", () => {
{ platform: "win32", concurrency: 1 },
{ platform: "win32", concurrency: 2 },
] as const)(
"preserves $platform cache ownership through preflight and retry (concurrency=$concurrency)",
"preserves $platform cache ownership through preflight and execution (concurrency=$concurrency)",
async ({ platform, concurrency }) => {
vi.spyOn(process, "platform", "get").mockReturnValue(platform);
const cacheRoot = tempDirs.make("cache-policy-");
vi.stubEnv("OPENCLAW_VITEST_FS_MODULE_CACHE_ROOT", cacheRoot);
vi.stubEnv("OPENCLAW_TEST_PROJECTS_PARALLEL", String(concurrency));
vi.stubEnv("OPENCLAW_VITEST_NO_OUTPUT_RETRY", "1");
const planner = await import("../../scripts/test-projects.test-support.mts");
const { runTestProjects } = await import("../../scripts/test-projects-run.mts");
if (concurrency === 1) {
@ -659,36 +677,29 @@ describe("cache lease completion", () => {
);
}
const firstPreflight = createDeferred<{ code: number; signal: null; groupJoined: boolean }>();
const retryPreflight = createDeferred<{ code: number; signal: null; groupJoined: boolean }>();
const firstExecution = createDeferred<{ code: number; signal: null; groupJoined: boolean }>();
const peer = createDeferred<{ code: number; signal: null; groupJoined: boolean }>();
const started = createDeferred();
const retryStarted = createDeferred();
const executionStarted = createDeferred();
const paths: string[] = [];
const uiPaths: string[] = [];
let peerPath: string | undefined;
let preflights = 0;
let attempts = 0;
const joined = { code: 0, signal: null, groupJoined: platform !== "win32" };
commands.reader.mockImplementation(({ env, pnpmArgs, onNoOutputTimeout }) => {
commands.reader.mockImplementation(({ env, pnpmArgs }) => {
const cache = env.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH;
paths.push(cache);
let completion;
if (pnpmArgs.includes("scripts/ensure-playwright-chromium.mts")) {
uiPaths.push(cache);
preflights += 1;
completion = preflights === 1 ? firstPreflight.promise : retryPreflight.promise;
if (preflights === 2) {
retryStarted.resolve();
}
completion = preflights === 1 ? firstPreflight.promise : Promise.resolve(joined);
} else if (pnpmArgs.includes("test/vitest/vitest.ui-e2e.config.ts")) {
uiPaths.push(cache);
attempts += 1;
if (attempts === 1) {
onNoOutputTimeout();
}
completion = Promise.resolve(
attempts === 1 ? { ...joined, code: 143, signal: "SIGTERM" } : joined,
);
completion = attempts === 1 ? firstExecution.promise : Promise.resolve(joined);
executionStarted.resolve();
} else {
peerPath = cache;
completion = peer.promise;
@ -714,30 +725,30 @@ describe("cache lease completion", () => {
expect(relative.split(path.sep)).not.toContain("..");
}
firstPreflight.resolve(joined);
await withTestTimeout(retryStarted.promise, 5_000, "retry preflight admission");
expect(uiPaths).toHaveLength(3);
await withTestTimeout(executionStarted.promise, 5_000, "execution admission");
expect(uiPaths).toHaveLength(2);
expect(new Set(uiPaths).size).toBe(1);
expect(uiPaths).not.toContain(peerPath);
expect(attempts).toBe(1);
} finally {
firstPreflight.resolve(joined);
retryPreflight.resolve(joined);
firstExecution.resolve(joined);
peer.resolve(joined);
await running;
}
expect(uiPaths).toHaveLength(concurrency === 1 ? 6 : 4);
expect(new Set(uiPaths.slice(0, 4)).size).toBe(1);
expect(uiPaths).toHaveLength(concurrency === 1 ? 4 : 2);
expect(new Set(uiPaths.slice(0, 2)).size).toBe(1);
if (concurrency === 1) {
expect(new Set(uiPaths.slice(4)).size).toBe(1);
expect(new Set(uiPaths.slice(2)).size).toBe(1);
if (platform === "win32") {
expect(uiPaths[4]).not.toBe(uiPaths[0]);
expect(uiPaths[2]).not.toBe(uiPaths[0]);
} else {
expect(uiPaths[4]).toBe(uiPaths[0]);
expect(uiPaths[2]).toBe(uiPaths[0]);
}
}
expect(uiPaths).not.toContain(peerPath);
expect(preflights).toBe(concurrency === 1 ? 3 : 2);
expect(attempts).toBe(concurrency === 1 ? 3 : 2);
expect(preflights).toBe(concurrency === 1 ? 2 : 1);
expect(attempts).toBe(concurrency === 1 ? 2 : 1);
expect(process.exitCode).toBe(0);
},
);

View file

@ -193,7 +193,6 @@ describe("project runner native empty-file policy", () => {
OPENCLAW_TEST_PROJECTS_TIMINGS: "0",
OPENCLAW_VITEST_MAX_WORKERS: "1",
OPENCLAW_VITEST_FS_MODULE_CACHE_PATH: path.join(root, "module-cache"),
OPENCLAW_VITEST_NO_OUTPUT_RETRY: "0",
...(scenario.parallel ? { OPENCLAW_TEST_PROJECTS_PARALLEL: "2" } : {}),
};
for (const name of ["home", "state", "tmp", "cache", "config"]) {

View file

@ -34,8 +34,6 @@ import {
resolveChangedTargetArgs,
resolveControlUiTestConsumers,
resolveParallelFullSuiteConcurrency,
shouldRetryVitestNoOutputTimeout,
withRetryNoOutputTimeout,
writeVitestIncludeFile,
} from "../../scripts/test-projects.test-support.mts";
import {
@ -5556,29 +5554,6 @@ describe("scripts/test-projects Vitest stall watchdog", () => {
expect(specs[1]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("0");
expect(specs[1]?.env.OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS).toBe("25000");
});
it("allows changed checks to disable automatic silent-run retries", () => {
expect(shouldRetryVitestNoOutputTimeout({})).toBe(true);
expect(shouldRetryVitestNoOutputTimeout({ CI: "true" })).toBe(false);
expect(shouldRetryVitestNoOutputTimeout({ CI: "1" })).toBe(false);
});
it("raises short shard no-output timeouts for the retry attempt", () => {
const spec = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "60000" } };
expect(withRetryNoOutputTimeout(spec).env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("300000");
const generous = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "600000" } };
expect(withRetryNoOutputTimeout(generous)).toBe(generous);
const disabled = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "0" } };
expect(withRetryNoOutputTimeout(disabled)).toBe(disabled);
const unset = { env: {} };
expect(withRetryNoOutputTimeout(unset)).toBe(unset);
expect(shouldRetryVitestNoOutputTimeout({ GITHUB_ACTIONS: "true" })).toBe(false);
expect(shouldRetryVitestNoOutputTimeout({ OPENCLAW_VITEST_NO_OUTPUT_RETRY: "1" })).toBe(true);
expect(shouldRetryVitestNoOutputTimeout({ OPENCLAW_VITEST_NO_OUTPUT_RETRY: "0" })).toBe(false);
expect(shouldRetryVitestNoOutputTimeout({ OPENCLAW_VITEST_NO_OUTPUT_RETRY: "false" })).toBe(
false,
);
});
});
describe("scripts/test-projects Vitest cache isolation", () => {

View file

@ -96,7 +96,6 @@ export function createVitestReportFixture(
OPENCLAW_TEST_PROJECTS_TIMINGS: "0",
OPENCLAW_VITEST_MAX_WORKERS: "1",
OPENCLAW_VITEST_FS_MODULE_CACHE_PATH: path.join(root, "cache"),
OPENCLAW_VITEST_NO_OUTPUT_RETRY: "0",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "20000",
};
for (const [key, value] of Object.entries(env)) {
@ -168,8 +167,7 @@ if(output&&path.basename(path.dirname(output))==='1'&&process.argv.some(arg=>arg
write(path.join(env.HOME!, "canary"), "synthetic caller home\n");
}
const isParallel = ["parallel", "batch-parallel", "failure", "overlap"].includes(mode);
// Report paths identify attempts before spawn; a marker written during config
// loading would move the intentional hang to a retry after slow first startup.
// Report paths identify the first attempt before config startup can record state.
for (const [index, name] of ["alpha", "beta"].entries()) {
const prelude = `import fs from 'node:fs';
${mode === "watchdog" ? "import path from 'node:path';" : ""}
@ -381,8 +379,6 @@ ${index === 0 ? "test('alpha/two',()=>expect(2).toBe(2));" : "test.skip('beta/sk
OPENCLAW_TEST_PROJECTS_PARALLEL: isParallel ? "2" : "1",
OPENCLAW_TEST_PROJECTS_SERIAL: isParallel ? "0" : "1",
OPENCLAW_EXTENSION_BATCH_PARALLEL: isParallel ? "2" : "1",
OPENCLAW_VITEST_NO_OUTPUT_RETRY:
mode === "watchdog" ? "1" : env.OPENCLAW_VITEST_NO_OUTPUT_RETRY,
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS:
mode === "watchdog" ? "1500" : env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS,
};

View file

@ -110,7 +110,6 @@ describe.skipIf(process.platform === "win32")("native multi-invocation report ow
"batch",
"batch-parallel",
"retry",
"watchdog",
"dotted",
"metadata",
"ignored-unhandled",
@ -162,16 +161,6 @@ describe.skipIf(process.platform === "win32")("native multi-invocation report ow
);
expect(index.merge).toMatchObject({ code: 0, signal: null });
}
if (mode === "watchdog") {
expect(
fs.readFileSync(
path.join(path.dirname(path.dirname(result.output)), "cold-started"),
"utf8",
),
).toBe("started");
expect(index.entries[0].attempts).toHaveLength(2);
expect(index.entries[0].attempts[0].outcome.noOutputTimedOut).toBe(true);
}
if (mode === "metadata") {
expect(parts.map((part: { json: string }) => json(part.json).snapshot.matched)).toEqual([
1, 1,
@ -188,6 +177,31 @@ describe.skipIf(process.platform === "win32")("native multi-invocation report ow
},
);
reportTest(
"preserves the first watchdog failure without publishing an aggregate",
{ timeout: 60000 },
async ({ expect, reports }) => {
const result = await reports.run("watchdog");
expect(result.code, result.stderr).toBe(143);
expect(
fs.readFileSync(
path.join(path.dirname(path.dirname(result.output)), "cold-started"),
"utf8",
),
).toBe("started");
const index = json(path.join(result.reportSet!, "index.json"));
expect(index.complete).toBe(false);
expect(index.entries[0].attempts).toHaveLength(1);
expect(index.entries[0].attempts[0].outcome).toMatchObject({
code: 143,
noOutputTimedOut: true,
});
expect(index.entries[1].attempts).toHaveLength(0);
expect(fs.existsSync(result.output)).toBe(false);
expect(result.stderr).not.toContain("[test] retrying");
},
);
reportTest(
"loads each file-backed merge project once and preserves its identity and caches",
async ({ expect, reports }) => {