ci: stabilize changed checks

This commit is contained in:
Peter Steinberger 2026-05-30 20:02:56 +01:00
parent c73e8eedf4
commit 602364f1c7
No known key found for this signature in database
7 changed files with 262 additions and 23 deletions

View file

@ -1,4 +1,4 @@
import { accessSync, chmodSync, constants, mkdtempSync, writeFileSync } from "node:fs";
import { accessSync, chmodSync, constants, existsSync, mkdtempSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { performance } from "node:perf_hooks";
@ -30,6 +30,10 @@ const LIVE_DOCKER_AUTH_SHELL_TARGETS = [
];
const SHRINKWRAP_POLICY_PATH_RE =
/^(?:npm-shrinkwrap\.json|package\.json|pnpm-lock\.yaml|pnpm-workspace\.yaml|scripts\/generate-npm-shrinkwrap\.mjs|extensions\/[^/]+\/(?:package\.json|npm-shrinkwrap\.json))$/u;
const CORE_OXLINT_TS_CONFIG = "config/tsconfig/oxlint.core.json";
const TARGETED_CORE_LINT_PATH_LIMIT = 8;
const LINTABLE_CORE_PATH_RE = /^(?:src|ui|packages)\/.+\.[cm]?[jt]sx?$/u;
const CORE_LINT_OPTIMIZATION_NEUTRAL_PATH_RE = /^(?:scripts|test\/scripts)\//u;
let corepackPnpmShimDir;
export function createChangedCheckChildEnv(baseEnv = process.env) {
@ -244,7 +248,17 @@ export function createChangedCheckPlan(result, options = {}) {
}
if (lanes.core || lanes.coreTests) {
addLint("lint core", ["lint:core"]);
const coreLintCommand = createTargetedCoreLintCommand(result.paths, baseEnv);
if (coreLintCommand) {
addCommand(
coreLintCommand.name,
coreLintCommand.bin,
coreLintCommand.args,
coreLintCommand.env,
);
} else {
addLint("lint core", ["lint:core"]);
}
}
if (
lanes.liveDockerTooling &&
@ -302,6 +316,34 @@ export function createChangedCheckPlan(result, options = {}) {
};
}
export function createTargetedCoreLintCommand(paths, env = process.env, options = {}) {
if (
paths.some(
(changedPath) =>
!LINTABLE_CORE_PATH_RE.test(changedPath) &&
!CORE_LINT_OPTIMIZATION_NEUTRAL_PATH_RE.test(changedPath),
)
) {
return null;
}
const targets = paths
.filter((changedPath) => LINTABLE_CORE_PATH_RE.test(changedPath))
.toSorted((left, right) => left.localeCompare(right));
if (targets.length === 0 || targets.length > TARGETED_CORE_LINT_PATH_LIMIT) {
return null;
}
const fileExists = options.fileExists ?? existsSync;
if (!targets.every((target) => fileExists(target))) {
return null;
}
return {
name: targets.length === 1 ? "lint core changed file" : "lint core changed files",
bin: "node",
args: ["scripts/run-oxlint.mjs", "--tsconfig", CORE_OXLINT_TS_CONFIG, ...targets],
env,
};
}
export async function runChangedCheck(result, options = {}) {
const baseEnv = resolveLocalHeavyCheckEnv(options.env ?? process.env);
const childEnv = createChangedCheckChildEnv(baseEnv);

View file

@ -18,6 +18,9 @@ const ANSI_CSI_PREFIX = `${String.fromCharCode(27)}[`;
const ANSI_CSI_SUFFIX_RE = /^[0-?]*[ -/]*[@-~]/u;
const SUPPRESSED_VITEST_STDERR_PATTERNS = ["[PLUGIN_TIMINGS]"];
export const DEFAULT_VITEST_NO_OUTPUT_TIMEOUT_MS = 300_000;
export const DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS = 120_000;
const VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS";
const VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS";
const UI_VITEST_CONFIG = "test/vitest/vitest.ui.config.ts";
const UNIT_UI_VITEST_CONFIG = "test/vitest/vitest.unit-ui.config.ts";
const TOOLING_VITEST_CONFIG = "test/vitest/vitest.tooling.config.ts";
@ -141,7 +144,11 @@ export function resolveVitestCliEntry({
}
export function resolveVitestNoOutputTimeoutMs(env = process.env) {
return parsePositiveInt(env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS);
return parsePositiveInt(env[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]);
}
export function resolveVitestNoOutputHeartbeatMs(env = process.env) {
return parsePositiveInt(env[VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY]);
}
function resolveBooleanModeFlag(argv, index, longName, shortName = null) {
@ -216,9 +223,6 @@ function resolveExplicitVitestMode(argv) {
}
export function resolveRunVitestSpawnEnv(env = process.env, argv = []) {
if (Object.hasOwn(env, "OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS")) {
return env;
}
const explicitMode = resolveExplicitVitestMode(argv);
if (explicitMode === "watch") {
return env;
@ -226,9 +230,19 @@ export function resolveRunVitestSpawnEnv(env = process.env, argv = []) {
if (explicitMode !== "run" && !isTruthyEnvValue(env.CI)) {
return env;
}
const hasTimeout = Object.hasOwn(env, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY);
const timeoutMs = hasTimeout
? parsePositiveInt(env[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY])
: DEFAULT_VITEST_NO_OUTPUT_TIMEOUT_MS;
const hasHeartbeat = Object.hasOwn(env, VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY);
return {
...env,
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: String(DEFAULT_VITEST_NO_OUTPUT_TIMEOUT_MS),
...(!hasTimeout
? { [VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]: String(DEFAULT_VITEST_NO_OUTPUT_TIMEOUT_MS) }
: {}),
...(!hasHeartbeat && timeoutMs !== null && DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS < timeoutMs
? { [VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY]: String(DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS) }
: {}),
};
}
@ -508,6 +522,10 @@ export function installVitestNoOutputWatchdog(params) {
const setTimeoutFn = params.setTimeoutFn ?? setTimeout;
const clearTimeoutFn = params.clearTimeoutFn ?? clearTimeout;
const forceKillAfterMs = params.forceKillAfterMs ?? 5_000;
const heartbeatMs =
params.heartbeatMs && params.heartbeatMs > 0 && params.heartbeatMs < timeoutMs
? params.heartbeatMs
: null;
const streams = params.streams?.filter(Boolean) ?? [];
const label = params.label?.trim();
const suffix = label ? ` (${label})` : "";
@ -515,6 +533,15 @@ export function installVitestNoOutputWatchdog(params) {
let active = true;
let silenceTimer = null;
let forceKillTimer = null;
let heartbeatTimer = null;
let silentForMs = 0;
const clearHeartbeatTimer = () => {
if (heartbeatTimer !== null) {
clearTimeoutFn(heartbeatTimer);
heartbeatTimer = null;
}
};
const clearForceKillTimer = () => {
if (forceKillTimer !== null) {
@ -530,15 +557,35 @@ export function installVitestNoOutputWatchdog(params) {
}
};
const scheduleHeartbeatTimer = () => {
if (!active || heartbeatMs === null) {
return;
}
clearHeartbeatTimer();
heartbeatTimer = setTimeoutFn(() => {
if (!active) {
return;
}
silentForMs += heartbeatMs;
params.log?.(`[vitest] still running with no output for ${silentForMs}ms${suffix}.`);
if (silentForMs + heartbeatMs < timeoutMs) {
scheduleHeartbeatTimer();
}
}, heartbeatMs);
};
const resetSilenceTimer = () => {
if (!active) {
return;
}
clearSilenceTimer();
silentForMs = 0;
scheduleHeartbeatTimer();
silenceTimer = setTimeoutFn(() => {
if (!active) {
return;
}
clearHeartbeatTimer();
params.log?.(
`[vitest] no output for ${timeoutMs}ms; terminating stalled Vitest process group${suffix}.`,
);
@ -580,6 +627,7 @@ export function installVitestNoOutputWatchdog(params) {
active = false;
clearSilenceTimer();
clearForceKillTimer();
clearHeartbeatTimer();
for (const { stream, handler } of listeners) {
stream.off("data", handler);
}
@ -629,6 +677,7 @@ export function spawnWatchedVitestProcess({
const teardownNoOutputWatchdog = installVitestNoOutputWatchdog({
streams: [child.stdout, child.stderr],
timeoutMs: resolveVitestNoOutputTimeoutMs(env),
heartbeatMs: resolveVitestNoOutputHeartbeatMs(env),
label,
log: (message) => {
console.error(message);

View file

@ -21,6 +21,7 @@ export type ChangedTestTargetOptions = {
};
export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS: string;
export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS: string;
export function parseTestProjectsArgs(
args: string[],

View file

@ -51,7 +51,7 @@ import {
} from "./changed-lanes.mjs";
import { isCiLikeEnv, resolveLocalFullSuiteProfile } from "./lib/vitest-local-scheduling.mjs";
import {
DEFAULT_VITEST_NO_OUTPUT_TIMEOUT_MS,
DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS,
resolveVitestCliEntry,
resolveVitestNodeArgs,
} from "./run-vitest.mjs";
@ -732,9 +732,11 @@ const IMPORT_SPECIFIER_PATTERN =
/\b(?:import|export)\s+(?:type\s+)?(?:[^'"]*?\s+from\s+)?["']([^"']+)["']|\bimport\s*\(\s*["']([^"']+)["']\s*\)/gu;
const BROAD_CHANGED_ENV_KEY = "OPENCLAW_TEST_CHANGED_BROAD";
const VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS";
const VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS";
const VITEST_NO_OUTPUT_RETRY_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_RETRY";
export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS = String(
DEFAULT_VITEST_NO_OUTPUT_TIMEOUT_MS,
export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS = String(900_000);
export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS = String(
DEFAULT_VITEST_NO_OUTPUT_HEARTBEAT_MS,
);
const EXPLICIT_SOURCE_FULL_IMPORT_GRAPH_THRESHOLD = 12;
const GATEWAY_SERVER_FULL_SUITE_TARGET_CHUNK_COUNT = 4;
@ -2321,19 +2323,34 @@ export function applyDefaultMultiSpecVitestCachePaths(specs, params = {}) {
export function applyDefaultVitestNoOutputTimeout(specs, params = {}) {
const baseEnv = params.env ?? process.env;
if (Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY)) {
if (
Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY) &&
Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY)
) {
return specs;
}
return specs.map((spec) => {
if (spec.watchMode || Object.hasOwn(spec.env ?? {}, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY)) {
if (spec.watchMode) {
return spec;
}
const env = spec.env ?? {};
const nextEnv = { ...env };
if (
!Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY) &&
!Object.hasOwn(env, VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY)
) {
nextEnv[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY] = DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS;
}
if (
!Object.hasOwn(baseEnv, VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY) &&
!Object.hasOwn(env, VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY)
) {
nextEnv[VITEST_NO_OUTPUT_HEARTBEAT_ENV_KEY] =
DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS;
}
return {
...spec,
env: {
...spec.env,
[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]: DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS,
},
env: nextEnv,
};
});
}

View file

@ -13,6 +13,7 @@ import {
createChangedCheckChildEnv,
createChangedCheckPlan,
createPnpmManagedCommand,
createTargetedCoreLintCommand,
shouldDelegateChangedCheckToCrabbox,
shouldRunShrinkwrapGuard,
createShrinkwrapGuardCommand,
@ -292,11 +293,76 @@ describe("scripts/changed-lanes", () => {
OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1",
OPENCLAW_TSGO_SPARSE_SKIP: "1",
});
expect(plan.commands.find((command) => command.args[0] === "lint:core")?.env).toEqual({
PATH: "/usr/bin",
OPENCLAW_OXLINT_SKIP_LOCK: "1",
OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD: "1",
OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1",
expect(plan.commands.find((command) => command.name === "lint core changed file")).toEqual({
name: "lint core changed file",
bin: "node",
args: [
"scripts/run-oxlint.mjs",
"--tsconfig",
"config/tsconfig/oxlint.core.json",
"src/shared/string-normalization.ts",
],
env: {
PATH: "/usr/bin",
OPENCLAW_OXLINT_SKIP_LOCK: "1",
OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD: "1",
OPENCLAW_TSGO_HEAVY_CHECK_LOCK_HELD: "1",
},
});
});
it("falls back to full core lint for broad core diffs", () => {
const targets = Array.from({ length: 9 }, (_, index) => `src/shared/file-${index}.ts`);
const command = createTargetedCoreLintCommand(targets, { PATH: "/usr/bin" });
expect(command).toBeNull();
});
it("falls back to full core lint when a changed core target was deleted", () => {
expect(
createTargetedCoreLintCommand(
["src/shared/deleted.ts"],
{ PATH: "/usr/bin" },
{
fileExists: () => false,
},
),
).toBeNull();
});
it("falls back to full core lint for mixed core lint configuration diffs", () => {
expect(
createTargetedCoreLintCommand(
["config/tsconfig/oxlint.core.json", "src/shared/string-normalization.ts"],
{ PATH: "/usr/bin" },
{ fileExists: () => true },
),
).toBeNull();
});
it("targets small core lint diffs", () => {
expect(
createTargetedCoreLintCommand(
[
"scripts/check-changed.mjs",
"src/agents/auth-profiles/usage.ts",
"test/scripts/changed-lanes.test.ts",
],
{ PATH: "/usr/bin" },
{ fileExists: () => true },
),
).toEqual({
name: "lint core changed file",
bin: "node",
args: [
"scripts/run-oxlint.mjs",
"--tsconfig",
"config/tsconfig/oxlint.core.json",
"src/agents/auth-profiles/usage.ts",
],
env: {
PATH: "/usr/bin",
},
});
});

View file

@ -11,6 +11,7 @@ import {
resolveTestProjectsDelegationArgs,
resolveTestProjectsRunnerEnv,
resolveVitestCliEntry,
resolveVitestNoOutputHeartbeatMs,
resolveVitestNodeArgs,
resolveVitestNoOutputTimeoutMs,
resolveVitestSpawnParams,
@ -301,28 +302,34 @@ describe("scripts/run-vitest", () => {
it("defaults direct non-watch runs to the stall watchdog", () => {
expect(resolveRunVitestSpawnEnv({ PATH: "/usr/bin" }, ["run"])).toEqual({
PATH: "/usr/bin",
OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "120000",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "300000",
});
expect(resolveRunVitestSpawnEnv({ PATH: "/usr/bin" }, ["run", "-t", "watch"])).toEqual({
PATH: "/usr/bin",
OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "120000",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "300000",
});
expect(resolveRunVitestSpawnEnv({ PATH: "/usr/bin" }, ["--watch=false"])).toEqual({
PATH: "/usr/bin",
OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "120000",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "300000",
});
expect(resolveRunVitestSpawnEnv({ PATH: "/usr/bin" }, ["--watch", "false"])).toEqual({
PATH: "/usr/bin",
OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "120000",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "300000",
});
expect(resolveRunVitestSpawnEnv({ PATH: "/usr/bin" }, ["--no-watch"])).toEqual({
PATH: "/usr/bin",
OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "120000",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "300000",
});
expect(resolveRunVitestSpawnEnv({ CI: "true", PATH: "/usr/bin" }, ["src/foo.test.ts"])).toEqual(
{
CI: "true",
PATH: "/usr/bin",
OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "120000",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "300000",
},
);
@ -530,6 +537,44 @@ describe("scripts/run-vitest", () => {
}
});
it("prints bounded heartbeats before killing silent vitest runs", () => {
vi.useFakeTimers();
try {
const stdout = new EventEmitter();
const timeoutSpy = vi.fn();
const logSpy = vi.fn();
installVitestNoOutputWatchdog({
streams: [stdout],
timeoutMs: 1000,
heartbeatMs: 400,
forceKillAfterMs: 0,
log: logSpy,
onTimeout: timeoutSpy,
setTimeoutFn: setTimeout,
clearTimeoutFn: clearTimeout,
});
vi.advanceTimersByTime(400);
expect(logSpy).toHaveBeenCalledWith("[vitest] still running with no output for 400ms.");
vi.advanceTimersByTime(400);
expect(logSpy).toHaveBeenCalledWith("[vitest] still running with no output for 800ms.");
stdout.emit("data", "still alive");
vi.advanceTimersByTime(400);
expect(logSpy).toHaveBeenCalledWith("[vitest] still running with no output for 400ms.");
vi.advanceTimersByTime(600);
expect(timeoutSpy).toHaveBeenCalledTimes(1);
expect(logSpy).toHaveBeenCalledWith(
"[vitest] no output for 1000ms; terminating stalled Vitest process group.",
);
} finally {
vi.useRealTimers();
}
});
it("includes the runner label in watchdog logs when provided", () => {
vi.useFakeTimers();
try {
@ -553,4 +598,13 @@ describe("scripts/run-vitest", () => {
vi.useRealTimers();
}
});
it("parses the optional watchdog heartbeat interval", () => {
expect(
resolveVitestNoOutputHeartbeatMs({ OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "120000" }),
).toBe(120000);
expect(
resolveVitestNoOutputHeartbeatMs({ OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "0" }),
).toBeNull();
});
});

View file

@ -5,6 +5,7 @@ import path from "node:path";
import fg from "fast-glob";
import { beforeAll, describe, expect, it, vi } from "vitest";
import {
DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS,
DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS,
applyDefaultMultiSpecVitestCachePaths,
applyDefaultVitestNoOutputTimeout,
@ -2234,7 +2235,7 @@ describe("scripts/test-projects failed shard digest", () => {
});
describe("scripts/test-projects Vitest stall watchdog", () => {
it("adds a default no-output timeout to non-watch specs", () => {
it("adds default no-output watchdog settings to non-watch specs", () => {
const [spec] = applyDefaultVitestNoOutputTimeout(
[
{
@ -2252,6 +2253,9 @@ describe("scripts/test-projects Vitest stall watchdog", () => {
expect(spec?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe(
DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS,
);
expect(spec?.env.OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS).toBe(
DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_HEARTBEAT_MS,
);
});
it("keeps explicit watchdog settings and watch mode untouched", () => {
@ -2267,7 +2271,11 @@ describe("scripts/test-projects Vitest stall watchdog", () => {
},
{
config: "test/vitest/vitest.extension-memory.config.ts",
env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "0", PATH: "/usr/bin" },
env: {
OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS: "25000",
OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "0",
PATH: "/usr/bin",
},
includeFilePath: null,
includePatterns: null,
pnpmArgs: [],
@ -2278,7 +2286,9 @@ describe("scripts/test-projects Vitest stall watchdog", () => {
);
expect(specs[0]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBeUndefined();
expect(specs[0]?.env.OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS).toBeUndefined();
expect(specs[1]?.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("0");
expect(specs[1]?.env.OPENCLAW_VITEST_NO_OUTPUT_HEARTBEAT_MS).toBe("25000");
});
it("allows changed checks to disable automatic silent-run retries", () => {