test: tighten changed test routing

This commit is contained in:
Peter Steinberger 2026-04-26 10:24:50 +01:00
parent 87ac8b0456
commit 9eb0934492
No known key found for this signature in database
8 changed files with 283 additions and 33 deletions

View file

@ -11,12 +11,13 @@ title: "Tests"
- `pnpm test:coverage`: Runs the unit suite with V8 coverage (via `vitest.unit.config.ts`). This is a loaded-file unit coverage gate, not whole-repo all-file coverage. Thresholds are 70% lines/functions/statements and 55% branches. Because `coverage.all` is false, the gate measures files loaded by the unit coverage suite instead of treating every split-lane source file as uncovered.
- `pnpm test:coverage:changed`: Runs unit coverage only for files changed since `origin/main`.
- `pnpm test:changed`: expands changed git paths into scoped Vitest lanes when the diff only touches routable source/test files. Config/setup changes still fall back to the native root projects run so wiring edits rerun broadly when needed.
- `pnpm test:changed:focused`: inner-loop changed test run. It only runs precise targets from direct test edits, sibling `*.test.ts` files, explicit source mappings, and the local import graph. Broad/config/package changes are skipped instead of expanding to the full changed-test fallback.
- `pnpm changed:lanes`: shows the architectural lanes triggered by the diff against `origin/main`.
- `pnpm check:changed`: runs the smart changed gate for the diff against `origin/main`. It runs core work with core test lanes, extension work with extension test lanes, test-only work with test typecheck/tests only, expands public Plugin SDK or plugin-contract changes to one extension validation pass, and keeps release metadata-only version bumps on targeted version/config/root-dependency checks.
- `pnpm test`: routes explicit file/directory targets through scoped Vitest lanes. Untargeted runs use fixed shard groups and expand to leaf configs for local parallel execution; the extension group always expands to the per-extension shard configs instead of one giant root-project process.
- Full, extension, and include-pattern shard runs update local timing data in `.artifacts/vitest-shard-timings.json`; later whole-config runs use those timings to balance slow and fast shards. Include-pattern CI shards append the shard name to the timing key, which keeps filtered shard timings visible without replacing whole-config timing data. Set `OPENCLAW_TEST_PROJECTS_TIMINGS=0` to ignore the local timing artifact.
- Selected `plugin-sdk` and `commands` test files now route through dedicated light lanes that keep only `test/setup.ts`, leaving runtime-heavy cases on their existing lanes.
- Selected `plugin-sdk` and `commands` helper source files also map `pnpm test:changed` to explicit sibling tests in those light lanes, so small helper edits avoid rerunning the heavy runtime-backed suites.
- Source files with sibling tests map to that sibling before falling back to wider directory globs. Helper edits under `test/helpers/channels` and `test/helpers/plugins` use a local import graph to run importing tests instead of broad-running every shard when the dependency path is precise.
- `auto-reply` now also splits into three dedicated configs (`core`, `top-level`, `reply`) so the reply harness does not dominate the lighter top-level status/token/helper tests.
- Base Vitest config now defaults to `pool: "threads"` and `isolate: false`, with the shared non-isolated runner enabled across the repo configs.
- `pnpm test:channels` runs `vitest.channels.config.ts`.

View file

@ -1478,6 +1478,7 @@
"test:build:singleton": "node scripts/test-built-plugin-singleton.mjs",
"test:bundled": "node scripts/run-vitest.mjs run --config test/vitest/vitest.bundled.config.ts",
"test:changed": "node scripts/test-projects.mjs --changed origin/main",
"test:changed:focused": "OPENCLAW_TEST_CHANGED_FOCUSED=1 node scripts/test-projects.mjs --changed origin/main",
"test:changed:max": "OPENCLAW_VITEST_MAX_WORKERS=8 node scripts/test-projects.mjs --changed origin/main",
"test:channels": "node scripts/run-vitest.mjs run --config test/vitest/vitest.channels.config.ts",
"test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins",

View file

@ -275,7 +275,9 @@ async function main() {
const baseEnv = resolveLocalVitestEnv(process.env);
const { targetArgs } = parseTestProjectsArgs(args, process.cwd());
const changedTargetArgs =
targetArgs.length === 0 ? resolveChangedTargetArgs(args, process.cwd()) : null;
targetArgs.length === 0
? resolveChangedTargetArgs(args, process.cwd(), undefined, { env: baseEnv })
: null;
const rawRunSpecs =
targetArgs.length === 0 && changedTargetArgs === null
? buildFullSuiteVitestRunPlans(args, process.cwd()).map((plan) => ({

View file

@ -14,6 +14,12 @@ export type VitestRunSpec = {
watchMode: boolean;
};
export type ChangedTestTargetOptions = {
cwd?: string;
env?: Record<string, string | undefined>;
focused?: boolean;
};
export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS: string;
export function parseTestProjectsArgs(
@ -29,15 +35,20 @@ export function buildVitestRunPlans(
args: string[],
cwd?: string,
listChangedPaths?: (baseRef: string, cwd: string) => string[],
options?: ChangedTestTargetOptions,
): VitestRunPlan[];
export function resolveChangedTargetArgs(
args: string[],
cwd?: string,
listChangedPaths?: (baseRef: string, cwd: string) => string[],
options?: ChangedTestTargetOptions,
): string[] | null;
export function resolveChangedTestTargetPlan(changedPaths: string[]): {
export function resolveChangedTestTargetPlan(
changedPaths: string[],
options?: ChangedTestTargetOptions,
): {
mode: "none" | "broad" | "targets";
targets: string[];
};

View file

@ -301,6 +301,11 @@ const GENERATED_CHANGED_TEST_TARGETS = new Set([
"src/canvas-host/a2ui/.bundle.hash",
"src/canvas-host/a2ui/a2ui.bundle.js",
]);
const SOURCE_ROOTS_FOR_IMPORT_GRAPH = ["src", "extensions", "packages", "ui/src", "test"];
const IMPORTABLE_FILE_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts"];
const IMPORT_SPECIFIER_PATTERN =
/\b(?:import|export)\s+(?:type\s+)?(?:[^'"]*?\s+from\s+)?["']([^"']+)["']|\bimport\s*\(\s*["']([^"']+)["']\s*\)/gu;
const FOCUSED_CHANGED_ENV_KEY = "OPENCLAW_TEST_CHANGED_FOCUSED";
const VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS";
const VITEST_NO_OUTPUT_RETRY_ENV_KEY = "OPENCLAW_VITEST_NO_OUTPUT_RETRY";
export const DEFAULT_TEST_PROJECTS_VITEST_NO_OUTPUT_TIMEOUT_MS = "180000";
@ -375,6 +380,10 @@ function isFileLikeTarget(arg) {
return /\.(?:test|spec)\.[cm]?[jt]sx?$/u.test(arg);
}
function isTestFileTarget(arg) {
return /\.(?:test|spec)\.[cm]?[jt]sx?$/u.test(arg);
}
function isLikelyFileTarget(arg) {
return /(?:^|\/)[^/]+\.[A-Za-z0-9]+$/u.test(arg);
}
@ -406,6 +415,128 @@ function toScopedIncludePattern(arg, cwd) {
return `${relative.replace(/\/+$/u, "")}/**/*.test.ts`;
}
function isSkippedImportGraphDirectory(name) {
return (
name === ".git" ||
name === "dist" ||
name === "node_modules" ||
name === "vendor" ||
name.startsWith(".openclaw-runtime-deps")
);
}
function listImportGraphFiles(cwd, directory, files = []) {
let entries;
try {
entries = fs.readdirSync(path.join(cwd, directory), { withFileTypes: true });
} catch {
return files;
}
for (const entry of entries) {
const relative = normalizePathPattern(path.posix.join(directory, entry.name));
if (entry.isDirectory()) {
if (!isSkippedImportGraphDirectory(entry.name)) {
listImportGraphFiles(cwd, relative, files);
}
continue;
}
if (entry.isFile() && IMPORTABLE_FILE_EXTENSIONS.some((ext) => relative.endsWith(ext))) {
files.push(relative);
}
}
return files;
}
function resolveImportSpecifier(importer, specifier, fileSet) {
if (!specifier.startsWith(".")) {
return null;
}
const importerDir = path.posix.dirname(importer);
const base = normalizePathPattern(path.posix.normalize(path.posix.join(importerDir, specifier)));
const candidates = [];
const ext = path.posix.extname(base);
if (ext) {
candidates.push(base);
if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) {
const withoutExt = base.slice(0, -ext.length);
candidates.push(
...IMPORTABLE_FILE_EXTENSIONS.map((candidateExt) => `${withoutExt}${candidateExt}`),
);
}
} else {
candidates.push(
...IMPORTABLE_FILE_EXTENSIONS.map((candidateExt) => `${base}${candidateExt}`),
...IMPORTABLE_FILE_EXTENSIONS.map((candidateExt) => `${base}/index${candidateExt}`),
);
}
return candidates.find((candidate) => fileSet.has(candidate)) ?? null;
}
let cachedImportGraph = null;
let cachedImportGraphCwd = null;
function getImportGraph(cwd) {
if (cachedImportGraph && cachedImportGraphCwd === cwd) {
return cachedImportGraph;
}
const files = SOURCE_ROOTS_FOR_IMPORT_GRAPH.flatMap((root) => listImportGraphFiles(cwd, root));
const fileSet = new Set(files);
const reverseImports = new Map();
const testFiles = new Set(
files.filter((file) => isTestFileTarget(file) && !file.endsWith(".live.test.ts")),
);
for (const file of files) {
let source = "";
try {
source = fs.readFileSync(path.join(cwd, file), "utf8");
} catch {
continue;
}
for (const match of source.matchAll(IMPORT_SPECIFIER_PATTERN)) {
const imported = resolveImportSpecifier(file, match[1] ?? match[2] ?? "", fileSet);
if (!imported) {
continue;
}
const importers = reverseImports.get(imported) ?? [];
importers.push(file);
reverseImports.set(imported, importers);
}
}
cachedImportGraph = { reverseImports, testFiles };
cachedImportGraphCwd = cwd;
return cachedImportGraph;
}
function resolveAffectedTestsFromImportGraph(changedPath, cwd) {
const normalized = normalizePathPattern(changedPath);
const { reverseImports, testFiles } = getImportGraph(cwd);
const queue = [normalized];
const seen = new Set(queue);
const targets = [];
for (let index = 0; index < queue.length; index += 1) {
const current = queue[index];
for (const importer of reverseImports.get(current) ?? []) {
if (seen.has(importer)) {
continue;
}
seen.add(importer);
if (testFiles.has(importer)) {
targets.push(importer);
}
queue.push(importer);
}
}
return [...new Set(targets)].toSorted((left, right) => left.localeCompare(right));
}
function resolveVitestConfigTargetKind(relative) {
return VITEST_CONFIG_TARGET_KIND_BY_PATH.get(relative) ?? null;
}
@ -554,6 +685,11 @@ function resolveToolingTestTargets(changedPath) {
return TOOLING_SOURCE_TEST_TARGETS.get(changedPath) ?? TOOLING_TEST_TARGETS.get(changedPath);
}
function shouldUseFocusedChangedTargets(env = process.env) {
const value = env[FOCUSED_CHANGED_ENV_KEY]?.trim().toLowerCase();
return ["1", "true", "yes", "on"].includes(value ?? "");
}
function isRoutableChangedTarget(changedPath) {
if (GENERATED_CHANGED_TEST_TARGETS.has(changedPath)) {
return false;
@ -564,7 +700,39 @@ function isRoutableChangedTarget(changedPath) {
return /^(?:src|test|extensions|ui|packages)(?:\/|$)/u.test(changedPath);
}
export function resolveChangedTestTargetPlan(changedPaths) {
function resolveSiblingTestTarget(changedPath, cwd) {
if (!/\.[cm]?tsx?$/u.test(changedPath) || isTestFileTarget(changedPath)) {
return null;
}
const withoutExtension = changedPath.replace(/\.[cm]?tsx?$/u, "");
const sibling = `${withoutExtension}.test.ts`;
return fs.existsSync(path.join(cwd, sibling)) ? sibling : null;
}
function resolvePreciseChangedTestTargets(changedPath, options) {
const cwd = options.cwd ?? process.cwd();
const mappedTargets =
resolveToolingTestTargets(changedPath) ?? SOURCE_TEST_TARGETS.get(changedPath);
if (mappedTargets) {
return mappedTargets;
}
if (isRoutableChangedTarget(changedPath) && isTestFileTarget(changedPath)) {
return [changedPath];
}
const siblingTest = resolveSiblingTestTarget(changedPath, cwd);
if (siblingTest) {
return [siblingTest];
}
if (/^(?:src|test\/helpers|extensions|packages|ui\/src)\//u.test(changedPath)) {
const affectedTests = resolveAffectedTestsFromImportGraph(changedPath, cwd);
if (affectedTests.length > 0) {
return affectedTests;
}
}
return null;
}
export function resolveChangedTestTargetPlan(changedPaths, options = {}) {
if (changedPaths.length === 0) {
return { mode: "none", targets: [] };
}
@ -572,22 +740,29 @@ export function resolveChangedTestTargetPlan(changedPaths) {
if (toolingTargets) {
return { mode: "targets", targets: toolingTargets };
}
if (shouldKeepBroadChangedRun(changedPaths)) {
return { mode: "broad", targets: [] };
}
const changedLanes = detectChangedLanes(changedPaths);
if (changedLanes.lanes.all) {
const focused = options.focused ?? shouldUseFocusedChangedTargets(options.env ?? {});
const targets = [];
for (const changedPath of changedPaths) {
const preciseTargets = resolvePreciseChangedTestTargets(changedPath, options);
if (preciseTargets) {
targets.push(...preciseTargets);
continue;
}
if (focused) {
continue;
}
if (shouldKeepBroadChangedRun([changedPath]) || changedLanes.lanes.all) {
return { mode: "broad", targets: [] };
}
if (isRoutableChangedTarget(changedPath)) {
targets.push(changedPath);
}
}
if (!focused && changedLanes.lanes.all) {
return { mode: "broad", targets: [] };
}
const targets = changedPaths.flatMap((changedPath) => {
const mappedTargets =
resolveToolingTestTargets(changedPath) ?? SOURCE_TEST_TARGETS.get(changedPath);
if (mappedTargets) {
return mappedTargets;
}
return isRoutableChangedTarget(changedPath) ? [changedPath] : [];
});
if (changedLanes.extensionImpactFromCore) {
if (!focused && changedLanes.extensionImpactFromCore) {
targets.push("extensions");
}
return { mode: "targets", targets: [...new Set(targets)] };
@ -604,13 +779,17 @@ export function resolveChangedTargetArgs(
args,
cwd = process.cwd(),
listChangedPaths = listChangedPathsFromGit,
options = {},
) {
const baseRef = extractChangedBaseRef(args);
if (!baseRef) {
return null;
}
const changedPaths = listChangedPaths(baseRef, cwd);
const plan = resolveChangedTestTargetPlan(changedPaths);
const plan = resolveChangedTestTargetPlan(changedPaths, {
cwd,
...options,
});
if (plan.mode === "broad") {
return null;
}
@ -877,10 +1056,11 @@ export function buildVitestRunPlans(
args,
cwd = process.cwd(),
listChangedPaths = listChangedPathsFromGit,
options = {},
) {
const { forwardedArgs, targetArgs, watchMode } = parseTestProjectsArgs(args, cwd);
const changedTargetArgs =
targetArgs.length === 0 ? resolveChangedTargetArgs(args, cwd, listChangedPaths) : null;
targetArgs.length === 0 ? resolveChangedTargetArgs(args, cwd, listChangedPaths, options) : null;
const activeTargetArgs = changedTargetArgs ?? targetArgs;
const activeForwardedArgs =
changedTargetArgs !== null ? stripChangedArgs(forwardedArgs) : forwardedArgs;
@ -1187,7 +1367,10 @@ export function shouldRetryVitestNoOutputTimeout(env = process.env) {
export function createVitestRunSpecs(args, params = {}) {
const cwd = params.cwd ?? process.cwd();
const baseEnv = params.baseEnv ?? process.env;
const plans = filterPlansForContractIncludeFile(buildVitestRunPlans(args, cwd), baseEnv);
const plans = filterPlansForContractIncludeFile(
buildVitestRunPlans(args, cwd, listChangedPathsFromGit, { env: baseEnv }),
baseEnv,
);
return plans.map((plan, index) => {
const includeFilePath = plan.includePatterns
? path.join(

View file

@ -908,11 +908,11 @@ describe("test-projects args", () => {
expect(
resolveChangedTargetArgs(["--changed=origin/main"], process.cwd(), () => changedPaths),
).toEqual(["src/plugin-sdk/core.ts", "extensions"]);
).toEqual(["src/plugin-sdk/core.test.ts", "extensions"]);
expect(plans[0]).toEqual({
config: "test/vitest/vitest.plugin-sdk.config.ts",
forwardedArgs: [],
includePatterns: ["src/plugin-sdk/**/*.test.ts"],
includePatterns: ["src/plugin-sdk/core.test.ts"],
watchMode: false,
});
expect(plans.map((plan) => plan.config)).toContain(
@ -932,7 +932,14 @@ describe("test-projects args", () => {
{
config: "test/vitest/vitest.extension-discord.config.ts",
forwardedArgs: [],
includePatterns: ["extensions/discord/src/monitor/**/*.test.ts"],
includePatterns: [
"extensions/discord/src/channel-actions.contract.test.ts",
"extensions/discord/src/channel.test.ts",
"extensions/discord/src/monitor/message-handler.bot-self-filter.test.ts",
"extensions/discord/src/monitor/message-handler.queue.test.ts",
"extensions/discord/src/monitor/provider.skill-dedupe.test.ts",
"extensions/discord/src/monitor/provider.test.ts",
],
watchMode: false,
},
]);

View file

@ -217,7 +217,7 @@ describe("scripts/changed-lanes", () => {
all: false,
});
expect(plan.runExtensionTests).toBe(true);
expect(plan.testTargets).toEqual(["src/plugin-sdk/core.ts"]);
expect(plan.testTargets).toEqual(["src/plugin-sdk/core.test.ts"]);
});
it("fails safe for root config changes", () => {

View file

@ -22,7 +22,7 @@ describe("scripts/test-projects changed-target routing", () => {
"src/shared/string-normalization.ts",
"src/utils/provider-utils.ts",
]),
).toEqual(["src/shared/string-normalization.ts", "src/utils/provider-utils.ts"]);
).toEqual(["src/shared/string-normalization.test.ts", "src/utils/provider-utils.test.ts"]);
});
it("keeps the broad changed run for Vitest wiring edits", () => {
@ -123,7 +123,7 @@ describe("scripts/test-projects changed-target routing", () => {
{
config: "test/vitest/vitest.extension-browser.config.ts",
forwardedArgs: [],
includePatterns: ["extensions/browser/src/browser/**/*.test.ts"],
includePatterns: ["extensions/browser/src/browser/cdp.helpers.test.ts"],
watchMode: false,
},
]);
@ -137,6 +137,32 @@ describe("scripts/test-projects changed-target routing", () => {
).toBeNull();
});
it("routes channel helper edits through the tests that import them", () => {
expect(resolveChangedTestTargetPlan(["test/helpers/channels/directory-ids.ts"])).toEqual({
mode: "targets",
targets: [
"extensions/discord/src/directory-contract.test.ts",
"extensions/slack/src/directory-contract.test.ts",
"extensions/telegram/src/directory-contract.test.ts",
],
});
});
it("routes channel contract helper edits through contract shards", () => {
const plan = resolveChangedTestTargetPlan([
"test/helpers/channels/registry-backed-contract-shards.ts",
]);
expect(plan.mode).toBe("targets");
expect(plan.targets).toContain(
"src/channels/plugins/contracts/plugin.registry-backed-shard-a.contract.test.ts",
);
expect(plan.targets).toContain(
"src/channels/plugins/contracts/threading.registry-backed-shard-h.contract.test.ts",
);
expect(plan.targets).not.toContain("extensions/discord/src/channel-actions.contract.test.ts");
});
it("routes precise plugin contract helpers without broad-running every shard", () => {
expect(
resolveChangedTargetArgs(["--changed", "origin/main"], process.cwd(), () => [
@ -208,7 +234,7 @@ describe("scripts/test-projects changed-target routing", () => {
{
config: "test/vitest/vitest.extension-providers.config.ts",
forwardedArgs: [],
includePatterns: ["extensions/lmstudio/src/**/*.test.ts"],
includePatterns: ["extensions/lmstudio/src/runtime.test.ts"],
watchMode: false,
},
]);
@ -392,7 +418,7 @@ describe("scripts/test-projects changed-target routing", () => {
{
config: "test/vitest/vitest.utils.config.ts",
forwardedArgs: [],
includePatterns: ["src/utils/**/*.test.ts"],
includePatterns: ["src/utils/provider-utils.test.ts"],
watchMode: false,
},
]);
@ -459,16 +485,16 @@ describe("scripts/test-projects changed-target routing", () => {
]);
});
it("keeps non-allowlisted plugin-sdk source files on the heavy lane plus extension tests", () => {
it("routes plugin-sdk source files with sibling tests narrowly plus extension tests", () => {
const plans = buildVitestRunPlans(["--changed", "origin/main"], process.cwd(), () => [
"src/plugin-sdk/facade-runtime.ts",
]);
expect(plans).toEqual([
{
config: "test/vitest/vitest.plugin-sdk.config.ts",
config: "test/vitest/vitest.bundled.config.ts",
forwardedArgs: [],
includePatterns: ["src/plugin-sdk/**/*.test.ts"],
includePatterns: ["src/plugin-sdk/facade-runtime.test.ts"],
watchMode: false,
},
...listFullExtensionVitestProjectConfigs().map((config) => ({
@ -480,7 +506,7 @@ describe("scripts/test-projects changed-target routing", () => {
]);
});
it("keeps non-allowlisted commands source files on the heavy lane", () => {
it("routes command source files with sibling tests narrowly on the command lane", () => {
const plans = buildVitestRunPlans(["--changed", "origin/main"], process.cwd(), () => [
"src/commands/channels.add.ts",
]);
@ -489,12 +515,31 @@ describe("scripts/test-projects changed-target routing", () => {
{
config: "test/vitest/vitest.commands.config.ts",
forwardedArgs: [],
includePatterns: ["src/commands/**/*.test.ts"],
includePatterns: ["src/commands/channels.add.test.ts"],
watchMode: false,
},
]);
});
it("keeps focused changed mode to precise targets only", () => {
expect(
resolveChangedTestTargetPlan(["package.json", "src/commands/channels.add.ts"], {
focused: true,
}),
).toEqual({
mode: "targets",
targets: ["src/commands/channels.add.test.ts"],
});
});
it("uses import-graph targets in focused changed mode", () => {
expect(
resolveChangedTestTargetPlan(["test/helpers/plugins/plugin-registration.ts"], {
focused: true,
}).targets,
).toContain("extensions/openrouter/index.test.ts");
});
it.each([
"src/gateway/gateway.test.ts",
"src/gateway/server.startup-matrix-migration.integration.test.ts",