mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
9137cfcc5f
* perf(ci): run qualified unit tests with native Bun * test(ci): keep a V8-only cache warm fixture * fix(ci): include shared timeout in PR wrapper inventory * perf(test): skip Bun bytecode cache during test teardown * test: handle worker fixture retirement notifications Handle rejected observer returns without delaying retirement, matching the worker pool contract, and avoid shadowing the lane fixture input helper. * perf(ci): qualify and tune Bun test workers Complete SQLite admission before Bun Vitest threads start and retain each worker generation's canonical cleanup decision in lifecycle fixtures. Keep proxy-agent fixture values out of Bun's native fetch cache and compare large socket payloads with full byte equality. Renew the 292-file native qualification from paired 2,518-case proof. Apply the existing UI JIT and allocator settings to ordinary unit-fast Bun/Vitest selections and their cache warmer, preserving Node and native Bun settings and dual-runtime release coverage. Proof: 208 routing/warming cases; 61 lifecycle cases on each runtime; 66 ordered proxy/fetch cases plus the canonical 11-case proxy owner on each runtime; independent P0-P2 review. The changed-file gate was interrupted by the local disk reserve guard after core typing and 14 core-test type shards passed. Tuned full-cohort performance remains to be measured; the completed untuned comparison was 4.60% slower than Node. * perf(test): expand native Bun coverage and reduce artifact IO * perf(test): yield between artifact verification batches * test: qualify native Bun cases after runtime integration * test: refresh native Bun qualification after main integration
1003 lines
37 KiB
TypeScript
1003 lines
37 KiB
TypeScript
// Runs one CI node test shard job: either explicit changed-test targets or a
|
|
// list of packed group plans. Extracted from .github/workflows/ci.yml so the
|
|
// execution policy is unit-testable and plans can run concurrently.
|
|
import { spawn, type ChildProcess } from "node:child_process";
|
|
import { randomUUID } from "node:crypto";
|
|
import {
|
|
constants,
|
|
cpSync,
|
|
existsSync,
|
|
lstatSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
rmSync,
|
|
statSync,
|
|
unlinkSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { rm } from "node:fs/promises";
|
|
import os, { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { Readable } from "node:stream";
|
|
import { StringDecoder } from "node:string_decoder";
|
|
import { fileURLToPath } from "node:url";
|
|
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
|
import { decodeNodeTestGroups } from "./lib/ci-node-test-groups-codec.mts";
|
|
import type { CiTestRuntimeSelection } from "./lib/ci-test-runtime.mts";
|
|
import { isDirectRunUrl } from "./lib/direct-run.mjs";
|
|
import { isConstrainedCiCheckHost, isExclusiveCiTestConfig } from "./lib/local-check-runtime.mts";
|
|
import { parsePositiveInt, readPositiveEnvInt } from "./lib/numeric-options.mjs";
|
|
import { isCiLikeEnv } from "./lib/vitest-local-scheduling.mts";
|
|
import {
|
|
MAX_CI_VITEST_PLAN_CONCURRENCY,
|
|
resolveCiVitestPlanConcurrency,
|
|
runVitestPlans,
|
|
} from "./lib/vitest-plan-scheduling.mts";
|
|
import type { VitestWorkerRun } from "./lib/vitest-worker-run.mts";
|
|
|
|
// CI admits at most two plans only when the actual host has room. Each plan
|
|
// normally keeps inner parallelism 1; qualified singleton envelopes have a separate admission.
|
|
const FS_MODULE_CACHE_ROOT_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_ROOT";
|
|
const FS_MODULE_CACHE_PATH_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_PATH";
|
|
const FS_MODULE_CACHE_WRITER_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_WRITER";
|
|
const NODE_COMPILE_CACHE_PATH_ENV_KEY = "NODE_COMPILE_CACHE";
|
|
const NODE_COMPILE_CACHE_WRITER_ENV_KEY = "OPENCLAW_NODE_COMPILE_CACHE_WRITER";
|
|
const VITEST_EXTRA_ARGS_ENV_KEY = "OPENCLAW_NODE_TEST_VITEST_ARGS_JSON";
|
|
const FS_MODULE_CACHE_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
|
const NODE_COMPILE_CACHE_MAX_BYTES = 1024 * 1024 * 1024;
|
|
const FS_MODULE_CACHE_PRUNE_TARGET_RATIO = 0.75;
|
|
const FS_MODULE_CACHE_METADATA_FILE = "_metadata.json";
|
|
const FS_MODULE_CACHE_GENERATION_FILE = ".openclaw-transform-generation";
|
|
|
|
function reportCiResourceSnapshot(phase: "start" | "end") {
|
|
const pressure = (resource: "cpu" | "memory" | "io") => {
|
|
if (process.platform !== "linux") {
|
|
return null;
|
|
}
|
|
try {
|
|
return readFileSync(`/proc/pressure/${resource}`, "utf8").trim();
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
console.log(
|
|
`[shard:resource-snapshot] ${JSON.stringify({
|
|
phase,
|
|
uptimeSeconds: os.uptime(),
|
|
cpuModel: os.cpus()[0]?.model ?? null,
|
|
loadAverage: os.loadavg(),
|
|
freeMemoryBytes: os.freemem(),
|
|
availableMemoryBytes: process.availableMemory?.() ?? null,
|
|
constrainedMemoryBytes: process.constrainedMemory?.() ?? null,
|
|
pressure: { cpu: pressure("cpu"), memory: pressure("memory"), io: pressure("io") },
|
|
})}`,
|
|
);
|
|
}
|
|
|
|
export type ShardTargetPlan = { kind: "target"; name: string; target: string };
|
|
type ShardGroupConfig = {
|
|
configs: string[];
|
|
requiresDist?: boolean;
|
|
pretestBuildMode?: "runtime" | "private-qa";
|
|
fallbackMaxWorkers?: number;
|
|
minTotalMemoryBytes?: number;
|
|
env?: Record<string, unknown> | null;
|
|
includePatterns?: string[] | null;
|
|
shard_name?: string;
|
|
timing_key?: string;
|
|
};
|
|
export type ShardGroupPlan = {
|
|
kind: "group";
|
|
name: string;
|
|
plan: ShardGroupConfig;
|
|
timingKey?: string;
|
|
};
|
|
export type ShardPlan = ShardTargetPlan | ShardGroupPlan;
|
|
type RunShardOptions = {
|
|
concurrency?: number;
|
|
continueOnFailure?: boolean;
|
|
env?: NodeJS.ProcessEnv;
|
|
fsModuleCacheMaxBytes?: number;
|
|
nodeCompileCacheMaxBytes?: number;
|
|
runChild?: typeof runChild;
|
|
scratchDir?: string;
|
|
};
|
|
|
|
function isStringArray(value: unknown): value is string[] {
|
|
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
}
|
|
|
|
function isShardGroupConfig(value: unknown): value is ShardGroupConfig {
|
|
return isRecord(value) && isStringArray(value.configs);
|
|
}
|
|
|
|
function parseJsonEnv(
|
|
env: Record<string, unknown>,
|
|
name: string,
|
|
fallback: unknown = null,
|
|
): unknown {
|
|
try {
|
|
const value = env[name];
|
|
return typeof value === "string" ? (JSON.parse(value) ?? fallback) : fallback;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
export function resolveShardPlans(env: NodeJS.ProcessEnv = process.env): ShardPlan[] {
|
|
const targets = parseJsonEnv(env, "OPENCLAW_NODE_TEST_TARGETS_JSON");
|
|
if (isStringArray(targets) && targets.length > 0) {
|
|
// One target per child process preserves the isolation boundaries encoded
|
|
// by full-suite include-pattern shards while keeping one runner job.
|
|
return targets.map((target) => ({ kind: "target", name: target, target }));
|
|
}
|
|
|
|
// The CI manifest packs matrix groups so preflight's job outputs stay under
|
|
// GitHub's 1 MiB UTF-16 cap. Plain JSON remains for the Vitest cache warmer,
|
|
// which writes its small group list straight into GITHUB_ENV.
|
|
const packedGroups = env.OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64?.trim();
|
|
const groups = packedGroups
|
|
? decodeNodeTestGroups(packedGroups)
|
|
: parseJsonEnv(env, "OPENCLAW_NODE_TEST_GROUPS_JSON");
|
|
const groupPlans = Array.isArray(groups) ? groups.filter(isShardGroupConfig) : [];
|
|
const configs = parseJsonEnv(env, "OPENCLAW_NODE_TEST_CONFIGS_JSON", []);
|
|
const groupEnv = parseJsonEnv(env, "OPENCLAW_NODE_TEST_ENV_JSON");
|
|
const includePatterns = parseJsonEnv(env, "OPENCLAW_NODE_TEST_INCLUDE_PATTERNS_JSON");
|
|
const plans: ShardGroupConfig[] =
|
|
groupPlans.length > 0
|
|
? groupPlans
|
|
: [
|
|
{
|
|
configs: isStringArray(configs) ? configs : [],
|
|
env: isRecord(groupEnv) ? groupEnv : null,
|
|
includePatterns: isStringArray(includePatterns) ? includePatterns : null,
|
|
shard_name: env.OPENCLAW_VITEST_SHARD_NAME,
|
|
},
|
|
];
|
|
return plans.map((plan) => {
|
|
const name = plan.shard_name ?? plan.configs?.[0] ?? "group";
|
|
return { kind: "group", name, plan, timingKey: plan.timing_key ?? name };
|
|
});
|
|
}
|
|
|
|
function mergePlanEnv(baseEnv: NodeJS.ProcessEnv, overrides: unknown): NodeJS.ProcessEnv {
|
|
const childEnv = { ...baseEnv };
|
|
if (isRecord(overrides)) {
|
|
for (const [key, value] of Object.entries(overrides)) {
|
|
if (typeof value === "string") {
|
|
const inherited = baseEnv[key]?.trim();
|
|
// Pins may lower the admitted job budget, never raise it. Compiler
|
|
// preparation and test children must inherit the same intersection.
|
|
childEnv[key] =
|
|
key === "OPENCLAW_VITEST_MAX_WORKERS" && inherited
|
|
? String(
|
|
Math.min(
|
|
parsePositiveInt(inherited, key),
|
|
parsePositiveInt(value.trim() || inherited, key),
|
|
),
|
|
)
|
|
: value;
|
|
}
|
|
}
|
|
}
|
|
return childEnv;
|
|
}
|
|
|
|
function prepareChildEnv(entry: ShardPlan, baseEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
|
return mergePlanEnv(
|
|
{
|
|
...baseEnv,
|
|
OPENCLAW_TEST_PROJECTS_PARALLEL: "1",
|
|
...(entry.kind === "group" && entry.plan.shard_name
|
|
? { OPENCLAW_VITEST_SHARD_NAME: entry.plan.shard_name }
|
|
: {}),
|
|
},
|
|
entry.kind === "group" ? entry.plan.env : undefined,
|
|
);
|
|
}
|
|
|
|
export function buildChildEnv(
|
|
entry: ShardPlan,
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
scratchDir: string,
|
|
index: number,
|
|
options: { serial?: boolean; cacheSlot?: number; runtime?: "node" | "bun" } = {},
|
|
) {
|
|
const configuredCacheRoot = baseEnv[FS_MODULE_CACHE_ROOT_ENV_KEY]?.trim();
|
|
const persistentCacheRoot = configuredCacheRoot || baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim();
|
|
const cachePrefix = options.runtime === "bun" ? "vitest-cache-bun" : "vitest-cache";
|
|
const cacheDirectory = persistentCacheRoot
|
|
? `${cachePrefix}-${options.cacheSlot ?? index}`
|
|
: options.serial
|
|
? `${cachePrefix}-shared`
|
|
: `${cachePrefix}-${index}`;
|
|
// Persistent worker slots let serial plans reuse transforms without concurrent
|
|
// writers. Scratch caches stay per-plan; group overrides still apply last.
|
|
const cacheEnv: NodeJS.ProcessEnv = {
|
|
...baseEnv,
|
|
[FS_MODULE_CACHE_ROOT_ENV_KEY]: join(persistentCacheRoot || scratchDir, cacheDirectory),
|
|
};
|
|
// Legacy shard callers supplied the archive root through PATH. With ROOT,
|
|
// PATH instead belongs to a caller that explicitly selected a final leaf.
|
|
if (!configuredCacheRoot) {
|
|
delete cacheEnv[FS_MODULE_CACHE_PATH_ENV_KEY];
|
|
}
|
|
const childEnv = prepareChildEnv(entry, cacheEnv);
|
|
if (options.runtime) {
|
|
childEnv.OPENCLAW_VITEST_RUNTIME = options.runtime;
|
|
}
|
|
if (entry.kind === "group") {
|
|
const plan = entry.plan;
|
|
if (Array.isArray(plan.includePatterns) && plan.includePatterns.length > 0) {
|
|
const includeFile = join(scratchDir, `node-test-include-${index}.json`);
|
|
writeFileSync(includeFile, JSON.stringify(plan.includePatterns), "utf8");
|
|
childEnv.OPENCLAW_VITEST_INCLUDE_FILE = includeFile;
|
|
} else {
|
|
delete childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
|
|
}
|
|
}
|
|
return childEnv;
|
|
}
|
|
|
|
export function pruneFsModuleCache(root: string, maxBytes = FS_MODULE_CACHE_MAX_BYTES) {
|
|
if (!root || !existsSync(root) || !Number.isFinite(maxBytes) || maxBytes < 0) {
|
|
return { beforeBytes: 0, afterBytes: 0, removedFiles: 0 };
|
|
}
|
|
|
|
const files: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
|
|
let totalBytes = 0;
|
|
const visit = (directory: string) => {
|
|
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
const filePath = join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
visit(filePath);
|
|
continue;
|
|
}
|
|
if (!entry.isFile()) {
|
|
continue;
|
|
}
|
|
const fileStat = statSync(filePath);
|
|
totalBytes += fileStat.size;
|
|
if (
|
|
entry.name !== FS_MODULE_CACHE_METADATA_FILE &&
|
|
entry.name !== FS_MODULE_CACHE_GENERATION_FILE
|
|
) {
|
|
files.push({ filePath, mtimeMs: fileStat.mtimeMs, size: fileStat.size });
|
|
}
|
|
}
|
|
};
|
|
visit(root);
|
|
|
|
const beforeBytes = totalBytes;
|
|
if (totalBytes <= maxBytes) {
|
|
return { beforeBytes, afterBytes: totalBytes, removedFiles: 0 };
|
|
}
|
|
|
|
const targetBytes = Math.floor(maxBytes * FS_MODULE_CACHE_PRUNE_TARGET_RATIO);
|
|
files.sort((left, right) => left.mtimeMs - right.mtimeMs);
|
|
let removedFiles = 0;
|
|
for (const file of files) {
|
|
if (totalBytes <= targetBytes) {
|
|
break;
|
|
}
|
|
unlinkSync(file.filePath);
|
|
totalBytes -= file.size;
|
|
removedFiles += 1;
|
|
}
|
|
return { beforeBytes, afterBytes: totalBytes, removedFiles };
|
|
}
|
|
|
|
export function clonePersistentCacheSlots(root: string | undefined, concurrency: number) {
|
|
if (!root || concurrency <= 1) {
|
|
return 0;
|
|
}
|
|
let clonedSlots = 0;
|
|
for (const prefix of ["vitest-cache", "vitest-cache-bun"]) {
|
|
const seed = join(root, `${prefix}-0`);
|
|
if (!existsSync(seed)) {
|
|
continue;
|
|
}
|
|
for (let cacheSlot = 1; cacheSlot < concurrency; cacheSlot += 1) {
|
|
const destination = join(root, `${prefix}-${cacheSlot}`);
|
|
rmSync(destination, { force: true, recursive: true });
|
|
// Clone before workers start. Reflinks make the common Linux path cheap;
|
|
// unsupported filesystems transparently fall back to a regular copy.
|
|
cpSync(seed, destination, {
|
|
mode: constants.COPYFILE_FICLONE,
|
|
recursive: true,
|
|
});
|
|
clonedSlots += 1;
|
|
}
|
|
}
|
|
return clonedSlots;
|
|
}
|
|
|
|
const MAX_PENDING_LINE_CHARS = 1_000_000;
|
|
|
|
function relayChildStream(stream: Readable, label: string) {
|
|
const decoder = new StringDecoder("utf8");
|
|
let pending = "";
|
|
const writeLine = (line: string) => {
|
|
// Actions only parses workflow commands at the start of a line.
|
|
const workflowCommand = /^::(?:error|warning|notice|group|endgroup)(?: .*?)?::/u.test(line);
|
|
const output = workflowCommand ? line : `[shard:${label}] ${line}`;
|
|
if (!process.stdout.write(`${output}\n`)) {
|
|
stream.pause();
|
|
process.stdout.once("drain", () => stream.resume());
|
|
}
|
|
};
|
|
stream.on("data", (chunk: Buffer | string) => {
|
|
pending += decoder.write(chunk);
|
|
const lines = pending.split("\n");
|
|
pending = lines.pop() ?? "";
|
|
for (const line of lines) {
|
|
writeLine(line);
|
|
}
|
|
if (pending.length > MAX_PENDING_LINE_CHARS) {
|
|
writeLine(pending);
|
|
pending = "";
|
|
}
|
|
});
|
|
return () => {
|
|
pending += decoder.end();
|
|
if (pending !== "") {
|
|
writeLine(pending);
|
|
pending = "";
|
|
}
|
|
};
|
|
}
|
|
|
|
const TEST_PROJECTS_ENTRYPOINTS = ["scripts/test-projects.mts", "scripts/test-projects.mjs"];
|
|
|
|
export function resolveTestProjectsEntrypoint(
|
|
fileExists: (path: string) => boolean = existsSync,
|
|
): string {
|
|
const entrypoint = TEST_PROJECTS_ENTRYPOINTS.find((candidate) => fileExists(candidate));
|
|
if (!entrypoint) {
|
|
throw new Error("CI target does not provide scripts/test-projects.mts or .mjs");
|
|
}
|
|
return entrypoint;
|
|
}
|
|
|
|
export function resolveShardChildCommand(
|
|
args: string[],
|
|
nodeExecPath = process.execPath,
|
|
testProjectsEntrypoint = resolveTestProjectsEntrypoint(),
|
|
workerRun?: VitestWorkerRun,
|
|
) {
|
|
if (args[0] === "--native-bun") {
|
|
return {
|
|
command: nodeExecPath,
|
|
args: [
|
|
"--import",
|
|
"tsx",
|
|
fileURLToPath(new URL("./run-vitest.mts", import.meta.url)),
|
|
...args,
|
|
],
|
|
};
|
|
}
|
|
const loaderArgs = testProjectsEntrypoint.endsWith(".mts") ? ["--import", "tsx"] : [];
|
|
return {
|
|
command: nodeExecPath,
|
|
args: [
|
|
...loaderArgs,
|
|
...(workerRun
|
|
? [
|
|
fileURLToPath(new URL("./lib/vitest-worker-bootstrap.mts", import.meta.url)),
|
|
workerRun.descriptor.directory,
|
|
]
|
|
: []),
|
|
testProjectsEntrypoint,
|
|
...args,
|
|
],
|
|
};
|
|
}
|
|
|
|
async function loadWorkerOwner() {
|
|
const ownedRunner = join(process.cwd(), "scripts/ci-run-node-test-shard.mts");
|
|
// ci.yml's frozen-release adapter must execute the old target,
|
|
// never load a modern compiler from its workflow-owned .ci-workflow checkout.
|
|
if (
|
|
fileURLToPath(import.meta.url) ===
|
|
join(process.cwd(), ".ci-workflow/scripts/ci-run-node-test-shard.mts") &&
|
|
!existsSync(ownedRunner)
|
|
) {
|
|
return undefined;
|
|
}
|
|
if (fileURLToPath(import.meta.url) !== ownedRunner) {
|
|
throw new Error("Compiled CI worker ownership requires the target's own shard runner");
|
|
}
|
|
const groupOwner = await import("./vitest-process-group.mts");
|
|
// Windows' portable entry keeps per-group owners: a close-only receipt cannot
|
|
// authorize deleting a generation shared with another group's descendants.
|
|
if (!groupOwner.shouldUseDetachedVitestProcessGroup()) {
|
|
return undefined;
|
|
}
|
|
const { resolveSharedVitestCompilerEnv } = await import("./lib/vitest-process-env.mts");
|
|
const [worker, processOwner] = await Promise.all([
|
|
import("./lib/vitest-worker-run.mts"),
|
|
import("./lib/vitest-process.mts"),
|
|
]);
|
|
return {
|
|
createWorkerRun: worker.createVitestWorkerRun,
|
|
resolveCompilerEnv: resolveSharedVitestCompilerEnv,
|
|
spawn: processOwner.spawnOwnedVitestProcess,
|
|
exitBySignal: processOwner.exitVitestBySignal,
|
|
installCleanup: groupOwner.installVitestProcessGroupCleanup,
|
|
};
|
|
}
|
|
|
|
function createWorkerContext(
|
|
owner: NonNullable<Awaited<ReturnType<typeof loadWorkerOwner>>>,
|
|
env: NodeJS.ProcessEnv,
|
|
plans: ShardPlan[],
|
|
) {
|
|
const compilerEnv = owner.resolveCompilerEnv(
|
|
plans.length > 0 ? plans.map((plan) => prepareChildEnv(plan, env)) : [env],
|
|
);
|
|
return {
|
|
workerRun: owner.createWorkerRun(compilerEnv),
|
|
normalCompletion: true,
|
|
spawn: owner.spawn,
|
|
exitBySignal: owner.exitBySignal,
|
|
installCleanup: owner.installCleanup,
|
|
};
|
|
}
|
|
|
|
async function runChild(
|
|
args: string[],
|
|
childEnv: NodeJS.ProcessEnv,
|
|
label: string,
|
|
timingKey: string,
|
|
context?: Awaited<ReturnType<typeof createWorkerContext>>,
|
|
) {
|
|
const spawnEnv =
|
|
args[0] === "--native-bun" && context
|
|
? { ...childEnv, OPENCLAW_NATIVE_BUN_PARENT_IPC: "1" }
|
|
: childEnv;
|
|
// Use Node directly. `pnpm exec node` may reconcile the workspace before
|
|
// tests, which destroys the sticky dependency fast path.
|
|
const childCommand = resolveShardChildCommand(
|
|
args,
|
|
process.execPath,
|
|
undefined,
|
|
context?.workerRun,
|
|
);
|
|
let child: ChildProcess;
|
|
let completion: Promise<number>;
|
|
let teardown: (() => void) | undefined;
|
|
if (context) {
|
|
const owned = context.spawn({
|
|
command: childCommand.command,
|
|
args: childCommand.args,
|
|
options: {
|
|
env: spawnEnv,
|
|
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
|
},
|
|
homeMode: "tooling",
|
|
});
|
|
child = owned.child;
|
|
teardown = context.installCleanup({
|
|
child,
|
|
forceSignal: "SIGKILL",
|
|
forceSignalDelayMs: 100,
|
|
}).teardown;
|
|
// The containing TMP owner preserves nested claims if a group parent dies.
|
|
completion = context.workerRun.borrow(
|
|
child,
|
|
owned.completion.then((result) => {
|
|
if (!result.groupJoined) {
|
|
throw new Error("CI group descendant completion is unverified");
|
|
}
|
|
context.normalCompletion &&= typeof result.code === "number" && result.signal === null;
|
|
return result.code ?? 1;
|
|
}),
|
|
);
|
|
} else {
|
|
child = spawn(childCommand.command, childCommand.args, {
|
|
env: spawnEnv,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
completion = new Promise<number>((resolve) => {
|
|
child.once("close", (code) => resolve(code ?? 1));
|
|
child.once("error", (error) => {
|
|
process.stdout.write(`[shard:${label}] failed to spawn: ${error}\n`);
|
|
resolve(1);
|
|
});
|
|
});
|
|
}
|
|
// Stream with a per-line label instead of buffering: children can run
|
|
// whole suites for hours and verbose output must not accumulate on the
|
|
// wrapper heap. Backpressure pauses the child stream while stdout drains,
|
|
// and an oversized newline-free tail is force-flushed so the pending
|
|
// partial line stays bounded too.
|
|
const flushers = [child.stdout!, child.stderr!].map((stream) => relayChildStream(stream, label));
|
|
process.stdout.write(`[shard:${timingKey}] begin\n`);
|
|
let code: number;
|
|
try {
|
|
code = await completion;
|
|
} finally {
|
|
for (const flush of flushers) {
|
|
flush();
|
|
}
|
|
teardown?.();
|
|
}
|
|
process.stdout.write(`[shard:${timingKey}] end (exit ${code})\n`);
|
|
return code;
|
|
}
|
|
|
|
function readUiNativeShardReceipt(
|
|
file: string,
|
|
requestId: string,
|
|
expectedFiles: ReadonlySet<string>,
|
|
): Set<string> | undefined {
|
|
try {
|
|
const stat = lstatSync(file);
|
|
if (!stat.isFile() || stat.size > 1024 * 1024) {
|
|
return undefined;
|
|
}
|
|
const receipt: unknown = JSON.parse(readFileSync(file, "utf8"));
|
|
if (
|
|
!isRecord(receipt) ||
|
|
receipt.version !== 1 ||
|
|
receipt.requestId !== requestId ||
|
|
receipt.config !== join(process.cwd(), "ui/vitest.config.ts").replaceAll("\\", "/") ||
|
|
receipt.root !== join(process.cwd(), "ui").replaceAll("\\", "/") ||
|
|
!isStringArray(receipt.files) ||
|
|
receipt.files.length === 0 ||
|
|
receipt.files.some((candidateFile) => !expectedFiles.has(candidateFile))
|
|
) {
|
|
return undefined;
|
|
}
|
|
return new Set(receipt.files);
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
export async function runShardPlans(plans: ShardPlan[], options: RunShardOptions = {}) {
|
|
const inheritedEnv = options.env ?? process.env;
|
|
const jobEnv = mergePlanEnv({}, parseJsonEnv(inheritedEnv, "OPENCLAW_NODE_TEST_ENV_JSON"));
|
|
const baseEnv = mergePlanEnv(inheritedEnv, jobEnv);
|
|
if (
|
|
baseEnv.OPENCLAW_E2E_USE_PREBUILT_DIST === "1" &&
|
|
fileURLToPath(import.meta.url) === join(process.cwd(), "scripts/ci-run-node-test-shard.mts")
|
|
) {
|
|
const { preparePrebuiltAiPackage } = await import("./lib/vitest-build-prerequisites.mts");
|
|
for (const entry of plans) {
|
|
const selection = entry.kind === "target" ? { includePatterns: [entry.target] } : entry.plan;
|
|
const code = await preparePrebuiltAiPackage([selection], prepareChildEnv(entry, baseEnv));
|
|
if (code !== 0) {
|
|
return code;
|
|
}
|
|
}
|
|
}
|
|
// Historical targets use a workflow-owned adapter. Their Node
|
|
// contract must not import current target discovery or runtime policy code.
|
|
const runtimePolicy = baseEnv.OPENCLAW_CI_TEST_RUNTIME_POLICY?.trim() || "node";
|
|
const runtimeOwner =
|
|
runtimePolicy === "node" ? undefined : await import("./lib/ci-test-runtime.mts");
|
|
const policy = runtimeOwner?.resolveCiTestRuntimePolicy(baseEnv) ?? "node";
|
|
// Respect serial timing-sensitive bins and never clone cache slots that
|
|
// cannot receive a plan.
|
|
const requestedConcurrency =
|
|
options.concurrency === undefined
|
|
? readPositiveEnvInt(
|
|
"OPENCLAW_NODE_TEST_PLAN_CONCURRENCY",
|
|
baseEnv,
|
|
MAX_CI_VITEST_PLAN_CONCURRENCY,
|
|
)
|
|
: parsePositiveInt(options.concurrency, "Shard plan concurrency");
|
|
const ci = isCiLikeEnv(baseEnv);
|
|
const hostResources = ci
|
|
? { logicalCpuCount: os.availableParallelism(), totalMemoryBytes: os.totalmem() }
|
|
: null;
|
|
const exclusive = (entry: ShardPlan) =>
|
|
entry.kind === "group" && entry.plan.configs.some(isExclusiveCiTestConfig);
|
|
let concurrency = hostResources
|
|
? resolveCiVitestPlanConcurrency(plans.length, hostResources, requestedConcurrency)
|
|
: Math.min(plans.length, requestedConcurrency);
|
|
const measuredHost =
|
|
hostResources !== null &&
|
|
!isConstrainedCiCheckHost(hostResources) &&
|
|
baseEnv.RUNNER_ENVIRONMENT === "self-hosted" &&
|
|
baseEnv.FROZEN_TARGET !== "true"
|
|
? hostResources
|
|
: null;
|
|
const workerOwner = await loadWorkerOwner();
|
|
// A portable close receipt cannot release a shared lane, and a caller's
|
|
// final cache leaf cannot be split into scheduler-owned writer slots.
|
|
const callerCacheLeaf =
|
|
Boolean(
|
|
baseEnv[FS_MODULE_CACHE_ROOT_ENV_KEY]?.trim() &&
|
|
baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim(),
|
|
) ||
|
|
plans.some(
|
|
(entry) =>
|
|
entry.kind === "group" &&
|
|
typeof entry.plan.env?.[FS_MODULE_CACHE_PATH_ENV_KEY] === "string" &&
|
|
entry.plan.env[FS_MODULE_CACHE_PATH_ENV_KEY].trim(),
|
|
);
|
|
if ((!workerOwner && plans.some(exclusive)) || callerCacheLeaf) {
|
|
concurrency = Math.min(plans.length, 1);
|
|
}
|
|
// Final plan admission owns both compiler and child worker budgets.
|
|
let admittedPlans = plans.map((entry): ShardPlan => {
|
|
if (entry.kind !== "group" || entry.plan.fallbackMaxWorkers === undefined) {
|
|
return entry;
|
|
}
|
|
const fallback = parsePositiveInt(entry.plan.fallbackMaxWorkers, "Fallback worker limit");
|
|
const minTotalMemoryBytes =
|
|
entry.plan.minTotalMemoryBytes === undefined
|
|
? 0
|
|
: parsePositiveInt(entry.plan.minTotalMemoryBytes, "Worker memory floor");
|
|
if (
|
|
measuredHost &&
|
|
(concurrency === 1 || exclusive(entry)) &&
|
|
measuredHost.totalMemoryBytes >= minTotalMemoryBytes
|
|
) {
|
|
return entry;
|
|
}
|
|
return {
|
|
...entry,
|
|
plan: {
|
|
...entry.plan,
|
|
env: mergePlanEnv(mergePlanEnv({}, entry.plan.env), {
|
|
OPENCLAW_VITEST_MAX_WORKERS: String(fallback),
|
|
}),
|
|
},
|
|
};
|
|
});
|
|
const preparedRuntimeSelections = new Map<ShardPlan, CiTestRuntimeSelection[]>();
|
|
const overlapRequests = admittedPlans.filter(
|
|
(entry): entry is ShardGroupPlan =>
|
|
entry.kind === "group" &&
|
|
entry.name.startsWith("changed-extensions-config") &&
|
|
entry.plan.env?.OPENCLAW_TEST_PROJECTS_PARALLEL === "2",
|
|
);
|
|
if (overlapRequests.length > 0) {
|
|
const constrained = process.constrainedMemory?.() ?? 0;
|
|
const memory = hostResources
|
|
? Math.min(
|
|
hostResources.totalMemoryBytes,
|
|
Number.isFinite(constrained) && constrained > 0
|
|
? constrained
|
|
: hostResources.totalMemoryBytes,
|
|
)
|
|
: 0;
|
|
const hostAdmitted =
|
|
ci &&
|
|
process.platform === "linux" &&
|
|
baseEnv.RUNNER_ENVIRONMENT === "self-hosted" &&
|
|
baseEnv.FROZEN_TARGET === "false" &&
|
|
hostResources !== null &&
|
|
Number.isFinite(hostResources.logicalCpuCount) &&
|
|
hostResources.logicalCpuCount >= 2 &&
|
|
Number.isFinite(memory) &&
|
|
memory >= 7.5 * 1024 ** 3 &&
|
|
concurrency === 1 &&
|
|
Boolean(workerOwner) &&
|
|
!callerCacheLeaf;
|
|
const canOverlap = hostAdmitted
|
|
? (await import("./lib/extension-test-plan.mts")).canOverlapTelegramSingletonProcesses
|
|
: undefined;
|
|
admittedPlans = admittedPlans.map((entry) => {
|
|
if (entry.kind !== "group" || !overlapRequests.includes(entry)) {
|
|
return entry;
|
|
}
|
|
const env = prepareChildEnv(entry, baseEnv);
|
|
const shapeAdmitted =
|
|
canOverlap !== undefined &&
|
|
entry.plan.configs.length === 1 &&
|
|
!entry.plan.requiresDist &&
|
|
entry.plan.pretestBuildMode === undefined &&
|
|
env.OPENCLAW_VITEST_MAX_WORKERS === "2" &&
|
|
[inheritedEnv[VITEST_EXTRA_ARGS_ENV_KEY], env[VITEST_EXTRA_ARGS_ENV_KEY]].every(
|
|
(args) => !args?.trim() || args.trim() === "[]",
|
|
) &&
|
|
canOverlap(entry.plan.configs[0]!, entry.plan.includePatterns);
|
|
const selections: CiTestRuntimeSelection[] = shapeAdmitted
|
|
? (runtimeOwner?.resolveCiTestRuntimeSelections(
|
|
{ ...entry.plan, env, vitestArgs: [] },
|
|
policy,
|
|
) ?? [{ runtime: "node" }])
|
|
: [];
|
|
const [selected] = selections;
|
|
const admitted =
|
|
selections.length === 1 &&
|
|
selected?.runtime === "node" &&
|
|
selected.configs === undefined &&
|
|
selected.includePatterns === undefined &&
|
|
selected.includeAfterShard === undefined &&
|
|
selected.env === undefined;
|
|
const prepared: ShardGroupPlan = {
|
|
...entry,
|
|
plan: {
|
|
...entry.plan,
|
|
env: { ...entry.plan.env, OPENCLAW_TEST_PROJECTS_PARALLEL: admitted ? "2" : "1" },
|
|
},
|
|
};
|
|
if (admitted) {
|
|
preparedRuntimeSelections.set(prepared, selections);
|
|
}
|
|
return prepared;
|
|
});
|
|
}
|
|
const scratchDir = options.scratchDir ?? mkdtempSync(join(tmpdir(), "openclaw-node-shard-"));
|
|
const persistentCacheRoot =
|
|
baseEnv[FS_MODULE_CACHE_ROOT_ENV_KEY]?.trim() || baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim();
|
|
const nodeCompileCacheRoot = baseEnv[NODE_COMPILE_CACHE_PATH_ENV_KEY]?.trim();
|
|
let context: ReturnType<typeof createWorkerContext> | undefined;
|
|
let unverifiedChild = false;
|
|
let scratchCleanupPending = options.scratchDir === undefined;
|
|
let interrupted: NodeJS.Signals | undefined;
|
|
let exitCode = 0;
|
|
const completion = {
|
|
version: 1,
|
|
planned: admittedPlans.length,
|
|
completed: 0,
|
|
invocations: 0,
|
|
failedInvocations: 0,
|
|
};
|
|
const onSignal = (signal: NodeJS.Signals) => {
|
|
interrupted ??= signal;
|
|
};
|
|
try {
|
|
context = workerOwner ? createWorkerContext(workerOwner, baseEnv, admittedPlans) : undefined;
|
|
if (hostResources) {
|
|
console.log(
|
|
`[shard:resources] logicalCpuCount=${hostResources.logicalCpuCount} totalMemoryBytes=${hostResources.totalMemoryBytes} requested plans=${requestedConcurrency} admitted plans=${concurrency}`,
|
|
);
|
|
reportCiResourceSnapshot("start");
|
|
}
|
|
const clonedCacheSlots = clonePersistentCacheSlots(persistentCacheRoot, concurrency);
|
|
if (clonedCacheSlots > 0) {
|
|
process.stdout.write(
|
|
`[shard:cache] cloned restored Vitest seed into ${clonedCacheSlots} isolated lane(s)\n`,
|
|
);
|
|
}
|
|
if (context) {
|
|
process.on("SIGINT", onSignal);
|
|
process.on("SIGTERM", onSignal);
|
|
}
|
|
const runner: typeof runChild =
|
|
options.runChild ??
|
|
((args, childEnv, label, timingKey) => runChild(args, childEnv, label, timingKey, context));
|
|
await runVitestPlans(admittedPlans, {
|
|
concurrency,
|
|
isExclusive: exclusive,
|
|
shouldStop: () => Boolean(interrupted) || (exitCode !== 0 && !options.continueOnFailure),
|
|
run: async (entry, index, cacheSlot) => {
|
|
const targetArgs = entry.kind === "target" ? [entry.target] : entry.plan.configs;
|
|
if (!Array.isArray(targetArgs) || targetArgs.length === 0) {
|
|
console.error(`Missing node test shard configs for ${entry.name}`);
|
|
exitCode = exitCode || 1;
|
|
return;
|
|
}
|
|
// A standalone plan already projects the job environment. Resolve its
|
|
// scoped override once, then append it to the inherited global flags.
|
|
const vitestExtraArgs = [
|
|
inheritedEnv,
|
|
mergePlanEnv(jobEnv, entry.kind === "group" ? entry.plan.env : undefined),
|
|
].flatMap((env) => {
|
|
const value = parseJsonEnv(env, VITEST_EXTRA_ARGS_ENV_KEY, []);
|
|
return isStringArray(value) ? value : [];
|
|
});
|
|
const selections = preparedRuntimeSelections.get(entry) ??
|
|
runtimeOwner?.resolveCiTestRuntimeSelections(
|
|
{
|
|
...(entry.kind === "target" ? { targets: [entry.target] } : entry.plan),
|
|
env: prepareChildEnv(entry, baseEnv),
|
|
vitestArgs: vitestExtraArgs,
|
|
},
|
|
policy,
|
|
) ?? [{ runtime: "node" as const }];
|
|
const [nodeSelection, bunSelection] = selections;
|
|
// Only the proven UI partition has native sharding after discovery.
|
|
// Run it once on Bun and reuse that owner's facts, never its algorithm.
|
|
const uiReceipt =
|
|
context &&
|
|
policy === "bun-compatible" &&
|
|
entry.kind === "group" &&
|
|
entry.plan.configs.length === 1 &&
|
|
entry.plan.configs[0] === "ui/vitest.config.ts" &&
|
|
selections.length === 2 &&
|
|
nodeSelection?.runtime === "node" &&
|
|
nodeSelection.includeAfterShard &&
|
|
nodeSelection.includePatterns?.length &&
|
|
bunSelection?.runtime === "bun" &&
|
|
bunSelection.includeAfterShard &&
|
|
bunSelection.includePatterns?.length
|
|
? {
|
|
directory: mkdtempSync(join(scratchDir, "ui-native-shard-")),
|
|
requestId: randomUUID(),
|
|
expectedFiles: new Set([
|
|
...nodeSelection.includePatterns,
|
|
...bunSelection.includePatterns,
|
|
]),
|
|
selections: [bunSelection, nodeSelection],
|
|
}
|
|
: undefined;
|
|
let nativeShardFiles: Set<string> | undefined;
|
|
for (const selection of uiReceipt?.selections ?? selections) {
|
|
if (interrupted) {
|
|
return;
|
|
}
|
|
const runtime = selection.runtime;
|
|
const nativeBun = selection.engine === "bun-test";
|
|
const nativeFiles = nativeShardFiles;
|
|
if (
|
|
runtime === "node" &&
|
|
nativeFiles &&
|
|
selection.includePatterns &&
|
|
!selection.includePatterns.some((file) => nativeFiles.has(file))
|
|
) {
|
|
process.stdout.write(
|
|
`[shard:node-subset:${entry.name}] skipped (native shard has no Node-only files)\n`,
|
|
);
|
|
continue;
|
|
}
|
|
const selectedEntry =
|
|
entry.kind === "group" && (selection.configs || selection.includePatterns)
|
|
? {
|
|
...entry,
|
|
plan: {
|
|
...entry.plan,
|
|
configs: selection.configs ?? entry.plan.configs,
|
|
includePatterns: selection.includePatterns ?? entry.plan.includePatterns,
|
|
},
|
|
}
|
|
: entry;
|
|
const selectedArgs =
|
|
selectedEntry.kind === "target" ? [selectedEntry.target] : selectedEntry.plan.configs;
|
|
const args = nativeBun
|
|
? ["--native-bun", ...selection.files.map((file) => `./${file}`)]
|
|
: vitestExtraArgs.length > 0
|
|
? [...selectedArgs, "--", ...vitestExtraArgs]
|
|
: selectedArgs;
|
|
const childEnv = nativeBun
|
|
? prepareChildEnv(entry, { ...baseEnv, OPENCLAW_VITEST_RUNTIME: "bun" })
|
|
: buildChildEnv(selectedEntry, baseEnv, scratchDir, index, {
|
|
serial: concurrency === 1,
|
|
cacheSlot,
|
|
runtime,
|
|
});
|
|
if (nativeBun) {
|
|
delete childEnv[FS_MODULE_CACHE_ROOT_ENV_KEY];
|
|
delete childEnv[FS_MODULE_CACHE_PATH_ENV_KEY];
|
|
delete childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
|
|
}
|
|
if (selection.includeAfterShard) {
|
|
childEnv.OPENCLAW_VITEST_POST_SHARD_INCLUDE_FILE =
|
|
childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
|
|
const includePatterns = entry.kind === "group" ? entry.plan.includePatterns : null;
|
|
if (includePatterns?.length) {
|
|
// Tier selection precedes native sharding; runtime membership follows it.
|
|
const includeFile = join(scratchDir, `node-test-pre-shard-include-${index}.json`);
|
|
writeFileSync(includeFile, JSON.stringify(includePatterns), "utf8");
|
|
childEnv.OPENCLAW_VITEST_INCLUDE_FILE = includeFile;
|
|
} else {
|
|
delete childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
|
|
}
|
|
}
|
|
Object.assign(childEnv, selection.env);
|
|
delete childEnv.OPENCLAW_VITEST_NATIVE_SHARD_RECEIPT;
|
|
delete childEnv.OPENCLAW_VITEST_NATIVE_SHARD_REQUEST_ID;
|
|
if (uiReceipt && runtime === "bun") {
|
|
childEnv.OPENCLAW_VITEST_NATIVE_SHARD_RECEIPT = join(uiReceipt.directory, "files.json");
|
|
childEnv.OPENCLAW_VITEST_NATIVE_SHARD_REQUEST_ID = uiReceipt.requestId;
|
|
}
|
|
const timingKey = entry.kind === "group" ? (entry.timingKey ?? entry.name) : entry.name;
|
|
const timingPrefix = nativeBun
|
|
? "bun-native:"
|
|
: runtime === "bun"
|
|
? "bun:"
|
|
: selection.configs || selection.includePatterns
|
|
? "node-subset:"
|
|
: "";
|
|
unverifiedChild ||= !context;
|
|
completion.invocations++;
|
|
const code = await runner(
|
|
args,
|
|
childEnv,
|
|
`${timingPrefix}${entry.name}`,
|
|
`${timingPrefix}${timingKey}`,
|
|
);
|
|
if (uiReceipt && runtime === "bun") {
|
|
// runner() has joined the child and its descendants before these
|
|
// facts can suppress a process or their scratch directory retires.
|
|
if (code === 0 && !interrupted) {
|
|
nativeShardFiles = readUiNativeShardReceipt(
|
|
join(uiReceipt.directory, "files.json"),
|
|
uiReceipt.requestId,
|
|
uiReceipt.expectedFiles,
|
|
);
|
|
}
|
|
rmSync(uiReceipt.directory, { recursive: true, force: true });
|
|
}
|
|
// A dual-runtime envelope always completes both ordinary test runs;
|
|
// its first failure still stops admission of later envelopes.
|
|
if (code !== 0) {
|
|
completion.failedInvocations++;
|
|
exitCode = exitCode || code;
|
|
}
|
|
}
|
|
completion.completed++;
|
|
},
|
|
});
|
|
if (persistentCacheRoot && baseEnv[FS_MODULE_CACHE_WRITER_ENV_KEY] === "1") {
|
|
try {
|
|
const pruned = pruneFsModuleCache(
|
|
persistentCacheRoot,
|
|
options.fsModuleCacheMaxBytes ?? FS_MODULE_CACHE_MAX_BYTES,
|
|
);
|
|
process.stdout.write(
|
|
`[shard:cache] vitest ${pruned.beforeBytes} -> ${pruned.afterBytes} bytes; removed ${pruned.removedFiles} files\n`,
|
|
);
|
|
} catch (error) {
|
|
console.warn(`[shard:cache] failed to prune Vitest cache: ${String(error)}`);
|
|
}
|
|
}
|
|
if (nodeCompileCacheRoot && baseEnv[NODE_COMPILE_CACHE_WRITER_ENV_KEY] === "1") {
|
|
try {
|
|
const pruned = pruneFsModuleCache(
|
|
nodeCompileCacheRoot,
|
|
options.nodeCompileCacheMaxBytes ?? NODE_COMPILE_CACHE_MAX_BYTES,
|
|
);
|
|
process.stdout.write(
|
|
`[shard:cache] node-compile ${pruned.beforeBytes} -> ${pruned.afterBytes} bytes; removed ${pruned.removedFiles} files\n`,
|
|
);
|
|
} catch (error) {
|
|
console.warn(`[shard:cache] failed to prune Node compile cache: ${String(error)}`);
|
|
}
|
|
}
|
|
} finally {
|
|
try {
|
|
await context?.workerRun.dispose();
|
|
if (scratchCleanupPending && !unverifiedChild) {
|
|
// Disposal proves compiler, borrower, and nested-resource settlement.
|
|
// Portable close-only launches cannot authorize deleting shared scratch.
|
|
try {
|
|
await rm(scratchDir, { recursive: true, force: true, maxRetries: 3 });
|
|
scratchCleanupPending = false;
|
|
} catch {
|
|
// Report retained scratch below without replacing the shard's result.
|
|
}
|
|
}
|
|
} finally {
|
|
if (scratchCleanupPending) {
|
|
console.warn(
|
|
`[shard:cache] retained ${scratchDir}: descendant or scratch cleanup is unverified`,
|
|
);
|
|
}
|
|
if (hostResources) {
|
|
reportCiResourceSnapshot("end");
|
|
}
|
|
process.off("SIGINT", onSignal);
|
|
process.off("SIGTERM", onSignal);
|
|
if (interrupted && context) {
|
|
await context.exitBySignal(interrupted);
|
|
}
|
|
}
|
|
}
|
|
// Publish after disposal; custom or portable runners cannot certify child joins.
|
|
if (
|
|
completion.completed === completion.planned &&
|
|
context?.normalCompletion &&
|
|
!interrupted &&
|
|
!unverifiedChild &&
|
|
!options.runChild &&
|
|
!scratchCleanupPending
|
|
) {
|
|
process.stdout.write(`[shard:completion] ${JSON.stringify(completion)}\n`);
|
|
}
|
|
return exitCode;
|
|
}
|
|
|
|
if (isDirectRunUrl(process.argv[1], import.meta.url)) {
|
|
const plans = resolveShardPlans();
|
|
process.exitCode = await runShardPlans(plans, {
|
|
continueOnFailure: process.env.OPENCLAW_NODE_TEST_PLAN_CONTINUE_ON_FAILURE === "1",
|
|
});
|
|
}
|