mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
9c69495d6f
qaRuntime intentionally omits declarations, but private-QA CI marks its output prebuilt and skips the normal typed E2E setup. The packed AI tarball then contains no declarations and fails its external TypeScript compilation. Check the selected consumer's manifest type entries and run the existing typed AI build only when an entry is absent. Share this preparation between local Vitest admission and the CI shard owner, before concurrent readers start. Keep explicit skip-build, complete typed output, unrelated shards, and the sparse historical workflow adapter unchanged. Proof: cold qaRuntime reproduced TS7016 locally and on Node 24.19.0 Testbox; the repaired concurrent shard passes with all 17 type entries. Linux typed AI compilation took 4.7s; the unrelated shard passed in 2.88s with no typed build and no declarations. Median unrelated preparation over 1,000 local calls changed from 0.001ms to 0.009ms, with zero added build commands. Validation: check:changed passed; 162 focused tests and all 203 shard-runner tests passed. New regression file: 8 tests, 2.19s pnpm wall; complete modified shard suite: 6.98s pnpm wall. Independent review's historical-adapter concern was rejected after executing the sparse adapter with the prebuilt flag and without the current prerequisite module. Canonical full private-QA pnpm build passed (665.84s), followed by the ordinary non-prebuilt packed-package E2E consumer (58.34s wall, 16.49s test duration).
976 lines
36 KiB
TypeScript
976 lines
36 KiB
TypeScript
// Runs one CI node test shard job: either explicit changed-test targets or a
|
|
// list of packed group plans. Extracted from .github/workflows/ci.yml so the
|
|
// execution policy is unit-testable and plans can run concurrently.
|
|
import { spawn, type ChildProcess } from "node:child_process";
|
|
import { randomUUID } from "node:crypto";
|
|
import {
|
|
constants,
|
|
cpSync,
|
|
existsSync,
|
|
lstatSync,
|
|
mkdtempSync,
|
|
readFileSync,
|
|
readdirSync,
|
|
rmSync,
|
|
statSync,
|
|
unlinkSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { rm } from "node:fs/promises";
|
|
import os, { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import type { Readable } from "node:stream";
|
|
import { StringDecoder } from "node:string_decoder";
|
|
import { fileURLToPath } from "node:url";
|
|
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
|
import { decodeNodeTestGroups } from "./lib/ci-node-test-groups-codec.mts";
|
|
import type { CiTestRuntimeSelection } from "./lib/ci-test-runtime.mts";
|
|
import { isDirectRunUrl } from "./lib/direct-run.mjs";
|
|
import { isConstrainedCiCheckHost, isExclusiveCiTestConfig } from "./lib/local-check-runtime.mts";
|
|
import { parsePositiveInt, readPositiveEnvInt } from "./lib/numeric-options.mjs";
|
|
import { isCiLikeEnv } from "./lib/vitest-local-scheduling.mts";
|
|
import {
|
|
MAX_CI_VITEST_PLAN_CONCURRENCY,
|
|
resolveCiVitestPlanConcurrency,
|
|
runVitestPlans,
|
|
} from "./lib/vitest-plan-scheduling.mts";
|
|
import type { VitestWorkerRun } from "./lib/vitest-worker-run.mts";
|
|
|
|
// CI admits at most two plans only when the actual host has room. Each plan
|
|
// normally keeps inner parallelism 1; qualified singleton envelopes have a separate admission.
|
|
const FS_MODULE_CACHE_ROOT_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_ROOT";
|
|
const FS_MODULE_CACHE_PATH_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_PATH";
|
|
const FS_MODULE_CACHE_WRITER_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_WRITER";
|
|
const NODE_COMPILE_CACHE_PATH_ENV_KEY = "NODE_COMPILE_CACHE";
|
|
const NODE_COMPILE_CACHE_WRITER_ENV_KEY = "OPENCLAW_NODE_COMPILE_CACHE_WRITER";
|
|
const VITEST_EXTRA_ARGS_ENV_KEY = "OPENCLAW_NODE_TEST_VITEST_ARGS_JSON";
|
|
const FS_MODULE_CACHE_MAX_BYTES = 2 * 1024 * 1024 * 1024;
|
|
const NODE_COMPILE_CACHE_MAX_BYTES = 1024 * 1024 * 1024;
|
|
const FS_MODULE_CACHE_PRUNE_TARGET_RATIO = 0.75;
|
|
const FS_MODULE_CACHE_METADATA_FILE = "_metadata.json";
|
|
const FS_MODULE_CACHE_GENERATION_FILE = ".openclaw-transform-generation";
|
|
|
|
function reportCiResourceSnapshot(phase: "start" | "end") {
|
|
const pressure = (resource: "cpu" | "memory" | "io") => {
|
|
if (process.platform !== "linux") {
|
|
return null;
|
|
}
|
|
try {
|
|
return readFileSync(`/proc/pressure/${resource}`, "utf8").trim();
|
|
} catch {
|
|
return null;
|
|
}
|
|
};
|
|
console.log(
|
|
`[shard:resource-snapshot] ${JSON.stringify({
|
|
phase,
|
|
uptimeSeconds: os.uptime(),
|
|
cpuModel: os.cpus()[0]?.model ?? null,
|
|
loadAverage: os.loadavg(),
|
|
freeMemoryBytes: os.freemem(),
|
|
availableMemoryBytes: process.availableMemory?.() ?? null,
|
|
constrainedMemoryBytes: process.constrainedMemory?.() ?? null,
|
|
pressure: { cpu: pressure("cpu"), memory: pressure("memory"), io: pressure("io") },
|
|
})}`,
|
|
);
|
|
}
|
|
|
|
export type ShardTargetPlan = { kind: "target"; name: string; target: string };
|
|
type ShardGroupConfig = {
|
|
configs: string[];
|
|
requiresDist?: boolean;
|
|
pretestBuildMode?: "runtime" | "private-qa";
|
|
fallbackMaxWorkers?: number;
|
|
minTotalMemoryBytes?: number;
|
|
env?: Record<string, unknown> | null;
|
|
includePatterns?: string[] | null;
|
|
shard_name?: string;
|
|
timing_key?: string;
|
|
};
|
|
export type ShardGroupPlan = {
|
|
kind: "group";
|
|
name: string;
|
|
plan: ShardGroupConfig;
|
|
timingKey?: string;
|
|
};
|
|
export type ShardPlan = ShardTargetPlan | ShardGroupPlan;
|
|
type RunShardOptions = {
|
|
concurrency?: number;
|
|
continueOnFailure?: boolean;
|
|
env?: NodeJS.ProcessEnv;
|
|
fsModuleCacheMaxBytes?: number;
|
|
nodeCompileCacheMaxBytes?: number;
|
|
runChild?: typeof runChild;
|
|
scratchDir?: string;
|
|
};
|
|
|
|
function isStringArray(value: unknown): value is string[] {
|
|
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
}
|
|
|
|
function isShardGroupConfig(value: unknown): value is ShardGroupConfig {
|
|
return isRecord(value) && isStringArray(value.configs);
|
|
}
|
|
|
|
function parseJsonEnv(
|
|
env: Record<string, unknown>,
|
|
name: string,
|
|
fallback: unknown = null,
|
|
): unknown {
|
|
try {
|
|
const value = env[name];
|
|
return typeof value === "string" ? (JSON.parse(value) ?? fallback) : fallback;
|
|
} catch {
|
|
return fallback;
|
|
}
|
|
}
|
|
|
|
export function resolveShardPlans(env: NodeJS.ProcessEnv = process.env): ShardPlan[] {
|
|
const targets = parseJsonEnv(env, "OPENCLAW_NODE_TEST_TARGETS_JSON");
|
|
if (isStringArray(targets) && targets.length > 0) {
|
|
// One target per child process preserves the isolation boundaries encoded
|
|
// by full-suite include-pattern shards while keeping one runner job.
|
|
return targets.map((target) => ({ kind: "target", name: target, target }));
|
|
}
|
|
|
|
// The CI manifest packs matrix groups so preflight's job outputs stay under
|
|
// GitHub's 1 MiB UTF-16 cap. Plain JSON remains for the Vitest cache warmer,
|
|
// which writes its small group list straight into GITHUB_ENV.
|
|
const packedGroups = env.OPENCLAW_NODE_TEST_GROUPS_GZIP_BASE64?.trim();
|
|
const groups = packedGroups
|
|
? decodeNodeTestGroups(packedGroups)
|
|
: parseJsonEnv(env, "OPENCLAW_NODE_TEST_GROUPS_JSON");
|
|
const groupPlans = Array.isArray(groups) ? groups.filter(isShardGroupConfig) : [];
|
|
const configs = parseJsonEnv(env, "OPENCLAW_NODE_TEST_CONFIGS_JSON", []);
|
|
const groupEnv = parseJsonEnv(env, "OPENCLAW_NODE_TEST_ENV_JSON");
|
|
const includePatterns = parseJsonEnv(env, "OPENCLAW_NODE_TEST_INCLUDE_PATTERNS_JSON");
|
|
const plans: ShardGroupConfig[] =
|
|
groupPlans.length > 0
|
|
? groupPlans
|
|
: [
|
|
{
|
|
configs: isStringArray(configs) ? configs : [],
|
|
env: isRecord(groupEnv) ? groupEnv : null,
|
|
includePatterns: isStringArray(includePatterns) ? includePatterns : null,
|
|
shard_name: env.OPENCLAW_VITEST_SHARD_NAME,
|
|
},
|
|
];
|
|
return plans.map((plan) => {
|
|
const name = plan.shard_name ?? plan.configs?.[0] ?? "group";
|
|
return { kind: "group", name, plan, timingKey: plan.timing_key ?? name };
|
|
});
|
|
}
|
|
|
|
function mergePlanEnv(baseEnv: NodeJS.ProcessEnv, overrides: unknown): NodeJS.ProcessEnv {
|
|
const childEnv = { ...baseEnv };
|
|
if (isRecord(overrides)) {
|
|
for (const [key, value] of Object.entries(overrides)) {
|
|
if (typeof value === "string") {
|
|
const inherited = baseEnv[key]?.trim();
|
|
// Pins may lower the admitted job budget, never raise it. Compiler
|
|
// preparation and test children must inherit the same intersection.
|
|
childEnv[key] =
|
|
key === "OPENCLAW_VITEST_MAX_WORKERS" && inherited
|
|
? String(
|
|
Math.min(
|
|
parsePositiveInt(inherited, key),
|
|
parsePositiveInt(value.trim() || inherited, key),
|
|
),
|
|
)
|
|
: value;
|
|
}
|
|
}
|
|
}
|
|
return childEnv;
|
|
}
|
|
|
|
function prepareChildEnv(entry: ShardPlan, baseEnv: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
|
|
return mergePlanEnv(
|
|
{
|
|
...baseEnv,
|
|
OPENCLAW_TEST_PROJECTS_PARALLEL: "1",
|
|
...(entry.kind === "group" && entry.plan.shard_name
|
|
? { OPENCLAW_VITEST_SHARD_NAME: entry.plan.shard_name }
|
|
: {}),
|
|
},
|
|
entry.kind === "group" ? entry.plan.env : undefined,
|
|
);
|
|
}
|
|
|
|
export function buildChildEnv(
|
|
entry: ShardPlan,
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
scratchDir: string,
|
|
index: number,
|
|
options: { serial?: boolean; cacheSlot?: number; runtime?: "node" | "bun" } = {},
|
|
) {
|
|
const configuredCacheRoot = baseEnv[FS_MODULE_CACHE_ROOT_ENV_KEY]?.trim();
|
|
const persistentCacheRoot = configuredCacheRoot || baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim();
|
|
const cachePrefix = options.runtime === "bun" ? "vitest-cache-bun" : "vitest-cache";
|
|
const cacheDirectory = persistentCacheRoot
|
|
? `${cachePrefix}-${options.cacheSlot ?? index}`
|
|
: options.serial
|
|
? `${cachePrefix}-shared`
|
|
: `${cachePrefix}-${index}`;
|
|
// Persistent worker slots let serial plans reuse transforms without concurrent
|
|
// writers. Scratch caches stay per-plan; group overrides still apply last.
|
|
const cacheEnv: NodeJS.ProcessEnv = {
|
|
...baseEnv,
|
|
[FS_MODULE_CACHE_ROOT_ENV_KEY]: join(persistentCacheRoot || scratchDir, cacheDirectory),
|
|
};
|
|
// Legacy shard callers supplied the archive root through PATH. With ROOT,
|
|
// PATH instead belongs to a caller that explicitly selected a final leaf.
|
|
if (!configuredCacheRoot) {
|
|
delete cacheEnv[FS_MODULE_CACHE_PATH_ENV_KEY];
|
|
}
|
|
const childEnv = prepareChildEnv(entry, cacheEnv);
|
|
if (options.runtime) {
|
|
childEnv.OPENCLAW_VITEST_RUNTIME = options.runtime;
|
|
}
|
|
if (entry.kind === "group") {
|
|
const plan = entry.plan;
|
|
if (Array.isArray(plan.includePatterns) && plan.includePatterns.length > 0) {
|
|
const includeFile = join(scratchDir, `node-test-include-${index}.json`);
|
|
writeFileSync(includeFile, JSON.stringify(plan.includePatterns), "utf8");
|
|
childEnv.OPENCLAW_VITEST_INCLUDE_FILE = includeFile;
|
|
} else {
|
|
delete childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
|
|
}
|
|
}
|
|
return childEnv;
|
|
}
|
|
|
|
export function pruneFsModuleCache(root: string, maxBytes = FS_MODULE_CACHE_MAX_BYTES) {
|
|
if (!root || !existsSync(root) || !Number.isFinite(maxBytes) || maxBytes < 0) {
|
|
return { beforeBytes: 0, afterBytes: 0, removedFiles: 0 };
|
|
}
|
|
|
|
const files: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
|
|
let totalBytes = 0;
|
|
const visit = (directory: string) => {
|
|
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
const filePath = join(directory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
visit(filePath);
|
|
continue;
|
|
}
|
|
if (!entry.isFile()) {
|
|
continue;
|
|
}
|
|
const fileStat = statSync(filePath);
|
|
totalBytes += fileStat.size;
|
|
if (
|
|
entry.name !== FS_MODULE_CACHE_METADATA_FILE &&
|
|
entry.name !== FS_MODULE_CACHE_GENERATION_FILE
|
|
) {
|
|
files.push({ filePath, mtimeMs: fileStat.mtimeMs, size: fileStat.size });
|
|
}
|
|
}
|
|
};
|
|
visit(root);
|
|
|
|
const beforeBytes = totalBytes;
|
|
if (totalBytes <= maxBytes) {
|
|
return { beforeBytes, afterBytes: totalBytes, removedFiles: 0 };
|
|
}
|
|
|
|
const targetBytes = Math.floor(maxBytes * FS_MODULE_CACHE_PRUNE_TARGET_RATIO);
|
|
files.sort((left, right) => left.mtimeMs - right.mtimeMs);
|
|
let removedFiles = 0;
|
|
for (const file of files) {
|
|
if (totalBytes <= targetBytes) {
|
|
break;
|
|
}
|
|
unlinkSync(file.filePath);
|
|
totalBytes -= file.size;
|
|
removedFiles += 1;
|
|
}
|
|
return { beforeBytes, afterBytes: totalBytes, removedFiles };
|
|
}
|
|
|
|
export function clonePersistentCacheSlots(root: string | undefined, concurrency: number) {
|
|
if (!root || concurrency <= 1) {
|
|
return 0;
|
|
}
|
|
let clonedSlots = 0;
|
|
for (const prefix of ["vitest-cache", "vitest-cache-bun"]) {
|
|
const seed = join(root, `${prefix}-0`);
|
|
if (!existsSync(seed)) {
|
|
continue;
|
|
}
|
|
for (let cacheSlot = 1; cacheSlot < concurrency; cacheSlot += 1) {
|
|
const destination = join(root, `${prefix}-${cacheSlot}`);
|
|
rmSync(destination, { force: true, recursive: true });
|
|
// Clone before workers start. Reflinks make the common Linux path cheap;
|
|
// unsupported filesystems transparently fall back to a regular copy.
|
|
cpSync(seed, destination, {
|
|
mode: constants.COPYFILE_FICLONE,
|
|
recursive: true,
|
|
});
|
|
clonedSlots += 1;
|
|
}
|
|
}
|
|
return clonedSlots;
|
|
}
|
|
|
|
const MAX_PENDING_LINE_CHARS = 1_000_000;
|
|
|
|
function relayChildStream(stream: Readable, label: string) {
|
|
const decoder = new StringDecoder("utf8");
|
|
let pending = "";
|
|
const writeLine = (line: string) => {
|
|
// Actions only parses workflow commands at the start of a line.
|
|
const workflowCommand = /^::(?:error|warning|notice|group|endgroup)(?: .*?)?::/u.test(line);
|
|
const output = workflowCommand ? line : `[shard:${label}] ${line}`;
|
|
if (!process.stdout.write(`${output}\n`)) {
|
|
stream.pause();
|
|
process.stdout.once("drain", () => stream.resume());
|
|
}
|
|
};
|
|
stream.on("data", (chunk: Buffer | string) => {
|
|
pending += decoder.write(chunk);
|
|
const lines = pending.split("\n");
|
|
pending = lines.pop() ?? "";
|
|
for (const line of lines) {
|
|
writeLine(line);
|
|
}
|
|
if (pending.length > MAX_PENDING_LINE_CHARS) {
|
|
writeLine(pending);
|
|
pending = "";
|
|
}
|
|
});
|
|
return () => {
|
|
pending += decoder.end();
|
|
if (pending !== "") {
|
|
writeLine(pending);
|
|
pending = "";
|
|
}
|
|
};
|
|
}
|
|
|
|
const TEST_PROJECTS_ENTRYPOINTS = ["scripts/test-projects.mts", "scripts/test-projects.mjs"];
|
|
|
|
export function resolveTestProjectsEntrypoint(
|
|
fileExists: (path: string) => boolean = existsSync,
|
|
): string {
|
|
const entrypoint = TEST_PROJECTS_ENTRYPOINTS.find((candidate) => fileExists(candidate));
|
|
if (!entrypoint) {
|
|
throw new Error("CI target does not provide scripts/test-projects.mts or .mjs");
|
|
}
|
|
return entrypoint;
|
|
}
|
|
|
|
export function resolveShardChildCommand(
|
|
args: string[],
|
|
nodeExecPath = process.execPath,
|
|
testProjectsEntrypoint = resolveTestProjectsEntrypoint(),
|
|
workerRun?: VitestWorkerRun,
|
|
) {
|
|
const loaderArgs = testProjectsEntrypoint.endsWith(".mts") ? ["--import", "tsx"] : [];
|
|
return {
|
|
command: nodeExecPath,
|
|
args: [
|
|
...loaderArgs,
|
|
...(workerRun
|
|
? [
|
|
fileURLToPath(new URL("./lib/vitest-worker-bootstrap.mts", import.meta.url)),
|
|
workerRun.descriptor.directory,
|
|
]
|
|
: []),
|
|
testProjectsEntrypoint,
|
|
...args,
|
|
],
|
|
};
|
|
}
|
|
|
|
async function loadWorkerOwner() {
|
|
const ownedRunner = join(process.cwd(), "scripts/ci-run-node-test-shard.mts");
|
|
// ci.yml's frozen-release adapter must execute the old target,
|
|
// never load a modern compiler from its workflow-owned .ci-workflow checkout.
|
|
if (
|
|
fileURLToPath(import.meta.url) ===
|
|
join(process.cwd(), ".ci-workflow/scripts/ci-run-node-test-shard.mts") &&
|
|
!existsSync(ownedRunner)
|
|
) {
|
|
return undefined;
|
|
}
|
|
if (fileURLToPath(import.meta.url) !== ownedRunner) {
|
|
throw new Error("Compiled CI worker ownership requires the target's own shard runner");
|
|
}
|
|
const groupOwner = await import("./vitest-process-group.mts");
|
|
// Windows' portable entry keeps per-group owners: a close-only receipt cannot
|
|
// authorize deleting a generation shared with another group's descendants.
|
|
if (!groupOwner.shouldUseDetachedVitestProcessGroup()) {
|
|
return undefined;
|
|
}
|
|
const { resolveSharedVitestCompilerEnv } = await import("./lib/vitest-process-env.mts");
|
|
const [worker, processOwner] = await Promise.all([
|
|
import("./lib/vitest-worker-run.mts"),
|
|
import("./lib/vitest-process.mts"),
|
|
]);
|
|
return {
|
|
createWorkerRun: worker.createVitestWorkerRun,
|
|
resolveCompilerEnv: resolveSharedVitestCompilerEnv,
|
|
spawn: processOwner.spawnOwnedVitestProcess,
|
|
exitBySignal: processOwner.exitVitestBySignal,
|
|
installCleanup: groupOwner.installVitestProcessGroupCleanup,
|
|
};
|
|
}
|
|
|
|
function createWorkerContext(
|
|
owner: NonNullable<Awaited<ReturnType<typeof loadWorkerOwner>>>,
|
|
env: NodeJS.ProcessEnv,
|
|
plans: ShardPlan[],
|
|
) {
|
|
const compilerEnv = owner.resolveCompilerEnv(
|
|
plans.length > 0 ? plans.map((plan) => prepareChildEnv(plan, env)) : [env],
|
|
);
|
|
return {
|
|
workerRun: owner.createWorkerRun(compilerEnv),
|
|
normalCompletion: true,
|
|
spawn: owner.spawn,
|
|
exitBySignal: owner.exitBySignal,
|
|
installCleanup: owner.installCleanup,
|
|
};
|
|
}
|
|
|
|
async function runChild(
|
|
args: string[],
|
|
childEnv: NodeJS.ProcessEnv,
|
|
label: string,
|
|
timingKey: string,
|
|
context?: Awaited<ReturnType<typeof createWorkerContext>>,
|
|
) {
|
|
// Use Node directly. `pnpm exec node` may reconcile the workspace before
|
|
// tests, which destroys the sticky dependency fast path.
|
|
const childCommand = resolveShardChildCommand(
|
|
args,
|
|
process.execPath,
|
|
undefined,
|
|
context?.workerRun,
|
|
);
|
|
let child: ChildProcess;
|
|
let completion: Promise<number>;
|
|
let teardown: (() => void) | undefined;
|
|
if (context) {
|
|
const owned = context.spawn({
|
|
command: childCommand.command,
|
|
args: childCommand.args,
|
|
options: {
|
|
env: childEnv,
|
|
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
|
},
|
|
homeMode: "tooling",
|
|
});
|
|
child = owned.child;
|
|
teardown = context.installCleanup({
|
|
child,
|
|
forceSignal: "SIGKILL",
|
|
forceSignalDelayMs: 100,
|
|
}).teardown;
|
|
// The containing TMP owner preserves nested claims if a group parent dies.
|
|
completion = context.workerRun.borrow(
|
|
child,
|
|
owned.completion.then((result) => {
|
|
if (!result.groupJoined) {
|
|
throw new Error("CI group descendant completion is unverified");
|
|
}
|
|
context.normalCompletion &&= typeof result.code === "number" && result.signal === null;
|
|
return result.code ?? 1;
|
|
}),
|
|
);
|
|
} else {
|
|
child = spawn(childCommand.command, childCommand.args, {
|
|
env: childEnv,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
completion = new Promise<number>((resolve) => {
|
|
child.once("close", (code) => resolve(code ?? 1));
|
|
child.once("error", (error) => {
|
|
process.stdout.write(`[shard:${label}] failed to spawn: ${error}\n`);
|
|
resolve(1);
|
|
});
|
|
});
|
|
}
|
|
// Stream with a per-line label instead of buffering: children can run
|
|
// whole suites for hours and verbose output must not accumulate on the
|
|
// wrapper heap. Backpressure pauses the child stream while stdout drains,
|
|
// and an oversized newline-free tail is force-flushed so the pending
|
|
// partial line stays bounded too.
|
|
const flushers = [child.stdout!, child.stderr!].map((stream) => relayChildStream(stream, label));
|
|
process.stdout.write(`[shard:${timingKey}] begin\n`);
|
|
let code: number;
|
|
try {
|
|
code = await completion;
|
|
} finally {
|
|
for (const flush of flushers) {
|
|
flush();
|
|
}
|
|
teardown?.();
|
|
}
|
|
process.stdout.write(`[shard:${timingKey}] end (exit ${code})\n`);
|
|
return code;
|
|
}
|
|
|
|
function readUiNativeShardReceipt(
|
|
file: string,
|
|
requestId: string,
|
|
expectedFiles: ReadonlySet<string>,
|
|
): Set<string> | undefined {
|
|
try {
|
|
const stat = lstatSync(file);
|
|
if (!stat.isFile() || stat.size > 1024 * 1024) {
|
|
return undefined;
|
|
}
|
|
const receipt: unknown = JSON.parse(readFileSync(file, "utf8"));
|
|
if (
|
|
!isRecord(receipt) ||
|
|
receipt.version !== 1 ||
|
|
receipt.requestId !== requestId ||
|
|
receipt.config !== join(process.cwd(), "ui/vitest.config.ts").replaceAll("\\", "/") ||
|
|
receipt.root !== join(process.cwd(), "ui").replaceAll("\\", "/") ||
|
|
!isStringArray(receipt.files) ||
|
|
receipt.files.length === 0 ||
|
|
receipt.files.some((candidateFile) => !expectedFiles.has(candidateFile))
|
|
) {
|
|
return undefined;
|
|
}
|
|
return new Set(receipt.files);
|
|
} catch {
|
|
return undefined;
|
|
}
|
|
}
|
|
|
|
export async function runShardPlans(plans: ShardPlan[], options: RunShardOptions = {}) {
|
|
const inheritedEnv = options.env ?? process.env;
|
|
const jobEnv = mergePlanEnv({}, parseJsonEnv(inheritedEnv, "OPENCLAW_NODE_TEST_ENV_JSON"));
|
|
const baseEnv = mergePlanEnv(inheritedEnv, jobEnv);
|
|
if (
|
|
baseEnv.OPENCLAW_E2E_USE_PREBUILT_DIST === "1" &&
|
|
fileURLToPath(import.meta.url) === join(process.cwd(), "scripts/ci-run-node-test-shard.mts")
|
|
) {
|
|
const { preparePrebuiltAiPackage } = await import("./lib/vitest-build-prerequisites.mts");
|
|
for (const entry of plans) {
|
|
const selection = entry.kind === "target" ? { includePatterns: [entry.target] } : entry.plan;
|
|
const code = await preparePrebuiltAiPackage([selection], prepareChildEnv(entry, baseEnv));
|
|
if (code !== 0) {
|
|
return code;
|
|
}
|
|
}
|
|
}
|
|
// Historical targets use a workflow-owned adapter. Their Node
|
|
// contract must not import current target discovery or runtime policy code.
|
|
const runtimePolicy = baseEnv.OPENCLAW_CI_TEST_RUNTIME_POLICY?.trim() || "node";
|
|
const runtimeOwner =
|
|
runtimePolicy === "node" ? undefined : await import("./lib/ci-test-runtime.mts");
|
|
const policy = runtimeOwner?.resolveCiTestRuntimePolicy(baseEnv) ?? "node";
|
|
// Respect serial timing-sensitive bins and never clone cache slots that
|
|
// cannot receive a plan.
|
|
const requestedConcurrency =
|
|
options.concurrency === undefined
|
|
? readPositiveEnvInt(
|
|
"OPENCLAW_NODE_TEST_PLAN_CONCURRENCY",
|
|
baseEnv,
|
|
MAX_CI_VITEST_PLAN_CONCURRENCY,
|
|
)
|
|
: parsePositiveInt(options.concurrency, "Shard plan concurrency");
|
|
const ci = isCiLikeEnv(baseEnv);
|
|
const hostResources = ci
|
|
? { logicalCpuCount: os.availableParallelism(), totalMemoryBytes: os.totalmem() }
|
|
: null;
|
|
const exclusive = (entry: ShardPlan) =>
|
|
entry.kind === "group" && entry.plan.configs.some(isExclusiveCiTestConfig);
|
|
let concurrency = hostResources
|
|
? resolveCiVitestPlanConcurrency(plans.length, hostResources, requestedConcurrency)
|
|
: Math.min(plans.length, requestedConcurrency);
|
|
const measuredHost =
|
|
hostResources !== null &&
|
|
!isConstrainedCiCheckHost(hostResources) &&
|
|
baseEnv.RUNNER_ENVIRONMENT === "self-hosted" &&
|
|
baseEnv.FROZEN_TARGET !== "true"
|
|
? hostResources
|
|
: null;
|
|
const workerOwner = await loadWorkerOwner();
|
|
// A portable close receipt cannot release a shared lane, and a caller's
|
|
// final cache leaf cannot be split into scheduler-owned writer slots.
|
|
const callerCacheLeaf =
|
|
Boolean(
|
|
baseEnv[FS_MODULE_CACHE_ROOT_ENV_KEY]?.trim() &&
|
|
baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim(),
|
|
) ||
|
|
plans.some(
|
|
(entry) =>
|
|
entry.kind === "group" &&
|
|
typeof entry.plan.env?.[FS_MODULE_CACHE_PATH_ENV_KEY] === "string" &&
|
|
entry.plan.env[FS_MODULE_CACHE_PATH_ENV_KEY].trim(),
|
|
);
|
|
if ((!workerOwner && plans.some(exclusive)) || callerCacheLeaf) {
|
|
concurrency = Math.min(plans.length, 1);
|
|
}
|
|
// Final plan admission owns both compiler and child worker budgets.
|
|
let admittedPlans = plans.map((entry): ShardPlan => {
|
|
if (entry.kind !== "group" || entry.plan.fallbackMaxWorkers === undefined) {
|
|
return entry;
|
|
}
|
|
const fallback = parsePositiveInt(entry.plan.fallbackMaxWorkers, "Fallback worker limit");
|
|
const minTotalMemoryBytes =
|
|
entry.plan.minTotalMemoryBytes === undefined
|
|
? 0
|
|
: parsePositiveInt(entry.plan.minTotalMemoryBytes, "Worker memory floor");
|
|
if (
|
|
measuredHost &&
|
|
(concurrency === 1 || exclusive(entry)) &&
|
|
measuredHost.totalMemoryBytes >= minTotalMemoryBytes
|
|
) {
|
|
return entry;
|
|
}
|
|
return {
|
|
...entry,
|
|
plan: {
|
|
...entry.plan,
|
|
env: mergePlanEnv(mergePlanEnv({}, entry.plan.env), {
|
|
OPENCLAW_VITEST_MAX_WORKERS: String(fallback),
|
|
}),
|
|
},
|
|
};
|
|
});
|
|
const preparedRuntimeSelections = new Map<ShardPlan, CiTestRuntimeSelection[]>();
|
|
const overlapRequests = admittedPlans.filter(
|
|
(entry): entry is ShardGroupPlan =>
|
|
entry.kind === "group" &&
|
|
entry.name.startsWith("changed-extensions-config") &&
|
|
entry.plan.env?.OPENCLAW_TEST_PROJECTS_PARALLEL === "2",
|
|
);
|
|
if (overlapRequests.length > 0) {
|
|
const constrained = process.constrainedMemory?.() ?? 0;
|
|
const memory = hostResources
|
|
? Math.min(
|
|
hostResources.totalMemoryBytes,
|
|
Number.isFinite(constrained) && constrained > 0
|
|
? constrained
|
|
: hostResources.totalMemoryBytes,
|
|
)
|
|
: 0;
|
|
const hostAdmitted =
|
|
ci &&
|
|
process.platform === "linux" &&
|
|
baseEnv.RUNNER_ENVIRONMENT === "self-hosted" &&
|
|
baseEnv.FROZEN_TARGET === "false" &&
|
|
hostResources !== null &&
|
|
Number.isFinite(hostResources.logicalCpuCount) &&
|
|
hostResources.logicalCpuCount >= 2 &&
|
|
Number.isFinite(memory) &&
|
|
memory >= 7.5 * 1024 ** 3 &&
|
|
concurrency === 1 &&
|
|
Boolean(workerOwner) &&
|
|
!callerCacheLeaf;
|
|
const canOverlap = hostAdmitted
|
|
? (await import("./lib/extension-test-plan.mts")).canOverlapTelegramSingletonProcesses
|
|
: undefined;
|
|
admittedPlans = admittedPlans.map((entry) => {
|
|
if (entry.kind !== "group" || !overlapRequests.includes(entry)) {
|
|
return entry;
|
|
}
|
|
const env = prepareChildEnv(entry, baseEnv);
|
|
const shapeAdmitted =
|
|
canOverlap !== undefined &&
|
|
entry.plan.configs.length === 1 &&
|
|
!entry.plan.requiresDist &&
|
|
entry.plan.pretestBuildMode === undefined &&
|
|
env.OPENCLAW_VITEST_MAX_WORKERS === "2" &&
|
|
[inheritedEnv[VITEST_EXTRA_ARGS_ENV_KEY], env[VITEST_EXTRA_ARGS_ENV_KEY]].every(
|
|
(args) => !args?.trim() || args.trim() === "[]",
|
|
) &&
|
|
canOverlap(entry.plan.configs[0]!, entry.plan.includePatterns);
|
|
const selections: CiTestRuntimeSelection[] = shapeAdmitted
|
|
? (runtimeOwner?.resolveCiTestRuntimeSelections(
|
|
{ ...entry.plan, env, vitestArgs: [] },
|
|
policy,
|
|
) ?? [{ runtime: "node" }])
|
|
: [];
|
|
const [selected] = selections;
|
|
const admitted =
|
|
selections.length === 1 &&
|
|
selected?.runtime === "node" &&
|
|
selected.configs === undefined &&
|
|
selected.includePatterns === undefined &&
|
|
selected.includeAfterShard === undefined &&
|
|
selected.env === undefined;
|
|
const prepared: ShardGroupPlan = {
|
|
...entry,
|
|
plan: {
|
|
...entry.plan,
|
|
env: { ...entry.plan.env, OPENCLAW_TEST_PROJECTS_PARALLEL: admitted ? "2" : "1" },
|
|
},
|
|
};
|
|
if (admitted) {
|
|
preparedRuntimeSelections.set(prepared, selections);
|
|
}
|
|
return prepared;
|
|
});
|
|
}
|
|
const scratchDir = options.scratchDir ?? mkdtempSync(join(tmpdir(), "openclaw-node-shard-"));
|
|
const persistentCacheRoot =
|
|
baseEnv[FS_MODULE_CACHE_ROOT_ENV_KEY]?.trim() || baseEnv[FS_MODULE_CACHE_PATH_ENV_KEY]?.trim();
|
|
const nodeCompileCacheRoot = baseEnv[NODE_COMPILE_CACHE_PATH_ENV_KEY]?.trim();
|
|
let context: ReturnType<typeof createWorkerContext> | undefined;
|
|
let unverifiedChild = false;
|
|
let scratchCleanupPending = options.scratchDir === undefined;
|
|
let interrupted: NodeJS.Signals | undefined;
|
|
let exitCode = 0;
|
|
const completion = {
|
|
version: 1,
|
|
planned: admittedPlans.length,
|
|
completed: 0,
|
|
invocations: 0,
|
|
failedInvocations: 0,
|
|
};
|
|
const onSignal = (signal: NodeJS.Signals) => {
|
|
interrupted ??= signal;
|
|
};
|
|
try {
|
|
context = workerOwner ? createWorkerContext(workerOwner, baseEnv, admittedPlans) : undefined;
|
|
if (hostResources) {
|
|
console.log(
|
|
`[shard:resources] logicalCpuCount=${hostResources.logicalCpuCount} totalMemoryBytes=${hostResources.totalMemoryBytes} requested plans=${requestedConcurrency} admitted plans=${concurrency}`,
|
|
);
|
|
reportCiResourceSnapshot("start");
|
|
}
|
|
const clonedCacheSlots = clonePersistentCacheSlots(persistentCacheRoot, concurrency);
|
|
if (clonedCacheSlots > 0) {
|
|
process.stdout.write(
|
|
`[shard:cache] cloned restored Vitest seed into ${clonedCacheSlots} isolated lane(s)\n`,
|
|
);
|
|
}
|
|
if (context) {
|
|
process.on("SIGINT", onSignal);
|
|
process.on("SIGTERM", onSignal);
|
|
}
|
|
const runner: typeof runChild =
|
|
options.runChild ??
|
|
((args, childEnv, label, timingKey) => runChild(args, childEnv, label, timingKey, context));
|
|
await runVitestPlans(admittedPlans, {
|
|
concurrency,
|
|
isExclusive: exclusive,
|
|
shouldStop: () => Boolean(interrupted) || (exitCode !== 0 && !options.continueOnFailure),
|
|
run: async (entry, index, cacheSlot) => {
|
|
const targetArgs = entry.kind === "target" ? [entry.target] : entry.plan.configs;
|
|
if (!Array.isArray(targetArgs) || targetArgs.length === 0) {
|
|
console.error(`Missing node test shard configs for ${entry.name}`);
|
|
exitCode = exitCode || 1;
|
|
return;
|
|
}
|
|
// A standalone plan already projects the job environment. Resolve its
|
|
// scoped override once, then append it to the inherited global flags.
|
|
const vitestExtraArgs = [
|
|
inheritedEnv,
|
|
mergePlanEnv(jobEnv, entry.kind === "group" ? entry.plan.env : undefined),
|
|
].flatMap((env) => {
|
|
const value = parseJsonEnv(env, VITEST_EXTRA_ARGS_ENV_KEY, []);
|
|
return isStringArray(value) ? value : [];
|
|
});
|
|
const selections = preparedRuntimeSelections.get(entry) ??
|
|
runtimeOwner?.resolveCiTestRuntimeSelections(
|
|
{
|
|
...(entry.kind === "target" ? { targets: [entry.target] } : entry.plan),
|
|
env: prepareChildEnv(entry, baseEnv),
|
|
vitestArgs: vitestExtraArgs,
|
|
},
|
|
policy,
|
|
) ?? [{ runtime: "node" as const }];
|
|
const [nodeSelection, bunSelection] = selections;
|
|
// Only the proven UI partition has native sharding after discovery.
|
|
// Run it once on Bun and reuse that owner's facts, never its algorithm.
|
|
const uiReceipt =
|
|
context &&
|
|
policy === "bun-compatible" &&
|
|
entry.kind === "group" &&
|
|
entry.plan.configs.length === 1 &&
|
|
entry.plan.configs[0] === "ui/vitest.config.ts" &&
|
|
selections.length === 2 &&
|
|
nodeSelection?.runtime === "node" &&
|
|
nodeSelection.includeAfterShard &&
|
|
nodeSelection.includePatterns?.length &&
|
|
bunSelection?.runtime === "bun" &&
|
|
bunSelection.includeAfterShard &&
|
|
bunSelection.includePatterns?.length
|
|
? {
|
|
directory: mkdtempSync(join(scratchDir, "ui-native-shard-")),
|
|
requestId: randomUUID(),
|
|
expectedFiles: new Set([
|
|
...nodeSelection.includePatterns,
|
|
...bunSelection.includePatterns,
|
|
]),
|
|
selections: [bunSelection, nodeSelection],
|
|
}
|
|
: undefined;
|
|
let nativeShardFiles: Set<string> | undefined;
|
|
for (const selection of uiReceipt?.selections ?? selections) {
|
|
if (interrupted) {
|
|
return;
|
|
}
|
|
const runtime = selection.runtime;
|
|
const nativeFiles = nativeShardFiles;
|
|
if (
|
|
runtime === "node" &&
|
|
nativeFiles &&
|
|
selection.includePatterns &&
|
|
!selection.includePatterns.some((file) => nativeFiles.has(file))
|
|
) {
|
|
process.stdout.write(
|
|
`[shard:node-subset:${entry.name}] skipped (native shard has no Node-only files)\n`,
|
|
);
|
|
continue;
|
|
}
|
|
const selectedEntry =
|
|
entry.kind === "group" && (selection.configs || selection.includePatterns)
|
|
? {
|
|
...entry,
|
|
plan: {
|
|
...entry.plan,
|
|
configs: selection.configs ?? entry.plan.configs,
|
|
includePatterns: selection.includePatterns ?? entry.plan.includePatterns,
|
|
},
|
|
}
|
|
: entry;
|
|
const selectedArgs =
|
|
selectedEntry.kind === "target" ? [selectedEntry.target] : selectedEntry.plan.configs;
|
|
const args =
|
|
vitestExtraArgs.length > 0 ? [...selectedArgs, "--", ...vitestExtraArgs] : selectedArgs;
|
|
const childEnv = buildChildEnv(selectedEntry, baseEnv, scratchDir, index, {
|
|
serial: concurrency === 1,
|
|
cacheSlot,
|
|
runtime,
|
|
});
|
|
if (selection.includeAfterShard) {
|
|
childEnv.OPENCLAW_VITEST_POST_SHARD_INCLUDE_FILE =
|
|
childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
|
|
const includePatterns = entry.kind === "group" ? entry.plan.includePatterns : null;
|
|
if (includePatterns?.length) {
|
|
// Tier selection precedes native sharding; runtime membership follows it.
|
|
const includeFile = join(scratchDir, `node-test-pre-shard-include-${index}.json`);
|
|
writeFileSync(includeFile, JSON.stringify(includePatterns), "utf8");
|
|
childEnv.OPENCLAW_VITEST_INCLUDE_FILE = includeFile;
|
|
} else {
|
|
delete childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
|
|
}
|
|
}
|
|
Object.assign(childEnv, selection.env);
|
|
delete childEnv.OPENCLAW_VITEST_NATIVE_SHARD_RECEIPT;
|
|
delete childEnv.OPENCLAW_VITEST_NATIVE_SHARD_REQUEST_ID;
|
|
if (uiReceipt && runtime === "bun") {
|
|
childEnv.OPENCLAW_VITEST_NATIVE_SHARD_RECEIPT = join(uiReceipt.directory, "files.json");
|
|
childEnv.OPENCLAW_VITEST_NATIVE_SHARD_REQUEST_ID = uiReceipt.requestId;
|
|
}
|
|
const timingKey = entry.kind === "group" ? (entry.timingKey ?? entry.name) : entry.name;
|
|
const timingPrefix =
|
|
runtime === "bun"
|
|
? "bun:"
|
|
: selection.configs || selection.includePatterns
|
|
? "node-subset:"
|
|
: "";
|
|
unverifiedChild ||= !context;
|
|
completion.invocations++;
|
|
const code = await runner(
|
|
args,
|
|
childEnv,
|
|
`${timingPrefix}${entry.name}`,
|
|
`${timingPrefix}${timingKey}`,
|
|
);
|
|
if (uiReceipt && runtime === "bun") {
|
|
// runner() has joined the child and its descendants before these
|
|
// facts can suppress a process or their scratch directory retires.
|
|
if (code === 0 && !interrupted) {
|
|
nativeShardFiles = readUiNativeShardReceipt(
|
|
join(uiReceipt.directory, "files.json"),
|
|
uiReceipt.requestId,
|
|
uiReceipt.expectedFiles,
|
|
);
|
|
}
|
|
rmSync(uiReceipt.directory, { recursive: true, force: true });
|
|
}
|
|
// A dual-runtime envelope always completes both ordinary test runs;
|
|
// its first failure still stops admission of later envelopes.
|
|
if (code !== 0) {
|
|
completion.failedInvocations++;
|
|
exitCode = exitCode || code;
|
|
}
|
|
}
|
|
completion.completed++;
|
|
},
|
|
});
|
|
if (persistentCacheRoot && baseEnv[FS_MODULE_CACHE_WRITER_ENV_KEY] === "1") {
|
|
try {
|
|
const pruned = pruneFsModuleCache(
|
|
persistentCacheRoot,
|
|
options.fsModuleCacheMaxBytes ?? FS_MODULE_CACHE_MAX_BYTES,
|
|
);
|
|
process.stdout.write(
|
|
`[shard:cache] vitest ${pruned.beforeBytes} -> ${pruned.afterBytes} bytes; removed ${pruned.removedFiles} files\n`,
|
|
);
|
|
} catch (error) {
|
|
console.warn(`[shard:cache] failed to prune Vitest cache: ${String(error)}`);
|
|
}
|
|
}
|
|
if (nodeCompileCacheRoot && baseEnv[NODE_COMPILE_CACHE_WRITER_ENV_KEY] === "1") {
|
|
try {
|
|
const pruned = pruneFsModuleCache(
|
|
nodeCompileCacheRoot,
|
|
options.nodeCompileCacheMaxBytes ?? NODE_COMPILE_CACHE_MAX_BYTES,
|
|
);
|
|
process.stdout.write(
|
|
`[shard:cache] node-compile ${pruned.beforeBytes} -> ${pruned.afterBytes} bytes; removed ${pruned.removedFiles} files\n`,
|
|
);
|
|
} catch (error) {
|
|
console.warn(`[shard:cache] failed to prune Node compile cache: ${String(error)}`);
|
|
}
|
|
}
|
|
} finally {
|
|
try {
|
|
await context?.workerRun.dispose();
|
|
if (scratchCleanupPending && !unverifiedChild) {
|
|
// Disposal proves compiler, borrower, and nested-resource settlement.
|
|
// Portable close-only launches cannot authorize deleting shared scratch.
|
|
try {
|
|
await rm(scratchDir, { recursive: true, force: true, maxRetries: 3 });
|
|
scratchCleanupPending = false;
|
|
} catch {
|
|
// Report retained scratch below without replacing the shard's result.
|
|
}
|
|
}
|
|
} finally {
|
|
if (scratchCleanupPending) {
|
|
console.warn(
|
|
`[shard:cache] retained ${scratchDir}: descendant or scratch cleanup is unverified`,
|
|
);
|
|
}
|
|
if (hostResources) {
|
|
reportCiResourceSnapshot("end");
|
|
}
|
|
process.off("SIGINT", onSignal);
|
|
process.off("SIGTERM", onSignal);
|
|
if (interrupted && context) {
|
|
await context.exitBySignal(interrupted);
|
|
}
|
|
}
|
|
}
|
|
// Publish after disposal; custom or portable runners cannot certify child joins.
|
|
if (
|
|
completion.completed === completion.planned &&
|
|
context?.normalCompletion &&
|
|
!interrupted &&
|
|
!unverifiedChild &&
|
|
!options.runChild &&
|
|
!scratchCleanupPending
|
|
) {
|
|
process.stdout.write(`[shard:completion] ${JSON.stringify(completion)}\n`);
|
|
}
|
|
return exitCode;
|
|
}
|
|
|
|
if (isDirectRunUrl(process.argv[1], import.meta.url)) {
|
|
const plans = resolveShardPlans();
|
|
process.exitCode = await runShardPlans(plans, {
|
|
continueOnFailure: process.env.OPENCLAW_NODE_TEST_PLAN_CONTINUE_ON_FAILURE === "1",
|
|
});
|
|
}
|