mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-11 18:08:03 +00:00
4ee008a026
Punchcard-Session: amber-workshop-workshop-36 Co-authored-by: Dallin Romney <6581799+RomneyDa@users.noreply.github.com>
2099 lines
70 KiB
TypeScript
2099 lines
70 KiB
TypeScript
// Docker E2E aggregate scheduler.
|
|
// Builds shared Docker images, prepares one OpenClaw npm tarball, assigns lanes
|
|
// to bare/functional images, and runs lanes through weighted resource pools.
|
|
|
|
import assert from "node:assert/strict";
|
|
import { execFileSync, spawn, type ChildProcess } from "node:child_process";
|
|
import fs from "node:fs";
|
|
import { mkdir, open, readFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { isRecord } from "@openclaw/normalization-core/record-coerce";
|
|
import {
|
|
DEFAULT_E2E_BARE_IMAGE,
|
|
DEFAULT_E2E_FUNCTIONAL_IMAGE,
|
|
DEFAULT_LIVE_RETRIES,
|
|
DEFAULT_PARALLELISM,
|
|
DEFAULT_PROFILE,
|
|
DEFAULT_RESOURCE_LIMITS,
|
|
DEFAULT_TAIL_PARALLELISM,
|
|
RELEASE_PATH_PROFILE,
|
|
findLaneByName,
|
|
laneResources,
|
|
laneSummary,
|
|
laneWeight,
|
|
lanesNeedE2eImageKind,
|
|
lanesNeedOpenClawPackage,
|
|
normalizeReleaseProfile,
|
|
parseLaneSelection,
|
|
parseLiveMode,
|
|
parseProfile,
|
|
resolveDockerE2ePlan,
|
|
} from "./lib/docker-e2e-plan.mts";
|
|
import type { DockerE2eLane } from "./lib/docker-e2e-scenarios.mts";
|
|
import { sleep } from "./lib/sleep.mjs";
|
|
import {
|
|
createPrepublishPluginRegistryArtifact,
|
|
inspectNpmPackageTarball,
|
|
validatePrepublishPluginRegistryArtifact,
|
|
} from "./prepublish-plugin-registry-artifact.mjs";
|
|
|
|
const SCRIPT_ROOT_DIR = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const ROOT_DIR = path.resolve(process.env.OPENCLAW_DOCKER_E2E_REPO_ROOT || SCRIPT_ROOT_DIR);
|
|
const LIVE_DOCKER_DEFAULT_HARNESS_DIR =
|
|
path.basename(SCRIPT_ROOT_DIR) === ".release-harness" && ROOT_DIR !== SCRIPT_ROOT_DIR
|
|
? ".release-harness"
|
|
: ".";
|
|
const DEFAULT_FAILURE_TAIL_LINES = 80;
|
|
const DEFAULT_LANE_TIMEOUT_MS = 120 * 60 * 1000;
|
|
const DEFAULT_LANE_START_STAGGER_MS = 2_000;
|
|
const DEFAULT_STATUS_INTERVAL_MS = 30_000;
|
|
const DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS = 60_000;
|
|
const CLEANUP_SMOKE_NAME = "cleanup-smoke";
|
|
export const SHELL_CAPTURE_MAX_CHARS = 1024 * 1024;
|
|
export const LOG_TAIL_MAX_BYTES = 1024 * 1024;
|
|
const SHELL_TIMEOUT_KILL_GRACE_MS = 10_000;
|
|
const SHELL_POST_FORCE_KILL_WAIT_MS = 1_000;
|
|
const SHELL_PROCESS_GROUP_EXIT_POLL_MS = 25;
|
|
const MAX_TIMER_TIMEOUT_MS = 2_147_000_000;
|
|
const DEFAULT_TIMINGS_FILE = path.join(ROOT_DIR, ".artifacts/docker-tests/lane-timings.json");
|
|
const DEFAULT_GITHUB_WORKFLOW = "openclaw-live-and-e2e-checks-reusable.yml";
|
|
const CANDIDATE_ENV_KEYS =
|
|
"OPENCLAW_DOCKER_E2E_SELECTED_SHA OPENCLAW_CURRENT_PACKAGE_TGZ OPENCLAW_CURRENT_PACKAGE_VERSION OPENCLAW_CURRENT_PACKAGE_SHA256".split(
|
|
" ",
|
|
);
|
|
const REGISTRY_ENV_KEYS =
|
|
"OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_CANDIDATE_VERSION OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256".split(
|
|
" ",
|
|
);
|
|
|
|
type SchedulerLimits = ReturnType<typeof parseSchedulerOptions>;
|
|
type DockerCandidatePlan = ReturnType<typeof resolveDockerE2ePlan>["plan"];
|
|
|
|
type SchedulerActiveState = {
|
|
count: number;
|
|
resources: Map<string, number>;
|
|
weight: number;
|
|
};
|
|
|
|
type SchedulerLane = Pick<DockerE2eLane, "name"> &
|
|
Partial<Pick<DockerE2eLane, "resources" | "weight">>;
|
|
|
|
type TimingStore = Awaited<ReturnType<typeof loadTimingStore>>;
|
|
|
|
type ShellCommandResult = Omit<ReturnType<typeof shellCommandSkippedForShutdown>, "signal"> & {
|
|
signal: NodeJS.Signals | null;
|
|
};
|
|
type ShellCaptureResult = Omit<ReturnType<typeof shellCaptureSkippedForShutdown>, "signal"> & {
|
|
signal: NodeJS.Signals | null;
|
|
};
|
|
|
|
type ShellCommandOptions = {
|
|
command: string;
|
|
env: NodeJS.ProcessEnv;
|
|
label: string;
|
|
logFile?: string;
|
|
noOutputTimeoutMs?: number;
|
|
timeoutKillGraceMs?: number;
|
|
timeoutMs?: number;
|
|
};
|
|
|
|
type ShellCaptureOptions = Omit<ShellCommandOptions, "logFile" | "noOutputTimeoutMs"> &
|
|
Required<Pick<ShellCommandOptions, "timeoutMs">>;
|
|
|
|
type LaneResult = Omit<Awaited<ReturnType<typeof runLane>>, "imageKind"> & {
|
|
imageKind?: DockerE2eLane["e2eImageKind"];
|
|
targetable?: boolean;
|
|
};
|
|
|
|
type RunLaneSummary = Pick<LaneResult, "name" | "status"> & Partial<LaneResult>;
|
|
|
|
type RunSummary = Record<string, unknown> & {
|
|
failures?: RunLaneSummary[];
|
|
github?: ReturnType<typeof githubRunSummary>;
|
|
images?: { bare?: string; functional?: string };
|
|
lanes?: RunLaneSummary[];
|
|
status?: string;
|
|
};
|
|
|
|
type LanePoolOptions = SchedulerLimits & {
|
|
failFast: boolean;
|
|
poolLabel: string;
|
|
startStaggerMs: number;
|
|
statusIntervalMs: number;
|
|
timeoutMs: number;
|
|
};
|
|
|
|
type ForegroundEntry = {
|
|
command: string;
|
|
env?: NodeJS.ProcessEnv;
|
|
label: string;
|
|
phaseDetails?: Record<string, unknown>;
|
|
phases?: Array<Record<string, unknown>>;
|
|
};
|
|
|
|
type ShutdownSignal = "SIGINT" | "SIGKILL" | "SIGTERM";
|
|
|
|
const IS_MAIN = (() => {
|
|
const entrypoint = process.argv[1];
|
|
if (!entrypoint) {
|
|
return false;
|
|
}
|
|
try {
|
|
// Node resolves ESM URLs through symlinks, but argv keeps the invoked path.
|
|
return fs.realpathSync(entrypoint) === fs.realpathSync(fileURLToPath(import.meta.url));
|
|
} catch {
|
|
return false;
|
|
}
|
|
})();
|
|
|
|
function dockerAllUsage() {
|
|
return [
|
|
"Usage: node scripts/test-docker-all.mjs [--plan-json | --prepare-only=<manifest>]",
|
|
"",
|
|
"Options:",
|
|
" --plan-json Print the resolved Docker E2E plan as JSON and exit.",
|
|
" --prepare-only Prepare one immutable candidate manifest and exit.",
|
|
" -h, --help Show this help.",
|
|
"",
|
|
"Lane selection and scheduler settings are configured with OPENCLAW_DOCKER_ALL_* env vars.",
|
|
].join("\n");
|
|
}
|
|
|
|
export function parseDockerAllCliArgs(argv: readonly string[]) {
|
|
const options: { help: boolean; planJson: boolean; prepareOnly?: string } = {
|
|
help: false,
|
|
planJson: false,
|
|
};
|
|
for (const arg of argv) {
|
|
if (arg === "--plan-json") {
|
|
options.planJson = true;
|
|
} else if (arg.startsWith("--prepare-only=")) {
|
|
options.prepareOnly = arg.slice("--prepare-only=".length);
|
|
if (!options.prepareOnly) {
|
|
throw new Error(`--prepare-only requires a manifest path\n\n${dockerAllUsage()}`);
|
|
}
|
|
} else if (arg === "--help" || arg === "-h") {
|
|
options.help = true;
|
|
} else {
|
|
throw new Error(`unknown argument: ${arg}\n\n${dockerAllUsage()}`);
|
|
}
|
|
}
|
|
assert(!(options.planJson && options.prepareOnly), "conflicting plan/prep options");
|
|
return options;
|
|
}
|
|
|
|
let cliOptions: ReturnType<typeof parseDockerAllCliArgs> = {
|
|
help: false,
|
|
planJson: false,
|
|
};
|
|
if (IS_MAIN) {
|
|
try {
|
|
cliOptions = parseDockerAllCliArgs(process.argv.slice(2));
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
}
|
|
if (cliOptions.help) {
|
|
console.log(dockerAllUsage());
|
|
process.exit(0);
|
|
}
|
|
}
|
|
|
|
function parsePositiveInt(raw: string | undefined, fallback: number, label: string) {
|
|
if (raw === undefined || raw === "") {
|
|
return fallback;
|
|
}
|
|
const text = raw.trim();
|
|
if (!/^\d+$/u.test(text)) {
|
|
throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`);
|
|
}
|
|
const parsed = Number(text);
|
|
if (!Number.isSafeInteger(parsed) || parsed < 1) {
|
|
throw new Error(`${label} must be a positive integer. Got: ${JSON.stringify(raw)}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseNonNegativeInt(raw: string | undefined, fallback: number, label: string) {
|
|
if (raw === undefined || raw === "") {
|
|
return fallback;
|
|
}
|
|
const text = raw.trim();
|
|
if (!/^\d+$/u.test(text)) {
|
|
throw new Error(`${label} must be a non-negative integer. Got: ${JSON.stringify(raw)}`);
|
|
}
|
|
const parsed = Number(text);
|
|
if (!Number.isSafeInteger(parsed) || parsed < 0) {
|
|
throw new Error(`${label} must be a non-negative integer. Got: ${JSON.stringify(raw)}`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
function parseBool(raw: string | undefined, fallback: boolean) {
|
|
if (raw === undefined || raw === "") {
|
|
return fallback;
|
|
}
|
|
return !/^(?:0|false|no)$/i.test(raw);
|
|
}
|
|
|
|
function normalizeReleaseProfileEnv(raw: string | undefined) {
|
|
const profile = raw?.trim();
|
|
if (!profile) {
|
|
return normalizeReleaseProfile(undefined);
|
|
}
|
|
if (profile === "minimum" || profile === "beta" || profile === "stable" || profile === "full") {
|
|
return normalizeReleaseProfile(profile);
|
|
}
|
|
throw new Error(
|
|
`release profile must be one of: beta, stable, full. Got: ${JSON.stringify(raw)}`,
|
|
);
|
|
}
|
|
|
|
function numericTimerValueMs(valueMs: unknown) {
|
|
const value = Number(valueMs);
|
|
return Number.isFinite(value) ? Math.floor(value) : undefined;
|
|
}
|
|
|
|
function resolveTimerTimeoutMs(valueMs: unknown, fallbackMs: unknown = MAX_TIMER_TIMEOUT_MS) {
|
|
const value = numericTimerValueMs(valueMs) ?? numericTimerValueMs(fallbackMs);
|
|
return Math.min(Math.max(value ?? MAX_TIMER_TIMEOUT_MS, 1), MAX_TIMER_TIMEOUT_MS);
|
|
}
|
|
|
|
function resolveOptionalTimerTimeoutMs(valueMs: unknown) {
|
|
const value = numericTimerValueMs(valueMs);
|
|
if (value === undefined || value <= 0) {
|
|
return undefined;
|
|
}
|
|
return resolveTimerTimeoutMs(value);
|
|
}
|
|
|
|
function resourceLimitsSummary(resourceLimits: Record<string, number>) {
|
|
return Object.entries(resourceLimits)
|
|
.map(([resource, limit]) => `${resource}=${String(limit)}`)
|
|
.join(" ");
|
|
}
|
|
|
|
export function describeDockerSchedulerLimits(parallelism: number, options: SchedulerLimits) {
|
|
return `parallelism=${parallelism} weightLimit=${options.weightLimit} resources=${resourceLimitsSummary(
|
|
options.resourceLimits,
|
|
)}`;
|
|
}
|
|
|
|
function parseSchedulerOptions(env: NodeJS.ProcessEnv, parallelism: number) {
|
|
const weightLimit = parsePositiveInt(
|
|
env.OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT,
|
|
parallelism,
|
|
"OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT",
|
|
);
|
|
const resourceLimits: Record<string, number> = {};
|
|
for (const [resource, fallback] of Object.entries(DEFAULT_RESOURCE_LIMITS)) {
|
|
const envName = `OPENCLAW_DOCKER_ALL_${resource.toUpperCase().replace(/[^A-Z0-9]+/g, "_")}_LIMIT`;
|
|
resourceLimits[resource] = parsePositiveInt(
|
|
env[envName],
|
|
Math.min(parallelism, fallback),
|
|
envName,
|
|
);
|
|
}
|
|
return {
|
|
resourceLimits,
|
|
weightLimit,
|
|
};
|
|
}
|
|
|
|
export function canStartSchedulerLane(
|
|
candidate: SchedulerLane,
|
|
active: SchedulerActiveState,
|
|
parallelism: number,
|
|
options: SchedulerLimits,
|
|
) {
|
|
const weight = Math.max(1, candidate.weight ?? 1);
|
|
if (active.count >= parallelism) {
|
|
return false;
|
|
}
|
|
|
|
const exceedsWeightLimit = active.weight + weight > options.weightLimit;
|
|
const resources = [...new Set(["docker", ...(candidate.resources ?? [])])];
|
|
const exceedsResourceLimit = resources.some((resource) => {
|
|
const limit = options.resourceLimits[resource] ?? options.weightLimit;
|
|
const current = active.resources.get(resource) ?? 0;
|
|
return current + weight > limit;
|
|
});
|
|
|
|
if (!exceedsWeightLimit && !exceedsResourceLimit) {
|
|
return true;
|
|
}
|
|
|
|
return active.count === 0;
|
|
}
|
|
|
|
function timingSeconds(timings: Record<string, unknown>, poolLane: DockerE2eLane) {
|
|
const timing = timings[poolLane.name];
|
|
const fromStore = isRecord(timing) ? timing.durationSeconds : undefined;
|
|
if (typeof fromStore === "number" && Number.isFinite(fromStore) && fromStore > 0) {
|
|
return fromStore;
|
|
}
|
|
return poolLane.estimateSeconds ?? 0;
|
|
}
|
|
|
|
function orderLanes(poolLanes: DockerE2eLane[], timingStore?: unknown) {
|
|
const timings = isRecord(timingStore) && isRecord(timingStore.lanes) ? timingStore.lanes : {};
|
|
return poolLanes
|
|
.map((poolLane, index) => ({
|
|
index,
|
|
poolLane,
|
|
seconds: timingSeconds(timings, poolLane),
|
|
}))
|
|
.toSorted((a, b) => b.seconds - a.seconds || a.index - b.index)
|
|
.map(({ poolLane }) => poolLane);
|
|
}
|
|
|
|
function utcStampForPath() {
|
|
return new Date().toISOString().replaceAll("-", "").replaceAll(":", "").replace(/\..*$/, "Z");
|
|
}
|
|
|
|
function utcStamp() {
|
|
return new Date().toISOString().replace(/\..*$/, "Z");
|
|
}
|
|
|
|
function appendExtension(env: NodeJS.ProcessEnv, extension: string) {
|
|
const current = env.OPENCLAW_DOCKER_BUILD_EXTENSIONS ?? env.OPENCLAW_EXTENSIONS ?? "";
|
|
const tokens = current.split(/\s+/).filter(Boolean);
|
|
if (!tokens.includes(extension)) {
|
|
tokens.push(extension);
|
|
}
|
|
env.OPENCLAW_DOCKER_BUILD_EXTENSIONS = tokens.join(" ");
|
|
}
|
|
|
|
function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
|
|
const env = {
|
|
...process.env,
|
|
...extra,
|
|
};
|
|
const pathEntries = [
|
|
env.PATH,
|
|
env.PNPM_HOME,
|
|
env.npm_execpath ? path.dirname(env.npm_execpath) : undefined,
|
|
path.dirname(process.execPath),
|
|
]
|
|
.flatMap((entry) => (entry ? entry.split(path.delimiter) : []))
|
|
.filter(Boolean);
|
|
env.PATH = [...new Set(pathEntries)].join(path.delimiter);
|
|
return env;
|
|
}
|
|
|
|
function gitOutput(repoRoot: string, args: string[]) {
|
|
return execFileSync("git", args, { cwd: repoRoot, encoding: "utf8" }).trim();
|
|
}
|
|
|
|
function rootPackageVersion(repoRoot: string) {
|
|
return JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")).version as string;
|
|
}
|
|
|
|
function readCompleteTuple(env: NodeJS.ProcessEnv, keys: readonly string[], label: string) {
|
|
const entries = keys.flatMap((key) => (env[key] ? [[key, env[key]]] : []));
|
|
const complete = entries.length === keys.length;
|
|
assert(!entries.length || complete, `${label} fields must be complete`);
|
|
return complete ? Object.fromEntries(entries) : undefined;
|
|
}
|
|
|
|
function validateRegistryEnvironment(baseEnv: NodeJS.ProcessEnv, plan: DockerCandidatePlan) {
|
|
validatePrepublishPluginRegistryArtifact({
|
|
artifactDir: baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR!,
|
|
expectedCandidateVersion: baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_CANDIDATE_VERSION!,
|
|
expectedManifestSha256: baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256!,
|
|
expectedSourceSha: baseEnv.OPENCLAW_DOCKER_E2E_SELECTED_SHA!,
|
|
requiredPackages: plan.requiredPrepublishPluginPackages,
|
|
});
|
|
}
|
|
|
|
export function validateDockerCandidateEnvironment(
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
plan: DockerCandidatePlan,
|
|
repoRoot = ROOT_DIR,
|
|
) {
|
|
const strictCandidate = CANDIDATE_ENV_KEYS.slice(2).some((key) => baseEnv[key]);
|
|
if (!strictCandidate || !plan.needs.package) {
|
|
baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ &&= path.resolve(baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ);
|
|
const registryDir = baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR;
|
|
if (!plan.needs.prepublishPluginRegistry || !registryDir) {
|
|
delete baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR;
|
|
return;
|
|
}
|
|
baseEnv.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR = path.resolve(registryDir);
|
|
validateRegistryEnvironment(baseEnv, plan);
|
|
return;
|
|
}
|
|
const candidate = readCompleteTuple(baseEnv, CANDIDATE_ENV_KEYS, "Docker candidate")!;
|
|
const registry = readCompleteTuple(baseEnv, REGISTRY_ENV_KEYS, "Docker candidate registry");
|
|
const packagePath = candidate.OPENCLAW_CURRENT_PACKAGE_TGZ;
|
|
if (
|
|
!path.isAbsolute(packagePath) ||
|
|
gitOutput(repoRoot, ["rev-parse", "HEAD"]) !== candidate.OPENCLAW_DOCKER_E2E_SELECTED_SHA
|
|
) {
|
|
throw new Error("Docker candidate path must be absolute and selected SHA must equal HEAD");
|
|
}
|
|
const packed = inspectNpmPackageTarball(packagePath);
|
|
if (
|
|
packed.packageJson.name !== "openclaw" ||
|
|
packed.packageJson.version !== rootPackageVersion(repoRoot) ||
|
|
packed.packageJson.version !== candidate.OPENCLAW_CURRENT_PACKAGE_VERSION ||
|
|
packed.sha256 !== candidate.OPENCLAW_CURRENT_PACKAGE_SHA256
|
|
) {
|
|
throw new Error("Docker candidate package identity differs from the immutable tuple");
|
|
}
|
|
if (registry) {
|
|
const registryDir = registry.OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR;
|
|
assert(path.isAbsolute(registryDir), "Docker candidate registry path must be absolute");
|
|
validateRegistryEnvironment(baseEnv, plan);
|
|
} else if (plan.needs.prepublishPluginRegistry) {
|
|
throw new Error("Docker plan requires a prepublish plugin registry tuple");
|
|
}
|
|
}
|
|
|
|
function shellQuote(value: string) {
|
|
return `'${value.replaceAll("'", "'\\''")}'`;
|
|
}
|
|
|
|
function maybeGhcrImage(value: unknown) {
|
|
return typeof value === "string" && value.startsWith("ghcr.io/") ? value : "";
|
|
}
|
|
|
|
export function githubWorkflowRerunCommand(
|
|
laneNames: readonly string[],
|
|
ref: string,
|
|
env: NodeJS.ProcessEnv = process.env,
|
|
) {
|
|
const workflowRef = env.OPENCLAW_DOCKER_E2E_WORKFLOW_REF || undefined;
|
|
const releasePath = env.OPENCLAW_DOCKER_ALL_PROFILE === RELEASE_PATH_PROFILE;
|
|
const allowUnreleasedChangelog = env.OPENCLAW_DOCKER_E2E_ALLOW_UNRELEASED_CHANGELOG === "true";
|
|
const bareImage = maybeGhcrImage(env.OPENCLAW_DOCKER_E2E_BARE_IMAGE);
|
|
const functionalImage = maybeGhcrImage(env.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE);
|
|
const fields = [
|
|
"gh workflow run",
|
|
shellQuote(env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW),
|
|
...(workflowRef ? ["--ref", shellQuote(workflowRef)] : []),
|
|
"-f",
|
|
`ref=${shellQuote(ref)}`,
|
|
"-f",
|
|
"include_repo_e2e=false",
|
|
"-f",
|
|
`include_release_path_suites=${releasePath ? "true" : "false"}`,
|
|
"-f",
|
|
"include_openwebui=false",
|
|
"-f",
|
|
`docker_lanes=${shellQuote(laneNames.join(" "))}`,
|
|
"-f",
|
|
"include_live_suites=false",
|
|
"-f",
|
|
"live_models_only=false",
|
|
];
|
|
if (allowUnreleasedChangelog) {
|
|
fields.push("-f", "allow_unreleased_changelog=true");
|
|
}
|
|
if (env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC) {
|
|
fields.push(
|
|
"-f",
|
|
`published_upgrade_survivor_baseline=${shellQuote(env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC)}`,
|
|
);
|
|
}
|
|
if (env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS) {
|
|
fields.push(
|
|
"-f",
|
|
`published_upgrade_survivor_baselines=${shellQuote(env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS)}`,
|
|
);
|
|
}
|
|
if (env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS) {
|
|
fields.push(
|
|
"-f",
|
|
`published_upgrade_survivor_scenarios=${shellQuote(env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS)}`,
|
|
);
|
|
}
|
|
if (bareImage) {
|
|
fields.push("-f", `docker_e2e_bare_image=${shellQuote(bareImage)}`);
|
|
}
|
|
if (functionalImage) {
|
|
fields.push("-f", `docker_e2e_functional_image=${shellQuote(functionalImage)}`);
|
|
}
|
|
if (bareImage || functionalImage) {
|
|
fields.push("-f", "shared_image_policy=existing-only");
|
|
}
|
|
return fields.join(" ");
|
|
}
|
|
|
|
export function buildLaneRerunCommand(name: string, baseEnv: NodeJS.ProcessEnv) {
|
|
const poolLane = findLaneByName(name);
|
|
const build = name.startsWith("live-") ? "1" : "0";
|
|
const image = poolLane ? e2eImageForLane(poolLane, baseEnv) : baseEnv.OPENCLAW_DOCKER_E2E_IMAGE;
|
|
const env: Array<readonly [string, string | undefined]> = [
|
|
["OPENCLAW_DOCKER_ALL_LANES", name],
|
|
["OPENCLAW_DOCKER_ALL_BUILD", build],
|
|
["OPENCLAW_DOCKER_ALL_PREFLIGHT", "0"],
|
|
["OPENCLAW_SKIP_DOCKER_BUILD", "1"],
|
|
["OPENCLAW_DOCKER_E2E_IMAGE", image || DEFAULT_E2E_FUNCTIONAL_IMAGE],
|
|
["OPENCLAW_DOCKER_E2E_BARE_IMAGE", baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE],
|
|
["OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE", baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE],
|
|
...CANDIDATE_ENV_KEYS.map((key) => [key, baseEnv[key]] as const),
|
|
...REGISTRY_ENV_KEYS.map((key) => [key, baseEnv[key]] as const),
|
|
["OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC", baseEnv.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC],
|
|
["OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS", baseEnv.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS],
|
|
["OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS", baseEnv.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS],
|
|
];
|
|
if (baseEnv.OPENCLAW_DOCKER_ALL_PNPM_COMMAND) {
|
|
env.push(["OPENCLAW_DOCKER_ALL_PNPM_COMMAND", baseEnv.OPENCLAW_DOCKER_ALL_PNPM_COMMAND]);
|
|
}
|
|
return `${env
|
|
.filter(
|
|
(entry): entry is readonly [string, string] => entry[1] !== undefined && entry[1] !== "",
|
|
)
|
|
.map(([key, value]) => `${key}=${shellQuote(value)}`)
|
|
.join(" ")} pnpm test:docker:all`;
|
|
}
|
|
|
|
function liveDockerHarnessScriptCommand(script: string) {
|
|
return `bash -c 'harness="\${OPENCLAW_DOCKER_E2E_TRUSTED_HARNESS_DIR:-${LIVE_DOCKER_DEFAULT_HARNESS_DIR}}"; OPENCLAW_LIVE_DOCKER_REPO_ROOT="\${OPENCLAW_DOCKER_E2E_REPO_ROOT:-$PWD}" bash "$harness/scripts/${script}"'`;
|
|
}
|
|
|
|
async function loadTimingStore(file: string, enabled: boolean) {
|
|
if (!enabled) {
|
|
return { enabled: false, file, lanes: {}, version: 1 };
|
|
}
|
|
const raw = await readFile(file, "utf8").catch(() => "");
|
|
if (!raw.trim()) {
|
|
return { enabled: true, file, lanes: {}, version: 1 };
|
|
}
|
|
try {
|
|
const parsed: unknown = JSON.parse(raw);
|
|
return {
|
|
enabled: true,
|
|
file,
|
|
lanes: isRecord(parsed) && isRecord(parsed.lanes) ? parsed.lanes : {},
|
|
version: 1,
|
|
};
|
|
} catch (error) {
|
|
console.warn(
|
|
`WARN: ignoring unreadable Docker lane timings ${file}: ${
|
|
error instanceof Error ? error.message : String(error)
|
|
}`,
|
|
);
|
|
return { enabled: true, file, lanes: {}, version: 1 };
|
|
}
|
|
}
|
|
|
|
async function writeTimingStore(timingStore: TimingStore, results: LaneResult[]) {
|
|
if (!timingStore.enabled || results.length === 0) {
|
|
return;
|
|
}
|
|
const next = {
|
|
lanes: { ...timingStore.lanes },
|
|
updatedAt: new Date().toISOString(),
|
|
version: 1,
|
|
};
|
|
for (const result of results) {
|
|
if (!result || typeof result.elapsedSeconds !== "number") {
|
|
continue;
|
|
}
|
|
next.lanes[result.name] = {
|
|
durationSeconds: result.elapsedSeconds,
|
|
status: result.status,
|
|
timedOut: result.timedOut,
|
|
updatedAt: new Date().toISOString(),
|
|
};
|
|
}
|
|
await mkdir(path.dirname(timingStore.file), { recursive: true });
|
|
await fs.promises.writeFile(timingStore.file, `${JSON.stringify(next, null, 2)}\n`);
|
|
timingStore.lanes = next.lanes;
|
|
console.log(`==> Docker lane timings: ${timingStore.file}`);
|
|
}
|
|
|
|
function githubRunSummary(env: NodeJS.ProcessEnv) {
|
|
return {
|
|
ref: env.GITHUB_REF_NAME || undefined,
|
|
repository: env.GITHUB_REPOSITORY || undefined,
|
|
runId: env.GITHUB_RUN_ID || undefined,
|
|
runUrl:
|
|
env.GITHUB_SERVER_URL && env.GITHUB_REPOSITORY && env.GITHUB_RUN_ID
|
|
? `${env.GITHUB_SERVER_URL}/${env.GITHUB_REPOSITORY}/actions/runs/${env.GITHUB_RUN_ID}`
|
|
: undefined,
|
|
selectedSha: env.OPENCLAW_DOCKER_E2E_SELECTED_SHA || undefined,
|
|
sha: env.GITHUB_SHA || undefined,
|
|
workflow: env.GITHUB_WORKFLOW || undefined,
|
|
};
|
|
}
|
|
|
|
export async function writeRunSummary(
|
|
logDir: string,
|
|
summary: RunSummary,
|
|
env: NodeJS.ProcessEnv = process.env,
|
|
) {
|
|
const file = path.join(logDir, "summary.json");
|
|
const payload = {
|
|
...summary,
|
|
// Summary reruns do not carry failure-index commands, so preserve this exact package intent.
|
|
allowUnreleasedChangelog:
|
|
env.OPENCLAW_DOCKER_E2E_ALLOW_UNRELEASED_CHANGELOG === "true" ? true : undefined,
|
|
packageArtifactName: env.OPENCLAW_DOCKER_E2E_PACKAGE_ARTIFACT_NAME || undefined,
|
|
finishedAt: new Date().toISOString(),
|
|
github: githubRunSummary(env),
|
|
version: 1,
|
|
};
|
|
await fs.promises.writeFile(file, `${JSON.stringify(payload, null, 2)}\n`);
|
|
await writeFailureIndex(logDir, payload, env);
|
|
console.log(`==> Docker run summary: ${file}`);
|
|
}
|
|
|
|
async function writeFailureIndex(logDir: string, summary: RunSummary, env: NodeJS.ProcessEnv) {
|
|
const ref =
|
|
summary.github?.selectedSha ||
|
|
env.OPENCLAW_DOCKER_E2E_SELECTED_SHA ||
|
|
summary.github?.sha ||
|
|
summary.github?.ref ||
|
|
env.GITHUB_SHA ||
|
|
"HEAD";
|
|
const failures = Array.isArray(summary.failures)
|
|
? summary.failures
|
|
: (summary.lanes ?? []).filter((lane) => lane.status !== 0);
|
|
const workflowRerunFailures = failures.filter((failure) => failure.targetable !== false);
|
|
const lanes = failures.map((failure) => ({
|
|
ghWorkflowCommand:
|
|
failure.targetable === false
|
|
? undefined
|
|
: githubWorkflowRerunCommand([failure.name], ref, env),
|
|
image: failure.image,
|
|
imageKind: failure.imageKind,
|
|
lane: failure.name,
|
|
logFile: failure.logFile,
|
|
name: failure.name,
|
|
noOutputTimedOut: failure.noOutputTimedOut,
|
|
rerunCommand: failure.rerunCommand,
|
|
status: failure.status,
|
|
targetable: failure.targetable,
|
|
timedOut: failure.timedOut,
|
|
}));
|
|
const failureIndex = {
|
|
combinedGhWorkflowCommand:
|
|
workflowRerunFailures.length > 0
|
|
? githubWorkflowRerunCommand(
|
|
workflowRerunFailures.map((failure) => failure.name),
|
|
ref,
|
|
env,
|
|
)
|
|
: undefined,
|
|
generatedAt: new Date().toISOString(),
|
|
lanes,
|
|
note: "Targeted GitHub reruns repack the exact selected ref and reuse only GHCR-backed shared images when the generated command includes docker_e2e_*_image inputs.",
|
|
images: summary.images,
|
|
packageArtifactName: env.OPENCLAW_DOCKER_E2E_PACKAGE_ARTIFACT_NAME || undefined,
|
|
ref,
|
|
runUrl: summary.github?.runUrl,
|
|
status: summary.status,
|
|
version: 1,
|
|
workflow: env.OPENCLAW_DOCKER_E2E_WORKFLOW || DEFAULT_GITHUB_WORKFLOW,
|
|
};
|
|
await fs.promises.writeFile(
|
|
path.join(logDir, "failures.json"),
|
|
`${JSON.stringify(failureIndex, null, 2)}\n`,
|
|
);
|
|
}
|
|
|
|
function phaseElapsedSeconds(startedAtMs: number) {
|
|
return Math.round((Date.now() - startedAtMs) / 1000);
|
|
}
|
|
|
|
function laneAttempt(
|
|
attempt: number,
|
|
startedAtMs: number,
|
|
result: Pick<ShellCommandResult, "noOutputTimedOut" | "status" | "timedOut">,
|
|
) {
|
|
return {
|
|
attempt,
|
|
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
|
|
finishedAt: new Date().toISOString(),
|
|
noOutputTimedOut: result.noOutputTimedOut,
|
|
startedAt: new Date(startedAtMs).toISOString(),
|
|
status: result.status,
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
function cleanupSmokeResult(
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
logFile: string,
|
|
command: string,
|
|
startedAtMs: number,
|
|
result: Pick<ShellCommandResult, "noOutputTimedOut" | "status" | "timedOut">,
|
|
) {
|
|
return {
|
|
command,
|
|
attempts: [laneAttempt(1, startedAtMs, result)],
|
|
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
|
|
finishedAt: new Date().toISOString(),
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
logFile,
|
|
name: CLEANUP_SMOKE_NAME,
|
|
noOutputTimedOut: result.noOutputTimedOut,
|
|
rerunCommand: command,
|
|
startedAt: new Date(startedAtMs).toISOString(),
|
|
status: result.status,
|
|
targetable: false,
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
async function runPhase<T>(
|
|
phases: Array<Record<string, unknown>>,
|
|
name: string,
|
|
details: Record<string, unknown>,
|
|
fn: () => Promise<T>,
|
|
): Promise<T> {
|
|
const startedAtMs = Date.now();
|
|
const startedAt = new Date(startedAtMs).toISOString();
|
|
let status: "failed" | "passed" = "passed";
|
|
let errorMessage: string | undefined;
|
|
try {
|
|
const result = await fn();
|
|
return result;
|
|
} catch (error) {
|
|
status = "failed";
|
|
errorMessage = error instanceof Error ? error.message : String(error);
|
|
throw error;
|
|
} finally {
|
|
const phase = {
|
|
...details,
|
|
name,
|
|
startedAt,
|
|
status,
|
|
...(errorMessage === undefined ? {} : { error: errorMessage }),
|
|
elapsedSeconds: phaseElapsedSeconds(startedAtMs),
|
|
finishedAt: new Date().toISOString(),
|
|
};
|
|
phases.push(phase);
|
|
console.log(`==> Phase ${phase.status}: ${name} ${phase.elapsedSeconds}s`);
|
|
}
|
|
}
|
|
|
|
function printLaneManifest(
|
|
label: string,
|
|
poolLanes: DockerE2eLane[],
|
|
timingStore: TimingStore,
|
|
): void {
|
|
console.log(`==> ${label} lanes (${poolLanes.length})`);
|
|
for (const [index, poolLane] of poolLanes.entries()) {
|
|
const seconds = timingSeconds(timingStore.lanes, poolLane);
|
|
const estimate = seconds > 0 ? ` last=${Math.round(seconds)}s` : "";
|
|
console.log(` ${index + 1}. ${laneSummary(poolLane)}${estimate}`);
|
|
}
|
|
}
|
|
|
|
export function dockerPreflightContainerNames(raw: string): string[] {
|
|
return raw.split(/\r?\n/).flatMap((line) => {
|
|
const name = line.trim().split(/\s+/, 1)[0];
|
|
return name &&
|
|
/^(?:openclaw-[a-z0-9-]+-e2e-\d+|openclaw-openwebui(?:-gateway)?-\d+)$/u.test(name)
|
|
? [name]
|
|
: [];
|
|
});
|
|
}
|
|
|
|
export function resolveDockerPreflightPlatform(arch: NodeJS.Architecture = process.arch) {
|
|
return arch === "arm64" ? "linux/arm64" : "linux/amd64";
|
|
}
|
|
|
|
export function dockerPreflightSmokeCommand(arch: NodeJS.Architecture = process.arch) {
|
|
const platform = resolveDockerPreflightPlatform(arch);
|
|
return `docker run --rm --platform ${shellQuote(platform)} alpine:3.20 true`;
|
|
}
|
|
|
|
export function runShellCommand({
|
|
command,
|
|
env,
|
|
label,
|
|
logFile,
|
|
timeoutMs,
|
|
noOutputTimeoutMs,
|
|
timeoutKillGraceMs = SHELL_TIMEOUT_KILL_GRACE_MS,
|
|
}: ShellCommandOptions) {
|
|
if (activeChildrenShutdownPromise) {
|
|
return activeChildrenShutdownPromise.then(() => shellCommandSkippedForShutdown());
|
|
}
|
|
return new Promise<ShellCommandResult>((resolve) => {
|
|
const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(timeoutMs);
|
|
const resolvedNoOutputTimeoutMs = resolveOptionalTimerTimeoutMs(noOutputTimeoutMs);
|
|
const resolvedTimeoutKillGraceMs = resolveTimerTimeoutMs(
|
|
timeoutKillGraceMs,
|
|
SHELL_TIMEOUT_KILL_GRACE_MS,
|
|
);
|
|
const pipeOutput = Boolean(logFile || resolvedNoOutputTimeoutMs);
|
|
const child = spawn("bash", ["-c", command], {
|
|
cwd: ROOT_DIR,
|
|
detached: process.platform !== "win32",
|
|
env,
|
|
stdio: pipeOutput ? ["ignore", "pipe", "pipe"] : "inherit",
|
|
});
|
|
activeChildren.set(child, resolvedTimeoutKillGraceMs);
|
|
let timedOut = false;
|
|
let noOutputTimedOut = false;
|
|
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
|
let killAt: number | undefined;
|
|
const stream = logFile ? fs.createWriteStream(logFile, { flags: "a" }) : undefined;
|
|
let noOutputTimer: ReturnType<typeof setTimeout> | undefined;
|
|
const terminateForTimeout = (message: string, options: { noOutput?: boolean } = {}) => {
|
|
if (timedOut) {
|
|
return;
|
|
}
|
|
timedOut = true;
|
|
noOutputTimedOut = options.noOutput === true;
|
|
if (stream) {
|
|
stream.write(`\n==> [${label}] ${message}; sending SIGTERM\n`);
|
|
} else {
|
|
console.error(`==> [${label}] ${message}; sending SIGTERM`);
|
|
}
|
|
terminateChild(child, "SIGTERM");
|
|
killAt = Date.now() + resolvedTimeoutKillGraceMs;
|
|
killTimer = setTimeout(() => terminateChild(child, "SIGKILL"), resolvedTimeoutKillGraceMs);
|
|
killTimer.unref?.();
|
|
};
|
|
const resetNoOutputTimer = () => {
|
|
if (!resolvedNoOutputTimeoutMs || timedOut) {
|
|
return;
|
|
}
|
|
if (noOutputTimer) {
|
|
clearTimeout(noOutputTimer);
|
|
}
|
|
noOutputTimer = setTimeout(() => {
|
|
terminateForTimeout(`no output for ${resolvedNoOutputTimeoutMs}ms`, { noOutput: true });
|
|
}, resolvedNoOutputTimeoutMs);
|
|
noOutputTimer.unref?.();
|
|
};
|
|
const timeoutTimer =
|
|
resolvedTimeoutMs !== undefined
|
|
? setTimeout(() => {
|
|
terminateForTimeout(`timeout after ${resolvedTimeoutMs}ms`);
|
|
}, resolvedTimeoutMs)
|
|
: undefined;
|
|
timeoutTimer?.unref?.();
|
|
|
|
if (stream) {
|
|
stream.write(`==> [${label}] command: ${command}\n`);
|
|
stream.write(`==> [${label}] started: ${utcStamp()}\n`);
|
|
}
|
|
if (pipeOutput && child.stdout && child.stderr) {
|
|
const writeOutput = (target: NodeJS.WriteStream, chunk: Uint8Array) => {
|
|
resetNoOutputTimer();
|
|
if (stream) {
|
|
stream.write(chunk);
|
|
} else {
|
|
target.write(chunk);
|
|
}
|
|
};
|
|
child.stdout.on("data", (chunk: Uint8Array) => writeOutput(process.stdout, chunk));
|
|
child.stderr.on("data", (chunk: Uint8Array) => writeOutput(process.stderr, chunk));
|
|
resetNoOutputTimer();
|
|
}
|
|
|
|
child.on("close", (status, signal) => {
|
|
if (timeoutTimer) {
|
|
clearTimeout(timeoutTimer);
|
|
}
|
|
if (noOutputTimer) {
|
|
clearTimeout(noOutputTimer);
|
|
}
|
|
const finish = () => {
|
|
if (killTimer) {
|
|
clearTimeout(killTimer);
|
|
}
|
|
killAt = undefined;
|
|
activeChildren.delete(child);
|
|
const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
|
|
if (stream) {
|
|
stream.write(
|
|
`\n==> [${label}] finished: ${utcStamp()} status=${exitCode}${
|
|
noOutputTimedOut ? " noOutputTimedOut=true" : ""
|
|
}\n`,
|
|
);
|
|
stream.end();
|
|
}
|
|
resolve({ signal, status: exitCode, timedOut, noOutputTimedOut });
|
|
};
|
|
if (timedOut) {
|
|
void finishTimedOutShellProcessTree(child, killAt, resolvedTimeoutKillGraceMs).then(
|
|
finish,
|
|
finish,
|
|
);
|
|
return;
|
|
}
|
|
finish();
|
|
});
|
|
});
|
|
}
|
|
|
|
export function appendBoundedShellCapture(
|
|
current: string,
|
|
chunk: unknown,
|
|
maxChars = SHELL_CAPTURE_MAX_CHARS,
|
|
) {
|
|
const combined = `${current}${String(chunk)}`;
|
|
if (combined.length <= maxChars) {
|
|
return { text: combined, truncated: false };
|
|
}
|
|
return { text: combined.slice(-maxChars), truncated: true };
|
|
}
|
|
|
|
export function runShellCaptureCommand({
|
|
command,
|
|
env,
|
|
label,
|
|
timeoutMs,
|
|
timeoutKillGraceMs = SHELL_TIMEOUT_KILL_GRACE_MS,
|
|
}: ShellCaptureOptions) {
|
|
if (activeChildrenShutdownPromise) {
|
|
return activeChildrenShutdownPromise.then(() => shellCaptureSkippedForShutdown(label));
|
|
}
|
|
return new Promise<ShellCaptureResult>((resolve) => {
|
|
const resolvedTimeoutMs = resolveOptionalTimerTimeoutMs(timeoutMs);
|
|
const resolvedTimeoutKillGraceMs = resolveTimerTimeoutMs(
|
|
timeoutKillGraceMs,
|
|
SHELL_TIMEOUT_KILL_GRACE_MS,
|
|
);
|
|
const child = spawn("bash", ["-c", command], {
|
|
cwd: ROOT_DIR,
|
|
detached: process.platform !== "win32",
|
|
env,
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
activeChildren.set(child, resolvedTimeoutKillGraceMs);
|
|
let stdout = "";
|
|
let stderr = "";
|
|
let stdoutTruncated = false;
|
|
let stderrTruncated = false;
|
|
let timedOut = false;
|
|
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
|
let killAt: number | undefined;
|
|
const timeoutTimer =
|
|
resolvedTimeoutMs !== undefined
|
|
? setTimeout(() => {
|
|
timedOut = true;
|
|
terminateChild(child, "SIGTERM");
|
|
killAt = Date.now() + resolvedTimeoutKillGraceMs;
|
|
killTimer = setTimeout(
|
|
() => terminateChild(child, "SIGKILL"),
|
|
resolvedTimeoutKillGraceMs,
|
|
);
|
|
killTimer.unref?.();
|
|
}, resolvedTimeoutMs)
|
|
: undefined;
|
|
timeoutTimer?.unref?.();
|
|
child.stdout.on("data", (chunk: Buffer) => {
|
|
const next = appendBoundedShellCapture(stdout, chunk);
|
|
stdout = next.text;
|
|
stdoutTruncated ||= next.truncated;
|
|
});
|
|
child.stderr.on("data", (chunk: Buffer) => {
|
|
const next = appendBoundedShellCapture(stderr, chunk);
|
|
stderr = next.text;
|
|
stderrTruncated ||= next.truncated;
|
|
});
|
|
child.on("close", (status, signal) => {
|
|
if (timeoutTimer) {
|
|
clearTimeout(timeoutTimer);
|
|
}
|
|
const finish = () => {
|
|
if (killTimer) {
|
|
clearTimeout(killTimer);
|
|
}
|
|
killAt = undefined;
|
|
activeChildren.delete(child);
|
|
const exitCode = typeof status === "number" ? status : signal ? 128 : 1;
|
|
resolve({
|
|
label,
|
|
signal,
|
|
status: exitCode,
|
|
stderr,
|
|
stderrTruncated,
|
|
stdout,
|
|
stdoutTruncated,
|
|
timedOut,
|
|
});
|
|
};
|
|
if (timedOut) {
|
|
void finishTimedOutShellProcessTree(child, killAt, resolvedTimeoutKillGraceMs).then(
|
|
finish,
|
|
finish,
|
|
);
|
|
return;
|
|
}
|
|
finish();
|
|
});
|
|
});
|
|
}
|
|
|
|
async function runForeground(label: string, command: string, env: NodeJS.ProcessEnv) {
|
|
console.log(`==> ${label}`);
|
|
const result = await runShellCommand({ command, env, label });
|
|
if (result.status !== 0) {
|
|
throw new Error(`${label} failed with status ${result.status}`);
|
|
}
|
|
}
|
|
|
|
export async function runCleanupSmokePhase(
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
logDir: string,
|
|
phases: Array<Record<string, unknown>>,
|
|
) {
|
|
const command = "pnpm test:docker:cleanup";
|
|
const logFile = path.join(logDir, `${CLEANUP_SMOKE_NAME}.log`);
|
|
const startedAtMs = Date.now();
|
|
let failure: LaneResult | undefined;
|
|
try {
|
|
await runPhase(phases, CLEANUP_SMOKE_NAME, {}, async () => {
|
|
const result = await runShellCommand({
|
|
command,
|
|
env: baseEnv,
|
|
label: CLEANUP_SMOKE_NAME,
|
|
logFile,
|
|
});
|
|
if (result.status !== 0) {
|
|
failure = cleanupSmokeResult(baseEnv, logFile, command, startedAtMs, result);
|
|
}
|
|
if (failure) {
|
|
throw new Error(
|
|
`Run cleanup smoke after parallel lanes failed with status ${failure.status}`,
|
|
);
|
|
}
|
|
});
|
|
} catch (error) {
|
|
if (!failure) {
|
|
const status = 1;
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
await fs.promises.writeFile(
|
|
logFile,
|
|
[
|
|
`==> [${CLEANUP_SMOKE_NAME}] command: ${command}`,
|
|
`==> [${CLEANUP_SMOKE_NAME}] status: ${status}`,
|
|
`==> [${CLEANUP_SMOKE_NAME}] error: ${message}`,
|
|
].join("\n"),
|
|
);
|
|
failure = cleanupSmokeResult(baseEnv, logFile, command, startedAtMs, {
|
|
noOutputTimedOut: false,
|
|
status,
|
|
timedOut: false,
|
|
});
|
|
}
|
|
}
|
|
return failure;
|
|
}
|
|
|
|
async function runForegroundGroup(entries: ForegroundEntry[], env: NodeJS.ProcessEnv) {
|
|
const failures: Array<{ entry: ForegroundEntry; error: unknown }> = [];
|
|
for (const entry of entries) {
|
|
try {
|
|
const { command, label, phaseDetails = {}, phases } = entry;
|
|
const entryEnv = { ...env, ...entry.env };
|
|
if (phases) {
|
|
await runPhase(phases, `build:${label}`, phaseDetails, async () => {
|
|
await runForeground(label, command, entryEnv);
|
|
});
|
|
} else {
|
|
await runForeground(label, command, entryEnv);
|
|
}
|
|
} catch (error) {
|
|
failures.push({ entry, error });
|
|
}
|
|
}
|
|
if (failures.length > 0) {
|
|
throw new Error(
|
|
failures
|
|
.map(
|
|
({ entry, error }) =>
|
|
`${entry.label}: ${error instanceof Error ? error.message : String(error)}`,
|
|
)
|
|
.join("\n"),
|
|
);
|
|
}
|
|
}
|
|
|
|
async function runDockerPreflight(
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
options: { cleanup: boolean; enabled: boolean; runTimeoutMs: number },
|
|
) {
|
|
if (!options.enabled) {
|
|
console.log("==> Docker preflight: skipped");
|
|
return;
|
|
}
|
|
console.log("==> Docker preflight");
|
|
const version = await runShellCaptureCommand({
|
|
command: "docker version --format '{{.Server.Version}}'",
|
|
env: baseEnv,
|
|
label: "docker-version",
|
|
timeoutMs: 20_000,
|
|
});
|
|
if (version.status !== 0) {
|
|
throw new Error(
|
|
`Docker preflight failed: docker version status=${version.status}\n${version.stderr}${version.stdout}`,
|
|
);
|
|
}
|
|
console.log(`==> Docker server: ${version.stdout.trim()}`);
|
|
|
|
if (options.cleanup) {
|
|
const stale = await runShellCaptureCommand({
|
|
command:
|
|
"docker ps -a --filter status=created --filter status=exited --filter status=dead --format '{{.Names}} {{.Status}}'",
|
|
env: baseEnv,
|
|
label: "docker-stale-list",
|
|
timeoutMs: 20_000,
|
|
});
|
|
if (stale.status === 0) {
|
|
const names = dockerPreflightContainerNames(stale.stdout);
|
|
if (names.length > 0) {
|
|
console.log(`==> Docker preflight cleanup: ${names.join(", ")}`);
|
|
const cleanup = await runShellCommand({
|
|
command: `docker rm -f ${names.map(shellQuote).join(" ")}`,
|
|
env: baseEnv,
|
|
label: "docker-stale-cleanup",
|
|
timeoutMs: 90_000,
|
|
});
|
|
if (cleanup.status !== 0) {
|
|
throw new Error(`Docker preflight cleanup failed with status ${cleanup.status}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const startedAt = Date.now();
|
|
const run = await runShellCommand({
|
|
command: dockerPreflightSmokeCommand(),
|
|
env: baseEnv,
|
|
label: "docker-run-smoke",
|
|
timeoutMs: options.runTimeoutMs,
|
|
});
|
|
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
|
|
if (run.status !== 0) {
|
|
throw new Error(
|
|
`Docker preflight failed: ${dockerPreflightSmokeCommand()} status=${run.status} elapsed=${elapsedSeconds}s`,
|
|
);
|
|
}
|
|
console.log(`==> Docker preflight run: ${elapsedSeconds}s`);
|
|
}
|
|
|
|
async function prepareOpenClawPackage(baseEnv: NodeJS.ProcessEnv, logDir: string) {
|
|
const existing = baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ;
|
|
if (existing) {
|
|
const packageTgz = path.resolve(existing);
|
|
baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ = packageTgz;
|
|
baseEnv.OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD = "0";
|
|
baseEnv.OPENCLAW_NPM_ONBOARD_HOST_BUILD = "0";
|
|
console.log(`==> OpenClaw package: ${packageTgz}`);
|
|
return;
|
|
}
|
|
|
|
const packDir = path.join(logDir, "openclaw-package");
|
|
await mkdir(packDir, { recursive: true });
|
|
const packageTgz = path.join(packDir, "openclaw-current.tgz");
|
|
await runForeground(
|
|
"Prepare OpenClaw package once",
|
|
`node ${shellQuote(path.join(ROOT_DIR, "scripts/package-openclaw-for-docker.mjs"))} --source-dir ${shellQuote(ROOT_DIR)} --allow-unreleased-changelog --output-dir ${shellQuote(packDir)} --output-name openclaw-current.tgz`,
|
|
baseEnv,
|
|
);
|
|
await fs.promises.access(packageTgz);
|
|
// Preserve current-tree package intent in generated GitHub reruns; otherwise a local QA
|
|
// failure would retry through the strict release path and fail before reaching its lane.
|
|
baseEnv.OPENCLAW_DOCKER_E2E_ALLOW_UNRELEASED_CHANGELOG = "true";
|
|
baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ = packageTgz;
|
|
baseEnv.OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD = "0";
|
|
baseEnv.OPENCLAW_NPM_ONBOARD_HOST_BUILD = "0";
|
|
console.log(`==> OpenClaw package: ${baseEnv.OPENCLAW_CURRENT_PACKAGE_TGZ}`);
|
|
}
|
|
|
|
async function prepareDockerCandidate(
|
|
plan: DockerCandidatePlan,
|
|
logDir: string,
|
|
manifestPath: string,
|
|
) {
|
|
const sourceSha = gitOutput(ROOT_DIR, ["rev-parse", "HEAD"]);
|
|
let candidate = null;
|
|
if (plan.needs.package) {
|
|
if (gitOutput(ROOT_DIR, ["status", "--porcelain=v1"])) {
|
|
throw new Error("repository has working-tree changes; refusing to prepare Docker candidate");
|
|
}
|
|
const candidateEnv = commandEnv();
|
|
for (const key of [...CANDIDATE_ENV_KEYS, ...REGISTRY_ENV_KEYS]) {
|
|
delete candidateEnv[key];
|
|
}
|
|
await prepareOpenClawPackage(candidateEnv, logDir);
|
|
const packagePath = candidateEnv.OPENCLAW_CURRENT_PACKAGE_TGZ!;
|
|
const packed = inspectNpmPackageTarball(packagePath);
|
|
const version = rootPackageVersion(ROOT_DIR);
|
|
if (packed.packageJson.name !== "openclaw" || packed.packageJson.version !== version) {
|
|
throw new Error("packed Docker candidate name or version differs from the root package");
|
|
}
|
|
let registry = null;
|
|
if (plan.needs.prepublishPluginRegistry) {
|
|
const registryDir = path.join(logDir, "prepublish-plugin-registry");
|
|
fs.rmSync(registryDir, { force: true, recursive: true });
|
|
const artifact = createPrepublishPluginRegistryArtifact({
|
|
repoRoot: ROOT_DIR,
|
|
outputDir: registryDir,
|
|
sourceSha,
|
|
candidateVersion: version,
|
|
requiredPackages: plan.requiredPrepublishPluginPackages,
|
|
});
|
|
registry = {
|
|
dir: registryDir,
|
|
candidateVersion: version,
|
|
manifestSha256: artifact.manifestSha256,
|
|
};
|
|
}
|
|
candidate = {
|
|
package: { path: packagePath, name: packed.packageJson.name, version, sha256: packed.sha256 },
|
|
registry,
|
|
};
|
|
}
|
|
await mkdir(path.dirname(manifestPath), { recursive: true });
|
|
fs.writeFileSync(
|
|
manifestPath,
|
|
`${JSON.stringify({ schema: "openclaw.qa-docker-candidate/v1", schemaVersion: 1, sourceSha, candidate }, null, 2)}\n`,
|
|
);
|
|
}
|
|
|
|
function e2eImageForLane(poolLane: DockerE2eLane, baseEnv: NodeJS.ProcessEnv) {
|
|
if (poolLane.e2eImageKind === "bare") {
|
|
return baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE;
|
|
}
|
|
if (poolLane.e2eImageKind === "functional") {
|
|
return baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE;
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
type DockerLaneEnv = {
|
|
[key: string]: string | undefined;
|
|
OPENCLAW_DOCKER_CACHE_HOME_DIR: string;
|
|
OPENCLAW_DOCKER_CLI_TOOLS_DIR: string;
|
|
};
|
|
|
|
function laneEnv(
|
|
poolLane: DockerE2eLane,
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
logDir: string,
|
|
cacheKey: string | undefined,
|
|
): DockerLaneEnv {
|
|
const name = poolLane.name;
|
|
const cacheName = cacheKey || name;
|
|
const env: DockerLaneEnv = {
|
|
...baseEnv,
|
|
OPENCLAW_DOCKER_CACHE_HOME_DIR:
|
|
process.env.OPENCLAW_DOCKER_CACHE_HOME_DIR ?? path.join(logDir, `${cacheName}-cache`),
|
|
OPENCLAW_DOCKER_CLI_TOOLS_DIR:
|
|
process.env.OPENCLAW_DOCKER_CLI_TOOLS_DIR ?? path.join(logDir, `${cacheName}-cli-tools`),
|
|
};
|
|
env.OPENCLAW_DOCKER_ALL_LANE_NAME = name;
|
|
const image = e2eImageForLane(poolLane, baseEnv);
|
|
if (image) {
|
|
env.OPENCLAW_DOCKER_E2E_IMAGE = image;
|
|
}
|
|
if (poolLane.e2eImageKind) {
|
|
env.OPENCLAW_DOCKER_E2E_IMAGE_KIND = poolLane.e2eImageKind;
|
|
}
|
|
return env;
|
|
}
|
|
|
|
async function runLane(
|
|
lane: DockerE2eLane,
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
logDir: string,
|
|
fallbackTimeoutMs: number,
|
|
) {
|
|
const { name } = lane;
|
|
const timeoutMs = lane.timeoutMs ?? fallbackTimeoutMs;
|
|
const noOutputTimeoutMs = lane.noOutputTimeoutMs;
|
|
const logFile = path.join(logDir, `${name}.log`);
|
|
const env = laneEnv(lane, baseEnv, logDir, lane.cacheKey);
|
|
const pnpmCommand = env.OPENCLAW_DOCKER_ALL_PNPM_COMMAND?.trim();
|
|
const command = pnpmCommand
|
|
? lane.command.replace(/(^|\s)pnpm(?=\s)/g, `$1${shellQuote(pnpmCommand)}`)
|
|
: lane.command;
|
|
await mkdir(env.OPENCLAW_DOCKER_CLI_TOOLS_DIR, { recursive: true });
|
|
await mkdir(env.OPENCLAW_DOCKER_CACHE_HOME_DIR, { recursive: true });
|
|
await fs.promises.writeFile(
|
|
logFile,
|
|
[
|
|
`==> [${name}] cli tools dir: ${env.OPENCLAW_DOCKER_CLI_TOOLS_DIR}`,
|
|
`==> [${name}] cache dir: ${env.OPENCLAW_DOCKER_CACHE_HOME_DIR}`,
|
|
`==> [${name}] timeout: ${timeoutMs}ms`,
|
|
`==> [${name}] no output timeout: ${noOutputTimeoutMs ?? 0}ms`,
|
|
`==> [${name}] retries: ${lane.retries ?? 0}`,
|
|
`==> [${name}] e2e image kind: ${lane.e2eImageKind ?? "none"}`,
|
|
`==> [${name}] e2e image: ${env.OPENCLAW_DOCKER_E2E_IMAGE ?? ""}`,
|
|
"",
|
|
].join("\n"),
|
|
);
|
|
console.log(`==> [${name}] start`);
|
|
const startedAt = Date.now();
|
|
const startedAtIso = new Date(startedAt).toISOString();
|
|
let result: ShellCommandResult;
|
|
const attempts: ReturnType<typeof laneAttempt>[] = [];
|
|
const maxAttempts = 1 + Math.max(0, lane.retries ?? 0);
|
|
for (let attempt = 1; ; attempt += 1) {
|
|
const attemptStartedAt = Date.now();
|
|
if (attempt > 1) {
|
|
await fs.promises.appendFile(logFile, `\n==> [${name}] retry attempt ${attempt}\n`);
|
|
console.log(`==> [${name}] retry ${attempt}/${maxAttempts}`);
|
|
}
|
|
result = await runShellCommand({
|
|
command,
|
|
env,
|
|
label: name,
|
|
logFile,
|
|
timeoutMs,
|
|
noOutputTimeoutMs,
|
|
});
|
|
attempts.push(laneAttempt(attempt, attemptStartedAt, result));
|
|
if (result.status === 0 || attempt >= maxAttempts) {
|
|
break;
|
|
}
|
|
const retryable =
|
|
result.timedOut || (await laneLogMatchesRetryPattern(logFile, lane.retryPatterns));
|
|
if (!retryable) {
|
|
break;
|
|
}
|
|
}
|
|
const elapsedSeconds = Math.round((Date.now() - startedAt) / 1000);
|
|
if (result.status === 0) {
|
|
console.log(`==> [${name}] pass ${elapsedSeconds}s`);
|
|
} else {
|
|
const timeoutLabel = result.timedOut ? " timeout" : "";
|
|
console.error(
|
|
`==> [${name}] fail${timeoutLabel} status=${result.status} ${elapsedSeconds}s log=${logFile}`,
|
|
);
|
|
}
|
|
return {
|
|
command,
|
|
attempts,
|
|
finishedAt: new Date().toISOString(),
|
|
image: env.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
imageKind: lane.e2eImageKind,
|
|
logFile,
|
|
name,
|
|
elapsedSeconds,
|
|
rerunCommand: buildLaneRerunCommand(name, baseEnv),
|
|
startedAt: startedAtIso,
|
|
status: result.status,
|
|
noOutputTimedOut: result.noOutputTimedOut,
|
|
timedOut: result.timedOut,
|
|
};
|
|
}
|
|
|
|
async function runLanePool(
|
|
poolLanes: DockerE2eLane[],
|
|
baseEnv: NodeJS.ProcessEnv,
|
|
logDir: string,
|
|
parallelism: number,
|
|
options: LanePoolOptions,
|
|
) {
|
|
const failures: LaneResult[] = [];
|
|
const results: LaneResult[] = [];
|
|
const pending = [...poolLanes];
|
|
const running = new Map<symbol, Promise<{ id: symbol; result: LaneResult }>>();
|
|
const active = {
|
|
count: 0,
|
|
resources: new Map<string, number>(),
|
|
weight: 0,
|
|
} satisfies SchedulerActiveState;
|
|
const activeLanes = new Map<string, number>();
|
|
let lastLaneStartAt = 0;
|
|
let laneStartQueue: Promise<void> = Promise.resolve();
|
|
const statusTimer =
|
|
options.statusIntervalMs > 0
|
|
? setInterval(() => {
|
|
const runningSummary = [...activeLanes]
|
|
.map(([name, startedAt]) => `${name}:${Math.round((Date.now() - startedAt) / 1000)}s`)
|
|
.join(", ");
|
|
const resources = [...active.resources.entries()]
|
|
.map(([resource, value]) => `${resource}=${value}`)
|
|
.join(" ");
|
|
console.log(
|
|
`==> [${options.poolLabel}] active=${active.count} pending=${pending.length} ${resources}${
|
|
runningSummary ? ` lanes=${runningSummary}` : ""
|
|
}`,
|
|
);
|
|
}, options.statusIntervalMs)
|
|
: undefined;
|
|
statusTimer?.unref?.();
|
|
|
|
async function waitForLaneStartSlot() {
|
|
if (options.startStaggerMs <= 0) {
|
|
return;
|
|
}
|
|
const previous = laneStartQueue;
|
|
let releaseNext = () => {};
|
|
laneStartQueue = new Promise<void>((resolve) => {
|
|
releaseNext = resolve;
|
|
});
|
|
await previous;
|
|
const waitMs = Math.max(0, lastLaneStartAt + options.startStaggerMs - Date.now());
|
|
if (waitMs > 0) {
|
|
await sleep(waitMs);
|
|
}
|
|
lastLaneStartAt = Date.now();
|
|
releaseNext();
|
|
}
|
|
|
|
function canStartLane(candidate: DockerE2eLane) {
|
|
return canStartSchedulerLane(candidate, active, parallelism, options);
|
|
}
|
|
|
|
function reserve(candidate: DockerE2eLane) {
|
|
const weight = laneWeight(candidate);
|
|
active.count += 1;
|
|
active.weight += weight;
|
|
for (const resource of laneResources(candidate)) {
|
|
active.resources.set(resource, (active.resources.get(resource) ?? 0) + weight);
|
|
}
|
|
}
|
|
|
|
function release(candidate: DockerE2eLane) {
|
|
const weight = laneWeight(candidate);
|
|
active.count -= 1;
|
|
active.weight -= weight;
|
|
for (const resource of laneResources(candidate)) {
|
|
const next = (active.resources.get(resource) ?? 0) - weight;
|
|
if (next > 0) {
|
|
active.resources.set(resource, next);
|
|
} else {
|
|
active.resources.delete(resource);
|
|
}
|
|
}
|
|
}
|
|
|
|
async function startLane(poolLane: DockerE2eLane) {
|
|
await waitForLaneStartSlot();
|
|
reserve(poolLane);
|
|
activeLanes.set(poolLane.name, Date.now());
|
|
const id = Symbol(poolLane.name);
|
|
const promise = runLane(poolLane, baseEnv, logDir, options.timeoutMs)
|
|
.then((result) => ({ id, result }))
|
|
.finally(() => {
|
|
activeLanes.delete(poolLane.name);
|
|
release(poolLane);
|
|
});
|
|
running.set(id, promise);
|
|
}
|
|
|
|
try {
|
|
while (pending.length > 0 || running.size > 0) {
|
|
let started = false;
|
|
if (!options.failFast || failures.length === 0) {
|
|
for (let index = 0; index < pending.length;) {
|
|
const candidate = pending[index];
|
|
if (!candidate) {
|
|
break;
|
|
}
|
|
if (!canStartLane(candidate)) {
|
|
index += 1;
|
|
continue;
|
|
}
|
|
pending.splice(index, 1);
|
|
await startLane(candidate);
|
|
started = true;
|
|
}
|
|
}
|
|
|
|
if (started) {
|
|
continue;
|
|
}
|
|
if (running.size === 0) {
|
|
const blocked = pending.map(laneSummary).join(", ");
|
|
throw new Error(
|
|
`No Docker lanes fit scheduler limits (${describeDockerSchedulerLimits(
|
|
parallelism,
|
|
options,
|
|
)}): ${blocked}. Tune OPENCLAW_DOCKER_ALL_PARALLELISM, OPENCLAW_DOCKER_ALL_WEIGHT_LIMIT, or OPENCLAW_DOCKER_ALL_<RESOURCE>_LIMIT.`,
|
|
);
|
|
}
|
|
|
|
const { id, result } = await Promise.race(running.values());
|
|
running.delete(id);
|
|
results.push(result);
|
|
if (result.status !== 0) {
|
|
failures.push(result);
|
|
}
|
|
if (options.failFast && failures.length > 0) {
|
|
const remainingResults = await Promise.all(running.values());
|
|
running.clear();
|
|
for (const remaining of remainingResults) {
|
|
results.push(remaining.result);
|
|
if (remaining.result.status !== 0) {
|
|
failures.push(remaining.result);
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
} finally {
|
|
if (statusTimer) {
|
|
clearInterval(statusTimer);
|
|
}
|
|
}
|
|
|
|
return { failures, results };
|
|
}
|
|
|
|
export async function tailFile(file: string, lines: number, maxBytes = LOG_TAIL_MAX_BYTES) {
|
|
let content: string;
|
|
try {
|
|
const handle = await open(file, "r");
|
|
try {
|
|
const stat = await handle.stat();
|
|
const bytesToRead = Math.min(Math.max(1, maxBytes), stat.size);
|
|
const buffer = Buffer.alloc(bytesToRead);
|
|
await handle.read(buffer, 0, bytesToRead, stat.size - bytesToRead);
|
|
content = buffer.toString("utf8");
|
|
} finally {
|
|
await handle.close().catch(() => {});
|
|
}
|
|
} catch {
|
|
content = "";
|
|
}
|
|
const tail = content.split(/\r?\n/).slice(-lines).join("\n");
|
|
return tail.trimEnd();
|
|
}
|
|
|
|
async function laneLogMatchesRetryPattern(logFile: string, patterns: RegExp[]) {
|
|
if (!patterns || patterns.length === 0) {
|
|
return false;
|
|
}
|
|
const tail = await tailFile(logFile, 160);
|
|
return patterns.some((pattern) => pattern.test(tail));
|
|
}
|
|
|
|
async function printFailureSummary(failures: LaneResult[], tailLines: number) {
|
|
console.error(`ERROR: ${failures.length} Docker lane(s) failed.`);
|
|
for (const failure of failures) {
|
|
console.error(`---- ${failure.name} failed (status=${failure.status}): ${failure.logFile}`);
|
|
const tail = await tailFile(failure.logFile, tailLines);
|
|
if (tail) {
|
|
console.error(tail);
|
|
}
|
|
}
|
|
}
|
|
|
|
const activeChildren = new Map<ChildProcess, number>();
|
|
let activeChildrenShutdownPromise: Promise<unknown> | undefined;
|
|
|
|
function shellCommandSkippedForShutdown(signal: ShutdownSignal | null = null) {
|
|
return {
|
|
noOutputTimedOut: false,
|
|
signal,
|
|
status: 143,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
|
|
function shellCaptureSkippedForShutdown(label: string, signal: ShutdownSignal | null = null) {
|
|
return {
|
|
label,
|
|
signal,
|
|
status: 143,
|
|
stderr: "",
|
|
stderrTruncated: false,
|
|
stdout: "",
|
|
stdoutTruncated: false,
|
|
timedOut: false,
|
|
};
|
|
}
|
|
|
|
function shellProcessGroupAlive(child: ChildProcess) {
|
|
if (process.platform === "win32" || !child.pid) {
|
|
return false;
|
|
}
|
|
try {
|
|
process.kill(-child.pid, 0);
|
|
return true;
|
|
} catch (error) {
|
|
return error instanceof Error && "code" in error && error.code === "EPERM";
|
|
}
|
|
}
|
|
|
|
async function waitForShellProcessGroupExit(child: ChildProcess, timeoutMs: number) {
|
|
const deadlineAt = Date.now() + timeoutMs;
|
|
while (Date.now() < deadlineAt) {
|
|
if (!shellProcessGroupAlive(child)) {
|
|
return true;
|
|
}
|
|
await new Promise((resolvePoll) => {
|
|
setTimeout(resolvePoll, SHELL_PROCESS_GROUP_EXIT_POLL_MS);
|
|
});
|
|
}
|
|
return !shellProcessGroupAlive(child);
|
|
}
|
|
|
|
async function finishTimedOutShellProcessTree(
|
|
child: ChildProcess,
|
|
killAt: number | undefined,
|
|
timeoutKillGraceMs: number,
|
|
) {
|
|
if (!shellProcessGroupAlive(child)) {
|
|
return;
|
|
}
|
|
const graceRemainingMs =
|
|
killAt === undefined ? timeoutKillGraceMs : Math.max(0, killAt - Date.now());
|
|
if (graceRemainingMs > 0) {
|
|
await waitForShellProcessGroupExit(child, graceRemainingMs);
|
|
}
|
|
if (shellProcessGroupAlive(child)) {
|
|
terminateChild(child, "SIGKILL");
|
|
}
|
|
await waitForShellProcessGroupExit(child, SHELL_POST_FORCE_KILL_WAIT_MS);
|
|
}
|
|
|
|
function terminateChild(child: ChildProcess, signal: ShutdownSignal) {
|
|
if (process.platform !== "win32" && child.pid) {
|
|
try {
|
|
process.kill(-child.pid, signal);
|
|
return;
|
|
} catch {
|
|
// Fall back to killing the direct child below.
|
|
}
|
|
}
|
|
child.kill(signal);
|
|
}
|
|
|
|
function terminateActiveChildren(signal: ShutdownSignal) {
|
|
for (const child of activeChildren.keys()) {
|
|
terminateChild(child, signal);
|
|
}
|
|
}
|
|
|
|
async function shutdownActiveChildren(signal: ShutdownSignal, exitCode: number) {
|
|
if (activeChildrenShutdownPromise) {
|
|
terminateActiveChildren("SIGKILL");
|
|
return activeChildrenShutdownPromise;
|
|
}
|
|
const children = [...activeChildren.entries()];
|
|
terminateActiveChildren(signal);
|
|
activeChildrenShutdownPromise = Promise.all(
|
|
children.map(([child, timeoutKillGraceMs]) =>
|
|
finishTimedOutShellProcessTree(child, Date.now() + timeoutKillGraceMs, timeoutKillGraceMs),
|
|
),
|
|
).finally(() => {
|
|
process.exit(exitCode);
|
|
});
|
|
return activeChildrenShutdownPromise;
|
|
}
|
|
|
|
process.on("SIGINT", () => {
|
|
void shutdownActiveChildren("SIGINT", 130);
|
|
});
|
|
process.on("SIGTERM", () => {
|
|
void shutdownActiveChildren("SIGTERM", 143);
|
|
});
|
|
|
|
async function main() {
|
|
const runStartedAt = new Date().toISOString();
|
|
const phases: Array<Record<string, unknown>> = [];
|
|
const parallelism = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_PARALLELISM,
|
|
DEFAULT_PARALLELISM,
|
|
"OPENCLAW_DOCKER_ALL_PARALLELISM",
|
|
);
|
|
const tailParallelism = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM,
|
|
Math.min(parallelism, DEFAULT_TAIL_PARALLELISM),
|
|
"OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM",
|
|
);
|
|
const tailLines = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES,
|
|
DEFAULT_FAILURE_TAIL_LINES,
|
|
"OPENCLAW_DOCKER_ALL_FAILURE_TAIL_LINES",
|
|
);
|
|
const laneTimeoutMs = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS,
|
|
DEFAULT_LANE_TIMEOUT_MS,
|
|
"OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS",
|
|
);
|
|
const laneStartStaggerMs = parseNonNegativeInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_START_STAGGER_MS,
|
|
DEFAULT_LANE_START_STAGGER_MS,
|
|
"OPENCLAW_DOCKER_ALL_START_STAGGER_MS",
|
|
);
|
|
const statusIntervalMs = parseNonNegativeInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS,
|
|
DEFAULT_STATUS_INTERVAL_MS,
|
|
"OPENCLAW_DOCKER_ALL_STATUS_INTERVAL_MS",
|
|
);
|
|
const preflightRunTimeoutMs = parsePositiveInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS,
|
|
DEFAULT_PREFLIGHT_RUN_TIMEOUT_MS,
|
|
"OPENCLAW_DOCKER_ALL_PREFLIGHT_RUN_TIMEOUT_MS",
|
|
);
|
|
const failFast = parseBool(process.env.OPENCLAW_DOCKER_ALL_FAIL_FAST, true);
|
|
const dryRun = parseBool(process.env.OPENCLAW_DOCKER_ALL_DRY_RUN, false);
|
|
const preflightEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT, true);
|
|
const preflightCleanup = parseBool(process.env.OPENCLAW_DOCKER_ALL_PREFLIGHT_CLEANUP, true);
|
|
const timingsEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_TIMINGS, true);
|
|
const buildEnabled = parseBool(process.env.OPENCLAW_DOCKER_ALL_BUILD, true);
|
|
const allowFrozenTargetScenarioOmissions = parseBool(
|
|
process.env.OPENCLAW_ALLOW_FROZEN_TARGET_SCENARIO_OMISSIONS,
|
|
false,
|
|
);
|
|
const planJson =
|
|
cliOptions.planJson || parseBool(process.env.OPENCLAW_DOCKER_ALL_PLAN_JSON, false);
|
|
const planReleaseAll = parseBool(process.env.OPENCLAW_DOCKER_ALL_PLAN_RELEASE_ALL, false);
|
|
const profile = parseProfile(process.env.OPENCLAW_DOCKER_ALL_PROFILE);
|
|
const releaseProfile = normalizeReleaseProfileEnv(
|
|
process.env.OPENCLAW_DOCKER_ALL_RELEASE_PROFILE || process.env.OPENCLAW_RELEASE_PROFILE,
|
|
);
|
|
const releaseChunk = process.env.OPENCLAW_DOCKER_ALL_CHUNK || process.env.DOCKER_E2E_CHUNK || "";
|
|
const includeOpenWebUI = parseBool(
|
|
process.env.OPENCLAW_DOCKER_ALL_INCLUDE_OPENWEBUI ?? process.env.INCLUDE_OPENWEBUI,
|
|
true,
|
|
);
|
|
const selectedLaneNamesRaw =
|
|
process.env.OPENCLAW_DOCKER_ALL_LANES || process.env.DOCKER_E2E_LANES || "";
|
|
const selectedLaneNames = parseLaneSelection(selectedLaneNamesRaw);
|
|
if (selectedLaneNamesRaw && selectedLaneNames.length === 0) {
|
|
throw new Error("OPENCLAW_DOCKER_ALL_LANES must include at least one lane name");
|
|
}
|
|
const liveMode = parseLiveMode(process.env.OPENCLAW_DOCKER_ALL_LIVE_MODE);
|
|
const liveRetries = parseNonNegativeInt(
|
|
process.env.OPENCLAW_DOCKER_ALL_LIVE_RETRIES,
|
|
DEFAULT_LIVE_RETRIES,
|
|
"OPENCLAW_DOCKER_ALL_LIVE_RETRIES",
|
|
);
|
|
const timingsFile = path.resolve(
|
|
process.env.OPENCLAW_DOCKER_ALL_TIMINGS_FILE || DEFAULT_TIMINGS_FILE,
|
|
);
|
|
const runId = process.env.OPENCLAW_DOCKER_ALL_RUN_ID || utcStampForPath();
|
|
const logDir = path.resolve(
|
|
process.env.OPENCLAW_DOCKER_ALL_LOG_DIR ||
|
|
path.join(ROOT_DIR, ".artifacts/docker-tests", runId),
|
|
);
|
|
|
|
const baseEnv = commandEnv({
|
|
OPENCLAW_DOCKER_E2E_BARE_IMAGE:
|
|
process.env.OPENCLAW_DOCKER_E2E_BARE_IMAGE ||
|
|
process.env.OPENCLAW_DOCKER_E2E_IMAGE ||
|
|
DEFAULT_E2E_BARE_IMAGE,
|
|
OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE:
|
|
process.env.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE ||
|
|
process.env.OPENCLAW_DOCKER_E2E_IMAGE ||
|
|
DEFAULT_E2E_FUNCTIONAL_IMAGE,
|
|
});
|
|
baseEnv.OPENCLAW_DOCKER_E2E_IMAGE =
|
|
process.env.OPENCLAW_DOCKER_E2E_IMAGE || baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE;
|
|
const writeSummary = (summary: RunSummary) => writeRunSummary(logDir, summary, baseEnv);
|
|
appendExtension(baseEnv, "matrix");
|
|
appendExtension(baseEnv, "acpx");
|
|
appendExtension(baseEnv, "codex");
|
|
|
|
const timingStore = await loadTimingStore(timingsFile, timingsEnabled);
|
|
const { omittedUnsupportedLaneNames, orderedLanes, orderedTailLanes, plan, scheduledLanes } =
|
|
resolveDockerE2ePlan({
|
|
includeOpenWebUI,
|
|
liveMode,
|
|
liveRetries,
|
|
orderLanes,
|
|
planReleaseAll: planJson && planReleaseAll,
|
|
profile,
|
|
releaseChunk,
|
|
releaseProfile,
|
|
selectedLaneNames,
|
|
timingStore,
|
|
upgradeSurvivorBaselines: process.env.OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS,
|
|
upgradeSurvivorScenarios: process.env.OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS,
|
|
upgradeSurvivorTargetRoot: process.env.OPENCLAW_UPGRADE_SURVIVOR_TARGET_ROOT,
|
|
allowFrozenTargetScenarioOmissions,
|
|
candidatePackageRoot: ROOT_DIR,
|
|
});
|
|
if (omittedUnsupportedLaneNames.length > 0 && !allowFrozenTargetScenarioOmissions) {
|
|
throw new Error(
|
|
`frozen target scenario omissions require trusted workflow opt-in: ${omittedUnsupportedLaneNames.join(", ")}`,
|
|
);
|
|
}
|
|
if (scheduledLanes.length === 0 && omittedUnsupportedLaneNames.length === 0) {
|
|
throw new Error(
|
|
[
|
|
"resolved zero Docker lanes",
|
|
`profile=${profile}`,
|
|
`releaseChunk=${releaseChunk || "<none>"}`,
|
|
`releaseProfile=${releaseProfile}`,
|
|
`liveMode=${liveMode}`,
|
|
`includeOpenWebUI=${includeOpenWebUI ? "1" : "0"}`,
|
|
`selectedLanes=${selectedLaneNames.length > 0 ? selectedLaneNames.join(",") : "<none>"}`,
|
|
].join("; "),
|
|
);
|
|
}
|
|
const omittedUnsupportedLanes =
|
|
omittedUnsupportedLaneNames.length > 0 ? omittedUnsupportedLaneNames : undefined;
|
|
|
|
if (planJson) {
|
|
process.stdout.write(`${JSON.stringify(plan, null, 2)}\n`);
|
|
return;
|
|
}
|
|
if (cliOptions.prepareOnly) {
|
|
await prepareDockerCandidate(plan, logDir, path.resolve(cliOptions.prepareOnly));
|
|
return;
|
|
}
|
|
|
|
await mkdir(logDir, { recursive: true });
|
|
console.log(`==> Docker test logs: ${logDir}`);
|
|
console.log(`==> Profile: ${profile}${releaseChunk ? ` chunk=${releaseChunk}` : ""}`);
|
|
if (profile === RELEASE_PATH_PROFILE) {
|
|
console.log(`==> Release profile: ${releaseProfile}`);
|
|
}
|
|
console.log(`==> Parallelism: ${parallelism}`);
|
|
console.log(`==> Tail parallelism: ${tailParallelism}`);
|
|
console.log(`==> Lane timeout: ${laneTimeoutMs}ms`);
|
|
console.log(`==> Live mode: ${liveMode}`);
|
|
console.log(`==> Live retries: ${liveRetries}`);
|
|
console.log(`==> Lane start stagger: ${laneStartStaggerMs}ms`);
|
|
console.log(`==> Status interval: ${statusIntervalMs}ms`);
|
|
console.log(`==> Fail fast: ${failFast ? "yes" : "no"}`);
|
|
console.log(`==> Dry run: ${dryRun ? "yes" : "no"}`);
|
|
console.log(
|
|
`==> Docker preflight: ${preflightEnabled ? "yes" : "no"}${
|
|
preflightCleanup ? " cleanup=yes" : " cleanup=no"
|
|
}`,
|
|
);
|
|
console.log(`==> Build shared Docker images: ${buildEnabled ? "yes" : "no"}`);
|
|
console.log(`==> Docker E2E bare image: ${baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE}`);
|
|
console.log(`==> Docker E2E functional image: ${baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}`);
|
|
if (profile === RELEASE_PATH_PROFILE) {
|
|
console.log(`==> Include Open WebUI: ${includeOpenWebUI ? "yes" : "no"}`);
|
|
}
|
|
if (selectedLaneNames.length > 0) {
|
|
console.log(`==> Selected lanes: ${selectedLaneNames.join(", ")}`);
|
|
}
|
|
console.log(`==> Docker lane timings: ${timingStore.enabled ? timingsFile : "disabled"}`);
|
|
console.log(`==> Live-test bundled plugins: ${baseEnv.OPENCLAW_DOCKER_BUILD_EXTENSIONS}`);
|
|
const schedulerOptions = parseSchedulerOptions(process.env, parallelism);
|
|
const tailSchedulerOptions = parseSchedulerOptions(process.env, tailParallelism);
|
|
console.log(
|
|
`==> Scheduler: weight=${schedulerOptions.weightLimit} ${resourceLimitsSummary(schedulerOptions.resourceLimits)}`,
|
|
);
|
|
console.log(
|
|
`==> Tail scheduler: weight=${tailSchedulerOptions.weightLimit} ${resourceLimitsSummary(tailSchedulerOptions.resourceLimits)}`,
|
|
);
|
|
printLaneManifest("Main", orderedLanes, timingStore);
|
|
printLaneManifest("Tail", orderedTailLanes, timingStore);
|
|
if (omittedUnsupportedLanes) {
|
|
console.log(
|
|
`==> Docker lanes omitted: target lacks ${omittedUnsupportedLanes.join(", ")} scenario support`,
|
|
);
|
|
}
|
|
if (dryRun) {
|
|
console.log("==> Dry run complete");
|
|
return;
|
|
}
|
|
validateDockerCandidateEnvironment(baseEnv, plan);
|
|
|
|
// Planning can report unsupported scenarios, but execution cannot pass when
|
|
// frozen-target omissions leave no selected lane to run.
|
|
if (scheduledLanes.length === 0) {
|
|
await writeSummary({
|
|
chunk: releaseChunk || undefined,
|
|
failures: [],
|
|
lanes: [],
|
|
omittedUnsupportedLanes,
|
|
phases,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "failed",
|
|
});
|
|
throw new Error(
|
|
`resolved zero runnable Docker lanes; frozen target does not support: ${omittedUnsupportedLaneNames.join(", ")}`,
|
|
);
|
|
}
|
|
|
|
await runPhase(
|
|
phases,
|
|
"docker-preflight",
|
|
{ cleanup: preflightCleanup, enabled: preflightEnabled },
|
|
async () => {
|
|
await runDockerPreflight(baseEnv, {
|
|
cleanup: preflightCleanup,
|
|
enabled: preflightEnabled,
|
|
runTimeoutMs: preflightRunTimeoutMs,
|
|
});
|
|
},
|
|
);
|
|
if (lanesNeedOpenClawPackage(scheduledLanes)) {
|
|
await runPhase(phases, "prepare-openclaw-package", {}, async () => {
|
|
await prepareOpenClawPackage(baseEnv, logDir);
|
|
});
|
|
} else {
|
|
console.log("==> OpenClaw package: not needed for selected lanes");
|
|
}
|
|
|
|
if (buildEnabled) {
|
|
const buildEntries: ForegroundEntry[] = [];
|
|
if (scheduledLanes.some((poolLane) => poolLane.needsLiveImage)) {
|
|
buildEntries.push({
|
|
command: liveDockerHarnessScriptCommand("test-live-build-docker.sh"),
|
|
label: "shared live-test image once",
|
|
phaseDetails: { imageKind: "live" },
|
|
phases,
|
|
});
|
|
}
|
|
if (lanesNeedE2eImageKind(scheduledLanes, "bare")) {
|
|
buildEntries.push({
|
|
command: "pnpm test:docker:e2e-build",
|
|
env: {
|
|
OPENCLAW_DOCKER_E2E_IMAGE: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
|
|
OPENCLAW_DOCKER_E2E_TARGET: "bare",
|
|
},
|
|
label: `shared bare Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE}`,
|
|
phaseDetails: { image: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE, imageKind: "bare" },
|
|
phases,
|
|
});
|
|
}
|
|
if (lanesNeedE2eImageKind(scheduledLanes, "functional")) {
|
|
buildEntries.push({
|
|
command: "pnpm test:docker:e2e-build",
|
|
env: {
|
|
OPENCLAW_DOCKER_E2E_IMAGE: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
|
|
OPENCLAW_DOCKER_E2E_TARGET: "functional",
|
|
},
|
|
label: `shared functional Docker E2E image once: ${baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE}`,
|
|
phaseDetails: {
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
|
|
imageKind: "functional",
|
|
},
|
|
phases,
|
|
});
|
|
}
|
|
await runForegroundGroup(buildEntries, baseEnv);
|
|
} else {
|
|
console.log(`==> Shared Docker image builds: skipped`);
|
|
}
|
|
|
|
const options = {
|
|
...schedulerOptions,
|
|
failFast,
|
|
poolLabel: "main",
|
|
startStaggerMs: laneStartStaggerMs,
|
|
statusIntervalMs,
|
|
timeoutMs: laneTimeoutMs,
|
|
} satisfies LanePoolOptions;
|
|
const mainResult = await runPhase(phases, "main-lane-pool", { lanes: orderedLanes.length }, () =>
|
|
runLanePool(orderedLanes, baseEnv, logDir, parallelism, options),
|
|
);
|
|
const failures = [...mainResult.failures];
|
|
const allResults = [...mainResult.results];
|
|
await writeTimingStore(timingStore, mainResult.results);
|
|
if (failFast && failures.length > 0) {
|
|
await writeSummary({
|
|
chunk: releaseChunk || undefined,
|
|
failures,
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
images: {
|
|
bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
|
|
functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
|
|
},
|
|
lanes: allResults,
|
|
omittedUnsupportedLanes,
|
|
phases,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "failed",
|
|
});
|
|
await printFailureSummary(failures, tailLines);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (orderedTailLanes.length > 0) {
|
|
console.log("==> Running provider-sensitive Docker tail lanes");
|
|
const tailResult = await runPhase(
|
|
phases,
|
|
"tail-lane-pool",
|
|
{ lanes: orderedTailLanes.length },
|
|
() =>
|
|
runLanePool(orderedTailLanes, baseEnv, logDir, tailParallelism, {
|
|
...options,
|
|
...tailSchedulerOptions,
|
|
poolLabel: "tail",
|
|
}),
|
|
);
|
|
failures.push(...tailResult.failures);
|
|
allResults.push(...tailResult.results);
|
|
await writeTimingStore(timingStore, tailResult.results);
|
|
} else {
|
|
console.log("==> Provider-sensitive Docker tail lanes: none");
|
|
}
|
|
if (failures.length > 0) {
|
|
await writeSummary({
|
|
chunk: releaseChunk || undefined,
|
|
failures,
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
images: {
|
|
bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
|
|
functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
|
|
},
|
|
lanes: allResults,
|
|
omittedUnsupportedLanes,
|
|
phases,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "failed",
|
|
});
|
|
await printFailureSummary(failures, tailLines);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (profile === DEFAULT_PROFILE && selectedLaneNames.length === 0) {
|
|
const cleanupFailure = await runCleanupSmokePhase(baseEnv, logDir, phases);
|
|
if (cleanupFailure) {
|
|
failures.push(cleanupFailure);
|
|
}
|
|
} else {
|
|
console.log("==> Cleanup smoke after parallel lanes: skipped for selected/release lanes");
|
|
}
|
|
await writeTimingStore(timingStore, allResults);
|
|
if (failures.length > 0) {
|
|
await writeSummary({
|
|
chunk: releaseChunk || undefined,
|
|
failures,
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
images: {
|
|
bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
|
|
functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
|
|
},
|
|
lanes: allResults,
|
|
omittedUnsupportedLanes,
|
|
phases,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "failed",
|
|
});
|
|
await printFailureSummary(failures, tailLines);
|
|
process.exit(1);
|
|
}
|
|
await writeSummary({
|
|
chunk: releaseChunk || undefined,
|
|
failures,
|
|
image: baseEnv.OPENCLAW_DOCKER_E2E_IMAGE,
|
|
images: {
|
|
bare: baseEnv.OPENCLAW_DOCKER_E2E_BARE_IMAGE,
|
|
functional: baseEnv.OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE,
|
|
},
|
|
lanes: allResults,
|
|
omittedUnsupportedLanes,
|
|
phases,
|
|
profile,
|
|
selectedLanes: selectedLaneNames.length > 0 ? selectedLaneNames : undefined,
|
|
startedAt: runStartedAt,
|
|
status: "passed",
|
|
});
|
|
console.log("==> Docker test suite passed");
|
|
}
|
|
|
|
if (IS_MAIN) {
|
|
await main().catch((error: unknown) => {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exit(1);
|
|
});
|
|
}
|