mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 17:53:39 +00:00
fix(test): Docker, e2e-instance, and CI tooling tests fail on loaded hosts when fixture children outlast polling deadlines (#162729)
Script-tooling tests for the SQLite reliability bench, CI workflow guards and git-owner auth fixture, Docker build helper, iOS node e2e, the OpenClaw e2e instance, Parallels smoke model, plugin SDK API diff, unchanged-plugin Docker update, Android Gradle runner, and Vitest worker-artifact verification polled files, ports, and PIDs against 500 ms to 15 s deadlines, and raced child readiness/exit against timers of up to 30 s, so a fixture child that outlasted the bound on a loaded host failed the test. Waits now await child IPC, inherited readiness pipes, durable fixture receipts, native exit/close promises, and existing read-admission deferreds; Python fixtures join their retained subprocess handles under the caller's lifetime guard. Suspended test bodies are bound to the Vitest signal and fixture roots are retained until native children settle; the tracked Docker build awaits its inherited pipe EOF through close before its TERM record is checked. Where no reaper handle is owned, one deadline-free, signal-bound foreign-PID check remains per file. ci-git-owner-settlement's native Windows Job cases keep main's bytes until a Windows host can prove them. No product source, test timeout, product timeout, or assertion meaning changed. Part of the polling audit from #162274. Proof on Blacksmith Testbox: delay probes fail the original bytes and pass these bytes, forced-abort probes reach cleanup, 20/20 standalone runs per touched file, 3/3 replays of each owning shard (core-tooling-12 on the rebased head), type-aware lint, base-aware timeout-race ratchet; Codex autoreview clean; ClawSweeper found no findings.
This commit is contained in:
Peter Steinberger
GitHub
parent
777421df55
commit
5c4e9aba92
12 changed files with 1060 additions and 661 deletions
|
|
@ -140,13 +140,11 @@ test/plugins/team-reports-http.gateway.test.ts 1
|
|||
test/scripts/ci-run-node-test-shard.test.ts 3
|
||||
test/scripts/compiler-input-snapshot.test.ts 1
|
||||
test/scripts/plugin-gateway-gauntlet.test.ts 3
|
||||
test/scripts/plugin-sdk-api-diff.test.ts 2
|
||||
test/scripts/run-tsgo.test.ts 2
|
||||
test/scripts/run-with-env.test.ts 1
|
||||
test/scripts/test-projects-build-admission.test.ts 7
|
||||
test/scripts/upstream-repository-advisories.test.ts 2
|
||||
test/scripts/vitest-fork-os-diagnostics.test.ts 1
|
||||
test/scripts/vitest-worker-artifacts.verification.test.ts 1
|
||||
test/slack-download-read-authority.integration.test.ts 2
|
||||
test/subagent-requester-owner.e2e.test.ts 2
|
||||
ui/src/e2e/chat-flow.catalog-bootstrap.e2e.test.ts 4
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import {
|
|||
openOpenClawStateDatabase,
|
||||
} from "../../src/state/openclaw-state-db.js";
|
||||
import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js";
|
||||
import { withinTest } from "../helpers/promise.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
import { toolingTsEntrypoints } from "./tooling-ts-runtime.test-support.js";
|
||||
|
||||
|
|
@ -63,16 +64,8 @@ function runProof(args: string[], env: NodeJS.ProcessEnv = {}) {
|
|||
|
||||
async function waitForChildReady(child: ChildProcess): Promise<void> {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("writer child did not become ready"));
|
||||
}, 10_000);
|
||||
const onMessage = (message: unknown) => {
|
||||
if (
|
||||
message &&
|
||||
typeof message === "object" &&
|
||||
(message as { kind?: unknown }).kind === "ready"
|
||||
) {
|
||||
if (message && typeof message === "object" && "kind" in message && message.kind === "ready") {
|
||||
cleanup();
|
||||
resolve();
|
||||
}
|
||||
|
|
@ -86,7 +79,6 @@ async function waitForChildReady(child: ChildProcess): Promise<void> {
|
|||
reject(new Error("writer child exited before ready"));
|
||||
};
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
child.off("message", onMessage);
|
||||
child.off("error", onError);
|
||||
child.off("exit", onExit);
|
||||
|
|
@ -102,10 +94,6 @@ async function waitForChildExit(child: ChildProcess): Promise<{
|
|||
signal: NodeJS.Signals | null;
|
||||
}> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("writer child did not exit after IPC disconnect"));
|
||||
}, 10_000);
|
||||
const onExit = (code: number | null, signal: NodeJS.Signals | null) => {
|
||||
cleanup();
|
||||
resolve({ code, signal });
|
||||
|
|
@ -115,7 +103,6 @@ async function waitForChildExit(child: ChildProcess): Promise<{
|
|||
reject(error);
|
||||
};
|
||||
const cleanup = () => {
|
||||
clearTimeout(timeout);
|
||||
child.off("exit", onExit);
|
||||
child.off("error", onError);
|
||||
};
|
||||
|
|
@ -493,7 +480,7 @@ if (isMainThread && !process.execArgv.includes("--no-concurrent-sparkplug")) {
|
|||
);
|
||||
});
|
||||
|
||||
it("stops the writer when its parent IPC channel disconnects", async () => {
|
||||
it("stops the writer when its parent IPC channel disconnects", async ({ signal }) => {
|
||||
const databasePath = path.join(
|
||||
tempDirs.make("openclaw-sqlite-reliability-test-"),
|
||||
"writer.sqlite",
|
||||
|
|
@ -510,15 +497,18 @@ if (isMainThread && !process.execArgv.includes("--no-concurrent-sparkplug")) {
|
|||
stdio: ["ignore", "ignore", "pipe", "ipc"],
|
||||
},
|
||||
);
|
||||
const exitPromise = waitForChildExit(child);
|
||||
// Readiness and exit are both owned by this child; the test signal owns the deadline.
|
||||
void exitPromise.catch(() => {});
|
||||
try {
|
||||
await waitForChildReady(child);
|
||||
const exitPromise = waitForChildExit(child);
|
||||
await withinTest(waitForChildReady(child), signal);
|
||||
child.disconnect();
|
||||
await expect(exitPromise).resolves.toEqual({ code: 0, signal: null });
|
||||
await expect(withinTest(exitPromise, signal)).resolves.toEqual({ code: 0, signal: null });
|
||||
} finally {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill();
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
await exitPromise;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ import { collectRuntimeImportClosure } from "../../scripts/lib/runtime-import-cl
|
|||
import { resolveRunVitestSpawnEnv } from "../../scripts/lib/vitest-process-env.mts";
|
||||
import { resolvePnpmRunner } from "../../scripts/pnpm-runner.mts";
|
||||
import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js";
|
||||
import { awaitGateBeforeSettlement, withinTest } from "../helpers/promise.js";
|
||||
import { createTempDirTracker, useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
import { createPrebuiltUiE2eVitestConfig } from "../vitest/vitest.ui-e2e-prebuilt.config.ts";
|
||||
import { uiE2eRealGatewayTestFiles } from "../vitest/vitest.ui-paths.mjs";
|
||||
|
|
@ -2919,11 +2920,9 @@ process.exit(JSON.parse(process.env.RECIPE_EXITS)[count] ?? 99);
|
|||
const fixtureDirs = createTempDirTracker();
|
||||
// oxlint-disable-next-line prefer-const -- Failure cleanup can run before the registry is started.
|
||||
let stopRegistry: (() => Promise<void>) | undefined;
|
||||
let readyTimeout: NodeJS.Timeout | undefined;
|
||||
// Timeout does not join the test body. Keep close and deletion in one hook,
|
||||
// outside afterEach, so a failed join cannot release the registry's files.
|
||||
onTestFinished(async () => {
|
||||
clearTimeout(readyTimeout);
|
||||
await stopRegistry?.();
|
||||
fixtureDirs.cleanup();
|
||||
});
|
||||
|
|
@ -3073,8 +3072,7 @@ server.listen(0, "127.0.0.1", () => {
|
|||
await registryClosed;
|
||||
};
|
||||
try {
|
||||
const port = await new Promise<number>((resolve, reject) => {
|
||||
readyTimeout = setTimeout(() => reject(new Error("fixture registry not ready")), 2_000);
|
||||
const ready = new Promise<number>((resolve, reject) => {
|
||||
registryServer.once("message", (message) => {
|
||||
if (typeof message !== "number") {
|
||||
reject(new Error("fixture registry sent an invalid port"));
|
||||
|
|
@ -3083,9 +3081,11 @@ server.listen(0, "127.0.0.1", () => {
|
|||
resolve(message);
|
||||
});
|
||||
registryServer.once("error", reject);
|
||||
void registryClosed.then(() => reject(new Error("fixture registry closed before ready")));
|
||||
});
|
||||
clearTimeout(readyTimeout);
|
||||
const port = await withinTest(
|
||||
awaitGateBeforeSettlement(ready, registryClosed, "fixture registry closed before ready"),
|
||||
signal,
|
||||
);
|
||||
signal.throwIfAborted();
|
||||
const registryUrl = `http://127.0.0.1:${port}`;
|
||||
writeFileSync(
|
||||
|
|
@ -3234,7 +3234,6 @@ server.listen(0, "127.0.0.1", () => {
|
|||
failures.unshift(error);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(readyTimeout);
|
||||
try {
|
||||
await stopRegistry();
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -22,10 +22,16 @@ import {
|
|||
import { basename, dirname, join } from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import { mainLanes } from "../../scripts/lib/docker-e2e-scenarios.mts";
|
||||
import { buildSystemdUnit } from "../../src/daemon/systemd-unit.js";
|
||||
import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js";
|
||||
import {
|
||||
fixtureReceiptClientSource,
|
||||
openFixtureReceiptChannel,
|
||||
type FixtureReceiptChannel,
|
||||
} from "../helpers/fixture-receipts.js";
|
||||
import { awaitGateBeforeSettlement, withinTest } from "../helpers/promise.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
import {
|
||||
copySurvivorCaptureClosure,
|
||||
|
|
@ -36,8 +42,69 @@ import {
|
|||
UPGRADE_SURVIVOR_PATHS_HELPER,
|
||||
} from "./upgrade-survivor-paths.test-support.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const pendingChildCompletions = new Set<Promise<unknown>>();
|
||||
const tempDirs = useAutoCleanupTempDirTracker((cleanup) =>
|
||||
afterEach(async () => {
|
||||
// Vitest starts afterEach without joining a timed-out body's async finally.
|
||||
// Keep stop-policy scripts and child state until their native owners settle.
|
||||
await Promise.allSettled(pendingChildCompletions);
|
||||
cleanup();
|
||||
}),
|
||||
);
|
||||
const testNodeExecPath = resolveTestNodeExecPath();
|
||||
let receipts: FixtureReceiptChannel;
|
||||
beforeAll(async () => {
|
||||
receipts = await openFixtureReceiptChannel();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await receipts?.close();
|
||||
});
|
||||
|
||||
function ownChildCompletion<T>(completion: Promise<T>): Promise<T> {
|
||||
const owned = completion.finally(() => {
|
||||
pendingChildCompletions.delete(owned);
|
||||
});
|
||||
pendingChildCompletions.add(owned);
|
||||
return owned;
|
||||
}
|
||||
|
||||
function writeFixtureReceiptReporter(workDir: string): string {
|
||||
const reporter = join(workDir, "fixture-receipt.mjs");
|
||||
writeFileSync(
|
||||
reporter,
|
||||
`${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
import fs from "node:fs";
|
||||
const [source, line] = process.argv.slice(2);
|
||||
fs.appendFileSync(source, line + "\\n");
|
||||
sendReceipt(source, line);
|
||||
fixtureReceiptSocket.ref();
|
||||
fixtureReceiptSocket.end();
|
||||
`,
|
||||
);
|
||||
return reporter;
|
||||
}
|
||||
|
||||
function fixtureEventBeforeSettlement(
|
||||
source: string,
|
||||
text: string,
|
||||
operation: PromiseLike<unknown>,
|
||||
message: string,
|
||||
): Promise<void> {
|
||||
const recorded = () => existsSync(source) && readFileSync(source, "utf8").includes(text);
|
||||
const settled = Promise.resolve(operation).then(
|
||||
() => {
|
||||
if (!recorded()) {
|
||||
throw new Error(message);
|
||||
}
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (!recorded()) {
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
);
|
||||
return Promise.race([receipts.waitFor(source, text), settled]);
|
||||
}
|
||||
|
||||
const PACKAGE_BUILDER_NODE_SCRIPT = `node() {
|
||||
local script="$1"
|
||||
|
|
@ -203,6 +270,8 @@ function containerCleanupFixture(scenario: string) {
|
|||
const log = join(temp, "runner log");
|
||||
const eventsPath = join(root, "events.jsonl");
|
||||
const pidPath = join(root, "docker.pid");
|
||||
const readyPath = join(root, "docker.ready");
|
||||
const reporter = writeFixtureReceiptReporter(root);
|
||||
const stdinPath = join(root, "stdin");
|
||||
mkdirSync(temp);
|
||||
writeFileSync(join(root, "retained-evidence"), "keep evidence");
|
||||
|
|
@ -263,6 +332,7 @@ console.log("fixture container output");
|
|||
if (scenario === "signal") {
|
||||
fs.writeFileSync(process.env.FIXTURE_PID, String(process.pid));
|
||||
process.on("SIGTERM", () => process.exit(143));
|
||||
require("node:child_process").spawn(process.execPath, [${JSON.stringify(reporter)}, ${JSON.stringify(readyPath)}, "ready"], { stdio: "ignore" });
|
||||
setInterval(() => {}, 1000);
|
||||
} else {
|
||||
console.log("Tests 4 passed");
|
||||
|
|
@ -292,7 +362,7 @@ if (scenario === "signal") {
|
|||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
return { root, temp, log, env, events, pidPath };
|
||||
return { root, temp, log, env, events, pidPath, readyPath };
|
||||
}
|
||||
|
||||
function expectContainerCleanup(
|
||||
|
|
@ -437,6 +507,37 @@ function spawnDockerSnippet(
|
|||
return spawnSync(...prepareDockerSnippet(script, options, args));
|
||||
}
|
||||
|
||||
async function runDockerSnippet(script: string, signal: AbortSignal): Promise<void> {
|
||||
const child = spawn("/bin/bash", ["--noprofile", "--norc", "-c", script], {
|
||||
detached: true,
|
||||
env: { ...process.env, BASH_ENV: "", ENV: "" },
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
let output = "";
|
||||
child.stdout.on("data", (chunk: Buffer) => (output += chunk.toString()));
|
||||
child.stderr.on("data", (chunk: Buffer) => (output += chunk.toString()));
|
||||
let closed = false;
|
||||
const completion = ownChildCompletion(
|
||||
new Promise<number | null>((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code) => {
|
||||
closed = true;
|
||||
resolve(code);
|
||||
});
|
||||
}),
|
||||
);
|
||||
try {
|
||||
expect(await withinTest(completion, signal), output).toBe(0);
|
||||
} finally {
|
||||
if (!closed && child.pid) {
|
||||
try {
|
||||
process.kill(-child.pid, "SIGKILL");
|
||||
} catch {}
|
||||
await completion;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function writeExecutables(directory: string, files: Record<string, string>): void {
|
||||
mkdirSync(directory, { recursive: true });
|
||||
for (const [name, contents] of Object.entries(files)) {
|
||||
|
|
@ -504,20 +605,28 @@ function installUpgradeSurvivorSystemctlShim(
|
|||
return join(prefix, "bin", "systemctl");
|
||||
}
|
||||
|
||||
async function waitForProcessExit(child: ChildProcess, timeoutMs = 5_000): Promise<number | null> {
|
||||
function waitForProcessExit(child: ChildProcess): Promise<number | null> {
|
||||
if (child.exitCode !== null || child.signalCode !== null) {
|
||||
return child.exitCode;
|
||||
return Promise.resolve(child.exitCode);
|
||||
}
|
||||
return ownChildCompletion(
|
||||
new Promise<number | null>((resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("exit", resolve);
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// The watchdog's escalation callback exits before reaping its foreign child.
|
||||
// Keep this exact-PID observation tied to the test lifetime, without another deadline.
|
||||
async function waitForForeignProcessExit(pid: number, signal: AbortSignal): Promise<void> {
|
||||
try {
|
||||
while (isProcessRunning(pid)) {
|
||||
await delay(10, undefined, { signal });
|
||||
}
|
||||
} catch (cause) {
|
||||
throw new Error(`process stayed alive: ${pid}`, { cause });
|
||||
}
|
||||
return await new Promise<number | null>((resolve, reject) => {
|
||||
const timeout = setTimeout(() => {
|
||||
child.kill("SIGKILL");
|
||||
reject(new Error("process did not exit before its test deadline"));
|
||||
}, timeoutMs);
|
||||
child.once("exit", (code) => {
|
||||
clearTimeout(timeout);
|
||||
resolve(code);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function isProcessRunning(pid: number): boolean {
|
||||
|
|
@ -559,6 +668,7 @@ process.on("SIGTERM", () => {});
|
|||
const pendingPid = process.env.DESCENDANT_PID_FILE + ".pending";
|
||||
fs.writeFileSync(pendingPid, String(process.pid));
|
||||
fs.renameSync(pendingPid, process.env.DESCENDANT_PID_FILE);
|
||||
process.send?.({ kind: "ready", pid: process.pid });
|
||||
setInterval(() => {}, 1_000);
|
||||
`,
|
||||
);
|
||||
|
|
@ -566,6 +676,7 @@ setInterval(() => {}, 1_000);
|
|||
}
|
||||
|
||||
async function forEachUpgradeSurvivorSystemctlShim(
|
||||
signal: AbortSignal,
|
||||
callback: (fixture: {
|
||||
pid: number;
|
||||
pidPath: string;
|
||||
|
|
@ -581,12 +692,17 @@ async function forEachUpgradeSurvivorSystemctlShim(
|
|||
const childPidPath = join(workDir, "child.pid");
|
||||
const child = spawn(process.execPath, [writeTermIgnoringDescendant(workDir)], {
|
||||
env: { ...process.env, DESCENDANT_PID_FILE: childPidPath },
|
||||
stdio: "ignore",
|
||||
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
||||
});
|
||||
const exited = waitForProcessExit(child);
|
||||
const ready = new Promise<void>((resolve) => {
|
||||
child.once("message", () => resolve());
|
||||
});
|
||||
try {
|
||||
for (let attempt = 0; attempt < 100 && !existsSync(childPidPath); attempt += 1) {
|
||||
await delay(10);
|
||||
}
|
||||
await withinTest(
|
||||
awaitGateBeforeSettlement(ready, exited, `file was not written: ${childPidPath}`),
|
||||
signal,
|
||||
);
|
||||
const pid = Number.parseInt(readFileSync(childPidPath, "utf8"), 10);
|
||||
writeFileSync(pidPath, `${pid}\n`);
|
||||
const daemonLog = join(workDir, "gateway.log");
|
||||
|
|
@ -1237,7 +1353,7 @@ grep -q '^build --progress=plain --build-arg GITHUB_ACTIONS -t demo-image .$' "$
|
|||
execDockerSnippet(script);
|
||||
});
|
||||
|
||||
it("stops the tracked build command without retrying when interrupted", async () => {
|
||||
it("stops the tracked build command without retrying when interrupted", async ({ signal }) => {
|
||||
const workDir = tempDirs.make("openclaw-docker-build-signal-");
|
||||
writeExecutables(join(workDir, "bin"), {
|
||||
docker: `#!/bin/bash
|
||||
|
|
@ -1253,6 +1369,7 @@ printf 'rpc error: code = Unavailable\\n'
|
|||
trap 'printf "term\\n" >"$TMPDIR/docker.term"; exit 0' TERM
|
||||
mkfifo "$TMPDIR/docker.block"
|
||||
printf 'ready\\n' >"$TMPDIR/docker.ready"
|
||||
printf 'ready\\n' >&3
|
||||
while true; do
|
||||
read -r -t 1 _ <> "$TMPDIR/docker.block" || true
|
||||
done
|
||||
|
|
@ -1272,31 +1389,7 @@ docker_build_run e2e-build -t demo-image .
|
|||
`,
|
||||
});
|
||||
|
||||
const waitForFile = async (filePath: string) => {
|
||||
for (let attempt = 0; attempt < 500; attempt += 1) {
|
||||
if (existsSync(filePath)) {
|
||||
return;
|
||||
}
|
||||
await delay(10);
|
||||
}
|
||||
throw new Error(`file was not written: ${filePath}`);
|
||||
};
|
||||
const waitForExit = async (child: ReturnType<typeof spawn>) =>
|
||||
await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => {
|
||||
child.once("exit", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const waitForDead = async (pid: number) => {
|
||||
for (let attempt = 0; attempt < 500; attempt += 1) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
await delay(10);
|
||||
}
|
||||
throw new Error(`process stayed alive: ${pid}`);
|
||||
};
|
||||
const runInterruptedBuild = async (signal: NodeJS.Signals, expectedCode: number) => {
|
||||
const runInterruptedBuild = async (childSignal: NodeJS.Signals, expectedCode: number) => {
|
||||
rmSync(join(workDir, "docker.pid"), { force: true });
|
||||
rmSync(join(workDir, "docker.term"), { force: true });
|
||||
rmSync(join(workDir, "docker.ready"), { force: true });
|
||||
|
|
@ -1304,25 +1397,44 @@ docker_build_run e2e-build -t demo-image .
|
|||
rmSync(join(workDir, "docker-count"), { force: true });
|
||||
const runner = spawn(join(workDir, "runner.sh"), {
|
||||
env: { ...process.env, TMPDIR: workDir },
|
||||
stdio: "ignore",
|
||||
stdio: ["ignore", "ignore", "ignore", "pipe"],
|
||||
});
|
||||
const exited = waitForProcessExit(runner);
|
||||
const closed = ownChildCompletion(
|
||||
new Promise<number | null>((resolve, reject) => {
|
||||
runner.once("error", reject);
|
||||
runner.once("close", resolve);
|
||||
}),
|
||||
);
|
||||
const ready = new Promise<void>((resolve, reject) => {
|
||||
runner.stdio[3]!.once("data", () => resolve());
|
||||
runner.stdio[3]!.once("error", reject);
|
||||
});
|
||||
try {
|
||||
const pidPath = join(workDir, "docker.pid");
|
||||
await waitForFile(pidPath);
|
||||
await waitForFile(join(workDir, "docker.ready"));
|
||||
// A builtin FD write leaves no foreground reporter that tree shutdown could
|
||||
// kill first, making errexit bypass the fixture's own TERM trap.
|
||||
await withinTest(
|
||||
awaitGateBeforeSettlement(ready, exited, `file was not written: ${pidPath}`),
|
||||
signal,
|
||||
);
|
||||
expect(existsSync(join(workDir, "docker.ready"))).toBe(true);
|
||||
const buildPid = Number.parseInt(readFileSync(pidPath, "utf8"), 10);
|
||||
|
||||
runner.kill(signal);
|
||||
const exit = await waitForExit(runner);
|
||||
|
||||
expect(exit).toEqual({ code: expectedCode, signal: null });
|
||||
await waitForFile(join(workDir, "docker.term"));
|
||||
runner.kill(childSignal);
|
||||
expect(await withinTest(closed, signal)).toBe(expectedCode);
|
||||
expect(runner.signalCode).toBeNull();
|
||||
// The timeout wrapper can exit first; the Docker fixture holds FD 3 until
|
||||
// its TERM trap finishes, so runner close also joins that fixture's output.
|
||||
expect(existsSync(join(workDir, "docker.term"))).toBe(true);
|
||||
expect(readFileSync(join(workDir, "docker-count"), "utf8").trim()).toBe("1");
|
||||
await waitForDead(buildPid);
|
||||
await waitForForeignProcessExit(buildPid, signal);
|
||||
expect(isProcessRunning(buildPid)).toBe(false);
|
||||
} finally {
|
||||
if (runner.exitCode === null && runner.signalCode === null) {
|
||||
runner.kill("SIGKILL");
|
||||
runner.kill("SIGTERM");
|
||||
}
|
||||
await closed;
|
||||
}
|
||||
};
|
||||
|
||||
|
|
@ -1454,7 +1566,7 @@ stdout="$(<"$TMPDIR/stdout")"
|
|||
execDockerSnippet(script);
|
||||
});
|
||||
|
||||
it.each([
|
||||
it.for([
|
||||
{
|
||||
title: "keeps reused Docker image probes behind the timeout-aware helper",
|
||||
tempPrefix: "openclaw-docker-image-reuse-timeout-",
|
||||
|
|
@ -1827,32 +1939,24 @@ export OPENCLAW_DOCKER_E2E_HEARTBEAT_TERM_GRACE_SECONDS=1
|
|||
source "$ROOT_DIR/scripts/lib/docker-e2e-logs.sh"
|
||||
|
||||
command_pid_file="$TMPDIR/command.pid"
|
||||
mkfifo "$TMPDIR/ready.pipe"
|
||||
(
|
||||
run_logged_print_heartbeat plugins-run 30 bash -c 'trap "exit 0" TERM; printf "%s" "$$" > "$1"; while true; do /bin/sleep 0.05; done' bash "$command_pid_file"
|
||||
) &
|
||||
run_logged_print_heartbeat plugins-run 30 bash -c 'trap "exit 0" TERM; printf "%s" "$$" > "$1"; printf "ready\\n" >&3; while true; do /bin/sleep 0.05; done' bash "$command_pid_file"
|
||||
) 3>"$TMPDIR/ready.pipe" &
|
||||
wrapper_pid="$!"
|
||||
for _ in $(seq 1 100); do
|
||||
[ -s "$command_pid_file" ] && break
|
||||
/bin/sleep 0.01
|
||||
done
|
||||
if [ ! -s "$command_pid_file" ]; then
|
||||
if ! IFS= read -r ready <"$TMPDIR/ready.pipe"; then
|
||||
kill -TERM "$wrapper_pid" 2>/dev/null || true
|
||||
echo "heartbeat command pid was not recorded" >&2
|
||||
exit 1
|
||||
fi
|
||||
command_pid="$(cat "$command_pid_file")"
|
||||
kill -TERM "$wrapper_pid"
|
||||
for _ in $(seq 1 50); do
|
||||
if ! kill -0 "$command_pid" 2>/dev/null; then
|
||||
wait "$wrapper_pid" 2>/dev/null || true
|
||||
exit 0
|
||||
fi
|
||||
/bin/sleep 0.01
|
||||
done
|
||||
kill -TERM "$command_pid" 2>/dev/null || true
|
||||
kill -TERM "$wrapper_pid" 2>/dev/null || true
|
||||
echo "heartbeat command still alive after wrapper termination: $command_pid" >&2
|
||||
exit 1
|
||||
wait "$wrapper_pid" 2>/dev/null || true
|
||||
# cleanup_heartbeat_command joins the exact command before the wrapper returns.
|
||||
if kill -0 "$command_pid" 2>/dev/null; then
|
||||
echo "heartbeat command still alive after wrapper termination: $command_pid" >&2
|
||||
exit 1
|
||||
fi
|
||||
`,
|
||||
},
|
||||
{
|
||||
|
|
@ -1892,26 +1996,21 @@ docker() {
|
|||
printf "started\\n" >"$TMPDIR/docker-started"
|
||||
printf "docker running\\n"
|
||||
trap 'exit 143' TERM
|
||||
printf "ready\\n" >&3
|
||||
while true; do /bin/sleep 0.05; done
|
||||
}
|
||||
export -f docker
|
||||
|
||||
mkfifo "$TMPDIR/ready.pipe"
|
||||
(
|
||||
docker_e2e_run_logged_print_with_harness plugins-run image-name bash -lc true
|
||||
) &
|
||||
) 3>"$TMPDIR/ready.pipe" &
|
||||
wrapper_pid="$!"
|
||||
for _ in $(seq 1 50); do
|
||||
[ -s "$TMPDIR/docker-started" ] && break
|
||||
/bin/sleep 0.01
|
||||
kill -0 "$wrapper_pid" 2>/dev/null || true
|
||||
done
|
||||
IFS= read -r ready <"$TMPDIR/ready.pipe"
|
||||
test -s "$TMPDIR/docker-started"
|
||||
kill -TERM "$wrapper_pid" 2>/dev/null || true
|
||||
wait "$wrapper_pid" 2>/dev/null || true
|
||||
for _ in $(seq 1 50); do
|
||||
grep -qx "container-term" "$TMPDIR/docker-rm-seen" 2>/dev/null && break
|
||||
/bin/sleep 0.01
|
||||
done
|
||||
# The joined harness removes its container before the heartbeat wrapper returns.
|
||||
grep -qx "container-term" "$TMPDIR/docker-rm-seen"
|
||||
test -z "$(find "$TMPDIR" -maxdepth 1 -name 'openclaw-docker-e2e-container.*' -print)"
|
||||
`,
|
||||
|
|
@ -1960,11 +2059,11 @@ heartbeat_elapsed="\${BASH_REMATCH[1]}"
|
|||
[[ -s "$stats_log" ]]
|
||||
`,
|
||||
},
|
||||
])("$title", ({ tempPrefix, scriptSource }) => {
|
||||
])("$title", async ({ tempPrefix, scriptSource }, { signal }) => {
|
||||
const workDir = tempDirs.make(tempPrefix);
|
||||
const script = scriptSource(workDir);
|
||||
|
||||
execDockerSnippet(script);
|
||||
await runDockerSnippet(script, signal);
|
||||
});
|
||||
|
||||
it("derives the browser CDP image from the shared functional image", () => {
|
||||
|
|
@ -2255,14 +2354,19 @@ OPENCLAW_DOCKER_E2E_DISABLE_RESOURCE_LIMITS=1 docker_e2e_docker_cmd run demo
|
|||
["TERM", "143"],
|
||||
["HUP", "129"],
|
||||
] as const) {
|
||||
it(`escalates Docker watchdog children that ignore parent SIG${shellSignal}`, () => {
|
||||
it(`escalates Docker watchdog children that ignore parent SIG${shellSignal}`, async ({
|
||||
signal,
|
||||
}) => {
|
||||
const workDir = tempDirs.make("openclaw-docker-node-signal-");
|
||||
const reporter = writeFixtureReceiptReporter(workDir);
|
||||
const readyPath = join(workDir, "ready");
|
||||
writeExecutables(join(workDir, "bin"), {
|
||||
node: `#!/bin/bash\nexec ${shellQuote(process.execPath)} "$@"\n`,
|
||||
docker: `#!/bin/bash
|
||||
trap "" TERM HUP
|
||||
printf "%s\\n" "$$" >"$TMPDIR/docker-pid"
|
||||
printf "%s\\n" "$PPID" >"$TMPDIR/watchdog-pid"
|
||||
${shellQuote(process.execPath)} ${shellQuote(reporter)} ${shellQuote(readyPath)} ready
|
||||
while true; do /bin/sleep 1; done
|
||||
`,
|
||||
});
|
||||
|
|
@ -2276,30 +2380,44 @@ export OPENCLAW_DOCKER_TIMEOUT_KILL_GRACE_MS=100
|
|||
|
||||
source "$ROOT_DIR/scripts/lib/docker-e2e-container.sh"
|
||||
|
||||
docker_e2e_docker_cmd run demo &
|
||||
watchdog_pid="$!"
|
||||
for ((i = 0; i < 100; i += 1)); do
|
||||
[ -s "$TMPDIR/docker-pid" ] && [ -s "$TMPDIR/watchdog-pid" ] && break
|
||||
/bin/sleep 0.02
|
||||
done
|
||||
[ -s "$TMPDIR/docker-pid" ]
|
||||
[ -s "$TMPDIR/watchdog-pid" ]
|
||||
kill -${shellSignal} "$(/bin/cat "$TMPDIR/watchdog-pid")"
|
||||
set +e
|
||||
wait "$watchdog_pid"
|
||||
status="$?"
|
||||
set -e
|
||||
[ "$status" = "${expectedStatus}" ]
|
||||
docker_pid="$(/bin/cat "$TMPDIR/docker-pid")"
|
||||
for ((i = 0; i < 100; i += 1)); do
|
||||
kill -0 "$docker_pid" 2>/dev/null || exit 0
|
||||
/bin/sleep 0.02
|
||||
done
|
||||
echo "docker child still alive after watchdog termination" >&2
|
||||
exit 1
|
||||
docker_e2e_docker_cmd run demo
|
||||
|
||||
`;
|
||||
|
||||
execDockerSnippet(script);
|
||||
const runner = spawn("/bin/bash", ["--noprofile", "--norc", "-c", script], {
|
||||
env: { ...process.env, BASH_ENV: "", ENV: "" },
|
||||
stdio: "ignore",
|
||||
});
|
||||
const exited = waitForProcessExit(runner);
|
||||
try {
|
||||
await withinTest(
|
||||
fixtureEventBeforeSettlement(
|
||||
readyPath,
|
||||
"ready",
|
||||
exited,
|
||||
"docker child PID was not recorded",
|
||||
),
|
||||
signal,
|
||||
);
|
||||
const watchdogPid = Number(readFileSync(join(workDir, "watchdog-pid"), "utf8"));
|
||||
const dockerPid = Number(readFileSync(join(workDir, "docker-pid"), "utf8"));
|
||||
process.kill(watchdogPid, `SIG${shellSignal}`);
|
||||
expect(await withinTest(exited, signal)).toBe(Number(expectedStatus));
|
||||
await waitForForeignProcessExit(dockerPid, signal);
|
||||
} finally {
|
||||
for (const name of ["watchdog-pid", "docker-pid"]) {
|
||||
const path = join(workDir, name);
|
||||
if (existsSync(path)) {
|
||||
try {
|
||||
process.kill(Number(readFileSync(path, "utf8")), "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
if (runner.exitCode === null && runner.signalCode === null) {
|
||||
runner.kill("SIGKILL");
|
||||
}
|
||||
await exited;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -2653,14 +2771,15 @@ printf '%s\\n' "$seconds"
|
|||
}
|
||||
});
|
||||
|
||||
it("records an interrupted upgrade survivor phase as failed", async () => {
|
||||
it("records an interrupted upgrade survivor phase as failed", async ({ signal }) => {
|
||||
const workDir = tempDirs.make("openclaw-upgrade-survivor-signal-");
|
||||
const binDir = join(workDir, "bin");
|
||||
const markerPath = join(workDir, "npm-started");
|
||||
const reporter = writeFixtureReceiptReporter(workDir);
|
||||
const summaryPath = join(workDir, "artifacts", "summary.json");
|
||||
writeExecutables(binDir, {
|
||||
npm: `#!/bin/sh
|
||||
touch "$FAKE_NPM_MARKER"
|
||||
${shellQuote(process.execPath)} ${shellQuote(reporter)} "$FAKE_NPM_MARKER" ready
|
||||
exec sleep 300
|
||||
`,
|
||||
timeout: `#!/bin/sh
|
||||
|
|
@ -2691,20 +2810,28 @@ exec "$@"
|
|||
if (!childPid) {
|
||||
throw new Error("upgrade survivor process did not start");
|
||||
}
|
||||
const exitPromise = new Promise<{
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
}>((resolve) => {
|
||||
child.once("exit", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
const exitPromise = ownChildCompletion(
|
||||
new Promise<{
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
}>((resolve) => {
|
||||
child.once("exit", (code, childSignal) => resolve({ code, signal: childSignal }));
|
||||
}),
|
||||
);
|
||||
|
||||
try {
|
||||
for (let attempt = 0; attempt < 500 && !existsSync(markerPath); attempt += 1) {
|
||||
await delay(10);
|
||||
}
|
||||
await withinTest(
|
||||
fixtureEventBeforeSettlement(
|
||||
markerPath,
|
||||
"ready",
|
||||
exitPromise,
|
||||
"npm-started marker was not written",
|
||||
),
|
||||
signal,
|
||||
);
|
||||
expect(existsSync(markerPath)).toBe(true);
|
||||
process.kill(-childPid, "SIGTERM");
|
||||
const exit = await exitPromise;
|
||||
const exit = await withinTest(exitPromise, signal);
|
||||
|
||||
expect(exit).toEqual({ code: 143, signal: null });
|
||||
const diagnostics = JSON.parse(
|
||||
|
|
@ -2734,6 +2861,7 @@ exec "$@"
|
|||
if (child.exitCode === null && child.signalCode === null) {
|
||||
process.kill(-childPid, "SIGKILL");
|
||||
}
|
||||
await exitPromise;
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -2882,9 +3010,10 @@ fi
|
|||
]);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform !== "linux").each(["published", "current"])(
|
||||
it.skipIf(process.platform !== "linux").for(["published", "current"])(
|
||||
"starts the %s auth probe under the manager that owns its restart and stop",
|
||||
async (lane) => {
|
||||
{ timeout: 60_000 },
|
||||
async (lane, { signal }) => {
|
||||
const workDir = tempDirs.make("survivor-managed-probe-");
|
||||
const paths = readUpgradeSurvivorPaths(workDir);
|
||||
const artifacts = paths.artifactRoot;
|
||||
|
|
@ -2897,16 +3026,22 @@ fi
|
|||
const childPath = join(workDir, "listener.mjs");
|
||||
const startsPath = join(workDir, "starts.jsonl");
|
||||
const portPath = join(workDir, "port");
|
||||
const readyPipe = join(workDir, "ready.pipe");
|
||||
execFileSync("mkfifo", [readyPipe]);
|
||||
writeFileSync(
|
||||
childPath,
|
||||
`import fs from "node:fs";
|
||||
`${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
import fs from "node:fs";
|
||||
import http from "node:http";
|
||||
const identity = { pid: process.pid, managed: process.env.OPENCLAW_SYSTEMD_UNIT === "openclaw-gateway.service" };
|
||||
const server = http.createServer((_req, res) => res.end(JSON.stringify(identity)));
|
||||
const firstStart = !fs.existsSync(process.env.PORT_FILE);
|
||||
const port = fs.existsSync(process.env.PORT_FILE) ? Number(fs.readFileSync(process.env.PORT_FILE, "utf8")) : 0;
|
||||
server.listen(port, "127.0.0.1", () => {
|
||||
fs.writeFileSync(process.env.PORT_FILE, String(server.address().port));
|
||||
fs.appendFileSync(process.env.STARTS_FILE, JSON.stringify(identity) + "\\n");
|
||||
sendReceipt(process.env.STARTS_FILE, "ready");
|
||||
if (firstStart) fs.writeFileSync(process.env.READY_PIPE, "ready\\n");
|
||||
console.log("[gateway] ready on 127.0.0.1:" + server.address().port);
|
||||
});
|
||||
`,
|
||||
|
|
@ -2966,6 +3101,7 @@ process.on("SIGTERM", () => {
|
|||
LISTENER_SCRIPT: childPath,
|
||||
STARTS_FILE: startsPath,
|
||||
PORT_FILE: portPath,
|
||||
READY_PIPE: readyPipe,
|
||||
};
|
||||
const source = readFileSync(UPGRADE_SURVIVOR_RUN_SCRIPT, "utf8");
|
||||
const setup =
|
||||
|
|
@ -2990,7 +3126,7 @@ ${tcpProbeAdapter}
|
|||
# This fixture chooses an ephemeral port; retain the actual readiness implementation.
|
||||
eval "$(declare -f openclaw_e2e_wait_gateway_ready | sed '1s/openclaw_e2e_wait_gateway_ready/fixture_wait_gateway_ready/')"
|
||||
openclaw_e2e_wait_gateway_ready() {
|
||||
for _ in {1..200}; do [ -s "$PORT_FILE" ] && break; sleep 0.01; done
|
||||
IFS= read -r ready <"$READY_PIPE"
|
||||
fixture_wait_gateway_ready "$1" "$2" 20 "$(cat "$PORT_FILE")" "\${5:-strict}"
|
||||
}
|
||||
${lane === "published" ? "prepare_update_restart_probe" : 'prepare_update_restart_probe_current_install 18789 "$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG"'}
|
||||
|
|
@ -3033,9 +3169,7 @@ openclaw_e2e_probe_tcp 127.0.0.1 18789 400`,
|
|||
expect(isProcessRunning(records()[0]!.pid)).toBe(false);
|
||||
await expect(fetch(url, { signal: AbortSignal.timeout(1_000) })).rejects.toThrow();
|
||||
expect(systemctl("start", "openclaw-gateway.service").status).toBe(0);
|
||||
for (let attempt = 0; attempt < 200 && records().length < 2; attempt++) {
|
||||
await delay(10);
|
||||
}
|
||||
await withinTest(receipts.waitFor(startsPath, "ready", 2), signal);
|
||||
expect(records()).toHaveLength(2);
|
||||
}
|
||||
const initial = (await (
|
||||
|
|
@ -3044,9 +3178,7 @@ openclaw_e2e_probe_tcp 127.0.0.1 18789 400`,
|
|||
expect(initial.managed).toBe(true);
|
||||
expect(systemctl("restart", "openclaw-gateway.service").status).toBe(0);
|
||||
const expectedStarts = lane === "published" ? 3 : 2;
|
||||
for (let attempt = 0; attempt < 200 && records().length < expectedStarts; attempt++) {
|
||||
await delay(10);
|
||||
}
|
||||
await withinTest(receipts.waitFor(startsPath, "ready", expectedStarts), signal);
|
||||
expect(records()).toHaveLength(expectedStarts);
|
||||
const replacement = (await (
|
||||
await fetch(url, { signal: AbortSignal.timeout(1_000) })
|
||||
|
|
@ -3067,7 +3199,6 @@ openclaw_e2e_probe_tcp 127.0.0.1 18789 400`,
|
|||
}
|
||||
}
|
||||
},
|
||||
60_000,
|
||||
);
|
||||
|
||||
it("returns the gateway readiness failure when startup is called conditionally", () => {
|
||||
|
|
@ -3325,8 +3456,8 @@ printf '%s\n' "$status" >"$TMPDIR/status"
|
|||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"stops promptly when the systemctl target is a zombie with spaces and parentheses in comm",
|
||||
async () => {
|
||||
await forEachUpgradeSurvivorSystemctlShim(({ pid, run, readLog, scriptPath }) => {
|
||||
async ({ signal }) => {
|
||||
await forEachUpgradeSurvivorSystemctlShim(signal, ({ pid, run, readLog, scriptPath }) => {
|
||||
const procTail = Array.from({ length: 49 }, (_, field) => field + 1).join(" ");
|
||||
expect(run(`${pid} (gateway (old) worker) Z ${procTail}`, true), scriptPath).toBe(0);
|
||||
expect(readLog()).toEqual([
|
||||
|
|
@ -3341,20 +3472,23 @@ printf '%s\n' "$status" >"$TMPDIR/status"
|
|||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"waits for a killable systemctl target when proc stat is unreadable or malformed",
|
||||
async () => {
|
||||
await forEachUpgradeSurvivorSystemctlShim(({ pid, pidPath, run, readLog, scriptPath }) => {
|
||||
for (const procStat of [undefined, `${pid} (gateway) Z`]) {
|
||||
// Reaching the wait sentinel proves the shell did not mistake missing stat data for exit.
|
||||
expect(run(procStat), `${scriptPath}: ${procStat ?? "unreadable"}`).toBe(97);
|
||||
expect(readLog()).toEqual([
|
||||
"--user stop openclaw-gateway.service",
|
||||
"proc-stat-read",
|
||||
"wait",
|
||||
]);
|
||||
expect(isProcessRunning(pid)).toBe(true);
|
||||
expect(readFileSync(pidPath, "utf8")).toBe(`${pid}\n`);
|
||||
}
|
||||
});
|
||||
async ({ signal }) => {
|
||||
await forEachUpgradeSurvivorSystemctlShim(
|
||||
signal,
|
||||
({ pid, pidPath, run, readLog, scriptPath }) => {
|
||||
for (const procStat of [undefined, `${pid} (gateway) Z`]) {
|
||||
// Reaching the wait sentinel proves the shell did not mistake missing stat data for exit.
|
||||
expect(run(procStat), `${scriptPath}: ${procStat ?? "unreadable"}`).toBe(97);
|
||||
expect(readLog()).toEqual([
|
||||
"--user stop openclaw-gateway.service",
|
||||
"proc-stat-read",
|
||||
"wait",
|
||||
]);
|
||||
expect(isProcessRunning(pid)).toBe(true);
|
||||
expect(readFileSync(pidPath, "utf8")).toBe(`${pid}\n`);
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -3994,9 +4128,9 @@ writePluginInstallIndexForE2E(${JSON.stringify(index)}, { stateDir: ${JSON.strin
|
|||
}
|
||||
});
|
||||
|
||||
it.each([true])(
|
||||
it.for([true])(
|
||||
"retains a failed service child and only sanitized diagnostics (candidate redactor: %s)",
|
||||
async (candidateRedactorPresent) => {
|
||||
async (candidateRedactorPresent, { signal }) => {
|
||||
const workDir = tempDirs.make("openclaw-survivor-diagnostics-");
|
||||
const artifacts = join(workDir, "artifacts");
|
||||
const state = join(workDir, "home", ".openclaw");
|
||||
|
|
@ -4043,7 +4177,15 @@ writePluginInstallIndexForE2E(${JSON.stringify(index)}, { stateDir: ${JSON.strin
|
|||
},
|
||||
stdio: "ignore",
|
||||
});
|
||||
expect(await waitForProcessExit(supervisor)).toBe(0);
|
||||
const exited = waitForProcessExit(supervisor);
|
||||
try {
|
||||
expect(await withinTest(exited, signal)).toBe(0);
|
||||
} finally {
|
||||
if (supervisor.exitCode === null && supervisor.signalCode === null) {
|
||||
supervisor.kill("SIGTERM");
|
||||
}
|
||||
await exited;
|
||||
}
|
||||
const observation = JSON.parse(readFileSync(`${logPath}.exit.json`, "utf8"));
|
||||
expect(observation.last).toMatchObject({ code: 78, signal: null });
|
||||
const managerEnv = {
|
||||
|
|
@ -4128,9 +4270,9 @@ writePluginInstallIndexForE2E(${JSON.stringify(index)}, { stateDir: ${JSON.strin
|
|||
},
|
||||
);
|
||||
|
||||
it.each([UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH])(
|
||||
it.for([UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH])(
|
||||
"retains supervisor bootstrap stderr without inventing a child exit in %s",
|
||||
async (scriptPath) => {
|
||||
async (scriptPath, { signal }) => {
|
||||
const workDir = tempDirs.make("openclaw-survivor-bootstrap-");
|
||||
const unitDir = join(workDir, ".config", "systemd", "user");
|
||||
mkdirSync(unitDir, { recursive: true });
|
||||
|
|
@ -4139,10 +4281,12 @@ writePluginInstallIndexForE2E(${JSON.stringify(index)}, { stateDir: ${JSON.strin
|
|||
`[Service]\nExecStart="${process.execPath}" unused\n`,
|
||||
);
|
||||
const binDir = join(workDir, "bin");
|
||||
const reporter = writeFixtureReceiptReporter(workDir);
|
||||
const readyPath = join(workDir, "bootstrap.ready");
|
||||
writeExecutables(binDir, {
|
||||
node: `#!/bin/sh
|
||||
case "$1" in
|
||||
*.supervisor.mjs) echo supervisor-bootstrap-failure >&2; exit 17 ;;
|
||||
*.supervisor.mjs) echo supervisor-bootstrap-failure >&2; ${shellQuote(process.execPath)} ${shellQuote(reporter)} ${shellQuote(readyPath)} ready; exit 17 ;;
|
||||
esac
|
||||
exec ${shellQuote(process.execPath)} "$@"
|
||||
`,
|
||||
|
|
@ -4162,15 +4306,7 @@ exec ${shellQuote(process.execPath)} "$@"
|
|||
});
|
||||
expect(started.status, started.stderr).toBe(0);
|
||||
const bootstrapPath = `${logPath}.bootstrap.log`;
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
if (
|
||||
existsSync(bootstrapPath) &&
|
||||
readFileSync(bootstrapPath, "utf8").includes("supervisor-bootstrap-failure")
|
||||
) {
|
||||
break;
|
||||
}
|
||||
await delay(10);
|
||||
}
|
||||
await withinTest(receipts.waitFor(readyPath, "ready"), signal);
|
||||
expect(readFileSync(bootstrapPath, "utf8")).toContain("supervisor-bootstrap-failure");
|
||||
expect(existsSync(`${logPath}.exit.json`)).toBe(false);
|
||||
const shown = spawnSync("bash", [shimPath, ...SURVIVOR_SERVICE_SHOW_ARGS], {
|
||||
|
|
@ -4514,7 +4650,7 @@ exit 0
|
|||
},
|
||||
);
|
||||
|
||||
it("stops supervised gateway restarts after the systemd burst limit", async () => {
|
||||
it("stops supervised gateway restarts after the systemd burst limit", async ({ signal }) => {
|
||||
const workDir = tempDirs.make("openclaw-update-restart-supervisor-");
|
||||
const scripts = [readFileSync(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH, "utf8")];
|
||||
|
||||
|
|
@ -4540,9 +4676,15 @@ exit 0
|
|||
},
|
||||
stdio: "ignore",
|
||||
});
|
||||
const exitCode = await waitForProcessExit(supervisor);
|
||||
|
||||
expect(exitCode).toBe(0);
|
||||
const exited = waitForProcessExit(supervisor);
|
||||
try {
|
||||
expect(await withinTest(exited, signal)).toBe(0);
|
||||
} finally {
|
||||
if (supervisor.exitCode === null && supervisor.signalCode === null) {
|
||||
supervisor.kill("SIGTERM");
|
||||
}
|
||||
await exited;
|
||||
}
|
||||
expect(readFileSync(countPath, "utf8")).toBe("xxxxx");
|
||||
expect(readFileSync(logPath, "utf8")).toContain(
|
||||
"[systemctl-shim] gateway restart limit reached",
|
||||
|
|
@ -4552,13 +4694,14 @@ exit 0
|
|||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"terminates supervised gateway descendants at the systemd stop timeout",
|
||||
async () => {
|
||||
async ({ signal }) => {
|
||||
const workDir = tempDirs.make("openclaw-update-restart-process-group-");
|
||||
const descendantPath = writeTermIgnoringDescendant(workDir);
|
||||
const gatewayPath = join(workDir, "gateway.mjs");
|
||||
writeFileSync(
|
||||
gatewayPath,
|
||||
`import fs from "node:fs";
|
||||
`${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
import fs from "node:fs";
|
||||
import { spawn } from "node:child_process";
|
||||
process.on("SIGTERM", () => {
|
||||
setTimeout(() => {
|
||||
|
|
@ -4566,12 +4709,13 @@ process.on("SIGTERM", () => {
|
|||
process.exit(0);
|
||||
}, 50);
|
||||
});
|
||||
spawn(process.execPath, [process.env.DESCENDANT_SCRIPT], { stdio: "ignore" });
|
||||
const ready = setInterval(() => {
|
||||
if (!fs.existsSync(process.env.DESCENDANT_PID_FILE)) return;
|
||||
clearInterval(ready);
|
||||
const descendant = spawn(process.execPath, [process.env.DESCENDANT_SCRIPT], {
|
||||
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
||||
});
|
||||
descendant.once("message", () => {
|
||||
fs.writeFileSync(process.env.STATE_FILE, "ready");
|
||||
}, 5);
|
||||
sendReceipt(process.env.STATE_FILE, "ready");
|
||||
});
|
||||
setInterval(() => {}, 1_000);
|
||||
`,
|
||||
);
|
||||
|
|
@ -4598,24 +4742,25 @@ setInterval(() => {}, 1_000);
|
|||
},
|
||||
stdio: "ignore",
|
||||
});
|
||||
const exited = waitForProcessExit(supervisor);
|
||||
try {
|
||||
for (let attempt = 0; attempt < 100 && !existsSync(statePath); attempt += 1) {
|
||||
await delay(10);
|
||||
}
|
||||
expect(
|
||||
existsSync(statePath),
|
||||
`${supervisorPath}: readiness missing (exit=${supervisor.exitCode}, signal=${supervisor.signalCode})`,
|
||||
).toBe(true);
|
||||
await withinTest(
|
||||
fixtureEventBeforeSettlement(
|
||||
statePath,
|
||||
"ready",
|
||||
exited,
|
||||
`${supervisorPath}: readiness missing (exit=${supervisor.exitCode}, signal=${supervisor.signalCode})`,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10);
|
||||
expect(descendantPid).toBeGreaterThan(1);
|
||||
expect(isProcessRunning(descendantPid)).toBe(true);
|
||||
|
||||
supervisor.kill("SIGTERM");
|
||||
expect(await waitForProcessExit(supervisor)).toBe(0);
|
||||
expect(await withinTest(exited, signal)).toBe(0);
|
||||
expect(readFileSync(statePath, "utf8")).toBe("ready-graceful");
|
||||
for (let attempt = 0; attempt < 100 && isProcessRunning(descendantPid); attempt += 1) {
|
||||
await delay(10);
|
||||
}
|
||||
// The supervisor exits only after drainProcessGroup observes the group absent.
|
||||
expect(isProcessRunning(descendantPid)).toBe(false);
|
||||
} finally {
|
||||
await stopUpgradeSurvivorSupervisor(supervisor, descendantPidPath);
|
||||
|
|
@ -4626,7 +4771,7 @@ setInterval(() => {}, 1_000);
|
|||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"drains the previous gateway process group before restarting",
|
||||
async () => {
|
||||
async ({ signal }) => {
|
||||
const workDir = tempDirs.make("openclaw-update-restart-process-group-restart-");
|
||||
const descendantPath = writeTermIgnoringDescendant(workDir);
|
||||
const gatewayPath = join(workDir, "restart-gateway.mjs");
|
||||
|
|
@ -4637,12 +4782,10 @@ import { spawn } from "node:child_process";
|
|||
fs.appendFileSync(process.env.STARTS_FILE, "x");
|
||||
const starts = fs.readFileSync(process.env.STARTS_FILE, "utf8").length;
|
||||
if (starts === 1) {
|
||||
spawn(process.execPath, [process.env.DESCENDANT_SCRIPT], { stdio: "ignore" });
|
||||
const ready = setInterval(() => {
|
||||
if (!fs.existsSync(process.env.DESCENDANT_PID_FILE)) return;
|
||||
clearInterval(ready);
|
||||
process.exit(1);
|
||||
}, 5);
|
||||
const descendant = spawn(process.execPath, [process.env.DESCENDANT_SCRIPT], {
|
||||
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
||||
});
|
||||
descendant.once("message", () => process.exit(1));
|
||||
setInterval(() => {}, 1_000);
|
||||
} else {
|
||||
const pid = Number.parseInt(fs.readFileSync(process.env.DESCENDANT_PID_FILE, "utf8"), 10);
|
||||
|
|
@ -4686,8 +4829,9 @@ if (starts === 1) {
|
|||
},
|
||||
stdio: "ignore",
|
||||
});
|
||||
const exited = waitForProcessExit(supervisor);
|
||||
try {
|
||||
expect(await waitForProcessExit(supervisor)).toBe(0);
|
||||
expect(await withinTest(exited, signal)).toBe(0);
|
||||
const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10);
|
||||
expect(descendantPid).toBeGreaterThan(1);
|
||||
expect(readFileSync(startsPath, "utf8")).toBe("xx");
|
||||
|
|
@ -5196,36 +5340,48 @@ process.exit(73);
|
|||
expectContainerCleanup(fixture, "openclaw-plugin-binding-command-escape-e2e-");
|
||||
});
|
||||
|
||||
it.each([
|
||||
it.for([
|
||||
["SIGINT", 130],
|
||||
["SIGTERM", 143],
|
||||
["SIGHUP", 129],
|
||||
] as const)("cleans the actual cron runner and its harness on %s", async (signal, status) => {
|
||||
const fixture = containerCleanupFixture("signal");
|
||||
const runner = spawn("/bin/bash", ["scripts/e2e/cron-cli-docker.sh"], {
|
||||
env: fixture.env,
|
||||
stdio: "ignore",
|
||||
});
|
||||
try {
|
||||
await expect.poll(() => existsSync(fixture.pidPath), { timeout: 5_000 }).toBe(true);
|
||||
runner.kill(signal);
|
||||
expect(await waitForProcessExit(runner)).toBe(status);
|
||||
expectContainerCleanup(fixture, "openclaw-cron-cli-e2e-");
|
||||
expect(isProcessRunning(Number(readFileSync(fixture.pidPath, "utf8")))).toBe(false);
|
||||
} finally {
|
||||
if (runner.exitCode === null && runner.signalCode === null) {
|
||||
runner.kill("SIGTERM");
|
||||
await waitForProcessExit(runner);
|
||||
}
|
||||
if (existsSync(fixture.pidPath)) {
|
||||
const pid = Number(readFileSync(fixture.pidPath, "utf8"));
|
||||
if (isProcessRunning(pid)) {
|
||||
process.kill(pid, "SIGKILL");
|
||||
await expect.poll(() => isProcessRunning(pid), { timeout: 5_000 }).toBe(false);
|
||||
] as const)(
|
||||
"cleans the actual cron runner and its harness on %s",
|
||||
async ([childSignal, status], { signal }) => {
|
||||
const fixture = containerCleanupFixture("signal");
|
||||
const runner = spawn("/bin/bash", ["scripts/e2e/cron-cli-docker.sh"], {
|
||||
env: fixture.env,
|
||||
stdio: "ignore",
|
||||
});
|
||||
const exited = waitForProcessExit(runner);
|
||||
try {
|
||||
await withinTest(
|
||||
fixtureEventBeforeSettlement(
|
||||
fixture.readyPath,
|
||||
"ready",
|
||||
exited,
|
||||
"Docker PID file was not written",
|
||||
),
|
||||
signal,
|
||||
);
|
||||
runner.kill(childSignal);
|
||||
expect(await withinTest(exited, signal)).toBe(status);
|
||||
expectContainerCleanup(fixture, "openclaw-cron-cli-e2e-");
|
||||
expect(isProcessRunning(Number(readFileSync(fixture.pidPath, "utf8")))).toBe(false);
|
||||
} finally {
|
||||
if (runner.exitCode === null && runner.signalCode === null) {
|
||||
runner.kill("SIGTERM");
|
||||
await exited;
|
||||
}
|
||||
if (existsSync(fixture.pidPath)) {
|
||||
const pid = Number(readFileSync(fixture.pidPath, "utf8"));
|
||||
if (isProcessRunning(pid)) {
|
||||
process.kill(pid, "SIGKILL");
|
||||
await waitForForeignProcessExit(pid, signal);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(
|
||||
[
|
||||
|
|
|
|||
|
|
@ -299,18 +299,15 @@ elif phase == "redirect":
|
|||
with subprocess.Popen([sys.executable, "-I", "-S", owner, "--policy", str(selected_policy),
|
||||
remote, token, phase], cwd=workspace, env=env,
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) as child:
|
||||
# The managed fixture caller owns the lifetime deadline. Keep the
|
||||
# server/root alive until this owner and its output pipes settle.
|
||||
try:
|
||||
stdout, stderr = child.communicate(timeout=25)
|
||||
stdout, stderr = child.communicate()
|
||||
except BaseException:
|
||||
# The Git owner must drain its separately owned Git groups
|
||||
# before this fixture closes the server or removes its root.
|
||||
child.terminate()
|
||||
try:
|
||||
child.communicate(timeout=12)
|
||||
except subprocess.TimeoutExpired:
|
||||
child.kill()
|
||||
child.wait()
|
||||
raise RuntimeError("checkout owner did not finish cancellation cleanup")
|
||||
child.communicate()
|
||||
raise
|
||||
return subprocess.CompletedProcess(child.args, child.returncode, stdout, stderr)
|
||||
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { createServer, type Server } from "node:http";
|
|||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { WebSocket, WebSocketServer, type RawData } from "ws";
|
||||
import { createBoundedChildOutput } from "../helpers/bounded-child-output.js";
|
||||
import { withinTest } from "../helpers/promise.js";
|
||||
|
||||
type ScriptResult = {
|
||||
status: number | null;
|
||||
|
|
@ -169,118 +170,95 @@ async function listenGateway(params: {
|
|||
return `ws://127.0.0.1:${address.port}`;
|
||||
}
|
||||
|
||||
function runScript(url: string, extraArgs: readonly string[] = []): Promise<ScriptResult> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
"tsx",
|
||||
"scripts/dev/ios-node-e2e.ts",
|
||||
"--url",
|
||||
url,
|
||||
"--token",
|
||||
"token",
|
||||
"--json",
|
||||
...extraArgs,
|
||||
],
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
const stdout = createBoundedChildOutput();
|
||||
const stderr = createBoundedChildOutput();
|
||||
let settled = false;
|
||||
const timeout = setTimeout(() => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
child.kill("SIGKILL");
|
||||
resolve({
|
||||
status: null,
|
||||
signal: "SIGKILL",
|
||||
stdout: stdout.text(),
|
||||
stderr: stderr.text(),
|
||||
timedOut: true,
|
||||
});
|
||||
}, 5000);
|
||||
timeout.unref?.();
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr.append(chunk);
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
resolve({ status, signal, stdout: stdout.text(), stderr: stderr.text(), timedOut: false });
|
||||
});
|
||||
});
|
||||
function runScript(
|
||||
url: string,
|
||||
signal: AbortSignal,
|
||||
extraArgs: readonly string[] = [],
|
||||
): Promise<ScriptResult> {
|
||||
return runScriptRaw(["--url", url, "--token", "token", "--json", ...extraArgs], signal);
|
||||
}
|
||||
|
||||
function runScriptRaw(args: readonly string[]): Promise<ScriptResult> {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/ios-node-e2e.ts", ...args],
|
||||
{
|
||||
stdio: "pipe",
|
||||
},
|
||||
);
|
||||
const stdout = createBoundedChildOutput();
|
||||
const stderr = createBoundedChildOutput();
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr.append(chunk);
|
||||
});
|
||||
child.on("close", (status, signal) => {
|
||||
resolve({ status, signal, stdout: stdout.text(), stderr: stderr.text(), timedOut: false });
|
||||
async function runScriptRaw(args: readonly string[], signal: AbortSignal): Promise<ScriptResult> {
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
["--import", "tsx", "scripts/dev/ios-node-e2e.ts", ...args],
|
||||
{ stdio: "pipe" },
|
||||
);
|
||||
const stdout = createBoundedChildOutput();
|
||||
const stderr = createBoundedChildOutput();
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout.append(chunk);
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr.append(chunk);
|
||||
});
|
||||
let spawnError: Error | undefined;
|
||||
child.once("error", (error) => {
|
||||
spawnError = error;
|
||||
});
|
||||
// Retain close from launch so abort cleanup joins the process and both output pipes.
|
||||
const closed = new Promise<ScriptResult>((resolve) => {
|
||||
child.once("close", (status, childSignal) => {
|
||||
resolve({
|
||||
status,
|
||||
signal: childSignal,
|
||||
stdout: stdout.text(),
|
||||
stderr: stderr.text(),
|
||||
timedOut: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
try {
|
||||
const result = await withinTest(closed, signal);
|
||||
if (spawnError) {
|
||||
throw spawnError;
|
||||
}
|
||||
return result;
|
||||
} finally {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
await closed;
|
||||
}
|
||||
}
|
||||
|
||||
describe("ios-node-e2e", () => {
|
||||
it("prints CLI help without connecting", async () => {
|
||||
const result = await runScriptRaw(["--help"]);
|
||||
it("prints CLI help without connecting", async ({ signal }) => {
|
||||
const result = await runScriptRaw(["--help"], signal);
|
||||
|
||||
expect(result).toMatchObject({ signal: null, status: 0, timedOut: false });
|
||||
expect(result.stdout).toContain("Usage: bun scripts/dev/ios-node-e2e.ts");
|
||||
expect(result.stderr).toBe("");
|
||||
});
|
||||
|
||||
it("rejects unknown CLI args before connecting", async () => {
|
||||
const result = await runScript("ws://127.0.0.1:9", ["--wat"]);
|
||||
it("rejects unknown CLI args before connecting", async ({ signal }) => {
|
||||
const result = await runScript("ws://127.0.0.1:9", signal, ["--wat"]);
|
||||
|
||||
expect(result).toMatchObject({ signal: null, status: 1, timedOut: false });
|
||||
expect(result.stderr.trim()).toBe("Unknown argument: --wat");
|
||||
expect(result.stdout).toBe("");
|
||||
});
|
||||
|
||||
it("rejects short flags as CLI option values before help handling", async () => {
|
||||
const result = await runScriptRaw(["--url", "-h", "--token", "token"]);
|
||||
it("rejects short flags as CLI option values before help handling", async ({ signal }) => {
|
||||
const result = await runScriptRaw(["--url", "-h", "--token", "token"], signal);
|
||||
|
||||
expect(result).toMatchObject({ signal: null, status: 1, timedOut: false });
|
||||
expect(result.stderr.trim()).toBe("--url requires a value");
|
||||
expect(result.stdout).toBe("");
|
||||
});
|
||||
|
||||
it("rejects malformed wait seconds before connecting", async () => {
|
||||
const result = await runScript("ws://127.0.0.1:9", ["--wait-seconds", "1e3"]);
|
||||
it("rejects malformed wait seconds before connecting", async ({ signal }) => {
|
||||
const result = await runScript("ws://127.0.0.1:9", signal, ["--wait-seconds", "1e3"]);
|
||||
|
||||
expect(result).toMatchObject({ signal: null, status: 1, timedOut: false });
|
||||
expect(result.stderr).toContain("--wait-seconds must be a positive integer; got: 1e3");
|
||||
expect(result.stdout).toBe("");
|
||||
});
|
||||
|
||||
it("fails empty node invoke payloads instead of counting them as proof", async () => {
|
||||
it("fails empty node invoke payloads instead of counting them as proof", async ({ signal }) => {
|
||||
const invokeParams: Array<{ command?: string; idempotencyKey?: string }> = [];
|
||||
const url = await listenGateway({ mode: "empty", invokeParams });
|
||||
const result = await runScript(url);
|
||||
const result = await runScript(url, signal);
|
||||
const report = JSON.parse(result.stdout) as {
|
||||
results: Array<{ error?: string; id: string; ok: boolean; payload?: unknown }>;
|
||||
};
|
||||
|
|
@ -299,10 +277,10 @@ describe("ios-node-e2e", () => {
|
|||
expect(invokeParams.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("fails malformed primitive device info payloads", async () => {
|
||||
it("fails malformed primitive device info payloads", async ({ signal }) => {
|
||||
const invokeParams: Array<{ command?: string; idempotencyKey?: string }> = [];
|
||||
const url = await listenGateway({ mode: "primitive-device-info", invokeParams });
|
||||
const result = await runScript(url);
|
||||
const result = await runScript(url, signal);
|
||||
const report = JSON.parse(result.stdout) as {
|
||||
results: Array<{ error?: string; id: string; ok: boolean; payload?: unknown }>;
|
||||
};
|
||||
|
|
@ -315,10 +293,10 @@ describe("ios-node-e2e", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("fails malformed nested payloadJSON payloads", async () => {
|
||||
it("fails malformed nested payloadJSON payloads", async ({ signal }) => {
|
||||
const invokeParams: Array<{ command?: string; idempotencyKey?: string }> = [];
|
||||
const url = await listenGateway({ mode: "invalid-payload-json", invokeParams });
|
||||
const result = await runScript(url);
|
||||
const result = await runScript(url, signal);
|
||||
const report = JSON.parse(result.stdout) as {
|
||||
results: Array<{ error?: string; id: string; ok: boolean; payload?: unknown }>;
|
||||
};
|
||||
|
|
@ -331,10 +309,10 @@ describe("ios-node-e2e", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it("accepts non-empty node invoke payloads and sends idempotency keys", async () => {
|
||||
it("accepts non-empty node invoke payloads and sends idempotency keys", async ({ signal }) => {
|
||||
const invokeParams: Array<{ command?: string; idempotencyKey?: string }> = [];
|
||||
const url = await listenGateway({ mode: "valid", invokeParams });
|
||||
const result = await runScript(url);
|
||||
const result = await runScript(url, signal);
|
||||
const report = JSON.parse(result.stdout) as {
|
||||
results: Array<{ id: string; ok: boolean }>;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,9 +1,33 @@
|
|||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { execFileSync, spawn, spawnSync, type ChildProcess } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
fixtureReceiptClientSource,
|
||||
openFixtureReceiptChannel,
|
||||
type FixtureReceiptChannel,
|
||||
} from "../helpers/fixture-receipts.js";
|
||||
import { awaitGateBeforeSettlement, withinTest } from "../helpers/promise.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const pendingChildCompletions = new Set<Promise<unknown>>();
|
||||
const tempDirs = useAutoCleanupTempDirTracker((cleanup) =>
|
||||
afterEach(async () => {
|
||||
// Timeout cleanup still needs fixture PID files until its native child closes.
|
||||
await Promise.allSettled(pendingChildCompletions);
|
||||
cleanup();
|
||||
}),
|
||||
);
|
||||
let receipts: FixtureReceiptChannel;
|
||||
beforeAll(async () => {
|
||||
receipts = await openFixtureReceiptChannel();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await receipts.close();
|
||||
});
|
||||
|
||||
const helperPath = path.resolve("scripts/lib/openclaw-e2e-instance.sh");
|
||||
const hostPath = [
|
||||
|
|
@ -18,6 +42,98 @@ function shellQuote(value: string): string {
|
|||
return `'${value.replace(/'/gu, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
function observeChild(child: ChildProcess) {
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout?.setEncoding("utf8").on("data", (chunk: string) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr?.setEncoding("utf8").on("data", (chunk: string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
const closed = new Promise<{ status: number | null; stdout: string; stderr: string }>(
|
||||
(resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (status) => resolve({ status, stdout, stderr }));
|
||||
},
|
||||
).finally(() => {
|
||||
pendingChildCompletions.delete(closed);
|
||||
});
|
||||
pendingChildCompletions.add(closed);
|
||||
return { child, closed };
|
||||
}
|
||||
|
||||
function startNodeFixture(script: string, args: string[]) {
|
||||
const fixture = observeChild(
|
||||
spawn(process.execPath, [script, ...args], {
|
||||
stdio: ["ignore", "pipe", "pipe", "ipc"],
|
||||
}),
|
||||
);
|
||||
const ready = new Promise<void>((resolve) => {
|
||||
fixture.child.once("message", () => resolve());
|
||||
});
|
||||
return {
|
||||
...fixture,
|
||||
ready: awaitGateBeforeSettlement(
|
||||
ready,
|
||||
fixture.closed,
|
||||
"fixture exited before publishing readiness",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function startBash(script: string, env: Record<string, string | undefined> = {}) {
|
||||
return observeChild(
|
||||
spawn("/bin/bash", ["-c", script], {
|
||||
env: shellTestEnv(env),
|
||||
detached: true,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function killFixturePid(pid: number) {
|
||||
if (pid > 1) {
|
||||
try {
|
||||
process.kill(pid, "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
function isProcessAlive(pid: number): boolean {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Shell stop/watchdog escalation can return before a foreign PID has been reaped.
|
||||
async function waitForProcessGone(
|
||||
pid: number,
|
||||
signal: AbortSignal,
|
||||
message: string,
|
||||
): Promise<void> {
|
||||
while (isProcessAlive(pid)) {
|
||||
await delay(5, undefined, { signal }).catch((error: unknown) => {
|
||||
throw new Error(`${message}: ${pid}`, { cause: error });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function fixtureReadyBeforeSettlement(marker: string, operation: PromiseLike<unknown>) {
|
||||
await Promise.race([
|
||||
receipts.waitFor(marker, "ready"),
|
||||
Promise.resolve(operation).then(() => {
|
||||
// Marker publication precedes the fixture receipt; a separate pipe may deliver it late.
|
||||
if (!fs.existsSync(marker)) {
|
||||
throw new Error(`fixture exited before publishing ${marker}`);
|
||||
}
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
function runHelper(payload: string) {
|
||||
return runSourcedHelper(
|
||||
[
|
||||
|
|
@ -361,51 +477,56 @@ describe("scripts/lib/openclaw-e2e-instance.sh", () => {
|
|||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
it.for([
|
||||
["accepts the March listening marker", "listening on", true, "legacy-ready-log-ok", 0],
|
||||
["rejects a closed March listener", "listening on", false, "legacy-ready-log-ok", 1],
|
||||
["accepts a live legacy ready marker", "ready", true, "legacy-ready-log-ok", 0],
|
||||
["rejects a closed legacy ready listener", "ready", false, "legacy-ready-log-ok", 1],
|
||||
["rejects the March marker in strict mode", "listening on", true, "strict", 1],
|
||||
] as const)("%s", (_label, marker, listening, mode, expectedStatus) => {
|
||||
withTempDir("openclaw-e2e-readyz-legacy-", (tempDir) => {
|
||||
const logPath = path.join(tempDir, "gateway.log");
|
||||
const portPath = path.join(tempDir, "port.txt");
|
||||
const resultPath = path.join(tempDir, "readiness-status.txt");
|
||||
const serverPath = path.join(tempDir, "gateway.cjs");
|
||||
fs.writeFileSync(
|
||||
serverPath,
|
||||
[
|
||||
"const fs = require('node:fs');",
|
||||
"const http = require('node:http');",
|
||||
"const [logPath, portPath, marker, listening, mode] = process.argv.slice(2);",
|
||||
"const server = http.createServer((_request, response) => {",
|
||||
// A working endpoint must not let strict mode accept the historical marker.
|
||||
" response.writeHead(mode === 'strict' ? 200 : 503);",
|
||||
" response.end('{}');",
|
||||
"});",
|
||||
"server.listen(0, '127.0.0.1', () => {",
|
||||
" const port = server.address().port;",
|
||||
" const publish = () => {",
|
||||
" fs.writeFileSync(logPath, `[gateway] ${marker} ws://127.0.0.1:${port} (PID ${process.pid})\\n`);",
|
||||
" fs.writeFileSync(portPath, String(port));",
|
||||
" };",
|
||||
" if (listening === 'true') publish(); else server.close(publish);",
|
||||
"});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
] as const)("%s", async ([_label, marker, listening, mode, expectedStatus], { signal }) => {
|
||||
const tempDir = tempDirs.make("openclaw-e2e-readyz-legacy-");
|
||||
const logPath = path.join(tempDir, "gateway.log");
|
||||
const portPath = path.join(tempDir, "port.txt");
|
||||
const resultPath = path.join(tempDir, "readiness-status.txt");
|
||||
const serverPath = path.join(tempDir, "gateway.cjs");
|
||||
fs.writeFileSync(
|
||||
serverPath,
|
||||
[
|
||||
"const fs = require('node:fs');",
|
||||
"const http = require('node:http');",
|
||||
"const [logPath, portPath, marker, listening, mode] = process.argv.slice(2);",
|
||||
"const server = http.createServer((_request, response) => {",
|
||||
// A working endpoint must not let strict mode accept the historical marker.
|
||||
" response.writeHead(mode === 'strict' ? 200 : 503);",
|
||||
" response.end('{}');",
|
||||
"});",
|
||||
"server.listen(0, '127.0.0.1', () => {",
|
||||
" const port = server.address().port;",
|
||||
" const publish = () => {",
|
||||
" fs.writeFileSync(logPath, `[gateway] ${marker} ws://127.0.0.1:${port} (PID ${process.pid})\\n`);",
|
||||
" fs.writeFileSync(portPath, String(port));",
|
||||
" process.send('ready');",
|
||||
" };",
|
||||
" if (listening === 'true') publish(); else server.close(publish);",
|
||||
"});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
const fixture = startNodeFixture(serverPath, [
|
||||
logPath,
|
||||
portPath,
|
||||
marker,
|
||||
String(listening),
|
||||
mode,
|
||||
]);
|
||||
try {
|
||||
await withinTest(fixture.ready, signal);
|
||||
const result = runBashWithHelper(
|
||||
[
|
||||
`${shellQuote(process.execPath)} ${shellQuote(serverPath)} ${shellQuote(logPath)} ${shellQuote(portPath)} ${shellQuote(marker)} ${listening} ${shellQuote(mode)} &`,
|
||||
'gateway_pid="$!"',
|
||||
'trap \'kill "$gateway_pid" >/dev/null 2>&1 || true; wait "$gateway_pid" >/dev/null 2>&1 || true\' EXIT',
|
||||
`for _ in $(seq 1 100); do [ -s ${shellQuote(portPath)} ] && break; sleep 0.02; done`,
|
||||
`[ -s ${shellQuote(portPath)} ]`,
|
||||
`port="$(cat ${shellQuote(portPath)})"`,
|
||||
`if openclaw_e2e_wait_gateway_ready "$gateway_pid" ${shellQuote(logPath)} 2 "$port" ${shellQuote(mode)}; then`,
|
||||
`if openclaw_e2e_wait_gateway_ready "${fixture.child.pid}" ${shellQuote(logPath)} 2 "$port" ${shellQuote(mode)}; then`,
|
||||
` printf '0' >${shellQuote(resultPath)}`,
|
||||
"else",
|
||||
` printf '%s' "$?" >${shellQuote(resultPath)}`,
|
||||
|
|
@ -414,10 +535,12 @@ describe("scripts/lib/openclaw-e2e-instance.sh", () => {
|
|||
{},
|
||||
5_000,
|
||||
);
|
||||
|
||||
expectShellSuccess(result);
|
||||
expect(fs.readFileSync(resultPath, "utf8")).toBe(String(expectedStatus));
|
||||
});
|
||||
} finally {
|
||||
fixture.child.kill("SIGKILL");
|
||||
await fixture.closed;
|
||||
}
|
||||
});
|
||||
|
||||
it("wraps package installs with the configured timeout", () => {
|
||||
|
|
@ -667,57 +790,59 @@ describe("scripts/lib/openclaw-e2e-instance.sh", () => {
|
|||
["TERM", "143"],
|
||||
["HUP", "129"],
|
||||
] as const) {
|
||||
it(`escalates Node watchdog children that ignore parent SIG${shellSignal}`, () => {
|
||||
withTempDir("openclaw-e2e-instance-node-watchdog-signal-", (tempDir) => {
|
||||
writeNodeShim(tempDir);
|
||||
const childPath = path.join(tempDir, "ignore-term.cjs");
|
||||
const pidPath = path.join(tempDir, "child.pid");
|
||||
const watchdogPidPath = path.join(tempDir, "watchdog.pid");
|
||||
fs.writeFileSync(
|
||||
childPath,
|
||||
[
|
||||
"const fs = require('node:fs');",
|
||||
"process.on('SIGTERM', () => {});",
|
||||
"process.on('SIGHUP', () => {});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
// PID publication lets the shell signal us; install handlers first.
|
||||
"fs.writeFileSync(process.argv[2], String(process.pid));",
|
||||
"fs.writeFileSync(process.argv[3], String(process.ppid));",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
const script = `
|
||||
it(`escalates Node watchdog children that ignore parent SIG${shellSignal}`, async ({
|
||||
signal,
|
||||
}) => {
|
||||
const tempDir = tempDirs.make("openclaw-e2e-instance-node-watchdog-signal-");
|
||||
writeNodeShim(tempDir);
|
||||
const childPath = path.join(tempDir, "ignore-term.mjs");
|
||||
const pidPath = path.join(tempDir, "child.pid");
|
||||
const watchdogPidPath = path.join(tempDir, "watchdog.pid");
|
||||
fs.writeFileSync(
|
||||
childPath,
|
||||
`${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
import fs from "node:fs";
|
||||
process.on("SIGTERM", () => {});
|
||||
process.on("SIGHUP", () => {});
|
||||
setInterval(() => {}, 1000);
|
||||
fs.writeFileSync(process.argv[2], String(process.pid));
|
||||
fs.writeFileSync(process.argv[3], String(process.ppid));
|
||||
sendReceipt(process.argv[2], "ready");
|
||||
`,
|
||||
);
|
||||
const runner = startBash(
|
||||
`
|
||||
set -euo pipefail
|
||||
source ${shellQuote(helperPath)}
|
||||
export OPENCLAW_E2E_TIMEOUT_KILL_GRACE_MS=100
|
||||
openclaw_e2e_maybe_timeout 30s node ${shellQuote(childPath)} ${shellQuote(pidPath)} ${shellQuote(watchdogPidPath)} &
|
||||
wrapper_pid="$!"
|
||||
for ((i = 0; i < 100; i += 1)); do
|
||||
[ -s ${shellQuote(pidPath)} ] && [ -s ${shellQuote(watchdogPidPath)} ] && break
|
||||
/bin/sleep 0.02
|
||||
done
|
||||
[ -s ${shellQuote(pidPath)} ]
|
||||
[ -s ${shellQuote(watchdogPidPath)} ]
|
||||
kill -${shellSignal} "$(/bin/cat ${shellQuote(watchdogPidPath)})"
|
||||
set +e
|
||||
wait "$wrapper_pid"
|
||||
status="$?"
|
||||
set -e
|
||||
[ "$status" = "${expectedStatus}" ]
|
||||
child_pid="$(/bin/cat ${shellQuote(pidPath)})"
|
||||
for ((i = 0; i < 100; i += 1)); do
|
||||
kill -0 "$child_pid" 2>/dev/null || exit 0
|
||||
/bin/sleep 0.02
|
||||
done
|
||||
echo "child still alive after watchdog termination" >&2
|
||||
exit 1
|
||||
`;
|
||||
|
||||
const result = runBash(script, { PATH: tempDir }, 5_000);
|
||||
|
||||
expectShellSuccess(result);
|
||||
});
|
||||
openclaw_e2e_maybe_timeout 30s node ${shellQuote(childPath)} ${shellQuote(pidPath)} ${shellQuote(watchdogPidPath)}
|
||||
`,
|
||||
{ PATH: tempDir },
|
||||
);
|
||||
try {
|
||||
await withinTest(fixtureReadyBeforeSettlement(pidPath, runner.closed), signal);
|
||||
const childPid = Number(fs.readFileSync(pidPath, "utf8"));
|
||||
process.kill(Number(fs.readFileSync(watchdogPidPath, "utf8")), `SIG${shellSignal}`);
|
||||
const result = await withinTest(runner.closed, signal);
|
||||
expect(result.status, result.stderr).toBe(Number(expectedStatus));
|
||||
await waitForProcessGone(childPid, signal, "child still alive after watchdog termination");
|
||||
} finally {
|
||||
for (const marker of [pidPath, watchdogPidPath]) {
|
||||
if (fs.existsSync(marker)) {
|
||||
killFixturePid(Number(fs.readFileSync(marker, "utf8")));
|
||||
}
|
||||
}
|
||||
if (
|
||||
runner.child.exitCode === null &&
|
||||
runner.child.signalCode === null &&
|
||||
runner.child.pid
|
||||
) {
|
||||
try {
|
||||
process.kill(-runner.child.pid, "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
await runner.closed;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -761,117 +886,143 @@ fi
|
|||
});
|
||||
});
|
||||
|
||||
it("terminates descendants in the tracked process group", () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-e2e-process-group-"));
|
||||
it("terminates descendants in the tracked process group", async ({ signal }) => {
|
||||
const tempDir = tempDirs.make("openclaw-e2e-process-group-");
|
||||
const parentPidPath = path.join(tempDir, "parent.pid");
|
||||
const childPidPath = path.join(tempDir, "child.pid");
|
||||
const childTermPath = path.join(tempDir, "child.term");
|
||||
const parentPath = path.join(tempDir, "parent.mjs");
|
||||
const childPath = path.join(tempDir, "child.mjs");
|
||||
const logPath = path.join(tempDir, "tracked.log");
|
||||
fs.writeFileSync(
|
||||
childPath,
|
||||
`${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
import fs from "node:fs";
|
||||
process.on("SIGTERM", () => {
|
||||
fs.writeFileSync(process.argv[3], "terminated");
|
||||
sendReceipt(process.argv[3], "terminated");
|
||||
process.exit(0);
|
||||
});
|
||||
setInterval(() => {}, 1000);
|
||||
fs.writeFileSync(process.argv[2], String(process.pid));
|
||||
process.send("ready");
|
||||
`,
|
||||
);
|
||||
fs.writeFileSync(
|
||||
parentPath,
|
||||
`${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
import fs from "node:fs";
|
||||
import { spawn } from "node:child_process";
|
||||
process.on("SIGTERM", () => process.exit(0));
|
||||
const child = spawn(process.execPath, [process.argv[2], process.argv[4], process.argv[5]], {
|
||||
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
||||
});
|
||||
child.once("message", () => {
|
||||
fs.writeFileSync(process.argv[3], String(process.pid));
|
||||
sendReceipt(process.argv[3], "ready");
|
||||
});
|
||||
setInterval(() => {}, 1000);
|
||||
`,
|
||||
);
|
||||
const starter = startBash(
|
||||
`source ${shellQuote(helperPath)}
|
||||
openclaw_e2e_start_tracked_process ${shellQuote(logPath)} ${shellQuote(process.execPath)} ${shellQuote(parentPath)} ${shellQuote(childPath)} ${shellQuote(parentPidPath)} ${shellQuote(childPidPath)} ${shellQuote(childTermPath)}`,
|
||||
{ PATH: hostPath },
|
||||
);
|
||||
let stopper: ReturnType<typeof startBash> | undefined;
|
||||
let trackedPid = 0;
|
||||
try {
|
||||
const parentPath = path.join(tempDir, "parent.cjs");
|
||||
const childPath = path.join(tempDir, "child.cjs");
|
||||
const logPath = path.join(tempDir, "tracked.log");
|
||||
fs.writeFileSync(
|
||||
childPath,
|
||||
[
|
||||
"const fs = require('node:fs');",
|
||||
"process.on('SIGTERM', () => {",
|
||||
" fs.writeFileSync(process.argv[3], 'terminated');",
|
||||
" process.exit(0);",
|
||||
"});",
|
||||
"setInterval(() => {}, 1000);",
|
||||
// Both PID files announce readiness for immediate group termination.
|
||||
"fs.writeFileSync(process.argv[2], String(process.pid));",
|
||||
"",
|
||||
].join("\n"),
|
||||
const started = await withinTest(starter.closed, signal);
|
||||
expect(started.status, started.stderr).toBe(0);
|
||||
trackedPid = Number(started.stdout.trim());
|
||||
// The start helper promises spawn only; its detached child reports actual readiness.
|
||||
await withinTest(receipts.waitFor(parentPidPath, "ready"), signal);
|
||||
const childPid = Number(fs.readFileSync(childPidPath, "utf8"));
|
||||
stopper = startBash(
|
||||
`source ${shellQuote(helperPath)}
|
||||
openclaw_e2e_stop_process ${trackedPid}`,
|
||||
{ PATH: hostPath },
|
||||
);
|
||||
fs.writeFileSync(
|
||||
parentPath,
|
||||
[
|
||||
"const fs = require('node:fs');",
|
||||
"const { spawn } = require('node:child_process');",
|
||||
"const child = spawn(process.execPath, [process.argv[2], process.argv[4], process.argv[5]], {",
|
||||
" stdio: 'ignore',",
|
||||
"});",
|
||||
"child.unref();",
|
||||
"process.on('SIGTERM', () => process.exit(0));",
|
||||
"setInterval(() => {}, 1000);",
|
||||
"fs.writeFileSync(process.argv[3], String(process.pid));",
|
||||
"",
|
||||
].join("\n"),
|
||||
await withinTest(
|
||||
Promise.race([
|
||||
receipts.waitFor(childTermPath, "terminated"),
|
||||
stopper.closed.then(() => {
|
||||
expect(fs.existsSync(childTermPath), "tracked child did not receive SIGTERM").toBe(
|
||||
true,
|
||||
);
|
||||
}),
|
||||
]),
|
||||
signal,
|
||||
);
|
||||
const stopped = await withinTest(stopper.closed, signal);
|
||||
expect(stopped.status, stopped.stderr).toBe(0);
|
||||
expect(
|
||||
fs.statSync(childTermPath).size,
|
||||
"tracked child did not receive SIGTERM",
|
||||
).toBeGreaterThan(0);
|
||||
await waitForProcessGone(
|
||||
childPid,
|
||||
signal,
|
||||
"tracked child still alive after group termination",
|
||||
);
|
||||
|
||||
const script = `
|
||||
set -euo pipefail
|
||||
source ${shellQuote(helperPath)}
|
||||
tracked_pid="$(openclaw_e2e_start_tracked_process ${shellQuote(logPath)} ${shellQuote(process.execPath)} ${shellQuote(parentPath)} ${shellQuote(childPath)} ${shellQuote(parentPidPath)} ${shellQuote(childPidPath)} ${shellQuote(childTermPath)})"
|
||||
for ((i = 0; i < 100; i += 1)); do
|
||||
[ -s ${shellQuote(parentPidPath)} ] && [ -s ${shellQuote(childPidPath)} ] && break
|
||||
/bin/sleep 0.02
|
||||
done
|
||||
[ -s ${shellQuote(parentPidPath)} ]
|
||||
[ -s ${shellQuote(childPidPath)} ]
|
||||
child_pid="$(/bin/cat ${shellQuote(childPidPath)})"
|
||||
openclaw_e2e_stop_process "$tracked_pid"
|
||||
for ((i = 0; i < 100; i += 1)); do
|
||||
[ -s ${shellQuote(childTermPath)} ] && break
|
||||
/bin/sleep 0.02
|
||||
done
|
||||
[ -s ${shellQuote(childTermPath)} ] || {
|
||||
echo "tracked child did not receive SIGTERM" >&2
|
||||
exit 1
|
||||
}
|
||||
for ((i = 0; i < 100; i += 1)); do
|
||||
kill -0 "$child_pid" 2>/dev/null || exit 0
|
||||
/bin/sleep 0.02
|
||||
done
|
||||
echo "tracked child still alive after group termination" >&2
|
||||
exit 1
|
||||
`;
|
||||
|
||||
const result = runBash(script, { PATH: hostPath }, 5_000);
|
||||
|
||||
expectShellSuccess(result);
|
||||
} finally {
|
||||
for (const pidPath of [childPidPath, parentPidPath]) {
|
||||
if (!fs.existsSync(pidPath)) {
|
||||
continue;
|
||||
}
|
||||
const pid = Number(fs.readFileSync(pidPath, "utf8"));
|
||||
if (Number.isInteger(pid) && pid > 1) {
|
||||
try {
|
||||
process.kill(pid, "SIGKILL");
|
||||
} catch {}
|
||||
if (fs.existsSync(pidPath)) {
|
||||
killFixturePid(Number(fs.readFileSync(pidPath, "utf8")));
|
||||
}
|
||||
}
|
||||
fs.rmSync(tempDir, { force: true, recursive: true });
|
||||
if (trackedPid > 1) {
|
||||
try {
|
||||
process.kill(-trackedPid, "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
killFixturePid(trackedPid);
|
||||
for (const command of [starter, stopper]) {
|
||||
if (!command) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
command.child.exitCode === null &&
|
||||
command.child.signalCode === null &&
|
||||
command.child.pid
|
||||
) {
|
||||
try {
|
||||
process.kill(-command.child.pid, "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
await command.closed;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds HTTP readiness probes when a server accepts connections but never responds", () => {
|
||||
withTempDir("openclaw-e2e-http-probe-", (tempDir) => {
|
||||
const portPath = path.join(tempDir, "port.txt");
|
||||
const serverPath = path.join(tempDir, "stalling-server.cjs");
|
||||
fs.writeFileSync(
|
||||
serverPath,
|
||||
[
|
||||
"const fs = require('node:fs');",
|
||||
"const net = require('node:net');",
|
||||
"const server = net.createServer((socket) => socket.on('data', () => {}));",
|
||||
"server.listen(0, '127.0.0.1', () => {",
|
||||
" fs.writeFileSync(process.argv[2], String(server.address().port));",
|
||||
"});",
|
||||
"process.on('SIGTERM', () => server.close(() => process.exit(0)));",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
it("bounds HTTP readiness probes when a server accepts connections but never responds", async ({
|
||||
signal,
|
||||
}) => {
|
||||
const tempDir = tempDirs.make("openclaw-e2e-http-probe-");
|
||||
const portPath = path.join(tempDir, "port.txt");
|
||||
const serverPath = path.join(tempDir, "stalling-server.cjs");
|
||||
fs.writeFileSync(
|
||||
serverPath,
|
||||
[
|
||||
"const fs = require('node:fs');",
|
||||
"const net = require('node:net');",
|
||||
"const server = net.createServer((socket) => socket.on('data', () => {}));",
|
||||
"server.listen(0, '127.0.0.1', () => {",
|
||||
" fs.writeFileSync(process.argv[2], String(server.address().port));",
|
||||
" process.send('ready');",
|
||||
"});",
|
||||
"process.on('SIGTERM', () => server.close(() => process.exit(0)));",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
const startedAt = Date.now();
|
||||
const startedAt = Date.now();
|
||||
const fixture = startNodeFixture(serverPath, [portPath]);
|
||||
try {
|
||||
await withinTest(fixture.ready, signal);
|
||||
const result = runBash(
|
||||
[
|
||||
"set -euo pipefail",
|
||||
`${shellQuote(process.execPath)} ${shellQuote(serverPath)} ${shellQuote(portPath)} & server_pid=$!`,
|
||||
'trap \'kill "$server_pid" 2>/dev/null || true; wait "$server_pid" 2>/dev/null || true\' EXIT',
|
||||
`for _ in $(seq 1 50); do [ -s ${shellQuote(portPath)} ] && break; sleep 0.02; done`,
|
||||
`port="$(cat ${shellQuote(portPath)})"`,
|
||||
`source ${shellQuote(helperPath)}`,
|
||||
'openclaw_e2e_probe_http_status "http://127.0.0.1:${port}/health" 200 100',
|
||||
|
|
@ -881,11 +1032,13 @@ exit 1
|
|||
"; ",
|
||||
);
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(elapsedMs).toBeLessThan(2_500);
|
||||
});
|
||||
} finally {
|
||||
fixture.child.kill("SIGKILL");
|
||||
await fixture.closed;
|
||||
}
|
||||
});
|
||||
|
||||
it("cancels HTTP readiness probe response bodies", () => {
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import {
|
|||
MAX_TIMER_TIMEOUT_MS,
|
||||
MAX_TIMER_TIMEOUT_SECONDS,
|
||||
} from "@openclaw/normalization-core/number-coercion";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
extractLastOpenClawVersionFromLog,
|
||||
isLikelyMacosDesktopHome,
|
||||
|
|
@ -77,6 +77,12 @@ import {
|
|||
import { withEnv } from "../../src/test-utils/env.js";
|
||||
import { resolveTestNodeExecPath, spawnNodeEvalSync } from "../../src/test-utils/node-process.js";
|
||||
import { acquireTestPortBlock } from "../../src/test-utils/port-claims.js";
|
||||
import {
|
||||
fixtureReceiptClientSource,
|
||||
openFixtureReceiptChannel,
|
||||
type FixtureReceiptChannel,
|
||||
} from "../helpers/fixture-receipts.js";
|
||||
import { withinTest } from "../helpers/promise.js";
|
||||
import { cleanupTempDirs, makeTempDir } from "../helpers/temp-dir.js";
|
||||
|
||||
const WRAPPERS = {
|
||||
|
|
@ -117,9 +123,20 @@ const TS_SOURCE = Object.fromEntries(
|
|||
|
||||
const OS_TS_PATHS = [TS_PATHS.linux, TS_PATHS.macos, TS_PATHS.windows];
|
||||
const tempDirs: string[] = [];
|
||||
const pendingChildCompletions = new Set<Promise<unknown>>();
|
||||
const testNodeExecPath = resolveTestNodeExecPath();
|
||||
|
||||
afterEach(() => {
|
||||
let receipts: FixtureReceiptChannel;
|
||||
beforeAll(async () => {
|
||||
receipts = await openFixtureReceiptChannel();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await receipts.close();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// A timed-out body's finally still needs the PID files while joining its native runner.
|
||||
await Promise.allSettled(pendingChildCompletions);
|
||||
cleanupTempDirs(tempDirs);
|
||||
});
|
||||
|
||||
|
|
@ -280,30 +297,30 @@ function isProcessAlive(pid: number): boolean {
|
|||
}
|
||||
}
|
||||
|
||||
async function waitFor(predicate: () => boolean, timeoutMs = 3_000): Promise<void> {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < timeoutMs) {
|
||||
if (predicate()) {
|
||||
return;
|
||||
}
|
||||
await delay(5);
|
||||
// The host wrapper escalates then settles; it exposes no foreign-PID reap signal.
|
||||
async function waitForProcessGone(pid: number, signal: AbortSignal): Promise<void> {
|
||||
while (isProcessAlive(pid)) {
|
||||
await delay(5, undefined, { signal }).catch((error: unknown) => {
|
||||
throw new Error(`condition was not met before timeout: process ${pid} is still alive`, {
|
||||
cause: error,
|
||||
});
|
||||
});
|
||||
}
|
||||
throw new Error("condition was not met before timeout");
|
||||
}
|
||||
|
||||
async function waitForProcessClose(
|
||||
function waitForProcessClose(
|
||||
child: ReturnType<typeof spawn>,
|
||||
timeoutMs = 3_000,
|
||||
): Promise<{ code: number | null; signal: NodeJS.Signals | null }> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
reject(new Error("child process did not close before timeout"));
|
||||
}, timeoutMs);
|
||||
child.once("close", (code, signal) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ code, signal });
|
||||
});
|
||||
const closed = new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
|
||||
(resolve, reject) => {
|
||||
child.once("error", reject);
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
},
|
||||
).finally(() => {
|
||||
pendingChildCompletions.delete(closed);
|
||||
});
|
||||
pendingChildCompletions.add(closed);
|
||||
return closed;
|
||||
}
|
||||
|
||||
function runNode(source: string, options: NonNullable<Parameters<typeof run>[2]> = {}) {
|
||||
|
|
@ -459,31 +476,49 @@ function createSignaledHostCommandFixture() {
|
|||
const tempDir = makeTempDir(tempDirs, "openclaw-parallels-host-command-signal-");
|
||||
const runnerPath = join(tempDir, "runner.mjs");
|
||||
const readyPath = join(tempDir, "ready");
|
||||
const parentPidPath = join(tempDir, "parent.pid");
|
||||
const grandchildPidPath = join(tempDir, "grandchild.pid");
|
||||
const hostCommandUrl = resolveRuntimeWorkerUrl(scriptProcessEntrypoints.parallelsHostCommand);
|
||||
const parentScript = `${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
import { spawn } from "node:child_process";
|
||||
import { writeFileSync } from "node:fs";
|
||||
process.on("SIGTERM", () => process.exit(0));
|
||||
writeFileSync(${JSON.stringify(parentPidPath)}, String(process.pid));
|
||||
const child = spawn(process.execPath, ["-e", ${JSON.stringify(`${SIGNAL_GRANDCHILD_SCRIPT} process.send("ready");`)}], {
|
||||
env: process.env,
|
||||
stdio: ["ignore", "ignore", "ignore", "ipc"],
|
||||
});
|
||||
child.once("message", () => {
|
||||
writeFileSync(process.env.OPENCLAW_TEST_READY_FILE, "ready");
|
||||
sendReceipt(process.env.OPENCLAW_TEST_READY_FILE, "ready");
|
||||
});
|
||||
setInterval(() => {}, 1000);`;
|
||||
writeFileSync(
|
||||
runnerPath,
|
||||
`import { run } from ${JSON.stringify(hostCommandUrl.href)};
|
||||
run(process.execPath, ['-e', ${JSON.stringify(SIGNAL_PARENT_SCRIPT)}], {
|
||||
run(process.execPath, ['--input-type=module', '-e', ${JSON.stringify(parentScript)}], {
|
||||
check: false,
|
||||
env: { ...process.env, OPENCLAW_TEST_GRANDCHILD_PID: ${JSON.stringify(grandchildPidPath)}, OPENCLAW_TEST_READY_FILE: ${JSON.stringify(readyPath)} },
|
||||
quiet: true,
|
||||
timeoutMs: 30_000,
|
||||
});`,
|
||||
);
|
||||
return {
|
||||
grandchildPidPath,
|
||||
readyPath,
|
||||
runner: spawn(
|
||||
testNodeExecPath,
|
||||
[...resolveRuntimeWorkerArgv(hostCommandUrl, testNodeExecPath).slice(0, -1), runnerPath],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
},
|
||||
),
|
||||
};
|
||||
const runner = spawn(
|
||||
testNodeExecPath,
|
||||
[...resolveRuntimeWorkerArgv(hostCommandUrl, testNodeExecPath).slice(0, -1), runnerPath],
|
||||
{ cwd: process.cwd(), detached: true, stdio: "ignore" },
|
||||
);
|
||||
const closed = waitForProcessClose(runner);
|
||||
const ready = Promise.race([
|
||||
receipts.waitFor(readyPath, "ready"),
|
||||
closed.then(() => {
|
||||
// PID publication precedes the parent's ready record; receipt delivery is unordered.
|
||||
if (!existsSync(readyPath) || !existsSync(grandchildPidPath)) {
|
||||
throw new Error("condition was not met before timeout");
|
||||
}
|
||||
}),
|
||||
]);
|
||||
return { grandchildPidPath, parentPidPath, readyPath, runner, ready, closed };
|
||||
}
|
||||
|
||||
function forceKillSignaledFixture(
|
||||
|
|
@ -2224,7 +2259,7 @@ if (commandArgs[0] === "list") {
|
|||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"kills timed-out host command process groups",
|
||||
async () => {
|
||||
async ({ signal }) => {
|
||||
const tempDir = makeTempDir(tempDirs, "openclaw-parallels-host-command-");
|
||||
const grandchildPidPath = join(tempDir, "grandchild.pid");
|
||||
const deadlineFile = join(tempDir, "deadline");
|
||||
|
|
@ -2249,7 +2284,7 @@ if (commandArgs[0] === "list") {
|
|||
expect(readFileSync(deadlineFile, "utf8")).toBe("elapsed");
|
||||
grandchildPid = Number.parseInt(readFileSync(grandchildPidPath, "utf8"), 10);
|
||||
expect(Number.isInteger(grandchildPid)).toBe(true);
|
||||
await waitFor(() => !isProcessAlive(grandchildPid));
|
||||
await waitForProcessGone(grandchildPid, signal);
|
||||
} finally {
|
||||
if (grandchildPid && isProcessAlive(grandchildPid)) {
|
||||
process.kill(grandchildPid, "SIGKILL");
|
||||
|
|
@ -2260,7 +2295,7 @@ if (commandArgs[0] === "list") {
|
|||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"settles timed host commands when an escaped descendant retains child pipes",
|
||||
async () => {
|
||||
() => {
|
||||
const tempDir = makeTempDir(tempDirs, "openclaw-parallels-host-command-pipes-");
|
||||
const grandchildPidPath = join(tempDir, "grandchild.pid");
|
||||
const deadlineFile = join(tempDir, "deadline");
|
||||
|
|
@ -2301,7 +2336,7 @@ if (commandArgs[0] === "list") {
|
|||
|
||||
const durationMs = Date.now() - startedAt;
|
||||
// The renamed path publishes a complete PID even if timeout settles first.
|
||||
await waitFor(() => existsSync(grandchildPidPath));
|
||||
expect(existsSync(grandchildPidPath)).toBe(true);
|
||||
grandchildPid = Number(readFileSync(grandchildPidPath, "utf8"));
|
||||
|
||||
expect(Number.isInteger(grandchildPid)).toBe(true);
|
||||
|
|
@ -2319,7 +2354,7 @@ if (commandArgs[0] === "list") {
|
|||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"reaps externally signaled timed host command descendants",
|
||||
async () => {
|
||||
async ({ signal }) => {
|
||||
const fixture = createSignaledHostCommandFixture();
|
||||
let runnerPid = 0;
|
||||
let grandchildPid = 0;
|
||||
|
|
@ -2327,21 +2362,28 @@ if (commandArgs[0] === "list") {
|
|||
try {
|
||||
runnerPid = fixture.runner.pid ?? 0;
|
||||
expect(runnerPid).toBeGreaterThan(0);
|
||||
await waitFor(
|
||||
() => existsSync(fixture.readyPath) && existsSync(fixture.grandchildPidPath),
|
||||
2_000,
|
||||
);
|
||||
await withinTest(fixture.ready, signal);
|
||||
grandchildPid = Number.parseInt(readFileSync(fixture.grandchildPidPath, "utf8"), 10);
|
||||
|
||||
process.kill(-runnerPid, "SIGTERM");
|
||||
|
||||
await expect(waitForProcessClose(fixture.runner, 3_000)).resolves.toEqual({
|
||||
await expect(withinTest(fixture.closed, signal)).resolves.toEqual({
|
||||
code: null,
|
||||
signal: "SIGTERM",
|
||||
});
|
||||
await waitFor(() => !isProcessAlive(grandchildPid), 3_000);
|
||||
await waitForProcessGone(grandchildPid, signal);
|
||||
} finally {
|
||||
if (!grandchildPid && existsSync(fixture.grandchildPidPath)) {
|
||||
grandchildPid = Number.parseInt(readFileSync(fixture.grandchildPidPath, "utf8"), 10);
|
||||
}
|
||||
forceKillSignaledFixture(runnerPid, grandchildPid, true);
|
||||
if (existsSync(fixture.parentPidPath)) {
|
||||
const parentPid = Number.parseInt(readFileSync(fixture.parentPidPath, "utf8"), 10);
|
||||
if (isProcessAlive(parentPid)) {
|
||||
process.kill(-parentPid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
await fixture.closed;
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import {
|
|||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { basename, delimiter, dirname, join, resolve } from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
|
||||
import {
|
||||
expandPluginSdkApiDiffSet,
|
||||
selectPluginSdkApiReleaseEvidence,
|
||||
|
|
@ -20,11 +20,31 @@ import {
|
|||
resolveRuntimeWorkerArgv,
|
||||
resolveRuntimeWorkerUrl,
|
||||
} from "../../src/infra/runtime-worker-url.js";
|
||||
import { withTestTimeout } from "../helpers/promise.js";
|
||||
import {
|
||||
fixtureReceiptClientSource,
|
||||
openFixtureReceiptChannel,
|
||||
type FixtureReceiptChannel,
|
||||
} from "../helpers/fixture-receipts.js";
|
||||
import { withinTest } from "../helpers/promise.js";
|
||||
import { withRuntimePreload } from "../helpers/runtime-preload.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const pendingChildCompletions = new Set<Promise<unknown>>();
|
||||
const tempDirs = useAutoCleanupTempDirTracker((cleanup) =>
|
||||
afterEach(async () => {
|
||||
// Timed-out bodies can still be joining installers and deleting worktrees in finally.
|
||||
await Promise.allSettled(pendingChildCompletions);
|
||||
cleanup();
|
||||
}),
|
||||
);
|
||||
|
||||
function ownChildCompletion<T>(completion: Promise<T>): Promise<T> {
|
||||
const owned = completion.finally(() => {
|
||||
pendingChildCompletions.delete(owned);
|
||||
});
|
||||
pendingChildCompletions.add(owned);
|
||||
return owned;
|
||||
}
|
||||
const emptyDiff = {
|
||||
entrypointsAdded: [],
|
||||
entrypointsRemoved: [],
|
||||
|
|
@ -78,24 +98,48 @@ function runCli(repo: string, runnerTemp: string, binDir: string, args: string[]
|
|||
);
|
||||
}
|
||||
|
||||
async function waitFor(
|
||||
check: () => boolean,
|
||||
timeoutMs: number,
|
||||
let receipts: FixtureReceiptChannel;
|
||||
beforeAll(async () => {
|
||||
receipts = await openFixtureReceiptChannel();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await receipts.close();
|
||||
});
|
||||
|
||||
function writeInstallReceipt(binDir: string): string {
|
||||
const receiptPath = join(binDir, "install-receipt.mjs");
|
||||
writeFileSync(
|
||||
receiptPath,
|
||||
`${fixtureReceiptClientSource(receipts.endpoint)}
|
||||
sendReceipt(process.argv[2], "ready");
|
||||
// This reporter is a short-lived shell child: retain the socket until its write drains.
|
||||
fixtureReceiptSocket.ref();
|
||||
fixtureReceiptSocket.end();
|
||||
`,
|
||||
);
|
||||
return `'${process.execPath.replace(/'/gu, `'\\''`)}' '${receiptPath.replace(/'/gu, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
async function installBeforeSettlement(
|
||||
marker: string,
|
||||
close: PromiseLike<unknown>,
|
||||
label = "Plugin SDK API diff child",
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (!check()) {
|
||||
if (Date.now() >= deadline) {
|
||||
throw new Error(`timed out waiting for ${label}`);
|
||||
}
|
||||
await new Promise((resolveWait) => {
|
||||
setTimeout(resolveWait, 25);
|
||||
});
|
||||
}
|
||||
// The fixture writes its marker before reporting; socket delivery may follow CLI close.
|
||||
await Promise.race([
|
||||
receipts.waitFor(marker, "ready"),
|
||||
Promise.resolve(close).then(() => {
|
||||
if (!existsSync(marker)) {
|
||||
throw new Error(`timed out waiting for ${label}`);
|
||||
}
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
describe("Plugin SDK API diff CLI", () => {
|
||||
it("finishes every revision install before starting declaration rendering", async () => {
|
||||
it("finishes every revision install before starting declaration rendering", async ({
|
||||
signal,
|
||||
}) => {
|
||||
const repo = tempDirs.make("plugin-sdk-install-order-repo-");
|
||||
const runnerTemp = tempDirs.make("plugin-sdk-install-order-temp-");
|
||||
const binDir = tempDirs.make("plugin-sdk-install-order-bin-");
|
||||
|
|
@ -142,6 +186,7 @@ if mkdir "$PNPM_MARKER" 2>/dev/null; then
|
|||
exit 0
|
||||
fi
|
||||
: > "$PNPM_BLOCKED"
|
||||
${writeInstallReceipt(binDir)} "$PNPM_BLOCKED"
|
||||
while [ ! -e "$PNPM_RELEASE" ]; do sleep 0.05; done
|
||||
`,
|
||||
);
|
||||
|
|
@ -207,18 +252,24 @@ if (process.argv.includes("--render-root")) {
|
|||
stdio: ["ignore", "ignore", "pipe"],
|
||||
},
|
||||
);
|
||||
const close = ownChildCompletion(
|
||||
new Promise<number | null>((resolveClose) => {
|
||||
child.once("close", resolveClose);
|
||||
}),
|
||||
);
|
||||
let stderr = "";
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
const close = new Promise<number | null>((resolveClose) => {
|
||||
child.once("close", resolveClose);
|
||||
});
|
||||
|
||||
let exitCode: number | null = null;
|
||||
try {
|
||||
await waitFor(() => existsSync(blockedMarker), 15_000, "second revision install");
|
||||
exitCode = await withTestTimeout(close, 30_000, "Plugin SDK API diff did not finish");
|
||||
await withinTest(
|
||||
installBeforeSettlement(blockedMarker, close, "second revision install"),
|
||||
signal,
|
||||
);
|
||||
exitCode = await withinTest(close, signal);
|
||||
} finally {
|
||||
writeFileSync(releaseMarker, "release\n");
|
||||
if (child.exitCode === null) {
|
||||
|
|
@ -329,7 +380,7 @@ if (process.argv.includes("--render-root")) {
|
|||
35_000,
|
||||
);
|
||||
|
||||
it("interrupts a running child and removes its registered worktree", async () => {
|
||||
it("interrupts a running child and removes its registered worktree", async ({ signal }) => {
|
||||
// Keep revision checkout bounded so startup reaches the child this test cancels.
|
||||
const repo = tempDirs.make("plugin-sdk-api-diff-repo-");
|
||||
const runnerTemp = tempDirs.make("plugin-sdk-api-diff-temp-");
|
||||
|
|
@ -349,7 +400,7 @@ if (process.argv.includes("--render-root")) {
|
|||
const fakePnpm = join(binDir, "pnpm");
|
||||
writeFileSync(
|
||||
fakePnpm,
|
||||
"#!/bin/sh\n: > \"$PNPM_MARKER\"\ntrap 'exit 143' INT TERM\nwhile :; do sleep 1; done\n",
|
||||
`#!/bin/sh\ntrap 'exit 143' INT TERM\n: > "$PNPM_MARKER"\n${writeInstallReceipt(binDir)} "$PNPM_MARKER"\nwhile :; do sleep 1; done\n`,
|
||||
);
|
||||
chmodSync(fakePnpm, 0o755);
|
||||
|
||||
|
|
@ -379,19 +430,22 @@ if (process.argv.includes("--render-root")) {
|
|||
);
|
||||
|
||||
let closed = false;
|
||||
const close = ownChildCompletion(
|
||||
new Promise<number | null>((resolveClose) => {
|
||||
child.once("close", (code) => {
|
||||
closed = true;
|
||||
resolveClose(code);
|
||||
});
|
||||
}),
|
||||
);
|
||||
let stderr = "";
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
const close = new Promise<number | null>((resolveClose) => {
|
||||
child.once("close", (code) => {
|
||||
closed = true;
|
||||
resolveClose(code);
|
||||
});
|
||||
});
|
||||
|
||||
try {
|
||||
await waitFor(() => existsSync(pnpmMarker) || closed, 10_000);
|
||||
await withinTest(installBeforeSettlement(pnpmMarker, close), signal);
|
||||
expect(closed, stderr).toBe(false);
|
||||
const revisionRoot = git(repo, ["worktree", "list", "--porcelain", "-z"])
|
||||
.split("\0")
|
||||
|
|
@ -403,7 +457,7 @@ if (process.argv.includes("--render-root")) {
|
|||
expect(existsSync(temporaryRoot)).toBe(true);
|
||||
const interruptedAt = Date.now();
|
||||
child.kill("SIGTERM");
|
||||
const exitCode = await withTestTimeout(close, 5_000, "Plugin SDK API diff ignored SIGTERM");
|
||||
const exitCode = await withinTest(close, signal);
|
||||
|
||||
expect(exitCode).toBe(143);
|
||||
expect(Date.now() - interruptedAt).toBeLessThan(5_000);
|
||||
|
|
@ -414,7 +468,8 @@ if (process.argv.includes("--render-root")) {
|
|||
expect(readFileSync(runnerSentinel, "utf8")).toBe("preserve\n");
|
||||
} finally {
|
||||
if (!closed) {
|
||||
child.kill("SIGKILL");
|
||||
// Let the CLI join its installer and remove revision worktrees on test cancellation.
|
||||
child.kill("SIGTERM");
|
||||
await close;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { loadInstalledPluginIndex } from "../../src/plugins/installed-plugin-ind
|
|||
import { createInstalledPluginOwnershipResolver } from "../../src/plugins/installed-plugin-package-ownership.js";
|
||||
import { closeOpenClawStateDatabaseByPath } from "../../src/state/openclaw-state-db-cache.js";
|
||||
import { openOpenClawStateDatabase } from "../../src/state/openclaw-state-db.js";
|
||||
import { awaitGateBeforeSettlement, withinTest } from "../helpers/promise.js";
|
||||
|
||||
const PLUGIN_UPDATE_SCENARIO_SCRIPT = "scripts/e2e/lib/plugin-update/unchanged-scenario.sh";
|
||||
const CORRUPT_UPDATE_SCENARIO_SCRIPT = "scripts/e2e/lib/plugin-update/corrupt-update-scenario.sh";
|
||||
|
|
@ -146,17 +147,22 @@ fi
|
|||
}
|
||||
}
|
||||
|
||||
async function waitForPortFile(portFile: string): Promise<number> {
|
||||
for (let attempt = 0; attempt < 50; attempt += 1) {
|
||||
async function waitForPortFile(portFile: string, signal: AbortSignal): Promise<number> {
|
||||
// The production registry publishes only this file; no IPC or stdout readiness is exposed.
|
||||
while (!signal.aborted) {
|
||||
if (existsSync(portFile)) {
|
||||
const port = Number.parseInt(readFileSync(portFile, "utf8"), 10);
|
||||
if (Number.isInteger(port) && port > 0) {
|
||||
return port;
|
||||
}
|
||||
}
|
||||
await delay(50);
|
||||
await delay(10, undefined, { signal }).catch((error: unknown) => {
|
||||
if (!signal.aborted) {
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
throw new Error("registry did not write a port file");
|
||||
throw new Error("registry did not write a port file", { cause: signal.reason });
|
||||
}
|
||||
|
||||
describe("plugin update unchanged Docker E2E", () => {
|
||||
|
|
@ -249,14 +255,26 @@ describe("plugin update unchanged Docker E2E", () => {
|
|||
expect(script).not.toContain("cat /tmp/openclaw-e2e-registry.log");
|
||||
});
|
||||
|
||||
it("serves plugin metadata from an ephemeral registry port", async () => {
|
||||
it("serves plugin metadata from an ephemeral registry port", async ({ signal }) => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-plugin-update-registry-"));
|
||||
const portFile = path.join(root, "registry.port");
|
||||
const child = spawn("node", [PLUGIN_UPDATE_REGISTRY_SCRIPT, portFile], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
const waiting = new AbortController();
|
||||
child.once("error", (error) => waiting.abort(error));
|
||||
const closed = new Promise<void>((resolve) => {
|
||||
child.once("close", () => resolve());
|
||||
});
|
||||
try {
|
||||
const port = await waitForPortFile(portFile);
|
||||
const port = await withinTest(
|
||||
awaitGateBeforeSettlement(
|
||||
waitForPortFile(portFile, AbortSignal.any([signal, waiting.signal])),
|
||||
closed,
|
||||
"registry did not write a port file",
|
||||
),
|
||||
signal,
|
||||
);
|
||||
|
||||
const response = await fetch(`http://127.0.0.1:${port}/@example%2flossless-claw`);
|
||||
expect(response.status).toBe(200);
|
||||
|
|
@ -267,7 +285,9 @@ describe("plugin update unchanged Docker E2E", () => {
|
|||
`http://127.0.0.1:${port}/@example/lossless-claw/-/lossless-claw-0.9.0.tgz`,
|
||||
);
|
||||
} finally {
|
||||
waiting.abort();
|
||||
child.kill("SIGTERM");
|
||||
await closed;
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import {
|
|||
shouldSkipLinuxArmAndroidGradle,
|
||||
splitAndroidGradleArgs,
|
||||
} from "../../scripts/run-android-gradle.mts";
|
||||
import { withinTest } from "../helpers/promise.js";
|
||||
|
||||
const posixIt = process.platform === "win32" ? it.skip : it;
|
||||
|
||||
|
|
@ -99,9 +100,11 @@ describe("run-android-gradle", () => {
|
|||
});
|
||||
});
|
||||
|
||||
posixIt("terminates the active command tree when the wrapper is terminated", async () => {
|
||||
const moduleUrl = pathToFileURL(path.resolve("scripts/run-android-gradle.mts")).href;
|
||||
const childSource = `
|
||||
posixIt(
|
||||
"terminates the active command tree when the wrapper is terminated",
|
||||
async ({ signal }) => {
|
||||
const moduleUrl = pathToFileURL(path.resolve("scripts/run-android-gradle.mts")).href;
|
||||
const childSource = `
|
||||
const { spawn } = require("node:child_process");
|
||||
const descendant = spawn(process.execPath, [
|
||||
"-e",
|
||||
|
|
@ -112,7 +115,7 @@ process.stdout.write(
|
|||
);
|
||||
setInterval(() => {}, 1_000);
|
||||
`;
|
||||
const runnerSource = `
|
||||
const runnerSource = `
|
||||
import { run } from ${JSON.stringify(moduleUrl)};
|
||||
process.exitCode = await run(
|
||||
process.execPath,
|
||||
|
|
@ -120,45 +123,48 @@ process.exitCode = await run(
|
|||
process.cwd(),
|
||||
);
|
||||
`;
|
||||
const runner = spawn(
|
||||
process.execPath,
|
||||
["--import", "tsx", "--input-type=module", "-e", runnerSource],
|
||||
{ stdio: ["ignore", "pipe", "ignore"] },
|
||||
);
|
||||
const runnerPid = expectPid(runner.pid);
|
||||
const processTreeReady = readProcessTree(runner);
|
||||
let childPid = 0;
|
||||
let descendantPid = 0;
|
||||
const runner = spawn(
|
||||
process.execPath,
|
||||
["--import", "tsx", "--input-type=module", "-e", runnerSource],
|
||||
{ stdio: ["ignore", "pipe", "ignore"] },
|
||||
);
|
||||
const runnerPid = expectPid(runner.pid);
|
||||
const processTreeReady = readProcessTree(runner);
|
||||
const closed = waitForClose(runner);
|
||||
let childPid = 0;
|
||||
let descendantPid = 0;
|
||||
|
||||
try {
|
||||
const processTree = await processTreeReady;
|
||||
childPid = processTree.childPid;
|
||||
descendantPid = processTree.descendantPid;
|
||||
expect(Number.isInteger(childPid)).toBe(true);
|
||||
expect(Number.isInteger(descendantPid)).toBe(true);
|
||||
expect(isProcessAlive(childPid)).toBe(true);
|
||||
expect(isProcessAlive(descendantPid)).toBe(true);
|
||||
try {
|
||||
const processTree = await withinTest(processTreeReady, signal);
|
||||
childPid = processTree.childPid;
|
||||
descendantPid = processTree.descendantPid;
|
||||
expect(Number.isInteger(childPid)).toBe(true);
|
||||
expect(Number.isInteger(descendantPid)).toBe(true);
|
||||
expect(isProcessAlive(childPid)).toBe(true);
|
||||
expect(isProcessAlive(descendantPid)).toBe(true);
|
||||
|
||||
process.kill(runnerPid, "SIGTERM");
|
||||
const result = await waitForClose(runner);
|
||||
await waitFor(() => !isProcessAlive(childPid), 1_500);
|
||||
await waitFor(() => !isProcessAlive(descendantPid), 1_500);
|
||||
process.kill(runnerPid, "SIGTERM");
|
||||
const result = await withinTest(closed, signal);
|
||||
// The owner joins the tree, but Darwin may still expose a reaping descendant's PID.
|
||||
await waitForProcessGone(descendantPid, signal);
|
||||
|
||||
expect(isProcessAlive(childPid)).toBe(false);
|
||||
expect(isProcessAlive(descendantPid)).toBe(false);
|
||||
expect(result).toEqual({ code: 143, signal: null });
|
||||
} finally {
|
||||
if (isProcessAlive(runnerPid)) {
|
||||
process.kill(runnerPid, "SIGKILL");
|
||||
expect(isProcessAlive(childPid)).toBe(false);
|
||||
expect(isProcessAlive(descendantPid)).toBe(false);
|
||||
expect(result).toEqual({ code: 143, signal: null });
|
||||
} finally {
|
||||
if (isProcessAlive(runnerPid)) {
|
||||
process.kill(runnerPid, "SIGKILL");
|
||||
}
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
if (descendantPid && isProcessAlive(descendantPid)) {
|
||||
process.kill(descendantPid, "SIGKILL");
|
||||
}
|
||||
await closed;
|
||||
}
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
if (descendantPid && isProcessAlive(descendantPid)) {
|
||||
process.kill(descendantPid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("reports spawn errors and returns a failure status", async () => {
|
||||
const error = vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
|
|
@ -209,7 +215,9 @@ async function readProcessTree(child: ReturnType<typeof spawn>): Promise<{
|
|||
try {
|
||||
resolve(JSON.parse(output.slice(0, newline)));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
reject(
|
||||
error instanceof Error ? error : new Error("invalid process tree", { cause: error }),
|
||||
);
|
||||
}
|
||||
};
|
||||
stdout.on("data", onData);
|
||||
|
|
@ -217,13 +225,13 @@ async function readProcessTree(child: ReturnType<typeof spawn>): Promise<{
|
|||
});
|
||||
}
|
||||
|
||||
async function waitFor(condition: () => boolean, timeoutMs = 3_000): Promise<void> {
|
||||
const startedAt = Date.now();
|
||||
while (!condition()) {
|
||||
if (Date.now() - startedAt > timeoutMs) {
|
||||
throw new Error("timed out waiting for condition");
|
||||
}
|
||||
await delay(5);
|
||||
async function waitForProcessGone(pid: number, signal: AbortSignal): Promise<void> {
|
||||
while (isProcessAlive(pid)) {
|
||||
await delay(5, undefined, { signal }).catch((error: unknown) => {
|
||||
throw new Error(`timed out waiting for condition: process ${pid} is still alive`, {
|
||||
cause: error,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import {
|
|||
type VitestWorkerManifest,
|
||||
} from "../../scripts/lib/vitest-worker-artifacts.mts";
|
||||
import { createVitestWorkerRun } from "../../scripts/lib/vitest-worker-run.mts";
|
||||
import { createDeferred, withTestTimeout } from "../helpers/promise.js";
|
||||
import { awaitGateBeforeSettlement, createDeferred, withinTest } from "../helpers/promise.js";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
|
@ -49,9 +49,9 @@ it("keeps the runner event loop responsive while verifying a completed generatio
|
|||
}
|
||||
});
|
||||
|
||||
it.each(["inputs", "outputs"] as const)(
|
||||
it.for(["inputs", "outputs"] as const)(
|
||||
"drains active %s reads before failed verification releases the generation",
|
||||
async (group) => {
|
||||
async (group, { signal }) => {
|
||||
const owner = createVitestWorkerRun();
|
||||
const directory = owner.descriptor.directory;
|
||||
const files = group === "inputs" ? directory : path.join(directory, "dist");
|
||||
|
|
@ -100,10 +100,13 @@ it.each(["inputs", "outputs"] as const)(
|
|||
completed = true;
|
||||
});
|
||||
try {
|
||||
await withTestTimeout(
|
||||
Promise.all([started.promise, failedRead.promise]),
|
||||
5_000,
|
||||
"verification did not admit both reads",
|
||||
await withinTest(
|
||||
awaitGateBeforeSettlement(
|
||||
Promise.all([started.promise, failedRead.promise]),
|
||||
disposal,
|
||||
"verification did not admit both reads",
|
||||
),
|
||||
signal,
|
||||
);
|
||||
await nextTurn();
|
||||
expect(completed).toBe(false);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue