diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 96fd8958fc0a..11e7727539df 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -4318,7 +4318,7 @@ jobs: profiles: stable full - suite_id: native-live-src-infra label: Native live infra - command: OPENCLAW_LIVE_APNS_REACHABILITY=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra + command: OPENCLAW_LIVE_APNS_REACHABILITY=1 OPENCLAW_LIVE_SESSION_EVENT_WAKE=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra timeout_minutes: 45 profile_env_only: false profiles: stable full diff --git a/scripts/e2e/doctor-install-switch-docker.sh b/scripts/e2e/doctor-install-switch-docker.sh index 371869fea14d..a0b8310dc94c 100755 --- a/scripts/e2e/doctor-install-switch-docker.sh +++ b/scripts/e2e/doctor-install-switch-docker.sh @@ -30,7 +30,7 @@ docker_e2e_build_or_reuse "$IMAGE_NAME" doctor-switch "$ROOT_DIR/scripts/e2e/Doc echo "Running doctor install switch E2E..." # Maintenance loads the installed unit's canonical PATH. Mount the shims there # so the unprivileged container keeps using the fixture manager during inspection. -SHIM_DIR="$ROOT_DIR/scripts/e2e/lib/doctor-install-switch/shims" +SHIM_DIR="$TARGET_CONTRACT_DIR/shims" docker_e2e_run_with_harness \ -v "$SHIM_DIR/systemctl:/usr/local/bin/systemctl:ro" \ -v "$SHIM_DIR/loginctl:/usr/local/bin/loginctl:ro" \ diff --git a/scripts/test-live-shard.mts b/scripts/test-live-shard.mts index 4d72c61d4df8..4df49c1b583e 100644 --- a/scripts/test-live-shard.mts +++ b/scripts/test-live-shard.mts @@ -410,6 +410,7 @@ export function resolveLiveShardPreparation(files: string[]): LiveShardPreparati if ( files.some(isSourceGatewayLiveTest) || files.some((file) => file.startsWith("test/e2e/qa-lab/runtime/")) || + files.includes("src/infra/heartbeat-runner.live.test.ts") || files.includes("src/agents/tools/image-tool.providers.live.test.ts") || files.includes("extensions/openai/openai.live.test.ts") ) { diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index c92ef6562893..67b4d86a1091 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -3525,11 +3525,21 @@ process.on("SIGTERM", () => { lane === "published" ? source.slice(0, source.indexOf("phase storage-preflight")) : `source ${shellQuote(OPENCLAW_E2E_INSTANCE_HELPER_PATH)}\nsource ${shellQuote(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH)}`; + // The published stop proof must inspect this fixture's listener, not a host Gateway. + const tcpProbeAdapter = ` +eval "$(declare -f openclaw_e2e_probe_tcp | sed '1s/openclaw_e2e_probe_tcp/fixture_probe_tcp/')" +openclaw_e2e_probe_tcp() { + local port="$2" + [ "$port" != 18789 ] || port="$(cat "$PORT_FILE")" + fixture_probe_tcp "$1" "$port" "\${3:-400}" +} +`; const script = `${setup} trap - EXIT ERR INT TERM assert_prepublish_fixture_idle() { :; } assert_baseline_state() { :; } check_gateway_status() { :; } +${tcpProbeAdapter} # This fixture chooses an ephemeral port; retain the actual readiness implementation. eval "$(declare -f openclaw_e2e_wait_gateway_ready | sed '1s/openclaw_e2e_wait_gateway_ready/fixture_wait_gateway_ready/')" openclaw_e2e_wait_gateway_ready() { @@ -3549,6 +3559,17 @@ ${lane === "published" ? "prepare_update_restart_probe" : 'prepare_update_restar encoding: "utf8", timeout: 40_000, }); + const probeListener = () => + spawnSync( + "bash", + [ + "-c", + `source ${shellQuote(OPENCLAW_E2E_INSTANCE_HELPER_PATH)} +${tcpProbeAdapter} +openclaw_e2e_probe_tcp 127.0.0.1 18789 400`, + ], + { env, encoding: "utf8", timeout: 5_000 }, + ); const records = (): Array<{ pid: number; managed: boolean }> => existsSync(startsPath) ? readFileSync(startsPath, "utf8") @@ -3593,7 +3614,9 @@ ${lane === "published" ? "prepare_update_restart_probe" : 'prepare_update_restar expect(replacement.managed).toBe(true); expect(replacement.pid).not.toBe(initial.pid); expect(isProcessRunning(initial.pid)).toBe(false); + expect(probeListener().status).toBe(0); expect(systemctl("stop", "openclaw-gateway.service").status).toBe(0); + expect(probeListener().status).toBe(1); await expect(fetch(url, { signal: AbortSignal.timeout(1_000) })).rejects.toThrow(); } finally { systemctl("stop", "openclaw-gateway.service"); @@ -7830,6 +7853,64 @@ done expect(staleObject.stdout).not.toContain('"loaded"'); }); + it.each(["selected", "default"] as const)( + "mounts the %s Doctor contract and canonical-path shims from the same checkout", + (targetMode) => { + const workDir = tempDirs.make("openclaw-doctor-contract-mounts-"); + const targetRoot = + targetMode === "selected" ? join(workDir, "selected target") : process.cwd(); + const contractPath = "scripts/e2e/lib/doctor-install-switch"; + const shimNames = ["systemctl", "loginctl", "busctl", "systemd-exec-start.mjs"]; + if (targetMode === "selected") { + const targetContract = join(targetRoot, contractPath); + mkdirSync(join(targetContract, "shims"), { recursive: true }); + copyFileSync(DOCTOR_SWITCH_SCENARIO_PATH, join(targetContract, "scenario.sh")); + for (const name of shimNames) { + copyFileSync(join(contractPath, "shims", name), join(targetContract, "shims", name)); + } + } + writeFileSync(join(workDir, "openclaw-current.tgz"), "unused package transport fixture"); + writeExecutables(join(workDir, "bin"), { + timeout: PASSTHROUGH_TIMEOUT_SCRIPT, + docker: `#!/bin/bash +set -euo pipefail +case "$1 \${2:-}" in + "image inspect") exit 0 ;; + "run "*) printf '%s\\0' "$@" >"$TMPDIR/docker-run-args" ;; + *) exit 9 ;; +esac +`, + }); + const script = repoShell(workDir)` +export PATH="$TMPDIR/bin:$PATH" +export OPENCLAW_SKIP_DOCKER_BUILD=1 +export OPENCLAW_DOCKER_E2E_IMAGE=doctor-contract-fixture +export OPENCLAW_CURRENT_PACKAGE_TGZ="$TMPDIR/openclaw-current.tgz" +unset DOCKER_E2E_HARNESS_ROOT_DIR OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR OPENCLAW_DOCKER_E2E_REPO_ROOT +${targetMode === "selected" ? `export OPENCLAW_DOCKER_E2E_REPO_ROOT=${shellQuote(targetRoot)}` : ""} +bash "$ROOT_DIR/scripts/e2e/doctor-install-switch-docker.sh" +`; + const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], { + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + const args = readFileSync(join(workDir, "docker-run-args"), "utf8").split("\0"); + const mounts = args.flatMap((arg, index) => (arg === "-v" ? [args[index + 1]] : [])); + // Service inspection uses /usr/local/bin, outside the target directory overlay. + // Both views must select one fixture while shared helpers stay trusted. + expect(mounts.filter((mount) => mount?.includes(":/usr/local/bin/"))).toEqual( + shimNames.map( + (name) => `${targetRoot}/${contractPath}/shims/${name}:/usr/local/bin/${name}:ro`, + ), + ); + expect(mounts).toContain(`${targetRoot}/${contractPath}:/app/${contractPath}:ro`); + expect(mounts).toContain(`${process.cwd()}/scripts/e2e:/app/scripts/e2e:ro`); + expect(mounts).toContain(`${process.cwd()}/scripts/lib:/app/scripts/lib:ro`); + expect(args).toContain("doctor-contract-fixture"); + expect(args).toContain(`${contractPath}/scenario.sh`); + }, + ); + it("routes doctor install switch commands through the E2E timeout helper", () => { const runner = readFileSync(DOCTOR_SWITCH_DOCKER_E2E_PATH, "utf8"); const scenario = readFileSync(DOCTOR_SWITCH_SCENARIO_PATH, "utf8"); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 5da5afabec3b..6d34c4623bda 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -9854,7 +9854,7 @@ describe("package artifact reuse", () => { ); expect(workflow).toContain("suite_id: native-live-src-infra"); expect(workflow).toContain( - "command: OPENCLAW_LIVE_APNS_REACHABILITY=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra", + "command: OPENCLAW_LIVE_APNS_REACHABILITY=1 OPENCLAW_LIVE_SESSION_EVENT_WAKE=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra", ); expect(workflow).toContain("suite_id: native-live-src-gateway-profiles-anthropic-smoke"); expect(workflow).toContain("OPENCLAW_LIVE_GATEWAY_SETUP_TIMEOUT_MS=300000"); diff --git a/test/scripts/test-live-shard.test.ts b/test/scripts/test-live-shard.test.ts index 4c1792b1324b..8c1a45bcdd52 100644 --- a/test/scripts/test-live-shard.test.ts +++ b/test/scripts/test-live-shard.test.ts @@ -1,6 +1,14 @@ // Test Live Shard tests cover test live shard script behavior. import { spawn, spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -256,7 +264,9 @@ describe("scripts/test-live-shard", () => { it.each([ "native-live-src-gateway-core", "native-live-src-gateway-backends", + "native-live-src-infra", "native-live-test", + "src/infra/heartbeat-runner.live.test.ts", "test/e2e/qa-lab/runtime/worker-skill-resources.live.test.ts", "test/e2e/qa-lab/runtime/gateway-node-mcp.live.test.ts", ])("prepares the built gateway runtime before %s starts Vitest", (target) => { @@ -276,9 +286,7 @@ describe("scripts/test-live-shard", () => { profile: "sourcePerformance", requiredArtifact: "dist/.runtime-postbuildstamp", }); - expect( - resolveLiveShardPreparation(selectLiveShardFiles("native-live-src-infra", allFiles)), - ).toBeNull(); + expect(resolveLiveShardPreparation(["src/infra/push-apns-http2.live.test.ts"])).toBeNull(); }); it("runs the frozen candidate's available build entrypoint and advertised profile", () => { @@ -671,32 +679,57 @@ describe("scripts/test-live-shard", () => { async () => { const root = mkdtempSync(path.join(tmpdir(), "openclaw-live-shard-signal-")); const fakePnpmPath = path.join(root, "pnpm"); + const argsPath = path.join(root, "args.json"); const childPidPath = path.join(root, "child.pid"); const descendantPidPath = path.join(root, "descendant.pid"); const signaledPath = path.join(root, "signaled"); let childPid = 0; let descendantPid = 0; let runner: ReturnType | undefined; + let runnerClosed: Promise<{ code: number | null; signal: NodeJS.Signals | null }> | undefined; try { writeFakePnpm(fakePnpmPath); - runner = spawn(process.execPath, ["scripts/test-live-shard.mjs", "native-live-src-infra"], { - env: { - ...process.env, - OPENCLAW_FAKE_PNPM_DESCENDANT_PID_PATH: descendantPidPath, - OPENCLAW_FAKE_PNPM_PID_PATH: childPidPath, - OPENCLAW_FAKE_PNPM_SIGNALED_PATH: signaledPath, - npm_execpath: fakePnpmPath, + // Give the real shard selector an APNs-only inventory: readiness must belong + // to the live-test child, not a build prerequisite of another infra test. + mkdirSync(path.join(root, "src/infra"), { recursive: true }); + writeFileSync(path.join(root, "src/infra/push-apns-http2.live.test.ts"), ""); + const startedRunner = spawn( + process.execPath, + [path.resolve("scripts/test-live-shard.mjs"), "native-live-src-infra"], + { + cwd: root, + env: { + ...process.env, + OPENCLAW_FAKE_PNPM_ARGS_PATH: argsPath, + OPENCLAW_FAKE_PNPM_DESCENDANT_PID_PATH: descendantPidPath, + OPENCLAW_FAKE_PNPM_PID_PATH: childPidPath, + OPENCLAW_FAKE_PNPM_SIGNALED_PATH: signaledPath, + npm_execpath: fakePnpmPath, + }, + stdio: "ignore", }, - stdio: "ignore", + ); + + runner = startedRunner; + runnerClosed = new Promise((resolve) => { + startedRunner.once("close", (code, signal) => resolve({ code, signal })); }); childPid = await waitForPidFile(childPidPath, 5_000); descendantPid = await waitForPidFile(descendantPidPath, 5_000); + expect(JSON.parse(readFileSync(argsPath, "utf8")).slice(0, 3)).toEqual([ + "test:live", + "--", + "src/infra/push-apns-http2.live.test.ts", + ]); runner.kill("SIGTERM"); - await expect(waitForClose(runner)).resolves.toEqual({ code: null, signal: "SIGTERM" }); + await expect(waitForClose(runnerClosed)).resolves.toEqual({ + code: null, + signal: "SIGTERM", + }); // Creation precedes the synchronous write; wait for the signal receipt itself. await waitFor( () => existsSync(signaledPath) && readFileSync(signaledPath, "utf8") === "SIGTERM", @@ -705,16 +738,32 @@ describe("scripts/test-live-shard", () => { await waitFor(() => !isProcessAlive(childPid), 5_000); await waitFor(() => !isProcessAlive(descendantPid), 5_000); } finally { - if (runner?.pid && isProcessAlive(runner.pid)) { - process.kill(runner.pid, "SIGKILL"); + try { + if (runner?.pid && isProcessAlive(runner.pid)) { + runner.kill("SIGTERM"); + } + // Join the shim so it can forward cancellation to its detached child group + // before the fixture directory (and its process receipts) disappears. + if (runnerClosed) { + await waitForClose(runnerClosed); + } + } finally { + // A failed shim join must not skip cleanup of already observed descendants. + for (const pid of [childPid, descendantPid]) { + if (pid && isProcessAlive(pid)) { + process.kill(pid, "SIGKILL"); + } + } + try { + await Promise.all( + [childPid, descendantPid] + .filter((pid) => pid > 0) + .map((pid) => waitFor(() => !isProcessAlive(pid), 5_000)), + ); + } finally { + rmSync(root, { force: true, recursive: true }); + } } - if (childPid && isProcessAlive(childPid)) { - process.kill(childPid, "SIGKILL"); - } - if (descendantPid && isProcessAlive(descendantPid)) { - process.kill(descendantPid, "SIGKILL"); - } - rmSync(root, { force: true, recursive: true }); } }, ); @@ -727,6 +776,8 @@ function writeFakePnpm(filePath: string): void { "#!/usr/bin/env node", 'const { spawn } = require("node:child_process");', 'const fs = require("node:fs");', + 'if (process.argv[2] !== "test:live") throw new Error("Expected the live-test invocation");', + "fs.writeFileSync(process.env.OPENCLAW_FAKE_PNPM_ARGS_PATH, JSON.stringify(process.argv.slice(2)));", "const child = spawn(process.execPath, [", ' "-e",', " \"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);\",", @@ -755,13 +806,11 @@ async function waitFor(condition: () => boolean, timeoutMs: number): Promise, + completion: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, timeoutMs = 5_000, ): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { return await Promise.race([ - new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { - child.once("close", (code, signal) => resolve({ code, signal })); - }), + completion, delay(timeoutMs, undefined, { ref: false }).then(() => { throw new Error("timed out waiting for child close"); }),