From 7444f28d2dc5da94fc7e52c4e240ae980968d29e Mon Sep 17 00:00:00 2001 From: RoboClaw Date: Thu, 10 Sep 2026 13:26:13 -0700 Subject: [PATCH] fix(ci): prepare selected release native fixtures (#144338) * fix(ci): prepare selected release native fixtures Enable and prepare the selected native heartbeat live test through its existing runtime build owner. Keep Doctor scenario and canonical-path service shims on the same selected checkout while retaining trusted shared helpers. Complete the managed test's ephemeral TCP endpoint adapter so shutdown verification cannot observe an unrelated host Gateway. Qualification context: https://github.com/openclaw/openclaw/actions/runs/34507645027 Co-authored-by: steipete <58493+steipete@users.noreply.github.com> Co-authored-by: vincentkoc <25068+vincentkoc@users.noreply.github.com> * test(ci): bind live-shard cancellation to the test child Use an isolated APNs-only inventory through the existing shard selector so build preparation cannot masquerade as live-child readiness. Assert the actual test:live arguments and join owned processes before fixture cleanup. Preserve the original signal, descendant-death, and timeout assertions. The original 20-file CI order passes all 332 tests after the correction. Co-authored-by: steipete <58493+steipete@users.noreply.github.com> Co-authored-by: vincentkoc <25068+vincentkoc@users.noreply.github.com> --------- Co-authored-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Co-authored-by: steipete <58493+steipete@users.noreply.github.com> Co-authored-by: vincentkoc <25068+vincentkoc@users.noreply.github.com> --- .../openclaw-live-and-e2e-checks-reusable.yml | 2 +- scripts/e2e/doctor-install-switch-docker.sh | 2 +- scripts/test-live-shard.mts | 1 + test/scripts/docker-build-helper.test.ts | 81 ++++++++++++++ .../package-acceptance-workflow.test.ts | 2 +- test/scripts/test-live-shard.test.ts | 101 +++++++++++++----- 6 files changed, 160 insertions(+), 29 deletions(-) diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 96fd8958fc0a..11e7727539df 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -4318,7 +4318,7 @@ jobs: profiles: stable full - suite_id: native-live-src-infra label: Native live infra - command: OPENCLAW_LIVE_APNS_REACHABILITY=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra + command: OPENCLAW_LIVE_APNS_REACHABILITY=1 OPENCLAW_LIVE_SESSION_EVENT_WAKE=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra timeout_minutes: 45 profile_env_only: false profiles: stable full diff --git a/scripts/e2e/doctor-install-switch-docker.sh b/scripts/e2e/doctor-install-switch-docker.sh index 371869fea14d..a0b8310dc94c 100755 --- a/scripts/e2e/doctor-install-switch-docker.sh +++ b/scripts/e2e/doctor-install-switch-docker.sh @@ -30,7 +30,7 @@ docker_e2e_build_or_reuse "$IMAGE_NAME" doctor-switch "$ROOT_DIR/scripts/e2e/Doc echo "Running doctor install switch E2E..." # Maintenance loads the installed unit's canonical PATH. Mount the shims there # so the unprivileged container keeps using the fixture manager during inspection. -SHIM_DIR="$ROOT_DIR/scripts/e2e/lib/doctor-install-switch/shims" +SHIM_DIR="$TARGET_CONTRACT_DIR/shims" docker_e2e_run_with_harness \ -v "$SHIM_DIR/systemctl:/usr/local/bin/systemctl:ro" \ -v "$SHIM_DIR/loginctl:/usr/local/bin/loginctl:ro" \ diff --git a/scripts/test-live-shard.mts b/scripts/test-live-shard.mts index 4d72c61d4df8..4df49c1b583e 100644 --- a/scripts/test-live-shard.mts +++ b/scripts/test-live-shard.mts @@ -410,6 +410,7 @@ export function resolveLiveShardPreparation(files: string[]): LiveShardPreparati if ( files.some(isSourceGatewayLiveTest) || files.some((file) => file.startsWith("test/e2e/qa-lab/runtime/")) || + files.includes("src/infra/heartbeat-runner.live.test.ts") || files.includes("src/agents/tools/image-tool.providers.live.test.ts") || files.includes("extensions/openai/openai.live.test.ts") ) { diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index c92ef6562893..67b4d86a1091 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -3525,11 +3525,21 @@ process.on("SIGTERM", () => { lane === "published" ? source.slice(0, source.indexOf("phase storage-preflight")) : `source ${shellQuote(OPENCLAW_E2E_INSTANCE_HELPER_PATH)}\nsource ${shellQuote(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH)}`; + // The published stop proof must inspect this fixture's listener, not a host Gateway. + const tcpProbeAdapter = ` +eval "$(declare -f openclaw_e2e_probe_tcp | sed '1s/openclaw_e2e_probe_tcp/fixture_probe_tcp/')" +openclaw_e2e_probe_tcp() { + local port="$2" + [ "$port" != 18789 ] || port="$(cat "$PORT_FILE")" + fixture_probe_tcp "$1" "$port" "\${3:-400}" +} +`; const script = `${setup} trap - EXIT ERR INT TERM assert_prepublish_fixture_idle() { :; } assert_baseline_state() { :; } check_gateway_status() { :; } +${tcpProbeAdapter} # This fixture chooses an ephemeral port; retain the actual readiness implementation. eval "$(declare -f openclaw_e2e_wait_gateway_ready | sed '1s/openclaw_e2e_wait_gateway_ready/fixture_wait_gateway_ready/')" openclaw_e2e_wait_gateway_ready() { @@ -3549,6 +3559,17 @@ ${lane === "published" ? "prepare_update_restart_probe" : 'prepare_update_restar encoding: "utf8", timeout: 40_000, }); + const probeListener = () => + spawnSync( + "bash", + [ + "-c", + `source ${shellQuote(OPENCLAW_E2E_INSTANCE_HELPER_PATH)} +${tcpProbeAdapter} +openclaw_e2e_probe_tcp 127.0.0.1 18789 400`, + ], + { env, encoding: "utf8", timeout: 5_000 }, + ); const records = (): Array<{ pid: number; managed: boolean }> => existsSync(startsPath) ? readFileSync(startsPath, "utf8") @@ -3593,7 +3614,9 @@ ${lane === "published" ? "prepare_update_restart_probe" : 'prepare_update_restar expect(replacement.managed).toBe(true); expect(replacement.pid).not.toBe(initial.pid); expect(isProcessRunning(initial.pid)).toBe(false); + expect(probeListener().status).toBe(0); expect(systemctl("stop", "openclaw-gateway.service").status).toBe(0); + expect(probeListener().status).toBe(1); await expect(fetch(url, { signal: AbortSignal.timeout(1_000) })).rejects.toThrow(); } finally { systemctl("stop", "openclaw-gateway.service"); @@ -7830,6 +7853,64 @@ done expect(staleObject.stdout).not.toContain('"loaded"'); }); + it.each(["selected", "default"] as const)( + "mounts the %s Doctor contract and canonical-path shims from the same checkout", + (targetMode) => { + const workDir = tempDirs.make("openclaw-doctor-contract-mounts-"); + const targetRoot = + targetMode === "selected" ? join(workDir, "selected target") : process.cwd(); + const contractPath = "scripts/e2e/lib/doctor-install-switch"; + const shimNames = ["systemctl", "loginctl", "busctl", "systemd-exec-start.mjs"]; + if (targetMode === "selected") { + const targetContract = join(targetRoot, contractPath); + mkdirSync(join(targetContract, "shims"), { recursive: true }); + copyFileSync(DOCTOR_SWITCH_SCENARIO_PATH, join(targetContract, "scenario.sh")); + for (const name of shimNames) { + copyFileSync(join(contractPath, "shims", name), join(targetContract, "shims", name)); + } + } + writeFileSync(join(workDir, "openclaw-current.tgz"), "unused package transport fixture"); + writeExecutables(join(workDir, "bin"), { + timeout: PASSTHROUGH_TIMEOUT_SCRIPT, + docker: `#!/bin/bash +set -euo pipefail +case "$1 \${2:-}" in + "image inspect") exit 0 ;; + "run "*) printf '%s\\0' "$@" >"$TMPDIR/docker-run-args" ;; + *) exit 9 ;; +esac +`, + }); + const script = repoShell(workDir)` +export PATH="$TMPDIR/bin:$PATH" +export OPENCLAW_SKIP_DOCKER_BUILD=1 +export OPENCLAW_DOCKER_E2E_IMAGE=doctor-contract-fixture +export OPENCLAW_CURRENT_PACKAGE_TGZ="$TMPDIR/openclaw-current.tgz" +unset DOCKER_E2E_HARNESS_ROOT_DIR OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR OPENCLAW_DOCKER_E2E_REPO_ROOT +${targetMode === "selected" ? `export OPENCLAW_DOCKER_E2E_REPO_ROOT=${shellQuote(targetRoot)}` : ""} +bash "$ROOT_DIR/scripts/e2e/doctor-install-switch-docker.sh" +`; + const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], { + encoding: "utf8", + }); + expect(result.status, result.stderr).toBe(0); + const args = readFileSync(join(workDir, "docker-run-args"), "utf8").split("\0"); + const mounts = args.flatMap((arg, index) => (arg === "-v" ? [args[index + 1]] : [])); + // Service inspection uses /usr/local/bin, outside the target directory overlay. + // Both views must select one fixture while shared helpers stay trusted. + expect(mounts.filter((mount) => mount?.includes(":/usr/local/bin/"))).toEqual( + shimNames.map( + (name) => `${targetRoot}/${contractPath}/shims/${name}:/usr/local/bin/${name}:ro`, + ), + ); + expect(mounts).toContain(`${targetRoot}/${contractPath}:/app/${contractPath}:ro`); + expect(mounts).toContain(`${process.cwd()}/scripts/e2e:/app/scripts/e2e:ro`); + expect(mounts).toContain(`${process.cwd()}/scripts/lib:/app/scripts/lib:ro`); + expect(args).toContain("doctor-contract-fixture"); + expect(args).toContain(`${contractPath}/scenario.sh`); + }, + ); + it("routes doctor install switch commands through the E2E timeout helper", () => { const runner = readFileSync(DOCTOR_SWITCH_DOCKER_E2E_PATH, "utf8"); const scenario = readFileSync(DOCTOR_SWITCH_SCENARIO_PATH, "utf8"); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 5da5afabec3b..6d34c4623bda 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -9854,7 +9854,7 @@ describe("package artifact reuse", () => { ); expect(workflow).toContain("suite_id: native-live-src-infra"); expect(workflow).toContain( - "command: OPENCLAW_LIVE_APNS_REACHABILITY=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra", + "command: OPENCLAW_LIVE_APNS_REACHABILITY=1 OPENCLAW_LIVE_SESSION_EVENT_WAKE=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra", ); expect(workflow).toContain("suite_id: native-live-src-gateway-profiles-anthropic-smoke"); expect(workflow).toContain("OPENCLAW_LIVE_GATEWAY_SETUP_TIMEOUT_MS=300000"); diff --git a/test/scripts/test-live-shard.test.ts b/test/scripts/test-live-shard.test.ts index 4c1792b1324b..8c1a45bcdd52 100644 --- a/test/scripts/test-live-shard.test.ts +++ b/test/scripts/test-live-shard.test.ts @@ -1,6 +1,14 @@ // Test Live Shard tests cover test live shard script behavior. import { spawn, spawnSync } from "node:child_process"; -import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { setTimeout as delay } from "node:timers/promises"; @@ -256,7 +264,9 @@ describe("scripts/test-live-shard", () => { it.each([ "native-live-src-gateway-core", "native-live-src-gateway-backends", + "native-live-src-infra", "native-live-test", + "src/infra/heartbeat-runner.live.test.ts", "test/e2e/qa-lab/runtime/worker-skill-resources.live.test.ts", "test/e2e/qa-lab/runtime/gateway-node-mcp.live.test.ts", ])("prepares the built gateway runtime before %s starts Vitest", (target) => { @@ -276,9 +286,7 @@ describe("scripts/test-live-shard", () => { profile: "sourcePerformance", requiredArtifact: "dist/.runtime-postbuildstamp", }); - expect( - resolveLiveShardPreparation(selectLiveShardFiles("native-live-src-infra", allFiles)), - ).toBeNull(); + expect(resolveLiveShardPreparation(["src/infra/push-apns-http2.live.test.ts"])).toBeNull(); }); it("runs the frozen candidate's available build entrypoint and advertised profile", () => { @@ -671,32 +679,57 @@ describe("scripts/test-live-shard", () => { async () => { const root = mkdtempSync(path.join(tmpdir(), "openclaw-live-shard-signal-")); const fakePnpmPath = path.join(root, "pnpm"); + const argsPath = path.join(root, "args.json"); const childPidPath = path.join(root, "child.pid"); const descendantPidPath = path.join(root, "descendant.pid"); const signaledPath = path.join(root, "signaled"); let childPid = 0; let descendantPid = 0; let runner: ReturnType | undefined; + let runnerClosed: Promise<{ code: number | null; signal: NodeJS.Signals | null }> | undefined; try { writeFakePnpm(fakePnpmPath); - runner = spawn(process.execPath, ["scripts/test-live-shard.mjs", "native-live-src-infra"], { - env: { - ...process.env, - OPENCLAW_FAKE_PNPM_DESCENDANT_PID_PATH: descendantPidPath, - OPENCLAW_FAKE_PNPM_PID_PATH: childPidPath, - OPENCLAW_FAKE_PNPM_SIGNALED_PATH: signaledPath, - npm_execpath: fakePnpmPath, + // Give the real shard selector an APNs-only inventory: readiness must belong + // to the live-test child, not a build prerequisite of another infra test. + mkdirSync(path.join(root, "src/infra"), { recursive: true }); + writeFileSync(path.join(root, "src/infra/push-apns-http2.live.test.ts"), ""); + const startedRunner = spawn( + process.execPath, + [path.resolve("scripts/test-live-shard.mjs"), "native-live-src-infra"], + { + cwd: root, + env: { + ...process.env, + OPENCLAW_FAKE_PNPM_ARGS_PATH: argsPath, + OPENCLAW_FAKE_PNPM_DESCENDANT_PID_PATH: descendantPidPath, + OPENCLAW_FAKE_PNPM_PID_PATH: childPidPath, + OPENCLAW_FAKE_PNPM_SIGNALED_PATH: signaledPath, + npm_execpath: fakePnpmPath, + }, + stdio: "ignore", }, - stdio: "ignore", + ); + + runner = startedRunner; + runnerClosed = new Promise((resolve) => { + startedRunner.once("close", (code, signal) => resolve({ code, signal })); }); childPid = await waitForPidFile(childPidPath, 5_000); descendantPid = await waitForPidFile(descendantPidPath, 5_000); + expect(JSON.parse(readFileSync(argsPath, "utf8")).slice(0, 3)).toEqual([ + "test:live", + "--", + "src/infra/push-apns-http2.live.test.ts", + ]); runner.kill("SIGTERM"); - await expect(waitForClose(runner)).resolves.toEqual({ code: null, signal: "SIGTERM" }); + await expect(waitForClose(runnerClosed)).resolves.toEqual({ + code: null, + signal: "SIGTERM", + }); // Creation precedes the synchronous write; wait for the signal receipt itself. await waitFor( () => existsSync(signaledPath) && readFileSync(signaledPath, "utf8") === "SIGTERM", @@ -705,16 +738,32 @@ describe("scripts/test-live-shard", () => { await waitFor(() => !isProcessAlive(childPid), 5_000); await waitFor(() => !isProcessAlive(descendantPid), 5_000); } finally { - if (runner?.pid && isProcessAlive(runner.pid)) { - process.kill(runner.pid, "SIGKILL"); + try { + if (runner?.pid && isProcessAlive(runner.pid)) { + runner.kill("SIGTERM"); + } + // Join the shim so it can forward cancellation to its detached child group + // before the fixture directory (and its process receipts) disappears. + if (runnerClosed) { + await waitForClose(runnerClosed); + } + } finally { + // A failed shim join must not skip cleanup of already observed descendants. + for (const pid of [childPid, descendantPid]) { + if (pid && isProcessAlive(pid)) { + process.kill(pid, "SIGKILL"); + } + } + try { + await Promise.all( + [childPid, descendantPid] + .filter((pid) => pid > 0) + .map((pid) => waitFor(() => !isProcessAlive(pid), 5_000)), + ); + } finally { + rmSync(root, { force: true, recursive: true }); + } } - if (childPid && isProcessAlive(childPid)) { - process.kill(childPid, "SIGKILL"); - } - if (descendantPid && isProcessAlive(descendantPid)) { - process.kill(descendantPid, "SIGKILL"); - } - rmSync(root, { force: true, recursive: true }); } }, ); @@ -727,6 +776,8 @@ function writeFakePnpm(filePath: string): void { "#!/usr/bin/env node", 'const { spawn } = require("node:child_process");', 'const fs = require("node:fs");', + 'if (process.argv[2] !== "test:live") throw new Error("Expected the live-test invocation");', + "fs.writeFileSync(process.env.OPENCLAW_FAKE_PNPM_ARGS_PATH, JSON.stringify(process.argv.slice(2)));", "const child = spawn(process.execPath, [", ' "-e",', " \"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);\",", @@ -755,13 +806,11 @@ async function waitFor(condition: () => boolean, timeoutMs: number): Promise, + completion: Promise<{ code: number | null; signal: NodeJS.Signals | null }>, timeoutMs = 5_000, ): Promise<{ code: number | null; signal: NodeJS.Signals | null }> { return await Promise.race([ - new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => { - child.once("close", (code, signal) => resolve({ code, signal })); - }), + completion, delay(timeoutMs, undefined, { ref: false }).then(() => { throw new Error("timed out waiting for child close"); }),