mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 17:53:39 +00:00
fix(ci): prepare selected release native fixtures (#144338)
* fix(ci): prepare selected release native fixtures Enable and prepare the selected native heartbeat live test through its existing runtime build owner. Keep Doctor scenario and canonical-path service shims on the same selected checkout while retaining trusted shared helpers. Complete the managed test's ephemeral TCP endpoint adapter so shutdown verification cannot observe an unrelated host Gateway. Qualification context: https://github.com/openclaw/openclaw/actions/runs/34507645027 Co-authored-by: steipete <58493+steipete@users.noreply.github.com> Co-authored-by: vincentkoc <25068+vincentkoc@users.noreply.github.com> * test(ci): bind live-shard cancellation to the test child Use an isolated APNs-only inventory through the existing shard selector so build preparation cannot masquerade as live-child readiness. Assert the actual test:live arguments and join owned processes before fixture cleanup. Preserve the original signal, descendant-death, and timeout assertions. The original 20-file CI order passes all 332 tests after the correction. Co-authored-by: steipete <58493+steipete@users.noreply.github.com> Co-authored-by: vincentkoc <25068+vincentkoc@users.noreply.github.com> --------- Co-authored-by: roboclaw-bot <309084314+roboclaw-bot@users.noreply.github.com> Co-authored-by: steipete <58493+steipete@users.noreply.github.com> Co-authored-by: vincentkoc <25068+vincentkoc@users.noreply.github.com>
This commit is contained in:
RoboClaw
GitHub
parent
5d3cc203b6
commit
7444f28d2d
6 changed files with 160 additions and 29 deletions
|
|
@ -4318,7 +4318,7 @@ jobs:
|
|||
profiles: stable full
|
||||
- suite_id: native-live-src-infra
|
||||
label: Native live infra
|
||||
command: OPENCLAW_LIVE_APNS_REACHABILITY=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra
|
||||
command: OPENCLAW_LIVE_APNS_REACHABILITY=1 OPENCLAW_LIVE_SESSION_EVENT_WAKE=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra
|
||||
timeout_minutes: 45
|
||||
profile_env_only: false
|
||||
profiles: stable full
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ docker_e2e_build_or_reuse "$IMAGE_NAME" doctor-switch "$ROOT_DIR/scripts/e2e/Doc
|
|||
echo "Running doctor install switch E2E..."
|
||||
# Maintenance loads the installed unit's canonical PATH. Mount the shims there
|
||||
# so the unprivileged container keeps using the fixture manager during inspection.
|
||||
SHIM_DIR="$ROOT_DIR/scripts/e2e/lib/doctor-install-switch/shims"
|
||||
SHIM_DIR="$TARGET_CONTRACT_DIR/shims"
|
||||
docker_e2e_run_with_harness \
|
||||
-v "$SHIM_DIR/systemctl:/usr/local/bin/systemctl:ro" \
|
||||
-v "$SHIM_DIR/loginctl:/usr/local/bin/loginctl:ro" \
|
||||
|
|
|
|||
|
|
@ -410,6 +410,7 @@ export function resolveLiveShardPreparation(files: string[]): LiveShardPreparati
|
|||
if (
|
||||
files.some(isSourceGatewayLiveTest) ||
|
||||
files.some((file) => file.startsWith("test/e2e/qa-lab/runtime/")) ||
|
||||
files.includes("src/infra/heartbeat-runner.live.test.ts") ||
|
||||
files.includes("src/agents/tools/image-tool.providers.live.test.ts") ||
|
||||
files.includes("extensions/openai/openai.live.test.ts")
|
||||
) {
|
||||
|
|
|
|||
|
|
@ -3525,11 +3525,21 @@ process.on("SIGTERM", () => {
|
|||
lane === "published"
|
||||
? source.slice(0, source.indexOf("phase storage-preflight"))
|
||||
: `source ${shellQuote(OPENCLAW_E2E_INSTANCE_HELPER_PATH)}\nsource ${shellQuote(UPGRADE_SURVIVOR_UPDATE_RESTART_AUTH_PATH)}`;
|
||||
// The published stop proof must inspect this fixture's listener, not a host Gateway.
|
||||
const tcpProbeAdapter = `
|
||||
eval "$(declare -f openclaw_e2e_probe_tcp | sed '1s/openclaw_e2e_probe_tcp/fixture_probe_tcp/')"
|
||||
openclaw_e2e_probe_tcp() {
|
||||
local port="$2"
|
||||
[ "$port" != 18789 ] || port="$(cat "$PORT_FILE")"
|
||||
fixture_probe_tcp "$1" "$port" "\${3:-400}"
|
||||
}
|
||||
`;
|
||||
const script = `${setup}
|
||||
trap - EXIT ERR INT TERM
|
||||
assert_prepublish_fixture_idle() { :; }
|
||||
assert_baseline_state() { :; }
|
||||
check_gateway_status() { :; }
|
||||
${tcpProbeAdapter}
|
||||
# This fixture chooses an ephemeral port; retain the actual readiness implementation.
|
||||
eval "$(declare -f openclaw_e2e_wait_gateway_ready | sed '1s/openclaw_e2e_wait_gateway_ready/fixture_wait_gateway_ready/')"
|
||||
openclaw_e2e_wait_gateway_ready() {
|
||||
|
|
@ -3549,6 +3559,17 @@ ${lane === "published" ? "prepare_update_restart_probe" : 'prepare_update_restar
|
|||
encoding: "utf8",
|
||||
timeout: 40_000,
|
||||
});
|
||||
const probeListener = () =>
|
||||
spawnSync(
|
||||
"bash",
|
||||
[
|
||||
"-c",
|
||||
`source ${shellQuote(OPENCLAW_E2E_INSTANCE_HELPER_PATH)}
|
||||
${tcpProbeAdapter}
|
||||
openclaw_e2e_probe_tcp 127.0.0.1 18789 400`,
|
||||
],
|
||||
{ env, encoding: "utf8", timeout: 5_000 },
|
||||
);
|
||||
const records = (): Array<{ pid: number; managed: boolean }> =>
|
||||
existsSync(startsPath)
|
||||
? readFileSync(startsPath, "utf8")
|
||||
|
|
@ -3593,7 +3614,9 @@ ${lane === "published" ? "prepare_update_restart_probe" : 'prepare_update_restar
|
|||
expect(replacement.managed).toBe(true);
|
||||
expect(replacement.pid).not.toBe(initial.pid);
|
||||
expect(isProcessRunning(initial.pid)).toBe(false);
|
||||
expect(probeListener().status).toBe(0);
|
||||
expect(systemctl("stop", "openclaw-gateway.service").status).toBe(0);
|
||||
expect(probeListener().status).toBe(1);
|
||||
await expect(fetch(url, { signal: AbortSignal.timeout(1_000) })).rejects.toThrow();
|
||||
} finally {
|
||||
systemctl("stop", "openclaw-gateway.service");
|
||||
|
|
@ -7830,6 +7853,64 @@ done
|
|||
expect(staleObject.stdout).not.toContain('"loaded"');
|
||||
});
|
||||
|
||||
it.each(["selected", "default"] as const)(
|
||||
"mounts the %s Doctor contract and canonical-path shims from the same checkout",
|
||||
(targetMode) => {
|
||||
const workDir = tempDirs.make("openclaw-doctor-contract-mounts-");
|
||||
const targetRoot =
|
||||
targetMode === "selected" ? join(workDir, "selected target") : process.cwd();
|
||||
const contractPath = "scripts/e2e/lib/doctor-install-switch";
|
||||
const shimNames = ["systemctl", "loginctl", "busctl", "systemd-exec-start.mjs"];
|
||||
if (targetMode === "selected") {
|
||||
const targetContract = join(targetRoot, contractPath);
|
||||
mkdirSync(join(targetContract, "shims"), { recursive: true });
|
||||
copyFileSync(DOCTOR_SWITCH_SCENARIO_PATH, join(targetContract, "scenario.sh"));
|
||||
for (const name of shimNames) {
|
||||
copyFileSync(join(contractPath, "shims", name), join(targetContract, "shims", name));
|
||||
}
|
||||
}
|
||||
writeFileSync(join(workDir, "openclaw-current.tgz"), "unused package transport fixture");
|
||||
writeExecutables(join(workDir, "bin"), {
|
||||
timeout: PASSTHROUGH_TIMEOUT_SCRIPT,
|
||||
docker: `#!/bin/bash
|
||||
set -euo pipefail
|
||||
case "$1 \${2:-}" in
|
||||
"image inspect") exit 0 ;;
|
||||
"run "*) printf '%s\\0' "$@" >"$TMPDIR/docker-run-args" ;;
|
||||
*) exit 9 ;;
|
||||
esac
|
||||
`,
|
||||
});
|
||||
const script = repoShell(workDir)`
|
||||
export PATH="$TMPDIR/bin:$PATH"
|
||||
export OPENCLAW_SKIP_DOCKER_BUILD=1
|
||||
export OPENCLAW_DOCKER_E2E_IMAGE=doctor-contract-fixture
|
||||
export OPENCLAW_CURRENT_PACKAGE_TGZ="$TMPDIR/openclaw-current.tgz"
|
||||
unset DOCKER_E2E_HARNESS_ROOT_DIR OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR OPENCLAW_DOCKER_E2E_REPO_ROOT
|
||||
${targetMode === "selected" ? `export OPENCLAW_DOCKER_E2E_REPO_ROOT=${shellQuote(targetRoot)}` : ""}
|
||||
bash "$ROOT_DIR/scripts/e2e/doctor-install-switch-docker.sh"
|
||||
`;
|
||||
const result = spawnSync("bash", ["--noprofile", "--norc", "-c", script], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
expect(result.status, result.stderr).toBe(0);
|
||||
const args = readFileSync(join(workDir, "docker-run-args"), "utf8").split("\0");
|
||||
const mounts = args.flatMap((arg, index) => (arg === "-v" ? [args[index + 1]] : []));
|
||||
// Service inspection uses /usr/local/bin, outside the target directory overlay.
|
||||
// Both views must select one fixture while shared helpers stay trusted.
|
||||
expect(mounts.filter((mount) => mount?.includes(":/usr/local/bin/"))).toEqual(
|
||||
shimNames.map(
|
||||
(name) => `${targetRoot}/${contractPath}/shims/${name}:/usr/local/bin/${name}:ro`,
|
||||
),
|
||||
);
|
||||
expect(mounts).toContain(`${targetRoot}/${contractPath}:/app/${contractPath}:ro`);
|
||||
expect(mounts).toContain(`${process.cwd()}/scripts/e2e:/app/scripts/e2e:ro`);
|
||||
expect(mounts).toContain(`${process.cwd()}/scripts/lib:/app/scripts/lib:ro`);
|
||||
expect(args).toContain("doctor-contract-fixture");
|
||||
expect(args).toContain(`${contractPath}/scenario.sh`);
|
||||
},
|
||||
);
|
||||
|
||||
it("routes doctor install switch commands through the E2E timeout helper", () => {
|
||||
const runner = readFileSync(DOCTOR_SWITCH_DOCKER_E2E_PATH, "utf8");
|
||||
const scenario = readFileSync(DOCTOR_SWITCH_SCENARIO_PATH, "utf8");
|
||||
|
|
|
|||
|
|
@ -9854,7 +9854,7 @@ describe("package artifact reuse", () => {
|
|||
);
|
||||
expect(workflow).toContain("suite_id: native-live-src-infra");
|
||||
expect(workflow).toContain(
|
||||
"command: OPENCLAW_LIVE_APNS_REACHABILITY=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra",
|
||||
"command: OPENCLAW_LIVE_APNS_REACHABILITY=1 OPENCLAW_LIVE_SESSION_EVENT_WAKE=1 node .release-harness/scripts/test-live-shard.mjs native-live-src-infra",
|
||||
);
|
||||
expect(workflow).toContain("suite_id: native-live-src-gateway-profiles-anthropic-smoke");
|
||||
expect(workflow).toContain("OPENCLAW_LIVE_GATEWAY_SETUP_TIMEOUT_MS=300000");
|
||||
|
|
|
|||
|
|
@ -1,6 +1,14 @@
|
|||
// Test Live Shard tests cover test live shard script behavior.
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
|
|
@ -256,7 +264,9 @@ describe("scripts/test-live-shard", () => {
|
|||
it.each([
|
||||
"native-live-src-gateway-core",
|
||||
"native-live-src-gateway-backends",
|
||||
"native-live-src-infra",
|
||||
"native-live-test",
|
||||
"src/infra/heartbeat-runner.live.test.ts",
|
||||
"test/e2e/qa-lab/runtime/worker-skill-resources.live.test.ts",
|
||||
"test/e2e/qa-lab/runtime/gateway-node-mcp.live.test.ts",
|
||||
])("prepares the built gateway runtime before %s starts Vitest", (target) => {
|
||||
|
|
@ -276,9 +286,7 @@ describe("scripts/test-live-shard", () => {
|
|||
profile: "sourcePerformance",
|
||||
requiredArtifact: "dist/.runtime-postbuildstamp",
|
||||
});
|
||||
expect(
|
||||
resolveLiveShardPreparation(selectLiveShardFiles("native-live-src-infra", allFiles)),
|
||||
).toBeNull();
|
||||
expect(resolveLiveShardPreparation(["src/infra/push-apns-http2.live.test.ts"])).toBeNull();
|
||||
});
|
||||
|
||||
it("runs the frozen candidate's available build entrypoint and advertised profile", () => {
|
||||
|
|
@ -671,32 +679,57 @@ describe("scripts/test-live-shard", () => {
|
|||
async () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "openclaw-live-shard-signal-"));
|
||||
const fakePnpmPath = path.join(root, "pnpm");
|
||||
const argsPath = path.join(root, "args.json");
|
||||
const childPidPath = path.join(root, "child.pid");
|
||||
const descendantPidPath = path.join(root, "descendant.pid");
|
||||
const signaledPath = path.join(root, "signaled");
|
||||
let childPid = 0;
|
||||
let descendantPid = 0;
|
||||
let runner: ReturnType<typeof spawn> | undefined;
|
||||
let runnerClosed: Promise<{ code: number | null; signal: NodeJS.Signals | null }> | undefined;
|
||||
|
||||
try {
|
||||
writeFakePnpm(fakePnpmPath);
|
||||
runner = spawn(process.execPath, ["scripts/test-live-shard.mjs", "native-live-src-infra"], {
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_FAKE_PNPM_DESCENDANT_PID_PATH: descendantPidPath,
|
||||
OPENCLAW_FAKE_PNPM_PID_PATH: childPidPath,
|
||||
OPENCLAW_FAKE_PNPM_SIGNALED_PATH: signaledPath,
|
||||
npm_execpath: fakePnpmPath,
|
||||
// Give the real shard selector an APNs-only inventory: readiness must belong
|
||||
// to the live-test child, not a build prerequisite of another infra test.
|
||||
mkdirSync(path.join(root, "src/infra"), { recursive: true });
|
||||
writeFileSync(path.join(root, "src/infra/push-apns-http2.live.test.ts"), "");
|
||||
const startedRunner = spawn(
|
||||
process.execPath,
|
||||
[path.resolve("scripts/test-live-shard.mjs"), "native-live-src-infra"],
|
||||
{
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
OPENCLAW_FAKE_PNPM_ARGS_PATH: argsPath,
|
||||
OPENCLAW_FAKE_PNPM_DESCENDANT_PID_PATH: descendantPidPath,
|
||||
OPENCLAW_FAKE_PNPM_PID_PATH: childPidPath,
|
||||
OPENCLAW_FAKE_PNPM_SIGNALED_PATH: signaledPath,
|
||||
npm_execpath: fakePnpmPath,
|
||||
},
|
||||
stdio: "ignore",
|
||||
},
|
||||
stdio: "ignore",
|
||||
);
|
||||
|
||||
runner = startedRunner;
|
||||
runnerClosed = new Promise((resolve) => {
|
||||
startedRunner.once("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
|
||||
childPid = await waitForPidFile(childPidPath, 5_000);
|
||||
descendantPid = await waitForPidFile(descendantPidPath, 5_000);
|
||||
expect(JSON.parse(readFileSync(argsPath, "utf8")).slice(0, 3)).toEqual([
|
||||
"test:live",
|
||||
"--",
|
||||
"src/infra/push-apns-http2.live.test.ts",
|
||||
]);
|
||||
|
||||
runner.kill("SIGTERM");
|
||||
|
||||
await expect(waitForClose(runner)).resolves.toEqual({ code: null, signal: "SIGTERM" });
|
||||
await expect(waitForClose(runnerClosed)).resolves.toEqual({
|
||||
code: null,
|
||||
signal: "SIGTERM",
|
||||
});
|
||||
// Creation precedes the synchronous write; wait for the signal receipt itself.
|
||||
await waitFor(
|
||||
() => existsSync(signaledPath) && readFileSync(signaledPath, "utf8") === "SIGTERM",
|
||||
|
|
@ -705,16 +738,32 @@ describe("scripts/test-live-shard", () => {
|
|||
await waitFor(() => !isProcessAlive(childPid), 5_000);
|
||||
await waitFor(() => !isProcessAlive(descendantPid), 5_000);
|
||||
} finally {
|
||||
if (runner?.pid && isProcessAlive(runner.pid)) {
|
||||
process.kill(runner.pid, "SIGKILL");
|
||||
try {
|
||||
if (runner?.pid && isProcessAlive(runner.pid)) {
|
||||
runner.kill("SIGTERM");
|
||||
}
|
||||
// Join the shim so it can forward cancellation to its detached child group
|
||||
// before the fixture directory (and its process receipts) disappears.
|
||||
if (runnerClosed) {
|
||||
await waitForClose(runnerClosed);
|
||||
}
|
||||
} finally {
|
||||
// A failed shim join must not skip cleanup of already observed descendants.
|
||||
for (const pid of [childPid, descendantPid]) {
|
||||
if (pid && isProcessAlive(pid)) {
|
||||
process.kill(pid, "SIGKILL");
|
||||
}
|
||||
}
|
||||
try {
|
||||
await Promise.all(
|
||||
[childPid, descendantPid]
|
||||
.filter((pid) => pid > 0)
|
||||
.map((pid) => waitFor(() => !isProcessAlive(pid), 5_000)),
|
||||
);
|
||||
} finally {
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
}
|
||||
if (childPid && isProcessAlive(childPid)) {
|
||||
process.kill(childPid, "SIGKILL");
|
||||
}
|
||||
if (descendantPid && isProcessAlive(descendantPid)) {
|
||||
process.kill(descendantPid, "SIGKILL");
|
||||
}
|
||||
rmSync(root, { force: true, recursive: true });
|
||||
}
|
||||
},
|
||||
);
|
||||
|
|
@ -727,6 +776,8 @@ function writeFakePnpm(filePath: string): void {
|
|||
"#!/usr/bin/env node",
|
||||
'const { spawn } = require("node:child_process");',
|
||||
'const fs = require("node:fs");',
|
||||
'if (process.argv[2] !== "test:live") throw new Error("Expected the live-test invocation");',
|
||||
"fs.writeFileSync(process.env.OPENCLAW_FAKE_PNPM_ARGS_PATH, JSON.stringify(process.argv.slice(2)));",
|
||||
"const child = spawn(process.execPath, [",
|
||||
' "-e",',
|
||||
" \"process.on('SIGTERM', () => {}); setInterval(() => {}, 1000);\",",
|
||||
|
|
@ -755,13 +806,11 @@ async function waitFor(condition: () => boolean, timeoutMs: number): Promise<voi
|
|||
}
|
||||
|
||||
async function waitForClose(
|
||||
child: ReturnType<typeof spawn>,
|
||||
completion: Promise<{ code: number | null; signal: NodeJS.Signals | null }>,
|
||||
timeoutMs = 5_000,
|
||||
): Promise<{ code: number | null; signal: NodeJS.Signals | null }> {
|
||||
return await Promise.race([
|
||||
new Promise<{ code: number | null; signal: NodeJS.Signals | null }>((resolve) => {
|
||||
child.once("close", (code, signal) => resolve({ code, signal }));
|
||||
}),
|
||||
completion,
|
||||
delay(timeoutMs, undefined, { ref: false }).then(() => {
|
||||
throw new Error("timed out waiting for child close");
|
||||
}),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue