From c7ff0090ea6e6780adaa1f2f4b5f07523901619a Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 28 Sep 2026 10:41:34 -0700 Subject: [PATCH] fix(e2e): release lanes fail on a missing scheduler entry and stale service state (#160306) * fix(e2e): emit scheduler entry for Docker clients Emit the scheduler imported by mounted release Docker clients and guard all Docker client runtime dist references against the configured build outputs. Validation: generic guard fails on the original config; all 33 config tests pass on Blacksmith Testbox. pnpm test wall: 7.73s warm, 31.90s cold worker preparation; test time 192ms. Scoped lint, formatting, scripts/root-test types, and independent review passed. * fix(e2e): reload first-hop service fixture between lanes Reload the lane systemd shim after deleting its unit so a published updater does not see a stale loaded definition. Print both captured service-install streams when setup fails, preserving effective-service admission checks. Validation: reset regression fails on original harness with the real fixture. All 17 systemd fixture tests pass on Blacksmith Testbox; pnpm test wall 13.62s, new case 556ms. Scoped formatting, lint, shell syntax, scripts/root-test types, and independent review passed. * fix(e2e): stop gateway before restart recovery preparation Standalone Doctor can start a stopped service. Use the existing confirmed-shutdown helper before preparing the separate recovery update, preserving restart, auth, and inactive-only start assertions. Validation: all four new recovery cases fail on original harness and pass with the fix, including failed stop, active service, and open listener refusal. All 14 phase tests pass on Blacksmith Testbox; pnpm test wall 1.85s, new cases 15ms. Scoped checks and independent review passed. * test(e2e): stop an inactive fixture service idempotently * test(e2e): align survivor fixtures with recovery shutdown Keep the verified shutdown phase before inference preparation. Model idempotent systemd stop in the prepared-service fixture and include the phase in companion and frozen-target harnesses without changing their auth, membership, restart, or survival contracts. Validation: origin/main passes all 55 targeted tests; original branch reproduces 17 failures. Blacksmith Testbox passes all 898 survivor tests across 42 files, including recovery-phase and cron-seed. Changed-file test walls: mobile 2.33s, parking 7.36s, membership 3.41s. Formatting, shell syntax, focused lint, and independent review pass. * test(e2e): include script clients in dist-entry guard Scan both Docker client roots and resolve each runtime import against its own client URL. This closes the release guard gap for scripts/e2e without adding runtime entries or duplicate tests. Validation: 33 config tests pass on Blacksmith Testbox; pnpm test wall 42.30s with cold worker compilation. Removing the scripts-only runtime-context entry passes the original guard and fails the extended guard. Formatting, scoped oxlint, and independent P2 review pass. --- scripts/e2e/lib/upgrade-survivor/run.sh | 2 + .../update-first-hop-compat.sh | 7 +- test/scripts/tsdown-runtime-config.test.ts | 38 +++++++++- .../upgrade-survivor-config-parking.test.ts | 28 ++++--- .../upgrade-survivor-cron-seed.test.ts | 5 +- ...upgrade-survivor-membership-compat.test.ts | 2 + .../upgrade-survivor-mobile-pairing.test.ts | 2 + .../upgrade-survivor-recovery-phase.test.ts | 75 +++++++++++++++++++ test/scripts/upgrade-survivor-systemd.test.ts | 30 ++++++++ tsdown.config.ts | 1 + 10 files changed, 177 insertions(+), 13 deletions(-) diff --git a/scripts/e2e/lib/upgrade-survivor/run.sh b/scripts/e2e/lib/upgrade-survivor/run.sh index 6535d00ba3ee..f33ce89d3d88 100644 --- a/scripts/e2e/lib/upgrade-survivor/run.sh +++ b/scripts/e2e/lib/upgrade-survivor/run.sh @@ -1857,6 +1857,8 @@ repair_update_restart_auth() { native | absent) ;; *) echo "invalid selected service membership mode: $membership_mode" >&2; return 2 ;; esac + # Standalone Doctor may have started the service after the first update. + phase stop-recovery-service stop_update_restart_probe_gateway "$COMMAND_TIMEOUT" || return "$?" # Historical preservation has already passed. This separate current-runtime # update needs a configured inference route for its real serving receipt. phase prepare-restart-inference prepare_restart_inference || return "$?" diff --git a/scripts/e2e/lib/upgrade-survivor/update-first-hop-compat.sh b/scripts/e2e/lib/upgrade-survivor/update-first-hop-compat.sh index 597c0d757a5b..4e073360e3a0 100755 --- a/scripts/e2e/lib/upgrade-survivor/update-first-hop-compat.sh +++ b/scripts/e2e/lib/upgrade-survivor/update-first-hop-compat.sh @@ -195,7 +195,11 @@ PLUGIN_CONFIG fi openclaw gateway install --force --json \ >"$ARTIFACT_DIR/$lane-service-install.json" \ - 2>"$ARTIFACT_DIR/$lane-service-install.err" + 2>"$ARTIFACT_DIR/$lane-service-install.err" || { + docker_e2e_print_log "$ARTIFACT_DIR/$lane-service-install.json" >&2 + docker_e2e_print_log "$ARTIFACT_DIR/$lane-service-install.err" >&2 + return 1 + } wait_service_active cp "$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE" "$ARTIFACT_DIR/$lane-before.pid" record_service_state "$ARTIFACT_DIR/$lane-service-before.txt" @@ -220,6 +224,7 @@ reset_lane() { "$HOME/.openclaw" \ "$HOME/.config/systemd/user/openclaw-gateway.service" \ "$HOME/.config/systemd/user/default.target.wants/openclaw-gateway.service" + systemctl --user daemon-reload || return "$?" } run_negative_control() { diff --git a/test/scripts/tsdown-runtime-config.test.ts b/test/scripts/tsdown-runtime-config.test.ts index 8bcdb6f15aca..96b8e507a598 100644 --- a/test/scripts/tsdown-runtime-config.test.ts +++ b/test/scripts/tsdown-runtime-config.test.ts @@ -1,6 +1,8 @@ // Covers bundling rules encoded in the root tsdown config. -import { readFileSync } from "node:fs"; +import { readdirSync, readFileSync } from "node:fs"; +import { stripTypeScriptTypes } from "node:module"; import path from "node:path"; +import { fileURLToPath } from "node:url"; import { expectDefined } from "@openclaw/normalization-core/expect"; import { bundledPluginRoot } from "openclaw/plugin-sdk/test-fixtures"; import type { TsdownPluginOption } from "tsdown"; @@ -360,6 +362,40 @@ describe("tsdown config", () => { expect(entrySources(distGraph)["docker-healthcheck"]).toBe("src/docker-healthcheck.ts"); }); + it("emits the dist modules referenced by every Docker client", () => { + const emittedPaths = new Set( + asConfigArray(tsdownConfig) + .filter((config) => !(typeof config.dts === "object" && config.dts.emitDtsOnly)) + .flatMap((config) => + entryKeys(config).map((entry) => + path.resolve( + config.outDir ?? "dist", + `${entry}${config.outExtensions?.().js ?? ".js"}`, + ), + ), + ), + ); + const clients = ["test/e2e", "scripts/e2e"].flatMap((root) => { + const clientRoot = new URL(`../../${root}/`, import.meta.url); + return readdirSync(clientRoot, { recursive: true, encoding: "utf8" }) + .filter((file) => file.endsWith("-docker-client.ts")) + .map((file) => new URL(file, clientRoot)); + }); + expect(clients.length).toBeGreaterThan(0); + for (const clientUrl of clients) { + const runtimeSource = stripTypeScriptTypes(readFileSync(clientUrl, "utf8")); + // Include literal paths assigned to variables used by dynamic imports, but not erased types. + for (const match of runtimeSource.matchAll( + /["'`]((?:\.{1,2}\/)+dist\/[^"'`\s]+\.[cm]?js)["'`]/gu, + )) { + const specifier = expectDefined(match[1], "Docker dist module path"); + expect(emittedPaths, `${fileURLToPath(clientUrl)}: ${specifier}`).toContain( + fileURLToPath(new URL(specifier, clientUrl)), + ); + } + } + }); + it("keeps root-package-excluded external plugins out of the root dist graph", () => { const distGraph = requireUnifiedDistGraph(); const keys = entryKeys(distGraph); diff --git a/test/scripts/upgrade-survivor-config-parking.test.ts b/test/scripts/upgrade-survivor-config-parking.test.ts index 29a66c9ecb32..1081c7d44f30 100644 --- a/test/scripts/upgrade-survivor-config-parking.test.ts +++ b/test/scripts/upgrade-survivor-config-parking.test.ts @@ -186,6 +186,8 @@ exit "$probe_status" `#!/usr/bin/env bash [ "$*" != '--user is-active --quiet openclaw-gateway.service' ] || exit "$PROBE_ACTIVE_STATUS" printf '%s\\n' "$*" >>"$PROBE_EVENTS" +# Stopping an inactive unit succeeds; PROBE_ACTIVE_STATUS models the state after stop. +[ "$*" != '--user stop openclaw-gateway.service' ] || exit 0 [ "$*" = '--user start openclaw-gateway.service' ] || exit 97 printf 'synthetic start diagnostic\\n' >&2 [ "$PROBE_START_STATUS" -eq 0 ] || exit "$PROBE_START_STATUS" @@ -221,6 +223,7 @@ printf 'original env\\n' >"$OPENCLAW_STATE_DIR/gateway.systemd.env" printf 'original dotenv\\n' >"$OPENCLAW_STATE_DIR/.env" printf 'baseline timeline\\n' >"$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG" : >"$PROBE_EVENTS" +openclaw_e2e_probe_tcp() { return 1; } openclaw_e2e_wait_gateway_ready() { printf 'readiness\\n' >>"$PROBE_EVENTS" return "$PROBE_READY_STATUS" @@ -294,8 +297,9 @@ exit "$probe_status" expect(result.status, result.stdout + result.stderr).toBe(expected); expect( readFileSync(path.join(root, "events"), "utf8").trimEnd().split("\n").filter(Boolean), - ).toEqual( - activeStatus !== 3 + ).toEqual([ + "--user stop openclaw-gateway.service", + ...(activeStatus !== 3 ? [] : startStatus || mutation !== "none" ? ["--user start openclaw-gateway.service"] @@ -310,8 +314,8 @@ exit "$probe_status" "serving-turn", "assert-survival", ...(scenario === "sqlite-volume" ? ["volume-doctor", "volume-state"] : []), - ], - ); + ]), + ]); if (activeStatus === 3) { expect( readFileSync( @@ -331,11 +335,16 @@ exit "$probe_status" .map((line) => JSON.parse(line)); expect( completedPhases.filter((event) => event.status === "passed").map((event) => event.phase), - ).toEqual([ - "prepare-restart-inference", - "prepare-restart-fixture", - "prepare-restart-manager", - ]); + ).toEqual( + activeStatus !== 3 + ? [] + : [ + "stop-recovery-service", + "prepare-restart-inference", + "prepare-restart-fixture", + "prepare-restart-manager", + ], + ); expect(phases).not.toContain("recovery-update-restart"); } }, @@ -364,6 +373,7 @@ update_repair_required=0 candidate_version=2026.9.3 baseline_version=2026.9.2 OPENCLAW_CLAWHUB_URL=fixture +stop_update_restart_probe_gateway() { :; } prepare_restart_inference() { :; } prepare_restart_fixture() { restart_fixture_package=/tmp/fixture.tgz; restart_fixture_version=2026.9.3; } install_update_restart_systemctl_shim() { :; } diff --git a/test/scripts/upgrade-survivor-cron-seed.test.ts b/test/scripts/upgrade-survivor-cron-seed.test.ts index e17fadf91b4e..343986cb3481 100644 --- a/test/scripts/upgrade-survivor-cron-seed.test.ts +++ b/test/scripts/upgrade-survivor-cron-seed.test.ts @@ -49,8 +49,9 @@ if (args[0] === "fixture-systemctl") { if (args[2] === "is-active") process.exit(fs.existsSync(live) ? 0 : 3); if (args[2] === "stop") { assert.equal(fs.existsSync(boot), true); - fs.unlinkSync(live); - fs.unlinkSync(process.env.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE); + // Like systemd, stopping an inactive unit succeeds. + fs.rmSync(live, { force: true }); + fs.rmSync(process.env.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE, { force: true }); } else { assert.equal(args[2], "start"); assert.equal(fs.existsSync(live), false); diff --git a/test/scripts/upgrade-survivor-membership-compat.test.ts b/test/scripts/upgrade-survivor-membership-compat.test.ts index cbb198e4327e..24e7b070bdcf 100644 --- a/test/scripts/upgrade-survivor-membership-compat.test.ts +++ b/test/scripts/upgrade-survivor-membership-compat.test.ts @@ -123,6 +123,7 @@ function runRecovery( "SCENARIO=base; UPDATE_RESTART_MODE=auto-auth; COMMAND_TIMEOUT=1; update_repair_required=0", "restart_fixture_package=fixture.tgz; restart_fixture_version=2100.1.0", 'phase() { printf "%s\\n" "$1" >> "$EVENTS"; shift; "$@"; }', + "stop_update_restart_probe_gateway() { :; }", "prepare_restart_inference() { :; }; prepare_restart_fixture() { :; }", 'install_update_restart_systemctl_shim() { printf "containment:%s\\n" "$1" >> "$EVENTS"; }', 'run_update_restart_probe_gateway() { [ "$1" = start ]; }', @@ -170,6 +171,7 @@ describe.skipIf(process.platform === "win32")("frozen managed membership contrac expect(run.status, run.stderr).toBe(0); expect(selectedMode(selected.stdout)).toBe("native"); expect(run.events).toEqual([ + "stop-recovery-service", "prepare-restart-inference", "prepare-restart-fixture", "prepare-restart-manager", diff --git a/test/scripts/upgrade-survivor-mobile-pairing.test.ts b/test/scripts/upgrade-survivor-mobile-pairing.test.ts index 239b3633da1e..0a75966df1a9 100644 --- a/test/scripts/upgrade-survivor-mobile-pairing.test.ts +++ b/test/scripts/upgrade-survivor-mobile-pairing.test.ts @@ -827,6 +827,7 @@ ARTIFACT_ROOT=/tmp update_repair_required=0 ${helper} phase() { printf '%s\\n' "$1"; shift; "$@"; } +stop_update_restart_probe_gateway() { :; } prepare_restart_inference() { :; } prepare_restart_fixture() { restart_fixture_package=/tmp/future-package.tgz @@ -849,6 +850,7 @@ repair_update_restart_auth ); expect(result.trim().split("\n")).toEqual([ + "stop-recovery-service", "prepare-restart-inference", "prepare-restart-fixture", "prepare-restart-manager", diff --git a/test/scripts/upgrade-survivor-recovery-phase.test.ts b/test/scripts/upgrade-survivor-recovery-phase.test.ts index 0d8941aa564f..1228c1256cfc 100644 --- a/test/scripts/upgrade-survivor-recovery-phase.test.ts +++ b/test/scripts/upgrade-survivor-recovery-phase.test.ts @@ -17,6 +17,81 @@ const update = source.slice( ); const outer = "recovery-update-restart"; +it.skipIf(process.platform === "win32").each([ + { fault: "none", code: 0 }, + { fault: "stop", code: 17 }, + { fault: "still-active", code: 1 }, + { fault: "listener", code: 1 }, +])("stops a Doctor-started service before recovery preparation ($fault)", ({ fault, code }) => { + const root = dirs.make("survivor-recovery-stop-"); + const repair = source.slice( + source.indexOf("repair_update_restart_auth() {"), + source.indexOf("assert_managed_membership_warning() {"), + ); + const result = spawnSync( + "bash", + [ + "-c", + `set -euo pipefail +exec 3>&1 +source scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh +ARTIFACT_ROOT="$1" +OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG="$1/gateway.log" +FAULT="$2" +SCENARIO=base +UPDATE_RESTART_MODE=auto-auth +OPENCLAW_FROZEN_UPGRADE_SURVIVOR_MEMBERSHIP_MODE=absent +COMMAND_TIMEOUT=30 +restart_fixture_package=synthetic.tgz +restart_fixture_version=2026.9.7 +update_repair_required=0 +active=1 +phase() { shift; "$@"; } +systemctl() { + case "$2" in + stop) + printf 'stop\n' >&3 + [ "$FAULT" != stop ] || return 17 + [ "$FAULT" = still-active ] || active=0 ;; + is-active) [ "$active" = 1 ] && return 0; return 3 ;; + *) return 99 ;; + esac +} +openclaw_e2e_maybe_timeout() { shift; "$@"; } +openclaw_e2e_probe_tcp() { [ "$FAULT" = listener ]; } +openclaw_e2e_print_log() { cat "$1"; } +prepare_restart_inference() { printf 'inference\n'; } +prepare_restart_fixture() { printf 'fixture\n'; } +install_update_restart_systemctl_shim() { printf 'manager\n'; } +run_update_restart_probe_gateway() { + assert_update_restart_probe_inactive || return "$?" + printf 'prepared\n' +} +check_gateway_status() { printf 'auth\n'; } +update_candidate() { printf 'update\n'; } +assert_managed_membership_warning() { :; } +node() { :; } +assert_survival() { :; } +${repair} +repair_update_restart_auth +`, + "fixture", + root, + fault, + ], + { env: { PATH: process.env.PATH, HOME: root }, encoding: "utf8", timeout: 5_000 }, + ); + expect(result.status, result.stderr).toBe(code); + expect(result.stdout.trim().split("\n")).toEqual( + code === 0 + ? ["stop", "inference", "fixture", "manager", "prepared", "auth", "update"] + : ["stop"], + ); + if (code !== 0) { + expect(result.stderr).toContain("gateway service shutdown could not be verified"); + } +}); + it.skipIf(process.platform === "win32").each([ { fault: "command", code: 17, stage: "command", checks: [] }, { fault: "command-one", code: 1, stage: "command", checks: [] }, diff --git a/test/scripts/upgrade-survivor-systemd.test.ts b/test/scripts/upgrade-survivor-systemd.test.ts index 1a5d483ba1a0..beb0b76d0708 100644 --- a/test/scripts/upgrade-survivor-systemd.test.ts +++ b/test/scripts/upgrade-survivor-systemd.test.ts @@ -84,6 +84,36 @@ function fixture(customPaths = true, registry?: string) { } describe.skipIf(process.platform === "win32")("survivor manager fixture", () => { + it("unloads the deleted service when resetting a first-hop lane", async () => { + const { home, env, unit, shell, systemctl } = fixture(); + const firstHop = readFileSync( + resolve("scripts/e2e/lib/upgrade-survivor/update-first-hop-compat.sh"), + "utf8", + ); + const resetLane = firstHop.slice( + firstHop.indexOf("reset_lane() {"), + firstHop.indexOf("run_negative_control() {"), + ); + writeFileSync(unit, buildSystemdUnit({ programArguments: ["/usr/bin/fixture", "gateway"] })); + expect(systemctl("daemon-reload").status).toBe(0); + rmSync(unit); + expect(await readLoadedSystemdServiceRuntime(env)).toMatchObject({ status: "stopped" }); + const reset = shell(` +ARTIFACT_DIR="$HOME" +openclaw() { return 1; } +${resetLane} +reset_lane negative +`); + expect(reset.status, reset.stderr).toBe(0); + expect(existsSync(`${unit}.loaded-unit`)).toBe(false); + expect(await readLoadedSystemdServiceRuntime(env)).toMatchObject({ status: "unknown" }); + expect(await readSystemdServiceRuntime(env)).toMatchObject({ + status: "stopped", + missingUnit: true, + }); + expect(existsSync(join(home, "negative-service-uninstall.json"))).toBe(true); + }); + it("keeps native placement by default and runs the no-identity recovery fixture outside it", () => { const { home, env, unit, shell, manager, execute } = fixture(); const preload = join(home, "available-cgroup.cjs"); diff --git a/tsdown.config.ts b/tsdown.config.ts index 88ba61b44754..2a1054bcf052 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -520,6 +520,7 @@ function buildDockerE2eHarnessEntries(): Record { "cli/run-main": "src/cli/run-main.ts", "commands/onboard-guided": "src/commands/onboard-guided.ts", "config/config": "src/config/config.ts", + "infra/gateway-scheduler": "src/infra/gateway-scheduler.ts", "infra/sqlite-audit-record-store": "src/infra/sqlite-audit-record-store.ts", "state/local-onboarding-state": "src/state/local-onboarding-state.ts", "system-agent/audit": "src/system-agent/audit.ts",