diff --git a/config/assertion-safety-baseline.txt b/config/assertion-safety-baseline.txt index c0f188203cd3..0a6942a57c4f 100644 --- a/config/assertion-safety-baseline.txt +++ b/config/assertion-safety-baseline.txt @@ -2809,7 +2809,7 @@ src/daemon/schtasks-process.ts 2 src/daemon/schtasks-runtime.ts 4 src/daemon/service.ts 1 src/daemon/systemd-exec.ts 2 -src/daemon/systemd-install.ts 5 +src/daemon/systemd-install.ts 2 src/daemon/systemd-lifecycle.ts 2 src/daemon/systemd-runtime.ts 1 src/entry.compile-cache.ts 3 diff --git a/config/knip.config.ts b/config/knip.config.ts index 59d052225be4..c00613d5b136 100644 --- a/config/knip.config.ts +++ b/config/knip.config.ts @@ -63,8 +63,13 @@ const repositoryScriptEntries = [ "scripts/e2e/lib/release-user-journey/write-clickclack-plugin.mjs!", "scripts/e2e/lib/run-with-pty.mjs!", "scripts/e2e/lib/sandbox-browser-sidecar/scenario.mjs!", + // systemd-sealed-service-definition.sh executes these via Node stdin and a container path. + "scripts/e2e/lib/systemd-sealed-service-definition/file-mount.mjs!", + "scripts/e2e/lib/systemd-sealed-service-definition/paired-mounts.mjs!", "scripts/e2e/lib/upgrade-survivor/config-parking.mjs!", "scripts/e2e/lib/upgrade-survivor/probe-gateway.mjs!", + // update-restart-auth.sh installs this manager/launch adapter into the fixture bin directory. + "scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs!", "scripts/embedded-run-abort-leak.ts!", "scripts/fixtures/packed-plugin-sdk-type-smoke.ts!", "scripts/ios-release-cut.ts!", diff --git a/docs/cli/gateway.md b/docs/cli/gateway.md index 1f3938c8aca1..c722d6e03ca4 100644 --- a/docs/cli/gateway.md +++ b/docs/cli/gateway.md @@ -151,6 +151,13 @@ On macOS and Windows, native service-managed profile names must be lowercase. Ru Named profiles must also use the native service identity derived from `OPENCLAW_PROFILE`. Unset `OPENCLAW_LAUNCHD_LABEL`, `OPENCLAW_SYSTEMD_UNIT`, or `OPENCLAW_WINDOWS_TASK_NAME` before service management; custom identities remain available for the default profile or runtime-only/external-supervisor setups. +On Linux, `openclaw gateway install --force` refuses a sealed systemd service +definition, or one whose write authority cannot be verified, before changing +configuration, authentication tokens, or service files. Ask the privileged deployment owner to repair or replace the definition. +Type-wide `service.d` defaults are inspected as shared read-only inputs and do +not require write access. Root-owned selected units and unit-specific drop-ins +remain protected. + ### External supervisors Set `OPENCLAW_SUPERVISOR_MODE=external` only when another process manager owns the Gateway lifecycle. In this mode: @@ -598,7 +605,7 @@ openclaw gateway restart - `gateway status`: `--url`, `--port`, `--token`, `--password`, `--timeout`, `--no-probe`, `--require-rpc`, `--deep`, `--json` - `gateway install`: `--port`, `--runtime ` (default: `node`), `--token`, `--wrapper `, `--force`, `--json` - - `gateway restart`: `--safe`, `--skip-deferral`, `--force`, `--wait `, `--json` + - `gateway restart`: `--safe`, `--skip-deferral`, `--force`, `--wait `, `--preserve-definition`, `--json` - `gateway uninstall|start`: `--json` - `gateway stop`: `--disable`, `--force`, `--json` @@ -613,6 +620,8 @@ openclaw gateway restart - If no managed service is installed, `gateway start` prints install hints and exits nonzero. `gateway restart` can first recover an installed-but-unloaded LaunchAgent or a verified unmanaged Gateway; if neither a managed service nor recovery handles the action, it prints the same hints and exits nonzero. Stopping an absent service remains a successful no-op. - If `gateway start` or `gateway restart` needs to repair a stale service definition, the command refuses when the invoking shell resolves a different state directory, config path, or port than the installed service. Match or unset the conflicting environment overrides, or use `openclaw gateway install --force` to retarget the service intentionally. - On Linux, `gateway start` and `gateway restart` also refuse ineffective repairs when an operator-owned systemd drop-in overrides the command or working directory. Inspect the effective unit with `systemctl --user cat .service`, then update or remove that drop-in. `gateway install --force` rewrites only the managed base unit and warns if the override remains; `Environment=` drop-ins remain supported. + - `gateway restart --preserve-definition` restarts only an inspectable native service, skips automatic definition repair, and checks health at the installed launcher's port. It does not recover an unmanaged listener and cannot be combined with `--safe` or external supervision. On macOS it can bootstrap an unloaded readable plist without rewriting the plist, environment, wrapper, or permissions; denied native activation fails without file repair. On Windows it also retains existing Startup entries. The legacy `daemon restart` command accepts the same option. Older CLIs reject the option before running restart or repair. + - During writable Linux service installs or refreshes, keep the unit and state directories stationary and avoid concurrent manual edits. OpenClaw serializes its own writers and aborts on detected changes, but cannot coordinate arbitrary filesystem edits. Moving or replacing a parent directory mid-publication can leave a temporary file inside the moved directory; inspect it before retrying. - Use `gateway restart` to restart a managed service. Do not chain `gateway stop` and `gateway start` as a restart substitute. - In a non-interactive shell, `gateway stop` requires `--force`. Interactive terminals keep the existing prompt-free behavior. For automation and tests, prefer `gateway run --dev` or an isolated `--profile` with a free port. - On macOS, `gateway stop` uses `launchctl bootout` by default, which removes the LaunchAgent from the current boot session without persisting a disable — KeepAlive auto-recovery stays active for future crashes and `gateway start` re-enables cleanly without a manual `launchctl enable`. Pass `--disable` to persistently suppress KeepAlive and RunAtLoad so the gateway does not respawn until the next explicit `gateway start`; use this when a manual stop should survive reboots. diff --git a/docs/cli/update.md b/docs/cli/update.md index 27c8cd1aaf22..2ff67df4d1c0 100644 --- a/docs/cli/update.md +++ b/docs/cli/update.md @@ -173,6 +173,29 @@ Package-manager updates additionally verify the restarted Gateway reports the expected package version; git-checkout updates verify gateway health and service readiness after the rebuild. +Code updates do not require permission to rewrite the native service definition. +On Linux, sealed or unverified definition-write authority skips metadata refresh, +even when metadata is stale. An inspectable service owned by the updated install +still uses its native manager for restart and health/version verification. +Activation runs the updated CLI with `gateway restart --preserve-definition` so +its own version guards apply and automatic repair stays disabled. If the target +CLI does not support that option, it rejects activation before repair. The code +update stays installed, but the command exits nonzero with the activation error +(on stderr in JSON mode). A service stopped for the update may remain stopped. +Run `openclaw gateway status --deep` and ask the deployment owner to restart it +through its native manager or repair stale metadata; do not retry without the +preservation option unless definition repair is intended. + +Shell installers do not establish the same service ownership proof. If their +service refresh is denied, they report code installation success, leave the +service untouched, and print guidance to inspect ownership and restart manually. + +If service inspection is unavailable, the code update continues with a warning +and leaves service control and definition files untouched; it does not assume +that no service exists. Run `openclaw gateway status --deep`, then restart manually +when access is restored. Services owned by another install remain untouched. +`--no-restart` still skips service restart. + Package-manager updates normally keep using the Node binary recorded in the managed service. If that Node cannot run the target release, but the current CLI Node can and the service is proven to belong to the package being updated, @@ -185,13 +208,15 @@ loaded/running for the active profile and the configured loopback port is healthy. If the plist is installed but launchd is not supervising it, OpenClaw re-bootstraps the LaunchAgent automatically and reruns the health/version/ channel readiness checks (a fresh bootstrap loads the `RunAtLoad` job directly, -so recovery does not immediately `kickstart -k` the newly spawned Gateway). If +so recovery does not immediately `kickstart -k` the newly spawned Gateway). +When preserving a definition, native restart/bootstrap runs without file repair; +a failed native activation or health check does not trigger a later plist rewrite. If the Gateway still does not become healthy, the command exits non-zero and prints the restart log path plus restart, reinstall, and package rollback instructions. If restart cannot run, the command prints `Gateway: restart skipped (...)` or -`Gateway: restart failed: ...` with a manual `openclaw gateway restart` hint. +`Gateway: restart failed: ...` with guidance to inspect the service and restart manually. With `--no-restart`, package replacement or git rebuild still runs, but the managed service is not stopped or restarted, so the running Gateway keeps old code until you restart it manually. diff --git a/docs/install/index.md b/docs/install/index.md index cc5d43fdbec2..7857a9068343 100644 --- a/docs/install/index.md +++ b/docs/install/index.md @@ -155,11 +155,13 @@ For contributors or anyone who wants to run from a local checkout: git clone https://github.com/openclaw/openclaw.git cd openclaw pnpm install && pnpm build && pnpm ui:build -pnpm link --global +pnpm add --global . openclaw onboard --install-daemon ``` -Or skip the link and use `pnpm openclaw ...` from inside the repo. See [Setup](/start/setup) for full development workflows. +`pnpm add --global .` links the CLI to this checkout without changing its package files. If pnpm reports that its global bin directory is not on `PATH`, run `pnpm setup`, reopen your shell, and retry. + +Or skip the global install and use `pnpm openclaw ...` from inside the repo. See [Setup](/start/setup) for full development workflows. ### Install from the GitHub main checkout diff --git a/docs/platforms/mac/remote.md b/docs/platforms/mac/remote.md index 8b12e0ef2a4f..aec26b10658e 100644 --- a/docs/platforms/mac/remote.md +++ b/docs/platforms/mac/remote.md @@ -28,7 +28,7 @@ Browser automation in remote mode is owned by the CLI node host, not the native ## Prereqs on the remote host -1. Install Node + pnpm and build/install the OpenClaw CLI (`pnpm install && pnpm build && pnpm link --global`). +1. Install Node + pnpm, then build/install the OpenClaw CLI from its checkout (`pnpm install && pnpm build && pnpm add --global .`). 2. Ensure `openclaw` is on PATH for non-interactive shells (symlink into `/usr/local/bin` or `/opt/homebrew/bin` if needed). 3. For SSH transport: set up key-based SSH auth. Tailscale IPs are recommended for stable reachability off-LAN. diff --git a/scripts/e2e/lib/doctor-install-switch/shims/busctl b/scripts/e2e/lib/doctor-install-switch/shims/busctl index a0f8e8599b49..cfb19aa4b5e8 100755 --- a/scripts/e2e/lib/doctor-install-switch/shims/busctl +++ b/scripts/e2e/lib/doctor-install-switch/shims/busctl @@ -11,6 +11,19 @@ function fail() { process.exit(1); } +function readUnitContent(unitPath, missingUnitName) { + try { + return fs.readFileSync(unitPath, "utf8"); + } catch (error) { + // Only a validated LoadUnit request may confirm absence; other read errors stay unavailable. + if (missingUnitName && error.code === "ENOENT") { + process.stderr.write(`Call failed: Unit ${missingUnitName} not found.\n`); + process.exit(1); + } + return fail(); + } +} + function splitWords(value) { const words = []; let word = ""; @@ -65,8 +78,14 @@ if (args[0] !== "--user" || args[1] !== "--json=short") { } const invocation = args.slice(2); +const unitName = `${process.env.OPENCLAW_SYSTEMD_UNIT || "openclaw-gateway"}`.replace( + /(?:\.service)?$/u, + ".service", +); +const unitPath = path.join(process.env.HOME, ".config/systemd/user", unitName); if (invocation[0] === "call") { - const [command, destination, objectPath, interfaceName, method, signature, unitName] = invocation; + const [command, destination, objectPath, interfaceName, method, signature, requestedUnit] = + invocation; if ( invocation.length !== 7 || command !== "call" || @@ -75,12 +94,12 @@ if (invocation[0] === "call") { interfaceName !== `${manager}.Manager` || method !== "LoadUnit" || signature !== "s" || - !unitName.endsWith(".service") || - path.basename(unitName) !== unitName || - !fs.existsSync(path.join(process.env.HOME, ".config/systemd/user", unitName)) + requestedUnit !== unitName || + path.basename(unitName) !== unitName ) { fail(); } + readUnitContent(unitPath, unitName); writeProperties([{ type: "o", data: [`${rootObjectPath}/unit/${encodeUnitName(unitName)}`] }]); process.exit(0); } @@ -89,19 +108,15 @@ if (invocation[0] !== "get-property" || invocation[1] !== manager) { fail(); } -const unitName = `${process.env.OPENCLAW_SYSTEMD_UNIT || "openclaw-gateway"}`.replace( - /(?:\.service)?$/u, - ".service", -); -const unitPath = path.join(process.env.HOME, ".config/systemd/user", unitName); if (invocation[2] !== `${rootObjectPath}/unit/${encodeUnitName(unitName)}`) { fail(); } +const content = readUnitContent(unitPath); if (invocation[3] === `${manager}.Unit`) { if ( - invocation.length !== 7 || - invocation.slice(4).join(" ") !== "FragmentPath DropInPaths NeedDaemonReload" + invocation.length !== 8 || + invocation.slice(4).join(" ") !== "FragmentPath DropInPaths NeedDaemonReload LoadState" ) { fail(); } @@ -109,6 +124,7 @@ if (invocation[3] === `${manager}.Unit`) { { type: "s", data: unitPath }, { type: "as", data: [] }, { type: "b", data: false }, + { type: "s", data: "loaded" }, ]); process.exit(0); } @@ -130,13 +146,6 @@ const unsetEnvironment = []; const expandHome = (value) => value.replace(/%%|%h/gu, (match) => (match === "%h" ? process.env.HOME : "%")); -let content; -try { - content = fs.readFileSync(unitPath, "utf8"); -} catch { - fail(); -} - for (const rawLine of content.split(/\r?\n/u)) { const line = rawLine.trim(); const separator = line.indexOf("="); diff --git a/scripts/e2e/lib/systemd-sealed-service-definition/file-mount.mjs b/scripts/e2e/lib/systemd-sealed-service-definition/file-mount.mjs new file mode 100644 index 000000000000..5ab0f04dd844 --- /dev/null +++ b/scripts/e2e/lib/systemd-sealed-service-definition/file-mount.mjs @@ -0,0 +1,184 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import fs from "node:fs/promises"; + +// The privileged setup runs only in a disposable container, without host mounts. +// Both the kernel probe and packaged CLI run as the ordinary appuser account. +const uid = Number(execFileSync("id", ["-u", "appuser"], { encoding: "utf8" }).trim()); +const gid = Number(execFileSync("id", ["-g", "appuser"], { encoding: "utf8" }).trim()); +const home = "/home/appuser"; +const state = `${home}/.openclaw`; +const unitDir = `${home}/.config/systemd/user`; +const unit = `${unitDir}/openclaw-gateway.service`; +const fixture = "/tmp/openclaw-file-mount"; +const shims = `${fixture}/bin`; +const kernelOnly = process.argv.includes("--kernel-only"); + +async function kernelProbe() { + const { default: check } = await import("node:assert/strict"); + const { promises: mountedFs, constants } = await import("node:fs"); + const { default: path } = await import("node:path"); + const file = process.argv[1]; + check.notEqual(process.geteuid(), 0); + check.equal((await mountedFs.stat(file)).uid, process.geteuid()); + await mountedFs.access(path.dirname(file), constants.W_OK | constants.X_OK); + const handle = await mountedFs.open(file, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const info = await mountedFs.readFile(`/proc/self/fdinfo/${handle.fd}`, "utf8"); + const mountId = /^mnt_id:\s+(\d+)$/m.exec(info)?.[1]; + const mount = (await mountedFs.readFile("/proc/self/mountinfo", "utf8")) + .split("\n") + .find((line) => line.startsWith(`${mountId} `)) + ?.split(" "); + check.equal(mount?.[4], file); + check(mount[5].split(",").includes("ro")); + } finally { + await handle.close(); + } + const temporary = `${file}.kernel-probe`; + await mountedFs.writeFile(temporary, "replacement"); + try { + await check.rejects(mountedFs.rename(temporary, file), { code: "EBUSY" }); + } finally { + await mountedFs.unlink(temporary); + } +} + +async function snapshot() { + const files = [ + unit, + `${unit}.bak`, + `${state}/openclaw.json`, + `${state}/gateway.systemd.env`, + `${state}/.env`, + ]; + return { + artifacts: await Promise.all( + files.map(async (file) => { + try { + const stat = await fs.stat(file); + return [ + file, + stat.uid, + stat.gid, + stat.mode, + stat.ino, + createHash("sha256") + .update(await fs.readFile(file)) + .digest("hex"), + ]; + } catch (error) { + if (error.code !== "ENOENT") { + throw error; + } + return [file, "missing"]; + } + }), + ), + unitEntries: (await fs.readdir(unitDir)).toSorted(), + stateEntries: (await fs.readdir(state)).filter((entry) => entry !== "state").toSorted(), + }; +} + +try { + assert.equal(process.geteuid(), 0); + for (const directory of [ + home, + `${home}/.config`, + `${home}/.config/systemd`, + unitDir, + state, + fixture, + shims, + ]) { + await fs.mkdir(directory, { recursive: true }); + await fs.chown(directory, uid, gid); + } + await fs.writeFile( + `${shims}/systemctl`, + `#!/bin/sh +case "$*" in + "--user is-enabled "*) echo enabled ;; + *--property=LoadState*) echo not-found ;; + *--property=UnitPath*) echo '/etc/systemd/system /usr/lib/systemd/system' ;; + "--user status") exit 0 ;; + *) exit 1 ;; +esac +`, + { mode: 0o755 }, + ); + await fs.writeFile( + `${shims}/busctl`, + `#!/bin/sh +printf '%s\n' 'Call failed: Unit openclaw-gateway.service not found.' >&2 +exit 1 +`, + { mode: 0o755 }, + ); + const childOptions = { + uid, + gid, + encoding: "utf8", + timeout: 60_000, + env: { + HOME: home, + USER: "appuser", + LOGNAME: "appuser", + PATH: `${shims}:/usr/local/bin:/usr/bin:/bin`, + }, + }; + for (const mode of [0o400, 0o644]) { + const source = `${fixture}/source-${mode}`; + await fs.writeFile( + source, + "[Service]\nExecStart=/usr/local/bin/node /app/openclaw.mjs gateway\n", + { mode }, + ); + await fs.chown(source, uid, gid); + await fs.writeFile(unit, ""); + for (const [file, contents] of [ + [`${state}/openclaw.json`, '{"gateway":{"mode":"local","auth":{"mode":"token"}}}'], + [`${state}/gateway.systemd.env`, "OPERATOR_VALUE=unchanged\n"], + [`${state}/.env`, "OPERATOR_VALUE=unchanged\n"], + ]) { + await fs.writeFile(file, contents, { mode: 0o600 }); + await fs.chown(file, uid, gid); + } + // Kernel setup must use Debian-owned tools, not PATH/npm fixture replacements. + execFileSync("/bin/mount", ["--bind", source, unit]); + try { + execFileSync("/bin/mount", ["-o", "remount,bind,ro", unit]); + const probe = spawnSync( + process.execPath, + ["--input-type=module", "-e", `await (${kernelProbe.toString()})();`, unit], + childOptions, + ); + assert.equal(probe.status, 0, probe.stderr); + if (!kernelOnly) { + const before = await snapshot(); + const result = spawnSync( + process.execPath, + ["/app/openclaw.mjs", "gateway", "install", "--force", "--json"], + childOptions, + ); + assert.notEqual(result.status, null, "packaged CLI must finish normally"); + assert.notEqual(result.status, 0); + assert.match(`${result.stdout}${result.stderr}`, /SERVICE_DEFINITION_SEALED/); + assert.deepEqual( + await snapshot(), + before, + "force-install must not mutate config/token/env/unit/backup", + ); + } + console.log( + `${kernelOnly ? "Kernel contract only" : "Packaged force-install denial"}: same-UID read-only file mount, mode=${mode.toString(8)}.`, + ); + } finally { + execFileSync("/bin/umount", [unit]); + } + } +} catch (error) { + console.error(error); + process.exitCode = 1; +} diff --git a/scripts/e2e/lib/systemd-sealed-service-definition/paired-mounts.mjs b/scripts/e2e/lib/systemd-sealed-service-definition/paired-mounts.mjs new file mode 100644 index 000000000000..acd8cc80f6e6 --- /dev/null +++ b/scripts/e2e/lib/systemd-sealed-service-definition/paired-mounts.mjs @@ -0,0 +1,54 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +try { + const shouldMatch = process.argv[2] === "same"; + const current = "/proof/current"; + const release = "/proof/releases/selected"; + assert.notEqual(process.geteuid(), 0, "proof must run unprivileged"); + assert.equal((await fs.lstat(current)).isSymbolicLink(), false); + assert.equal(await fs.realpath(current), current); + const selectedStat = await fs.stat(release); + const currentStat = await fs.stat(current); + assert.equal( + currentStat.dev === selectedStat.dev && currentStat.ino === selectedStat.ino, + shouldMatch, + ); + await assert.rejects(fs.writeFile(`${current}/must-not-write`, "forbidden"), { code: "EROFS" }); + await assert.rejects(fs.writeFile(`${release}/must-not-write`, "forbidden"), { code: "EROFS" }); + + // Exercise the update lifecycle's root-ownership boundary from the installed package. + let ownsRoot; + const bundles = (await fs.readdir("/app/dist")) + .filter((file) => /^update-command-service-.*\.js$/.test(file)) + .toSorted(); + for (const file of bundles) { + const module = await import(pathToFileURL(`/app/dist/${file}`).href); + ownsRoot = Object.values(module).find( + (value) => typeof value === "function" && value.name === "gatewayServiceCommandUsesRoot", + ); + if (ownsRoot) { + break; + } + } + assert.equal(typeof ownsRoot, "function", "packaged update ownership export is required"); + const result = await ownsRoot({ + root: "/proof/openclaw", + command: { + programArguments: [process.execPath, `${current}/dist/index.js`, "gateway"], + managedDefinition: { + programArguments: [process.execPath, "/proof/openclaw/dist/index.js", "gateway"], + }, + managedOverrides: { launcher: "command" }, + }, + }); + assert.equal(result, shouldMatch); + console.log( + `Paired read-only bind mount proof passed: ${shouldMatch ? "matching release accepted" : "different release rejected"}.`, + ); +} catch (error) { + console.error(error); + console.error("[systemd-paired-mounts] FAILED (exit 1)"); + process.exitCode = 1; +} diff --git a/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs b/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs new file mode 100644 index 000000000000..83f0a43fa735 --- /dev/null +++ b/scripts/e2e/lib/upgrade-survivor/systemd-fixture.mjs @@ -0,0 +1,292 @@ +// The direct survivor lane supports generated user units, not arbitrary systemd configuration. +// Inspection and launch share this parser so reported argv/environment cannot drift from execution. +import fs from "node:fs"; +import path from "node:path"; + +const unitName = "openclaw-gateway.service"; +const unitPath = path.join(process.env.HOME, ".config/systemd/user", unitName); +const loadedPath = `${process.env.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE || "/tmp/openclaw-systemctl-shim.pid"}.loaded-unit`; +const manager = "org.freedesktop.systemd1"; +const root = "/org/freedesktop/systemd1"; +const object = `${root}/unit/openclaw_2dgateway_2eservice`; + +function fail(message = "Unsupported survivor manager request or generated unit grammar.") { + throw new Error(message); +} + +// buildSystemdUnit quotes whole words and escapes only quotes/backslashes. +function words(value) { + const result = []; + const pattern = /(?:"(?:[^"\\]|\\["\\])*"|[^\s"\\]+)(?:\s+|$)/gy; + let offset = 0; + while (offset < value.length) { + pattern.lastIndex = offset; + const match = pattern.exec(value); + if (!match) { + fail(); + } + const word = match[0].trimEnd(); + result.push(word.startsWith('"') ? word.slice(1, -1).replace(/\\(["\\])/g, "$1") : word); + offset = pattern.lastIndex; + } + return result.map((word) => + word.replace(/%%|%h|%/g, (specifier) => { + if (specifier === "%") { + fail(); + } + return specifier === "%h" ? process.env.HOME : "%"; + }), + ); +} + +function assignments(values) { + return Object.fromEntries( + values.map((value) => { + const separator = value.indexOf("="); + if (separator <= 0 || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(value.slice(0, separator))) { + fail(); + } + return [value.slice(0, separator), value.slice(separator + 1)]; + }), + ); +} + +function parseUnit(content) { + const directives = new Map(); + let section = ""; + for (const raw of content.split(/\r?\n/)) { + const line = raw.trim(); + if (!line || line.startsWith("#")) { + continue; + } + if (line.startsWith("[")) { + section = line; + continue; + } + if (section !== "[Service]") { + continue; + } + const separator = line.indexOf("="); + const key = line.slice(0, separator); + if (separator < 0) { + fail(); + } + const values = directives.get(key) || []; + values.push(line.slice(separator + 1)); + directives.set(key, values); + } + const single = (key) => { + const values = directives.get(key) || []; + if (values.length > 1) { + fail(); + } + return values[0] || ""; + }; + const programArguments = words(single("ExecStart")); + if (!programArguments.length || !path.isAbsolute(programArguments[0])) { + fail(); + } + const workingDirectories = words(single("WorkingDirectory")); + if (workingDirectories.length > 1) { + fail(); + } + const environment = assignments( + (directives.get("Environment") || []).flatMap((value) => { + if (!value) { + fail(); + } + return words(value); + }), + ); + const environmentFiles = (directives.get("EnvironmentFile") || []).map((value) => { + const optional = value.startsWith("-"); + const filenames = words(optional ? value.slice(1) : value); + if (filenames.length !== 1 || !path.isAbsolute(filenames[0])) { + fail(); + } + return [filenames[0], optional]; + }); + const supported = new Set([ + "ExecStart", + "WorkingDirectory", + "Environment", + "EnvironmentFile", + "Restart", + "RestartSec", + "RestartPreventExitStatus", + "TimeoutStopSec", + "TimeoutStartSec", + "SuccessExitStatus", + "OOMPolicy", + "KillMode", + ]); + if ([...directives.keys()].some((key) => !supported.has(key))) { + fail(); + } + return { + programArguments, + workingDirectory: workingDirectories[0] || "", + environment, + environmentFiles, + }; +} + +function readUnit(reload = false) { + for (const directory of [`${unitPath}.d`, path.join(path.dirname(unitPath), "service.d")]) { + if (fs.existsSync(directory) && fs.readdirSync(directory).length) { + fail(); + } + } + let content; + try { + content = fs.readFileSync(unitPath, "utf8"); + } catch (error) { + if (error.code !== "ENOENT") { + throw error; + } + fs.rmSync(loadedPath, { force: true }); + return null; + } + parseUnit(content); + // Keep the manager's loaded command until daemon-reload; file edits alone do not activate it. + if (reload || !fs.existsSync(loadedPath)) { + fs.writeFileSync(loadedPath, content); + } + const loaded = fs.readFileSync(loadedPath, "utf8"); + return { ...parseUnit(loaded), reloadPending: loaded !== content }; +} + +function writeProperties(properties) { + for (const [type, data] of properties) { + console.log(JSON.stringify({ type, data })); + } +} + +function run() { + const [operation, ...args] = process.argv.slice(2); + if (operation === "reload" && !args.length) { + readUnit(true); + return; + } + if (operation === "command" && !args.length) { + const unit = readUnit(); + if (!unit) { + fail("Cannot launch an absent fixture unit."); + } + const environment = { ...unit.environment }; + for (const [filename, optional] of unit.environmentFiles) { + let content; + try { + content = fs.readFileSync(filename, "utf8"); + } catch (error) { + if (optional && error.code === "ENOENT") { + continue; + } + throw error; + } + for (const line of content.split(/\r?\n/)) { + if (!line.trim() || line.startsWith("#")) { + continue; + } + const separator = line.indexOf("="); + if (separator <= 0) { + fail(); + } + const raw = line.slice(separator + 1); + // serializeSystemdEnvironmentFile escapes exactly these four characters. + const value = + raw.startsWith('"') && raw.endsWith('"') + ? raw.slice(1, -1).replace(/\\(["\\`$])/g, "$1") + : raw; + Object.assign(environment, assignments([`${line.slice(0, separator)}=${value}`])); + } + } + const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`; + const command = [ + "env", + ...Object.entries(environment).map(([key, value]) => `${key}=${value}`), + ...unit.programArguments, + ]; + console.log( + `cd ${quote(unit.workingDirectory || process.env.HOME)} && exec ${command.map(quote).join(" ")}`, + ); + return; + } + if (operation !== "busctl") { + fail(); + } + const matches = (expected) => + args.length === expected.length && args.every((arg, index) => arg === expected[index]); + const prefix = ["--user", "--json=short"]; + const load = matches([ + ...prefix, + "call", + manager, + root, + `${manager}.Manager`, + "LoadUnit", + "s", + unitName, + ]); + const unitQuery = matches([ + ...prefix, + "get-property", + manager, + object, + `${manager}.Unit`, + "FragmentPath", + "DropInPaths", + "NeedDaemonReload", + "LoadState", + ]); + const serviceQuery = matches([ + ...prefix, + "get-property", + manager, + object, + `${manager}.Service`, + "ExecStart", + "WorkingDirectory", + "Environment", + "EnvironmentFiles", + "UnsetEnvironment", + ]); + if (!load && !unitQuery && !serviceQuery) { + fail(); + } + const unit = readUnit(); + if (!unit) { + if (load) { + fail(`Call failed: Unit ${unitName} not found.`); + } + fail("Fixture unit is not loaded."); + } + if (load) { + writeProperties([["o", [object]]]); + } else if (unitQuery) { + writeProperties([ + ["s", unitPath], + ["as", []], + ["b", unit.reloadPending], + ["s", "loaded"], + ]); + } else { + writeProperties([ + [ + "a(sasbttttuii)", + [[unit.programArguments[0], unit.programArguments, false, 0, 0, 0, 0, 0, 0, 0]], + ], + ["s", unit.workingDirectory], + ["as", Object.entries(unit.environment).map(([key, value]) => `${key}=${value}`)], + ["a(sb)", unit.environmentFiles], + ["as", []], + ]); + } +} + +try { + run(); +} catch (error) { + console.error(error.message); + process.exitCode = 1; +} diff --git a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh index 75e5609af6a5..3553730c6569 100644 --- a/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh +++ b/scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh @@ -3,6 +3,11 @@ install_update_restart_systemctl_shim() { local shim_dir="$npm_config_prefix/bin" mkdir -p "$shim_dir" + cp "$(dirname "${BASH_SOURCE[0]}")/systemd-fixture.mjs" "$shim_dir/systemd-fixture.mjs" + cat >"$shim_dir/busctl" <<'BUSCTL' +#!/usr/bin/env bash +exec node "$(dirname "$0")/systemd-fixture.mjs" busctl "$@" +BUSCTL cat >"$shim_dir/systemctl" <<'SHIM' #!/usr/bin/env bash set -euo pipefail @@ -11,6 +16,7 @@ log_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG:-/tmp/openclaw-systemct pid_file="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE:-/tmp/openclaw-systemctl-shim.pid}" daemon_log="${OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG:-/tmp/openclaw-systemctl-shim-gateway.log}" supervisor_script="${pid_file}.supervisor.mjs" +manager_script="$(dirname "$0")/systemd-fixture.mjs" printf '%s\n' "$*" >>"$log_file" filtered=() @@ -38,6 +44,12 @@ for ((i = 1; i <= $#; i++)); do done command="${filtered[0]:-status}" +unit_name="${filtered[1]:-}" +if [ "${#filtered[@]}" -gt 2 ] || + { [ -n "$unit_name" ] && [ "$unit_name" != openclaw-gateway.service ] && [ "$unit_name" != openclaw.service ]; }; then + echo "systemctl shim unsupported unit or arguments: $*" >&2 + exit 1 +fi is_running() { [ -s "$pid_file" ] || return 1 @@ -71,40 +83,9 @@ unit_path() { printf '%s/.config/systemd/user/openclaw-gateway.service\n' "${HOME:?missing HOME}" } -load_unit_environment() { - local unit="$1" - while IFS= read -r line; do - case "$line" in - EnvironmentFile=*) - local spec="${line#EnvironmentFile=}" - for token in $spec; do - local file="${token#-}" - [ -f "$file" ] || continue - set -a - # shellcheck disable=SC1090 - . "$file" - set +a - done - ;; - Environment=*) - local assignment="${line#Environment=}" - assignment="${assignment#\"}" - assignment="${assignment%\"}" - export "$assignment" - ;; - esac - done <"$unit" -} - start_gateway() { - local unit local exec_start - unit="$(unit_path)" - exec_start="$(sed -n 's/^ExecStart=//p' "$unit" | tail -n 1)" - [ -n "$exec_start" ] || { - echo "systemctl shim could not find ExecStart in $unit" >&2 - return 1 - } + exec_start="$(node "$manager_script" command)" rm -f "$pid_file" "$supervisor_script" cat >"$supervisor_script" <<'SUPERVISOR' import fs from "node:fs"; @@ -218,7 +199,7 @@ const start = () => { return finish(); } starts.push(now); - child = spawn("bash", ["-lc", `exec ${command}`], { + child = spawn("bash", ["-c", command], { detached: true, env: childEnv, stdio: ["ignore", output, output], @@ -244,7 +225,6 @@ process.on("SIGTERM", stop); start(); SUPERVISOR ( - load_unit_environment "$unit" OPENCLAW_SYSTEMCTL_SHIM_EXEC_START="$exec_start" \ OPENCLAW_SYSTEMCTL_SHIM_DAEMON_LOG="$daemon_log" \ nohup node "$supervisor_script" /dev/null 2>&1 & @@ -253,26 +233,48 @@ SUPERVISOR } case "$command" in - daemon-reload | enable | disable) + daemon-reload) + [ "$system_scope" = 0 ] && [ -z "$unit_name" ] || exit 1 + node "$manager_script" reload + exit 0 + ;; + enable | disable | reset-failed) + [ "$system_scope" = 0 ] && [ "$unit_name" = openclaw-gateway.service ] || exit 1 + [ -f "$(unit_path)" ] || exit 1 + if [ "$command" = enable ]; then + mkdir -p "$(dirname "$(unit_path)")/default.target.wants" + ln -sf ../openclaw-gateway.service "$(dirname "$(unit_path)")/default.target.wants/openclaw-gateway.service" + elif [ "$command" = disable ]; then + stop_gateway + rm -f "$(dirname "$(unit_path)")/default.target.wants/openclaw-gateway.service" + fi exit 0 ;; status) - is_running && exit 0 - exit 0 + [ "$system_scope" = 0 ] || exit 1 + [ -z "$unit_name" ] && exit 0 + [ "$unit_name" = openclaw-gateway.service ] && is_running && exit 0 + exit 3 ;; stop) + [ "$system_scope" = 0 ] && [ "$unit_name" = openclaw-gateway.service ] || exit 1 stop_gateway exit 0 ;; restart | start) + [ "$system_scope" = 0 ] && [ "$unit_name" = openclaw-gateway.service ] || exit 1 stop_gateway start_gateway exit 0 ;; is-enabled) - exit 0 + [ "$system_scope" = 0 ] && [ "$unit_name" = openclaw-gateway.service ] && + [ -f "$(unit_path)" ] && [ -L "$(dirname "$(unit_path)")/default.target.wants/openclaw-gateway.service" ] && exit 0 + printf 'disabled\n' + exit 1 ;; is-active) + [ "$system_scope" = 0 ] && [ "$unit_name" = openclaw-gateway.service ] || exit 1 is_running && exit 0 exit 3 ;; @@ -280,9 +282,11 @@ case "$command" in if [ "$system_scope" = "1" ]; then case "$property" in LoadState) + [ -n "$unit_name" ] || exit 1 printf 'not-found\n' ;; UnitPath) + [ -z "$unit_name" ] || exit 1 printf '/etc/systemd/system /usr/lib/systemd/system\n' ;; *) @@ -292,6 +296,11 @@ case "$command" in esac exit 0 fi + [ "$unit_name" = openclaw-gateway.service ] || exit 1 + [ "$property" = 'Id,ActiveState,SubState,Result,NRestarts,StartLimitBurst,MainPID,ExecMainStatus,ExecMainCode,KillMode,TasksCurrent,MemoryCurrent' ] || { + echo "systemctl shim unsupported user-scope show: $*" >&2 + exit 1 + } if is_running; then printf 'ActiveState=active\nSubState=running\nMainPID=%s\nExecMainStatus=0\nExecMainCode=0\n' "$(cat "$pid_file")" else @@ -305,10 +314,23 @@ case "$command" in ;; esac SHIM - chmod +x "$shim_dir/systemctl" + chmod +x "$shim_dir/systemctl" "$shim_dir/busctl" export PATH="$shim_dir:$PATH" } +assert_update_restart_service_replaced() { + local previous_pid="$1" previous_log_lines="$2" current_pid + current_pid="$(cat "$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE")" + if [ "$current_pid" = "$previous_pid" ] || + ! systemctl --user is-active --quiet openclaw-gateway.service || + ! tail -n +"$((previous_log_lines + 1))" "$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG" | + grep -Fx -- '--user restart openclaw-gateway.service' >/dev/null; then + echo "Update did not replace the managed gateway supervisor through restart." >&2 + return 1 + fi + echo "Update-owned fixture restart replaced supervisor $previous_pid with $current_pid." +} + seed_update_restart_probe_device_auth() { node --input-type=module <<'NODE' import crypto from "node:crypto"; diff --git a/scripts/e2e/systemd-sealed-service-definition.sh b/scripts/e2e/systemd-sealed-service-definition.sh new file mode 100644 index 000000000000..7c3c3bff2fb2 --- /dev/null +++ b/scripts/e2e/systemd-sealed-service-definition.sh @@ -0,0 +1,204 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh" + +IMAGE_NAME="$(docker_e2e_resolve_image \ + "openclaw-systemd-sealed-service-definition-e2e" \ + OPENCLAW_SYSTEMD_SEALED_SERVICE_DEFINITION_E2E_IMAGE)" + +docker_e2e_build_or_reuse "$IMAGE_NAME" systemd-sealed-service-definition + +echo "Proving root-owned systemd service definitions reject unprivileged installation without writes..." +docker_e2e_run_with_harness -i --user root "$IMAGE_NAME" bash -s <<'SCENARIO' +set -euo pipefail + +service_home=/home/appuser +state_dir="$service_home/.openclaw" +unit_dir="$service_home/.config/systemd/user" +unit_path="$unit_dir/openclaw-gateway.service" +environment_path="$state_dir/gateway.systemd.env" +config_path="$state_dir/openclaw.json" +shim_dir=/tmp/openclaw-sealed-systemd-bin +token_canary=sealed-docker-proof-token + +[[ "$(id -u)" == 0 && "$(runuser -u appuser -- id -u)" != 0 ]] || { + echo "The fixture requires root setup and an unprivileged appuser." >&2 + exit 1 +} +install -d -o appuser -g appuser -m 0755 "$service_home/.config" "$service_home/.config/systemd" +install -d -o appuser -g appuser -m 0700 "$state_dir" +install -d -o root -g root -m 0555 "$unit_dir" +install -d -o root -g root -m 0755 "$shim_dir" +install -o root -g root -m 0755 scripts/e2e/lib/doctor-install-switch/shims/systemctl "$shim_dir/systemctl" +install -o root -g root -m 0755 /dev/stdin "$shim_dir/busctl" <<'BUSCTL' +#!/usr/bin/env bash +set -euo pipefail + +manager=org.freedesktop.systemd1 +object_path=/org/freedesktop/systemd1/unit/openclaw_2dgateway_2eservice +actual=("$@") +expected=(--user --json=short) + +reject_invocation() { + printf '%s\n' 'Unsupported sealed systemd busctl invocation.' >&2 + exit 1 +} + +case "${actual[2]:-}" in + call) + expected+=(call "$manager" /org/freedesktop/systemd1 "$manager.Manager" LoadUnit s openclaw-gateway.service) + response=manager + ;; + get-property) + case "${actual[5]:-}" in + "$manager.Service") + expected+=(get-property "$manager" "$object_path" "$manager.Service" ExecStart WorkingDirectory Environment EnvironmentFiles UnsetEnvironment) + response=service + ;; + "$manager.Unit") + expected+=(get-property "$manager" "$object_path" "$manager.Unit" FragmentPath DropInPaths NeedDaemonReload LoadState) + response=unit + ;; + *) reject_invocation ;; + esac + ;; + *) reject_invocation ;; +esac + +[[ "${#actual[@]}" -eq "${#expected[@]}" ]] || reject_invocation +for index in "${!expected[@]}"; do + [[ "${actual[$index]}" == "${expected[$index]}" ]] || reject_invocation +done + +case "$response" in + manager) + printf '%s\n' '{"type":"o","data":["/org/freedesktop/systemd1/unit/openclaw_2dgateway_2eservice"]}' + ;; + service) + printf '%s\n' \ + '{"type":"a(sasbttttuii)","data":[["/usr/local/bin/node",["/usr/local/bin/node","/app/openclaw.mjs","gateway","--port","18789"],false,0,0,0,0,0,0,0]]}' \ + '{"type":"s","data":"/app"}' \ + '{"type":"as","data":["OPENCLAW_GATEWAY_PORT=18789"]}' \ + '{"type":"a(sb)","data":[["/home/appuser/.openclaw/gateway.systemd.env",false]]}' \ + '{"type":"as","data":[]}' + ;; + unit) + printf '%s\n' \ + '{"type":"s","data":"/home/appuser/.config/systemd/user/openclaw-gateway.service"}' \ + '{"type":"as","data":[]}' \ + '{"type":"b","data":false}' \ + '{"type":"s","data":"loaded"}' + ;; +esac +BUSCTL + +install_sealed_unit() { + install -o root -g "$1" -m "$2" /dev/stdin "$unit_path" <<'UNIT' +[Unit] +Description=OpenClaw Gateway (sealed ownership proof) +[Service] +ExecStart=/usr/local/bin/node /app/openclaw.mjs gateway --port 18789 +WorkingDirectory=/app +Environment=OPENCLAW_GATEWAY_PORT=18789 +EnvironmentFile=/home/appuser/.openclaw/gateway.systemd.env +[Install] +WantedBy=default.target +UNIT +} + +install_sealed_unit root 0444 +printf '%s\n' 'OPENCLAW_SEALED_DOCKER_PROOF=from-state-dotenv' | + install -o appuser -g appuser -m 0600 /dev/stdin "$state_dir/.env" +printf '%s\n' 'OPENCLAW_SEALED_DOCKER_PROOF=preserve-original-generated-environment' | + install -o appuser -g appuser -m 0600 /dev/stdin "$environment_path" + +snapshot_managed_state() { + stat -c '%n:%u:%g:%a:%i' "$unit_dir" "$state_dir" "$unit_path" "$environment_path" "$state_dir/.env" + sha256sum "$unit_path" "$environment_path" "$state_dir/.env" + if [[ -e "$config_path" ]]; then + stat -c '%n:%u:%g:%a:%i' "$config_path" + sha256sum "$config_path" + else + printf 'config=absent\n' + fi + find "$unit_dir" -mindepth 1 -maxdepth 1 -printf 'unit-entry=%f\n' | sort + find "$state_dir" -mindepth 1 -maxdepth 1 ! -name state -printf 'state-entry=%f\n' | sort +} + +for scenario in missing-mode missing-token missing-config group-writable-root-owned; do + case "$scenario" in + missing-mode) config='{"gateway":{"port":18789,"auth":{"mode":"token","token":"sealed-docker-proof-token"}}}' ;; + missing-token) config='{"gateway":{"mode":"local","port":18789,"auth":{"mode":"token"}}}' ;; + missing-config) config='' ;; + group-writable-root-owned) + rm -f "$unit_path" + rmdir "$unit_dir" + install -d -o appuser -g appuser -m 0755 "$unit_dir" + install_sealed_unit appuser 0664 + runuser -u appuser -- test -w "$unit_dir" + runuser -u appuser -- test -w "$unit_path" + config='{"gateway":{"mode":"local","port":18789,"auth":{"mode":"token","token":"sealed-docker-proof-token"}}}' + ;; + esac + if [[ -n "$config" ]]; then + printf '%s\n' "$config" | install -o appuser -g appuser -m 0600 /dev/stdin "$config_path" + else + rm -f "$config_path" + fi + + state_before="$(snapshot_managed_state)" + install_result=0 + output="$(runuser -u appuser -- env -u OPENCLAW_GATEWAY_TOKEN -u OPENCLAW_GATEWAY_PASSWORD \ + HOME="$service_home" USER=appuser LOGNAME=appuser PATH="$shim_dir:$PATH" \ + OPENCLAW_STATE_DIR="$state_dir" OPENCLAW_CONFIG_PATH="$config_path" \ + node /app/openclaw.mjs gateway install --force --json 2>&1)" || install_result=$? + + if [[ "$install_result" == 0 || "$output" != *SERVICE_DEFINITION_SEALED* || + "$output" != *"privileged deployment owner"* || "$output" == *"$token_canary"* ]]; then + echo "Unprivileged sealed service install did not fail safely ($scenario)." >&2 + exit 1 + fi + if [[ "$state_before" != "$(snapshot_managed_state)" ]]; then + echo "Sealed service install changed protected bytes, metadata, or directory entries ($scenario)." >&2 + exit 1 + fi + echo "Sealed systemd ownership proof passed without writes ($scenario)." +done +SCENARIO + +# File mounts need SYS_ADMIN only inside this disposable fixture. Feed the helper +# over stdin: no host paths, private data, or Docker socket enter this container. +# Docker tmpfs defaults to noexec; only the fixture's shim directory needs execution. +file_mount_cid_dir="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-file-mount-cid.XXXXXX")" +trap 'docker_e2e_cleanup_container_cidfile "$file_mount_cid_dir/container.cid"; rm -rf "$file_mount_cid_dir"' EXIT +docker_e2e_docker_run_cmd run --rm -i --cidfile "$file_mount_cid_dir/container.cid" \ + --network none --read-only --tmpfs /tmp:rw,mode=1777 --tmpfs /home/appuser:rw \ + --tmpfs /tmp/openclaw-file-mount/bin:rw,exec,mode=0755 \ + --cap-drop ALL --cap-add SYS_ADMIN --cap-add CHOWN --cap-add DAC_OVERRIDE \ + --cap-add SETUID --cap-add SETGID --security-opt seccomp=unconfined \ + --security-opt no-new-privileges --user 0 --entrypoint node "$IMAGE_NAME" --input-type=module \ + <"$ROOT_DIR/scripts/e2e/lib/systemd-sealed-service-definition/file-mount.mjs" +docker_e2e_cleanup_container_cidfile "$file_mount_cid_dir/container.cid" +rmdir "$file_mount_cid_dir" +trap - EXIT + +mount_fixture="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-systemd-mounts.XXXXXX")" +trap 'rm -rf "$mount_fixture"' EXIT +for release in "$mount_fixture/releases/selected" "$mount_fixture/unrelated" "$mount_fixture/openclaw"; do + mkdir -p "$release/dist" + printf '%s\n' '{"name":"openclaw"}' >"$release/package.json" + printf '%s\n' '// inert release fixture' >"$release/dist/index.js" +done +for relationship in same different; do + current_source="$mount_fixture/releases/selected" + if [[ "$relationship" == different ]]; then + current_source="$mount_fixture/unrelated" + fi + docker_e2e_run_with_harness --network none --user appuser \ + --mount "type=bind,src=$mount_fixture/openclaw,dst=/proof/openclaw,readonly" \ + --mount "type=bind,src=$mount_fixture/releases,dst=/proof/releases,readonly" \ + --mount "type=bind,src=$current_source,dst=/proof/current,readonly" \ + "$IMAGE_NAME" node scripts/e2e/lib/systemd-sealed-service-definition/paired-mounts.mjs "$relationship" +done diff --git a/scripts/e2e/upgrade-survivor-docker.sh b/scripts/e2e/upgrade-survivor-docker.sh index c84b16a0d6b3..616a3ac7c82b 100755 --- a/scripts/e2e/upgrade-survivor-docker.sh +++ b/scripts/e2e/upgrade-survivor-docker.sh @@ -4,6 +4,32 @@ # baseline first and upgrades it to the selected candidate. set -euo pipefail +PACKAGE_TGZ="" +AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT="" +run_completed="0" +cleanup_outer() { + local exit_status="$?" + trap - EXIT + set +e + if [ -n "$PACKAGE_TGZ" ]; then + docker_e2e_cleanup_package_tgz "$PACKAGE_TGZ" + fi + if [ -n "$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" ]; then + rm -rf "$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" + fi + # Bash 3.2 can enter EXIT with status 0 after a fatal nounset expansion. + # Only a successfully joined scenario may turn cleanup into a successful exit. + if [ "$exit_status" -eq 0 ] && [ "$run_completed" != "1" ]; then + echo "Upgrade survivor exited before the scenario completed." >&2 + exit_status=1 + fi + if [ "$exit_status" -ne 0 ]; then + printf '[upgrade-survivor] FAILED (exit %s)\n' "$exit_status" >&2 + fi + exit "$exit_status" +} +trap cleanup_outer EXIT + HARNESS_ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" ROOT_DIR="$(cd "${OPENCLAW_DOCKER_E2E_REPO_ROOT:-$HARNESS_ROOT_DIR}" && pwd)" DOCKER_E2E_HARNESS_ROOT_DIR="$HARNESS_ROOT_DIR" @@ -93,7 +119,6 @@ LANE_ARTIFACT_SUFFIX="${LANE_ARTIFACT_SUFFIX//[^A-Za-z0-9_.-]/_}" ARTIFACT_DIR="${OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_DIR:-$ROOT_DIR/.artifacts/upgrade-survivor/$LANE_ARTIFACT_SUFFIX}" DOCKER_RUN_USER_ARGS=() OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DOCKER_ARGS=() -AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT="" PROBE_ENV_ARGS=( -e OPENCLAW_UPGRADE_SURVIVOR_PROBE_TIMEOUT_MS="$PROBE_TIMEOUT_MS" -e OPENCLAW_UPGRADE_SURVIVOR_PROBE_ATTEMPT_TIMEOUT_MS="$PROBE_ATTEMPT_TIMEOUT_MS" @@ -113,14 +138,6 @@ if [ -n "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR:-}" ]; then openclaw_prepublish_plugin_registry_configure_docker_args \ "$OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR" fi -cleanup_outer() { - docker_e2e_cleanup_package_tgz "${PACKAGE_TGZ:-}" - if [ -n "$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" ]; then - rm -rf "$AUTO_PREPUBLISH_PLUGIN_REGISTRY_ROOT" - fi -} -trap cleanup_outer EXIT - if [ "$ROOT_MANAGED_VPS" = "1" ]; then if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" != "1" ]; then echo "OPENCLAW_UPGRADE_SURVIVOR_ROOT_MANAGED_VPS=1 requires OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE=1" >&2 @@ -249,6 +266,7 @@ if [ "${OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE:-0}" = "1" ]; then ${DOCKER_RUN_USER_ARGS[@]+"${DOCKER_RUN_USER_ARGS[@]}"} \ "$IMAGE_NAME" \ timeout --kill-after=30s "$DOCKER_RUN_TIMEOUT" bash /tmp/openclaw-upgrade-survivor-run.sh + run_completed="1" exit 0 fi @@ -293,9 +311,9 @@ docker_e2e_run_with_harness \ -v "$ARTIFACT_DIR:/tmp/openclaw-upgrade-survivor-artifacts" \ -v "$HARNESS_ROOT_DIR/scripts/e2e/lib/clawhub-fixture-server.cjs:/tmp/openclaw-clawhub-fixture-server.cjs:ro" \ -v "$HARNESS_ROOT_DIR/scripts/e2e/lib/upgrade-survivor/config-parking.mjs:/tmp/openclaw-config-parking.mjs:ro" \ - "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DOCKER_ARGS[@]}" \ - "${DOCKER_E2E_PACKAGE_ARGS[@]}" \ - "${DOCKER_RUN_USER_ARGS[@]}" \ + ${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DOCKER_ARGS[@]+"${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DOCKER_ARGS[@]}"} \ + ${DOCKER_E2E_PACKAGE_ARGS[@]+"${DOCKER_E2E_PACKAGE_ARGS[@]}"} \ + ${DOCKER_RUN_USER_ARGS[@]+"${DOCKER_RUN_USER_ARGS[@]}"} \ "$IMAGE_NAME" \ timeout --kill-after=30s "$DOCKER_RUN_TIMEOUT" bash -lc 'set -euo pipefail source scripts/lib/openclaw-e2e-instance.sh @@ -355,7 +373,11 @@ export OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SERVICE_INSTALL_ERR="$BASELINE_SERVICE gateway_pid="" plugin_registry_pid="" clawhub_fixture_pid="" +run_completed="0" cleanup() { + local exit_status="$?" + trap - EXIT + set +e if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then systemctl --user stop openclaw-gateway.service >/dev/null 2>&1 || true fi @@ -365,6 +387,11 @@ cleanup() { fi openclaw_e2e_stop_process "${plugin_registry_pid:-}" openclaw_e2e_stop_process "${clawhub_fixture_pid:-}" + if [ "$exit_status" -eq 0 ] && [ "$run_completed" != "1" ]; then + echo "Upgrade survivor exited before all assertions completed." >&2 + exit_status=1 + fi + exit "$exit_status" } trap cleanup EXIT @@ -468,7 +495,7 @@ NODE "${OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256:-}" \ "$fixture_root" \ plugin_registry_pid \ - "${registry_args[@]}" + ${registry_args[@]+"${registry_args[@]}"} } install_companion_plugins() { @@ -549,6 +576,8 @@ if [ "$UPDATE_RESTART_MODE" = "auto-auth" ]; then # shellcheck disable=SC1091 source scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh prepare_update_restart_probe_current_install "$PORT" "$GATEWAY_LOG" + pre_update_service_pid="$(cat "$SYSTEMCTL_SHIM_PID_FILE")" + pre_update_systemctl_lines="$(wc -l <"$SYSTEMCTL_SHIM_LOG")" fi echo "Running package update against the mounted tarball..." @@ -593,6 +622,7 @@ node scripts/e2e/lib/upgrade-survivor/assertions.mjs assert-state startup_summary="n/a" if [ "$UPDATE_RESTART_MODE" = "auto-auth" ]; then + assert_update_restart_service_replaced "$pre_update_service_pid" "$pre_update_systemctl_lines" echo "Gateway restart was handled by openclaw update." else echo "Starting gateway from upgraded state..." @@ -650,4 +680,6 @@ fi node scripts/e2e/lib/upgrade-survivor/assertions.mjs assert-status-json /tmp/openclaw-upgrade-survivor-status.json echo "Upgrade survivor Docker E2E passed scenario=${OPENCLAW_UPGRADE_SURVIVOR_SCENARIO:-base} updateRestartMode=${UPDATE_RESTART_MODE} startup=${startup_summary} status=${status_seconds}s." +run_completed="1" ' +run_completed="1" diff --git a/scripts/install-cli.sh b/scripts/install-cli.sh index efeb07b2f11c..c0fe252856db 100755 --- a/scripts/install-cli.sh +++ b/scripts/install-cli.sh @@ -1713,7 +1713,7 @@ try { } refresh_gateway_service_if_loaded() { - local claw="${PREFIX}/bin/openclaw" + local claw="${PREFIX}/bin/openclaw" refresh_output if [[ ! -x "$claw" ]]; then return 0 fi @@ -1726,7 +1726,13 @@ refresh_gateway_service_if_loaded() { emit_json step name gateway-service status start log "Refreshing loaded gateway service..." - if ! "$claw" gateway install --force >/dev/null 2>&1; then + if ! refresh_output="$({ set +x; "$claw" gateway install --force; } 2>&1 | sed -n -e 's/.*SERVICE_DEFINITION_SEALED:.*/ask the privileged deployment owner to manually repair it/p' -e 's/.*SERVICE_DEFINITION_UNKNOWN:.*/inspect service-definition access and manually repair it/p')"; then + if [[ -n "$refresh_output" ]]; then + emit_json step name gateway-service status warn reason definition-mutation-denied + printf '%s\n' "Code installed; gateway service definition left unchanged; ${refresh_output}." >&2 + printf '%s\n' "Run openclaw gateway status --deep, verify the installation owner, and restart it manually if needed." >&2 + return 0 + fi emit_json step name gateway-service status warn reason install-failed log "Warning: gateway service refresh failed; continuing." return 0 diff --git a/scripts/install.sh b/scripts/install.sh index c1038f525e05..af176d5047ee 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -3574,7 +3574,7 @@ try { } refresh_gateway_service_if_loaded() { - local claw="${OPENCLAW_BIN:-}" + local claw="${OPENCLAW_BIN:-}" refresh_output if [[ -z "$claw" ]]; then claw="$(resolve_openclaw_bin || true)" fi @@ -3587,11 +3587,17 @@ refresh_gateway_service_if_loaded() { fi ui_info "Refreshing loaded gateway service" - if run_quiet_step "Refreshing gateway service" "$claw" gateway install --force; then - ui_success "Gateway service metadata refreshed" + if ! refresh_output="$({ set +x; "$claw" gateway install --force; } 2>&1 | sed -n -e 's/.*SERVICE_DEFINITION_SEALED:.*/ask the privileged deployment owner to manually repair it/p' -e 's/.*SERVICE_DEFINITION_UNKNOWN:.*/inspect service-definition access and manually repair it/p')"; then + if [[ -n "$refresh_output" ]]; then + ui_warn "Code installed; gateway service definition left unchanged; ${refresh_output}" + ui_info "Run openclaw gateway status --deep, verify the installation owner, and restart it manually if needed." + return 0 + else + ui_warn "Gateway service refresh failed; continuing" + return 0 + fi else - ui_warn "Gateway service refresh failed; continuing" - return 0 + ui_success "Gateway service metadata refreshed" fi # `gateway install --force` activates the replacement service. Keep the diff --git a/src/cli/daemon-cli/install.integration.test.ts b/src/cli/daemon-cli/install.integration.test.ts index caed96c362c9..e6812f80876e 100644 --- a/src/cli/daemon-cli/install.integration.test.ts +++ b/src/cli/daemon-cli/install.integration.test.ts @@ -4,15 +4,31 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { buildServiceEnvironment } from "../../daemon/service-env.js"; import type { GatewayServiceCommandConfig, GatewayServiceInstallArgs, } from "../../daemon/service-types.js"; +import { + buildSystemdManagerPropertyOutput, + buildSystemdUnitPropertyOutput, +} from "../../daemon/service.test-helpers.js"; import { makeTempWorkspace } from "../../test-helpers/workspace.js"; import { captureEnv } from "../../test-utils/env.js"; import { createCliRuntimeCapture } from "../test-runtime-capture.js"; const { runtimeLogs, defaultRuntime, resetRuntimeCapture } = createCliRuntimeCapture(); +const busctl = vi.hoisted(() => + vi.fn(), +); +vi.mock("../../daemon/systemd-exec.js", async (importOriginal) => ({ + ...(await importOriginal()), + execBusctlUser: busctl, +})); +vi.mock("../../daemon/systemd-system.js", async (importOriginal) => ({ + ...(await importOriginal()), + assertNoSystemSystemdOwnership: async () => {}, +})); const serviceMock = vi.hoisted(() => ({ label: "Gateway", @@ -24,7 +40,17 @@ const serviceMock = vi.hoisted(() => ({ stop: vi.fn(async () => {}), restart: vi.fn(async () => {}), isLoaded: vi.fn(async () => false), - readCommand: vi.fn<() => Promise>(async () => null), + readDefinitionMutationCapability: vi.fn< + (args?: { + env?: NodeJS.ProcessEnv; + environment?: NodeJS.ProcessEnv; + }) => Promise + >(async (_args?: { env?: NodeJS.ProcessEnv; environment?: NodeJS.ProcessEnv }) => ({ + kind: "writable" as const, + })), + readCommand: vi.fn< + typeof import("../../daemon/systemd-service-files.js").readSystemdServiceExecStart + >(async () => null), readRuntime: vi.fn(async () => ({ status: "stopped" as const })), })); @@ -43,17 +69,48 @@ vi.mock("../../runtime.js", () => ({ })); const { mergeInstallInvocationEnv, runDaemonInstall } = await import("./install.js"); -const { clearConfigCache, clearRuntimeConfigSnapshot } = await import("../../config/config.js"); +const { clearConfigCache, clearRuntimeConfigSnapshot, readConfigFileSnapshot } = + await import("../../config/config.js"); +const { readSystemdDefinitionMutationCapability } = + await import("../../daemon/systemd-definition-mutation.js"); +const { readSystemdServiceExecStart } = await import("../../daemon/systemd-service-files.js"); +const { assertServiceDefinitionWritable } = await import("../../daemon/service-types.js"); async function readJson(filePath: string): Promise> { return JSON.parse(await fs.readFile(filePath, "utf8")) as Record; } +async function createInstalledServiceCommand() { + // An installed service has already observed its config; include that health store in snapshots. + await readConfigFileSnapshot(); + const programArguments = ["openclaw", "gateway", "run"]; + const environment = buildServiceEnvironment({ + env: process.env, + port: 18789, + execPath: programArguments[0], + }); + return { + programArguments, + // Service readers return only persisted strings, including the host's required TLS CA bundle. + environment: Object.fromEntries( + Object.entries(environment).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ), + }; +} + describe("runDaemonInstall integration", () => { let envSnapshot: ReturnType; let tempHome: string; let configPath: string; + async function snapshotConfig() { + const contents = await fs.readFile(configPath); + const { ino, mode, uid } = await fs.lstat(configPath); + return { contents, ino, mode, uid, entries: (await fs.readdir(tempHome)).toSorted() }; + } + beforeAll(async () => { envSnapshot = captureEnv([ "HOME", @@ -82,6 +139,8 @@ describe("runDaemonInstall integration", () => { process.env.OPENCLAW_GATEWAY_TOKEN = ""; process.env.OPENCLAW_GATEWAY_PASSWORD = ""; serviceMock.isLoaded.mockResolvedValue(false); + serviceMock.readDefinitionMutationCapability.mockResolvedValue({ kind: "writable" }); + serviceMock.readCommand.mockReset(); serviceMock.readCommand.mockResolvedValue(null); await fs.writeFile(configPath, JSON.stringify({}, null, 2)); clearConfigCache(); @@ -121,6 +180,146 @@ describe("runDaemonInstall integration", () => { expect(joined).toContain("MISSING_GATEWAY_TOKEN"); }); + it.each(["fragment", "drop-in"])( + "blocks a root-owned manager %s before config or token writes", + async (kind) => { + const fixture = await fs.realpath(await fs.mkdtemp(path.join(tempHome, "manager-owner-"))); + const unitPath = path.join(fixture, ".config/systemd/user/openclaw-gateway.service"); + const extra = path.join(fixture, "global-user", "operator.conf"); + await fs.mkdir(path.dirname(unitPath), { recursive: true }); + await fs.mkdir(path.dirname(extra)); + await fs.writeFile(extra, "[Service]\nEnvironment=TOKEN=operator-secret-canary\n"); + if (kind === "drop-in") { + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/node gateway\n"); + } + const originalLstat = fs.lstat.bind(fs); + const lstat = vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + const stat = await originalLstat(...args); + if (args[0] === extra) { + Object.defineProperty(stat, "uid", { value: 0 }); + } + return stat; + }); + busctl.mockImplementation(async (_env, args) => ({ + code: 0, + termination: "exit", + stderr: "", + stdout: args.includes("LoadUnit") + ? JSON.stringify({ type: "o", data: ["/org/freedesktop/systemd1/unit/owned"] }) + : args.includes("org.freedesktop.systemd1.Unit") + ? buildSystemdUnitPropertyOutput({ + fragmentPath: kind === "fragment" ? extra : unitPath, + dropInPaths: kind === "fragment" ? [] : [extra], + }) + : buildSystemdManagerPropertyOutput({ programArguments: ["/usr/bin/node", "gateway"] }), + })); + const env = { ...process.env, HOME: fixture, OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway" }; + serviceMock.readCommand.mockImplementationOnce((_env, options) => + readSystemdServiceExecStart(env, options), + ); + serviceMock.readDefinitionMutationCapability.mockImplementationOnce(() => + readSystemdDefinitionMutationCapability(env), + ); + const before = await snapshotConfig(); + const managedEntries = await fs.readdir(path.dirname(unitPath)); + try { + await expect(runDaemonInstall({ json: true, force: true })).rejects.toThrow("__exit__:1"); + expect(await snapshotConfig()).toEqual(before); + expect(await fs.readdir(path.dirname(unitPath))).toEqual(managedEntries); + expect(await fs.readFile(extra, "utf8")).toContain("operator-secret-canary"); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(runtimeLogs.join("\n")).toContain("SERVICE_DEFINITION_SEALED"); + expect(runtimeLogs.join("\n")).not.toContain("secret-canary"); + } finally { + lstat.mockRestore(); + await fs.rm(fixture, { recursive: true, force: true }); + } + }, + ); + + it("checks the planned generated environment after a drop-in redirects effective state", async () => { + const fixture = await fs.realpath(await fs.mkdtemp(path.join(tempHome, "planned-owner-"))); + const plannedState = path.join(fixture, "planned"); + const effectiveState = path.join(fixture, "effective"); + const unit = path.join(fixture, ".config/systemd/user/openclaw-gateway.service"); + const dropIn = `${unit}.d/override.conf`; + const plannedFile = path.join(plannedState, "gateway.systemd.env"); + const effectiveFile = path.join(effectiveState, "gateway.systemd.env"); + const invocation = captureEnv(["HOME", "OPENCLAW_STATE_DIR"]); + await fs.mkdir(path.dirname(dropIn), { recursive: true }); + await fs.mkdir(plannedState); + await fs.mkdir(effectiveState); + await fs.writeFile(plannedFile, "OPERATOR_VALUE=planned\n"); + await fs.writeFile(effectiveFile, "OPERATOR_VALUE=effective\n"); + await fs.writeFile( + unit, + `[Service]\nExecStart=/usr/bin/node gateway\nEnvironment=OPENCLAW_STATE_DIR=${plannedState}\nEnvironmentFile=${plannedFile}\n`, + ); + await fs.writeFile( + dropIn, + `[Service]\nEnvironment=OPENCLAW_STATE_DIR=${effectiveState}\nEnvironmentFile=\nEnvironmentFile=${effectiveFile}\n`, + ); + await fs.writeFile( + configPath, + JSON.stringify({ gateway: { auth: { mode: "token", token: "existing-token" } } }), + ); + process.env.HOME = fixture; + process.env.OPENCLAW_STATE_DIR = plannedState; + clearConfigCache(); + const lstat = fs.lstat.bind(fs); + const owner = vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + const stat = await lstat(...args); + if (args[0] === plannedFile) { + Object.defineProperty(stat, "uid", { value: 0 }); + } + return stat; + }); + busctl.mockImplementation(async (_env, args) => ({ + code: 0, + termination: "exit", + stderr: "", + stdout: args.includes("LoadUnit") + ? JSON.stringify({ type: "o", data: ["/org/freedesktop/systemd1/unit/owned"] }) + : args.includes("org.freedesktop.systemd1.Unit") + ? buildSystemdUnitPropertyOutput({ fragmentPath: unit, dropInPaths: [dropIn] }) + : buildSystemdManagerPropertyOutput({ + programArguments: ["/usr/bin/node", "gateway"], + environment: [`OPENCLAW_STATE_DIR=${effectiveState}`], + environmentFiles: [[effectiveFile, false]], + }), + })); + serviceMock.readCommand.mockImplementation(readSystemdServiceExecStart); + serviceMock.readDefinitionMutationCapability.mockImplementation((args) => + readSystemdDefinitionMutationCapability(args?.env ?? process.env, { + environment: args?.environment, + }), + ); + // Model the actual writer's planned scope without operating a native manager. + serviceMock.install.mockImplementationOnce(async (args) => { + assertServiceDefinitionWritable( + await readSystemdDefinitionMutationCapability(process.env, { + environment: args?.environment, + }), + ); + }); + const before = await snapshotConfig(); + try { + await expect(runDaemonInstall({ json: true, force: true })).rejects.toThrow("__exit__:1"); + expect(await snapshotConfig()).toEqual(before); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(runtimeLogs.join("\n")).toContain("SERVICE_DEFINITION_SEALED"); + expect(await fs.readdir(plannedState)).toEqual(["gateway.systemd.env"]); + expect(await fs.readdir(effectiveState)).toEqual(["gateway.systemd.env"]); + } finally { + owner.mockRestore(); + invocation.restore(); + serviceMock.install.mockReset().mockResolvedValue(undefined); + clearConfigCache(); + clearRuntimeConfigSnapshot(); + await fs.rm(fixture, { recursive: true, force: true }); + } + }); + it("refuses service install when config was written by a newer OpenClaw", async () => { await fs.writeFile( configPath, @@ -148,21 +347,202 @@ describe("runDaemonInstall integration", () => { }); it.each([ - { force: undefined, label: "normal install" }, - { force: true, label: "forced reinstall" }, - ])("does not bypass system ownership during $label", async ({ force }) => { - serviceMock.install.mockRejectedValueOnce( - new Error( - "System systemd unit openclaw-gateway.service already owns this gateway unit name. --force does not override system ownership.", - ), + { + name: "gateway.mode is missing", + capability: { kind: "sealed" as const, detail: "unit definition is owned by root" }, + config: { gateway: { auth: { mode: "token", token: "existing-token" } } }, + marker: "SERVICE_DEFINITION_SEALED", + }, + { + name: "the gateway token is missing", + capability: { kind: "sealed" as const, detail: "unit definition is owned by root" }, + config: { gateway: { mode: "local", auth: { mode: "token" } } }, + marker: "SERVICE_DEFINITION_SEALED", + }, + { + name: "gateway.mode is missing and definition authority is unknown", + capability: { kind: "unknown" as const, detail: "unit definition cannot be inspected" }, + config: { gateway: { auth: { mode: "token" } } }, + marker: "SERVICE_DEFINITION_UNKNOWN", + }, + ])( + "preserves config bytes and directory entries when definition access is refused and $name", + async ({ capability, config, marker }) => { + await fs.writeFile(configPath, JSON.stringify(config, null, 2)); + clearConfigCache(); + serviceMock.readDefinitionMutationCapability.mockResolvedValueOnce(capability as never); + const before = await snapshotConfig(); + + await expect(runDaemonInstall({ json: true, force: true })).rejects.toThrow("__exit__:1"); + + expect(await snapshotConfig()).toEqual(before); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(serviceMock.readCommand).toHaveBeenCalledOnce(); + expect(runtimeLogs.join("\n")).toContain(marker); + expect(runtimeLogs.join("\n")).toContain( + capability.kind === "sealed" ? "deployment owner" : "Inspect service definition access", + ); + }, + ); + + it.each([ + { name: "forced fresh install", loaded: false, force: true }, + { name: "loaded auto-refresh", loaded: true, force: false }, + { name: "forced loaded refresh", loaded: true, force: true }, + ])( + "preserves config, token, and state when $name cannot inspect its command", + async ({ loaded, force }) => { + const secret = "service-command-inspection-secret-canary"; + await fs.writeFile(configPath, JSON.stringify({ gateway: { auth: { mode: "token" } } })); + clearConfigCache(); + serviceMock.isLoaded.mockResolvedValue(loaded); + serviceMock.readCommand.mockRejectedValueOnce(new Error(secret)); + const before = await snapshotConfig(); + + await expect(runDaemonInstall({ json: true, force })).rejects.toThrow("__exit__:1"); + + expect(await snapshotConfig()).toEqual(before); + expect(serviceMock.readCommand).toHaveBeenCalledWith(expect.any(Object), { + requireEffective: true, + }); + expect(serviceMock.readDefinitionMutationCapability).not.toHaveBeenCalled(); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(runtimeLogs.join("\n")).toContain("SERVICE_DEFINITION_UNKNOWN"); + expect(runtimeLogs.join("\n")).not.toContain(secret); + }, + ); + + it("keeps an already-installed service read-only without probing definition authority", async () => { + await fs.writeFile( + configPath, + JSON.stringify({ gateway: { mode: "local", auth: { mode: "token", token: "existing" } } }), ); + clearConfigCache(); + serviceMock.isLoaded.mockResolvedValue(true); + serviceMock.readCommand.mockResolvedValue(await createInstalledServiceCommand()); + const before = await snapshotConfig(); - await expect(runDaemonInstall({ json: true, force })).rejects.toThrow("__exit__:1"); + await runDaemonInstall({ json: true }); - expect(serviceMock.install).toHaveBeenCalledTimes(1); - const joined = runtimeLogs.join("\n"); - expect(joined).toContain("System systemd unit openclaw-gateway.service"); - expect(joined).toContain("--force does not override system ownership"); + expect(runtimeLogs.join("\n")).toContain('"result": "already-installed"'); + expect(serviceMock.readDefinitionMutationCapability).not.toHaveBeenCalled(); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(await snapshotConfig()).toEqual(before); + }); + + it("repairs missing gateway mode for a loaded sealed service without rewriting its definition", async () => { + const config = { gateway: { auth: { mode: "token", token: "existing-token" } } }; + await fs.writeFile(configPath, JSON.stringify(config)); + clearConfigCache(); + serviceMock.isLoaded.mockResolvedValue(true); + serviceMock.readDefinitionMutationCapability.mockResolvedValue({ + kind: "sealed", + detail: "unit definition is owned by root", + } as never); + serviceMock.readCommand.mockResolvedValue(await createInstalledServiceCommand()); + + await runDaemonInstall({ json: true }); + + expect((await readJson(configPath)).gateway).toEqual({ ...config.gateway, mode: "local" }); + expect(runtimeLogs.join("\n")).toContain('"result": "already-installed"'); + expect(serviceMock.readDefinitionMutationCapability).not.toHaveBeenCalled(); + expect(serviceMock.install).not.toHaveBeenCalled(); + }); + + it("refuses loaded-service auto-refresh before persisting missing gateway defaults", async () => { + await fs.writeFile( + configPath, + JSON.stringify({ gateway: { auth: { mode: "token", token: "existing-token" } } }), + ); + clearConfigCache(); + serviceMock.isLoaded.mockResolvedValue(true); + serviceMock.readCommand.mockResolvedValue({ + programArguments: ["openclaw", "gateway", "run"], + environment: { OPENCLAW_GATEWAY_TOKEN: "outdated-token" }, + } as never); + serviceMock.readDefinitionMutationCapability.mockResolvedValueOnce({ + kind: "sealed", + detail: "unit definition is owned by root", + } as never); + const before = await snapshotConfig(); + + await expect(runDaemonInstall({ json: true })).rejects.toThrow("__exit__:1"); + + expect(runtimeLogs.join("\n")).toContain("SERVICE_DEFINITION_SEALED"); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(await snapshotConfig()).toEqual(before); + }); + + it("refuses a loaded service's sealed effective state before persisting config or a token", async () => { + const effectiveStateDir = path.join(tempHome, "sealed-service-state"); + await fs.writeFile(configPath, JSON.stringify({ gateway: { auth: { mode: "token" } } })); + clearConfigCache(); + serviceMock.isLoaded.mockResolvedValue(true); + serviceMock.readCommand.mockResolvedValue({ + programArguments: ["openclaw", "gateway", "run"], + environment: { OPENCLAW_STATE_DIR: effectiveStateDir }, + } as never); + serviceMock.readDefinitionMutationCapability.mockImplementationOnce( + async (args) => + (args?.environment?.OPENCLAW_STATE_DIR === effectiveStateDir + ? { kind: "sealed", detail: "effective state is owned by root" } + : { kind: "writable" }) as never, + ); + const before = await snapshotConfig(); + + await expect(runDaemonInstall({ json: true, force: true })).rejects.toThrow("__exit__:1"); + + expect(serviceMock.readDefinitionMutationCapability).toHaveBeenCalledWith( + expect.objectContaining({ + env: expect.objectContaining({ OPENCLAW_STATE_DIR: tempHome }), + environment: expect.objectContaining({ OPENCLAW_STATE_DIR: effectiveStateDir }), + }), + ); + expect(await snapshotConfig()).toEqual(before); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(runtimeLogs.join("\n")).toContain("SERVICE_DEFINITION_SEALED"); + }); + + it.each([ + { name: "sealed definition without force", kind: "sealed", force: false }, + { name: "sealed definition with force", kind: "sealed", force: true }, + { name: "uninspectable definition", kind: "unknown", force: true }, + { name: "rejected definition inspection", kind: "rejected", force: false }, + ])("leaves absent config and state untouched for $name", async ({ kind, force }) => { + const stateDir = await fs.mkdtemp(path.join(tempHome, "sealed-install-")); + const missingConfigPath = path.join(stateDir, "openclaw.json"); + const originalStateDir = process.env.OPENCLAW_STATE_DIR; + const originalConfigPath = process.env.OPENCLAW_CONFIG_PATH; + const secret = "direct-install-capability-secret-canary"; + process.env.OPENCLAW_STATE_DIR = stateDir; + process.env.OPENCLAW_CONFIG_PATH = missingConfigPath; + clearConfigCache(); + if (kind === "rejected") { + serviceMock.readDefinitionMutationCapability.mockRejectedValueOnce(new Error(secret)); + } else { + serviceMock.readDefinitionMutationCapability.mockResolvedValueOnce({ + kind, + detail: secret, + } as never); + } + + try { + await expect(runDaemonInstall({ json: true, force })).rejects.toThrow("__exit__:1"); + + expect(await fs.readdir(stateDir)).toEqual([]); + await expect(fs.access(missingConfigPath)).rejects.toMatchObject({ code: "ENOENT" }); + expect(serviceMock.readCommand).toHaveBeenCalledOnce(); + expect(serviceMock.install).not.toHaveBeenCalled(); + expect(runtimeLogs.join("\n")).toContain( + kind === "sealed" ? "SERVICE_DEFINITION_SEALED" : "SERVICE_DEFINITION_UNKNOWN", + ); + expect(runtimeLogs.join("\n")).not.toContain(secret); + } finally { + process.env.OPENCLAW_STATE_DIR = originalStateDir; + process.env.OPENCLAW_CONFIG_PATH = originalConfigPath; + clearConfigCache(); + await fs.rm(stateDir, { recursive: true, force: true }); + } }); it("auto-mints token when no source exists without embedding it into service env", async () => { @@ -195,6 +575,23 @@ describe("runDaemonInstall integration", () => { expect(installEnv?.OPENCLAW_GATEWAY_TOKEN).toBeUndefined(); }); + it("logs a generated-token warning without callback indexes or warning arrays", async () => { + await fs.writeFile( + configPath, + JSON.stringify({ gateway: { mode: "local", auth: { mode: "token" } } }), + ); + clearConfigCache(); + serviceMock.isLoaded.mockResolvedValueOnce(false).mockResolvedValueOnce(true); + + await runDaemonInstall({}); + + expect( + defaultRuntime.log.mock.calls.filter(([message]) => + String(message).includes("No gateway token found"), + ), + ).toEqual([["No gateway token found. Auto-generated one and saving to config."]]); + }); + it.each([ { name: "operator heap cap", diff --git a/src/cli/daemon-cli/install.test.ts b/src/cli/daemon-cli/install.test.ts index d67b5b3dc81c..1606357466ae 100644 --- a/src/cli/daemon-cli/install.test.ts +++ b/src/cli/daemon-cli/install.test.ts @@ -82,6 +82,7 @@ const service = vi.hoisted(() => ({ uninstall: vi.fn(async () => {}), restart: vi.fn(async () => {}), stop: vi.fn(async () => {}), + readDefinitionMutationCapability: vi.fn(async () => ({ kind: "writable" as const })), readCommand: vi.fn(async () => null), readRuntime: vi.fn(async () => ({ status: "stopped" as const })), })); @@ -270,6 +271,7 @@ describe("runDaemonInstall", () => { service.isLoaded.mockReset(); service.stage.mockReset(); service.install.mockReset(); + service.readDefinitionMutationCapability.mockReset(); service.readCommand.mockReset(); resetRuntimeCapture(); actionState.warnings.length = 0; @@ -302,6 +304,7 @@ describe("runDaemonInstall", () => { service.isLoaded.mockResolvedValue(false); service.stage.mockResolvedValue(undefined); service.install.mockResolvedValue(undefined); + service.readDefinitionMutationCapability.mockResolvedValue({ kind: "writable" }); service.readCommand.mockResolvedValue(null); resolveNodeStartupTlsEnvironmentMock.mockReturnValue({ NODE_EXTRA_CA_CERTS: undefined, @@ -342,6 +345,15 @@ describe("runDaemonInstall", () => { expect(installDaemonServiceAndEmitMock).not.toHaveBeenCalled(); }); + it("blocks inaccessible definitions before config reads or credential generation", async () => { + service.readDefinitionMutationCapability.mockRejectedValueOnce(new Error("secret-canary")); + await runDaemonInstall({ json: true, force: true }); + expect(actionState.failed[0]?.message).toContain("SERVICE_DEFINITION_UNKNOWN"); + expect(readConfigFileSnapshotMock).not.toHaveBeenCalled(); + expect(randomTokenMock).not.toHaveBeenCalled(); + expect(service.readCommand).toHaveBeenCalledOnce(); + }); + it("blocks non-default install identities before inspecting host services", async () => { isDefaultInstallIdentityMock.mockReturnValue(false); @@ -1000,29 +1012,14 @@ describe("runDaemonInstall", () => { OPENAI_API_KEY: "service-openai-key", }, } as never); - const previous = process.env.OPENAI_API_KEY; - const previousNodeOptions = process.env.NODE_OPTIONS; delete process.env.OPENAI_API_KEY; process.env.NODE_OPTIONS = "--require /tmp/untrusted.js"; - try { - await runDaemonInstall({ json: true, force: true }); + await runDaemonInstall({ json: true, force: true }); - expectFields(readFirstInstallPlanArg().env, { - OPENAI_API_KEY: "service-openai-key", - }); - expect(installDaemonServiceAndEmitMock).toHaveBeenCalledTimes(1); - } finally { - if (previous === undefined) { - delete process.env.OPENAI_API_KEY; - } else { - process.env.OPENAI_API_KEY = previous; - } - if (previousNodeOptions === undefined) { - delete process.env.NODE_OPTIONS; - } else { - process.env.NODE_OPTIONS = previousNodeOptions; - } - } + expectFields(readFirstInstallPlanArg().env, { + OPENAI_API_KEY: "service-openai-key", + }); + expect(installDaemonServiceAndEmitMock).toHaveBeenCalledTimes(1); }); it("does not reuse stale service control env during forced reinstall", async () => { @@ -1039,27 +1036,18 @@ describe("runDaemonInstall", () => { }, } as never); - const previous = process.env.OPENAI_API_KEY; delete process.env.OPENAI_API_KEY; - try { - await runDaemonInstall({ json: true, force: true }); + await runDaemonInstall({ json: true, force: true }); - expectFields(readFirstInstallPlanArg().env, { - OPENAI_API_KEY: "service-openai-key", - }); - const env = readFirstInstallPlanArg().env as Record; - expect(env.OPENCLAW_STATE_DIR).toBeUndefined(); - expect(env.OPENCLAW_CONFIG_PATH).toBeUndefined(); - expect(env.OPENCLAW_GATEWAY_TOKEN).toBeUndefined(); - expect(env.NODE_OPTIONS).toBeUndefined(); - expect(env.PATH).not.toContain("/tmp/doctor-bin"); - expect(installDaemonServiceAndEmitMock).toHaveBeenCalledTimes(1); - } finally { - if (previous === undefined) { - delete process.env.OPENAI_API_KEY; - } else { - process.env.OPENAI_API_KEY = previous; - } - } + expectFields(readFirstInstallPlanArg().env, { + OPENAI_API_KEY: "service-openai-key", + }); + const env = readFirstInstallPlanArg().env as Record; + expect(env.OPENCLAW_STATE_DIR).toBeUndefined(); + expect(env.OPENCLAW_CONFIG_PATH).toBeUndefined(); + expect(env.OPENCLAW_GATEWAY_TOKEN).toBeUndefined(); + expect(env.NODE_OPTIONS).toBeUndefined(); + expect(env.PATH).not.toContain("/tmp/doctor-bin"); + expect(installDaemonServiceAndEmitMock).toHaveBeenCalledTimes(1); }); }); diff --git a/src/cli/daemon-cli/install.ts b/src/cli/daemon-cli/install.ts index 2f5f540a2a74..9b360f5d3a72 100644 --- a/src/cli/daemon-cli/install.ts +++ b/src/cli/daemon-cli/install.ts @@ -16,7 +16,11 @@ import type { GatewayBindMode } from "../../config/types.gateway.js"; import type { OpenClawConfig } from "../../config/types.js"; import { OPENCLAW_WRAPPER_ENV_KEY, resolveOpenClawWrapperPath } from "../../daemon/program-args.js"; import { readEmbeddedGatewayToken } from "../../daemon/service-audit.js"; -import { resolveManagedGatewayServiceCommand } from "../../daemon/service-types.js"; +import { mergeGatewayServiceEnv } from "../../daemon/service-env-merge.js"; +import { + assertServiceDefinitionWritable, + resolveManagedGatewayServiceCommand, +} from "../../daemon/service-types.js"; import { resolveGatewayService, type GatewayServiceCommandConfig } from "../../daemon/service.js"; import { isNonFatalSystemdInstallProbeError } from "../../daemon/systemd.js"; import { resolveGatewayAuth } from "../../gateway/auth.js"; @@ -32,6 +36,7 @@ import { normalizeEnvVarKey, } from "../../infra/host-env-security.js"; import { defaultRuntime } from "../../runtime.js"; +import { createLazyPromise } from "../../shared/lazy-promise.js"; import { formatCliCommand } from "../command-format.js"; import { formatInvalidConfigPort, formatInvalidPortOption } from "../error-format.js"; import { buildDaemonServiceSnapshot, installDaemonServiceAndEmit } from "./response.js"; @@ -146,6 +151,13 @@ export function mergeInstallInvocationEnv(params: { /** Install or refresh the managed Gateway service. */ export async function runDaemonInstall(opts: DaemonInstallOptions) { const { json, stdout, warnings, emit, fail } = createDaemonInstallActionContext(opts.json); + const warn = (message: string) => { + if (json) { + warnings.push(message); + } else { + defaultRuntime.log(message); + } + }; if (failIfNixDaemonInstallMode(fail)) { return; } @@ -156,6 +168,55 @@ export async function runDaemonInstall(opts: DaemonInstallOptions) { return; } + const service = resolveGatewayService(); + let loaded; + try { + loaded = await service.isLoaded({ env: process.env }); + } catch (error) { + if (!isNonFatalSystemdInstallProbeError(error)) { + fail(`Gateway service check failed: ${String(error)}`); + return; + } + loaded = false; + } + let existingServiceCommand: GatewayServiceCommandConfig | null; + try { + existingServiceCommand = await service.readCommand(process.env, { requireEffective: true }); + } catch { + fail("SERVICE_DEFINITION_UNKNOWN: Service definition cannot be safely inspected."); + return; + } + const existingManagedCommand = resolveManagedGatewayServiceCommand(existingServiceCommand); + const existingServiceEnv = existingManagedCommand?.environment; + const installEnv = mergeInstallInvocationEnv({ + env: process.env, + existingServiceEnv, + }); + const effectiveServiceEnv = mergeGatewayServiceEnv(process.env, existingServiceCommand); + const assertWritable = async () => { + try { + // Drop-ins can redirect effective state away from the files this install will publish. + for (const environment of [effectiveServiceEnv, installEnv]) { + const capability = await service + .readDefinitionMutationCapability?.({ env: process.env, environment }) + .catch(() => ({ kind: "unknown" as const, detail: "" })); + if (capability && capability.kind !== "writable") { + assertServiceDefinitionWritable({ + kind: capability.kind, + detail: "Service definition cannot be safely modified.", + }); + } + } + return true; + } catch (error) { + fail(`Gateway install blocked: ${String(error)}`); + return false; + } + }; + if ((opts.force || !loaded) && !(await assertWritable())) { + return; + } + let { snapshot: configSnapshot, writeOptions: configWriteOptions } = await readConfigFileSnapshotForWrite(); const futureBlock = resolveFutureConfigActionBlock({ @@ -195,56 +256,6 @@ export async function runDaemonInstall(opts: DaemonInstallOptions) { return; } } - if (configSnapshot.valid && cfg.gateway?.mode === undefined) { - const baseConfig = configSnapshot.sourceConfig ?? configSnapshot.config; - await replaceConfigFile({ - nextConfig: { - ...baseConfig, - gateway: { - ...baseConfig.gateway, - mode: "local", - }, - }, - snapshot: configSnapshot, - writeOptions: { - baseSnapshot: configSnapshot, - ...configWriteOptions, - skipRuntimeSnapshotRefresh: true, - }, - afterWrite: { mode: "auto" }, - }); - const refreshed = await readConfigFileSnapshotForWrite(); - configSnapshot = refreshed.snapshot; - configWriteOptions = refreshed.writeOptions; - cfg = configSnapshot.valid ? configSnapshot.sourceConfig : configSnapshot.config; - const message = "No gateway.mode found. Set gateway.mode=local for managed gateway install."; - if (json) { - warnings.push(message); - } else { - defaultRuntime.log(message); - } - } - - const service = resolveGatewayService(); - let loaded; - try { - loaded = await service.isLoaded({ env: process.env }); - } catch (err) { - if (isNonFatalSystemdInstallProbeError(err)) { - loaded = false; - } else { - fail(`Gateway service check failed: ${String(err)}`); - return; - } - } - const existingServiceCommand = await service.readCommand(process.env).catch(() => null); - const existingManagedCommand = resolveManagedGatewayServiceCommand(existingServiceCommand); - const existingServiceEnv: Record | undefined = - existingManagedCommand?.environment; - const installEnv = mergeInstallInvocationEnv({ - env: process.env, - existingServiceEnv, - }); if (!wrapperPath) { try { wrapperPath = await resolveOpenClawWrapperPath(installEnv[OPENCLAW_WRAPPER_ENV_KEY]); @@ -265,43 +276,60 @@ export async function runDaemonInstall(opts: DaemonInstallOptions) { fail(`Gateway install blocked: ${noAuthNonLoopbackBlock}`); return; } - if (loaded) { - if (!opts.force) { - const autoRefreshMessage = await getGatewayServiceAutoRefreshMessage({ - currentCommand: existingServiceCommand, - env: process.env, - installEnv, - port, - runtime: runtimeRaw, - wrapperPath, - existingEnvironment: existingServiceEnv, - existingEnvironmentValueSources: existingManagedCommand?.environmentValueSources, - config: cfg, - }); - if (autoRefreshMessage) { - if (json) { - warnings.push(autoRefreshMessage); - } else { - defaultRuntime.log(autoRefreshMessage); - } - } else { - emit({ - ok: true, - result: "already-installed", - message: `Gateway service already ${service.loadedText}.`, - service: buildDaemonServiceSnapshot(service, loaded), - }); - if (!json) { - defaultRuntime.log(`Gateway service already ${service.loadedText}.`); - defaultRuntime.log( - `Reinstall with: ${formatCliCommand("openclaw gateway install --force")}`, - ); - } + let autoRefreshMessage: string | undefined; + if (loaded && !opts.force) { + autoRefreshMessage = await getGatewayServiceAutoRefreshMessage({ + currentCommand: existingServiceCommand, + env: process.env, + installEnv, + port, + runtime: runtimeRaw, + wrapperPath, + existingEnvironment: existingServiceEnv, + existingEnvironmentValueSources: existingManagedCommand?.environmentValueSources, + config: cfg, + }); + if (autoRefreshMessage) { + if (!(await assertWritable())) { return; } + warn(autoRefreshMessage); } } + if (configSnapshot.valid && cfg.gateway?.mode === undefined) { + const baseConfig = configSnapshot.sourceConfig ?? configSnapshot.config; + await replaceConfigFile({ + nextConfig: { ...baseConfig, gateway: { ...baseConfig.gateway, mode: "local" } }, + snapshot: configSnapshot, + writeOptions: { + baseSnapshot: configSnapshot, + ...configWriteOptions, + skipRuntimeSnapshotRefresh: true, + }, + afterWrite: { mode: "auto" }, + }); + const refreshed = await readConfigFileSnapshotForWrite(); + configSnapshot = refreshed.snapshot; + configWriteOptions = refreshed.writeOptions; + cfg = configSnapshot.valid ? configSnapshot.sourceConfig : configSnapshot.config; + warn("No gateway.mode found. Set gateway.mode=local for managed gateway install."); + } + + if (loaded && !opts.force && !autoRefreshMessage) { + emit({ + ok: true, + result: "already-installed", + message: `Gateway service already ${service.loadedText}.`, + service: buildDaemonServiceSnapshot(service, loaded), + }); + if (!json) { + defaultRuntime.log(`Gateway service already ${service.loadedText}.`); + defaultRuntime.log(`Reinstall with: ${formatCliCommand("openclaw gateway install --force")}`); + } + return; + } + const tokenResolution = await resolveGatewayInstallToken({ config: cfg, configSnapshot, @@ -317,11 +345,7 @@ export async function runDaemonInstall(opts: DaemonInstallOptions) { return; } for (const warning of tokenResolution.warnings) { - if (json) { - warnings.push(warning); - } else { - defaultRuntime.log(warning); - } + warn(warning); } const { programArguments, workingDirectory, environment, environmentValueSources } = @@ -333,23 +357,9 @@ export async function runDaemonInstall(opts: DaemonInstallOptions) { existingCommand: existingServiceCommand, existingEnvironment: existingServiceEnv, existingEnvironmentValueSources: existingManagedCommand?.environmentValueSources, - warn: (message) => { - if (json) { - warnings.push(message); - } else { - defaultRuntime.log(message); - } - }, + warn, config: cfg, }); - const warn = (message: string) => { - if (json) { - warnings.push(message); - } else { - defaultRuntime.log(message); - } - }; - await installDaemonServiceAndEmit({ serviceNoun: "Gateway", service, @@ -386,9 +396,8 @@ async function getGatewayServiceAutoRefreshMessage(params: { if (!currentCommand) { return undefined; } - const currentEmbeddedToken = readEmbeddedGatewayToken(currentCommand); - if (currentEmbeddedToken) { - const plannedInstall = await buildGatewayInstallPlan({ + const getPlannedInstall = createLazyPromise(() => + buildGatewayInstallPlan({ env: params.installEnv, port: params.port, runtime: params.runtime, @@ -398,7 +407,11 @@ async function getGatewayServiceAutoRefreshMessage(params: { existingEnvironmentValueSources: params.existingEnvironmentValueSources, warn: () => undefined, config: params.config, - }); + }), + ); + const currentEmbeddedToken = readEmbeddedGatewayToken(currentCommand); + if (currentEmbeddedToken) { + const plannedInstall = await getPlannedInstall(); const plannedEmbeddedToken = normalizeOptionalString( plannedInstall.environment.OPENCLAW_GATEWAY_TOKEN, ); @@ -410,17 +423,7 @@ async function getGatewayServiceAutoRefreshMessage(params: { params.wrapperPath || normalizeOptionalString(params.installEnv[OPENCLAW_WRAPPER_ENV_KEY]), ); if (wrapperRequested) { - const plannedInstall = await buildGatewayInstallPlan({ - env: params.installEnv, - port: params.port, - runtime: params.runtime, - wrapperPath: params.wrapperPath, - existingCommand: params.currentCommand, - existingEnvironment: params.existingEnvironment, - existingEnvironmentValueSources: params.existingEnvironmentValueSources, - warn: () => undefined, - config: params.config, - }); + const plannedInstall = await getPlannedInstall(); if ( plannedInstall.programArguments.join("\u0000") !== currentCommand.programArguments.join("\u0000") diff --git a/src/cli/daemon-cli/lifecycle-core.ts b/src/cli/daemon-cli/lifecycle-core.ts index c2e4c0f87a1c..9c60acda9573 100644 --- a/src/cli/daemon-cli/lifecycle-core.ts +++ b/src/cli/daemon-cli/lifecycle-core.ts @@ -46,6 +46,7 @@ type DaemonLifecycleOptions = { force?: boolean; wait?: string; restartIntent?: GatewayRestartIntent; + preserveDefinition?: boolean; disable?: boolean; }; @@ -659,6 +660,7 @@ export async function runServiceRestart(params: { await prepareGatewayRestartIntent(); try { restartResult = await params.service.restart({ + preserveDefinition: params.opts?.preserveDefinition, env: process.env, stdout, warn, diff --git a/src/cli/daemon-cli/lifecycle.external-supervision.test.ts b/src/cli/daemon-cli/lifecycle.external-supervision.test.ts index 1efae1c3c5f9..ab66e7c69fa6 100644 --- a/src/cli/daemon-cli/lifecycle.external-supervision.test.ts +++ b/src/cli/daemon-cli/lifecycle.external-supervision.test.ts @@ -118,13 +118,7 @@ async function expectRestartError(promise: Promise): Promise { describe("external gateway supervision lifecycle", () => { let runDaemonStart: (opts?: { json?: boolean }) => Promise; - let runDaemonRestart: (opts?: { - json?: boolean; - force?: boolean; - safe?: boolean; - skipDeferral?: boolean; - wait?: string; - }) => Promise; + let runDaemonRestart: typeof import("./lifecycle.js").runDaemonRestart; let runDaemonStop: (opts?: { json?: boolean }) => Promise; let runDaemonUninstall: (opts?: { json?: boolean }) => Promise; let envSnapshot: ReturnType; @@ -414,12 +408,18 @@ describe("external gateway supervision lifecycle", () => { ["start", () => runDaemonStart({ json: true })], ["stop", () => runDaemonStop({ json: true })], ["uninstall", () => runDaemonUninstall({ json: true })], + ["preserved restart", () => runDaemonRestart({ json: true, preserveDefinition: true })], ])("blocks native %s lifecycle access", async (_action, run) => { await expect(run()).rejects.toThrow("gateway lifecycle is managed by an external supervisor"); expect(runServiceStart).not.toHaveBeenCalled(); + expect(runServiceRestart).not.toHaveBeenCalled(); expect(runServiceStop).not.toHaveBeenCalled(); expect(runServiceUninstall).not.toHaveBeenCalled(); expect(service.readCommand).not.toHaveBeenCalled(); + expect(readActiveGatewayLockIdentity).not.toHaveBeenCalled(); + expect(callGatewayCli).not.toHaveBeenCalled(); + expect(writeGatewayRestartIntentSync).not.toHaveBeenCalled(); + expect(signalVerifiedGatewayPidSync).not.toHaveBeenCalled(); }); }); diff --git a/src/cli/daemon-cli/lifecycle.ts b/src/cli/daemon-cli/lifecycle.ts index f730b4f245ca..86846b2bb962 100644 --- a/src/cli/daemon-cli/lifecycle.ts +++ b/src/cli/daemon-cli/lifecycle.ts @@ -3,10 +3,8 @@ import { expectDefined } from "@openclaw/normalization-core"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { theme } from "../../../packages/terminal-core/src/theme.js"; import { isRestartEnabled } from "../../config/commands.flags.js"; -import { readBestEffortConfig, resolveGatewayPort } from "../../config/config.js"; -import { createConfigIO } from "../../config/io.js"; +import { readBestEffortConfig } from "../../config/config.js"; import { resolveGatewayServiceProbeHosts } from "../../daemon/gateway-service-probe-hosts.js"; -import { mergeGatewayServiceEnv } from "../../daemon/service-env-merge.js"; import { resolveGatewayService } from "../../daemon/service.js"; import { findInstalledSystemdGatewayScope, @@ -71,7 +69,11 @@ import { waitForGatewayHealthyListener, waitForGatewayHealthyRestart, } from "./restart-health.js"; -import { parsePortFromArgs, renderGatewayServiceStartHints } from "./shared.js"; +import { + resolveGatewayLifecycleContext, + resolveGatewayConfigPorts, + renderGatewayServiceStartHints, +} from "./shared.js"; import { repairLoadedGatewayServiceForStart } from "./start-repair.js"; import type { DaemonLifecycleOptions } from "./types.js"; @@ -109,30 +111,6 @@ function formatRestartFailure(params: { }; } -async function resolveGatewayLifecycleContext(service = resolveGatewayService()) { - const command = await service.readCommand(process.env).catch(() => null); - const env = mergeGatewayServiceEnv(process.env, command); - const config = await createConfigIO({ - env, - observe: false, - pluginValidation: "skip", - suppressFutureVersionWarning: true, - }) - .readBestEffortConfig() - .catch(() => undefined); - const port = parsePortFromArgs(command?.programArguments) ?? resolveGatewayPort(config, env); - return { port, env, command }; -} - -async function resolveGatewayPortFallback(): Promise { - const config = await readBestEffortConfig({ observe: false }).catch(() => undefined); - return resolveGatewayPort(config, process.env); -} - -async function resolveExplicitGatewayConfigPort(): Promise { - return (await readBestEffortConfig({ observe: false }).catch(() => undefined))?.gateway?.port; -} - async function assertUnmanagedGatewayRestartEnabled(port: number): Promise { const cfg = await readBestEffortConfig({ observe: false }).catch(() => undefined); const scheme = cfg?.gateway?.tls?.enabled ? "wss" : "ws"; @@ -480,7 +458,7 @@ export async function runDaemonUninstall(opts: DaemonLifecycleOptions = {}) { export async function runDaemonStart(opts: DaemonLifecycleOptions = {}) { assertGatewayServiceMutationAllowed("start the gateway"); const service = resolveGatewayService(); - const expectedPort = await resolveExplicitGatewayConfigPort(); + const expectedPort = (await resolveGatewayConfigPorts()).explicit; return await runServiceStart({ serviceNoun: "Gateway", service, @@ -546,7 +524,7 @@ export async function runDaemonStop(opts: DaemonLifecycleOptions = {}) { // override makes the running gateway look like it is already stopped. const lock = await readActiveGatewayLockIdentity().catch(() => undefined); const ctx = lock ? null : await resolveGatewayLifecycleContext(service).catch(() => null); - const port = lock?.port ?? ctx?.port ?? (await resolveGatewayPortFallback()); + const port = lock?.port ?? ctx?.port ?? (await resolveGatewayConfigPorts()).fallback; return await stopGatewayWithoutServiceManager(port, lock?.pid, ctx ?? undefined); }, }); @@ -554,6 +532,13 @@ export async function runDaemonStop(opts: DaemonLifecycleOptions = {}) { /** Restart the Gateway service or a verified unmanaged listener, then prove health. */ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promise { + const preserveDefinition = Boolean(opts.preserveDefinition); + if (preserveDefinition) { + assertGatewayServiceMutationAllowed("restart the gateway"); + if (opts.safe) { + throw new Error("--preserve-definition requires a native restart without --safe"); + } + } if (opts.skipDeferral && !opts.safe) { throw new Error("--skip-deferral requires --safe"); } @@ -568,12 +553,19 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi let restartedWithoutServiceManager = false; let unmanagedPreviousLockIdentity: GatewayLockIdentity | undefined; const restartIntent = resolveGatewayRestartIntentOptions(opts); - const configuredPort = await resolveExplicitGatewayConfigPort(); - let managedRestartContext = await resolveGatewayLifecycleContext(service).catch(async () => ({ - port: await resolveGatewayPortFallback(), - env: process.env, - })); - let managedRestartPort = configuredPort ?? managedRestartContext.port; + const { explicit: configuredPort, fallback: fallbackPort } = await resolveGatewayConfigPorts(); + let managedRestartContext = await resolveGatewayLifecycleContext( + service, + preserveDefinition, + ).catch(async (error: unknown) => { + if (preserveDefinition) { + throw error; + } + return { port: fallbackPort, env: process.env }; + }); + let managedRestartPort = preserveDefinition + ? managedRestartContext.port + : (configuredPort ?? managedRestartContext.port); // An unmanaged run loop keeps its lock port across in-process restarts, even // when config changes underneath it. Use that port for both the signal and // health proof or a valid CLI/env override looks like a failed restart. @@ -597,23 +589,28 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi checkTokenDrift: true, expectedPort: configuredPort, beforeServiceMutation: () => assertGatewayServiceMutationAllowed("restart the gateway"), - repairLoadedService: async ({ json, stdout, warn, state, issues }) => { - const result = await repairLoadedGatewayServiceForStart({ - action: "restart", - service, - json, - stdout, - warn, - state, - issues, - }); - // Repair rewrites the service definition, so the old command environment - // no longer identifies where the restarted gateway publishes readiness. - managedRestartContext = await resolveGatewayLifecycleContext(service); - managedRestartPort = configuredPort ?? managedRestartContext.port; - return result; - }, + repairLoadedService: preserveDefinition + ? undefined + : async ({ json, stdout, warn, state, issues }) => { + const result = await repairLoadedGatewayServiceForStart({ + action: "restart", + service, + json, + stdout, + warn, + state, + issues, + }); + // Repair rewrites the service definition, so the old command environment + // no longer identifies where the restarted gateway publishes readiness. + managedRestartContext = await resolveGatewayLifecycleContext(service); + managedRestartPort = configuredPort ?? managedRestartContext.port; + return result; + }, onNotLoaded: async () => { + if (preserveDefinition) { + return null; + } const mutationError = resolveGatewayServiceMutationError("restart the gateway"); if (process.platform === "darwin" && !mutationError) { const recovered = await recoverInstalledLaunchAgent({ result: "restarted" }); @@ -681,15 +678,17 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi throw new Error("unreachable after gateway restart health failure"); } - let health = await waitForGatewayHealthyRestart({ - service, - port: managedRestartPort, - attempts: restartHealthAttempts, - delayMs: POST_RESTART_HEALTH_DELAY_MS, - env: managedRestartContext.env, - includeUnknownListenersAsStale: process.platform === "win32", - supervisorKeepsAlive: process.platform === "darwin", - }); + const waitForHealthy = async () => + await waitForGatewayHealthyRestart({ + service, + port: managedRestartPort, + attempts: restartHealthAttempts, + delayMs: POST_RESTART_HEALTH_DELAY_MS, + env: managedRestartContext.env, + includeUnknownListenersAsStale: process.platform === "win32", + supervisorKeepsAlive: process.platform === "darwin", + }); + let health = await waitForHealthy(); if (!health.healthy && health.staleGatewayPids.length > 0) { // On Windows service restarts can leave stale listeners behind; kill verified stale @@ -703,6 +702,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi await terminateStaleGatewayPids(health.staleGatewayPids); const retryRestart = await service.restart({ + preserveDefinition, env: process.env, stdout, warn, @@ -711,15 +711,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi if (retryRestart.outcome === "scheduled") { return retryRestart; } - health = await waitForGatewayHealthyRestart({ - service, - port: managedRestartPort, - attempts: restartHealthAttempts, - delayMs: POST_RESTART_HEALTH_DELAY_MS, - env: managedRestartContext.env, - includeUnknownListenersAsStale: process.platform === "win32", - supervisorKeepsAlive: process.platform === "darwin", - }); + health = await waitForHealthy(); } if (health.healthy) { diff --git a/src/cli/daemon-cli/register-service-commands.test.ts b/src/cli/daemon-cli/register-service-commands.test.ts index bd78b7e3f637..058eaa45343e 100644 --- a/src/cli/daemon-cli/register-service-commands.test.ts +++ b/src/cli/daemon-cli/register-service-commands.test.ts @@ -26,8 +26,8 @@ vi.mock("./lifecycle.runtime.js", () => ({ runDaemonUninstall: (opts: unknown) => runDaemonUninstall(opts), })); -function createGatewayParentLikeCommand() { - const gateway = new Command().name("gateway"); +function createGatewayParentLikeCommand(program?: Command) { + const gateway = program ? program.command("gateway") : new Command().name("gateway"); // Mirror overlapping root gateway options that conflict with service subcommand options. gateway.option("--port ", "Port for the gateway WebSocket"); gateway.option("--token ", "Gateway token"); @@ -124,6 +124,34 @@ describe("addGatewayServiceCommands", () => { assert(); }); + it.each(["gateway", "daemon"])("parses preservation only on %s restart", async (name) => { + const program = new Command() + .enablePositionalOptions() + .exitOverride() + .configureOutput({ writeErr: () => {} }); + if (name === "daemon") { + registerDaemonCli(program); + } else { + createGatewayParentLikeCommand(program); + } + await program.parseAsync([name, "restart", "--preserve-definition", "--json"], { + from: "user", + }); + expect(expectSingleDaemonCall(runDaemonRestart)).toMatchObject({ + preserveDefinition: true, + json: true, + }); + for (const verb of ["install", "start", "stop", "uninstall"]) { + await expect( + program.parseAsync([name, verb, "--preserve-definition"], { from: "user" }), + ).rejects.toMatchObject({ code: "commander.unknownOption" }); + } + expect(runDaemonInstall).not.toHaveBeenCalled(); + expect(runDaemonStart).not.toHaveBeenCalled(); + expect(runDaemonStop).not.toHaveBeenCalled(); + expect(runDaemonUninstall).not.toHaveBeenCalled(); + }); + it.each( [ { leaf: "status", runner: runDaemonStatus }, diff --git a/src/cli/daemon-cli/register-service-commands.ts b/src/cli/daemon-cli/register-service-commands.ts index 2ff487c34028..bdd6f9b7ae1e 100644 --- a/src/cli/daemon-cli/register-service-commands.ts +++ b/src/cli/daemon-cli/register-service-commands.ts @@ -126,6 +126,7 @@ export function addGatewayServiceCommands(parent: Command, opts?: { statusDescri parent .command("restart") .description("Restart the Gateway service (launchd/systemd/schtasks)") + .option("--preserve-definition", "Keep the native service definition", false) .option("--force", "Restart immediately without waiting for active gateway work", false) .option( "--safe", diff --git a/src/cli/daemon-cli/restart-health-probe.test.ts b/src/cli/daemon-cli/restart-health-probe.test.ts index c6089f3b9a66..67a639791f03 100644 --- a/src/cli/daemon-cli/restart-health-probe.test.ts +++ b/src/cli/daemon-cli/restart-health-probe.test.ts @@ -21,6 +21,10 @@ import { sleep, } from "./restart-health.test-helpers.js"; +// Load the real client's dependency graph before timing its socket/probe behavior. +const actualProbe = + await vi.importActual("../../gateway/probe.js"); + describe("restart health", () => { beforeEach(resetRestartHealthMocks); afterEach(restoreRestartHealthMocks); @@ -61,11 +65,7 @@ describe("restart health", () => { } }); }); - probeGateway.mockImplementation(async (...args: unknown[]) => { - const actual = - await vi.importActual("../../gateway/probe.js"); - return actual.probeGateway(...(args as Parameters)); - }); + probeGateway.mockImplementation(actualProbe.probeGateway); inspectPortUsage.mockResolvedValue({ port, status: "busy", diff --git a/src/cli/daemon-cli/shared.ts b/src/cli/daemon-cli/shared.ts index 739c61032358..9f7a93e267a6 100644 --- a/src/cli/daemon-cli/shared.ts +++ b/src/cli/daemon-cli/shared.ts @@ -1,5 +1,7 @@ // Shared Gateway service CLI helpers: status styles, env filtering, port parsing, and hints. import { colorize, isRich, theme } from "../../../packages/terminal-core/src/theme.js"; +import { readBestEffortConfig, resolveGatewayPort } from "../../config/config.js"; +import { createConfigIO } from "../../config/io.js"; import { resolveIsNixMode } from "../../config/paths.js"; import { resolveGatewayLaunchAgentLabel, @@ -12,6 +14,8 @@ import { buildPlatformRuntimeLogHints, buildPlatformServiceStartHints, } from "../../daemon/runtime-hints.js"; +import { mergeGatewayServiceEnv } from "../../daemon/service-env-merge.js"; +import { resolveGatewayService } from "../../daemon/service.js"; import { parseTcpPortFromArgs } from "../../infra/tcp-port.js"; import { formatCliCommand } from "../command-format.js"; import { parsePort } from "../shared/parse-port.js"; @@ -229,3 +233,33 @@ export function filterContainerGenericHints( !hint.includes("systemd user services are unavailable; install/enable systemd"), ); } + +export async function resolveGatewayLifecycleContext( + service = resolveGatewayService(), + requireEffective = false, +) { + const command = requireEffective + ? await service.readCommand(process.env, { requireEffective: true }) + : await service.readCommand(process.env).catch(() => null); + if (requireEffective && !command) { + throw new Error( + "Updated gateway service could not be inspected; run `openclaw gateway status --deep`.", + ); + } + const env = mergeGatewayServiceEnv(process.env, command); + const config = await createConfigIO({ + env, + observe: false, + pluginValidation: "skip", + suppressFutureVersionWarning: true, + }) + .readBestEffortConfig() + .catch(() => undefined); + const port = parsePortFromArgs(command?.programArguments) ?? resolveGatewayPort(config, env); + return { port, env, command }; +} + +export async function resolveGatewayConfigPorts() { + const config = await readBestEffortConfig({ observe: false }).catch(() => undefined); + return { explicit: config?.gateway?.port, fallback: resolveGatewayPort(config, process.env) }; +} diff --git a/src/cli/daemon-cli/start-repair.test.ts b/src/cli/daemon-cli/start-repair.test.ts index 006053a7875a..bd1a0f5db092 100644 --- a/src/cli/daemon-cli/start-repair.test.ts +++ b/src/cli/daemon-cli/start-repair.test.ts @@ -1,6 +1,6 @@ // Start repair tests cover stale service repair install-plan wiring. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { GatewayService, GatewayServiceState } from "../../daemon/service.js"; +import type { GatewayServiceState } from "../../daemon/service.js"; const buildGatewayInstallPlanMock = vi.hoisted(() => vi.fn( @@ -146,13 +146,47 @@ describe("repairLoadedGatewayServiceForStart", () => { vi.unstubAllEnvs(); }); + it.each(["sealed", "unknown"] as const)( + "denies definition repair before config or token work when authority is %s", + async (kind) => { + const install = vi.fn(); + const service = { + install, + isLoaded: vi.fn(async () => true), + readDefinitionMutationCapability: vi.fn(async () => ({ kind, detail: "protected" })), + }; + const state: GatewayServiceState = { + installed: true, + loadState: { status: "loaded" }, + running: false, + env: { HOME: "/home/openclaw" }, + command: { + programArguments: ["/usr/bin/openclaw", "gateway"], + environment: { HOME: "/home/openclaw" }, + }, + }; + await expect( + repairLoadedGatewayServiceForStart({ + service, + state, + issues: [{ code: "missing-program", message: "missing" }], + json: true, + stdout: process.stdout, + }), + ).rejects.toThrow(`SERVICE_DEFINITION_${kind.toUpperCase()}`); + expect(readConfigFileSnapshotForWriteMock).not.toHaveBeenCalled(); + expect(resolveGatewayInstallTokenMock).not.toHaveBeenCalled(); + expect(install).not.toHaveBeenCalled(); + }, + ); + it("preserves the managed base environment when an environment-only drop-in overrides it", async () => { const installMock = vi.fn(async () => {}); const isLoadedMock = vi.fn(async () => true); const service = { install: installMock, isLoaded: isLoadedMock, - } as unknown as GatewayService; + }; const existingEnvironment = { HOME: "/home/openclaw", OPENCLAW_SERVICE_VERSION: "2026.4.24", @@ -229,7 +263,7 @@ describe("repairLoadedGatewayServiceForStart", () => { const service = { install: vi.fn(async () => {}), isLoaded: vi.fn(async () => true), - } as unknown as GatewayService; + }; const state: GatewayServiceState = { installed: true, loadState: { status: "loaded" }, @@ -275,7 +309,7 @@ describe("repairLoadedGatewayServiceForStart", () => { "refuses an ineffective stopped-service repair for a %s drop-in", async (_, overrides, effectiveEnvironment) => { const installMock = vi.fn(async () => {}); - const service = { install: installMock } as unknown as GatewayService; + const service = { install: installMock, isLoaded: vi.fn(async () => true) }; const managedDefinition = { programArguments: ["/usr/bin/openclaw", "gateway", "run"], workingDirectory: "/srv/openclaw", @@ -339,7 +373,7 @@ describe("repairLoadedGatewayServiceForStart", () => { const service = { install: installMock, isLoaded: vi.fn(async () => true), - } as unknown as GatewayService; + }; const state: GatewayServiceState = { installed: true, loadState: { status: "loaded" }, @@ -407,7 +441,7 @@ describe("repairLoadedGatewayServiceForStart", () => { const service = { install: installMock, isLoaded: vi.fn(async () => true), - } as unknown as GatewayService; + }; const state: GatewayServiceState = { installed: true, loadState: { status: "loaded" }, @@ -438,7 +472,7 @@ describe("repairLoadedGatewayServiceForStart", () => { const service = { install: installMock, isLoaded: vi.fn(async () => true), - } as unknown as GatewayService; + }; const state: GatewayServiceState = { installed: true, loadState: { status: "loaded" }, @@ -473,7 +507,7 @@ describe("repairLoadedGatewayServiceForStart", () => { const service = { install: installMock, isLoaded: vi.fn(async () => true), - } as unknown as GatewayService; + }; const state: GatewayServiceState = { installed: true, loadState: { status: "loaded" }, diff --git a/src/cli/daemon-cli/start-repair.ts b/src/cli/daemon-cli/start-repair.ts index 65fbf860a4a7..4a31f65e8084 100644 --- a/src/cli/daemon-cli/start-repair.ts +++ b/src/cli/daemon-cli/start-repair.ts @@ -12,6 +12,7 @@ import { import { OPENCLAW_WRAPPER_ENV_KEY, resolveOpenClawWrapperPath } from "../../daemon/program-args.js"; import { resolveBunRuntimeInfo } from "../../daemon/runtime-paths.js"; import { + assertServiceDefinitionWritable, hasGatewayServiceEnvironmentDifference, hasGatewayServiceLauncherOverride, resolveManagedGatewayServiceCommand, @@ -29,7 +30,7 @@ import { defaultRuntime } from "../../runtime.js"; import { mergeInstallInvocationEnv } from "./install.js"; type GatewayServiceRepairParams = { - service: GatewayService; + service: Pick; state: GatewayServiceState; issues: GatewayServiceStartRepairIssue[]; json: boolean; @@ -149,6 +150,16 @@ export async function repairLoadedGatewayServiceForStart( loaded: boolean; }> { assertGatewayServiceMutationAllowed("repair the gateway service"); + // Repair can persist a generated token; check definition authority before planning it. + const capability = await params.service + .readDefinitionMutationCapability?.({ env: process.env, environment: params.state.env }) + .catch(() => ({ kind: "unknown" as const, detail: "" })); + if (capability && capability.kind !== "writable") { + assertServiceDefinitionWritable({ + kind: capability.kind, + detail: "Service definition cannot be safely modified.", + }); + } if ( hasGatewayServiceLauncherOverride(params.state.command) || hasGatewayServiceEnvironmentDifference(params.state.command, GATEWAY_TARGET_ENV_KEYS) diff --git a/src/cli/daemon-cli/status.print.test.ts b/src/cli/daemon-cli/status.print.test.ts index 2d550527594d..c1faf722b508 100644 --- a/src/cli/daemon-cli/status.print.test.ts +++ b/src/cli/daemon-cli/status.print.test.ts @@ -149,6 +149,7 @@ describe("printDaemonStatus", () => { environment: { OPENCLAW_GATEWAY_TOKEN: "managed-base-gateway-token" }, }, managedOverrides: { launcher: "command", environment: { keys: ["OPENCLAW_GATEWAY_TOKEN"] } }, + definitionPaths: ["/etc/systemd/user/private-definition.conf"], reloadPending: true, }; for (const server of servers) { @@ -176,6 +177,7 @@ describe("printDaemonStatus", () => { for (const [payload] of runtime.writeJson.mock.calls) { expect(payload).not.toHaveProperty("service.command.managedDefinition"); expect(payload).not.toHaveProperty("service.command.managedOverrides"); + expect(payload).not.toHaveProperty("service.command.definitionPaths"); expect(payload).toHaveProperty("service.command.reloadPending", true); expect(JSON.stringify(payload)).not.toContain("gateway-token"); } diff --git a/src/cli/daemon-cli/status.print.ts b/src/cli/daemon-cli/status.print.ts index 7cbb614d022a..4388afd21f0f 100644 --- a/src/cli/daemon-cli/status.print.ts +++ b/src/cli/daemon-cli/status.print.ts @@ -52,6 +52,7 @@ function sanitizeDaemonStatusForJson(status: DaemonStatus): DaemonStatus { }; delete nextCommand.managedDefinition; delete nextCommand.managedOverrides; + delete nextCommand.definitionPaths; return { ...status, service: { diff --git a/src/cli/daemon-cli/types.ts b/src/cli/daemon-cli/types.ts index d38f82ab2abd..265d5576751a 100644 --- a/src/cli/daemon-cli/types.ts +++ b/src/cli/daemon-cli/types.ts @@ -31,6 +31,7 @@ export type DaemonLifecycleOptions = { force?: boolean; safe?: boolean; skipDeferral?: boolean; + preserveDefinition?: boolean; wait?: string; disable?: boolean; }; diff --git a/src/cli/update-cli.test.ts b/src/cli/update-cli.test.ts index 82d12ad52204..d4433324b950 100644 --- a/src/cli/update-cli.test.ts +++ b/src/cli/update-cli.test.ts @@ -8,6 +8,7 @@ import { expectDefined } from "@openclaw/normalization-core"; import { Command } from "commander"; import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { writePackageDistInventory } from "../../scripts/lib/package-dist-inventory.ts"; +import { createDeferred } from "../../test/helpers/promise.js"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { OpenClawConfig, ConfigFileSnapshot } from "../config/types.openclaw.js"; import type { PluginInstallRecord } from "../config/types.plugins.js"; @@ -41,6 +42,8 @@ const readPackageVersion = vi.fn(); const resolveGlobalManager = vi.fn(); const serviceLoaded = vi.fn(); const serviceEnabled = vi.fn(); +const serviceDefinitionMutationCapability = vi.fn(); +const serviceStart = vi.fn(); const serviceStop = vi.fn(); const serviceRestart = vi.fn(); // A fixed Gateway PID can collide with the updater and trigger its self-stop safeguard. @@ -59,6 +62,7 @@ const mockedRunDaemonInstall = vi.fn(); const serviceReadCommand = vi.fn(); const serviceReadRuntime = vi.fn(); const mockGetSelfAndAncestorPidsSync = vi.fn(() => new Set([process.pid])); +const terminateStaleGatewayPids = vi.fn(); const inspectPortUsage = vi.fn(); const classifyPortListener = vi.fn(); const formatPortDiagnostics = vi.fn(); @@ -243,6 +247,7 @@ vi.mock("../infra/runtime-guard.js", () => ({ vi.mock("../infra/restart-stale-pids.js", () => ({ getSelfAndAncestorPidsSync: () => mockGetSelfAndAncestorPidsSync(), + terminateStaleGatewayPids: (...args: unknown[]) => terminateStaleGatewayPids(...args), })); vi.mock("../infra/update-managed-service-handoff-cleanup.js", () => ({ @@ -366,12 +371,16 @@ vi.mock("../daemon/service.js", () => ({ readGatewayServiceState: async ( _service: unknown, args?: { + requireEffective?: boolean; validateEnvBeforeStatusRead?: (env: NodeJS.ProcessEnv) => void; }, ) => { - const command = await serviceReadCommand(); + const command = await serviceReadCommand( + args?.requireEffective ? { requireEffective: true } : undefined, + ); const env = { ...process.env, + ...(process.platform === "win32" ? { PATH: path.dirname(process.execPath) } : undefined), ...(command && typeof command === "object" && "environment" in command ? (command.environment as NodeJS.ProcessEnv | undefined) : undefined), @@ -392,6 +401,7 @@ vi.mock("../daemon/service.js", () => ({ env, command, runtime, + definitionMutationCapability: await serviceDefinitionMutationCapability(), }; }, resolveGatewayService: vi.fn(() => ({ @@ -399,6 +409,7 @@ vi.mock("../daemon/service.js", () => ({ isEnabled: (...args: unknown[]) => serviceEnabled(...args), readCommand: (...args: unknown[]) => serviceReadCommand(...args), readRuntime: (...args: unknown[]) => serviceReadRuntime(...args), + start: (...args: unknown[]) => serviceStart(...args), stop: (...args: unknown[]) => serviceStop(...args), restart: (...args: unknown[]) => serviceRestart(...args), })), @@ -717,6 +728,11 @@ describe("update-cli", () => { (argv.length === 4 || argv[4] === "--fix"), ); + const freshRestartCalls = () => + vi + .mocked(runCommandWithTimeout) + .mock.calls.filter(([argv]) => argv[2] === "gateway" && argv[3] === "restart"); + const gatewayCommandCall = (entryPath: string, action: "install" | "restart") => commandCalls().find( ([argv]) => argv[1] === entryPath && argv[2] === "gateway" && argv[3] === action, @@ -1264,7 +1280,7 @@ describe("update-cli", () => { const serviceNode = path.join(params.prefix, "bin", "node"); const serviceNpm = path.join(params.prefix, "bin", "npm"); await fs.mkdir(path.dirname(serviceNode), { recursive: true }); - await fs.writeFile(serviceNode, "", "utf-8"); + await fs.writeFile(serviceNode, "#!/bin/sh\n", { encoding: "utf-8", mode: 0o755 }); const serviceNpmReal = params.withNpm === false ? undefined @@ -1336,14 +1352,12 @@ describe("update-cli", () => { "gateway", "run", ]); - serviceLoaded.mockResolvedValue(false); - serviceLoaded.mockResolvedValueOnce(true); - serviceReadRuntime.mockResolvedValue({ status: "stopped", pid: null, state: "stopped" }); - serviceReadRuntime.mockResolvedValueOnce({ - status: "running", - pid: gatewayFixturePid, - state: "running", - }); + serviceLoaded.mockImplementation(async () => serviceStop.mock.calls.length === 0); + serviceReadRuntime.mockImplementation(async () => + serviceStop.mock.calls.length === 0 + ? { status: "running", pid: gatewayFixturePid, state: "running" } + : { status: "stopped", pid: null, state: "stopped" }, + ); }; const expectFailedManagedGitRestart = (message: string) => { @@ -1351,7 +1365,9 @@ describe("update-cli", () => { expect(serviceStop).toHaveBeenCalledTimes(1); expect(runRestartScript).toHaveBeenCalledTimes(1); expect(defaultRuntime.exit).toHaveBeenCalledWith(1); - expect(logs).toContain(message); + expect([logs, ...vi.mocked(defaultRuntime.error).mock.calls.flat()].join("\n")).toContain( + message, + ); expect(logs).not.toContain("Gateway: restarted and verified."); expect(logs).not.toContain("Update Result: OK"); }; @@ -1477,7 +1493,14 @@ describe("update-cli", () => { return readRestartSentinel({ OPENCLAW_STATE_DIR: stateDir } as NodeJS.ProcessEnv); }; - beforeEach(() => { + beforeEach(async () => { + const gatewayEntrypoint = await import("../daemon/gateway-entrypoint.js"); + const actualGatewayEntrypoint = await vi.importActual< + typeof import("../daemon/gateway-entrypoint.js") + >("../daemon/gateway-entrypoint.js"); + vi.mocked(gatewayEntrypoint.resolveGatewayInstallEntrypoint).mockImplementation( + actualGatewayEntrypoint.resolveGatewayInstallEntrypoint, + ); delete process.env.OPENCLAW_SERVICE_MARKER; delete process.env.OPENCLAW_SERVICE_KIND; delete process.env[GATEWAY_SERVICE_RUNTIME_PID_ENV]; @@ -1487,6 +1510,7 @@ describe("update-cli", () => { restartHealthTestControl.snapshot = undefined; vi.resetAllMocks(); serviceEnabled.mockResolvedValue(true); + serviceDefinitionMutationCapability.mockResolvedValue(undefined); readPersistedInstalledPluginIndex.mockResolvedValue(null); restorePersistedInstalledPluginIndexIfCurrent.mockResolvedValue(true); writePersistedInstalledPluginIndexInstallRecords.mockResolvedValue(undefined); @@ -1573,7 +1597,9 @@ describe("update-cli", () => { readPackageName.mockResolvedValue("openclaw"); readPackageVersion.mockResolvedValue("1.0.0"); resolveGlobalManager.mockResolvedValue("npm"); + serviceStart.mockResolvedValue(undefined); serviceStop.mockResolvedValue(undefined); + terminateStaleGatewayPids.mockResolvedValue(undefined); serviceRestart.mockResolvedValue({ outcome: "completed" }); isDefaultInstallIdentity.mockReset(); isDefaultInstallIdentity.mockReturnValue(true); @@ -1583,11 +1609,11 @@ describe("update-cli", () => { serviceReadCommand.mockImplementation(async () => (await serviceLoaded()) ? { programArguments: ["openclaw", "gateway", "run"] } : null, ); - serviceReadRuntime.mockResolvedValue({ - status: "running", - pid: gatewayFixturePid, - state: "running", - }); + serviceReadRuntime.mockImplementation(async () => + (await serviceLoaded()) + ? { status: "running", pid: gatewayFixturePid, state: "running" } + : { status: "stopped", state: "stopped", missingUnit: true }, + ); mockGetSelfAndAncestorPidsSync.mockReturnValue(new Set([process.pid])); prepareRestartScript.mockResolvedValue("/tmp/openclaw-restart-test.sh"); runRestartScript.mockResolvedValue(undefined); @@ -1647,6 +1673,229 @@ describe("update-cli", () => { tempDirsToCleanup.clear(); }); + it("recovers a stopped sealed service after a restart-safe failure", async () => { + mockRunningManagedGateway(["node", path.join(process.cwd(), "dist", "index.js"), "gateway"]); + serviceDefinitionMutationCapability.mockResolvedValue({ kind: "sealed", detail: "root owner" }); + const { + maybeStopManagedServiceBeforeMutableUpdate, + maybeRestartServiceAfterFailedMutableUpdate, + } = await import("./update-cli/update-command-service.js"); + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + root: process.cwd(), + updateInstallKind: "git", + shouldRestart: true, + jsonMode: true, + }); + await maybeRestartServiceAfterFailedMutableUpdate({ + preManagedServiceStop: before, + jsonMode: true, + }); + expect(serviceRestart).toHaveBeenCalledOnce(); + }); + + it.each([ + { kind: "git", restart: false }, + { kind: "git", restart: true }, + { kind: "package", restart: false }, + { kind: "package", restart: true }, + ] as const)( + "updates $kind with restart=$restart when service inspection is unavailable", + async ({ kind, restart }) => { + if (kind === "package") { + mockPackageInstallAtCaseDir(); + mockCurrentProcessFreshDoctor(); + } else { + mockGitUpdateAfterMutation(); + } + serviceReadCommand.mockRejectedValue(new Error("inspection-secret-canary")); + + await updateCommand({ yes: true, json: true, restart }); + + if (kind === "package") { + expectPackageInstallSpec("openclaw@9999.0.0"); + } else { + expect(runGatewayUpdate).toHaveBeenCalledOnce(); + } + expectNoSideEffects( + serviceStop, + serviceStart, + serviceRestart, + runDaemonInstall, + runDaemonRestart, + prepareRestartScript, + runRestartScript, + ); + expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); + expect(getErrorOutput()).toContain( + "Gateway service management skipped: inspection is unavailable", + ); + expect(getErrorOutput()).toContain("gateway status --deep"); + expect(getErrorOutput()).not.toContain("inspection-secret-canary"); + expect(lastWriteJsonCall()).toMatchObject({ status: "ok" }); + }, + ); + + it.each([ + { kind: "git", restart: false, capability: "sealed" }, + { kind: "git", restart: true, capability: "sealed" }, + { kind: "package", restart: false, capability: "sealed" }, + { kind: "package", restart: true, capability: "sealed" }, + { kind: "git", restart: true, capability: "unknown" }, + { kind: "package", restart: true, capability: "unknown" }, + ] as const)( + "updates $kind with stale $capability metadata and restart=$restart", + async ({ kind, restart, capability }) => { + const root = + kind === "package" ? createCaseDir("openclaw-sealed-code-update") : process.cwd(); + const entrypoint = + kind === "package" + ? await writeOpenClawPackageFixture(root, "1.0.0", { + entrySource: "export {};\n", + inventory: true, + }) + : path.join(root, "dist", "index.js"); + if (kind === "package") { + mockPackageInstallStatus(root); + mockCurrentProcessFreshDoctor(); + mockGatewayProbe("9999.0.0", "updated-service"); + } else { + mockGitUpdateAfterMutation(makeOkUpdateResult({ mode: "git", root })); + } + vi.mocked(resolveGatewayInstallEntrypoint).mockResolvedValue(entrypoint); + // No managed mode, token, or env-key metadata: this must not become an install-plan veto. + mockRunningManagedGateway(["node", entrypoint, "gateway", "--port", "18789"]); + serviceDefinitionMutationCapability.mockResolvedValue({ + kind: capability, + detail: "definition-owner-secret-canary", + }); + + await updateCommand({ yes: true, json: true, restart }); + + if (kind === "package") { + expectPackageInstallSpec("openclaw@9999.0.0"); + } else { + expect(runGatewayUpdate).toHaveBeenCalledOnce(); + } + expect(serviceStop).toHaveBeenCalledTimes(restart ? 1 : 0); + expect(freshRestartCalls().length).toBe(restart ? 1 : 0); + expect(serviceStart).not.toHaveBeenCalled(); + expectNoSideEffects( + runDaemonInstall, + runDaemonRestart, + prepareRestartScript, + runRestartScript, + ); + expect(getErrorOutput()).toContain("service definition left unchanged"); + expect(getErrorOutput()).not.toContain("definition-owner-secret-canary"); + expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); + expect(lastWriteJsonCall()).toMatchObject({ status: "ok" }); + }, + ); + + it.each([ + ["sealed", "writable"], + ["writable", "unknown"], + ] as const)( + "retains activation but not refresh when authority changes %s -> %s", + async (beforeKind, afterKind) => { + mockRunningManagedGateway(["node", path.join(process.cwd(), "dist", "index.js"), "gateway"]); + serviceDefinitionMutationCapability.mockResolvedValue({ kind: beforeKind, detail: "owner" }); + const { + maybeStopManagedServiceBeforeMutableUpdate, + revalidateManagedGatewayServiceAfterUpdate, + } = await import("./update-cli/update-command-service.js"); + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + root: process.cwd(), + updateInstallKind: "git", + shouldRestart: true, + jsonMode: true, + }); + serviceDefinitionMutationCapability.mockResolvedValue({ kind: afterKind, detail: "owner" }); + const { readGatewayServiceState, resolveGatewayService } = + await import("../daemon/service.js"); + const state = await readGatewayServiceState(resolveGatewayService(), { + requireEffective: true, + }); + await expect( + revalidateManagedGatewayServiceAfterUpdate({ + state, + root: process.cwd(), + preManagedServiceStop: before, + }), + ).resolves.toMatchObject({ kind: "owned", refreshDefinition: false }); + }, + ); + + it.each(["unchanged", "changed", "unreadable"] as const)( + "recovers stopped unresolved services only with unchanged inspection (%s)", + async (inspection) => { + mockRunningManagedGateway(); + const { + maybeStopManagedServiceBeforeMutableUpdate, + maybeRestartServiceAfterFailedMutableUpdate, + } = await import("./update-cli/update-command-service.js"); + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + root: process.cwd(), + updateInstallKind: "git", + shouldRestart: true, + jsonMode: true, + }); + expect(before).toMatchObject({ + stopped: true, + serviceUpdateVerdict: { kind: "unresolved" }, + }); + if (inspection === "changed") { + mockRunningManagedGateway(["foreign-openclaw", "gateway", "run"]); + } else if (inspection === "unreadable") { + serviceReadCommand.mockRejectedValueOnce(new Error("manager unavailable")); + } + + await maybeRestartServiceAfterFailedMutableUpdate({ + preManagedServiceStop: before, + jsonMode: true, + }); + + expect(serviceRestart).toHaveBeenCalledTimes(inspection === "unchanged" ? 1 : 0); + if (inspection !== "unchanged") { + expect(defaultRuntime.error).toHaveBeenCalledWith( + expect.stringContaining("Failed to restart managed gateway service after failed update"), + ); + } + }, + ); + + it("fails sealed-service activation without claiming a successful restart", async () => { + vi.mocked(runCommandWithTimeout).mockResolvedValueOnce( + commandResult({ code: 1, stderr: "systemctl restart denied" }), + ); + const { maybeRestartService } = await import("./update-cli/update-command-service.js"); + vi.mocked(resolveGatewayInstallEntrypoint).mockResolvedValue("/updated/dist/index.js"); + + await expect( + maybeRestartService({ + channel: "stable", + shouldRestart: true, + result: makeOkUpdateResult({ mode: "npm", after: { version: "2026.4.24" } }), + opts: { json: true }, + refreshServiceEnv: false, + serviceUpdateVerdict: { + kind: "owned", + root: process.cwd(), + refreshDefinition: false, + fingerprint: "sealed", + }, + serviceEnv: { MANAGED_VALUE: "revalidated" }, + gatewayPort: 18789, + requireRunningServiceAfterRestart: true, + timeoutMs: 1_000, + }), + ).resolves.toBe(false); + + expect(freshRestartCalls().length).toBe(1); + expect(serviceStart).not.toHaveBeenCalled(); + expectNoSideEffects(runRestartScript, runDaemonInstall, runDaemonRestart, serviceRestart); + }); + it("reads the initial update config without schema validation or observation", async () => { await updateCommand({ yes: true, restart: false }); @@ -2074,18 +2323,38 @@ describe("update-cli", () => { expect(stderrPipe).toHaveBeenCalledWith(process.stderr); }); - it("finishes package updates when the post-core process writes a result but keeps handles open", async () => { + it("stops a post-core process with open handles only once when result reads overlap", async () => { setupUpdatedRootRefresh(); const kill = vi.fn(); + let resultPath: string | undefined; + const readsReady = createDeferred(); + const releaseReads = createDeferred(); + const jsonFiles = await import("../infra/json-files.js"); + const readJsonIfExists = jsonFiles.readJsonIfExists; + const pendingReads: Promise[] = []; + let resultReads = 0; + const readSpy = vi + .spyOn(jsonFiles, "readJsonIfExists") + .mockImplementation((...args: Parameters) => { + const read = readJsonIfExists(...args).then(async (result) => { + if (args[0] === resultPath) { + if (++resultReads === 2) { + readsReady.resolve(); + } + await releaseReads.promise; + } + return result; + }); + pendingReads.push(read); + return read; + }); spawn.mockImplementationOnce((_command: unknown, _argv: unknown, options: unknown) => { - const resultPath = (options as { env?: NodeJS.ProcessEnv }).env + resultPath = (options as { env?: NodeJS.ProcessEnv }).env ?.OPENCLAW_UPDATE_POST_CORE_RESULT_PATH; if (!resultPath) { throw new Error("missing post-core result path"); } - queueMicrotask(() => { - void fs.writeFile(resultPath, `${JSON.stringify({ status: "ok" })}\n`, "utf-8"); - }); + fsSync.writeFileSync(resultPath, `${JSON.stringify({ status: "ok" })}\n`, "utf-8"); const child = new EventEmitter() as EventEmitter & { kill: typeof kill; once: EventEmitter["once"]; @@ -2094,11 +2363,26 @@ describe("update-cli", () => { return child; }); - await updateCommand({ yes: true, restart: false }); + const updating = updateCommand({ yes: true, restart: false }); + try { + await Promise.race([ + readsReady.promise, + updating.then(() => { + throw new Error("update finished before overlapping result reads"); + }), + ]); + releaseReads.resolve(); + await updating; + await Promise.all(pendingReads); - expect(kill).toHaveBeenCalledTimes(1); - expect(updateNpmInstalledPlugins).not.toHaveBeenCalled(); - expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); + expect(kill).toHaveBeenCalledTimes(1); + expect(updateNpmInstalledPlugins).not.toHaveBeenCalled(); + expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); + } finally { + releaseReads.resolve(); + await Promise.allSettled([updating, ...pendingReads]); + readSpy.mockRestore(); + } }); it("does not restart a stopped managed gateway after post-core plugin errors", async () => { @@ -2465,7 +2749,6 @@ describe("update-cli", () => { readPackageVersion.mockResolvedValue("2026.4.14"); primeNpmChannelTag("latest", "2026.4.10"); mockCurrentProcessFreshDoctor(); - mockGatewayProbe("2026.4.10", "downgraded-gateway"); await updateCommand({ yes: true, tag: "2026.4.10", restart: false }); @@ -2477,30 +2760,6 @@ describe("update-cli", () => { expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); }); - it("runs the fresh doctor for a core-changing downgrade without plugin changes", async () => { - const downgradedRoot = createCaseDir("openclaw-downgraded-fresh-doctor-root"); - setupUpdatedRootRefresh({ - gatewayUpdateImpl: async () => - makeOkUpdateResult({ - mode: "npm", - root: downgradedRoot, - before: { version: "2026.4.14" }, - after: { version: "2026.4.10" }, - }), - }); - readPackageVersion.mockResolvedValue("2026.4.14"); - primeNpmChannelTag("latest", "2026.4.10"); - mockCurrentProcessFreshDoctor(); - - await updateCommand({ yes: true, tag: "2026.4.10", restart: false }); - - expect(spawn).not.toHaveBeenCalled(); - expect(syncPluginsForUpdateChannel).toHaveBeenCalledTimes(1); - expect(updateNpmInstalledPlugins).toHaveBeenCalledTimes(1); - expectFreshPostUpdateDoctor({ yes: true }); - expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); - }); - it("pins the compatibility host version to the downgraded target during current-process post-core plugin convergence (#87914)", async () => { const downgradedRoot = createCaseDir("openclaw-downgraded-compat-root"); setupUpdatedRootRefresh({ @@ -5094,6 +5353,7 @@ describe("update-cli", () => { "guards a %s Windows Scheduled Task during a no-restart package update", async (runtimeStatus) => { const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const homeSpy = vi.spyOn(os, "homedir").mockReturnValue(fixtureRoot); mockPackageInstallStatus(createCaseDir("openclaw-update-stopped-task")); primeServiceCommand(["openclaw", "gateway", "run"], { OPENCLAW_SERVICE_MARKER: "openclaw", @@ -5107,8 +5367,12 @@ describe("update-cli", () => { suspendScheduledTaskAutoStartForUpdate.mockResolvedValue(true); resumeScheduledTaskAutoStartAfterUpdate.mockResolvedValue(true); - await updateCommand({ yes: true, restart: false }); - platformSpy.mockRestore(); + try { + await updateCommand({ yes: true, restart: false }); + } finally { + homeSpy.mockRestore(); + platformSpy.mockRestore(); + } expect(serviceStop).not.toHaveBeenCalled(); expect(packageInstallCommandCall()).toBeDefined(); @@ -5130,6 +5394,46 @@ describe("update-cli", () => { }, ); + it("does not suspend a foreign Windows Scheduled Task during a no-restart package update", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("win32"); + const updateRoot = tempDirs.make("openclaw-update-foreign-task-"); + const foreignRoot = tempDirs.make("openclaw-update-foreign-task-owner-"); + const foreignEntrypoint = await writeOpenClawPackageFixture(foreignRoot, "2026.4.21", { + entrySource: "export {};\n", + }); + primeServiceCommand(["node", foreignEntrypoint, "gateway", "run"]); + serviceLoaded.mockResolvedValue(true); + serviceReadRuntime.mockResolvedValue({ + status: "running", + state: "running", + pid: gatewayFixturePid, + }); + suspendScheduledTaskAutoStartForUpdate.mockResolvedValue(true); + + try { + const { maybeStopManagedServiceBeforeMutableUpdate } = + await import("./update-cli/update-command-service.js"); + await expect( + maybeStopManagedServiceBeforeMutableUpdate({ + root: updateRoot, + updateInstallKind: "package", + shouldRestart: false, + jsonMode: false, + }), + ).resolves.toMatchObject({ + inspected: true, + running: true, + serviceUpdateVerdict: { kind: "foreign" }, + }); + } finally { + platformSpy.mockRestore(); + } + + expect(suspendScheduledTaskAutoStartForUpdate).not.toHaveBeenCalled(); + expect(resumeScheduledTaskAutoStartAfterUpdate).not.toHaveBeenCalled(); + expect(serviceStop).not.toHaveBeenCalled(); + }); + it("stops a running managed gateway when git checkout rebuild starts", async () => { const serviceEntrypoint = path.join(process.cwd(), "dist", "index.js"); mockRunningManagedGateway(["node", serviceEntrypoint, "gateway", "run"]); @@ -5158,27 +5462,131 @@ describe("update-cli", () => { ]); }); - it("stops a running managed git gateway when wrapper commands hide the service root", async () => { - const wrapperPath = path.join( - createCaseDir("openclaw-update-wrapper-service"), - "gateway-wrapper", + it("uses a manager-effective global user unit during update preflight", async () => { + const entrypoint = path.join(process.cwd(), "dist", "index.js"); + const command = { + programArguments: ["node", entrypoint, "gateway", "--port", "18789"], + environment: { + OPENCLAW_SERVICE_MARKER: "openclaw", + OPENCLAW_SERVICE_KIND: "gateway", + }, + sourcePath: "/etc/systemd/user/openclaw-gateway.service", + definitionPaths: ["/etc/systemd/user/openclaw-gateway.service"], + }; + serviceReadCommand.mockImplementation(async (options) => + options?.requireEffective ? command : null, ); - mockRunningManagedGateway([wrapperPath, "gateway", "run"]); - const preparations = mockGitUpdateAfterMutation(); + serviceLoaded.mockResolvedValue(true); + serviceReadRuntime.mockResolvedValue({ + status: "running", + pid: gatewayFixturePid, + state: "running", + }); + serviceDefinitionMutationCapability.mockResolvedValue({ + kind: "sealed", + detail: "privileged global user unit", + }); await updateCommand({ yes: true }); + expect(serviceReadCommand).toHaveBeenCalledWith({ requireEffective: true }); + }); + + it("uses an explicit service wrapper when openclaw is absent from PATH", async () => { + const wrapperDir = createCaseDir("openclaw-update-wrapper-service"); + const wrapperPath = path.join(wrapperDir, "gateway-wrapper"); + await fs.mkdir(wrapperDir, { recursive: true }); + await fs.writeFile(wrapperPath, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + const configPath = path.join(wrapperDir, "openclaw.json"); + await fs.writeFile(configPath, JSON.stringify(baseSnapshot.config)); + const serviceEnv = { + ...process.env, + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_STATE_DIR: wrapperDir, + OPENCLAW_WRAPPER: wrapperPath, + PATH: path.dirname(process.execPath), + }; + const { buildGatewayInstallPlan } = await import("../commands/daemon-install-helpers.js"); + const initialPlan = await buildGatewayInstallPlan({ + env: serviceEnv, + config: baseSnapshot.config, + port: 18789, + runtime: "node", + runtimePath: process.execPath, + wrapperPath, + }); + const existingEnvironment = Object.fromEntries( + Object.entries(initialPlan.environment).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ); + const { resolveOwnedManagedUpdateEnv } = + await import("./update-cli/update-command-service-env.js"); + const { mergeInstallInvocationEnv } = await import("./daemon-cli/install.js"); + const ownedEnv = resolveOwnedManagedUpdateEnv({ + serviceEnv: { ...process.env, ...existingEnvironment }, + serviceDefinitionEnv: existingEnvironment, + invocationCwd: process.cwd(), + }); + const installEnv = mergeInstallInvocationEnv({ + env: ownedEnv, + existingServiceEnv: existingEnvironment, + }); + const servicePlan = await buildGatewayInstallPlan({ + env: installEnv, + config: baseSnapshot.config, + port: 18789, + runtime: "node", + wrapperPath, + existingEnvironment, + existingEnvironmentValueSources: initialPlan.environmentValueSources, + }); + const serviceCommand = { + ...servicePlan, + environment: Object.fromEntries( + Object.entries(servicePlan.environment).filter( + (entry): entry is [string, string] => typeof entry[1] === "string", + ), + ), + }; + serviceReadCommand.mockResolvedValue(serviceCommand); + serviceLoaded.mockResolvedValue(true); + serviceReadRuntime.mockImplementation(async () => + serviceStop.mock.calls.length === 0 || freshRestartCalls().length > 0 + ? { status: "running", pid: gatewayFixturePid, state: "running" } + : { status: "stopped", pid: null, state: "stopped" }, + ); + serviceDefinitionMutationCapability.mockResolvedValue({ + kind: "sealed", + detail: "privileged wrapper owner", + }); + const { resolveExecutablePath } = await import("../infra/executable-path.js"); + expect(resolveExecutablePath("openclaw", { env: serviceCommand.environment })).toBeUndefined(); + const envSnapshot = captureEnv(Object.keys(serviceCommand.environment)); + mockGitUpdateAfterMutation(makeOkUpdateResult({ mode: "git", root: process.cwd() })); + vi.mocked(resolveGatewayInstallEntrypoint).mockResolvedValue( + path.join(process.cwd(), "dist", "index.js"), + ); + + try { + await updateCommand({ yes: true }); + } finally { + envSnapshot.restore(); + const { clearConfigCache } = await import("../config/io.js"); + const { clearRuntimeConfigSnapshot } = await import("../config/runtime-snapshot.js"); + clearConfigCache(); + clearRuntimeConfigSnapshot(); + } + + expect(getErrorOutput()).toContain("service definition left unchanged"); expect(serviceStop).toHaveBeenCalledTimes(1); expect(runGatewayUpdate).toHaveBeenCalledTimes(1); - expect(prepareRestartScript).toHaveBeenCalledWith( - expect.anything(), - expect.any(Number), - undefined, + const restartOptions = freshRestartCalls()[0]?.[1]; + expect(typeof restartOptions === "object" && restartOptions.env?.OPENCLAW_WRAPPER).toBe( + wrapperPath, ); - expectNoSideEffects(runDaemonInstall, runDaemonRestart); - expect(preparations).toEqual([ - { allowGatewayServiceRepair: false, allowGatewayActivation: false }, - ]); + expect(serviceStart).not.toHaveBeenCalled(); + expectNoSideEffects(prepareRestartScript, runRestartScript, runDaemonInstall, runDaemonRestart); }); it("fails managed git restart when the gateway responds but the service stays stopped", async () => { @@ -5251,12 +5659,27 @@ describe("update-cli", () => { const packageEntrypoint = await writeOpenClawPackageFixture(packageRoot, "2026.4.20", { entrySource: "export {};\n", }); - await writeOpenClawPackageFixture(gitRoot, "2026.4.21", { git: true }); + const gitEntrypoint = await writeOpenClawPackageFixture(gitRoot, "2026.4.21", { + entrySource: "export {};\n", + git: true, + }); const canonicalGitRoot = await fs.realpath(gitRoot); mockPackageInstallStatus(packageRoot); pathExists.mockImplementation(async (candidate: string) => candidate === gitRoot); mockRunningManagedGateway(["node", packageEntrypoint, "gateway", "run"]); - mockGitUpdateAfterMutation( + serviceReadCommand.mockImplementation(async () => ({ + programArguments: [ + "node", + serviceStop.mock.calls.length > 0 ? gitEntrypoint : packageEntrypoint, + "gateway", + "run", + ], + environment: { + OPENCLAW_SERVICE_MARKER: "openclaw", + OPENCLAW_SERVICE_KIND: "gateway", + }, + })); + const preparations = mockGitUpdateAfterMutation( makeOkUpdateResult({ mode: "git", root: gitRoot, @@ -5269,6 +5692,17 @@ describe("update-cli", () => { expect(serviceStop).toHaveBeenCalledTimes(1); expect(runGatewayUpdate).toHaveBeenCalledTimes(1); + expect(preparations).toEqual([ + { allowGatewayServiceRepair: true, allowGatewayActivation: true }, + ]); + expect(prepareRestartScript).toHaveBeenCalledWith(expect.anything(), expect.any(Number), [ + "node", + gitEntrypoint, + "gateway", + "run", + ]); + expect(runRestartScript).toHaveBeenCalledTimes(1); + expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); const updateCall = vi.mocked(runGatewayUpdate).mock.calls[0]?.[0]; expect(updateCall?.cwd).toBe(canonicalGitRoot); expect(updateCall?.beforeGitMutation).toEqual(expect.any(Function)); @@ -5779,7 +6213,7 @@ describe("update-cli", () => { expect(errors).toContain("Upgrade the Node runtime that owns the managed Gateway service"); }); - it("runs managed service package follow-up commands with the service Node", async () => { + it("runs managed service package follow-up commands with the service Node despite heap argv", async () => { const shellRoot = createCaseDir("openclaw-shell-root"); const servicePrefix = tempDirs.make("openclaw-service-prefix-"); const { @@ -5791,7 +6225,7 @@ describe("update-cli", () => { entrypoint, } = await setupServicePackageAtPrefix({ prefix: servicePrefix }); mockPackageInstallStatus(shellRoot); - primeServiceCommand([serviceNode, entrypoint, "gateway"]); + primeServiceCommand([serviceNode, "--max-old-space-size=16384", entrypoint, "gateway"]); serviceLoaded.mockResolvedValue(true); primeNpmChannelTag("latest", "2026.5.20"); mockFileBackedPathExists(); @@ -5813,26 +6247,34 @@ describe("update-cli", () => { expect(serviceInstallCall?.[0][0]).toBe(serviceNode); }); - it("uses the managed service Node when package roots match but node binaries differ", async () => { - const root = createCaseDir("openclaw-same-root"); - // Service is baked with a different node than the current process.execPath. - const serviceNode = "/opt/other-node/bin/node"; - const entrypoint = path.join(root, "dist", "index.js"); - mockPackageInstallStatus(root); - primeServiceCommand([serviceNode, entrypoint, "gateway"]); + it.each([ + { command: "gateway", selected: true }, + { command: "agent", selected: false }, + ])( + "selects service Node only for a Gateway command ($command)", + async ({ command, selected }) => { + const root = createCaseDir("openclaw-same-root"); + const serviceNode = "/opt/other-node/bin/node"; + const entrypoint = path.join(root, "dist", "index.js"); + mockPackageInstallStatus(root); + primeServiceCommand([serviceNode, "--import", "tsx", entrypoint, command]); - await updateCommand({ dryRun: true }); + await updateCommand({ dryRun: true }); - const logs = getLogOutput(); - // Should NOT log root redirect messages since the package root is the same. - expect(logs).not.toContain("Targeting managed gateway service package root"); - // Should warn about the node binary mismatch. - expect(logs).toContain("differs from the managed gateway service Node"); - expect(logs).toContain(serviceNode); - expect(logs).toContain( - "Using the managed service Node for this update so the gateway can start after the upgrade", - ); - }); + const logs = getLogOutput(); + expect(logs).not.toContain("Targeting managed gateway service package root"); + if (selected) { + expect(logs).toContain("differs from the managed gateway service Node"); + expect(logs).toContain(serviceNode); + expect(logs).toContain( + "Using the managed service Node for this update so the gateway can start after the upgrade", + ); + } else { + expect(logs).not.toContain("differs from the managed gateway service Node"); + expect(logs).not.toContain(serviceNode); + } + }, + ); it("refreshes the managed service to current Node when its baked Node cannot run the target", async () => { const servicePrefix = tempDirs.make("openclaw-service-prefix-"); @@ -6652,7 +7094,7 @@ describe("update-cli", () => { }, ] as const)("updateCommand service refresh behavior: $name", runUpdateCliScenario); - it("restores an unknown package service without rewriting its missing updated entrypoint", async () => { + it("reports activation failure when the updated CLI entrypoint is missing", async () => { mockPackageInstallAtCaseDir(); mockCurrentProcessFreshDoctor(); serviceLoaded.mockResolvedValue(true); @@ -6661,8 +7103,10 @@ describe("update-cli", () => { await updateCommand({ yes: true }); expect(runDaemonInstall).not.toHaveBeenCalled(); - expect(runRestartScript).toHaveBeenCalledWith("/tmp/openclaw-restart-test.sh"); - expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); + expect(serviceStart).not.toHaveBeenCalled(); + expect(freshRestartCalls().length).toBe(0); + expectNoSideEffects(prepareRestartScript, runRestartScript); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); }); it("tries the updated install restart when package service refresh fails", async () => { @@ -6714,7 +7158,7 @@ describe("update-cli", () => { expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); }); - it("restores a same-version service without rewriting when its root is ambiguous", async () => { + it("leaves a same-version service untouched when its package root is foreign", async () => { const oldRoot = createCaseDir("openclaw-old-root"); const updatedRoot = createCaseDir("openclaw-updated-root"); const oldEntrypoint = path.join(oldRoot, "dist", "entry.js"); @@ -6743,7 +7187,14 @@ describe("update-cli", () => { expect(gatewayCommandCall(updatedEntrypoint, "install")).toBeUndefined(); expect(gatewayCommandCall(updatedEntrypoint, "restart")).toBeUndefined(); - expect(runRestartScript).toHaveBeenCalledWith("/tmp/openclaw-restart-test.sh"); + expectNoSideEffects( + serviceStop, + serviceStart, + serviceRestart, + prepareRestartScript, + runRestartScript, + ); + expect(getErrorOutput()).toContain("service belongs to a different OpenClaw installation"); expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); }); @@ -6991,7 +7442,7 @@ describe("update-cli", () => { }; }, }); - primeServiceCommand(["node", setup.entrypoints[0], "gateway", "run"], { + primeServiceCommand([process.execPath, setup.entrypoints[0], "gateway", "run"], { OPENCLAW_STATE_DIR: "./service-state", OPENCLAW_CONFIG_PATH: "./service-config/openclaw.json", PATH: "/service/bin", @@ -7113,7 +7564,13 @@ describe("update-cli", () => { const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0); try { await withEnvAsync({ OPENCLAW_UPDATE_IN_PROGRESS: undefined }, async () => { - vi.mocked(runGatewayUpdate).mockResolvedValue(makeOkUpdateResult()); + mockRunningManagedGateway([ + "node", + path.join(process.cwd(), "dist", "index.js"), + "gateway", + ]); + mockGitUpdateAfterMutation(); + prepareRestartScript.mockResolvedValue(null); vi.mocked(runDaemonRestart).mockResolvedValue(true); vi.mocked(doctorCommand).mockResolvedValue(undefined); vi.mocked(defaultRuntime.log).mockClear(); @@ -7342,10 +7799,12 @@ describe("update-cli", () => { spec: "post-plugin@1.0.0", }, } satisfies Record; - vi.mocked(readConfigFileSnapshot) - .mockResolvedValueOnce(preDoctorSnapshot) - .mockResolvedValueOnce(postDoctorSnapshot) - .mockResolvedValueOnce(postDoctorSnapshot); + let currentSnapshot = preDoctorSnapshot; + vi.mocked(readConfigFileSnapshot).mockImplementation(async () => currentSnapshot); + vi.mocked(runExec).mockImplementationOnce(async () => { + currentSnapshot = postDoctorSnapshot; + return { stdout: "", stderr: "" }; + }); loadInstalledPluginIndexInstallRecords.mockResolvedValueOnce(postDoctorRecords); syncPluginsForUpdateChannel.mockImplementationOnce( async (params: { config?: OpenClawConfig }) => @@ -7446,15 +7905,16 @@ describe("update-cli", () => { parsed: baseSnapshot.parsed, hash: "post-doctor", }); - vi.mocked(readConfigFileSnapshot) - .mockResolvedValueOnce(preDoctorSnapshot) - .mockResolvedValueOnce(preDoctorSnapshot) - .mockResolvedValueOnce(preDoctorSnapshot) - .mockResolvedValueOnce(postDoctorSnapshot) - .mockResolvedValueOnce(postDoctorSnapshot); + let currentSnapshot = preDoctorSnapshot; + vi.mocked(readConfigFileSnapshot).mockImplementation(async () => currentSnapshot); + vi.mocked(runExec).mockImplementationOnce(async () => { + currentSnapshot = postDoctorSnapshot; + return { stdout: "", stderr: "" }; + }); await updateFinalizeCommand({ channel: "dev", json: true, restart: false }); + expectFreshPostUpdateDoctor({ yes: false, workspaceSuggestions: true }); expect(replaceConfigCall(0)?.baseHash).toBe("pre-doctor"); expect(replaceConfigCall(0)?.nextConfig).toEqual({ update: { channel: "dev" } }); expect(replaceConfigCall(1)?.baseHash).toBe("post-doctor"); diff --git a/src/cli/update-cli/managed-gateway-update.runtime.ts b/src/cli/update-cli/managed-gateway-update.runtime.ts new file mode 100644 index 000000000000..3d771cfe94f8 --- /dev/null +++ b/src/cli/update-cli/managed-gateway-update.runtime.ts @@ -0,0 +1,8 @@ +export { + maybeRestartService, + resolvePreparedGatewayUpdatePolicy, + resolveUpdatedGatewayRestartPort, + maybeRestartServiceAfterFailedMutableUpdate, + maybeStopManagedServiceBeforeMutableUpdate, + revalidateManagedGatewayServiceAfterUpdate, +} from "./update-command-service.js"; diff --git a/src/cli/update-cli/update-command-execution.ts b/src/cli/update-cli/update-command-execution.ts index 9b1a92f6712a..d68637fa438c 100644 --- a/src/cli/update-cli/update-command-execution.ts +++ b/src/cli/update-cli/update-command-execution.ts @@ -24,6 +24,7 @@ import { maybeRestartServiceAfterFailedMutableUpdate, maybeResumeWindowsTaskAutoStartAfterPackageUpdate, maybeStopManagedServiceBeforeMutableUpdate, + resolvePreparedGatewayUpdatePolicy, shouldBlockMutableUpdateFromGatewayServiceEnv, UpdateCommandAbort, type ManagedServiceRootRedirect, @@ -76,6 +77,7 @@ export async function executeMutableUpdate(params: { : null; const stopManagedServiceBeforeMutableUpdate = async ( mutationRoots: readonly string[] = [params.root], + phase: "inspect" | "prepare" = "prepare", ) => { if (params.updateInstallKind !== "package" && params.updateInstallKind !== "git") { return; @@ -88,6 +90,8 @@ export async function executeMutableUpdate(params: { root: mutationRoot, shouldRestart: params.shouldRestart, jsonMode: Boolean(params.opts.json), + timeoutMs: params.updateStepTimeoutMs, + phase, }); if (preManagedServiceStop.windowsTaskAutoStartRecovery) { params.recoveryState.windowsTaskAutoStartRecovery = @@ -114,6 +118,10 @@ export async function executeMutableUpdate(params: { throw new UpdateCommandAbort(); } + if (phase === "inspect" && preManagedServiceStop?.serviceUpdateVerdict?.kind === "foreign") { + preManagedServiceStop = undefined; + } + try { ownedManagedUpdateContext = await captureOwnedManagedUpdateContext({ stopState: preManagedServiceStop, @@ -131,13 +139,6 @@ export async function executeMutableUpdate(params: { throw new UpdateCommandAbort(); } - if (preManagedServiceStop?.blockMessage) { - params.stop(); - defaultRuntime.error(preManagedServiceStop.blockMessage); - defaultRuntime.exit(1); - throw new UpdateCommandAbort(); - } - if (shouldBlockMutableUpdateFromGatewayServiceEnv({ preManagedServiceStop })) { params.stop(); const updateLabel = params.updateInstallKind === "git" ? "Git updates" : "Package updates"; @@ -151,11 +152,21 @@ export async function executeMutableUpdate(params: { defaultRuntime.exit(1); throw new UpdateCommandAbort(); } + + if (preManagedServiceStop?.blockMessage) { + params.stop(); + defaultRuntime.error(preManagedServiceStop.blockMessage); + defaultRuntime.exit(1); + throw new UpdateCommandAbort(); + } }; - if (params.updateInstallKind === "package") { + if (params.updateInstallKind === "package" || params.updateInstallKind === "git") { try { - await stopManagedServiceBeforeMutableUpdate(); + await stopManagedServiceBeforeMutableUpdate( + gitMutationRoots ?? undefined, + params.updateInstallKind === "git" ? "inspect" : "prepare", + ); } catch (err) { if (err instanceof UpdateCommandAbort) { return null; @@ -198,11 +209,7 @@ export async function executeMutableUpdate(params: { startedAt: params.startedAt, progress: params.progress, jsonMode: Boolean(params.opts.json), - allowGatewayServiceRepair: preManagedServiceStop?.serviceMatchesMutationRoot === true, - allowGatewayActivation: - params.shouldRestart && - preManagedServiceStop?.stopped === true && - preManagedServiceStop.serviceMatchesMutationRoot === true, + ...resolvePreparedGatewayUpdatePolicy(preManagedServiceStop, params.shouldRestart), managedServiceEnv: preManagedServiceStop?.serviceEnv, invocationCwd: params.invocationCwd, honorPackageRoot: diff --git a/src/cli/update-cli/update-command-git.ts b/src/cli/update-cli/update-command-git.ts index 4f5ca4c8ec2e..e73e33afab10 100644 --- a/src/cli/update-cli/update-command-git.ts +++ b/src/cli/update-cli/update-command-git.ts @@ -31,7 +31,11 @@ import { runUpdateStep, type UpdateCommandOptions, } from "./shared.js"; -import { UpdateCommandAbort, type PreManagedServiceStop } from "./update-command-service.js"; +import { + resolvePreparedGatewayUpdatePolicy, + UpdateCommandAbort, + type PreManagedServiceStop, +} from "./update-command-service.js"; const DEFAULT_UPDATE_STEP_TIMEOUT_MS = 30 * 60_000; @@ -186,15 +190,7 @@ export function createBeforeGitMutation(params: { defaultRuntime.error(formatSchemaRefusalLines(postStopSchemas).join("\n")); throw new UpdateCommandAbort(); } - return { - // Only a positively owned service may be rewritten. Activation - // additionally requires this update to have stopped it. - allowGatewayServiceRepair: preManagedServiceStop?.serviceMatchesMutationRoot === true, - allowGatewayActivation: - params.shouldRestart && - preManagedServiceStop?.stopped === true && - preManagedServiceStop.serviceMatchesMutationRoot === true, - }; + return resolvePreparedGatewayUpdatePolicy(preManagedServiceStop, params.shouldRestart); }; } diff --git a/src/cli/update-cli/update-command-lease.test-support.ts b/src/cli/update-cli/update-command-lease.test-support.ts new file mode 100644 index 000000000000..66605b593775 --- /dev/null +++ b/src/cli/update-cli/update-command-lease.test-support.ts @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { once } from "node:events"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawConfig } from "../../config/types.openclaw.js"; +import type { PluginInstallRecord } from "../../config/types.plugins.js"; + +export type LeaseScenario = { + lane: "resume" | "current-process" | "repair"; + preDoctorChannel?: string; + invalidConfig?: boolean; + failDoctor?: "pre" | "post"; + hostVersion?: string; + doctorWrites?: boolean; + writerConfig?: OpenClawConfig; + writerRecords?: Record; +}; + +// A narrow child substitutes for the CLI, not for its cross-process lease. +export async function runUpdateLeaseChild(): Promise { + const stateDir = process.env.OPENCLAW_STATE_DIR; + const configPath = process.env.OPENCLAW_CONFIG_PATH; + assert.ok(stateDir && configPath); + const scenario = JSON.parse( + await fs.readFile(path.join(stateDir, "scenario.json"), "utf8"), + ) as LeaseScenario; + const record = async (event: string) => + fs.appendFile( + path.join(stateDir, "events.jsonl"), + `${JSON.stringify({ event, pid: process.pid })}\n`, + ); + const publish = async () => { + assert.ok(scenario.writerConfig && scenario.writerRecords); + const { writePersistedInstalledPluginIndexInstallRecords } = + await import("../../plugins/installed-plugin-index-records.js"); + await fs.writeFile(configPath, JSON.stringify(scenario.writerConfig)); + await writePersistedInstalledPluginIndexInstallRecords(scenario.writerRecords, { + config: scenario.writerConfig, + }); + await record("writer-committed"); + }; + const command = process.argv[2]; + if (command === "config") { + assert.deepEqual(process.argv.slice(2), ["config", "validate", "--json"]); + assert.equal(process.env.OPENCLAW_UPDATE_IN_PROGRESS, "0"); + await record("validate"); + process.exitCode = scenario.invalidConfig ? 1 : 0; + return; + } + const { withPluginLifecycleLease } = await import("../../plugins/plugin-lifecycle-lease.js"); + if (command === "doctor") { + const phase = process.env.OPENCLAW_UPDATE_POST_CORE_CONVERGENCE === "1" ? "post" : "pre"; + assert.deepEqual(process.argv.slice(3), [ + "--repair", + "--non-interactive", + ...(scenario.lane === "repair" && phase === "pre" ? [] : ["--no-workspace-suggestions"]), + "--yes", + ]); + assert.equal(process.env.OPENCLAW_UPDATE_IN_PROGRESS, "1"); + assert.equal(process.env.OPENCLAW_UPDATE_DEFER_CONFIGURED_PLUGIN_INSTALL_REPAIR, "1"); + assert.equal(process.env.OPENCLAW_UPDATE_PARENT_SUPPORTS_DOCTOR_CONFIG_WRITE, "1"); + if (scenario.hostVersion) { + assert.equal(process.env.OPENCLAW_COMPATIBILITY_HOST_VERSION, scenario.hostVersion); + } + await record(`${phase}-attempt`); + // One real acquisition attempt makes the regression fail promptly, without changing parent budgets. + await withPluginLifecycleLease({ waitMs: 0 }, async () => { + await record(`${phase}-acquired`); + if (phase === "pre" && scenario.preDoctorChannel !== undefined) { + const { readConfigFileSnapshot } = await import("../../config/config.js"); + assert.equal( + (await readConfigFileSnapshot()).config.update?.channel, + scenario.preDoctorChannel, + ); + } + if (phase === "pre" && scenario.doctorWrites) { + await publish(); + } + }); + process.stdout.write("doctor fixture output\n"); + process.stderr.write("doctor fixture diagnostic\n"); + if (scenario.failDoctor === phase) { + throw new Error("doctor fixture failure"); + } + return; + } + if (command === "probe") { + try { + await withPluginLifecycleLease({ waitMs: 0 }, async () => record("probe-acquired")); + process.stdout.write("acquired"); + } catch (error) { + if (!(error instanceof Error) || !("code" in error)) { + throw error; + } + assert.equal(error.code, "OPENCLAW_STATE_LEASE_TIMEOUT"); + process.stdout.write("excluded"); + } + return; + } + assert.equal(command, "writer"); + assert.ok(process.connected, "writer requires an IPC channel"); + await withPluginLifecycleLease({}, async () => { + const release = once(process, "message"); + process.send?.("acquired"); + await release; + await publish(); + }); + process.disconnect?.(); +} diff --git a/src/cli/update-cli/update-command-lease.test.ts b/src/cli/update-cli/update-command-lease.test.ts new file mode 100644 index 000000000000..84f41767fa65 --- /dev/null +++ b/src/cli/update-cli/update-command-lease.test.ts @@ -0,0 +1,420 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createDeferred } from "../../../test/helpers/promise.js"; +import { readConfigFileSnapshot } from "../../config/config.js"; +import type { PluginInstallRecord } from "../../config/types.plugins.js"; +import { + loadInstalledPluginIndexInstallRecords, + writePersistedInstalledPluginIndexInstallRecords, +} from "../../plugins/installed-plugin-index-records.js"; +import { runExec } from "../../process/exec.js"; +import { defaultRuntime } from "../../runtime.js"; +import { + createOpenClawTestState, + type OpenClawTestState, +} from "../../test-utils/openclaw-test-state.js"; + +const mocks = vi.hoisted(() => ({ + entrypoint: vi.fn(), + root: vi.fn(), + plugins: vi.fn(), + restart: vi.fn(async () => true), + print: vi.fn(), +})); + +vi.mock("../../daemon/gateway-entrypoint.js", () => ({ + resolveGatewayInstallEntrypoint: mocks.entrypoint, +})); +vi.mock("./update-command-plugins.js", () => ({ updatePluginsAfterCoreUpdate: mocks.plugins })); +vi.mock("./progress.js", () => ({ printResult: mocks.print })); +vi.mock("./shared.js", async (importOriginal) => ({ + ...(await importOriginal()), + resolveUpdateRoot: mocks.root, + tryWriteCompletionCache: vi.fn(async () => "skipped"), +})); +vi.mock("./update-command-service.js", async (importOriginal) => ({ + ...(await importOriginal()), + maybeRestartService: mocks.restart, + tryInstallShellCompletion: vi.fn(), +})); + +import { updateFinalizeCommand } from "./update-command-finalize.js"; +import type { LeaseScenario } from "./update-command-lease.test-support.js"; +import type { PostCorePluginUpdateResult } from "./update-command-plugins.js"; +import { finishUpdate } from "./update-command-post-update.js"; +import { resumePostCoreUpdate } from "./update-command-resume.js"; + +const pluginResult: PostCorePluginUpdateResult = { + status: "ok", + changed: true, + sync: { changed: false, switchedToBundled: [], switchedToNpm: [], warnings: [], errors: [] }, + npm: { changed: false, outcomes: [] }, + integrityDrifts: [], +}; +type Lane = LeaseScenario["lane"]; +let state: OpenClawTestState; +let entrypoint: string; + +beforeEach(async () => { + vi.clearAllMocks(); + state = await createOpenClawTestState({ + label: "update-lease", + env: { + OPENCLAW_COMPATIBILITY_HOST_VERSION: undefined, + OPENCLAW_UPDATE_POST_CORE_RESULT_PATH: undefined, + OPENCLAW_UPDATE_POST_CORE_INSTALL_RECORDS_PATH: undefined, + OPENCLAW_UPDATE_POST_CORE_SOURCE_CONFIG_PATH: undefined, + OPENCLAW_UPDATE_POST_CORE_REQUESTED_CHANNEL: undefined, + OPENCLAW_UPDATE_POST_CORE_STARTED_AT_MS: undefined, + }, + }); + await state.writeConfig({ plugins: { enabled: false }, update: { channel: "stable" } }); + await fs.writeFile(state.path("package.json"), JSON.stringify({ version: "1.0.0" })); + entrypoint = await state.writeText( + "entry.mjs", + ` + import { tsImport } from ${JSON.stringify(import.meta.resolve("tsx/esm/api"))}; + const { runUpdateLeaseChild } = await tsImport(${JSON.stringify(new URL("./update-command-lease.test-support.ts", import.meta.url).href)}, { parentURL: import.meta.url, tsconfig: ${JSON.stringify(path.resolve("tsconfig.json"))} }); + await runUpdateLeaseChild(); + `, + ); + mocks.entrypoint.mockResolvedValue(entrypoint); + mocks.root.mockResolvedValue(state.root); + mocks.plugins.mockReset().mockResolvedValue(pluginResult); + vi.spyOn(defaultRuntime, "exit").mockImplementation(() => undefined as never); + vi.spyOn(defaultRuntime, "writeJson").mockImplementation(() => undefined); + vi.spyOn(defaultRuntime, "log").mockImplementation(() => undefined); + vi.spyOn(defaultRuntime, "error").mockImplementation(() => undefined); +}); + +afterEach(async () => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + await state.cleanup(); +}); + +async function writeScenario( + lane: Lane, + scenario: Omit = {}, +): Promise { + await state.writeJson("scenario.json", { ...scenario, lane }); +} + +async function invoke(lane: Lane): Promise { + if (lane === "resume") { + return resumePostCoreUpdate({ + root: state.root, + channel: "stable", + opts: { json: true, yes: true }, + timeoutMs: 15_000, + }); + } + if (lane === "repair") { + return updateFinalizeCommand({ + json: true, + yes: true, + restart: false, + timeout: "15", + deferCompletionCache: true, + }); + } + return finishUpdate({ + result: { + status: "ok", + mode: "npm", + root: state.root, + before: { version: "2.0.0" }, + after: { version: "1.0.0" }, + steps: [], + durationMs: 1, + }, + root: state.root, + installKindChanged: false, + configSnapshot: await readConfigFileSnapshot({ skipPluginValidation: true }), + requestedChannel: null, + storedChannel: "stable", + channel: "stable", + downgradeRisk: true, + shouldRestart: false, + opts: { json: true, yes: true }, + showProgress: false, + ownedManagedUpdateEnv: { ...process.env }, + controlPlaneUpdateSentinelMeta: null, + preUpdatePluginInstallRecords: { stale: { source: "path", sourcePath: state.path("stale") } }, + startedAt: Date.now(), + updateStepTimeoutMs: 15_000, + }); +} + +async function events(): Promise { + return (await fs.readFile(state.statePath("events.jsonl"), "utf8")) + .trim() + .split("\n") + .map((line) => { + const event = JSON.parse(line) as { event: string; pid: number }; + expect(event.pid).not.toBe(process.pid); + return event.event; + }); +} + +function expectDoctorDiagnostics(): void { + expect(defaultRuntime.log).not.toHaveBeenCalledWith(expect.stringContaining("doctor fixture")); + expect(defaultRuntime.error).toHaveBeenCalledWith("doctor fixture output"); + expect(defaultRuntime.error).toHaveBeenCalledWith( + expect.stringContaining("doctor fixture diagnostic"), + ); +} + +function expectSuccess(lane: Lane): void { + expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1); + const output = + lane === "current-process" + ? mocks.print.mock.lastCall?.[0] + : vi.mocked(defaultRuntime.writeJson).mock.lastCall?.[0]; + expect(output).toMatchObject({ status: "ok", postUpdate: { plugins: { status: "ok" } } }); + expectDoctorDiagnostics(); +} + +describe("update orchestration lifecycle ownership", () => { + it.each(["resume", "current-process", "repair"] as const)( + "%s releases ownership for fresh doctor and strict validation", + async (lane) => { + await writeScenario(lane, { + hostVersion: lane === "repair" ? undefined : "1.0.0", + }); + await invoke(lane); + expectSuccess(lane); + expect(await events()).toEqual([ + ...(lane === "current-process" ? [] : ["pre-attempt", "pre-acquired"]), + "post-attempt", + "post-acquired", + "validate", + ]); + if (lane === "current-process") { + expect(process.env.OPENCLAW_COMPATIBILITY_HOST_VERSION).toBeUndefined(); + expect(mocks.restart).toHaveBeenCalledWith( + expect.objectContaining({ shouldRestart: false }), + ); + } + }, + ); + + it.each(["resume", "current-process", "repair"] as const)( + "%s keeps plugin mutation exclusive to its parent", + async (lane) => { + await writeScenario(lane); + mocks.plugins.mockImplementationOnce(async () => { + const result = await runExec(process.execPath, [entrypoint, "probe"], { + timeoutMs: 15_000, + }); + expect(result.stdout).toBe("excluded"); + return pluginResult; + }); + await invoke(lane); + expectSuccess(lane); + expect(mocks.plugins).toHaveBeenCalledOnce(); + const after = await runExec(process.execPath, [entrypoint, "probe"], { timeoutMs: 15_000 }); + expect(after.stdout).toBe("acquired"); + }, + ); + + it.each(["current-process", "repair"] as const)( + "%s reloads config and records after a competing writer commits", + async (lane) => { + await writePersistedInstalledPluginIndexInstallRecords({ old: { source: "path" } }); + expect(await loadInstalledPluginIndexInstallRecords()).toHaveProperty("old"); + const writerRecords: Record = { + current: { source: "path", sourcePath: state.path("current") }, + }; + await writeScenario(lane, { + writerConfig: { + plugins: { enabled: false }, + update: { channel: "beta" }, + gateway: { port: 19002 }, + }, + writerRecords, + }); + const acquired = createDeferred(); + const completed = createDeferred(); + const child = spawn(process.execPath, [entrypoint, "writer"], { + env: process.env, + stdio: ["ignore", "pipe", "pipe", "ipc"], + }); + if (!child.stderr) { + throw new Error("writer stderr pipe was not created"); + } + let stderr = ""; + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.once("message", () => acquired.resolve()); + child.once("error", (error) => { + acquired.reject(error); + completed.reject(error); + }); + child.once("close", (code) => { + if (code === 0) { + completed.resolve(); + } else { + const error = new Error(`writer exited ${code}: ${stderr}`); + acquired.reject(error); + completed.reject(error); + } + }); + void completed.promise.catch(() => {}); + try { + await acquired.promise; + const update = invoke(lane); + void update.catch(() => {}); + child.send("commit"); + await completed.promise; + await update; + expectSuccess(lane); + expect(mocks.plugins).toHaveBeenCalledWith( + expect.objectContaining({ + configSnapshot: expect.objectContaining({ + config: expect.objectContaining({ + gateway: expect.objectContaining({ port: 19002 }), + }), + }), + pluginInstallRecords: writerRecords, + }), + ); + } finally { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + await completed.promise.catch(() => {}); + } + }, + ); + + it.each([false, true])( + "resume reads the doctor's committed generation (empty=%s)", + async (empty) => { + const old = { old: { source: "path" as const } }; + await writePersistedInstalledPluginIndexInstallRecords(old); + expect(await loadInstalledPluginIndexInstallRecords()).toEqual(old); + const recordsPath = await state.writeJson("forwarded.json", old); + vi.stubEnv("OPENCLAW_UPDATE_POST_CORE_INSTALL_RECORDS_PATH", recordsPath); + vi.stubEnv("OPENCLAW_UPDATE_POST_CORE_STARTED_AT_MS", String(Date.now())); + const current: Record = empty + ? {} + : { current: { source: "path" } }; + await writeScenario("resume", { + doctorWrites: true, + writerConfig: { plugins: { enabled: false }, gateway: { port: 19003 } }, + writerRecords: current, + }); + await invoke("resume"); + expectSuccess("resume"); + expect(mocks.plugins).toHaveBeenCalledWith( + expect.objectContaining({ + configSnapshot: expect.objectContaining({ + config: expect.objectContaining({ gateway: expect.objectContaining({ port: 19003 }) }), + }), + pluginInstallRecords: current, + }), + ); + expect(await events()).toEqual([ + "pre-attempt", + "pre-acquired", + "writer-committed", + "post-attempt", + "post-acquired", + "validate", + ]); + }, + ); + + it.each(["resume", "repair"] as const)( + "%s does not run a final doctor when no plugins changed", + async (lane) => { + await writeScenario(lane); + mocks.plugins.mockResolvedValueOnce({ ...pluginResult, changed: false }); + await invoke(lane); + expectSuccess(lane); + expect(await events()).toEqual(["pre-attempt", "pre-acquired"]); + }, + ); + + it.each(["resume", "current-process", "repair"] as const)( + "%s retains strict fresh validation after releasing the lease", + async (lane) => { + await writeScenario(lane, { invalidConfig: true }); + await invoke(lane); + const output = + lane === "current-process" + ? mocks.print.mock.lastCall?.[0] + : vi.mocked(defaultRuntime.writeJson).mock.lastCall?.[0]; + expect(output).toMatchObject({ + status: "error", + postUpdate: { plugins: { reason: "post-plugin-doctor-invalid-config" } }, + }); + expect(mocks.restart).not.toHaveBeenCalled(); + expect(await events()).toContain("post-acquired"); + expect((await events()).at(-1)).toBe("validate"); + if (lane !== "resume") { + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); + } + }, + ); + + it("repair persists a requested channel before its fresh doctor and retains timings", async () => { + await writeScenario("repair", { preDoctorChannel: "beta" }); + await updateFinalizeCommand({ + channel: "beta", + json: true, + yes: true, + restart: false, + deferCompletionCache: true, + }); + expectSuccess("repair"); + expect(await events()).toContain("pre-acquired"); + expect(vi.mocked(defaultRuntime.writeJson).mock.lastCall?.[0]).toMatchObject({ + channel: "beta", + restart: false, + phaseTimings: [ + "targetConfigValidation", + "configSnapshot", + "doctor", + "plugins", + "targetConfigConvergence", + "completionCache", + ].map((phase) => + expect.objectContaining({ + phase, + outcome: phase === "completionCache" ? "deferred" : "completed", + }), + ), + }); + }); + + it.each(["resume", "repair"] as const)( + "%s propagates a pre-plugin doctor failure before parent mutation", + async (lane) => { + await writeScenario(lane, { failDoctor: "pre" }); + await expect(invoke(lane)).rejects.toThrow("doctor fixture failure"); + expect(mocks.plugins).not.toHaveBeenCalled(); + expect(defaultRuntime.writeJson).not.toHaveBeenCalled(); + expectDoctorDiagnostics(); + expect(await events()).toEqual(["pre-attempt", "pre-acquired"]); + }, + ); + + it("keeps a failed final doctor fatal even when strict validation succeeds", async () => { + await writeScenario("current-process", { failDoctor: "post", hostVersion: "1.0.0" }); + await invoke("current-process"); + expect(mocks.print.mock.lastCall?.[0]).toMatchObject({ + status: "error", + postUpdate: { plugins: { reason: "post-plugin-doctor-execution-failed" } }, + }); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); + expect(mocks.restart).not.toHaveBeenCalled(); + expect(process.env.OPENCLAW_COMPATIBILITY_HOST_VERSION).toBeUndefined(); + expectDoctorDiagnostics(); + expect(await events()).toEqual(["post-attempt", "post-acquired", "validate"]); + }); +}); diff --git a/src/cli/update-cli/update-command-managed-context.ts b/src/cli/update-cli/update-command-managed-context.ts index 61518d193704..bc6a7c5e88c9 100644 --- a/src/cli/update-cli/update-command-managed-context.ts +++ b/src/cli/update-cli/update-command-managed-context.ts @@ -47,7 +47,7 @@ export async function captureOwnedManagedUpdateContext(params: { const stopState = params.stopState; if ( stopState?.stopped !== true || - stopState.serviceMatchesMutationRoot !== true || + stopState.serviceUpdateVerdict?.kind !== "owned" || !stopState.serviceEnv ) { return undefined; diff --git a/src/cli/update-cli/update-command-post-core.ts b/src/cli/update-cli/update-command-post-core.ts index 6b654ac789d2..33a391f7726a 100644 --- a/src/cli/update-cli/update-command-post-core.ts +++ b/src/cli/update-cli/update-command-post-core.ts @@ -396,18 +396,18 @@ export async function continuePostCoreUpdateInFreshProcess(params: { settled = true; clearInterval(resultPoll); resolve(result); + if (result.kind === "plugin-update") { + // Only the winning result stops the child. Claim completion first so its + // signal cannot reject committed work and roll the plugin index back. + stopPostCoreUpdateChild(child); + } }; const resultPoll = setInterval(() => { void readPostCorePluginUpdateResultFile(resultPath) .then((pluginUpdate) => { - if (!pluginUpdate) { - return; + if (pluginUpdate) { + finish({ kind: "plugin-update", pluginUpdate }); } - // Claim the settle before stopping: the stop delivers a signal, and the exit - // handler below rejects on any signal it still owns. Stopping first would fail - // an update this child already committed and roll its plugin index back. - finish({ kind: "plugin-update", pluginUpdate }); - stopPostCoreUpdateChild(child); }) .catch(() => undefined); }, POST_CORE_UPDATE_RESULT_POLL_MS); diff --git a/src/cli/update-cli/update-command-post-update.test.ts b/src/cli/update-cli/update-command-post-update.test.ts index 3c56e72c2673..c85d6fa973e7 100644 --- a/src/cli/update-cli/update-command-post-update.test.ts +++ b/src/cli/update-cli/update-command-post-update.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { beforeEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { UpdateRunResult } from "../../infra/update-runner.js"; import { defaultRuntime } from "../../runtime.js"; @@ -10,11 +10,19 @@ const mocks = vi.hoisted(() => ({ leaseActive: false, loadPluginRecords: vi.fn(), markSentinelFailure: vi.fn(async () => undefined), + prepareRestartScript: vi.fn(async () => null), printResult: vi.fn(), readConfig: vi.fn(), + createServiceConfigIO: vi.fn(), readServiceState: vi.fn(), restart: vi.fn(async () => undefined), - restartService: vi.fn(async (_params: { serviceInstallEnv?: NodeJS.ProcessEnv | null }) => true), + restartService: vi.fn( + async () => true, + ), + revalidateService: + vi.fn< + typeof import("./update-command-service.js").revalidateManagedGatewayServiceAfterUpdate + >(), restoreWindowsAutoStart: vi.fn(async () => true), tryInstallCompletion: vi.fn(async () => undefined), tryWriteCompletionCache: vi.fn(async () => undefined), @@ -27,6 +35,10 @@ vi.mock("../../config/config.js", async (importOriginal) => ({ ...(await importOriginal()), readConfigFileSnapshot: mocks.readConfig, })); +vi.mock("../../config/io.js", async (importOriginal) => ({ + ...(await importOriginal()), + createConfigIO: mocks.createServiceConfigIO, +})); vi.mock("../../daemon/service.js", async (importOriginal) => ({ ...(await importOriginal()), readGatewayServiceState: mocks.readServiceState, @@ -65,12 +77,13 @@ vi.mock("./shared.js", async (importOriginal) => ({ tryWriteCompletionCache: mocks.tryWriteCompletionCache, })); vi.mock("./restart-helper.js", () => ({ - prepareRestartScript: vi.fn(async () => null), + prepareRestartScript: mocks.prepareRestartScript, })); vi.mock("./update-command-service.js", async (importOriginal) => ({ ...(await importOriginal()), maybeRestartService: mocks.restartService, maybeRestartServiceAfterFailedMutableUpdate: mocks.restart, + revalidateManagedGatewayServiceAfterUpdate: mocks.revalidateService, restoreWindowsTaskAutoStartOrExit: mocks.restoreWindowsAutoStart, tryInstallShellCompletion: mocks.tryInstallCompletion, })); @@ -82,6 +95,7 @@ vi.mock("./update-command-post-core.js", async (importOriginal) => ({ import { retireStandaloneGitWrapper } from "./update-command-git.js"; import { finishUpdate } from "./update-command-post-update.js"; +import { resolveUpdatedGatewayRestartPort } from "./update-command-service.js"; type FinishUpdateParams = Parameters[0]; @@ -115,12 +129,22 @@ async function finishSuccessfulPackageSwitch(params: { previousRoot: string; packageRoot: string; restartEnvironment?: NodeJS.ProcessEnv; + sealed?: boolean; + updateMode?: UpdateRunResult["mode"]; + stoppedForUpdate?: boolean; }): Promise { await finishUpdate({ result: { status: "ok", - mode: "npm", + mode: params.updateMode ?? "npm", root: params.packageRoot, + ...(params.sealed && { + before: { version: "2026.4.23" }, + after: { + version: "2026.4.24", + ...(params.updateMode === "git" ? { buildId: "new-build" } : {}), + }, + }), steps: [], durationMs: 1, }, @@ -130,7 +154,7 @@ async function finishSuccessfulPackageSwitch(params: { configSnapshot: validConfigSnapshot, requestedChannel: null, storedChannel: null, - channel: "stable", + channel: params.updateMode === "git" ? "dev" : "stable", downgradeRisk: true, shouldRestart: Boolean(params.restartEnvironment), opts: {}, @@ -140,7 +164,17 @@ async function finishSuccessfulPackageSwitch(params: { startedAt: Date.now(), updateStepTimeoutMs: 1_000, ...(params.restartEnvironment && { - preManagedServiceStop: { stopped: true, serviceMatchesMutationRoot: true }, + preManagedServiceStop: { + stopped: params.stoppedForUpdate ?? true, + ...(params.sealed && { + serviceUpdateVerdict: { + kind: "owned", + root: params.previousRoot, + refreshDefinition: false, + fingerprint: "sealed", + }, + }), + }, ownedManagedUpdateEnv: params.restartEnvironment, }), } as unknown as FinishUpdateParams); @@ -237,7 +271,17 @@ describe("successful update finalization ordering", () => { vi.clearAllMocks(); mocks.leaseActive = false; mocks.loadPluginRecords.mockResolvedValue({}); + mocks.revalidateService.mockImplementation(async ({ root, preManagedServiceStop }) => ({ + kind: "owned", + root, + fingerprint: "sealed", + refreshDefinition: + preManagedServiceStop?.serviceUpdateVerdict?.kind === "owned" + ? preManagedServiceStop.serviceUpdateVerdict.refreshDefinition + : true, + })); mocks.readConfig.mockResolvedValue(validConfigSnapshot); + mocks.createServiceConfigIO.mockReturnValue({ readBestEffortConfig: async () => ({}) }); mocks.updatePlugins.mockResolvedValue(successfulPluginUpdate); mocks.completePluginUpdate.mockResolvedValue({ pluginUpdate: successfulPluginUpdate, @@ -452,29 +496,63 @@ describe("successful update finalization ordering", () => { } }); - it.each([ - ["unknown", true], - ["inline reset", { resetInline: true }], - ["environment-file reset", { resetFiles: true }], - ] as const)("skips unsafe metadata refresh for %s ownership", async (_, environment) => { - const programArguments = ["/usr/bin/node", "/tmp/openclaw-update/dist/index.js", "gateway"]; - mocks.readServiceState.mockResolvedValueOnce({ - installed: true, - loadState: { status: "loaded" }, - env: {}, - command: { - programArguments, - managedDefinition: { programArguments }, - managedOverrides: { environment }, - }, - }); - - vi.stubEnv("HOME", os.homedir()); - vi.stubEnv("OPENCLAW_PROFILE", "default"); - for (const key of ["OPENCLAW_HOME", "OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH"]) { - vi.stubEnv(key, ""); - } + it("reads the preserved service config without using the caller config or writing state", async () => { + const { createConfigIO } = + await vi.importActual("../../config/io.js"); + mocks.createServiceConfigIO.mockImplementation(createConfigIO); + const home = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-restart-config-")); + const configPath = path.join(home, "openclaw.json"); + await fs.writeFile(configPath, JSON.stringify({ gateway: { mode: "local", port: 19600 } })); try { + expect( + await resolveUpdatedGatewayRestartPort({ + config: { gateway: { port: 19601 } }, + processEnv: { OPENCLAW_GATEWAY_PORT: "19602" }, + serviceEnv: { HOME: home, OPENCLAW_STATE_DIR: home, OPENCLAW_CONFIG_PATH: configPath }, + serviceCommand: { + programArguments: ["/usr/bin/node", "/srv/openclaw/dist/index.js", "gateway"], + }, + }), + ).toBe(19600); + expect(await fs.readdir(home)).toEqual(["openclaw.json"]); + } finally { + await fs.rm(home, { recursive: true, force: true }); + } + }); + + describe("managed service finalization", () => { + beforeEach(() => { + vi.stubEnv("HOME", os.homedir()); + vi.stubEnv("OPENCLAW_PROFILE", "default"); + for (const key of ["OPENCLAW_HOME", "OPENCLAW_STATE_DIR", "OPENCLAW_CONFIG_PATH"]) { + vi.stubEnv(key, ""); + } + }); + afterEach(() => vi.unstubAllEnvs()); + + it.each([ + ["unknown", true], + ["inline reset", { resetInline: true }], + ["environment-file reset", { resetFiles: true }], + ] as const)("skips unsafe metadata refresh for %s ownership", async (_, environment) => { + const programArguments = [ + "/usr/bin/node", + "/tmp/openclaw-update/dist/index.js", + "gateway", + "--port", + "19305", + ]; + mocks.readServiceState.mockResolvedValueOnce({ + installed: true, + loadState: { status: "loaded" }, + env: {}, + command: { + programArguments, + managedDefinition: { programArguments }, + managedOverrides: { environment }, + }, + }); + await finishSuccessfulPackageSwitch({ previousRoot: "/tmp/openclaw-update", packageRoot: "/tmp/openclaw-update", @@ -488,12 +566,170 @@ describe("successful update finalization ordering", () => { serviceInstallEnv: null, }), ); - expect(defaultRuntime.log).toHaveBeenCalledWith( - expect.stringContaining("metadata refresh was skipped because systemd drop-in environment"), + expect(mocks.restartService).toHaveBeenCalledWith( + expect.objectContaining({ + serviceUpdateVerdict: expect.objectContaining({ refreshDefinition: false }), + }), ); - } finally { - vi.unstubAllEnvs(); - } + expect(mocks.restartService.mock.lastCall?.[0].gatewayPort).toBe(19305); + }); + + it.each([ + { source: "preserved ExecStart", sealed: true, args: ["--port", "19301"], expected: 19301 }, + { source: "preserved config", sealed: true, args: [], expected: 19304 }, + { source: "writable refresh", sealed: false, args: ["--port=19301"], expected: 19303 }, + ])("verifies the CLI service port for $source", async ({ sealed, args, expected }) => { + const serviceEnv = { HOME: os.homedir() }; + mocks.readServiceState.mockResolvedValue({ + installed: true, + loadState: { status: "loaded" }, + env: serviceEnv, + command: { + programArguments: [ + "/usr/bin/node", + "/tmp/openclaw-update/dist/index.js", + "gateway", + ...args, + ], + environment: serviceEnv, + }, + }); + mocks.readConfig.mockResolvedValue({ + ...validConfigSnapshot, + config: { gateway: { port: 19303 } }, + }); + mocks.completePluginUpdate.mockResolvedValue({ + pluginUpdate: successfulPluginUpdate, + configSnapshot: { ...validConfigSnapshot, config: { gateway: { port: 19303 } } }, + }); + mocks.createServiceConfigIO.mockReturnValue({ + readBestEffortConfig: async () => ({ gateway: { port: 19304 } }), + }); + vi.stubEnv("OPENCLAW_GATEWAY_PORT", ""); + await finishSuccessfulPackageSwitch({ + previousRoot: "/tmp/openclaw-update", + packageRoot: "/tmp/openclaw-update", + restartEnvironment: { ...process.env }, + sealed, + }); + + const restart = mocks.restartService.mock.calls.at(-1)?.[0]; + expect({ port: restart?.gatewayPort, refresh: restart?.refreshServiceEnv }).toEqual({ + port: expected, + refresh: !sealed, + }); + if (!sealed) { + expect(mocks.prepareRestartScript).toHaveBeenCalledWith( + serviceEnv, + expected, + expect.any(Array), + ); + expect(mocks.createServiceConfigIO).not.toHaveBeenCalled(); + } + }); + + it.each(["inspection", "revalidation"] as const)( + "does not restart a stopped sealed service when fresh %s fails", + async (failure) => { + const error = new Error("inspection-secret-canary"); + mocks.readServiceState.mockResolvedValue({ + installed: true, + loadState: { status: "loaded" }, + env: {}, + command: { + programArguments: ["/usr/bin/node", "/tmp/openclaw-update/dist/index.js", "gateway"], + }, + }); + if (failure === "inspection") { + mocks.readServiceState.mockRejectedValueOnce(error); + } else { + mocks.revalidateService.mockRejectedValueOnce(error); + } + await finishSuccessfulPackageSwitch({ + previousRoot: "/tmp/openclaw-update", + packageRoot: "/tmp/openclaw-update", + restartEnvironment: { ...process.env }, + sealed: true, + }); + + expect(mocks.restartService).not.toHaveBeenCalled(); + expect(mocks.prepareRestartScript).not.toHaveBeenCalled(); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); + expect(defaultRuntime.error).toHaveBeenCalledWith( + "Stopped gateway service could not be revalidated; inspect it before restarting manually.", + ); + expect(mocks.printResult).not.toHaveBeenCalled(); + }, + ); + + it.each([ + { name: "finalizes only after healthy activation", activated: true, unloaded: false }, + { + name: "marks failed activation without finalizing success", + activated: false, + unloaded: false, + }, + { + name: "preserves the native context of an unloaded git service", + activated: true, + unloaded: true, + }, + ])("canonical sealed post-update $name", async ({ activated, unloaded }) => { + const serviceEnv = { MANAGED_VALUE: "revalidated" }; + const programArguments = ["/usr/bin/node", "/tmp/openclaw-update/dist/index.js", "gateway"]; + mocks.readServiceState.mockResolvedValueOnce({ + installed: true, + loadState: { status: unloaded ? "not-loaded" : "loaded" }, + env: serviceEnv, + command: { programArguments, environment: serviceEnv }, + }); + mocks.restartService.mockResolvedValueOnce(activated); + await finishSuccessfulPackageSwitch({ + previousRoot: "/tmp/openclaw-update", + packageRoot: "/tmp/openclaw-update", + restartEnvironment: { ...process.env }, + sealed: true, + updateMode: unloaded ? "git" : "npm", + stoppedForUpdate: !unloaded, + }); + + expect(mocks.revalidateService).toHaveBeenCalledOnce(); + expect(mocks.prepareRestartScript).not.toHaveBeenCalled(); + expect(mocks.restartService).toHaveBeenCalledWith( + expect.objectContaining({ + refreshServiceEnv: false, + serviceEnv, + serviceUpdateVerdict: { + kind: "owned", + root: "/tmp/openclaw-update", + refreshDefinition: false, + fingerprint: "sealed", + }, + channel: unloaded ? "dev" : "stable", + result: expect.objectContaining({ + after: { version: "2026.4.24", ...(unloaded ? { buildId: "new-build" } : {}) }, + }), + requireRunningServiceAfterRestart: !unloaded, + }), + ); + expect(mocks.revalidateService.mock.invocationCallOrder[0]).toBeLessThan( + mocks.restartService.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + if (activated) { + expect(mocks.writeSentinel).toHaveBeenCalledTimes(2); + expect(mocks.restartService.mock.invocationCallOrder[0]).toBeLessThan( + mocks.writeSentinel.mock.invocationCallOrder[1] ?? Number.POSITIVE_INFINITY, + ); + expect(mocks.markSentinelFailure).not.toHaveBeenCalled(); + } else { + expect(mocks.writeSentinel).toHaveBeenCalledOnce(); + expect(mocks.markSentinelFailure).toHaveBeenCalledWith( + expect.objectContaining({ reason: "restart-unhealthy" }), + ); + expect(mocks.printResult).not.toHaveBeenCalled(); + expect(defaultRuntime.exit).toHaveBeenCalledWith(1); + } + }); }); }); @@ -502,6 +738,7 @@ function failedResult(recovery: UpdateRunResult["recovery"]): UpdateRunResult { status: "error", mode: "git", reason: "doctor-failed", + root: "/repo", recovery, steps: [], durationMs: 1, @@ -566,7 +803,7 @@ describe("failed Git update recovery restart", () => { it("restarts a managed Gateway after verified rollback recovery", async () => { await finishFailedUpdate(failedResult({ serviceRestartSafe: true })); - expect(mocks.restart).toHaveBeenCalledOnce(); + expect(mocks.restart).toHaveBeenCalledWith(expect.objectContaining({ root: "/repo" })); }); it("leaves a managed Gateway stopped after unverified rollback recovery", async () => { diff --git a/src/cli/update-cli/update-command-post-update.ts b/src/cli/update-cli/update-command-post-update.ts index a8d4eea94309..10a6291776ff 100644 --- a/src/cli/update-cli/update-command-post-update.ts +++ b/src/cli/update-cli/update-command-post-update.ts @@ -44,10 +44,10 @@ import { POST_PLUGIN_DOCTOR_EXECUTION_FAILED_REASON } from "./update-command-pos import { assertGatewayServiceManagementAllowedForUpdate, GatewayServiceUpdateOwnershipError, - gatewayServiceCommandUsesRoot, isGatewayServiceManagementAllowedForUpdate, maybeRestartService, maybeRestartServiceAfterFailedMutableUpdate, + revalidateManagedGatewayServiceAfterUpdate, resolveGatewayServiceManagementBlockMessageForUpdate, resolvePostUpdateServiceStateReadEnv, resolveUpdatedGatewayRestartPort, @@ -132,6 +132,7 @@ export async function finishUpdate(params: { } } else { await maybeRestartServiceAfterFailedMutableUpdate({ + root: params.result.root, preManagedServiceStop: params.preManagedServiceStop, jsonMode: Boolean(params.opts.json), }); @@ -150,6 +151,7 @@ export async function finishUpdate(params: { jsonMode: Boolean(params.opts.json), }); await maybeRestartServiceAfterFailedMutableUpdate({ + root: params.result.root, preManagedServiceStop: params.preManagedServiceStop, jsonMode: Boolean(params.opts.json), }); @@ -344,6 +346,7 @@ export async function finishUpdate(params: { // stopped by this update. Invalid post-migration config intentionally remains stopped. if (postCorePluginUpdate.reason === POST_PLUGIN_DOCTOR_EXECUTION_FAILED_REASON) { await maybeRestartServiceAfterFailedMutableUpdate({ + root: params.result.root, preManagedServiceStop: params.preManagedServiceStop, jsonMode: Boolean(params.opts.json), }); @@ -365,66 +368,60 @@ export async function finishUpdate(params: { let refreshGatewayServiceEnv = false; let gatewayServiceEnv: NodeJS.ProcessEnv | undefined; let gatewayServiceInstallEnv: NodeJS.ProcessEnv | null | undefined; - let skipLegacyServiceRestart = false; + let serviceUpdateVerdict = params.preManagedServiceStop?.serviceUpdateVerdict; + let skipLegacyServiceRestart = serviceUpdateVerdict?.kind === "absent"; const serviceStateReadEnv = resolvePostUpdateServiceStateReadEnv({ updateMode: resultWithPostUpdate.mode, processEnv: process.env, preManagedServiceEnv: params.preManagedServiceStop?.serviceEnv, }); - const serviceMutationAllowed = + let serviceMutationAllowed = params.preManagedServiceStop?.serviceMutationAllowed !== false && isGatewayServiceManagementAllowedForUpdate(process.env) && isGatewayServiceManagementAllowedForUpdate(serviceStateReadEnv); - const serviceMutationSkipMessage = - params.shouldRestart && !serviceMutationAllowed - ? (params.preManagedServiceStop?.serviceMutationSkipMessage ?? - resolveGatewayServiceManagementBlockMessageForUpdate(process.env) ?? - resolveGatewayServiceManagementBlockMessageForUpdate(serviceStateReadEnv)) - : undefined; - let gatewayPort = resolveUpdatedGatewayRestartPort({ + let serviceMutationSkipMessage = !serviceMutationAllowed + ? (params.preManagedServiceStop?.serviceMutationSkipMessage ?? + resolveGatewayServiceManagementBlockMessageForUpdate(process.env) ?? + resolveGatewayServiceManagementBlockMessageForUpdate(serviceStateReadEnv)) + : undefined; + let gatewayPort = await resolveUpdatedGatewayRestartPort({ config: restartConfigSnapshot.valid ? restartConfigSnapshot.config : undefined, processEnv: process.env, serviceEnv: params.ownedManagedUpdateEnv, }); - if (params.shouldRestart && serviceMutationAllowed) { + if (params.shouldRestart && serviceMutationAllowed && !skipLegacyServiceRestart) { try { const serviceState = await readGatewayServiceState(resolveGatewayService(), { env: serviceStateReadEnv, + requireEffective: true, validateEnvBeforeStatusRead: assertGatewayServiceManagementAllowedForUpdate, + timeoutMs: params.updateStepTimeoutMs, }); - const serviceMatchesUpdateRoot = - (await gatewayServiceCommandUsesRoot({ - root: postUpdateRoot, - command: serviceState.command, - })) ?? undefined; - const serviceOwnershipConfirmed = - params.preManagedServiceStop?.serviceMatchesMutationRoot === true || - serviceMatchesUpdateRoot === true; - const knownForeignService = - params.preManagedServiceStop?.serviceMatchesMutationRoot === false && - serviceMatchesUpdateRoot !== true; - const serviceLoaded = serviceState.loadState.status === "loaded"; + serviceUpdateVerdict = await revalidateManagedGatewayServiceAfterUpdate({ + state: serviceState, + root: postUpdateRoot, + preManagedServiceStop: params.preManagedServiceStop, + }); + gatewayServiceEnv = serviceState.env; skipLegacyServiceRestart = - knownForeignService || - (resultWithPostUpdate.mode === "git" && - serviceState.installed && - serviceLoaded && - params.preManagedServiceStop?.stopped !== true && - serviceMatchesUpdateRoot === false); - if ( - !knownForeignService && + serviceUpdateVerdict.kind === "foreign" || serviceUpdateVerdict.kind === "absent"; + if (serviceUpdateVerdict.kind === "unavailable") { + serviceMutationAllowed = false; + serviceMutationSkipMessage = serviceUpdateVerdict.message; + } else if (serviceUpdateVerdict.kind === "foreign") { + serviceMutationAllowed = false; + serviceMutationSkipMessage = + "Gateway service management skipped: the service belongs to a different OpenClaw installation and was left untouched."; + } else if ( + !skipLegacyServiceRestart && shouldPrepareUpdatedInstallRestart({ updateMode: resultWithPostUpdate.mode, serviceInstalled: serviceState.installed, - serviceLoaded, + serviceLoaded: serviceState.loadState.status === "loaded", serviceStoppedForUpdate: params.preManagedServiceStop?.stopped, - serviceMatchesMutationRoot: serviceOwnershipConfirmed - ? true - : params.preManagedServiceStop?.serviceMatchesMutationRoot, - serviceMatchesUpdateRoot, + serviceMatchesUpdateRoot: serviceUpdateVerdict.kind === "owned", }) ) { - gatewayServiceEnv = serviceState.env; gatewayServiceInstallEnv = resolveManagedGatewayServiceProcessEnv( serviceState.command, params.ownedManagedUpdateEnv ?? process.env, @@ -432,37 +429,43 @@ export async function finishUpdate(params: { if (gatewayServiceInstallEnv) { gatewayServiceInstallEnv = stripGatewayServiceMarkerEnv(gatewayServiceInstallEnv); } - gatewayPort = resolveUpdatedGatewayRestartPort({ - config: restartConfigSnapshot.valid ? restartConfigSnapshot.config : undefined, - processEnv: process.env, - serviceEnv: gatewayServiceEnv, - }); + refreshGatewayServiceEnv = + serviceUpdateVerdict.kind === "owned" && serviceUpdateVerdict.refreshDefinition; + if (serviceUpdateVerdict.kind === "owned" && gatewayServiceInstallEnv === null) { + refreshGatewayServiceEnv = false; + serviceUpdateVerdict = { ...serviceUpdateVerdict, refreshDefinition: false }; + } + } + gatewayPort = await resolveUpdatedGatewayRestartPort({ + config: restartConfigSnapshot.valid ? restartConfigSnapshot.config : undefined, + serviceEnv: gatewayServiceEnv, + serviceCommand: + serviceUpdateVerdict.kind === "unresolved" || + (serviceUpdateVerdict.kind === "owned" && !serviceUpdateVerdict.refreshDefinition) + ? serviceState.command + : undefined, + }); + if (refreshGatewayServiceEnv) { restartScriptPath = await prepareRestartScript( serviceState.env, gatewayPort, - serviceOwnershipConfirmed ? serviceState.command?.programArguments : undefined, + serviceState.command?.programArguments, ); - // An ambiguous wrapper may be stopped and restored, but only proven - // ownership authorizes rewriting the service definition. - refreshGatewayServiceEnv = serviceOwnershipConfirmed; - if (refreshGatewayServiceEnv && gatewayServiceInstallEnv === null) { - refreshGatewayServiceEnv = false; - const message = - "Gateway service metadata refresh was skipped because systemd drop-in environment ownership could not be inspected."; - if (params.opts.json) { - defaultRuntime.error(message); - } else { - defaultRuntime.log(theme.warn(message)); - } - } } } catch (err) { - if (err instanceof GatewayServiceUpdateOwnershipError) { - defaultRuntime.error(formatErrorMessage(err)); + if (params.preManagedServiceStop?.stopped) { + const message = + err instanceof GatewayServiceUpdateOwnershipError + ? formatErrorMessage(err) + : "Stopped gateway service could not be revalidated; inspect it before restarting manually."; + defaultRuntime.error(message); defaultRuntime.exit(1); return; } - // Ignore errors during pre-check; fallback to standard restart + serviceMutationAllowed = false; + serviceMutationSkipMessage = + "Code update completed; gateway service management skipped because its current ownership could not be inspected. " + + "Run `openclaw gateway status --deep` before restarting it manually."; } } @@ -488,6 +491,7 @@ export async function finishUpdate(params: { channel: params.channel, opts: params.opts, refreshServiceEnv: refreshGatewayServiceEnv, + serviceUpdateVerdict, serviceEnv: gatewayServiceEnv, serviceInstallEnv: gatewayServiceInstallEnv, gatewayPort, @@ -495,8 +499,7 @@ export async function finishUpdate(params: { invocationCwd: params.invocationCwd, nodeRunner: params.packageUpdateNodeRunner, skipLegacyServiceRestart, - requireRunningServiceAfterRestart: - resultWithPostUpdate.mode === "git" && params.preManagedServiceStop?.stopped === true, + requireRunningServiceAfterRestart: params.preManagedServiceStop?.stopped === true, serviceMutationSkipMessage, timeoutMs: params.updateStepTimeoutMs, }), diff --git a/src/cli/update-cli/update-command-service-recovery.ts b/src/cli/update-cli/update-command-service-recovery.ts index 61a6308ba866..16e9b8a3b488 100644 --- a/src/cli/update-cli/update-command-service-recovery.ts +++ b/src/cli/update-cli/update-command-service-recovery.ts @@ -32,6 +32,7 @@ type PostUpdateGatewayHealthRecoveryDeps = { }; export async function recoverLaunchAgentAndRecheckGatewayHealth(params: { + preserveDefinition?: boolean; health: GatewayRestartSnapshot; service: GatewayService; port: number; @@ -43,7 +44,7 @@ export async function recoverLaunchAgentAndRecheckGatewayHealth(params: { health: GatewayRestartSnapshot; launchAgentRecovery: PostUpdateLaunchAgentRecoveryResult | null; }> { - if (params.health.healthy) { + if (params.health.healthy || params.preserveDefinition) { return { health: params.health, launchAgentRecovery: null }; } diff --git a/src/cli/update-cli/update-command-service.integration.test.ts b/src/cli/update-cli/update-command-service.integration.test.ts new file mode 100644 index 000000000000..e87e5c9fc709 --- /dev/null +++ b/src/cli/update-cli/update-command-service.integration.test.ts @@ -0,0 +1,1015 @@ +// Keep the real lifecycle/version guards across the old-parent and fresh-CLI boundaries. +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { Command } from "commander"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { clearConfigCache, clearRuntimeConfigSnapshot } from "../../config/config.js"; +import { stampConfigWriteMetadata } from "../../config/io.meta.js"; +import { buildLaunchAgentPlist } from "../../daemon/launchd-plist.js"; +import { + resolveLaunchAgentPlistPath, + resolveLaunchAgentEnvFilePath, + resolveLaunchAgentEnvWrapperPath, +} from "../../daemon/launchd-service-files.js"; +import { readGatewayServiceState, resolveGatewayService } from "../../daemon/service.js"; +import { captureEnv } from "../../test-utils/env.js"; +import { mockProcessPlatform } from "../../test-utils/vitest-spies.js"; +import { VERSION } from "../../version.js"; +import { runDaemonRestart } from "../daemon-cli/lifecycle.js"; +import { addGatewayServiceCommands } from "../daemon-cli/register-service-commands.js"; +import * as startRepair from "../daemon-cli/start-repair.js"; +import { + assertGatewayServiceManagementAllowedForUpdate, + maybeRestartService, + maybeRestartServiceAfterFailedMutableUpdate, + maybeStopManagedServiceBeforeMutableUpdate, + revalidateManagedGatewayServiceAfterUpdate, +} from "./update-command-service.js"; + +const mocks = vi.hoisted(() => ({ + launchctl: vi.fn(), + handoff: + vi.fn< + typeof import("../../daemon/launchd-restart-handoff.js").scheduleDetachedLaunchdRestartHandoff + >(), + inLaunchd: false, + terminateStale: vi.fn(async (pids: number[]) => pids), + running: true, + loaded: true, + listenerPids: vi.fn(() => [4242]), + ports: vi.fn(), + probe: vi.fn(), + signal: vi.fn(), + events: [] as string[], + command: vi.fn(), + restart: vi.fn(async () => { + mocks.events.push("native restart"); + mocks.running = true; + return { outcome: "completed" as const }; + }), + start: vi.fn(), + install: vi.fn(), + script: vi.fn(), + child: vi.fn(), + health: vi.fn(), + doctor: vi.fn(), + error: vi.fn(), + log: vi.fn(), + capability: + vi.fn< + typeof import("../../daemon/systemd-definition-mutation.js").readSystemdDefinitionMutationCapability + >(), +})); + +vi.mock( + "../daemon-cli/lifecycle.runtime.js", + async () => await import("../daemon-cli/lifecycle.js"), +); + +vi.mock("../../daemon/launchd-exec.js", async (importOriginal) => ({ + ...(await importOriginal()), + execLaunchctl: mocks.launchctl, +})); +vi.mock("../../daemon/launchd-current-service.js", () => ({ + isCurrentProcessLaunchdServiceLabel: () => mocks.inLaunchd, +})); +vi.mock("../../daemon/launchd-restart-handoff.js", () => ({ + scheduleDetachedLaunchdRestartHandoff: mocks.handoff, +})); +vi.mock("../../daemon/launchd-system.js", async (importOriginal) => ({ + ...(await importOriginal()), + assertNoSystemLaunchDaemonOwnership: async () => {}, + inspectSystemLaunchDaemonOwnership: async (label: string) => ({ + status: "absent", + serviceTarget: `system/${label}`, + }), +})); +vi.mock("../../infra/restart-stale-pids.js", async (importOriginal) => ({ + ...(await importOriginal()), + cleanStaleGatewayProcessesSync: () => [], + terminateStaleGatewayPids: mocks.terminateStale, +})); +vi.mock("../../infra/ports-inspect.js", () => ({ + inspectPortUsage: mocks.ports, +})); + +vi.mock("../../gateway/probe.js", () => ({ probeGateway: mocks.probe })); + +vi.mock("../../daemon/systemd.js", async (importOriginal) => ({ + ...(await importOriginal()), + readSystemdServiceExecStart: mocks.command, + readSystemdServiceRuntime: async () => ({ status: mocks.running ? "running" : "stopped" }), + isSystemdServiceEnabled: async () => mocks.loaded, + findInstalledSystemdGatewayScope: async () => null, + isSystemdUserServiceAvailable: async () => true, + stopSystemdService: async () => { + mocks.events.push("native stop"); + mocks.running = false; + }, + restartSystemdService: mocks.restart, + startSystemdService: mocks.start, + installSystemdService: mocks.install, +})); +vi.mock("../../daemon/systemd-definition-mutation.js", () => ({ + readSystemdDefinitionMutationCapability: mocks.capability, +})); +vi.mock("../../process/exec.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCommandWithTimeout: mocks.child, +})); +vi.mock("../../infra/gateway-processes.js", async (importOriginal) => ({ + ...(await importOriginal()), + findVerifiedGatewayListenerPidsOnPortSync: mocks.listenerPids, + signalVerifiedGatewayPidSync: mocks.signal, +})); +vi.mock("../../commands/doctor.js", () => ({ doctorCommand: mocks.doctor })); +vi.mock("./restart-helper.js", () => ({ runRestartScript: mocks.script })); +vi.mock("../../runtime.js", () => ({ + defaultRuntime: { log: mocks.log, error: mocks.error, exit: vi.fn(), writeJson: vi.fn() }, +})); +vi.mock("../daemon-cli/restart-health.js", async (importOriginal) => ({ + ...(await importOriginal()), + waitForGatewayHealthyRestart: mocks.health, +})); +vi.mock("../daemon-cli/lifecycle-audit.js", () => ({ + appendServiceLifecycleRepairAudit: vi.fn(), + createServiceLifecycleMutationAudit: vi.fn(), + createGatewayLifecycleMutationAudit: vi.fn(), +})); + +let root: string; +let configPath: string; +let envSnapshot: ReturnType; +async function writeConfig(version: string) { + await fs.writeFile( + configPath, + JSON.stringify(stampConfigWriteMetadata({ gateway: { port: 19001 } }, undefined, version)), + ); + clearConfigCache(); + clearRuntimeConfigSnapshot(); +} + +beforeEach(async () => { + vi.clearAllMocks(); + mockProcessPlatform("linux"); + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-activation-"))); + vi.spyOn(os, "userInfo").mockReturnValue({ ...os.userInfo(), homedir: root }); + const keys = [ + "HOME", + "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", + "OPENCLAW_CONFIG_PATH", + "OPENCLAW_PROFILE", + "OPENCLAW_GATEWAY_PORT", + "OPENCLAW_SERVICE_MARKER", + "OPENCLAW_SERVICE_KIND", + "OPENCLAW_SUPERVISOR_MODE", + "OPENCLAW_SYSTEMD_UNIT", + "OPENCLAW_LAUNCHD_LABEL", + "OPENCLAW_UPDATE_IN_PROGRESS", + "OPENCLAW_UPDATE_PARENT_ALLOWS_GATEWAY_SERVICE_REPAIR", + "OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS", + ]; + envSnapshot = captureEnv(keys); + for (const key of keys) { + delete process.env[key]; + } + process.env.HOME = root; + configPath = path.join(root, ".openclaw", "openclaw.json"); + await fs.mkdir(path.dirname(configPath)); + await fs.mkdir(path.join(root, "dist")); + await fs.writeFile( + path.join(root, "package.json"), + JSON.stringify({ name: "openclaw", version: VERSION }), + ); + await fs.writeFile(path.join(root, "dist", "index.js"), "export {};\n"); + await writeConfig(VERSION); + mocks.ports.mockImplementation(async (port) => ({ + port, + status: "free", + listeners: [], + hints: [], + })); + mocks.probe.mockReset(); + mocks.running = true; + mocks.loaded = true; + mocks.inLaunchd = false; + mocks.launchctl.mockImplementation(async () => { + throw new Error("Unexpected native control in fixture"); + }); + mocks.handoff.mockReturnValue({ ok: true, value: Promise.resolve(true) }); + mocks.events = []; + mocks.capability.mockResolvedValue({ kind: "sealed", detail: "operator-owned definition" }); + mocks.command.mockResolvedValue({ + programArguments: [ + process.execPath, + path.join(root, "dist", "index.js"), + "gateway", + "--port", + "19305", + ], + environment: { HOME: root }, + sourcePath: "/etc/systemd/system/openclaw-gateway.service", + }); + mocks.child.mockImplementation(async (args) => { + if (!args.includes("restart")) { + throw new Error("Unexpected subprocess in activation fixture"); + } + mocks.events.push("fresh CLI restart"); + mocks.running = true; + return { code: 0, stdout: "", stderr: "", signal: null, killed: false, termination: "exit" }; + }); + mocks.health.mockImplementation(async ({ port }) => ({ + healthy: true, + staleGatewayPids: [], + runtime: { status: mocks.running ? "running" : "stopped" }, + portUsage: { port, status: "busy", listeners: [], hints: [] }, + })); +}); +afterEach(async () => { + envSnapshot.restore(); + clearConfigCache(); + clearRuntimeConfigSnapshot(); + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); +}); + +describe("preserved update activation with real version guards", () => { + it.each([ + ...( + [ + { mode: "git", outcome: "healthy" }, + { mode: "npm", outcome: "healthy" }, + { mode: "npm", outcome: "stale retry" }, + ] as const + ).map(({ mode, outcome }) => ({ + mode, + outcome, + denial: "sealed" as const, + channel: "stable" as const, + phase: "initial", + })), + ...(["git", "npm", "pnpm", "bun"] as const).flatMap((mode) => + (["sealed", "unknown"] as const).flatMap((denial) => + (mode === "git" || mode === "npm" + ? ["healthy", "json denial", "stale retry", "uninspectable", "foreign"] + : ["healthy"] + ).map((outcome) => ({ + mode, + denial, + outcome, + channel: "stable" as const, + phase: "late", + })), + ), + ), + ...(["sealed", "unknown"] as const).flatMap((denial) => + ["initial", "late"].flatMap((phase) => + ["healthy", "stale build", "missing build", "stale retry"].map((outcome) => ({ + mode: "git" as const, + denial, + outcome, + channel: "dev" as const, + phase, + })), + ), + ), + ])( + "handles $phase $denial denial for $channel $mode activation ($outcome)", + async ({ mode, denial, outcome, channel, phase }) => { + const late = phase === "late"; + const serviceCommand = await mocks.command(process.env); + if (!serviceCommand) { + throw new Error("missing fixture command"); + } + mocks.command.mockResolvedValue({ + ...serviceCommand, + environment: { HOME: root, MANAGED_VALUE: "revalidated" }, + }); + mocks.capability.mockResolvedValue( + late ? { kind: "writable" } : { kind: denial, detail: "owner denial" }, + ); + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + updateInstallKind: mode === "git" ? "git" : "package", + root, + shouldRestart: true, + jsonMode: true, + }); + expect(before.serviceUpdateVerdict).toMatchObject({ kind: "owned", refreshDefinition: late }); + const repair = vi.spyOn(startRepair, "repairLoadedGatewayServiceForStart"); + mocks.child.mockImplementation(async (args) => { + if (args.includes("install")) { + mocks.capability.mockResolvedValue({ kind: denial, detail: "late owner denial" }); + if (outcome === "uninspectable") { + mocks.command.mockRejectedValue(new Error("manager inspection failed")); + } else if (outcome === "foreign") { + const command = await mocks.command(process.env); + mocks.command.mockResolvedValue({ + ...command, + programArguments: ["/foreign/openclaw", "gateway"], + }); + } + return { + code: 1, + stdout: + outcome === "json denial" + ? JSON.stringify({ + ok: false, + error: `SERVICE_DEFINITION_${denial.toUpperCase()}: late owner denial`, + }) + : "", + stderr: + outcome === "json denial" + ? "runtime warning" + : `SERVICE_DEFINITION_${denial.toUpperCase()}: late owner denial`, + signal: null, + killed: false, + termination: "exit", + }; + } + const program = new Command().exitOverride(); + addGatewayServiceCommands(program.command("gateway")); + await program.parseAsync(args.slice(2), { from: "user" }); + return { + code: 0, + stdout: "", + stderr: "", + signal: null, + killed: false, + termination: "exit", + }; + }); + const commandBefore = await mocks.command(process.env); + mocks.ports.mockImplementation(async (port) => ({ + port, + status: "busy", + listeners: [{ pid: 4242, command: "openclaw-gateway" }], + hints: [], + })); + mocks.probe.mockImplementation(async ({ url }) => ({ + ok: true, + url, + connectLatencyMs: 1, + error: null, + close: null, + auth: { role: "operator", scopes: ["operator.read"], capability: "read_only" }, + server: { + version: VERSION, + connId: "fixture", + ...(outcome === "missing build" + ? {} + : { buildId: outcome === "stale build" ? "old-build" : "new-build" }), + }, + health: {}, + status: {}, + presence: [], + configSnapshot: null, + })); + const { waitForGatewayHealthyRestart } = await vi.importActual< + typeof import("../daemon-cli/restart-health.js") + >("../daemon-cli/restart-health.js"); + let retried = false; + mocks.health.mockImplementation(async (params) => { + const stale = outcome === "stale retry" && params.expectedVersion === VERSION && !retried; + retried ||= stale; + if (stale) { + return { + healthy: false, + staleGatewayPids: [4242], + runtime: { status: "running" }, + portUsage: { port: params.port, status: "busy", listeners: [], hints: [] }, + }; + } + return await waitForGatewayHealthyRestart(params); + }); + const activated = await maybeRestartService({ + channel, + shouldRestart: true, + result: { + status: "ok", + mode, + root, + steps: [], + durationMs: 0, + before: { version: "2026.1.1" }, + after: { version: VERSION, buildId: "new-build" }, + }, + opts: { json: outcome === "json denial" || (!late && channel === "stable") }, + refreshServiceEnv: late, + serviceUpdateVerdict: before.serviceUpdateVerdict, + serviceEnv: before.serviceEnv, + gatewayPort: late ? 19001 : 19305, + requireRunningServiceAfterRestart: true, + restartScriptPath: "/fixture/prepared-restart.sh", + timeoutMs: 1000, + }); + const allowed = !["uninspectable", "foreign"].includes(outcome); + const buildMismatch = ["stale build", "missing build"].includes(outcome); + expect(activated).toBe(allowed && !buildMismatch); + const restarts = mocks.child.mock.calls.filter(([args]) => args.includes("restart")); + expect(restarts).toHaveLength(allowed ? (retried ? 2 : 1) : 0); + for (const [args, options] of restarts) { + expect(args).toContain("--preserve-definition"); + expect(typeof options === "object" && options.env?.MANAGED_VALUE).toBe("revalidated"); + if (!late && channel === "stable") { + expect(args).toContain("--json"); + } + } + expect(mocks.start).not.toHaveBeenCalled(); + expect(mocks.child.mock.calls.filter(([args]) => args.includes("install"))).toHaveLength( + late ? 1 : 0, + ); + expect(mocks.restart).toHaveBeenCalledTimes(restarts.length); + expect(mocks.health.mock.calls.every(([args]) => args.port === 19305)).toBe(true); + if (allowed) { + expect(mocks.health.mock.calls.some(([args]) => args.expectedVersion === VERSION)).toBe( + true, + ); + } + if (retried) { + expect(mocks.terminateStale).toHaveBeenCalledWith([4242]); + } + if (allowed) { + expect(await mocks.command(process.env)).toEqual(commandBefore); + const verification = mocks.health.mock.calls.filter( + ([args]) => args.expectedVersion === VERSION, + ); + expect(verification.length).toBe(retried ? 2 : 1); + expect(verification.every(([args]) => args.requireRunningService === true)).toBe(true); + expect( + verification.every( + ([args]) => args.expectedBuildId === (channel === "dev" ? "new-build" : undefined), + ), + ).toBe(true); + expect(mocks.probe.mock.calls.every(([args]) => args.url === "ws://127.0.0.1:19305")).toBe( + true, + ); + } + if (buildMismatch) { + expect( + mocks.error.mock.calls.flat().concat(mocks.log.mock.calls.flat()).join("\n"), + ).toContain( + `Gateway build mismatch: expected new-build, running gateway reported ${outcome === "missing build" ? "unavailable" : "old-build"}.`, + ); + } + expect(repair).not.toHaveBeenCalled(); + expect(mocks.script).not.toHaveBeenCalled(); + expect(mocks.install).not.toHaveBeenCalled(); + expect(mocks.doctor).not.toHaveBeenCalled(); + }, + ); + + it("rejects a target without preservation support before automatic repair, with a JSON diagnostic", async () => { + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + updateInstallKind: "package", + root, + shouldRestart: true, + jsonMode: true, + }); + expect(before.stopped).toBe(true); + // Permissions can change after the original preservation verdict. + mocks.capability.mockResolvedValue({ kind: "writable" }); + const repair = vi + .spyOn(startRepair, "repairLoadedGatewayServiceForStart") + .mockRejectedValue(new Error("automatic repair reached")); + mocks.child.mockImplementation(async (args, options) => { + const snapshot = captureEnv([ + "OPENCLAW_UPDATE_IN_PROGRESS", + "OPENCLAW_UPDATE_PARENT_ALLOWS_GATEWAY_SERVICE_REPAIR", + ]); + if (typeof options === "object") { + for (const key of [ + "OPENCLAW_UPDATE_IN_PROGRESS", + "OPENCLAW_UPDATE_PARENT_ALLOWS_GATEWAY_SERVICE_REPAIR", + ]) { + const value = options.env?.[key]; + if (value !== undefined) { + process.env[key] = value; + } + } + } + let stderr = ""; + const program = new Command().exitOverride().configureOutput({ + writeErr: (text) => { + stderr += text; + }, + }); + // A target without this option must reject before its normal restart action. + program + .command("gateway") + .command("restart") + .option("--json") + .action(async (opts: { json?: boolean }) => { + await runDaemonRestart(opts); + }); + try { + await program.parseAsync(args.slice(2), { from: "user" }); + return { code: 0, stdout: "", stderr, signal: null, killed: false, termination: "exit" }; + } catch (error) { + if (!(error instanceof Error) || !error.message.includes("unknown option")) { + throw error; + } + return { code: 1, stdout: "", stderr, signal: null, killed: false, termination: "exit" }; + } finally { + snapshot.restore(); + } + }); + const activated = await maybeRestartService({ + channel: "stable", + shouldRestart: true, + result: { + status: "ok", + mode: "npm", + root, + steps: [], + durationMs: 0, + after: { version: VERSION }, + }, + opts: { json: true }, + refreshServiceEnv: false, + serviceUpdateVerdict: before.serviceUpdateVerdict, + serviceEnv: before.serviceEnv, + gatewayPort: 19305, + requireRunningServiceAfterRestart: true, + timeoutMs: 1000, + }); + expect(repair).not.toHaveBeenCalled(); + expect(activated).toBe(false); + expect(mocks.error.mock.calls.flat().join("\n")).toContain("unknown option"); + expect(mocks.error.mock.calls.flat().join("\n")).toContain("stopped"); + expect(mocks.restart).not.toHaveBeenCalled(); + expect(mocks.health).not.toHaveBeenCalled(); + }); + + it.each(["foreign", "metadata", "unit", "unavailable", "replacement root", "profile"])( + "revalidates writable failed-update recovery after %s changes", + async (change) => { + mocks.capability.mockResolvedValue({ kind: "writable" }); + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + updateInstallKind: "package", + root, + shouldRestart: true, + jsonMode: true, + }); + expect(before.stopped).toBe(true); + const command = await mocks.command(process.env); + if (!command) { + throw new Error("missing fixture command"); + } + if (change === "unavailable") { + mocks.command.mockRejectedValue(new Error("manager unavailable")); + } else { + const foreign = path.join(root, "foreign"); + await fs.mkdir(path.join(foreign, "dist"), { recursive: true }); + await fs.writeFile( + path.join(foreign, "package.json"), + JSON.stringify({ name: "openclaw", version: VERSION }), + ); + await fs.writeFile(path.join(foreign, "dist", "index.js"), "export {};\n"); + mocks.command.mockResolvedValue({ + ...command, + programArguments: [ + process.execPath, + path.join( + ["foreign", "replacement root"].includes(change) ? foreign : root, + "dist", + "index.js", + ), + "gateway", + "--port", + "19002", + ], + environment: { + HOME: root, + OPENCLAW_PROFILE: "default", + OPENCLAW_STATE_DIR: path.dirname(configPath), + OPENCLAW_CONFIG_PATH: configPath, + OPENCLAW_SYSTEMD_UNIT: + change === "unit" ? "openclaw-other.service" : "openclaw-gateway.service", + ...(change === "profile" + ? { + OPENCLAW_PROFILE: "second", + OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway-second.service", + OPENCLAW_STATE_DIR: path.join(root, ".openclaw-second"), + OPENCLAW_CONFIG_PATH: path.join(root, ".openclaw-second", "openclaw.json"), + } + : {}), + }, + }); + } + mocks.events.push("update failed after definition changed"); + await maybeRestartServiceAfterFailedMutableUpdate({ + root: change === "replacement root" ? path.join(root, "foreign") : undefined, + preManagedServiceStop: before, + jsonMode: true, + }); + if (change === "metadata" || change === "replacement root") { + expect(mocks.restart).toHaveBeenCalledOnce(); + } else { + expect(mocks.restart).not.toHaveBeenCalled(); + expect(mocks.error.mock.calls.flat().join("\n")).toContain("Failed to restart"); + expect(mocks.events).toEqual(["native stop", "update failed after definition changed"]); + } + }, + ); + + it.each(["metadata", "profile", "unit"])( + "pins writable service identity across %s changes", + async (change) => { + mocks.capability.mockResolvedValue({ kind: "writable" }); + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + updateInstallKind: "package", + root, + shouldRestart: true, + jsonMode: true, + }); + expect(before.stopped).toBe(true); + expect(before.serviceEnv?.OPENCLAW_SYSTEMD_UNIT).toBeUndefined(); + const command = await mocks.command(process.env); + if (!command) { + throw new Error("missing fixture command"); + } + mocks.command.mockResolvedValue({ + ...command, + programArguments: [ + process.execPath, + path.join(root, "dist", "index.js"), + "gateway", + "--port", + "19002", + ], + environment: { + HOME: root, + OPENCLAW_GATEWAY_PORT: "19002", + ...(change === "profile" + ? { + OPENCLAW_PROFILE: "second", + OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway-second.service", + OPENCLAW_STATE_DIR: path.join(root, ".openclaw-second"), + OPENCLAW_CONFIG_PATH: path.join(root, ".openclaw-second", "openclaw.json"), + } + : { + OPENCLAW_PROFILE: "default", + OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service", + OPENCLAW_STATE_DIR: path.join(root, ".openclaw"), + OPENCLAW_CONFIG_PATH: configPath, + }), + ...(change === "unit" + ? { OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway-custom.service" } + : {}), + }, + }); + const state = await readGatewayServiceState(resolveGatewayService(), { + env: before.serviceEnv, + requireEffective: true, + validateEnvBeforeStatusRead: assertGatewayServiceManagementAllowedForUpdate, + }); + const revalidated = revalidateManagedGatewayServiceAfterUpdate({ + state, + root, + preManagedServiceStop: before, + }); + if (change !== "metadata") { + expect(state.env.OPENCLAW_SYSTEMD_UNIT).toBe( + change === "profile" + ? "openclaw-gateway-second.service" + : "openclaw-gateway-custom.service", + ); + await expect(revalidated).rejects.toThrow("manager identity changed"); + } else { + await expect(revalidated).resolves.toMatchObject({ + kind: "owned", + refreshDefinition: true, + }); + } + expect(mocks.events).toEqual(["native stop"]); + }, + ); + + it.each(["git", "npm"] as const)( + "delegates %s activation after candidate doctor stamps newer config", + async (mode) => { + const before = await maybeStopManagedServiceBeforeMutableUpdate({ + updateInstallKind: mode === "git" ? "git" : "package", + root, + shouldRestart: true, + jsonMode: true, + }); + expect(before.stopped).toBe(true); + mocks.events.push("core updated"); + await writeConfig("9999.1.1"); + mocks.events.push("candidate doctor stamped config"); + const service = resolveGatewayService(); + const state = await readGatewayServiceState(service, { requireEffective: true }); + const verdict = await revalidateManagedGatewayServiceAfterUpdate({ + state, + root, + preManagedServiceStop: before, + }); + + const activated = await maybeRestartService({ + channel: "stable", + shouldRestart: true, + result: { + status: "ok", + mode, + root, + steps: [], + durationMs: 0, + before: { version: VERSION }, + after: { version: "9999.1.1" }, + }, + opts: {}, + refreshServiceEnv: false, + serviceUpdateVerdict: verdict, + serviceEnv: state.env, + gatewayPort: 19305, + requireRunningServiceAfterRestart: true, + timeoutMs: 1000, + }); + + expect(activated, mocks.log.mock.calls.flat().join("\n")).toBe(true); + expect(mocks.events).toEqual([ + "native stop", + "core updated", + "candidate doctor stamped config", + "fresh CLI restart", + ]); + const child = mocks.child.mock.calls[0]; + expect(child?.[0].slice(1)).toEqual([ + path.join(root, "dist", "index.js"), + "gateway", + "restart", + "--preserve-definition", + ]); + expect(mocks.health.mock.calls[0]?.[0]).toMatchObject({ + port: 19305, + expectedVersion: "9999.1.1", + requireRunningService: true, + }); + expect(mocks.start).not.toHaveBeenCalled(); + expect(mocks.restart).not.toHaveBeenCalled(); + expect(mocks.doctor).not.toHaveBeenCalled(); + // The old adapter still refuses the same config: delegation must not weaken its guard. + await expect(service.restart({ env: state.env, stdout: process.stdout })).rejects.toThrow( + "older than the config", + ); + }, + ); + + it.each(["sealed", "writable"] as const)( + "fresh restart keeps the preserved launcher even when authority is %s", + async (kind) => { + mocks.capability.mockResolvedValue( + kind === "sealed" ? { kind, detail: "operator-owned definition" } : { kind }, + ); + await expect(runDaemonRestart({ json: true, preserveDefinition: true })).resolves.toBe(true); + expect(mocks.restart).toHaveBeenCalledOnce(); + expect(mocks.install).not.toHaveBeenCalled(); + expect(mocks.health.mock.calls[0]?.[0].port).toBe(19305); + }, + ); + + it.each(["missing", "uninspectable", "disappeared after inspection"])( + "does not fall through to an unmanaged listener when the selected service is %s", + async (scenario) => { + if (scenario === "uninspectable") { + mocks.command.mockRejectedValue(new Error("manager unavailable")); + await expect(runDaemonRestart({ json: true, preserveDefinition: true })).rejects.toThrow( + "manager unavailable", + ); + } else if (scenario === "missing") { + mocks.command.mockResolvedValue(null); + await expect(runDaemonRestart({ json: true, preserveDefinition: true })).rejects.toThrow( + "could not be inspected", + ); + } else { + const command = await mocks.command(process.env); + mocks.command.mockResolvedValueOnce(command).mockResolvedValue(null); + mocks.loaded = false; + await expect(runDaemonRestart({ json: true, preserveDefinition: true })).resolves.toBe( + false, + ); + } + expect(mocks.restart).not.toHaveBeenCalled(); + expect(mocks.install).not.toHaveBeenCalled(); + expect(mocks.listenerPids).not.toHaveBeenCalled(); + expect(mocks.signal).not.toHaveBeenCalled(); + expect(mocks.health).not.toHaveBeenCalled(); + }, + ); + + it.each(["safe", "external"])( + "refuses preserved activation through %s process signaling", + async (mode) => { + if (mode === "external") { + process.env.OPENCLAW_SUPERVISOR_MODE = "external"; + } + await expect( + runDaemonRestart({ preserveDefinition: true, safe: mode === "safe" }), + ).rejects.toThrow(); + expect(mocks.restart).not.toHaveBeenCalled(); + expect(mocks.listenerPids).not.toHaveBeenCalled(); + expect(mocks.signal).not.toHaveBeenCalled(); + }, + ); + + it.each([ + "loaded", + "unloaded", + "unloaded demand", + "start demand", + "handoff", + "bootstrap denied", + "parent unhealthy", + "parent late sealed", + "parent late unknown", + "stale retry", + ])("preserves actual LaunchAgent artifacts during %s activation", async (scenario) => { + mockProcessPlatform("darwin"); + const label = "ai.openclaw.gateway"; + const plistPath = resolveLaunchAgentPlistPath(process.env); + const envPath = resolveLaunchAgentEnvFilePath(process.env, label); + const wrapperPath = resolveLaunchAgentEnvWrapperPath(process.env, label); + const demandOnly = scenario.endsWith("demand"); + let plist = buildLaunchAgentPlist({ + label, + programArguments: [ + process.execPath, + path.join(root, "dist", "index.js"), + "gateway", + "--port", + "19305", + ], + stdoutPath: path.join(root, "gateway.log"), + stderrPath: path.join(root, "gateway.err"), + environment: { + HOME: root, + OPENCLAW_GATEWAY_TOKEN: "fixture-inline-token", + OPENCLAW_SERVICE_VERSION: "legacy", + }, + }); + if (demandOnly) { + plist = plist.replace(/((?:RunAtLoad|KeepAlive)<\/key>\s*)/g, "$1"); + expect(plist.match(/(?:RunAtLoad|KeepAlive)<\/key>\s*/g)).toHaveLength(2); + } + const plistMode = scenario === "bootstrap denied" ? 0o400 : 0o444; + for (const [file, content, mode] of [ + [plistPath, plist, plistMode], + [envPath, "EXISTING=env\n", 0o600], + [wrapperPath, "#!/bin/sh\n# existing wrapper\n", 0o700], + ] as const) { + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, content, { mode }); + } + const snapshot = async () => + Promise.all( + [plistPath, envPath, wrapperPath, configPath].map(async (file) => ({ + file, + bytes: await fs.readFile(file), + mode: (await fs.stat(file)).mode, + })), + ); + const before = await snapshot(); + const writeFile = vi.spyOn(fs, "writeFile"); + const chmod = vi.spyOn(fs, "chmod"); + const rename = vi.spyOn(fs, "rename"); + mocks.inLaunchd = scenario === "handoff"; + let loaded = ["loaded", "handoff", "stale retry"].includes(scenario); + let nativeRunning = loaded; + mocks.launchctl.mockImplementation(async (args) => { + if (args[0] === "bootstrap") { + if (scenario === "bootstrap denied") { + return { code: 13, stdout: "", stderr: "permission denied", termination: "exit" }; + } + loaded = true; + nativeRunning = !demandOnly; + } + if ((args[0] === "print" || args[0] === "kickstart") && !loaded) { + return { code: 113, stdout: "", stderr: "Could not find service", termination: "exit" }; + } + if (args[0] === "kickstart") { + nativeRunning = true; + } + const state = nativeRunning ? "running" : "stopped"; + return { + code: 0, + stdout: args[0] === "print" ? `state = ${state}\n` : "", + stderr: "", + termination: "exit", + }; + }); + if (demandOnly) { + mocks.health.mockImplementation(async ({ port }) => ({ + healthy: nativeRunning, + staleGatewayPids: [], + runtime: { status: nativeRunning ? "running" : "stopped" }, + portUsage: { port, status: nativeRunning ? "busy" : "free", listeners: [], hints: [] }, + })); + } + if (scenario === "stale retry") { + mocks.health.mockResolvedValueOnce({ + healthy: false, + staleGatewayPids: [4242], + runtime: { status: "stopped" }, + portUsage: { port: 19305, status: "busy", listeners: [], hints: [] }, + }); + } + let result: boolean; + if (scenario === "start demand") { + await resolveGatewayService().start({ env: process.env, stdout: process.stdout }); + result = nativeRunning; + } else if (scenario.startsWith("parent")) { + const lateDenial = scenario.startsWith("parent late"); + const verdict = lateDenial + ? await revalidateManagedGatewayServiceAfterUpdate({ + state: await readGatewayServiceState(resolveGatewayService(), { + requireEffective: true, + }), + root, + }) + : { kind: "unresolved" as const, root, fingerprint: "fixture" }; + if (lateDenial) { + expect(verdict).toMatchObject({ kind: "owned", refreshDefinition: true }); + mocks.child.mockResolvedValueOnce({ + code: 1, + stdout: "", + stderr: `SERVICE_DEFINITION_${scenario.endsWith("sealed") ? "SEALED" : "UNKNOWN"}: late denial`, + signal: null, + killed: false, + termination: "exit", + }); + } + mocks.health.mockImplementation(async ({ port }) => ({ + healthy: false, + staleGatewayPids: [], + runtime: { status: "stopped" }, + portUsage: { port, status: "free", listeners: [], hints: [] }, + })); + result = await maybeRestartService({ + channel: "stable", + shouldRestart: true, + result: { + status: "ok", + mode: "npm", + root, + steps: [], + durationMs: 0, + after: { version: VERSION }, + }, + opts: { json: true }, + refreshServiceEnv: lateDenial, + serviceUpdateVerdict: verdict, + serviceEnv: process.env, + gatewayPort: lateDenial ? 19001 : 19305, + restartScriptPath: "/fixture/prepared-restart.sh", + requireRunningServiceAfterRestart: true, + timeoutMs: 1000, + }); + expect(mocks.error.mock.calls.flat().join("\n")).toContain("did not become healthy"); + expect(mocks.health.mock.calls.every(([args]) => args.port === 19305)).toBe(true); + expect(mocks.script).not.toHaveBeenCalled(); + expect(mocks.doctor).not.toHaveBeenCalled(); + } else { + result = await runDaemonRestart({ json: true, preserveDefinition: true }); + } + expect(result).toBe(scenario !== "bootstrap denied" && !scenario.startsWith("parent")); + expect(await snapshot()).toEqual(before); + expect(writeFile).not.toHaveBeenCalled(); + expect(chmod).not.toHaveBeenCalled(); + expect(rename).not.toHaveBeenCalled(); + if (demandOnly || scenario === "unloaded") { + const calls = mocks.launchctl.mock.calls.map(([args]) => args); + const afterBootstrap = calls.slice(calls.findIndex((args) => args[0] === "bootstrap") + 1); + const target = `gui/${process.getuid?.() ?? 501}/${label}`; + expect(afterBootstrap).toContainEqual(["kickstart", target]); + expect(afterBootstrap).not.toContainEqual(["kickstart", "-k", target]); + } + if (scenario === "stale retry") { + expect(mocks.terminateStale).toHaveBeenCalledWith([4242]); + expect(mocks.launchctl.mock.calls.filter(([args]) => args[0] === "kickstart")).toHaveLength( + 2, + ); + expect(mocks.health).toHaveBeenCalledTimes(2); + } + if (scenario.startsWith("parent")) { + expect(mocks.launchctl.mock.calls.every(([args]) => args[0] === "print")).toBe(true); + } else if (scenario === "handoff") { + expect(mocks.handoff).toHaveBeenCalledWith(expect.objectContaining({ mode: "kickstart" })); + } else { + expect(mocks.launchctl.mock.calls.some(([args]) => args[0] === "kickstart")).toBe(true); + expect(mocks.launchctl.mock.calls.some(([args]) => args[0] === "bootstrap")).toBe( + !["loaded", "stale retry"].includes(scenario), + ); + } + }); + + it("fresh restart still rejects config newer than its own binary", async () => { + await writeConfig("9999.1.1"); + await expect(runDaemonRestart({ json: true, preserveDefinition: true })).resolves.toBe(false); + expect(mocks.restart).not.toHaveBeenCalled(); + expect(mocks.install).not.toHaveBeenCalled(); + expect(mocks.health).not.toHaveBeenCalled(); + }); +}); diff --git a/src/cli/update-cli/update-command-service.ts b/src/cli/update-cli/update-command-service.ts index 4ba480622d7b..5165a4a2399d 100644 --- a/src/cli/update-cli/update-command-service.ts +++ b/src/cli/update-cli/update-command-service.ts @@ -5,6 +5,7 @@ import { Writable } from "node:stream"; import { confirm, isCancel } from "@clack/prompts"; import { parseStrictPositiveInteger } from "@openclaw/normalization-core/number-coercion"; import { err as resultError, ok, type Result } from "@openclaw/normalization-core/result"; +import { stableStringify } from "@openclaw/normalization-core/stable-stringify"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; import { stylePromptMessage } from "../../../packages/terminal-core/src/prompt-style.js"; import { theme } from "../../../packages/terminal-core/src/theme.js"; @@ -15,10 +16,17 @@ import { import { doctorCommand } from "../../commands/doctor.js"; import { UPDATE_PARENT_SUPPORTS_DOCTOR_CONFIG_WRITE_ENV } from "../../commands/doctor/shared/update-phase.js"; import { resolveGatewayPort } from "../../config/config.js"; +import { createConfigIO } from "../../config/io.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { GATEWAY_SERVICE_RUNTIME_PID_ENV, isGatewayServiceEnv } from "../../daemon/constants.js"; +import { + GATEWAY_SERVICE_RUNTIME_PID_ENV, + isGatewayServiceEnv, + resolveGatewayProfileSuffix, +} from "../../daemon/constants.js"; import { resolveGatewayInstallEntrypoint } from "../../daemon/gateway-entrypoint.js"; +import { resolveLaunchAgentLabel } from "../../daemon/launchd-label.js"; import { resolveGatewayRestartLogPath } from "../../daemon/restart-logs.js"; +import { resolveTaskName } from "../../daemon/schtasks-layout.js"; import { resumeScheduledTaskAutoStartAfterUpdate, suspendScheduledTaskAutoStartForUpdate, @@ -27,11 +35,15 @@ import { summarizeGatewayServiceLayout } from "../../daemon/service-layout.js"; import { resolveManagedGatewayServiceCommand, type GatewayServiceCommandConfig, + type GatewayServiceState, } from "../../daemon/service-types.js"; import { readGatewayServiceState, resolveGatewayService } from "../../daemon/service.js"; +import { resolveSystemdServiceName } from "../../daemon/systemd-service-files.js"; +import { sha256Hex } from "../../infra/crypto-digest.js"; import { assertGatewayServiceMutationAllowed } from "../../infra/gateway-supervision.js"; import { getSelfAndAncestorPidsSync } from "../../infra/restart-stale-pids.js"; import { nodeVersionSatisfiesEngine } from "../../infra/runtime-guard.js"; +import { parseTcpPortFromArgs } from "../../infra/tcp-port.js"; import type { UpdateChannel } from "../../infra/update-channels.js"; import { fetchNpmPackageTargetStatus } from "../../infra/update-check-package-target.js"; import { canResolveRegistryVersionForPackageTarget } from "../../infra/update-global.js"; @@ -68,6 +80,7 @@ export { isPackageManagerUpdateMode } from "./update-command-service-recovery.js const CLI_NAME = resolveCliName(); const SERVICE_REFRESH_TIMEOUT_MS = 60_000; +const DEFINITION_DENIAL = /\bSERVICE_DEFINITION_(?:SEALED|UNKNOWN):[^\n]*/; const POST_REFRESH_ALREADY_HEALTHY_ATTEMPTS = 10; const POST_REFRESH_ALREADY_HEALTHY_DELAY_MS = 500; const JSON_MODE_SERVICE_STDOUT = new Writable({ @@ -81,22 +94,15 @@ export function shouldPrepareUpdatedInstallRestart(params: { serviceInstalled: boolean; serviceLoaded: boolean; serviceStoppedForUpdate?: boolean; - serviceMatchesMutationRoot?: boolean; serviceMatchesUpdateRoot?: boolean; }): boolean { - if (params.serviceMatchesMutationRoot === false) { - return false; - } - if (isPackageManagerUpdateMode(params.updateMode)) { - return params.serviceInstalled; - } - if (params.updateMode === "git" && params.serviceStoppedForUpdate) { - return params.serviceInstalled; - } - if (params.updateMode === "git") { - return params.serviceLoaded && params.serviceMatchesUpdateRoot === true; - } - return params.serviceLoaded; + const useInstalledState = + isPackageManagerUpdateMode(params.updateMode) || + (params.updateMode === "git" && params.serviceStoppedForUpdate); + return useInstalledState + ? params.serviceInstalled + : params.serviceLoaded && + (params.updateMode !== "git" || params.serviceMatchesUpdateRoot === true); } export type PreManagedServiceStop = { @@ -106,13 +112,129 @@ export type PreManagedServiceStop = { running: boolean; serviceMutationAllowed?: boolean; serviceMutationSkipMessage?: string; - serviceMatchesMutationRoot?: boolean; + serviceUpdateVerdict?: ManagedGatewayUpdateVerdict; blockMessage?: string; serviceEnv?: NodeJS.ProcessEnv; serviceDefinitionEnv?: NodeJS.ProcessEnv; windowsTaskAutoStartRecovery?: WindowsTaskAutoStartRecovery; }; +export function resolvePreparedGatewayUpdatePolicy( + stopState: PreManagedServiceStop | undefined, + shouldRestart: boolean, +) { + const verdict = stopState?.serviceUpdateVerdict; + // Root ownership permits activation; rewriting also requires definition authority. + return { + allowGatewayServiceRepair: verdict?.kind === "owned" && verdict.refreshDefinition, + allowGatewayActivation: + shouldRestart && stopState?.stopped === true && verdict?.kind === "owned", + }; +} + +type ManagedGatewayUpdateVerdict = + | { kind: "absent" | "foreign" } + | { kind: "owned"; root: string; fingerprint: string; refreshDefinition: boolean } + | { kind: "unresolved"; root: string; fingerprint: string } + | { kind: "unavailable"; message: string }; + +async function inspectManagedGatewayServiceBeforeUpdate(params: { + root: string; + state: GatewayServiceState; +}): Promise { + const { state, root } = params; + const { command } = state; + const unavailable = (): ManagedGatewayUpdateVerdict => ({ + kind: "unavailable", + message: + "Gateway service management skipped: its owner or runtime could not be inspected. " + + "Code update can continue; run `openclaw gateway status --deep` and restart the gateway manually when service access is restored.", + }); + if (!command) { + return !state.installed && !state.running && state.runtime?.missingUnit + ? { kind: "absent" } + : unavailable(); + } + // Lifecycle authority follows the effective launcher, not the writable base + // that a drop-in may replace with a different installation. + const ownsRoot = await gatewayServiceCommandUsesRoot({ root, command }); + if (ownsRoot === false) { + return { kind: "foreign" }; + } + if ( + state.loadState.status === "unknown" || + (state.runtime?.status !== "running" && state.runtime?.status !== "stopped") + ) { + return unavailable(); + } + const serialized = stableStringify(command); + if (Buffer.byteLength(serialized) > 4 * 1024 * 1024) { + return unavailable(); + } + const fingerprint = sha256Hex(serialized); + return ownsRoot + ? { + kind: "owned", + root, + fingerprint, + refreshDefinition: (state.definitionMutationCapability?.kind ?? "writable") === "writable", + } + : { kind: "unresolved", root, fingerprint }; +} + +function matchesStoppedService( + before: Pick, + state: GatewayServiceState, + inspection: ManagedGatewayUpdateVerdict, +): boolean { + const verdict = before.serviceUpdateVerdict; + const refreshDefinition = verdict?.kind === "owned" && verdict.refreshDefinition; + const resolveName = + process.platform === "darwin" + ? resolveLaunchAgentLabel + : process.platform === "win32" + ? resolveTaskName + : resolveSystemdServiceName; + // Explicit default metadata selects the same manager; protected command hashes + // still pin the effective launcher and its environment through normalization. + return Boolean( + before.serviceEnv && + state.command && + verdict && + "fingerprint" in verdict && + resolveGatewayProfileSuffix(before.serviceEnv.OPENCLAW_PROFILE) === + resolveGatewayProfileSuffix(state.env.OPENCLAW_PROFILE) && + resolveName(before.serviceEnv) === resolveName(state.env) && + (refreshDefinition || + ("fingerprint" in inspection && inspection.fingerprint === verdict.fingerprint)), + ); +} + +export async function revalidateManagedGatewayServiceAfterUpdate(params: { + state: GatewayServiceState; + root: string; + preManagedServiceStop?: Pick; +}): Promise { + const before = params.preManagedServiceStop; + const verdict = before?.serviceUpdateVerdict; + assertGatewayServiceManagementAllowedForUpdate(params.state.env); + const inspection = await inspectManagedGatewayServiceBeforeUpdate(params); + if ( + before && + verdict && + (verdict.kind === "owned" || verdict.kind === "unresolved") && + (inspection.kind !== verdict.kind || !matchesStoppedService(before, params.state, inspection)) + ) { + throw new GatewayServiceUpdateOwnershipError( + "Gateway service ownership or manager identity changed; inspect it before restarting manually.", + undefined, + ); + } + return inspection.kind === "owned" && verdict?.kind === "owned" && !verdict.refreshDefinition + ? { ...inspection, refreshDefinition: false } + : inspection; +} + type WindowsTaskAutoStartRecovery = { suspended: Promise; restore: () => Promise; @@ -180,49 +302,28 @@ export type ManagedServiceRootRedirect = { nodeRunner?: string; }; -function formatGatewayAncestryBlockMessage(pid: number): string { - return `openclaw update detected it is running inside the gateway process tree. -Gateway PID ${pid} is an ancestor of this process, so this updater cannot safely stop or restart the gateway that owns it. -Run \`${replaceCliName(formatCliCommand("openclaw update"), CLI_NAME)}\` from a shell outside the gateway service, or stop the gateway service first and then update.`; -} - function parsePositivePid(value: unknown): number | null { - if (typeof value === "number" && Number.isFinite(value) && value > 0) { - return Math.floor(value); + if (typeof value === "number") { + return Number.isFinite(value) && value > 0 ? Math.floor(value) : null; } - if (typeof value !== "string") { - return null; - } - const trimmed = value.trim(); - if (!/^\d+$/u.test(trimmed)) { - return null; - } - return parseStrictPositiveInteger(trimmed) ?? null; -} - -function isInheritedGatewayRuntimePid( - pid: number, - env: Record = process.env, -): boolean { - if (!isRunningInsideGatewayService(env)) { - return false; - } - return parsePositivePid(env[GATEWAY_SERVICE_RUNTIME_PID_ENV]) === pid; -} - -function isGatewayAncestorPid( - pid: unknown, - env: Record = process.env, -): pid is number { - const parsed = parsePositivePid(pid); - if (parsed === null) { - return false; - } - return isInheritedGatewayRuntimePid(parsed, env) || getSelfAndAncestorPidsSync().has(parsed); + const trimmed = typeof value === "string" ? value.trim() : ""; + return /^\d+$/u.test(trimmed) ? (parseStrictPositiveInteger(trimmed) ?? null) : null; } function gatewayAncestryBlockMessage(pid: unknown): string | undefined { - return isGatewayAncestorPid(pid) ? formatGatewayAncestryBlockMessage(pid) : undefined; + const gatewayPid = parsePositivePid(pid); + if (gatewayPid === null) { + return undefined; + } + const inherited = + isGatewayServiceEnv(process.env) && + parsePositivePid(process.env[GATEWAY_SERVICE_RUNTIME_PID_ENV]) === gatewayPid; + if (!inherited && !getSelfAndAncestorPidsSync().has(gatewayPid)) { + return undefined; + } + return `openclaw update detected it is running inside the gateway process tree. +Gateway PID ${gatewayPid} is an ancestor of this process, so this updater cannot safely stop or restart the gateway that owns it. +Run \`${replaceCliName(formatCliCommand("openclaw update"), CLI_NAME)}\` from a shell outside the gateway service, or stop the gateway service first and then update.`; } function serviceControlStdoutForMode(jsonMode: boolean): NodeJS.WritableStream { @@ -364,174 +465,103 @@ export async function maybeStopManagedServiceBeforeMutableUpdate(params: { root: string; shouldRestart: boolean; jsonMode: boolean; + phase?: "inspect" | "prepare"; + timeoutMs?: number; }): Promise { + const uninspected = { stopped: false, inspected: false, runtimeInspected: false, running: false }; const serviceMutationSkipMessage = resolveGatewayServiceManagementBlockMessageForUpdate( process.env, ); if (serviceMutationSkipMessage) { - return { - stopped: false, - inspected: false, - runtimeInspected: false, - running: false, - serviceMutationAllowed: false, - serviceMutationSkipMessage, - }; + return { ...uninspected, serviceMutationAllowed: false, serviceMutationSkipMessage }; } let service: ReturnType; - let serviceState: Awaited>; + let serviceState: GatewayServiceState; try { service = resolveGatewayService(); serviceState = await readGatewayServiceState(service, { env: process.env, + requireEffective: true, validateEnvBeforeStatusRead: assertGatewayServiceManagementAllowedForUpdate, + timeoutMs: params.timeoutMs, }); } catch (err) { if (err instanceof GatewayServiceUpdateOwnershipError) { - return { - stopped: false, - inspected: false, - runtimeInspected: false, - running: false, - serviceMutationAllowed: false, - blockMessage: err.message, - }; + return { ...uninspected, serviceMutationAllowed: false, blockMessage: err.message }; } - return { stopped: false, inspected: false, runtimeInspected: false, running: false }; - } - - const runtimeStatus = serviceState.runtime?.status; - const runtimeInspected = runtimeStatus === "running" || runtimeStatus === "stopped"; - if (!serviceState.installed) { return { - stopped: false, - inspected: true, - runtimeInspected, - running: serviceState.running, - serviceEnv: serviceState.env, + ...uninspected, + serviceMutationAllowed: false, + serviceMutationSkipMessage: + "Gateway service management skipped: inspection is unavailable. Code update can continue; " + + "run `openclaw gateway status --deep` and restart the gateway manually when service access is restored.", }; } - - const serviceMatchesMutationRoot = await gatewayServiceCommandUsesRoot({ + const serviceUpdateVerdict = await inspectManagedGatewayServiceBeforeUpdate({ root: params.root, - command: serviceState.command, + state: serviceState, }); - const serviceOwnership = - serviceMatchesMutationRoot === null ? {} : { serviceMatchesMutationRoot }; - - if (!params.shouldRestart) { - if (!params.jsonMode && serviceState.running) { - defaultRuntime.log( - theme.warn( - `--no-restart is set while the managed gateway service is running; the ${params.updateInstallKind} update will not stop or restart that process.`, - ), - ); - } - const windowsTaskAutoStartRecovery = isRunningInsideGatewayService() - ? undefined - : await maybeSuspendWindowsTaskAutoStartForPackageUpdate({ - updateInstallKind: params.updateInstallKind, - serviceEnv: serviceState.env, - }); + const inspected = { + stopped: false, + inspected: true, + runtimeInspected: ["running", "stopped"].includes(serviceState.runtime?.status ?? ""), + running: serviceState.running, + serviceEnv: serviceState.env, + serviceUpdateVerdict, + }; + if (serviceUpdateVerdict.kind === "unavailable") { return { - stopped: false, - inspected: true, - runtimeInspected, - running: serviceState.running, - ...serviceOwnership, - serviceEnv: serviceState.env, - ...(windowsTaskAutoStartRecovery ? { windowsTaskAutoStartRecovery } : {}), + ...inspected, + serviceMutationAllowed: false, + serviceMutationSkipMessage: serviceUpdateVerdict.message, }; } - - if (!runtimeInspected) { - // An inherited gateway process cannot safely update and will be rejected below. - // Do not leave its task disabled while returning that rejection. - const windowsTaskAutoStartRecovery = isRunningInsideGatewayService() - ? undefined - : await maybeSuspendWindowsTaskAutoStartForPackageUpdate({ - updateInstallKind: params.updateInstallKind, - serviceEnv: serviceState.env, - }); + if (serviceUpdateVerdict.kind === "foreign") { return { - stopped: false, - inspected: true, - runtimeInspected: false, - running: false, - ...serviceOwnership, - serviceEnv: serviceState.env, - ...(windowsTaskAutoStartRecovery ? { windowsTaskAutoStartRecovery } : {}), + ...inspected, + serviceMutationAllowed: false, + serviceMutationSkipMessage: + "Gateway service management skipped: the service belongs to a different OpenClaw installation and was left untouched.", }; } - - if (serviceMatchesMutationRoot === false) { - if (!params.jsonMode) { - defaultRuntime.log( - theme.muted( - `Managed gateway service points at a different OpenClaw root; leaving it running during this ${params.updateInstallKind} update.`, - ), - ); - } - return { - stopped: false, - inspected: true, - runtimeInspected: true, - // Keep checking additional git mutation roots for this active supervisor. - running: true, - ...serviceOwnership, - serviceEnv: serviceState.env, - }; + if (serviceUpdateVerdict.kind === "absent" || params.phase === "inspect") { + return inspected; } - - // A loaded LaunchAgent can be between KeepAlive respawns. Other supervisors - // need the handoff marker to distinguish that transition from operator-stopped state. - const serviceLoaded = serviceState.loadState.status === "loaded"; - const launchAgentMayRespawn = - process.platform === "darwin" && - serviceLoaded && - (await service.isEnabled?.({ env: serviceState.env })) === true; - const handoffSupervisorMayRespawn = - process.platform !== "darwin" && process.env.OPENCLAW_UPDATE_RUN_HANDOFF === "1"; - const supervisorMayRespawn = - serviceLoaded && (launchAgentMayRespawn || handoffSupervisorMayRespawn); - if (!serviceState.running && !supervisorMayRespawn) { - const windowsTaskAutoStartRecovery = await maybeSuspendWindowsTaskAutoStartForPackageUpdate({ + const suspendTask = () => + maybeSuspendWindowsTaskAutoStartForPackageUpdate({ updateInstallKind: params.updateInstallKind, serviceEnv: serviceState.env, }); + // A loaded LaunchAgent can be between KeepAlive respawns. Other supervisors + // need the handoff marker to distinguish that transition from operator-stopped state. + const supervisorMayRespawn = + params.shouldRestart && + serviceState.loadState.status === "loaded" && + (process.platform === "darwin" + ? (await service.isEnabled?.({ env: serviceState.env })) === true + : process.env.OPENCLAW_UPDATE_RUN_HANDOFF === "1"); + if (!params.shouldRestart || (!serviceState.running && !supervisorMayRespawn)) { + if (!params.shouldRestart && !params.jsonMode && serviceState.running) { + const warning = `--no-restart is set while the managed gateway service is running; the ${params.updateInstallKind} update will not stop or restart that process.`; + defaultRuntime.log(theme.warn(warning)); + } + const windowsTaskAutoStartRecovery = + !params.shouldRestart && isGatewayServiceEnv(process.env) ? undefined : await suspendTask(); return { - stopped: false, - inspected: true, - runtimeInspected: true, - running: false, - ...serviceOwnership, - serviceEnv: serviceState.env, + ...inspected, ...(windowsTaskAutoStartRecovery ? { windowsTaskAutoStartRecovery } : {}), }; } - const blockMessage = gatewayAncestryBlockMessage(serviceState.runtime?.pid); if (blockMessage) { - return { - stopped: false, - inspected: true, - runtimeInspected: true, - running: true, - ...serviceOwnership, - blockMessage, - serviceEnv: serviceState.env, - }; + return { ...inspected, running: true, blockMessage }; } if (!params.jsonMode) { - defaultRuntime.log( - theme.muted(`Stopping managed gateway service before ${params.updateInstallKind} update...`), - ); + const message = `Stopping managed gateway service before ${params.updateInstallKind} update...`; + defaultRuntime.log(theme.muted(message)); } - const windowsTaskAutoStartRecovery = await maybeSuspendWindowsTaskAutoStartForPackageUpdate({ - updateInstallKind: params.updateInstallKind, - serviceEnv: serviceState.env, - }); + const windowsTaskAutoStartRecovery = await suspendTask(); try { await service.stop({ env: serviceState.env, @@ -563,12 +593,8 @@ export async function maybeStopManagedServiceBeforeMutableUpdate(params: { throw err; } return { + ...inspected, stopped: true, - inspected: true, - runtimeInspected: true, - running: serviceState.running, - ...serviceOwnership, - serviceEnv: serviceState.env, serviceDefinitionEnv: resolveManagedGatewayServiceCommand(serviceState.command)?.environment ?? {}, ...(windowsTaskAutoStartRecovery ? { windowsTaskAutoStartRecovery } : {}), @@ -577,60 +603,63 @@ export async function maybeStopManagedServiceBeforeMutableUpdate(params: { export async function maybeRestartServiceAfterFailedMutableUpdate(params: { preManagedServiceStop: PreManagedServiceStop | undefined; + root?: string; jsonMode: boolean; }): Promise { - if (!params.preManagedServiceStop?.stopped || !params.preManagedServiceStop.serviceEnv) { + const before = params.preManagedServiceStop; + if (!before?.stopped || !before.serviceEnv) { return; } try { - await resolveGatewayService().restart({ - env: params.preManagedServiceStop.serviceEnv, + const verdict = before.serviceUpdateVerdict; + if (!verdict || !("root" in verdict)) { + throw new Error( + "Stopped service ownership is unknown; restart it manually after inspection.", + ); + } + const service = resolveGatewayService(); + const state = await readGatewayServiceState(service, { + env: before.serviceEnv, + requireEffective: true, + validateEnvBeforeStatusRead: assertGatewayServiceManagementAllowedForUpdate, + }); + // Recovery follows the verified installation or the update's returned replacement root. + const revalidated = await revalidateManagedGatewayServiceAfterUpdate({ + state, + root: params.root ?? verdict.root, + preManagedServiceStop: before, + }); + await service.restart({ + env: state.env, + preserveDefinition: revalidated.kind !== "owned" || !revalidated.refreshDefinition, stdout: serviceControlStdoutForMode(params.jsonMode), }); if (!params.jsonMode) { defaultRuntime.log(theme.muted("Restarted managed gateway service after failed update.")); } } catch (err) { - const message = `Failed to restart managed gateway service after failed update: ${String(err)}`; - if (params.jsonMode) { - defaultRuntime.error(message); - } else { - defaultRuntime.log(theme.warn(message)); - } + defaultRuntime.error( + `Failed to restart managed gateway service after failed update: ${String(err)}`, + ); } } -function isRunningInsideGatewayService( - env: Record = process.env, -): boolean { - return isGatewayServiceEnv(env); -} - export function shouldBlockMutableUpdateFromGatewayServiceEnv(params: { preManagedServiceStop: PreManagedServiceStop | undefined; }): boolean { - if (!isRunningInsideGatewayService()) { - return false; - } const stopState = params.preManagedServiceStop; - if (!stopState?.inspected) { - return true; - } - if (stopState.stopped) { - return false; - } - if (!stopState.runtimeInspected) { - return true; - } - return stopState.running; + return ( + isGatewayServiceEnv(process.env) && + (!stopState?.inspected || + (!stopState.stopped && + (!stopState.runtimeInspected || (stopState.running && !stopState.blockMessage)))) + ); } function formatCommandFailure(stdout: string, stderr: string): string { - const detail = (stderr || stdout).trim(); - if (!detail) { - return "command returned a non-zero exit code"; - } - return detail.split("\n").slice(-3).join("\n"); + // Keep the stable denial even when JSON stdout accompanies unrelated stderr warnings. + const detail = `${stderr}\n${stdout}`.match(DEFINITION_DENIAL)?.[0] ?? (stderr || stdout).trim(); + return detail ? detail.split("\n").slice(-3).join("\n") : "command returned a non-zero exit code"; } export function tryResolveInvocationCwd(): string | undefined { @@ -659,14 +688,12 @@ export async function resolvePackageRuntimePreflight(params: { }): Promise> { const nodeRunner = normalizeOptionalString(params.nodeRunner); const unchanged = (): PackageRuntimePreflight => (nodeRunner ? { nodeRunner } : {}); - if (!canResolveRegistryVersionForPackageTarget(params.tag)) { - return ok(unchanged()); - } - if (params.spec && !canResolveRegistryVersionForPackageTarget(params.spec)) { - return ok(unchanged()); - } const target = params.tag.trim(); - if (!target) { + if ( + !target || + !canResolveRegistryVersionForPackageTarget(params.tag) || + (params.spec && !canResolveRegistryVersionForPackageTarget(params.spec)) + ) { return ok(unchanged()); } const status = await fetchNpmPackageTargetStatus({ @@ -686,11 +713,9 @@ export async function resolvePackageRuntimePreflight(params: { }); const satisfies = nodeVersionSatisfiesEngine(runtime.version, status.nodeEngine); const targetVersion = status.version ?? target; + const unchangedRuntime = { ...unchanged(), targetVersion }; if (satisfies === true) { - return ok({ - ...(nodeRunner ? { nodeRunner } : {}), - targetVersion, - }); + return ok(unchangedRuntime); } const fallbackNodeRunner = normalizeOptionalString(params.fallbackNodeRunner); if (nodeRunner && fallbackNodeRunner && fallbackNodeRunner !== nodeRunner) { @@ -711,10 +736,7 @@ export async function resolvePackageRuntimePreflight(params: { } } if (satisfies !== false) { - return ok({ - ...(nodeRunner ? { nodeRunner } : {}), - targetVersion, - }); + return ok(unchangedRuntime); } const runtimeLabel = runtime.nodeRunner ? `Node ${runtime.version ?? "unknown"} at ${runtime.nodeRunner}` @@ -743,9 +765,10 @@ async function resolvePackageRuntimeForPreflight(params: { const res = await runCommandWithTimeout([nodeRunner, "--version"], { timeoutMs: Math.min(params.timeoutMs ?? 10_000, 10_000), }).catch(() => null); - const rawVersion = res?.code === 0 ? res.stdout.trim() : ""; - const version = rawVersion.replace(/^v/u, "") || null; - return { version, nodeRunner }; + return { + version: res?.code === 0 ? res.stdout.trim().replace(/^v/u, "") || null : null, + nodeRunner, + }; } export { disableUpdatedPackageCompileCacheEnv } from "./update-command-service-env.js"; @@ -758,90 +781,67 @@ export function stripGatewayServiceMarkerEnv(env: NodeJS.ProcessEnv): NodeJS.Pro return resolvedEnv; } -export function resolveUpdatedGatewayRestartPort(params: { +export async function resolveUpdatedGatewayRestartPort(params: { config?: OpenClawConfig; processEnv?: NodeJS.ProcessEnv; serviceEnv?: NodeJS.ProcessEnv; -}): number { - return resolveGatewayPort(params.config, params.serviceEnv ?? params.processEnv ?? process.env); + serviceCommand?: GatewayServiceCommandConfig | null; +}): Promise { + const env = params.serviceEnv ?? params.processEnv ?? process.env; + let config = params.config; + if (params.serviceCommand) { + // Preserved launchers keep their explicit port and their own config context; + // refresh callers omit the old command and use the intended new configuration. + const port = parseTcpPortFromArgs(params.serviceCommand.programArguments); + if (port !== null) { + return port; + } + config = await createConfigIO({ + env, + observe: false, + pluginValidation: "skip", + suppressFutureVersionWarning: true, + }).readBestEffortConfig(); + } + return resolveGatewayPort(config, env); } export function resolvePostUpdateServiceStateReadEnv(params: { updateMode: UpdateRunResult["mode"]; processEnv?: NodeJS.ProcessEnv; preManagedServiceEnv?: NodeJS.ProcessEnv; - prePackageServiceEnv?: NodeJS.ProcessEnv; }): NodeJS.ProcessEnv { - if (params.updateMode === "git" && params.preManagedServiceEnv) { - return params.preManagedServiceEnv; - } - if (isPackageManagerUpdateMode(params.updateMode)) { - return ( - params.preManagedServiceEnv ?? params.prePackageServiceEnv ?? params.processEnv ?? process.env - ); - } - return params.processEnv ?? process.env; + const fallbackEnv = params.processEnv ?? process.env; + const usesServiceEnv = + params.updateMode === "git" || isPackageManagerUpdateMode(params.updateMode); + return usesServiceEnv ? (params.preManagedServiceEnv ?? fallbackEnv) : fallbackEnv; } -async function refreshGatewayServiceEnv(params: { - result: UpdateRunResult; - jsonMode: boolean; - invocationCwd?: string; - env?: NodeJS.ProcessEnv; - nodeRunner?: string; -}): Promise { - const args = ["gateway", "install", "--force"]; - if (params.jsonMode) { - args.push("--json"); - } - - const entrypoint = await resolveGatewayInstallEntrypoint(params.result.root); - if (entrypoint) { - const res = await runCommandWithTimeout( - [params.nodeRunner ?? resolveNodeRunner(), entrypoint, ...args], - { - cwd: params.result.root, - env: resolveUpdatedInstallCommandEnv({ - processEnv: params.env ?? process.env, - invocationCwd: params.invocationCwd, - }), - timeoutMs: SERVICE_REFRESH_TIMEOUT_MS, - }, - ); - if (res.code === 0) { - return; - } - throw new Error( - `updated install refresh failed (${entrypoint}): ${formatCommandFailure(res.stdout, res.stderr)}`, - ); - } - - if (isPackageManagerUpdateMode(params.result.mode)) { - throw new Error( - `updated install entrypoint not found under ${params.result.root ?? "unknown"}`, - ); - } - - await runDaemonInstall({ force: true, json: params.jsonMode || undefined }); -} - -async function runUpdatedInstallGatewayRestart(params: { - result: UpdateRunResult; - jsonMode: boolean; - invocationCwd?: string; - env?: NodeJS.ProcessEnv; - nodeRunner?: string; - timeoutMs: number; -}): Promise { +// Use the candidate's version guards for both refresh and activation. The parsed +// preservation option makes older targets reject before repair, without a retry. +async function runUpdatedInstallGatewayCommand( + params: Parameters[0], + action: "install" | "restart", + preserveDefinition = false, +): Promise { + const installing = action === "install"; const entrypoint = await resolveGatewayInstallEntrypoint(params.result.root); if (!entrypoint) { + if (installing && !isPackageManagerUpdateMode(params.result.mode)) { + await runDaemonInstall({ force: true, json: params.opts.json || undefined }); + return true; + } throw new Error( `updated install entrypoint not found under ${params.result.root ?? "unknown"}`, ); } - - const args = ["gateway", "restart"]; - if (params.jsonMode) { + const args = ["gateway", action]; + if (installing) { + args.push("--force"); + } else if (preserveDefinition) { + args.push("--preserve-definition"); + } + if (params.opts.json) { args.push("--json"); } const res = await runCommandWithTimeout( @@ -849,20 +849,20 @@ async function runUpdatedInstallGatewayRestart(params: { { cwd: params.result.root, env: resolveUpdatedInstallCommandEnv({ - processEnv: process.env, - serviceEnv: params.env, + processEnv: installing ? (params.serviceInstallEnv ?? process.env) : process.env, + serviceEnv: installing ? undefined : params.serviceEnv, invocationCwd: params.invocationCwd, }), - // Restart health owns migration-aware readiness. Keep only the caller's bounded update - // budget outside it so the former fixed 60-second watchdog cannot preempt that wait. - timeoutMs: params.timeoutMs, + // Restart owns migration-aware readiness; only refresh has the fixed watchdog. + timeoutMs: installing ? SERVICE_REFRESH_TIMEOUT_MS : params.timeoutMs, }, ); if (res.code === 0) { return true; } + const operation = installing ? "refresh" : "restart"; throw new Error( - `updated install restart failed (${entrypoint}): ${formatCommandFailure(res.stdout, res.stderr)}`, + `updated install ${operation} failed (${entrypoint}): ${formatCommandFailure(res.stdout, res.stderr)}`, ); } @@ -932,31 +932,17 @@ async function installShellCompletionForUpdate(shell: string, yes: boolean): Pro } async function tryRealpathOrResolve(value: string): Promise { - try { - return await fs.realpath(path.resolve(value)); - } catch { - return path.resolve(value); - } -} - -function isNodeExecutable(value: string | undefined): boolean { - const base = normalizeOptionalString(value ? path.basename(value) : undefined)?.toLowerCase(); - return base === "node" || base === "node.exe"; + return await fs.realpath(path.resolve(value)).catch(() => path.resolve(value)); } function resolveManagedServiceNodeRunner( command: GatewayServiceCommandConfig | null, ): string | undefined { - const args = command?.programArguments; - if (!args?.length) { - return undefined; - } - const gatewayIndex = args.indexOf("gateway"); - if (gatewayIndex <= 1) { - return undefined; - } - const runner = args[gatewayIndex - 2]; - return isNodeExecutable(runner) ? runner : undefined; + const args = command?.programArguments ?? []; + // Native heap flags and dev loaders separate the executable from the entrypoint. + const runner = args.indexOf("gateway") > 1 ? args[0] : undefined; + const executable = normalizeOptionalString(runner ? path.basename(runner) : undefined); + return ["node", "node.exe"].includes(executable?.toLowerCase() ?? "") ? runner : undefined; } /** @@ -970,7 +956,7 @@ export async function resolveManagedServiceNodeRunnerOverride(): Promise null); const serviceNode = resolveManagedServiceNodeRunner(command); if (!serviceNode) { @@ -981,10 +967,7 @@ export async function resolveManagedServiceNodeRunnerOverride(): Promise null); const layout = await summarizeGatewayServiceLayout(command); const serviceRoot = layout?.packageRoot; @@ -1029,7 +1012,7 @@ export async function gatewayServiceCommandUsesRoot(params: { params.command === undefined ? isGatewayServiceManagementAllowedForUpdate(params.env ?? process.env) ? await resolveGatewayService() - .readCommand(params.env ?? process.env) + .readCommand(params.env ?? process.env, { requireEffective: true }) .catch(() => null) : null : params.command; @@ -1047,7 +1030,52 @@ export async function gatewayServiceCommandUsesRoot(params: { tryRealpathOrResolve(expectedRoot), tryRealpathOrResolve(serviceRoot), ]); - return expectedRootReal === serviceRootReal; + if (expectedRootReal === serviceRootReal) { + return true; + } + // Paired read-only release mounts have different paths but the same directory + // identity. Copies of another release must remain foreign. + const [expected, actual] = await Promise.all( + [expectedRootReal, serviceRootReal].map((root) => fs.stat(root).catch(() => null)), + ); + if (expected && actual && expected.dev === actual.dev && expected.ino === actual.ino) { + return true; + } + const managed = command?.managedDefinition; + if ( + !managed || + (await gatewayServiceCommandUsesRoot({ root: expectedRoot, command: managed })) !== true + ) { + return false; + } + const namespace = path.dirname(expectedRootReal); + const managedLayout = await summarizeGatewayServiceLayout(managed); + const stableEntry = path.join( + namespace, + "current", + "dist", + path.basename(managedLayout?.entrypoint ?? ""), + ); + if (serviceEntrypoint !== stableEntry) { + return false; + } + // Deployment-owned current points into this installation's releases, either + // by symlink or by a paired bind mount. Unrelated namespaces remain foreign. + const releases = path.join(namespace, "releases"); + if (serviceRootReal.startsWith(`${releases}${path.sep}`)) { + return true; + } + try { + for await (const entry of await fs.opendir(releases)) { + const candidate = await fs.lstat(path.join(releases, entry.name)); + if (actual && candidate.dev === actual.dev && candidate.ino === actual.ino) { + return true; + } + } + } catch { + // Without directory identity proof, the override cannot authorize lifecycle actions. + } + return false; } export async function maybeRestartService(params: { @@ -1058,6 +1086,7 @@ export async function maybeRestartService(params: { refreshServiceEnv: boolean; serviceEnv?: NodeJS.ProcessEnv; serviceInstallEnv?: NodeJS.ProcessEnv | null; + serviceUpdateVerdict?: ManagedGatewayUpdateVerdict; gatewayPort: number; restartScriptPath?: string | null; invocationCwd?: string; @@ -1080,44 +1109,50 @@ export async function maybeRestartService(params: { } return false; } - const canRestartUpdatedInstall = params.refreshServiceEnv || params.serviceInstallEnv === null; + let activation = params; + const verdict = activation.serviceUpdateVerdict; + let preserveDefinition = + verdict?.kind === "unresolved" || (verdict?.kind === "owned" && !verdict.refreshDefinition); + const isPackageUpdate = isPackageManagerUpdateMode(activation.result.mode); + const requiresVerifiedRestart = () => + preserveDefinition || isPackageUpdate || activation.requireRunningServiceAfterRestart; + const canRestartUpdatedInstall = () => + preserveDefinition || + (isPackageUpdate && + (activation.refreshServiceEnv || + activation.serviceInstallEnv === null || + activation.requireRunningServiceAfterRestart)); + if (preserveDefinition) { + defaultRuntime.error( + "Gateway service definition left unchanged; ask its deployment owner to repair stale metadata if needed.", + ); + } + if (activation.serviceMutationSkipMessage) { + defaultRuntime.error(activation.serviceMutationSkipMessage); + return true; + } const verifyRestartedGateway = async ( expectedGatewayVersion: string | undefined, expectedGatewayBuildId: string | undefined, opts: { requireRunningService?: boolean } = {}, ) => { - const restartAfterStaleCleanup = async () => { - if (canRestartUpdatedInstall && isPackageManagerUpdateMode(params.result.mode)) { - await runUpdatedInstallGatewayRestart({ - result: params.result, - jsonMode: Boolean(params.opts.json), - invocationCwd: params.invocationCwd, - env: params.serviceEnv, - nodeRunner: params.nodeRunner, - timeoutMs: params.timeoutMs, - }); - return; - } - if (shouldUseLegacyProcessRestartAfterUpdate({ updateMode: params.result.mode })) { - await runDaemonRestart(); - } - }; const service = resolveGatewayService(); - let supervisorKeepsAlive = await hasLoadedLaunchdKeepAliveSupervisor({ - service, - env: params.serviceEnv, - }); - let health = await waitForGatewayHealthyRestart({ - service, - port: params.gatewayPort, - expectedVersion: expectedGatewayVersion, - ...(expectedGatewayBuildId ? { expectedBuildId: expectedGatewayBuildId } : {}), - env: params.serviceEnv, - requireRunningService: opts.requireRunningService, - supervisorKeepsAlive, - }); + const waitForHealthy = async () => + await waitForGatewayHealthyRestart({ + service, + port: activation.gatewayPort, + expectedVersion: expectedGatewayVersion, + ...(expectedGatewayBuildId ? { expectedBuildId: expectedGatewayBuildId } : {}), + env: activation.serviceEnv, + requireRunningService: opts.requireRunningService, + supervisorKeepsAlive: await hasLoadedLaunchdKeepAliveSupervisor({ + service, + env: activation.serviceEnv, + }), + }); + let health = await waitForHealthy(); if (!health.healthy && health.staleGatewayPids.length > 0) { - if (!params.opts.json) { + if (!activation.opts.json) { defaultRuntime.log( theme.warn( `Found stale gateway process(es) after restart: ${health.staleGatewayPids.join(", ")}. Cleaning up...`, @@ -1125,50 +1160,35 @@ export async function maybeRestartService(params: { ); } await terminateStaleGatewayPids(health.staleGatewayPids); - await restartAfterStaleCleanup(); - supervisorKeepsAlive = await hasLoadedLaunchdKeepAliveSupervisor({ - service, - env: params.serviceEnv, - }); - health = await waitForGatewayHealthyRestart({ - service, - port: params.gatewayPort, - expectedVersion: expectedGatewayVersion, - ...(expectedGatewayBuildId ? { expectedBuildId: expectedGatewayBuildId } : {}), - env: params.serviceEnv, - requireRunningService: opts.requireRunningService, - supervisorKeepsAlive, - }); + if (canRestartUpdatedInstall()) { + await runUpdatedInstallGatewayCommand(activation, "restart", preserveDefinition); + } else if (shouldUseLegacyProcessRestartAfterUpdate({ updateMode: activation.result.mode })) { + await runDaemonRestart(); + } + health = await waitForHealthy(); } const recoveryVerification = await recoverLaunchAgentAndRecheckGatewayHealth({ + preserveDefinition, health, service, - port: params.gatewayPort, + port: activation.gatewayPort, expectedVersion: expectedGatewayVersion, ...(expectedGatewayBuildId ? { expectedBuildId: expectedGatewayBuildId } : {}), - env: params.serviceEnv, + env: activation.serviceEnv, }); health = recoveryVerification.health; const launchAgentRecovery = recoveryVerification.launchAgentRecovery; if (launchAgentRecovery?.attempted) { - if (!params.opts.json) { - defaultRuntime.log( - launchAgentRecovery.recovered - ? theme.warn(launchAgentRecovery.message) - : theme.warn(launchAgentRecovery.detail), - ); - } else { - defaultRuntime.error( - launchAgentRecovery.recovered ? launchAgentRecovery.message : launchAgentRecovery.detail, - ); - } + defaultRuntime.error( + launchAgentRecovery.recovered ? launchAgentRecovery.message : launchAgentRecovery.detail, + ); } const serviceRuntimeHealthy = !opts.requireRunningService || health.runtime.status === "running"; if (health.healthy && serviceRuntimeHealthy) { - if (!params.opts.json) { + if (!activation.opts.json) { defaultRuntime.log(theme.success("Gateway: restarted and verified.")); } return true; @@ -1187,11 +1207,11 @@ export async function maybeRestartService(params: { : `LaunchAgent recovery failed: ${launchAgentRecovery.detail}`, ] : []), - `Restart log: ${resolveGatewayRestartLogPath(params.serviceEnv ?? process.env)}`, + `Restart log: ${resolveGatewayRestartLogPath(activation.serviceEnv ?? process.env)}`, `Run \`${replaceCliName(formatCliCommand("openclaw gateway status --deep"), CLI_NAME)}\` for details.`, - ...formatPostUpdateGatewayRecoveryInstructions(params.result), + ...formatPostUpdateGatewayRecoveryInstructions(activation.result), ]; - if (params.opts.json) { + if (activation.opts.json) { defaultRuntime.error(diagnosticLines.join("\n")); } else { defaultRuntime.log(theme.warn(diagnosticLines[0] ?? "Gateway did not become healthy.")); @@ -1200,7 +1220,7 @@ export async function maybeRestartService(params: { } } - if (isPackageManagerUpdateMode(params.result.mode) || opts.requireRunningService) { + if (requiresVerifiedRestart() || opts.requireRunningService) { return false; } @@ -1211,49 +1231,42 @@ export async function maybeRestartService(params: { ); }; - if (params.shouldRestart) { - if (!params.opts.json) { + if (activation.shouldRestart) { + if (!activation.opts.json) { defaultRuntime.log(""); defaultRuntime.log(theme.heading("Restarting service...")); } try { - const expectedGatewayVersion = isPackageManagerUpdateMode(params.result.mode) - ? normalizeOptionalString(params.result.after?.version) + let expectedGatewayVersion = requiresVerifiedRestart() + ? normalizeOptionalString(activation.result.after?.version) : undefined; const expectedGatewayBuildId = - params.channel === "dev" && params.result.mode === "git" - ? normalizeOptionalString(params.result.after?.buildId) + activation.channel === "dev" && activation.result.mode === "git" + ? normalizeOptionalString(activation.result.after?.buildId) : undefined; - const isPackageUpdate = isPackageManagerUpdateMode(params.result.mode); const canVerifyUpdatedGatewayByVersion = expectedGatewayVersion !== undefined && - expectedGatewayVersion !== normalizeOptionalString(params.result.before?.version); + expectedGatewayVersion !== normalizeOptionalString(activation.result.before?.version); let restarted = false; let restartInitiated = false; let refreshedGatewayAlreadyHealthy = false; let updatedInstallRestartNeedsServiceRootProof = false; - let restartScriptPath = params.restartScriptPath; - if (params.refreshServiceEnv && params.serviceInstallEnv !== null) { + let restartScriptPath = preserveDefinition ? null : activation.restartScriptPath; + if (activation.refreshServiceEnv && activation.serviceInstallEnv !== null) { try { - await refreshGatewayServiceEnv({ - result: params.result, - jsonMode: Boolean(params.opts.json), - invocationCwd: params.invocationCwd, - env: params.serviceInstallEnv, - nodeRunner: params.nodeRunner, - }); + await runUpdatedInstallGatewayCommand(activation, "install"); if (isPackageUpdate && expectedGatewayVersion) { const health = await waitForGatewayHealthyRestart({ service: resolveGatewayService(), - port: params.gatewayPort, + port: activation.gatewayPort, expectedVersion: expectedGatewayVersion, - env: params.serviceEnv, + env: activation.serviceEnv, attempts: POST_REFRESH_ALREADY_HEALTHY_ATTEMPTS, delayMs: POST_REFRESH_ALREADY_HEALTHY_DELAY_MS, }); refreshedGatewayAlreadyHealthy = health.healthy; - if (refreshedGatewayAlreadyHealthy && !params.opts.json) { + if (refreshedGatewayAlreadyHealthy && !activation.opts.json) { defaultRuntime.log( theme.muted( "Gateway already reports the updated version after service refresh; skipped redundant restart.", @@ -1262,14 +1275,40 @@ export async function maybeRestartService(params: { } } } catch (err) { - // Always log the refresh failure so callers can detect it (issue #56772). - // Previously this was silently suppressed in --json mode, hiding the root - // cause and preventing auto-update callers from detecting the failure. - const message = `Failed to refresh gateway service environment from updated install: ${String(err)}`; - if (params.opts.json) { - defaultRuntime.error(message); - } else { - defaultRuntime.log(theme.warn(message)); + defaultRuntime.error( + `Failed to refresh gateway service environment from updated install: ${String(err)}`, + ); + if (DEFINITION_DENIAL.test(String(err))) { + // A writer denial is not a lifecycle grant: revalidate the retained + // command and manager before using native activation without repair. + preserveDefinition = true; + if (verdict?.kind !== "owned") { + throw err; + } + const state = await readGatewayServiceState(resolveGatewayService(), { + env: activation.serviceEnv, + requireEffective: true, + validateEnvBeforeStatusRead: assertGatewayServiceManagementAllowedForUpdate, + timeoutMs: activation.timeoutMs, + }); + await revalidateManagedGatewayServiceAfterUpdate({ + state, + root: activation.result.root ?? verdict.root, + preManagedServiceStop: { + serviceEnv: activation.serviceEnv, + serviceUpdateVerdict: { ...verdict, refreshDefinition: false }, + }, + }); + activation = { + ...activation, + serviceEnv: state.env, + gatewayPort: await resolveUpdatedGatewayRestartPort({ + serviceEnv: state.env, + serviceCommand: state.command, + }), + }; + expectedGatewayVersion = normalizeOptionalString(activation.result.after?.version); + restartScriptPath = null; } if (isPackageUpdate) { restartScriptPath = null; @@ -1284,24 +1323,21 @@ export async function maybeRestartService(params: { await createUpdateConfigSnapshot(); await runRestartScript(restartScriptPath); restartInitiated = true; - } else if (!refreshedGatewayAlreadyHealthy && canRestartUpdatedInstall && isPackageUpdate) { + } else if (!refreshedGatewayAlreadyHealthy && canRestartUpdatedInstall()) { await createUpdateConfigSnapshot(); - restarted = await runUpdatedInstallGatewayRestart({ - result: params.result, - jsonMode: Boolean(params.opts.json), - invocationCwd: params.invocationCwd, - env: params.serviceEnv, - nodeRunner: params.nodeRunner, - timeoutMs: params.timeoutMs, - }); + restarted = await runUpdatedInstallGatewayCommand( + activation, + "restart", + preserveDefinition, + ); if ( updatedInstallRestartNeedsServiceRootProof && (await gatewayServiceCommandUsesRoot({ - root: params.result.root, - env: params.serviceEnv, + root: activation.result.root, + env: activation.serviceEnv, })) !== true ) { - if (!params.opts.json) { + if (!activation.opts.json) { defaultRuntime.log( theme.warn("Gateway service did not point at the updated install after restart."), ); @@ -1310,12 +1346,12 @@ export async function maybeRestartService(params: { } } else if ( !refreshedGatewayAlreadyHealthy && - shouldUseLegacyProcessRestartAfterUpdate({ updateMode: params.result.mode }) && - !params.skipLegacyServiceRestart + shouldUseLegacyProcessRestartAfterUpdate({ updateMode: activation.result.mode }) && + !activation.skipLegacyServiceRestart ) { await createUpdateConfigSnapshot(); restarted = await runDaemonRestart(); - } else if (!refreshedGatewayAlreadyHealthy && !params.opts.json) { + } else if (!refreshedGatewayAlreadyHealthy && !activation.opts.json) { defaultRuntime.log(theme.muted("Gateway: restart skipped (no installed service found).")); } @@ -1323,32 +1359,32 @@ export async function maybeRestartService(params: { refreshedGatewayAlreadyHealthy || restartInitiated || (restarted && - (expectedGatewayVersion !== undefined || - expectedGatewayBuildId !== undefined || - params.result.mode === "git")); + (preserveDefinition || + expectedGatewayVersion !== undefined || + activation.result.mode === "git")) || + activation.requireRunningServiceAfterRestart; if (shouldVerifyRestart) { const requireRunningService = - updatedInstallRestartNeedsServiceRootProof || params.requireRunningServiceAfterRestart; + updatedInstallRestartNeedsServiceRootProof || + activation.requireRunningServiceAfterRestart; const restartHealthy = await verifyRestartedGateway( expectedGatewayVersion, expectedGatewayBuildId, - { - requireRunningService, - }, + { requireRunningService }, ); if (!restartHealthy) { - if (!params.opts.json) { + if (!activation.opts.json) { defaultRuntime.log(""); } return false; } - if (!params.opts.json && restartInitiated) { + if (!activation.opts.json && restartInitiated) { defaultRuntime.log(theme.success("Daemon restart completed.")); defaultRuntime.log(""); } } - if (!params.opts.json && restarted) { + if (!activation.opts.json && restarted && !preserveDefinition) { defaultRuntime.log(theme.success("Daemon restarted successfully.")); defaultRuntime.log(""); await createUpdateConfigSnapshot(); @@ -1356,7 +1392,7 @@ export async function maybeRestartService(params: { process.env[UPDATE_PARENT_SUPPORTS_DOCTOR_CONFIG_WRITE_ENV] = "1"; try { const interactiveDoctor = - process.stdin.isTTY && !params.opts.json && params.opts.yes !== true; + process.stdin.isTTY && !activation.opts.json && activation.opts.yes !== true; await doctorCommand(defaultRuntime, { nonInteractive: !interactiveDoctor, }); @@ -1368,40 +1404,21 @@ export async function maybeRestartService(params: { } } } catch (err) { - if (!params.opts.json) { - defaultRuntime.log(theme.warn(`Gateway: restart failed: ${String(err)}`)); - defaultRuntime.log( - theme.muted( - `You may need to restart the service manually: ${replaceCliName(formatCliCommand("openclaw gateway restart"), CLI_NAME)}`, - ), - ); - } - if ( - isPackageManagerUpdateMode(params.result.mode) || - params.requireRunningServiceAfterRestart - ) { + defaultRuntime.error( + `Gateway: restart failed: ${String(err)}. Code update remains installed; a service stopped for update may still be stopped. ` + + "Run `openclaw gateway status --deep` and ask its service owner to restart it manually.", + ); + if (requiresVerifiedRestart()) { return false; } } return true; } - if (params.serviceMutationSkipMessage) { - if (params.opts.json) { - defaultRuntime.error(params.serviceMutationSkipMessage); - } else { - defaultRuntime.log(""); - defaultRuntime.log( - theme.warn(`Gateway: restart skipped: ${params.serviceMutationSkipMessage}`), - ); - } - return true; - } - - if (!params.opts.json) { + if (!activation.opts.json) { defaultRuntime.log(""); defaultRuntime.log(theme.muted("Gateway: restart skipped (--no-restart).")); - if (params.result.mode === "npm" || params.result.mode === "pnpm") { + if (activation.result.mode === "npm" || activation.result.mode === "pnpm") { defaultRuntime.log( theme.muted( `Tip: Run \`${replaceCliName(formatCliCommand("openclaw doctor"), CLI_NAME)}\`, then \`${replaceCliName(formatCliCommand("openclaw gateway restart"), CLI_NAME)}\` to apply updates to a running gateway.`, diff --git a/src/cli/update-cli/update-command.test.ts b/src/cli/update-cli/update-command.test.ts index 27b45d89074a..a508a5dd3884 100644 --- a/src/cli/update-cli/update-command.test.ts +++ b/src/cli/update-cli/update-command.test.ts @@ -109,17 +109,6 @@ describe("shouldPrepareUpdatedInstallRestart", () => { ).toBe(false); }); - it("does not prepare package restart for a service owned by another root", () => { - expect( - shouldPrepareUpdatedInstallRestart({ - updateMode: "npm", - serviceInstalled: true, - serviceLoaded: true, - serviceMatchesMutationRoot: false, - }), - ).toBe(false); - }); - it("keeps non-package updates tied to the matching loaded service state", () => { expect( shouldPrepareUpdatedInstallRestart({ @@ -159,9 +148,9 @@ describe("shouldPrepareUpdatedInstallRestart", () => { }); describe("resolveUpdatedGatewayRestartPort", () => { - it("uses the managed service port ahead of the caller environment", () => { + it("uses the managed service port ahead of the caller environment", async () => { expect( - resolveUpdatedGatewayRestartPort({ + await resolveUpdatedGatewayRestartPort({ config: { gateway: { port: 19000 } } as never, processEnv: { OPENCLAW_GATEWAY_PORT: "19001" }, serviceEnv: { OPENCLAW_GATEWAY_PORT: "19002" }, @@ -169,9 +158,9 @@ describe("resolveUpdatedGatewayRestartPort", () => { ).toBe(19002); }); - it("falls back to the post-update config when no service port is available", () => { + it("falls back to the post-update config when no service port is available", async () => { expect( - resolveUpdatedGatewayRestartPort({ + await resolveUpdatedGatewayRestartPort({ config: { gateway: { port: 19000 } } as never, processEnv: {}, serviceEnv: {}, @@ -181,49 +170,22 @@ describe("resolveUpdatedGatewayRestartPort", () => { }); describe("resolvePostUpdateServiceStateReadEnv", () => { - it("keeps package restart preparation anchored to the pre-update service env", () => { - const processEnv = { - OPENCLAW_STATE_DIR: "/source/state", - OPENCLAW_CONFIG_PATH: "/source/openclaw.json", - } as NodeJS.ProcessEnv; - const prePackageServiceEnv = { - OPENCLAW_STATE_DIR: "/managed/state", - OPENCLAW_CONFIG_PATH: "/managed/openclaw.json", - } as NodeJS.ProcessEnv; + it.each(["git", "npm", "pnpm", "bun"] as const)( + "keeps %s restart preparation anchored to the pre-update service env", + (updateMode) => { + const processEnv = { OPENCLAW_STATE_DIR: "/source/state" }; + const preManagedServiceEnv = { OPENCLAW_STATE_DIR: "/managed/state" }; + expect( + resolvePostUpdateServiceStateReadEnv({ updateMode, processEnv, preManagedServiceEnv }), + ).toEqual(preManagedServiceEnv); + }, + ); - expect( - resolvePostUpdateServiceStateReadEnv({ - updateMode: "npm", - processEnv, - prePackageServiceEnv, - }), - ).toBe(prePackageServiceEnv); - }); - - it("keeps git updates tied to the caller environment", () => { - const processEnv = { OPENCLAW_STATE_DIR: "/source/state" } as NodeJS.ProcessEnv; - const prePackageServiceEnv = { OPENCLAW_STATE_DIR: "/managed/state" } as NodeJS.ProcessEnv; - - expect( - resolvePostUpdateServiceStateReadEnv({ - updateMode: "git", - processEnv, - prePackageServiceEnv, - }), - ).toBe(processEnv); - }); - - it("uses the managed service environment for git updates stopped by this updater", () => { - const processEnv = { OPENCLAW_STATE_DIR: "/source/state" } as NodeJS.ProcessEnv; - const preManagedServiceEnv = { OPENCLAW_STATE_DIR: "/managed/state" } as NodeJS.ProcessEnv; - - expect( - resolvePostUpdateServiceStateReadEnv({ - updateMode: "git", - processEnv, - preManagedServiceEnv, - }), - ).toBe(preManagedServiceEnv); + it("uses the caller environment when no managed service context was captured", () => { + const processEnv = { OPENCLAW_STATE_DIR: "/source/state" }; + expect(resolvePostUpdateServiceStateReadEnv({ updateMode: "git", processEnv })).toEqual( + processEnv, + ); }); }); @@ -718,43 +680,52 @@ describe("formatPostUpdateGatewayRecoveryInstructions", () => { }); describe("recoverInstalledLaunchAgentAfterUpdate", () => { - it("re-bootstraps an installed-but-not-loaded macOS LaunchAgent after update", async () => { - const service = {} as never; - const serviceEnv = { OPENCLAW_PROFILE: "stomme" } as NodeJS.ProcessEnv; - const recoveredEnv = { ...serviceEnv, OPENCLAW_PORT: "18790" } as NodeJS.ProcessEnv; - const readState = vi.fn(async () => ({ - installed: true, - loadState: { status: "not-loaded" }, - running: false, - env: recoveredEnv, - command: null, - runtime: { status: "unknown", missingSupervision: true }, - })); - const recover = vi.fn(async () => ({ - result: "restarted" as const, - loaded: true as const, - message: "Gateway LaunchAgent was installed but not loaded; re-bootstrapped launchd service.", - })); + it.each(["recovered", "failed", "system owner"] as const)( + "reports installed-but-not-loaded LaunchAgent recovery: %s", + async (outcome) => { + const service = {} as never; + const serviceEnv = { OPENCLAW_PROFILE: "stomme" }; + const recoveredEnv = { ...serviceEnv, OPENCLAW_PORT: "18790" }; + const readState = vi.fn(async () => ({ + installed: true, + loadState: { status: "not-loaded" }, + running: false, + env: recoveredEnv, + command: null, + runtime: { status: "unknown", missingSupervision: true }, + })); + const message = + "Gateway LaunchAgent was installed but not loaded; re-bootstrapped launchd service."; + const guidance = "System LaunchDaemon system/ai.openclaw.stomme owns this label"; + const recover = vi.fn(async () => { + if (outcome === "system owner") { + throw new Error(guidance); + } + return outcome === "recovered" ? { result: "restarted", loaded: true, message } : null; + }); - await expect( - updateCommandServiceTesting.recoverInstalledLaunchAgentAfterUpdate({ - service, - env: serviceEnv, - deps: { - platform: "darwin", - readState: readState as never, - recover: recover as never, - }, - }), - ).resolves.toEqual({ - attempted: true, - recovered: true, - message: "Gateway LaunchAgent was installed but not loaded; re-bootstrapped launchd service.", - }); - - expect(readState).toHaveBeenCalledWith(service, { env: serviceEnv }); - expect(recover).toHaveBeenCalledWith({ result: "restarted", env: recoveredEnv }); - }); + await expect( + updateCommandServiceTesting.recoverInstalledLaunchAgentAfterUpdate({ + service, + env: serviceEnv, + deps: { platform: "darwin", readState: readState as never, recover: recover as never }, + }), + ).resolves.toEqual( + outcome === "recovered" + ? { attempted: true, recovered: true, message } + : { + attempted: true, + recovered: false, + detail: + outcome === "system owner" + ? guidance + : "LaunchAgent was installed but not loaded; automatic bootstrap/kickstart recovery failed.", + }, + ); + expect(readState).toHaveBeenCalledWith(service, { env: serviceEnv }); + expect(recover).toHaveBeenCalledWith({ result: "restarted", env: recoveredEnv }); + }, + ); it("does not touch non-macOS service managers", async () => { const readState = vi.fn(); @@ -799,63 +770,6 @@ describe("recoverInstalledLaunchAgentAfterUpdate", () => { expect(recover).not.toHaveBeenCalled(); }); - - it("returns an explicit failed recovery state when bootstrap repair fails", async () => { - const readState = vi.fn(async () => ({ - installed: true, - loadState: { status: "not-loaded" }, - running: false, - env: { OPENCLAW_PROFILE: "stomme" } as NodeJS.ProcessEnv, - command: null, - runtime: { status: "unknown", missingSupervision: true }, - })); - const recover = vi.fn(async () => null); - - await expect( - updateCommandServiceTesting.recoverInstalledLaunchAgentAfterUpdate({ - service: {} as never, - deps: { - platform: "darwin", - readState: readState as never, - recover: recover as never, - }, - }), - ).resolves.toEqual({ - attempted: true, - recovered: false, - detail: - "LaunchAgent was installed but not loaded; automatic bootstrap/kickstart recovery failed.", - }); - }); - - it("preserves system LaunchDaemon recovery guidance", async () => { - const readState = vi.fn(async () => ({ - installed: true, - loadState: { status: "not-loaded" }, - running: false, - env: { OPENCLAW_PROFILE: "stomme" } as NodeJS.ProcessEnv, - command: null, - runtime: { status: "unknown", missingSupervision: true }, - })); - const recover = vi.fn(async () => { - throw new Error("System LaunchDaemon system/ai.openclaw.stomme owns this label"); - }); - - await expect( - updateCommandServiceTesting.recoverInstalledLaunchAgentAfterUpdate({ - service: {} as never, - deps: { - platform: "darwin", - readState: readState as never, - recover: recover as never, - }, - }), - ).resolves.toEqual({ - attempted: true, - recovered: false, - detail: "System LaunchDaemon system/ai.openclaw.stomme owns this label", - }); - }); }); describe("recoverLaunchAgentAndRecheckGatewayHealth", () => { @@ -889,6 +803,7 @@ describe("recoverLaunchAgentAndRecheckGatewayHealth", () => { service, port: 18790, expectedVersion: "2026.5.3", + expectedBuildId: "new-build", env: { OPENCLAW_PROFILE: "stomme", OPENCLAW_PORT: "18790" }, deps: { recoverLaunchAgent, waitForHealthy }, }), @@ -906,6 +821,7 @@ describe("recoverLaunchAgentAndRecheckGatewayHealth", () => { service, port: 18790, expectedVersion: "2026.5.3", + expectedBuildId: "new-build", env: { OPENCLAW_PROFILE: "stomme", OPENCLAW_PORT: "18790" }, supervisorKeepsAlive: true, }); diff --git a/src/commands/doctor-update.test.ts b/src/commands/doctor-update.test.ts index 392d48cda4ca..3c94438c02fd 100644 --- a/src/commands/doctor-update.test.ts +++ b/src/commands/doctor-update.test.ts @@ -1,7 +1,7 @@ // Doctor update tests cover pre-doctor update prompts, state files, and declined update flows. import fs from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { RuntimeEnv } from "../runtime.js"; +import { defaultRuntime, type RuntimeEnv } from "../runtime.js"; import { EXTERNAL_SERVICE_REPAIR_NOTE } from "./doctor-service-repair-policy.js"; import { maybeOfferUpdateBeforeDoctor } from "./doctor-update.js"; @@ -11,12 +11,20 @@ const originalServiceRepairPolicy = process.env.OPENCLAW_SERVICE_REPAIR_POLICY; const mocks = vi.hoisted(() => ({ createUpdateProgress: vi.fn(), + gitMutationPolicy: vi.fn(), isDefaultInstallIdentity: vi.fn(() => true), + maybeRestartServiceAfterFailedMutableUpdate: vi.fn(), + maybeStopManagedServiceBeforeMutableUpdate: vi.fn(), note: vi.fn(), readGatewayServiceState: vi.fn(), - restartGatewayService: vi.fn(), + revalidateManagedGatewayServiceAfterUpdate: vi.fn(), + restartUpdatedGateway: vi.fn(), + stopGatewayService: vi.fn(), + waitForHealthyRestart: vi.fn(), + doctorCommand: vi.fn(), + createUpdateConfigSnapshot: vi.fn(), + createServiceConfigIO: vi.fn(), resolveGatewayService: vi.fn(), - summarizeGatewayServiceLayout: vi.fn(), runCommandWithTimeout: vi.fn(), runGatewayUpdate: vi.fn(), })); @@ -30,6 +38,9 @@ vi.mock("../config/paths.js", async () => { return { ...actual, isDefaultInstallIdentity: mocks.isDefaultInstallIdentity }; }); +vi.mock("../daemon/gateway-entrypoint.js", () => ({ + resolveGatewayInstallEntrypoint: async (root: string) => `${root}/dist/index.js`, +})); vi.mock("../process/exec.js", () => ({ runCommandWithTimeout: mocks.runCommandWithTimeout, })); @@ -38,15 +49,42 @@ vi.mock("../infra/update-runner.js", () => ({ runGatewayUpdate: mocks.runGatewayUpdate, })); +vi.mock("../config/io.js", async (importOriginal) => ({ + ...(await importOriginal()), + createConfigIO: mocks.createServiceConfigIO, +})); + +vi.mock("../cli/update-cli/managed-gateway-update.runtime.js", async () => ({ + ...(await vi.importActual( + "../cli/update-cli/update-command-service.js", + )), + maybeRestartServiceAfterFailedMutableUpdate: mocks.maybeRestartServiceAfterFailedMutableUpdate, + maybeStopManagedServiceBeforeMutableUpdate: mocks.maybeStopManagedServiceBeforeMutableUpdate, + revalidateManagedGatewayServiceAfterUpdate: mocks.revalidateManagedGatewayServiceAfterUpdate, +})); + +vi.mock("./doctor.js", () => ({ doctorCommand: mocks.doctorCommand })); +vi.mock("../cli/daemon-cli.js", () => ({ + runDaemonInstall: vi.fn(), + runDaemonRestart: vi.fn(), +})); +vi.mock("../cli/update-cli/update-command-config.js", () => ({ + createUpdateConfigSnapshot: mocks.createUpdateConfigSnapshot, +})); +vi.mock("../cli/daemon-cli/restart-health.js", () => ({ + waitForGatewayHealthyRestart: mocks.waitForHealthyRestart, + renderRestartDiagnostics: () => ["gateway not ready"], + terminateStaleGatewayPids: vi.fn(), +})); +vi.mock("../cli/update-cli/update-command-launch-agent-recovery.js", () => ({ + recoverInstalledLaunchAgentAfterUpdate: async () => ({ attempted: false, recovered: false }), +})); + vi.mock("../daemon/service.js", () => ({ readGatewayServiceState: mocks.readGatewayServiceState, resolveGatewayService: mocks.resolveGatewayService, })); -vi.mock("../daemon/service-layout.js", () => ({ - summarizeGatewayServiceLayout: mocks.summarizeGatewayServiceLayout, -})); - vi.mock("../../packages/terminal-core/src/note.js", () => ({ note: mocks.note, })); @@ -73,28 +111,46 @@ async function runOffer(params?: { beforeEach(async () => { mocks.createUpdateProgress.mockReset(); mocks.createUpdateProgress.mockReturnValue({ progress: {}, stop: vi.fn() }); + mocks.gitMutationPolicy.mockReset(); mocks.isDefaultInstallIdentity.mockReturnValue(true); + mocks.maybeRestartServiceAfterFailedMutableUpdate.mockReset(); + mocks.maybeStopManagedServiceBeforeMutableUpdate.mockReset(); mocks.note.mockReset(); mocks.readGatewayServiceState.mockReset(); - mocks.restartGatewayService.mockReset(); + mocks.revalidateManagedGatewayServiceAfterUpdate.mockReset(); + mocks.restartUpdatedGateway.mockReset(); + mocks.stopGatewayService.mockReset(); mocks.resolveGatewayService.mockReset(); - mocks.summarizeGatewayServiceLayout.mockReset(); mocks.runCommandWithTimeout.mockReset(); mocks.runGatewayUpdate.mockReset(); mocks.resolveGatewayService.mockReturnValue({ - restart: mocks.restartGatewayService, + restart: vi.fn(), + start: vi.fn(), + isLoaded: async () => false, }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: false, - loadState: { status: "not-loaded" }, + mocks.readGatewayServiceState.mockResolvedValue({ env: { OPENCLAW_PROFILE: "work" } }); + mocks.revalidateManagedGatewayServiceAfterUpdate.mockImplementation( + async ({ preManagedServiceStop }) => preManagedServiceStop.serviceUpdateVerdict, + ); + mocks.waitForHealthyRestart.mockReset().mockResolvedValue({ + healthy: true, + runtime: { status: "running" }, + staleGatewayPids: [], + gatewayVersion: "2026.4.24", + }); + mocks.doctorCommand.mockReset(); + mocks.createUpdateConfigSnapshot.mockReset().mockResolvedValue(undefined); + mocks.createServiceConfigIO + .mockReset() + .mockReturnValue({ readBestEffortConfig: async () => ({}) }); + vi.spyOn(defaultRuntime, "log").mockImplementation(() => {}); + vi.spyOn(defaultRuntime, "error").mockImplementation(() => {}); + mocks.maybeStopManagedServiceBeforeMutableUpdate.mockResolvedValue({ + stopped: false, + inspected: true, + runtimeInspected: true, running: false, - env: {}, - }); - mocks.summarizeGatewayServiceLayout.mockResolvedValue({ - execStart: "node /repo/link/dist/index.js gateway run", - entrypoint: "/repo/link/dist/index.js", - packageRoot: "/repo/link", - packageRootReal: "/repo/link", + serviceUpdateVerdict: { kind: "absent" }, }); Object.defineProperty(process.stdin, "isTTY", { configurable: true, @@ -128,17 +184,75 @@ afterEach(() => { describe("maybeOfferUpdateBeforeDoctor", () => { function mockGitCheckout() { vi.spyOn(fs, "realpath").mockImplementation(async (candidate) => String(candidate)); - mocks.runCommandWithTimeout.mockResolvedValue({ - stdout: "/repo/link\n", - stderr: "", - code: 0, - killed: false, - signal: null, - termination: "exit", - noOutputTimedOut: false, + mocks.runCommandWithTimeout.mockImplementation(async (argv, options) => { + if (argv[2] === "gateway" && argv[3] === "restart") { + await mocks.restartUpdatedGateway(options.env); + } + return { + stdout: "/repo/link\n", + stderr: "", + code: 0, + killed: false, + signal: null, + termination: "exit", + noOutputTimedOut: false, + }; }); } + function mockManagedService(params: { + verdict: + | { kind: "owned"; refreshDefinition: boolean; fingerprint: string } + | { kind: "unresolved"; fingerprint: string } + | { kind: "foreign" } + | { kind: "unavailable"; message: string }; + running?: boolean; + env?: NodeJS.ProcessEnv; + stopUnresolved?: boolean; + }) { + const running = params.running ?? true; + const owned = params.verdict.kind === "owned"; + const serviceEnv = params.env ?? { OPENCLAW_PROFILE: "work" }; + mocks.maybeStopManagedServiceBeforeMutableUpdate.mockImplementation( + async ({ phase }: { phase: "inspect" | "prepare" }) => { + const stopped = phase === "prepare" && running && (owned || params.stopUnresolved === true); + if (stopped) { + await mocks.stopGatewayService({ env: serviceEnv, stdout: process.stdout }); + } + return { + stopped, + inspected: true, + runtimeInspected: true, + running, + serviceEnv, + serviceUpdateVerdict: params.verdict, + ...(params.verdict.kind === "unavailable" + ? { serviceMutationAllowed: false, serviceMutationSkipMessage: params.verdict.message } + : {}), + }; + }, + ); + } + + function mockUpdateResult(result: { + status: "ok" | "error" | "skipped"; + mode: "git"; + root: string; + after?: { version: string; buildId?: string }; + recovery?: { serviceRestartSafe: false; reason: "source-rollback-failed" }; + }) { + mocks.runGatewayUpdate.mockImplementation( + async ({ + beforeGitMutation, + }: { + beforeGitMutation?: (target: object) => Promise; + }) => { + mocks.gitMutationPolicy(await beforeGitMutation?.({})); + return result; + }, + ); + } + it("treats a linked package root as a git checkout when realpaths match", async () => { const confirm = vi.fn().mockResolvedValue(false); vi.spyOn(fs, "realpath").mockImplementation(async (candidate) => { @@ -174,16 +288,7 @@ describe("maybeOfferUpdateBeforeDoctor", () => { const stop = vi.fn(); const progress = {}; mocks.createUpdateProgress.mockReturnValue({ progress, stop }); - vi.spyOn(fs, "realpath").mockImplementation(async (candidate) => String(candidate)); - mocks.runCommandWithTimeout.mockResolvedValue({ - stdout: "/repo/link\n", - stderr: "", - code: 0, - killed: false, - signal: null, - termination: "exit", - noOutputTimedOut: false, - }); + mockGitCheckout(); mocks.runGatewayUpdate.mockRejectedValue(new Error("update exploded")); const confirm = vi.fn().mockResolvedValue(true); @@ -198,6 +303,11 @@ describe("maybeOfferUpdateBeforeDoctor", () => { ); expect(mocks.createUpdateProgress).toHaveBeenCalledWith(true); expect(stop).toHaveBeenCalledTimes(1); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).not.toHaveBeenCalled(); + expect(mocks.note).not.toHaveBeenCalledWith( + expect.stringContaining("source checkout may be partially mutated"), + "Update", + ); }); it("disables update progress when stdout is not a TTY", async () => { @@ -205,16 +315,7 @@ describe("maybeOfferUpdateBeforeDoctor", () => { configurable: true, value: false, }); - vi.spyOn(fs, "realpath").mockImplementation(async (candidate) => String(candidate)); - mocks.runCommandWithTimeout.mockResolvedValue({ - stdout: "/repo/link\n", - stderr: "", - code: 0, - killed: false, - signal: null, - termination: "exit", - noOutputTimedOut: false, - }); + mockGitCheckout(); mocks.runGatewayUpdate.mockResolvedValue({ status: "skipped", mode: "git", @@ -254,257 +355,427 @@ describe("maybeOfferUpdateBeforeDoctor", () => { ); }); - it("restarts a running managed gateway after a successful git update", async () => { + it.each([ + { definition: "writable", refreshDefinition: true }, + { definition: "sealed", refreshDefinition: false }, + ])( + "restarts an owned $definition gateway using its current environment", + async ({ refreshDefinition }) => { + mockGitCheckout(); + const verdict = { kind: "owned" as const, refreshDefinition, fingerprint: "opaque" }; + mockManagedService({ verdict }); + mockUpdateResult({ status: "ok", mode: "git", root: "/repo/link" }); + const currentEnv = { + OPENCLAW_PROFILE: "work", + ...(refreshDefinition ? { CURRENT_MANAGED_VALUE: "validated" } : {}), + }; + mocks.readGatewayServiceState.mockResolvedValueOnce({ env: currentEnv }); + + await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ + updated: true, + handled: true, + }); + expect(mocks.maybeStopManagedServiceBeforeMutableUpdate.mock.calls).toEqual([ + [expect.objectContaining({ phase: "inspect", root: "/repo/link" })], + [expect.objectContaining({ phase: "prepare", root: "/repo/link" })], + ]); + expect(mocks.stopGatewayService).toHaveBeenCalledOnce(); + expect(mocks.restartUpdatedGateway).toHaveBeenCalledOnce(); + expect( + mocks.runCommandWithTimeout.mock.calls.some(([args]) => + args.includes("--preserve-definition"), + ), + ).toBe(true); + expect(mocks.restartUpdatedGateway.mock.calls[0]?.[0]).toMatchObject(currentEnv); + const policy = { allowGatewayServiceRepair: refreshDefinition, allowGatewayActivation: true }; + expect(mocks.runGatewayUpdate).toHaveBeenCalledWith(expect.objectContaining(policy)); + expect(mocks.gitMutationPolicy).toHaveBeenCalledWith(policy); + expect(mocks.revalidateManagedGatewayServiceAfterUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + root: "/repo/link", + preManagedServiceStop: expect.objectContaining({ serviceUpdateVerdict: verdict }), + }), + ); + expect(mocks.note).toHaveBeenCalledWith( + "Restarted the running gateway service after updating OpenClaw.", + "Update", + ); + }, + ); + + it.each(["healthy", "exited", "old-version"] as const)( + "verifies doctor update restart readiness: %s", + async (outcome) => { + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + mockGitCheckout(); + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: false, fingerprint: "opaque" }, + }); + mockUpdateResult({ + status: "ok", + mode: "git", + root: "/repo/link", + after: { version: "2026.4.24", buildId: "new-build" }, + }); + mocks.waitForHealthyRestart.mockResolvedValue({ + healthy: outcome === "healthy", + runtime: { status: outcome === "exited" ? "stopped" : "running" }, + gatewayVersion: outcome === "old-version" ? "2026.4.23" : "2026.4.24", + versionMismatch: outcome === "old-version", + staleGatewayPids: [], + }); + + await expect( + runOffer({ confirm: vi.fn().mockResolvedValue(true), runtime }), + ).resolves.toEqual({ updated: true, handled: true }); + + expect(mocks.runGatewayUpdate).toHaveBeenCalledOnce(); + expect(mocks.waitForHealthyRestart).toHaveBeenCalledWith( + expect.objectContaining({ + expectedVersion: "2026.4.24", + expectedBuildId: "new-build", + env: { OPENCLAW_PROFILE: "work" }, + requireRunningService: true, + }), + ); + expect(mocks.doctorCommand).not.toHaveBeenCalled(); + if (outcome === "healthy") { + expect(runtime.exit).not.toHaveBeenCalled(); + expect(mocks.note).toHaveBeenCalledWith( + "Restarted the running gateway service after updating OpenClaw.", + "Update", + ); + expect(mocks.waitForHealthyRestart.mock.invocationCallOrder[0]).toBeLessThan( + mocks.note.mock.invocationCallOrder.at(-1)!, + ); + } else { + expect(runtime.exit).toHaveBeenCalledWith(1); + expect(runtime.error).toHaveBeenCalledWith( + expect.stringContaining("Update completed, but gateway service restart failed"), + ); + expect(mocks.note).not.toHaveBeenCalledWith( + "Restarted the running gateway service after updating OpenClaw.", + "Update", + ); + } + }, + ); + + it.each([ + { source: "ExecStart", args: ["--port=19201"], envPort: "19202", expected: 19201 }, + { source: "service environment", args: [], envPort: "19202", expected: 19202 }, + { source: "service config", args: [], envPort: undefined, expected: 19203 }, + ])( + "verifies the preserved doctor service port from $source", + async ({ args, envPort, expected }) => { + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + const serviceEnv = { OPENCLAW_PROFILE: "work", OPENCLAW_GATEWAY_PORT: envPort }; + mockGitCheckout(); + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: false, fingerprint: "opaque" }, + env: serviceEnv, + }); + mockUpdateResult({ + status: "ok", + mode: "git", + root: "/repo/link", + after: { version: "2026.4.24" }, + }); + mocks.readGatewayServiceState.mockResolvedValue({ + env: serviceEnv, + command: { + programArguments: ["/usr/bin/node", "/repo/link/dist/index.js", "gateway", ...args], + }, + }); + mocks.createServiceConfigIO.mockReturnValue({ + readBestEffortConfig: async () => ({ gateway: { port: 19203 } }), + }); + + await expect( + runOffer({ confirm: vi.fn().mockResolvedValue(true), runtime }), + ).resolves.toEqual({ updated: true, handled: true }); + + expect(mocks.waitForHealthyRestart).toHaveBeenCalledWith( + expect.objectContaining({ port: expected, expectedVersion: "2026.4.24", env: serviceEnv }), + ); + if (envPort === undefined) { + expect(mocks.createServiceConfigIO).toHaveBeenCalledWith( + expect.objectContaining({ env: serviceEnv, observe: false }), + ); + } + expect(runtime.exit).not.toHaveBeenCalled(); + expect(mocks.doctorCommand).not.toHaveBeenCalled(); + }, + ); + + it.each([ + { definition: "preserved", refreshDefinition: false, failure: "ownership revalidation" }, + { definition: "writable", refreshDefinition: true, failure: "ownership revalidation" }, + { definition: "writable", refreshDefinition: true, failure: "service inspection" }, + ])( + "leaves a stopped $definition gateway down after failed $failure", + async ({ refreshDefinition, failure }) => { + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + mockGitCheckout(); + mockManagedService({ verdict: { kind: "owned", refreshDefinition, fingerprint: "opaque" } }); + mockUpdateResult({ status: "ok", mode: "git", root: "/repo/link" }); + const inspectionError = new Error(`${failure} unavailable`); + if (failure === "service inspection") { + mocks.readGatewayServiceState.mockRejectedValueOnce(inspectionError); + } else { + mocks.revalidateManagedGatewayServiceAfterUpdate.mockRejectedValueOnce(inspectionError); + } + + await expect( + runOffer({ confirm: vi.fn().mockResolvedValue(true), runtime }), + ).resolves.toEqual({ + updated: true, + handled: true, + }); + expect(mocks.stopGatewayService).toHaveBeenCalledOnce(); + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).not.toHaveBeenCalled(); + expect(runtime.error).toHaveBeenCalledWith(expect.stringContaining(inspectionError.message)); + expect(runtime.error.mock.invocationCallOrder[0]).toBeLessThan( + runtime.exit.mock.invocationCallOrder[0] ?? Number.POSITIVE_INFINITY, + ); + expect(runtime.exit).toHaveBeenCalledWith(1); + }, + ); + + it.each([{ kind: "foreign" as const }, { kind: "unresolved" as const, fingerprint: "opaque" }])( + "does not stop, repair or activate a $kind service", + async (verdict) => { + mockGitCheckout(); + mockManagedService({ verdict }); + mockUpdateResult({ status: "ok", mode: "git", root: "/repo/link" }); + + await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ + updated: true, + handled: true, + }); + expect(mocks.runGatewayUpdate).toHaveBeenCalledWith( + expect.objectContaining({ + allowGatewayServiceRepair: false, + allowGatewayActivation: false, + }), + ); + expect(mocks.stopGatewayService).not.toHaveBeenCalled(); + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); + }, + ); + + it.each([false, true])( + "restores a stopped unresolved gateway only when its identity survives the doctor update (changed: %s)", + async (identityChanged) => { + const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + const serviceEnv = { + OPENCLAW_PROFILE: "work", + OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway-work.service", + }; + mockGitCheckout(); + mockManagedService({ + verdict: { kind: "unresolved", fingerprint: "opaque" }, + env: serviceEnv, + stopUnresolved: true, + }); + mockUpdateResult({ status: "ok", mode: "git", root: "/repo/link" }); + mocks.readGatewayServiceState.mockResolvedValueOnce({ env: serviceEnv }); + if (identityChanged) { + mocks.revalidateManagedGatewayServiceAfterUpdate.mockRejectedValueOnce( + new Error("The stopped gateway service-manager identity changed."), + ); + } + + await expect( + runOffer({ confirm: vi.fn().mockResolvedValue(true), runtime }), + ).resolves.toEqual({ + updated: true, + handled: true, + }); + + expect(mocks.stopGatewayService).toHaveBeenCalledOnce(); + expect(mocks.gitMutationPolicy).toHaveBeenCalledWith({ + allowGatewayServiceRepair: false, + allowGatewayActivation: false, + }); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).not.toHaveBeenCalled(); + if (identityChanged) { + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); + expect(runtime.error).toHaveBeenCalledWith( + expect.stringContaining("service-manager identity changed"), + ); + expect(runtime.exit).toHaveBeenCalledWith(1); + } else { + expect(mocks.restartUpdatedGateway.mock.calls[0]?.[0]).toMatchObject({ + ...serviceEnv, + }); + expect(runtime.exit).not.toHaveBeenCalled(); + } + }, + ); + + it("repairs without activating a stopped gateway owned by this checkout", async () => { mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", - }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: true, - loadState: { status: "loaded" }, - running: true, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/link/dist/index.js", "gateway", "run"], - }, + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: true, fingerprint: "opaque" }, + running: false, }); + mockUpdateResult({ status: "ok", mode: "git", root: "/repo/link" }); await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ updated: true, handled: true, }); - expect(mocks.restartGatewayService).toHaveBeenCalledWith({ - env: { OPENCLAW_PROFILE: "work" }, - stdout: process.stdout, - }); expect(mocks.runGatewayUpdate).toHaveBeenCalledWith( expect.objectContaining({ allowGatewayServiceRepair: true, - allowGatewayActivation: true, + allowGatewayActivation: false, }), ); + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); + }); + + it("updates with a visible skip when service inspection is unavailable", async () => { + const message = + "Gateway service management skipped; inspect service access before restarting manually."; + mockGitCheckout(); + mockManagedService({ verdict: { kind: "unavailable", message } }); + mockUpdateResult({ status: "ok", mode: "git", root: "/repo/link" }); + + await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ + updated: true, + handled: true, + }); + + expect(mocks.note).toHaveBeenCalledWith(message, "Update"); + expect(mocks.runGatewayUpdate).toHaveBeenCalledOnce(); + expect(mocks.gitMutationPolicy).toHaveBeenCalledWith({ + allowGatewayServiceRepair: false, + allowGatewayActivation: false, + }); + expect(mocks.stopGatewayService).not.toHaveBeenCalled(); + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); + }); + + it("leaves the stopped gateway down when a git mutation throws without recovery proof", async () => { + mockGitCheckout(); + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: false, fingerprint: "opaque" }, + }); + mocks.runGatewayUpdate.mockImplementation( + async ({ + beforeGitMutation, + }: { + beforeGitMutation: (target: object) => Promise; + }) => { + await beforeGitMutation({}); + throw new Error("checkout mutation failed"); + }, + ); + + await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).rejects.toThrow( + "checkout mutation failed", + ); + + expect(mocks.stopGatewayService).toHaveBeenCalledOnce(); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).not.toHaveBeenCalled(); + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); expect(mocks.note).toHaveBeenCalledWith( - "Restarted the running gateway service after updating OpenClaw.", + expect.stringContaining("source checkout may be partially mutated"), + "Update", + ); + expect(mocks.note).toHaveBeenCalledWith( + expect.stringContaining("restart the gateway manually"), "Update", ); }); - it("restarts an owned gateway that stops during the git update", async () => { + it("recovers a stopped gateway when mutation preparation itself fails before authorization", async () => { mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: false, fingerprint: "opaque" }, }); - mocks.readGatewayServiceState - .mockResolvedValueOnce({ - installed: true, - loadState: { status: "loaded" }, - running: true, + mocks.maybeStopManagedServiceBeforeMutableUpdate.mockImplementationOnce(async () => ({ + stopped: false, + inspected: true, + runtimeInspected: true, + running: true, + serviceEnv: { OPENCLAW_PROFILE: "work" }, + serviceUpdateVerdict: { kind: "owned", refreshDefinition: false, fingerprint: "opaque" }, + })); + mocks.maybeStopManagedServiceBeforeMutableUpdate.mockImplementationOnce(async () => { + await mocks.stopGatewayService({ env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/link/dist/index.js", "gateway", "run"], - }, - }) - .mockResolvedValueOnce({ - installed: true, - loadState: { status: "loaded" }, - running: false, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/link/dist/index.js", "gateway", "run"], - }, + stdout: process.stdout, }); - - await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ - updated: true, - handled: true, - }); - - expect(mocks.restartGatewayService).toHaveBeenCalledWith({ - env: { OPENCLAW_PROFILE: "work" }, - stdout: process.stdout, - }); - }); - - it("does not activate or restart a running gateway owned by another checkout", async () => { - mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", - }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: true, - loadState: { status: "loaded" }, - running: true, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/other/dist/index.js", "gateway", "run"], - }, - }); - mocks.summarizeGatewayServiceLayout.mockResolvedValue({ - execStart: "node /repo/other/dist/index.js gateway run", - entrypoint: "/repo/other/dist/index.js", - packageRoot: "/repo/other", - packageRootReal: "/repo/other", - }); - - await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ - updated: true, - handled: true, - }); - - expect(mocks.runGatewayUpdate).toHaveBeenCalledWith( - expect.objectContaining({ - allowGatewayServiceRepair: false, - allowGatewayActivation: false, - }), - ); - expect(mocks.restartGatewayService).not.toHaveBeenCalled(); - }); - - it("does not repair or restart a service with an ambiguous relative entrypoint", async () => { - mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", - }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: true, - loadState: { status: "loaded" }, - running: true, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "dist/index.js", "gateway", "run"], - }, - }); - mocks.summarizeGatewayServiceLayout.mockResolvedValue({ - execStart: "node dist/index.js gateway run", - entrypoint: "dist/index.js", - packageRoot: "/repo/link", - packageRootReal: "/repo/link", - }); - - await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ - updated: true, - handled: true, - }); - - expect(mocks.runGatewayUpdate).toHaveBeenCalledWith( - expect.objectContaining({ - allowGatewayServiceRepair: false, - allowGatewayActivation: false, - }), - ); - expect(mocks.restartGatewayService).not.toHaveBeenCalled(); - }); - - it("repairs without activating a stopped gateway owned by this checkout", async () => { - mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", - }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: true, - loadState: { status: "loaded" }, - running: false, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/link/dist/index.js", "gateway", "run"], - }, - }); - - await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ - updated: true, - handled: true, - }); - - expect(mocks.runGatewayUpdate).toHaveBeenCalledWith( - expect.objectContaining({ - allowGatewayServiceRepair: true, - allowGatewayActivation: false, - }), - ); - expect(mocks.restartGatewayService).not.toHaveBeenCalled(); - }); - - it("does not repair or activate when the initial service inspection is unknown", async () => { - mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", - }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: true, - loadState: { status: "unknown", detail: "systemctl is-enabled failed" }, - running: true, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/link/dist/index.js", "gateway", "run"], - }, - }); - - await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ - updated: true, - handled: true, - }); - - expect(mocks.runGatewayUpdate).toHaveBeenCalledWith( - expect.objectContaining({ - allowGatewayServiceRepair: false, - allowGatewayActivation: false, - }), - ); - expect(mocks.restartGatewayService).not.toHaveBeenCalled(); - }); - - it("does not restart when the post-update service inspection is unknown", async () => { - mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", - }); - mocks.readGatewayServiceState - .mockResolvedValueOnce({ - installed: true, - loadState: { status: "loaded" }, + return { + stopped: true, + inspected: true, + runtimeInspected: true, running: true, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/link/dist/index.js", "gateway", "run"], - }, - }) - .mockResolvedValueOnce({ - installed: true, - loadState: { status: "unknown", detail: "launchctl inspection failed" }, - running: true, - env: { OPENCLAW_PROFILE: "work" }, - command: { - programArguments: ["node", "/repo/link/dist/index.js", "gateway", "run"], - }, - }); + serviceEnv: { OPENCLAW_PROFILE: "work" }, + serviceUpdateVerdict: { kind: "owned", refreshDefinition: false, fingerprint: "opaque" }, + blockMessage: "mutation preparation blocked", + }; + }); + mocks.runGatewayUpdate.mockImplementation( + async ({ beforeGitMutation }: { beforeGitMutation: (target: object) => Promise }) => + await beforeGitMutation({}), + ); + + await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).rejects.toThrow( + "mutation preparation blocked", + ); + + expect(mocks.stopGatewayService).toHaveBeenCalledOnce(); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).toHaveBeenCalledWith({ + preManagedServiceStop: expect.objectContaining({ stopped: true }), + jsonMode: false, + }); + expect(mocks.note).not.toHaveBeenCalledWith( + expect.stringContaining("source checkout may be partially mutated"), + "Update", + ); + }); + + it("recovers the previously stopped service when the update returns an error", async () => { + mockGitCheckout(); + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: true, fingerprint: "opaque" }, + }); + mockUpdateResult({ status: "error", mode: "git", root: "/repo/link" }); await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ updated: true, - handled: true, + handled: false, }); - expect(mocks.runGatewayUpdate).toHaveBeenCalledWith( - expect.objectContaining({ - allowGatewayServiceRepair: true, - allowGatewayActivation: true, - }), - ); - expect(mocks.restartGatewayService).not.toHaveBeenCalled(); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).toHaveBeenCalledWith({ + root: "/repo/link", + preManagedServiceStop: expect.objectContaining({ stopped: true }), + jsonMode: false, + }); + }); + + it("does not restart a stopped service when source rollback could not be verified", async () => { + mockGitCheckout(); + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: true, fingerprint: "opaque" }, + }); + mockUpdateResult({ + status: "error", + mode: "git", + root: "/repo/link", + recovery: { serviceRestartSafe: false, reason: "source-rollback-failed" }, + }); + + await runOffer({ confirm: vi.fn().mockResolvedValue(true) }); + + expect(mocks.stopGatewayService).toHaveBeenCalledOnce(); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).not.toHaveBeenCalled(); + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); }); it("leaves a running gateway alone when service repair is externally managed", async () => { @@ -515,12 +786,6 @@ describe("maybeOfferUpdateBeforeDoctor", () => { mode: "git", root: "/repo/link", }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: true, - loadState: { status: "loaded" }, - running: true, - env: { OPENCLAW_PROFILE: "work" }, - }); await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true) })).resolves.toEqual({ updated: true, @@ -528,8 +793,8 @@ describe("maybeOfferUpdateBeforeDoctor", () => { }); expect(mocks.resolveGatewayService).not.toHaveBeenCalled(); - expect(mocks.readGatewayServiceState).not.toHaveBeenCalled(); - expect(mocks.restartGatewayService).not.toHaveBeenCalled(); + expect(mocks.maybeStopManagedServiceBeforeMutableUpdate).not.toHaveBeenCalled(); + expect(mocks.restartUpdatedGateway).not.toHaveBeenCalled(); expect(mocks.note).toHaveBeenCalledWith(EXTERNAL_SERVICE_REPAIR_NOTE, "Update"); }); @@ -540,18 +805,11 @@ describe("maybeOfferUpdateBeforeDoctor", () => { exit: vi.fn(), }; mockGitCheckout(); - mocks.runGatewayUpdate.mockResolvedValue({ - status: "ok", - mode: "git", - root: "/repo/link", + mockManagedService({ + verdict: { kind: "owned", refreshDefinition: true, fingerprint: "opaque" }, }); - mocks.readGatewayServiceState.mockResolvedValue({ - installed: true, - loadState: { status: "loaded" }, - running: true, - env: { OPENCLAW_PROFILE: "work" }, - }); - mocks.restartGatewayService.mockRejectedValue(new Error("schtasks failed")); + mockUpdateResult({ status: "ok", mode: "git", root: "/repo/link" }); + mocks.restartUpdatedGateway.mockRejectedValue(new Error("schtasks failed")); await expect(runOffer({ confirm: vi.fn().mockResolvedValue(true), runtime })).resolves.toEqual({ updated: true, @@ -559,8 +817,10 @@ describe("maybeOfferUpdateBeforeDoctor", () => { }); expect(runtime.error).toHaveBeenCalledWith( - "Update completed, but gateway service restart failed: Error: schtasks failed", + expect.stringContaining("Update completed, but gateway service restart failed"), ); + expect(defaultRuntime.error).toHaveBeenCalledWith(expect.stringContaining("schtasks failed")); + expect(mocks.maybeRestartServiceAfterFailedMutableUpdate).not.toHaveBeenCalled(); expect(runtime.exit).toHaveBeenCalledWith(1); }); }); diff --git a/src/commands/doctor-update.ts b/src/commands/doctor-update.ts index a93a14295b8a..e57e5ddcc796 100644 --- a/src/commands/doctor-update.ts +++ b/src/commands/doctor-update.ts @@ -6,9 +6,9 @@ import { note } from "../../packages/terminal-core/src/note.js"; import { formatCliCommand } from "../cli/command-format.js"; import { createUpdateProgress } from "../cli/update-cli/progress.js"; import { isDefaultInstallIdentity } from "../config/paths.js"; -import { summarizeGatewayServiceLayout } from "../daemon/service-layout.js"; import { readGatewayServiceState, resolveGatewayService } from "../daemon/service.js"; import { isTruthyEnvValue } from "../infra/env.js"; +import { UPDATE_RUNNER_TIMEOUT_MS } from "../infra/update-runner-command.js"; import { runGatewayUpdate } from "../infra/update-runner.js"; import type { UpdateRunResult } from "../infra/update-runner.js"; import { runCommandWithTimeout } from "../process/exec.js"; @@ -44,93 +44,6 @@ async function detectOpenClawGitCheckout(root: string): Promise<"git" | "not-git : "not-git"; } -type GatewayServiceUpdatePolicy = { - allowGatewayServiceRepair: boolean; - allowGatewayActivation: boolean; -}; - -type GatewayServiceUpdateInspection = GatewayServiceUpdatePolicy & { - service?: ReturnType; - state?: Awaited>; -}; - -const NO_GATEWAY_SERVICE_UPDATE: GatewayServiceUpdatePolicy = { - allowGatewayServiceRepair: false, - allowGatewayActivation: false, -}; - -async function inspectGatewayServiceForUpdate( - root: string, -): Promise { - if (!isDefaultInstallIdentity(process.env) || isServiceRepairExternallyManaged()) { - return NO_GATEWAY_SERVICE_UPDATE; - } - try { - const service = resolveGatewayService(); - const state = await readGatewayServiceState(service, { env: process.env }); - if (state.loadState.status === "unknown" || !state.installed) { - return NO_GATEWAY_SERVICE_UPDATE; - } - const layout = await summarizeGatewayServiceLayout(state.command); - const serviceRoot = layout?.packageRootReal ?? layout?.packageRoot; - const serviceEntrypoint = layout?.entrypoint; - if ( - !serviceRoot || - !serviceEntrypoint || - (!path.isAbsolute(serviceEntrypoint) && !path.win32.isAbsolute(serviceEntrypoint)) - ) { - return NO_GATEWAY_SERVICE_UPDATE; - } - const [serviceRootReal, updateRootReal] = await Promise.all([ - resolveComparablePath(serviceRoot), - resolveComparablePath(root), - ]); - if (serviceRootReal !== updateRootReal) { - return NO_GATEWAY_SERVICE_UPDATE; - } - return { - allowGatewayServiceRepair: true, - allowGatewayActivation: state.running, - service, - state, - }; - } catch { - // Repair or activation can disrupt a different checkout, so unknown ownership fails closed. - return NO_GATEWAY_SERVICE_UPDATE; - } -} - -async function restartRunningGatewayServiceAfterUpdate( - runtime: RuntimeEnv, - root: string, - wasOwnedAndRunning: boolean, -): Promise { - if (isServiceRepairExternallyManaged()) { - note(EXTERNAL_SERVICE_REPAIR_NOTE, "Update"); - return true; - } - if (!wasOwnedAndRunning) { - return true; - } - const inspection = await inspectGatewayServiceForUpdate(root); - // Revalidate ownership after checkout replacement, but restart even when the - // previously running service stopped during the update. - if (!inspection.allowGatewayServiceRepair || !inspection.service || !inspection.state) { - return true; - } - try { - await inspection.service.restart({ - env: inspection.state.env, - stdout: process.stdout, - }); - note("Restarted the running gateway service after updating OpenClaw.", "Update"); - return true; - } catch (err) { - runtime.error(`Update completed, but gateway service restart failed: ${String(err)}`); - return false; - } -} - /** Offers to update OpenClaw before doctor when running interactively from an updatable install. */ export async function maybeOfferUpdateBeforeDoctor(params: { runtime: RuntimeEnv; @@ -159,50 +72,146 @@ export async function maybeOfferUpdateBeforeDoctor(params: { if (!shouldUpdate) { return { updated: false }; } + const updateRoot = params.root; + const externallyManaged = isServiceRepairExternallyManaged(); + const serviceLifecycle = + isDefaultInstallIdentity(process.env) && !externallyManaged + ? await import("../cli/update-cli/managed-gateway-update.runtime.js") + : undefined; + let inspection = await serviceLifecycle?.maybeStopManagedServiceBeforeMutableUpdate({ + updateInstallKind: "git", + root: updateRoot, + shouldRestart: true, + jsonMode: false, + phase: "inspect", + }); + if (inspection?.blockMessage) { + note(inspection.blockMessage, "Update"); + return { updated: false }; + } + if (inspection?.serviceMutationSkipMessage) { + note(inspection.serviceMutationSkipMessage, "Update"); + } + let gitMutationAuthorized = false; note("Running update…", "Update"); - const serviceInspection = await inspectGatewayServiceForUpdate(params.root); - const serviceUpdatePolicy: GatewayServiceUpdatePolicy = { - allowGatewayServiceRepair: serviceInspection.allowGatewayServiceRepair, - allowGatewayActivation: serviceInspection.allowGatewayActivation, - }; const { progress, stop } = createUpdateProgress(process.stdout.isTTY); let result: UpdateRunResult; try { result = await runGatewayUpdate({ - cwd: params.root, + cwd: updateRoot, argv1: process.argv[1], progress, - ...serviceUpdatePolicy, + allowGatewayServiceRepair: + inspection?.serviceUpdateVerdict?.kind === "owned" && + inspection.serviceUpdateVerdict.refreshDefinition, + allowGatewayActivation: Boolean( + inspection?.running && inspection.serviceUpdateVerdict?.kind === "owned", + ), + beforeGitMutation: serviceLifecycle + ? async () => { + const previousSkip = inspection?.serviceMutationSkipMessage; + inspection = await serviceLifecycle.maybeStopManagedServiceBeforeMutableUpdate({ + updateInstallKind: "git", + root: updateRoot, + shouldRestart: true, + jsonMode: false, + phase: "prepare", + }); + if (inspection.blockMessage) { + throw new Error(inspection.blockMessage); + } + if ( + inspection.serviceMutationSkipMessage !== previousSkip && + inspection.serviceMutationSkipMessage + ) { + note(inspection.serviceMutationSkipMessage, "Update"); + } + gitMutationAuthorized = true; + return serviceLifecycle.resolvePreparedGatewayUpdatePolicy(inspection, true); + } + : undefined, }); + } catch (err) { + if (inspection?.stopped && gitMutationAuthorized) { + note( + "The gateway service remains stopped because the source checkout may be partially mutated. " + + `Inspect and repair the checkout, then restart the gateway manually with \`${formatCliCommand("openclaw gateway restart")}\`.`, + "Update", + ); + } else if (inspection?.stopped) { + await serviceLifecycle?.maybeRestartServiceAfterFailedMutableUpdate({ + preManagedServiceStop: inspection, + jsonMode: false, + }); + } + throw err; } finally { stop(); } - note( - [ - `Status: ${result.status}`, - `Mode: ${result.mode}`, - result.root ? `Root: ${result.root}` : null, - result.reason ? `Reason: ${result.reason}` : null, - ] - .filter(Boolean) - .join("\n"), - "Update result", - ); - if (result.status === "ok") { - const restarted = await restartRunningGatewayServiceAfterUpdate( - params.runtime, - params.root, - serviceUpdatePolicy.allowGatewayActivation, - ); - if (!restarted) { - params.outro("Update completed, but gateway service restart failed."); + const resultDetails = [ + `Status: ${result.status}`, + `Mode: ${result.mode}`, + result.root && `Root: ${result.root}`, + result.reason && `Reason: ${result.reason}`, + ].filter(Boolean); + note(resultDetails.join("\n"), "Update result"); + if (result.status !== "ok") { + if (result.recovery?.serviceRestartSafe !== false) { + await serviceLifecycle?.maybeRestartServiceAfterFailedMutableUpdate({ + root: result.root, + preManagedServiceStop: inspection, + jsonMode: false, + }); + } + return { updated: true, handled: false }; + } + if (externallyManaged) { + note(EXTERNAL_SERVICE_REPAIR_NOTE, "Update"); + } else if (inspection?.stopped && inspection.serviceEnv && serviceLifecycle) { + try { + const service = resolveGatewayService(); + const serviceState = await readGatewayServiceState(service, { + env: inspection.serviceEnv, + requireEffective: true, + }); + const verdict = await serviceLifecycle.revalidateManagedGatewayServiceAfterUpdate({ + state: serviceState, + root: updateRoot, + preManagedServiceStop: inspection, + }); + // Doctor already ran during the update; reuse activation/health without another repair. + const activated = await serviceLifecycle.maybeRestartService({ + shouldRestart: true, + result, + channel: "dev", + opts: {}, + refreshServiceEnv: false, + serviceUpdateVerdict: + verdict.kind === "owned" ? { ...verdict, refreshDefinition: false } : verdict, + serviceEnv: serviceState.env, + gatewayPort: await serviceLifecycle.resolveUpdatedGatewayRestartPort({ + serviceEnv: serviceState.env, + serviceCommand: serviceState.command, + }), + requireRunningServiceAfterRestart: true, + timeoutMs: UPDATE_RUNNER_TIMEOUT_MS, + }); + if (!activated) { + throw new Error( + "Gateway restart was not verified; run `openclaw gateway status --deep` before restarting manually.", + ); + } + note("Restarted the running gateway service after updating OpenClaw.", "Update"); + } catch (err) { + const message = "Update completed, but gateway service restart failed"; + params.runtime.error(`${message}: ${String(err)}`); + params.outro(`${message}.`); params.runtime.exit(1); return { updated: true, handled: true }; } - params.outro("Update completed (doctor already ran as part of the update)."); - return { updated: true, handled: true }; } - return { updated: true, handled: false }; + params.outro("Update completed (doctor already ran as part of the update)."); + return { updated: true, handled: true }; } if (git === "not-git") { diff --git a/src/daemon/launchd-lifecycle.ts b/src/daemon/launchd-lifecycle.ts index 241a8571bcd6..ea6b39b8e138 100644 --- a/src/daemon/launchd-lifecycle.ts +++ b/src/daemon/launchd-lifecycle.ts @@ -215,10 +215,8 @@ export async function startLaunchAgent({ reportMutation("enable"); } - const start = await execLaunchctl(["kickstart", serviceTarget]); - if (start.code === 0) { - reportMutation("kickstart"); - } else if (isLaunchctlNotLoaded(start)) { + let start = await execLaunchctl(["kickstart", serviceTarget]); + if (isLaunchctlNotLoaded(start)) { await bootstrapLaunchAgentOrThrow({ domain, serviceTarget, @@ -227,14 +225,19 @@ export async function startLaunchAgent({ onMutation: reportMutation, skipEnable: enabled, }); - } else { + // Loading does not start demand-only jobs. Without -k, an auto-started job is left running. + start = await execLaunchctl(["kickstart", serviceTarget]); + } + if (start.code !== 0) { throw new Error(`launchctl kickstart failed: ${start.stderr || start.stdout}`.trim()); } + reportMutation("kickstart"); writeLaunchAgentActionLine(stdout, "Started LaunchAgent", serviceTarget); } export async function restartLaunchAgent({ + preserveDefinition, stdout, env, warn, @@ -248,17 +251,53 @@ export async function restartLaunchAgent({ const reportMutation = createGatewayLifecycleMutationReporter(onMutation); await assertNoSystemLaunchDaemonOwnership(label); - // Restart requests issued from inside the managed gateway process tree need a - // detached handoff. A direct `kickstart -k` would terminate the caller before - // it can finish the restart command. - if (isCurrentProcessLaunchdServiceLabel(label)) { - const plistReloadNeeded = await rewriteLaunchAgentPlistForRestart({ + const detached = isCurrentProcessLaunchdServiceLabel(label); + if (!detached) { + const { port: cleanupPort, probeHosts } = await resolveLaunchAgentGatewayContext(serviceEnv); + if (cleanupPort !== null) { + cleanStaleGatewayProcessesSync(cleanupPort, { + // Resolve after lsof captures its listener snapshot. A KeepAlive respawn + // during enumeration must be protected before candidate filtering/signals. + resolveProtectedPid: () => readLaunchAgentPidForCleanupSync(serviceTarget), + }); + const diagnostics = await inspectPortUsage(cleanupPort, { + probeHosts, + }).catch(() => null); + if (diagnostics?.status === "busy") { + const runtime = await readLaunchAgentRuntime(serviceEnv); + const managedPid = runtime.pid; + // Only the current supervised PID may keep the port busy before a + // disruptive restart. Re-read after cleanup to close over a concurrent + // launchd respawn rather than trusting the protected pre-cleanup PID. + const ownedByLaunchAgent = + managedPid !== undefined && + diagnostics.listeners.length > 0 && + diagnostics.listeners.every((listener) => listener.pid === managedPid); + if (!ownedByLaunchAgent) { + throw new Error( + [ + `gateway port ${cleanupPort} is busy but is not verifiably owned by LaunchAgent ${label}`, + ...formatPortDiagnostics(diagnostics), + ].join("\n"), + ); + } + } + } + } + // Preservation permits native activation only, including detached handoffs. + const plistReloadNeeded = + !preserveDefinition && + (await rewriteLaunchAgentPlistForRestart({ env: serviceEnv, label, plistPath, stdout, warn, - }); + })); + // Restart requests issued from inside the managed gateway process tree need a + // detached handoff. A direct `kickstart -k` would terminate the caller before + // it can finish the restart command. + if (detached) { const handoff = scheduleDetachedLaunchdRestartHandoff({ env: serviceEnv, mode: plistReloadNeeded ? "reload" : "kickstart", @@ -272,44 +311,6 @@ export async function restartLaunchAgent({ return { outcome: "scheduled" }; } - const { port: cleanupPort, probeHosts } = await resolveLaunchAgentGatewayContext(serviceEnv); - if (cleanupPort !== null) { - cleanStaleGatewayProcessesSync(cleanupPort, { - // Resolve after lsof captures its listener snapshot. A KeepAlive respawn - // during enumeration must be protected before candidate filtering/signals. - resolveProtectedPid: () => readLaunchAgentPidForCleanupSync(serviceTarget), - }); - const diagnostics = await inspectPortUsage(cleanupPort, { - probeHosts, - }).catch(() => null); - if (diagnostics?.status === "busy") { - const runtime = await readLaunchAgentRuntime(serviceEnv); - const managedPid = runtime.pid; - // Only the current supervised PID may keep the port busy before a - // disruptive restart. Re-read after cleanup to close over a concurrent - // launchd respawn rather than trusting the protected pre-cleanup PID. - const ownedByLaunchAgent = - managedPid !== undefined && - diagnostics.listeners.length > 0 && - diagnostics.listeners.every((listener) => listener.pid === managedPid); - if (!ownedByLaunchAgent) { - throw new Error( - [ - `gateway port ${cleanupPort} is busy but is not verifiably owned by LaunchAgent ${label}`, - ...formatPortDiagnostics(diagnostics), - ].join("\n"), - ); - } - } - } - const plistReloadNeeded = await rewriteLaunchAgentPlistForRestart({ - env: serviceEnv, - label, - plistPath, - stdout, - warn, - }); - // `openclaw gateway restart` is an explicit operator request to bring the // LaunchAgent back, so clear any persisted disabled state before restart. const enable = await execLaunchctl(["enable", serviceTarget]); @@ -391,7 +392,7 @@ export async function restartLaunchAgent({ ); } - // If the service was previously booted out, re-register the rewritten plist and retry. + // A preserved plist may be demand-only; bootstrap alone only registers it. await bootstrapLaunchAgentOrThrow({ domain, serviceTarget, @@ -399,6 +400,13 @@ export async function restartLaunchAgent({ actionHint: "openclaw gateway restart", onMutation: reportMutation, }); + if (preserveDefinition) { + const kick = await execLaunchctl(["kickstart", serviceTarget]); + if (kick.code !== 0) { + throw new Error(`launchctl kickstart failed: ${kick.stderr || kick.stdout}`.trim()); + } + reportMutation("kickstart"); + } writeLaunchAgentActionLine(stdout, "Restarted LaunchAgent", serviceTarget); return { outcome: "completed" }; } diff --git a/src/daemon/launchd.test.ts b/src/daemon/launchd.test.ts index ebf407ebb7b0..037e9de578c0 100644 --- a/src/daemon/launchd.test.ts +++ b/src/daemon/launchd.test.ts @@ -3105,8 +3105,13 @@ describe("launchd install", () => { ["enable", serviceId], ["kickstart", serviceId], ["bootstrap", domain, resolveLaunchAgentPlistPath(env)], + ["kickstart", serviceId], + ]); + expect(onMutation.mock.calls).toEqual([ + [{ mode: "enable" }], + [{ mode: "bootstrap" }], + [{ mode: "kickstart" }], ]); - expect(onMutation.mock.calls).toEqual([[{ mode: "enable" }], [{ mode: "bootstrap" }]]); }); it("fails an already-loaded bootstrap immediately instead of waiting out the teardown deadline", async () => { diff --git a/src/daemon/schtasks-control.ts b/src/daemon/schtasks-control.ts index b586f0097734..3d304517aff2 100644 --- a/src/daemon/schtasks-control.ts +++ b/src/daemon/schtasks-control.ts @@ -327,6 +327,7 @@ export async function startScheduledTask({ } export async function restartRegisteredScheduledTask(params: { + preserveDefinition?: boolean; env: GatewayServiceEnv; stdout: NodeJS.WritableStream; mode: { kind: "standard" } | { kind: "fallback-takeover" }; @@ -402,7 +403,7 @@ export async function restartRegisteredScheduledTask(params: { } throw new Error("Replacement Windows Scheduled Task did not produce running evidence."); } - if (startupEntryInstalled && hasRunningEvidence) { + if (startupEntryInstalled && hasRunningEvidence && !params.preserveDefinition) { await removeStartupEntries(params.env, params.stdout); } params.stdout.write(`${formatLine("Restarted Scheduled Task", taskName)}\n`); @@ -410,6 +411,7 @@ export async function restartRegisteredScheduledTask(params: { } export async function restartScheduledTask({ + preserveDefinition, stdout, env, onMutation, @@ -422,6 +424,7 @@ export async function restartScheduledTask({ ); } return restartRegisteredScheduledTask({ + preserveDefinition, env: effectiveEnv, stdout, mode: { kind: "standard" }, diff --git a/src/daemon/schtasks.startup-fallback.test.ts b/src/daemon/schtasks.startup-fallback.test.ts index d5e6cc4c212b..23ae116c3fc9 100644 --- a/src/daemon/schtasks.startup-fallback.test.ts +++ b/src/daemon/schtasks.startup-fallback.test.ts @@ -1426,19 +1426,36 @@ describe("Windows startup fallback", () => { }); }); - it("removes an old Startup-folder launcher after Scheduled Task restart is proven", async () => { - await withWindowsEnv("openclaw-win-startup-", async ({ env }) => { - const startupEntryPath = await writeStartupFallbackEntry(env); - const hiddenStartupEntryPath = await writeStartupFallbackEntry(env, "vbs"); - await writeGatewayScript(env); - addSuccessfulScheduledTaskRestartResponses(); + it.each([false, true])( + "preserves Startup definitions only when requested (%s)", + async (preserveDefinition) => { + await withWindowsEnv("openclaw-win-startup-", async ({ env }) => { + const startupEntryPath = await writeStartupFallbackEntry(env); + const hiddenStartupEntryPath = await writeStartupFallbackEntry(env, "vbs"); + const files = [startupEntryPath, hiddenStartupEntryPath]; + const snapshot = () => + Promise.all( + files.map(async (file) => ({ + bytes: await fs.readFile(file), + mode: (await fs.stat(file)).mode, + })), + ); + const before = await snapshot(); + await writeGatewayScript(env); + addSuccessfulScheduledTaskRestartResponses(); - await restartScheduledTask({ env, stdout: new PassThrough() }); + await restartScheduledTask({ env, stdout: new PassThrough(), preserveDefinition }); - await expect(fs.access(startupEntryPath)).rejects.toThrow(); - await expect(fs.access(hiddenStartupEntryPath)).rejects.toThrow(); - }); - }); + if (preserveDefinition) { + expect(await snapshot()).toEqual(before); + } else { + for (const file of files) { + await expect(fs.access(file)).rejects.toThrow(); + } + } + }); + }, + ); it("waits for running evidence before removing a Startup-folder launcher", async () => { await withWindowsEnv("openclaw-win-startup-", async ({ env }) => { diff --git a/src/daemon/service-layout.test.ts b/src/daemon/service-layout.test.ts index d0aab6da1034..5cdb34029b4c 100644 --- a/src/daemon/service-layout.test.ts +++ b/src/daemon/service-layout.test.ts @@ -1,5 +1,8 @@ +import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { gatewayServiceCommandUsesRoot } from "../cli/update-cli/update-command-service.js"; import { summarizeGatewayServiceLayout } from "./service-layout.js"; describe("summarizeGatewayServiceLayout", () => { @@ -39,3 +42,59 @@ describe("summarizeGatewayServiceLayout", () => { ).resolves.not.toHaveProperty("entrypoint"); }); }); + +describe("gatewayServiceCommandUsesRoot release ownership", () => { + it.each(["stable", "foreign", "pinned", "paired", "different-mount"] as const)( + "checks the effective launcher against the managed installation (%s)", + async (layout) => { + const root = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-update-release-owner-")), + ); + try { + const managedRoot = path.join(root, "openclaw"); + const release = path.join(root, "releases", "selected"); + const foreign = path.join(root, "foreign", "releases", "selected"); + const current = path.join(root, "current"); + const mounted = layout === "paired" || layout === "different-mount"; + for (const packageRoot of [managedRoot, release, foreign, ...(mounted ? [current] : [])]) { + await fs.mkdir(path.join(packageRoot, "dist"), { recursive: true }); + await fs.writeFile(path.join(packageRoot, "package.json"), '{"name":"openclaw"}'); + await fs.writeFile(path.join(packageRoot, "dist", "index.js"), "gateway"); + } + if (!mounted) { + await fs.symlink(layout === "foreign" ? foreign : release, current); + } else if (layout === "paired") { + // The Docker proof supplies real bind mounts; this isolates directory identity. + const selected = await fs.stat(release); + const stat = fs.stat.bind(fs); + vi.spyOn(fs, "stat").mockImplementation(async (target) => + path.resolve(String(target)) === current ? selected : stat(target), + ); + } + const command = { + programArguments: [ + process.execPath, + path.join(layout === "pinned" ? release : current, "dist", "index.js"), + "gateway", + ], + managedDefinition: { + programArguments: [ + process.execPath, + path.join(managedRoot, "dist", "index.js"), + "gateway", + ], + }, + }; + await expect(gatewayServiceCommandUsesRoot({ root: managedRoot, command })).resolves.toBe( + layout === "stable" || layout === "paired", + ); + await expect( + gatewayServiceCommandUsesRoot({ root: managedRoot, command: command.managedDefinition }), + ).resolves.toBe(true); + } finally { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); + } + }, + ); +}); diff --git a/src/daemon/service-types.ts b/src/daemon/service-types.ts index fff28eb18b08..9e07e9228867 100644 --- a/src/daemon/service-types.ts +++ b/src/daemon/service-types.ts @@ -30,6 +30,7 @@ export type GatewayServiceControlArgs = { stdout: NodeJS.WritableStream; env?: GatewayServiceEnv; disable?: boolean; + preserveDefinition?: boolean; warn?: (message: string) => void; onMutation?: (mutation: GatewayLifecycleMutation) => void; }; @@ -82,6 +83,7 @@ export type GatewayServiceEnvArgs = { /** Options for read-only service inspection that should fail soft under a deadline. */ export type GatewayServiceReadOptions = { timeoutMs?: number; + requireEffective?: boolean; }; export type GatewayServiceEnvironmentValueSource = "inline" | "file" | "inline-and-file"; @@ -91,6 +93,22 @@ export type GatewayServiceLoadState = | { status: "not-loaded" } | { status: "unknown"; detail: string }; +export type ServiceDefinitionMutationCapability = + | { kind: "writable" } + | { kind: "sealed" | "unknown"; detail: string }; + +export function assertServiceDefinitionWritable(capability: ServiceDefinitionMutationCapability) { + if (capability.kind !== "writable") { + const guidance = + capability.kind === "sealed" + ? "Ask the privileged deployment owner." + : "Inspect service definition access."; + throw new Error( + `SERVICE_DEFINITION_${capability.kind.toUpperCase()}: ${capability.detail} ${guidance}`, + ); + } +} + export type GatewayServiceCommandSnapshot = { programArguments: string[]; workingDirectory?: string; @@ -106,6 +124,7 @@ export type GatewayServiceManagedOverrides = { /** Effective platform service command and, when externally owned, its managed base definition. */ export type GatewayServiceCommandConfig = GatewayServiceCommandSnapshot & { sourcePath?: string; + definitionPaths?: string[]; managedDefinition?: GatewayServiceCommandSnapshot; managedOverrides?: GatewayServiceManagedOverrides; reloadPending?: true; @@ -220,6 +239,7 @@ export type GatewayServiceState = { running: boolean; env: GatewayServiceEnv; command: GatewayServiceCommandConfig | null; + definitionMutationCapability?: ServiceDefinitionMutationCapability; runtime?: GatewayServiceRuntime; }; diff --git a/src/daemon/service.test-helpers.ts b/src/daemon/service.test-helpers.ts index 73f0bb6cdedc..784263d5d19e 100644 --- a/src/daemon/service.test-helpers.ts +++ b/src/daemon/service.test-helpers.ts @@ -2,6 +2,46 @@ import { vi } from "vitest"; import type { GatewayService } from "./service.js"; +export type SystemdManagerSnapshotFixture = { + programArguments: string[]; + workingDirectory?: string; + environment?: string[]; + environmentFiles?: Array<[string, boolean]>; + unsetEnvironment?: string[]; + fragmentPath?: string; + dropInPaths?: string[]; + needDaemonReload?: boolean; + loadState?: string; +}; + +export function buildSystemdManagerPropertyOutput(snapshot: SystemdManagerSnapshotFixture): string { + return [ + { + type: "a(sasbttttuii)", + data: [[snapshot.programArguments[0], snapshot.programArguments, false, 0, 0, 0, 0, 0, 0, 0]], + }, + { type: "s", data: snapshot.workingDirectory ?? "" }, + { type: "as", data: snapshot.environment ?? [] }, + { type: "a(sb)", data: snapshot.environmentFiles ?? [] }, + { type: "as", data: snapshot.unsetEnvironment ?? [] }, + ] + .map((property) => JSON.stringify(property)) + .join("\n"); +} + +export function buildSystemdUnitPropertyOutput( + params: Pick & { + fragmentPath: string; + }, +): string { + return [ + JSON.stringify({ type: "s", data: params.fragmentPath }), + JSON.stringify({ type: "as", data: params.dropInPaths ?? [] }), + JSON.stringify({ type: "b", data: params.needDaemonReload ?? false }), + JSON.stringify({ type: "s", data: params.loadState ?? "loaded" }), + ].join("\n"); +} + /** Creates a mock gateway service implementation for daemon service tests. */ export function createMockGatewayService(overrides: Partial = {}): GatewayService { return { diff --git a/src/daemon/service.test.ts b/src/daemon/service.test.ts index dac7ec979e94..19c604c58eaa 100644 --- a/src/daemon/service.test.ts +++ b/src/daemon/service.test.ts @@ -139,28 +139,117 @@ describe("resolveGatewayService", () => { }); describe("readGatewayServiceState", () => { - it("tracks installed, loaded, and running separately", async () => { - const hasInstalledDefinition = vi.fn(async () => false); - const service = createService({ - hasInstalledDefinition, - isLoaded: vi.fn(async () => true), - readCommand: vi.fn(async () => ({ - programArguments: ["openclaw", "gateway", "run"], - environment: { OPENCLAW_GATEWAY_PORT: "18789" }, - })), - readRuntime: vi.fn(async () => ({ status: "running" })), - }); + it.each([ + { updateInstallKind: "git" as const, shouldRestart: false }, + { updateInstallKind: "git" as const, shouldRestart: true }, + { updateInstallKind: "package" as const, shouldRestart: false }, + { updateInstallKind: "package" as const, shouldRestart: true }, + ])( + "allows managerless Linux preflight for $updateInstallKind restart=$shouldRestart", + async ({ updateInstallKind, shouldRestart }) => { + const { maybeStopManagedServiceBeforeMutableUpdate } = + await import("../cli/update-cli/update-command-service.js"); + const home = await makeTempWorkspace("openclaw-managerless-preflight-"); + const keys = [ + "HOME", + "PATH", + "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", + "OPENCLAW_CONFIG_PATH", + "OPENCLAW_PROFILE", + "OPENCLAW_SUPERVISOR_MODE", + "OPENCLAW_SERVICE_MARKER", + "OPENCLAW_SERVICE_KIND", + "OPENCLAW_SYSTEMD_UNIT", + "DBUS_SESSION_BUS_ADDRESS", + "DBUS_SYSTEM_BUS_ADDRESS", + "XDG_RUNTIME_DIR", + "SUDO_USER", + ]; + const snapshot = captureEnv(keys); + try { + setPlatform("linux"); + for (const key of keys) { + delete process.env[key]; + } + process.env.HOME = home; + process.env.PATH = home; + const result = await maybeStopManagedServiceBeforeMutableUpdate({ + root: home, + updateInstallKind, + shouldRestart, + jsonMode: true, + phase: "inspect", + timeoutMs: 2_000, + }); + expect(result.blockMessage).toBeUndefined(); + expect(result.serviceMutationAllowed).toBe(false); + expect(result.serviceMutationSkipMessage).toContain("inspection is unavailable"); + expect(result.serviceMutationSkipMessage).toContain("gateway status --deep"); + expect(result.serviceUpdateVerdict?.kind).not.toBe("absent"); + expect(result.stopped).toBe(false); + } finally { + snapshot.restore(); + await fs.rm(home, { recursive: true, force: true }); + } + }, + ); - const state = await readGatewayServiceState(service, { - env: { OPENCLAW_GATEWAY_PORT: "1" }, - }); + it.each([ + { read: "ordinary", requireEffective: undefined, capabilityFails: false }, + { read: "strict", requireEffective: true, capabilityFails: false }, + { read: "strict with unavailable capability", requireEffective: true, capabilityFails: true }, + ])( + "tracks service state and reads only needed capability for $read reads", + async ({ requireEffective, capabilityFails }) => { + const hasInstalledDefinition = vi.fn(async () => false); + const readDefinitionMutationCapability = vi.fn< + NonNullable + >(async () => { + if (capabilityFails) { + throw new Error("capability unavailable"); + } + return { kind: "sealed", detail: "deployment owned" }; + }); + const service = createService({ + hasInstalledDefinition, + readDefinitionMutationCapability, + isLoaded: vi.fn(async () => true), + readCommand: vi.fn(async () => ({ + programArguments: ["openclaw", "gateway", "run"], + environment: { OPENCLAW_GATEWAY_PORT: "18789" }, + })), + readRuntime: vi.fn(async () => ({ status: "running" })), + }); - expect(state.installed).toBe(true); - expect(state.loadState).toEqual({ status: "loaded" }); - expect(state.running).toBe(true); - expect(state.env.OPENCLAW_GATEWAY_PORT).toBe("18789"); - expect(hasInstalledDefinition).not.toHaveBeenCalled(); - }); + const state = await readGatewayServiceState(service, { + env: { OPENCLAW_GATEWAY_PORT: "1" }, + requireEffective, + timeoutMs: 100, + }); + + expect(state.installed).toBe(true); + expect(state.loadState).toEqual({ status: "loaded" }); + expect(state.running).toBe(true); + expect(state.env.OPENCLAW_GATEWAY_PORT).toBe("18789"); + expect(hasInstalledDefinition).not.toHaveBeenCalled(); + if (requireEffective) { + expect(readDefinitionMutationCapability).toHaveBeenCalledWith({ + env: { OPENCLAW_GATEWAY_PORT: "1" }, + environment: { OPENCLAW_GATEWAY_PORT: "18789" }, + timeoutMs: 100, + }); + expect(state.definitionMutationCapability).toEqual( + capabilityFails + ? { kind: "unknown", detail: "Cannot inspect service definition." } + : { kind: "sealed", detail: "deployment owned" }, + ); + } else { + expect(readDefinitionMutationCapability).not.toHaveBeenCalled(); + expect(state.definitionMutationCapability).toBeUndefined(); + } + }, + ); it.each([ { name: "system-scoped OpenClaw service", definition: true, installed: true }, @@ -211,6 +300,21 @@ describe("readGatewayServiceState", () => { ); }); + it("propagates required effective command inspection failures", async () => { + const readCommand = vi.fn(async () => { + throw new Error("manager unavailable"); + }); + const service = createService({ readCommand }); + + await expect(readGatewayServiceState(service, { requireEffective: true })).rejects.toThrow( + "manager unavailable", + ); + expect(readCommand).toHaveBeenCalledWith(process.env, { + timeoutMs: undefined, + requireEffective: true, + }); + }); + it("preserves runtime probe failures as an explicit unknown state", async () => { const readCommand = vi.fn(async () => null); const service = createService({ @@ -249,8 +353,11 @@ describe("readGatewayServiceState", () => { it("validates merged service env before native status probes", async () => { const isLoaded = vi.fn(async () => true); const readRuntime = vi.fn(async () => ({ status: "running" as const })); + const readDefinitionMutationCapability = + vi.fn>(); const service = createService({ isLoaded, + readDefinitionMutationCapability, readCommand: vi.fn(async () => ({ programArguments: ["openclaw", "gateway", "run"], environment: { OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway.service" }, @@ -261,6 +368,7 @@ describe("readGatewayServiceState", () => { await expect( readGatewayServiceState(service, { env: {}, + requireEffective: true, validateEnvBeforeStatusRead: (env) => { throw new Error(`refused ${env.OPENCLAW_SYSTEMD_UNIT}`); }, @@ -269,6 +377,7 @@ describe("readGatewayServiceState", () => { expect(isLoaded).not.toHaveBeenCalled(); expect(readRuntime).not.toHaveBeenCalled(); + expect(readDefinitionMutationCapability).not.toHaveBeenCalled(); }); }); diff --git a/src/daemon/service.ts b/src/daemon/service.ts index 974163b9aa14..e2db7c563d6f 100644 --- a/src/daemon/service.ts +++ b/src/daemon/service.ts @@ -47,6 +47,7 @@ import type { GatewayServiceStageArgs, GatewayServiceState, } from "./service-types.js"; +import { readSystemdDefinitionMutationCapability } from "./systemd-definition-mutation.js"; import { findInstalledSystemdGatewayScope, installSystemdService, @@ -88,6 +89,9 @@ export type GatewayService = { isLoaded: (args: GatewayServiceEnvArgs) => Promise; isEnabled?: (args: GatewayServiceEnvArgs) => Promise; hasInstalledDefinition?: (args: GatewayServiceEnvArgs) => Promise; + readDefinitionMutationCapability?: ( + args: GatewayServiceEnvArgs & { environment?: GatewayServiceEnv }, + ) => ReturnType; readCommand: ( env: GatewayServiceEnv, opts?: GatewayServiceReadOptions, @@ -99,6 +103,7 @@ export type GatewayService = { }; type ReadGatewayServiceStateArgs = GatewayServiceEnvArgs & { + requireEffective?: boolean; validateEnvBeforeStatusRead?: (env: GatewayServiceEnv) => void; }; @@ -171,10 +176,7 @@ export async function inspectGatewayServiceStartRepair( expectedPort?: number, ): Promise<{ state: GatewayServiceState; issues: GatewayServiceStartRepairIssue[] }> { const state = await readGatewayServiceState(service, args); - return { - state, - issues: collectGatewayServiceStartRepairIssues(state, expectedPort), - }; + return { state, issues: collectGatewayServiceStartRepairIssues(state, expectedPort) }; } export function formatGatewayServiceStartRepairIssues( @@ -188,8 +190,7 @@ export async function readGatewayServiceLoadState( args: GatewayServiceEnvArgs = {}, ): Promise { try { - const loaded = await service.isLoaded(args); - return { status: loaded ? "loaded" : "not-loaded" }; + return { status: (await service.isLoaded(args)) ? "loaded" : "not-loaded" }; } catch (error) { return { status: "unknown", detail: String(error) }; } @@ -201,24 +202,27 @@ export async function readGatewayServiceState( ): Promise { const baseEnv = args.env ?? (process.env as GatewayServiceEnv); const { timeoutMs } = args; - // Keep command and status probes on the same fail-soft manager deadline. - const command = await service.readCommand(baseEnv, { timeoutMs }).catch(() => null); + const command = args.requireEffective + ? await service.readCommand(baseEnv, { timeoutMs, requireEffective: true }) + : await service.readCommand(baseEnv, { timeoutMs }).catch(() => null); const env = mergeGatewayServiceEnv(baseEnv, command); - // Callers that may mutate the selected service can reject persisted selector - // drift before isLoaded/readRuntime invoke the native service manager. + // Reject persisted selector drift before invoking the native service manager. args.validateEnvBeforeStatusRead?.(env); - const [installed, loadState, runtime] = await Promise.all([ + const [installed, loadState, runtime, definitionMutationCapability] = await Promise.all([ command !== null ? true : (service.hasInstalledDefinition?.({ env, timeoutMs }).catch(() => false) ?? false), readGatewayServiceLoadState(service, { env, timeoutMs }), - service.readRuntime(env, { timeoutMs }).catch( - (error: unknown) => - ({ - status: "unknown", - detail: String(error), - }) satisfies GatewayServiceRuntime, - ), + service.readRuntime(env, { timeoutMs }).catch((error: unknown) => ({ + status: "unknown" as const, + detail: String(error), + })), + // Update policy needs definition authority; ordinary status/start reads do not. + args.requireEffective + ? service + .readDefinitionMutationCapability?.({ env: baseEnv, environment: env, timeoutMs }) + .catch(() => ({ kind: "unknown" as const, detail: "Cannot inspect service definition." })) + : undefined, ]); return { installed, @@ -226,6 +230,7 @@ export async function readGatewayServiceState( running: runtime?.status === "running", env, command, + ...(definitionMutationCapability ? { definitionMutationCapability } : {}), runtime, }; } @@ -386,6 +391,8 @@ const GATEWAY_SERVICE_REGISTRY: Record (await findInstalledSystemdGatewayScope(env ?? process.env)) !== null, + readDefinitionMutationCapability: ({ env, environment, timeoutMs }) => + readSystemdDefinitionMutationCapability(env ?? process.env, { environment, timeoutMs }), readCommand: readSystemdServiceExecStart, readRuntime: readSystemdServiceRuntime, }, diff --git a/src/daemon/systemd-definition-mutation.creation.test.ts b/src/daemon/systemd-definition-mutation.creation.test.ts new file mode 100644 index 000000000000..221a9c927390 --- /dev/null +++ b/src/daemon/systemd-definition-mutation.creation.test.ts @@ -0,0 +1,76 @@ +import { execFileSync } from "node:child_process"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +vi.mock("./systemd-exec.js", async (importOriginal) => ({ + ...(await importOriginal()), + execBusctlUser: async (env: Record) => ({ + code: 1, + stdout: "", + stderr: `Call failed: Unit ${env.OPENCLAW_SYSTEMD_UNIT}.service not found.`, + }), +})); + +import { withSystemdDefinitionMutation } from "./systemd-definition-mutation.js"; + +describe.skipIf(process.platform === "win32")("systemd publication directory creation", () => { + let root: string; + + beforeEach(async () => { + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-unit-mode-"))); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); + }); + + it.each([false, true])( + "uses safe new directory modes without changing existing ones under umask 0002 (existing=%s)", + async (existing) => { + const env = { + HOME: path.join(root, "home"), + OPENCLAW_STATE_DIR: path.join(root, "state"), + OPENCLAW_SYSTEMD_UNIT: "openclaw-mode-proof", + }; + const directory = path.join(env.HOME, ".config/systemd/user"); + const unit = path.join(directory, `${env.OPENCLAW_SYSTEMD_UNIT}.service`); + await fs.mkdir(env.HOME, { mode: 0o700 }); + if (existing) { + await fs.mkdir(directory, { recursive: true, mode: 0o700 }); + } + const mkdir = fs.mkdir.bind(fs); + vi.spyOn(fs, "mkdir").mockImplementation(async (...args) => { + if (args[0] !== directory) { + return mkdir(...args); + } + // Vitest uses threads, where changing umask is forbidden. A child applies + // the real kernel mask to the exact mkdir options supplied by publication. + execFileSync( + process.execPath, + [ + "--input-type=module", + "-e", + "import { mkdirSync } from 'node:fs'; process.umask(0o002); mkdirSync(process.argv[1], JSON.parse(process.argv[2]));", + directory, + JSON.stringify(args[1] ?? {}), + ], + { timeout: 10_000 }, + ); + return undefined; + }); + const chmod = vi.spyOn(fs, "chmod"); + const contents = "[Service]\nExecStart=/usr/bin/node gateway\n"; + + await withSystemdDefinitionMutation(env, env, (mutation) => + mutation.publish(unit, contents, 0o644), + ); + + expect(await fs.readFile(unit, "utf8")).toBe(contents); + expect((await fs.stat(directory)).mode & 0o777).toBe(existing ? 0o700 : 0o755); + expect(chmod).not.toHaveBeenCalled(); + }, + ); +}); diff --git a/src/daemon/systemd-definition-mutation.test.ts b/src/daemon/systemd-definition-mutation.test.ts new file mode 100644 index 000000000000..f43ee2529aa9 --- /dev/null +++ b/src/daemon/systemd-definition-mutation.test.ts @@ -0,0 +1,1014 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { Writable } from "node:stream"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createDeferred } from "../../test/helpers/promise.js"; +import { + buildSystemdManagerPropertyOutput, + buildSystemdUnitPropertyOutput, +} from "./service.test-helpers.js"; + +const assertNoSystemOwnership = vi.hoisted(() => + vi.fn(), +); +const busctl = vi.hoisted(() => vi.fn()); + +vi.mock("./systemd-system.js", async (importOriginal) => ({ + ...(await importOriginal()), + assertNoSystemSystemdOwnership: assertNoSystemOwnership, +})); +vi.mock("./systemd-exec.js", async (importOriginal) => ({ + ...(await importOriginal()), + assertSystemdAvailable: async () => {}, + execBusctlUser: busctl, +})); + +import { + readSystemdDefinitionMutationCapability, + withSystemdDefinitionMutation, +} from "./systemd-definition-mutation.js"; +import { stageSystemdService } from "./systemd-install.js"; + +describe.skipIf(process.platform === "win32")("systemd definition mutation ownership", () => { + let root: string; + let stateDir: string; + let unitPath: string; + let environmentPath: string; + let env: Record; + const artifacts = [ + { artifact: "unit", select: () => unitPath }, + { artifact: "environment", select: () => environmentPath }, + { artifact: "backup", select: () => `${unitPath}.bak` }, + ]; + + beforeEach(async () => { + assertNoSystemOwnership.mockReset().mockResolvedValue(undefined); + busctl.mockReset().mockImplementation(async (serviceEnv) => ({ + code: 1, + termination: "exit", + stdout: "", + stderr: `Call failed: Unit ${serviceEnv.OPENCLAW_SYSTEMD_UNIT}.service not found.`, + })); + root = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-definition-"))); + stateDir = path.join(root, "state"); + env = { + HOME: path.join(root, "home"), + OPENCLAW_STATE_DIR: stateDir, + OPENCLAW_SYSTEMD_UNIT: "openclaw-owned", + }; + unitPath = path.join(env.HOME!, ".config/systemd/user/openclaw-owned.service"); + environmentPath = path.join(stateDir, "gateway.systemd.env"); + await fs.mkdir(path.dirname(unitPath), { recursive: true }); + await fs.mkdir(stateDir); + }); + + afterEach(async () => { + vi.restoreAllMocks(); + await fs.rm(root, { recursive: true, force: true }); + }); + + const stage = (environmentOverrides: Record = {}) => + stageSystemdService({ + env, + stdout: new Writable({ + write(_chunk, _encoding, done) { + done(); + }, + }), + programArguments: [ + "/usr/bin/node", + "/srv/openclaw/dist/index.js", + "gateway", + "--port", + "18789", + ], + environment: { + OPENCLAW_GATEWAY_PORT: "18789", + OPENCLAW_GATEWAY_TOKEN: "replacement-secret-canary", + ...environmentOverrides, + }, + environmentValueSources: { + OPENCLAW_GATEWAY_PORT: "inline", + OPENCLAW_GATEWAY_TOKEN: "file", + }, + }); + + function managerDefinition( + fragmentPath: string, + dropInPaths: string[] = [], + environmentFiles: Array<[string, boolean]> = [], + isLoaded = async () => true, + ) { + busctl.mockImplementation(async (_env, args) => { + const loaded = await isLoaded(); + return { + code: 0, + termination: "exit", + stderr: "", + stdout: args.includes("LoadUnit") + ? JSON.stringify({ type: "o", data: ["/org/freedesktop/systemd1/unit/owned"] }) + : args.includes("org.freedesktop.systemd1.Unit") + ? buildSystemdUnitPropertyOutput({ + fragmentPath: loaded ? fragmentPath : "", + dropInPaths: loaded ? dropInPaths : [], + loadState: loaded ? "loaded" : "not-found", + }) + : buildSystemdManagerPropertyOutput({ + programArguments: ["/usr/bin/node", "gateway"], + environment: ["TOKEN=manager-secret-canary"], + environmentFiles, + }), + }; + }); + } + + it.each(["unit", "state", "ancestor"])( + "publishes a first unit through a %s directory alias discovered by the manager", + async (alias) => { + const directory = + alias === "state" + ? stateDir + : alias === "ancestor" + ? path.dirname(path.dirname(unitPath)) + : path.dirname(unitPath); + const target = path.join(root, "unit-directory"); + await fs.rename(directory, target); + await fs.symlink(target, directory); + managerDefinition(unitPath, [], [], () => + fs.access(unitPath).then( + () => true, + () => false, + ), + ); + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toEqual({ + kind: "writable", + }); + const rename = fs.rename.bind(fs); + let published = false; + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + if (destination === unitPath) { + expect(await fs.readFile(environmentPath, "utf8")).toContain("replacement-secret-canary"); + published = true; + } + await rename(source, destination); + }); + await expect(stage()).resolves.toMatchObject({ unitPath }); + expect(published).toBe(true); + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toEqual({ + kind: "writable", + }); + expect(await fs.readFile(unitPath, "utf8")).toContain("ExecStart="); + expect(await fs.readdir(path.dirname(unitPath))).toEqual([path.basename(unitPath)]); + expect(await fs.readdir(stateDir)).toEqual([path.basename(environmentPath)]); + }, + ); + + it.each( + ["unit", "environment"].flatMap((artifact) => + ["root-owned", "unsafe mode", "changed alias", "retargeted alias"].map((scenario) => ({ + artifact, + scenario, + })), + ), + )("protects an aliased $artifact directory with $scenario", async ({ artifact, scenario }) => { + const file = artifact === "unit" ? unitPath : environmentPath; + const directory = path.dirname(file); + const target = path.join(root, "alias-target"); + const replacement = path.join(root, "alias-replacement"); + await fs.rename(directory, target); + await fs.mkdir(replacement); + await fs.symlink(target, directory); + managerDefinition(unitPath, [], [], () => + fs.access(unitPath).then( + () => true, + () => false, + ), + ); + if (scenario === "root-owned") { + const stat = fs.stat.bind(fs); + vi.spyOn(fs, "stat").mockImplementation(async (...args) => { + const value = await stat(...args); + if (args[0] === directory) { + Object.defineProperty(value, "uid", { value: 0 }); + } + return value; + }); + } else if (scenario === "unsafe mode") { + await fs.chmod(target, 0o777); + } else { + const writeFile = fs.writeFile.bind(fs); + let replaced = false; + vi.spyOn(fs, "writeFile").mockImplementation(async (...args) => { + await writeFile(...args); + if ( + !replaced && + typeof args[0] === "string" && + path.basename(args[0]).startsWith(`${path.basename(file)}.`) && + args[0].endsWith(".tmp") + ) { + replaced = true; + // Replacing an alias must reject publication and clean the original directory. + await fs.rename(directory, `${directory}.previous`); + await fs.symlink(scenario === "retargeted alias" ? replacement : target, directory); + } + }); + } + if (scenario === "retargeted alias") { + await expect(stage()).rejects.toThrow(); + } else { + await expect(stage()).rejects.toThrow( + scenario === "changed alias" ? "changed during publication" : "SERVICE_DEFINITION_", + ); + } + expect(await fs.readdir(target)).toEqual([]); + expect(await fs.readdir(replacement)).toEqual([]); + expect(await fs.readdir(stateDir)).toEqual([]); + }); + + it.each(["fragment", "drop-in", "parent"])( + "seals a root-owned manager %s before publication", + async (kind) => { + const extra = path.join( + root, + "global-user", + kind === "fragment" ? "service.d" : "owned.service.d", + "operator.conf", + ); + await fs.mkdir(path.dirname(extra), { recursive: true }); + await fs.writeFile(extra, "[Service]\nEnvironment=TOKEN=operator-secret-canary\n"); + if (kind !== "fragment") { + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/node gateway\n"); + } + managerDefinition(kind === "fragment" ? extra : unitPath, kind === "fragment" ? [] : [extra]); + const originalLstat = fs.lstat.bind(fs); + vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + const stat = await originalLstat(...args); + if (args[0] === (kind === "parent" ? path.dirname(extra) : extra)) { + Object.defineProperty(stat, "uid", { value: 0 }); + } + return stat; + }); + const before = await fs.readFile(extra); + const entries = await fs.readdir(path.dirname(unitPath)); + const capability = await readSystemdDefinitionMutationCapability(env); + expect(capability).toMatchObject({ kind: "sealed" }); + expect(JSON.stringify(capability)).not.toContain("secret-canary"); + await expect(stage()).rejects.toThrow("SERVICE_DEFINITION_SEALED"); + expect(await fs.readFile(extra)).toEqual(before); + expect(await fs.readdir(path.dirname(unitPath))).toEqual(entries); + expect(await fs.readdir(stateDir)).toEqual([]); + }, + ); + + it("refuses an uninspectable user manager before publication and redacts its diagnostics", async () => { + busctl.mockRejectedValue(new Error("manager-secret-canary")); + const capability = await readSystemdDefinitionMutationCapability(env); + expect(capability).toMatchObject({ kind: "unknown" }); + expect(JSON.stringify(capability)).not.toContain("secret-canary"); + await expect(stage()).rejects.toThrow("SERVICE_DEFINITION_UNKNOWN"); + expect(await fs.readdir(stateDir)).toEqual([]); + expect(await fs.readdir(path.dirname(unitPath))).toEqual([]); + }); + + it.each(["unchanged", "changed", "first install"])( + "reads root-owned type-wide defaults without write authority (%s)", + async (scenario) => { + const changed = scenario === "changed"; + const firstInstall = scenario === "first install"; + const shared = path.join(root, "distribution-user", "service.d", "default.conf"); + await fs.mkdir(path.dirname(shared), { recursive: true, mode: 0o755 }); + await fs.writeFile(shared, "[Service]\nTimeoutStopSec=30s\n", { mode: 0o644 }); + if (!firstInstall) { + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/node gateway\n"); + } + await fs.writeFile(environmentPath, "OPERATOR=unchanged\n"); + managerDefinition(unitPath, [shared], [], async () => + firstInstall + ? fs.stat(unitPath).then( + () => true, + () => false, + ) + : true, + ); + const lstat = fs.lstat.bind(fs); + const open = fs.open.bind(fs); + const readFile = fs.readFile.bind(fs); + let sharedFd: number | undefined; + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + const stat = await lstat(...args); + if (args[0] === shared || args[0] === path.dirname(shared)) { + Object.defineProperty(stat, "uid", { value: 0 }); + Object.defineProperty(stat, "gid", { value: 0 }); + } + return stat; + }); + vi.spyOn(fs, "open").mockImplementation(async (...args) => { + const handle = await open(...args); + sharedFd = args[0] === shared ? handle.fd : undefined; + if (args[0] === shared) { + const stat = handle.stat.bind(handle); + vi.spyOn(handle, "stat").mockImplementation(async () => { + const opened = await stat(); + Object.defineProperty(opened, "uid", { value: 0 }); + Object.defineProperty(opened, "gid", { value: 0 }); + return opened; + }); + } + return handle; + }); + vi.spyOn(fs, "readFile").mockImplementation(async (...args) => { + if (typeof args[0] === "string" && args[0].startsWith("/proc/self/fdinfo/")) { + return `mnt_id:\t${args[0] === `/proc/self/fdinfo/${sharedFd}` ? 2 : 1}\n`; + } + if (args[0] === "/proc/self/mountinfo") { + return `1 0 0:1 / / rw - tmpfs tmpfs rw\n2 1 0:2 / ${shared} ro - tmpfs tmpfs ro\n`; + } + return readFile(...args); + }); + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toEqual({ + kind: "writable", + }); + if (firstInstall) { + const rename = fs.rename.bind(fs); + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await rename(source, destination); + if (destination === unitPath) { + expect(await fs.readFile(environmentPath, "utf8")).toContain( + "replacement-secret-canary", + ); + // Native LoadUnit reveals type-wide defaults only once the base exists, + // without daemon-reload between the not-found and loaded observations. + expect(await fs.readFile(shared, "utf8")).toContain("30s"); + } + }); + } + if (changed) { + const originalUnit = await fs.readFile(unitPath, "utf8"); + const rename = fs.rename.bind(fs); + let edited = false; + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await rename(source, destination); + if (destination === unitPath && !edited) { + edited = true; + await fs.writeFile(shared, "[Service]\nTimeoutStopSec=60s\n"); + } + }); + await expect(stage()).rejects.toThrow("changed during publication"); + expect(await fs.readFile(unitPath, "utf8")).toBe(originalUnit); + expect(await fs.readFile(environmentPath, "utf8")).toBe("OPERATOR=unchanged\n"); + } else { + await stage(); + expect(await fs.readFile(unitPath, "utf8")).toContain("/srv/openclaw/dist/index.js"); + expect(await fs.readFile(environmentPath, "utf8")).toContain("replacement-secret-canary"); + } + expect(await fs.readFile(shared, "utf8")).toContain(changed ? "60s" : "30s"); + }, + ); + + it.each(["unit", "unit replacement", "environment", "backup"])( + "preserves a concurrent %s change during first-load discovery", + async (artifact) => { + const shared = path.join(root, "service.d", "default.conf"); + await fs.mkdir(path.dirname(shared)); + await fs.writeFile(shared, "[Service]\nTimeoutStopSec=30s\n"); + managerDefinition(unitPath, [shared], [], () => + fs.stat(unitPath).then( + () => true, + () => false, + ), + ); + const edited = + artifact === "environment" + ? environmentPath + : artifact === "backup" + ? `${unitPath}.bak` + : unitPath; + const rename = fs.rename.bind(fs); + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await rename(source, destination); + if (destination === unitPath) { + expect(await fs.readFile(environmentPath, "utf8")).toContain("replacement-secret-canary"); + if (artifact === "unit replacement") { + const replacement = path.join(root, "operator-replacement"); + await fs.writeFile(replacement, "OPERATOR=concurrent\n"); + await rename(replacement, edited); + } else { + await fs.writeFile(edited, "OPERATOR=concurrent\n"); + } + } + }); + + await expect(stage()).rejects.toThrow("changed during publication"); + + expect(await fs.readFile(edited, "utf8")).toBe("OPERATOR=concurrent\n"); + for (const target of [unitPath, environmentPath]) { + if (target !== edited) { + await expect(fs.stat(target)).rejects.toMatchObject({ code: "ENOENT" }); + } + } + expect(await fs.readFile(shared, "utf8")).toContain("30s"); + for (const directory of [path.dirname(unitPath), stateDir]) { + expect((await fs.readdir(directory)).filter((file) => file.endsWith(".tmp"))).toEqual([]); + } + }, + ); + + it.each(["existing unit", "unit-specific drop-in", "selected fragment"])( + "rejects new manager inputs after publication for a %s", + async (scenario) => { + const existing = scenario === "existing unit"; + const extra = path.join( + root, + scenario === "unit-specific drop-in" ? "owned.service.d" : "service.d", + "operator.conf", + ); + await fs.mkdir(path.dirname(extra)); + await fs.writeFile(extra, "[Service]\nTimeoutStopSec=30s\n"); + if (existing) { + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/node gateway\n"); + } + let published = false; + managerDefinition(unitPath, [], [], async () => existing); + const rename = fs.rename.bind(fs); + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await rename(source, destination); + if (destination === unitPath && !published) { + published = true; + managerDefinition( + scenario === "selected fragment" ? extra : unitPath, + scenario === "selected fragment" ? [] : [extra], + [], + () => + fs.stat(unitPath).then( + () => true, + () => false, + ), + ); + } + }); + + await expect(stage()).rejects.toThrow("changed during publication"); + expect(await fs.readFile(extra, "utf8")).toContain("30s"); + if (existing) { + expect(await fs.readFile(unitPath, "utf8")).toBe( + "[Service]\nExecStart=/usr/bin/node gateway\n", + ); + } else { + await expect(fs.stat(unitPath)).rejects.toMatchObject({ code: "ENOENT" }); + } + await expect(fs.stat(environmentPath)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it.each(["uid", "gid", "mode"] as const)( + "rejects changed %s between lstat and open", + async (field) => { + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/node gateway\n"); + const open = fs.open.bind(fs); + vi.spyOn(fs, "open").mockImplementation(async (...args) => { + const handle = await open(...args); + if (args[0] === unitPath) { + const stat = handle.stat.bind(handle); + vi.spyOn(handle, "stat").mockImplementation(async () => { + const opened = await stat(); + Object.defineProperty(opened, field, { + value: field === "mode" ? opened.mode | 0o022 : opened[field] + 1, + }); + return opened; + }); + } + return handle; + }); + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toMatchObject({ + kind: "unknown", + }); + expect(await fs.readdir(stateDir)).toEqual([]); + }, + ); + + it.each(["fragment", "drop-in"])( + "fingerprints a safe same-owner manager %s without snapshotting or restoring it", + async (kind) => { + const extra = path.join(root, "operator.conf"); + await fs.writeFile(extra, "[Service]\nEnvironment=OWNER=first\n"); + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/node gateway\n"); + await fs.writeFile(environmentPath, "OPERATOR=preserved\n"); + managerDefinition(kind === "fragment" ? extra : unitPath, kind === "fragment" ? [] : [extra]); + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toEqual({ + kind: "writable", + }); + await withSystemdDefinitionMutation(env, env, async (mutation) => { + expect([...mutation.snapshots.keys()]).toEqual([unitPath, environmentPath]); + await expect(mutation.publish(extra, "must not publish", 0o600)).rejects.toThrow( + "Not a managed service publication target", + ); + await expect( + mutation.restore(extra, { contents: Buffer.from("must not restore"), mode: 0o600 }), + ).rejects.toThrow("Not a managed service publication target"); + await mutation.publish(unitPath, "managed definition", 0o644); + await mutation.restore(extra, null); + expect(await fs.readFile(extra, "utf8")).toContain("OWNER=first"); + const writeFile = fs.writeFile.bind(fs); + vi.spyOn(fs, "writeFile").mockImplementation(async (...args) => { + await writeFile(...args); + if ( + typeof args[0] === "string" && + args[0].startsWith(`${unitPath}.`) && + args[0].endsWith(".tmp") + ) { + await writeFile(extra, "[Service]\nEnvironment=OWNER=second\n"); + } + }); + await expect(mutation.publish(unitPath, "must not publish", 0o644)).rejects.toThrow( + "changed during publication", + ); + }); + expect(await fs.readFile(unitPath, "utf8")).toBe("managed definition"); + expect(await fs.readFile(extra, "utf8")).toContain("OWNER=second"); + expect(await fs.readFile(environmentPath, "utf8")).toBe("OPERATOR=preserved\n"); + expect( + (await fs.readdir(path.dirname(unitPath))).filter((file) => file.endsWith(".tmp")), + ).toEqual([]); + }, + ); + + it.each( + artifacts.flatMap(({ artifact, select }) => + ["between publications", "after rename", "replacement after rename"].map((change) => ({ + artifact, + select, + change, + })), + ), + )("rejects a concurrent $artifact edit $change", async ({ select, change }) => { + const target = select(); + await withSystemdDefinitionMutation(env, env, async (mutation) => { + if (change === "between publications") { + await mutation.publish(target, "first publication", 0o600); + await fs.writeFile(target, "operator edit"); + } else { + const rename = fs.rename.bind(fs); + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await rename(source, destination); + if (destination === target) { + if (change === "replacement after rename") { + const replacement = path.join(root, "operator-replacement"); + await fs.writeFile(replacement, "operator edit", { mode: 0o600 }); + await rename(replacement, target); + } else { + await fs.writeFile(target, "operator edit"); + } + } + }); + } + await expect(mutation.publish(target, "must not accept", 0o600)).rejects.toThrow( + "changed during publication", + ); + }); + expect(await fs.readFile(target, "utf8")).toBe("operator edit"); + expect( + (await fs.readdir(path.dirname(target))).filter((file) => file.endsWith(".tmp")), + ).toEqual([]); + }); + + it.each( + artifacts.flatMap(({ artifact, select }) => + [false, true].map((existed) => ({ artifact, select, existed })), + ), + )( + "rolls back $artifact after a post-rename failure (existed=$existed)", + async ({ select, existed }) => { + const target = select(); + const extra = path.join(root, "operator.conf"); + await fs.writeFile(extra, "[Service]\nEnvironment=OWNER=first\n"); + managerDefinition(extra); + if (existed) { + await fs.writeFile(target, "previous definition", { mode: 0o400 }); + } + const rename = fs.rename.bind(fs); + let changed = false; + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await rename(source, destination); + if (destination === target && !changed) { + changed = true; + await fs.writeFile(extra, "[Service]\nEnvironment=OWNER=second\n"); + } + }); + await expect( + withSystemdDefinitionMutation(env, env, (mutation) => + mutation.publish(target, "must not remain", 0o600), + ), + ).rejects.toThrow("changed during publication"); + if (existed) { + expect(await fs.readFile(target, "utf8")).toBe("previous definition"); + expect((await fs.stat(target)).mode & 0o777).toBe(0o400); + } else { + await expect(fs.stat(target)).rejects.toMatchObject({ code: "ENOENT" }); + } + expect(await fs.readFile(extra, "utf8")).toContain("OWNER=second"); + expect( + (await fs.readdir(path.dirname(target))).filter((file) => file.endsWith(".tmp")), + ).toEqual([]); + }, + ); + + it.each( + artifacts.flatMap(({ artifact, select }) => + [false, true].map((environmentExisted) => ({ artifact, select, environmentExisted })), + ), + )( + "stage rollback preserves a concurrent $artifact edit (env existed=$environmentExisted)", + async ({ artifact, select, environmentExisted }) => { + const previousUnit = "[Service]\nExecStart=/usr/bin/node /old/index.js gateway\n"; + const previousEnvironment = "OPERATOR=original\n"; + await fs.writeFile(unitPath, previousUnit); + if (environmentExisted) { + await fs.writeFile(environmentPath, previousEnvironment); + } + const edited = select(); + const rename = fs.rename.bind(fs); + let changed = false; + vi.spyOn(fs, "rename").mockImplementation(async (source, destination) => { + await rename(source, destination); + if (destination === unitPath && !changed) { + changed = true; + expect(await fs.readFile(environmentPath, "utf8")).toContain("replacement-secret-canary"); + await fs.writeFile(edited, "OPERATOR=concurrent\n"); + } + }); + + await expect(stage()).rejects.toThrow("changed during publication"); + + expect(await fs.readFile(edited, "utf8")).toBe("OPERATOR=concurrent\n"); + if (artifact !== "unit") { + expect(await fs.readFile(unitPath, "utf8")).toBe(previousUnit); + } + if (artifact !== "environment") { + if (environmentExisted) { + expect(await fs.readFile(environmentPath, "utf8")).toBe(previousEnvironment); + } else { + await expect(fs.stat(environmentPath)).rejects.toMatchObject({ code: "ENOENT" }); + } + } + for (const directory of [path.dirname(unitPath), stateDir]) { + expect((await fs.readdir(directory)).filter((file) => file.endsWith(".tmp"))).toEqual([]); + } + }, + ); + + it.each([ + { mount: "file-ro", mode: 0o644, kind: "sealed" }, + { mount: "file-ro", mode: 0o400, kind: "sealed" }, + { mount: "file-rw", mode: 0o644, kind: "sealed" }, + { mount: "ordinary", mode: 0o400, kind: "writable" }, + { mount: "unavailable", mode: 0o644, kind: "unknown" }, + ])("inspects a mounted target before staging ($mount, $mode)", async ({ mount, mode, kind }) => { + await fs.writeFile(unitPath, "original definition", { mode }); + const open = fs.open.bind(fs); + const readFile = fs.readFile.bind(fs); + let targetFd: number | undefined; + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.spyOn(fs, "open").mockImplementation(async (...args) => { + const handle = await open(...args); + if (args[0] === unitPath) { + targetFd = handle.fd; + } + return handle; + }); + vi.spyOn(fs, "readFile").mockImplementation(async (...args) => { + if (typeof args[0] === "string" && args[0].startsWith("/proc/self/fdinfo/")) { + return `mnt_id:\t${args[0] === `/proc/self/fdinfo/${targetFd}` && mount !== "ordinary" ? 2 : 1}\n`; + } + if (args[0] === "/proc/self/mountinfo") { + if (mount === "unavailable") { + throw Object.assign(new Error("proc unavailable"), { code: "EACCES" }); + } + const escaped = unitPath.replaceAll(" ", "\\040"); + return `1 0 0:1 / / rw - tmpfs tmpfs rw\n2 1 0:1 /unit ${escaped} ${mount === "file-ro" ? "ro" : "rw"} - tmpfs tmpfs rw\n`; + } + return readFile(...args); + }); + + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toMatchObject({ kind }); + if (kind === "writable") { + await stage(); + expect(await fs.readFile(unitPath, "utf8")).toContain("/srv/openclaw/dist/index.js"); + expect((await fs.stat(unitPath)).mode & 0o777).toBe(mode); + } else { + await expect(stage()).rejects.toThrow(`SERVICE_DEFINITION_${kind.toUpperCase()}`); + expect(await fs.readFile(unitPath, "utf8")).toBe("original definition"); + expect(await fs.readdir(stateDir)).toEqual([]); + expect(await fs.readdir(path.dirname(unitPath))).toEqual([path.basename(unitPath)]); + } + }); + + it("cleans unpublished temporary files after a write failure", async () => { + await fs.writeFile(unitPath, "previous definition"); + const writeFile = fs.writeFile.bind(fs); + vi.spyOn(fs, "writeFile").mockImplementation(async (...args) => { + await writeFile(...args); + if ( + typeof args[0] === "string" && + args[0].startsWith(`${unitPath}.`) && + args[0].endsWith(".tmp") + ) { + throw new Error("write failed"); + } + }); + await expect( + withSystemdDefinitionMutation(env, env, (mutation) => + mutation.publish(unitPath, "must not remain", 0o600), + ), + ).rejects.toThrow("write failed"); + expect(await fs.readFile(unitPath, "utf8")).toBe("previous definition"); + expect( + (await fs.readdir(path.dirname(unitPath))).filter((file) => file.endsWith(".tmp")), + ).toEqual([]); + }); + + it("rejects manager definition path changes during publication", async () => { + const first = path.join(root, "first.conf"); + const second = path.join(root, "second.conf"); + await fs.writeFile(first, "[Service]\nEnvironment=OWNER=first\n"); + await fs.writeFile(second, "[Service]\nEnvironment=OWNER=second\n"); + await fs.writeFile(unitPath, "original definition"); + managerDefinition(unitPath, [first]); + + await expect( + withSystemdDefinitionMutation(env, env, async (mutation) => { + const writeFile = fs.writeFile.bind(fs); + vi.spyOn(fs, "writeFile").mockImplementation(async (...args) => { + await writeFile(...args); + if ( + typeof args[0] === "string" && + args[0].startsWith(`${unitPath}.`) && + args[0].endsWith(".tmp") + ) { + managerDefinition(unitPath, [second]); + } + }); + await mutation.publish(unitPath, "must not publish", 0o644); + }), + ).rejects.toThrow("changed during publication"); + expect(await fs.readFile(unitPath, "utf8")).toBe("original definition"); + }); + + it.each(["file symlink", "unsafe mode", "uninspectable", "missing", "directory"])( + "rejects a manager definition with %s without publication", + async (kind) => { + const directory = path.join(root, "operator"); + const target = path.join(directory, "operator.conf"); + await fs.mkdir(directory); + await fs.writeFile(target, "[Service]\nEnvironment=TOKEN=protected-secret-canary\n"); + let extra = target; + if (kind === "file symlink") { + extra = path.join(root, "linked.conf"); + await fs.symlink(target, extra); + } else if (kind === "unsafe mode") { + await fs.chmod(target, 0o666); + } else if (kind === "missing") { + extra = path.join(directory, "missing.conf"); + } else if (kind === "directory") { + extra = stateDir; + } else { + const lstat = fs.lstat.bind(fs); + vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + if (args[0] === extra) { + throw Object.assign(new Error("inspection-secret-canary"), { code: "EACCES" }); + } + return lstat(...args); + }); + } + managerDefinition(extra); + const capability = await readSystemdDefinitionMutationCapability(env); + expect(capability).toMatchObject({ kind: "unknown" }); + expect(JSON.stringify(capability)).not.toContain("secret-canary"); + await expect(stage()).rejects.toThrow("SERVICE_DEFINITION_UNKNOWN"); + expect(await fs.readFile(target, "utf8")).toContain("protected-secret-canary"); + expect(await fs.readdir(path.dirname(unitPath))).toEqual([]); + expect(await fs.readdir(stateDir)).toEqual([]); + }, + ); + + it("accepts the ownership owner's proven absence on a fresh non-systemd install", async () => { + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toEqual({ + kind: "writable", + }); + expect(assertNoSystemOwnership).toHaveBeenCalledWith("openclaw-owned.service", undefined); + }); + + it.each([ + { parent: "unit", select: () => path.dirname(unitPath) }, + { parent: "environment", select: () => stateDir }, + ])("seals a foreign-owned $parent parent before creating service files", async ({ select }) => { + const protectedParent = select(); + const originalEntries = await fs.readdir(protectedParent); + const originalLstat = fs.lstat.bind(fs); + vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + const stat = await originalLstat(...args); + if (args[0] === protectedParent) { + Object.defineProperty(stat, "uid", { value: (process.geteuid?.() ?? 0) + 1 }); + } + return stat; + }); + + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toMatchObject({ + kind: "sealed", + }); + await expect(stage()).rejects.toThrow("SERVICE_DEFINITION_SEALED"); + expect(await fs.readdir(protectedParent)).toEqual(originalEntries); + }); + + it.each(artifacts)("seals a foreign-owned $artifact before publication", async ({ select }) => { + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/node gateway\n"); + const protectedPath = select(); + if (protectedPath !== unitPath) { + await fs.writeFile(protectedPath, "protected-secret-canary\n"); + } + const original = await fs.readFile(protectedPath); + const originalLstat = fs.lstat.bind(fs); + vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + const stat = await originalLstat(...args); + if (args[0] === protectedPath) { + Object.defineProperty(stat, "uid", { value: (process.geteuid?.() ?? 0) + 1 }); + Object.defineProperty(stat, "mode", { value: Number(stat.mode) | 0o022 }); + } + return stat; + }); + + const capability = await readSystemdDefinitionMutationCapability(env); + expect(capability).toMatchObject({ kind: "sealed" }); + expect(JSON.stringify(capability)).not.toContain("secret-canary"); + await expect(stage()).rejects.toThrow("SERVICE_DEFINITION_SEALED"); + expect(await fs.readFile(protectedPath)).toEqual(original); + }); + + it.each([ + ...artifacts.map(({ artifact, select }) => ({ artifact, select, fresh: false })), + { artifact: "fresh environment", select: () => environmentPath, fresh: true }, + ])("rejects a symlinked $artifact without changing its target", async ({ select, fresh }) => { + const file = select(); + if (file !== unitPath && !fresh) { + await fs.writeFile(unitPath, "[Service]\n"); + } + const target = path.join(root, "operator-target"); + await fs.writeFile(target, "operator-secret-canary\n"); + await fs.symlink(target, file); + + await expect(readSystemdDefinitionMutationCapability(env)).resolves.toMatchObject({ + kind: "unknown", + detail: `Refusing to rewrite symlinked managed systemd file: ${file}`, + }); + await expect(stage()).rejects.toThrow( + `SERVICE_DEFINITION_UNKNOWN: Refusing to rewrite symlinked managed systemd file: ${file}`, + ); + expect(await fs.readlink(file)).toBe(target); + expect(await fs.readFile(target, "utf8")).toBe("operator-secret-canary\n"); + if (fresh) { + await expect(fs.stat(unitPath)).rejects.toMatchObject({ code: "ENOENT" }); + } + }); + + it("publishes the unit, backup, and generated environment without chmod or secret disclosure", async () => { + const previous = "[Service]\nExecStart=/usr/bin/node /old/index.js gateway\n"; + await fs.writeFile(unitPath, previous); + await fs.writeFile(environmentPath, "OPERATOR_SECRET=preserved-canary\n"); + const chmod = vi.spyOn(fs, "chmod"); + + await stage(); + + expect(await fs.readFile(`${unitPath}.bak`, "utf8")).toBe(previous); + expect(await fs.readFile(unitPath, "utf8")).toContain("/srv/openclaw/dist/index.js"); + expect(await fs.readFile(unitPath, "utf8")).not.toContain("replacement-secret-canary"); + expect(await fs.readFile(environmentPath, "utf8")).toContain( + "OPERATOR_SECRET=preserved-canary", + ); + expect(chmod).not.toHaveBeenCalled(); + expect( + (await fs.readdir(path.dirname(unitPath))).filter((file) => file.includes(".tmp")), + ).toEqual([]); + }); + + it("publishes only the environment selected by the effective service state dir", async () => { + const effectiveStateDir = path.join(root, "effective-state"); + const effectiveEnvironmentPath = path.join(effectiveStateDir, "gateway.systemd.env"); + await fs.mkdir(effectiveStateDir); + await fs.writeFile(environmentPath, "CALLER_SECRET=caller-canary\n"); + await fs.writeFile(effectiveEnvironmentPath, "OPERATOR_SECRET=preserved-canary\n"); + + await stage({ OPENCLAW_STATE_DIR: effectiveStateDir }); + + expect(await fs.readFile(environmentPath, "utf8")).toBe("CALLER_SECRET=caller-canary\n"); + expect(await fs.readFile(effectiveEnvironmentPath, "utf8")).toContain( + "OPERATOR_SECRET=preserved-canary", + ); + expect(await fs.readFile(effectiveEnvironmentPath, "utf8")).toContain( + "OPENCLAW_GATEWAY_TOKEN=replacement-secret-canary", + ); + expect(await fs.readFile(unitPath, "utf8")).toContain(effectiveEnvironmentPath); + }); + + it("keeps a retired generated environment file readable until the unit drops it", async () => { + await fs.writeFile( + unitPath, + `[Service]\nExecStart=/usr/bin/node gateway\nEnvironmentFile=${environmentPath}\n`, + ); + await fs.writeFile(environmentPath, "OPENCLAW_GATEWAY_TOKEN=retired-secret-canary\n"); + managerDefinition(unitPath, [], [[environmentPath, false]]); + + await stageSystemdService({ + env, + stdout: new Writable({ + write(_chunk, _encoding, done) { + done(); + }, + }), + programArguments: ["/usr/bin/node", "/srv/openclaw/dist/index.js", "gateway"], + environment: { OPENCLAW_GATEWAY_PORT: "18789" }, + environmentValueSources: { OPENCLAW_GATEWAY_TOKEN: "file" }, + }); + + expect(await fs.readFile(environmentPath, "utf8")).toBe(""); + expect(await fs.readFile(unitPath, "utf8")).not.toContain("EnvironmentFile="); + }); + + it.each(["environment", "unit", "directory alias", "retargeted unit", "retargeted environment"])( + "serializes canonical targets across concurrent writers (%s)", + async (shared) => { + const retarget = shared.startsWith("retargeted"); + const file = shared.includes("environment") ? environmentPath : unitPath; + const directory = path.dirname(file); + const original = path.join(root, "original"); + const replacement = path.join(root, "replacement"); + const other = { ...env }; + if (retarget) { + await fs.rename(directory, original); + await fs.mkdir(replacement); + await fs.symlink(original, directory); + } else if (shared === "environment") { + other.OPENCLAW_SYSTEMD_UNIT = "openclaw-secondary"; + } else { + other.OPENCLAW_STATE_DIR = path.join(root, "other-state"); + await fs.mkdir(other.OPENCLAW_STATE_DIR); + if (shared === "directory alias") { + other.HOME = path.join(root, "home-alias"); + await fs.symlink(env.HOME!, other.HOME); + } + } + const events: string[] = []; + const { promise: barrier, resolve: release } = createDeferred(); + const { promise: firstStarted, resolve: entered } = createDeferred(); + const first = withSystemdDefinitionMutation(env, env, async (mutation) => { + events.push("first-start"); + entered(); + await barrier; + if (!retarget) { + await mutation.publish(file, "first writer", 0o600); + } + events.push("first-finish"); + }); + await firstStarted; + const open = fs.open.bind(fs); + let contended = false; + vi.spyOn(fs, "open").mockImplementation(async (...args) => { + const handle = await open(...args); + // Reading a held sidecar follows the real owner's failed exclusive acquire. + if (typeof args[0] === "string" && args[0].endsWith(".lock")) { + contended = true; + } + return handle; + }); + const second = withSystemdDefinitionMutation(other, other, async (mutation) => { + events.push("second-start"); + const target = shared === "directory alias" ? file.replace(env.HOME!, other.HOME!) : file; + if (!retarget) { + expect(mutation.snapshots.get(target)?.contents.toString()).toBe("first writer"); + } + await mutation.publish(target, "second writer", 0o600); + }); + try { + await vi.waitFor(() => expect(contended).toBe(true)); + expect(events).toEqual(["first-start"]); + if (retarget) { + await fs.unlink(directory); + await fs.symlink(replacement, directory); + } + } finally { + release(); + } + await first; + if (retarget) { + await expect(second).rejects.toThrow("lock targets changed"); + expect(events).toEqual(["first-start", "first-finish"]); + expect(await fs.readdir(original)).toEqual([]); + expect(await fs.readdir(replacement)).toEqual([]); + } else { + await second; + expect(events).toEqual(["first-start", "first-finish", "second-start"]); + expect(await fs.readFile(file, "utf8")).toBe("second writer"); + } + }, + ); +}); diff --git a/src/daemon/systemd-definition-mutation.ts b/src/daemon/systemd-definition-mutation.ts new file mode 100644 index 000000000000..e5bc1047e2cc --- /dev/null +++ b/src/daemon/systemd-definition-mutation.ts @@ -0,0 +1,318 @@ +/** Systemd service-definition authority and atomic, cross-process publication. */ +import { randomUUID } from "node:crypto"; +import { constants, promises as fs, type Stats } from "node:fs"; +import path from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { decodeMountInfoPath } from "@openclaw/normalization-core/mountinfo-path"; +import { resolveStateDir } from "../config/paths.js"; +import { sha256Hex } from "../infra/crypto-digest.js"; +import { hasErrnoCode } from "../infra/errno.js"; +import { withFileLock } from "../infra/file-lock.js"; +import { canonicalPathFromExistingAncestor, findExistingAncestor } from "../infra/fs-safe.js"; +import { + assertServiceDefinitionWritable, + type GatewayServiceEnv, + type ServiceDefinitionMutationCapability, +} from "./service-types.js"; +import { + readSystemdServiceExecStart, + resolveSystemdEnvironmentFilePath, + resolveSystemdUnitPath, +} from "./systemd-service-files.js"; +import { assertNoSystemSystemdOwnership, isSystemSystemdOwnershipError } from "./systemd-system.js"; + +type Snapshot = { contents: Buffer; mode: number } | null; +type SystemdDefinitionMutation = { + snapshots: Map; + publish: (file: string, contents: string | Buffer, mode: number) => Promise; + restore: (file: string, snapshot: Snapshot) => Promise; +}; +const identity = (stat: Stats, contents?: Buffer) => + [stat.dev, stat.ino, stat.uid, stat.gid, stat.mode, contents && sha256Hex(contents)].join(":"); + +function resolveMutationTargets(env: GatewayServiceEnv, environment: GatewayServiceEnv) { + const unit = resolveSystemdUnitPath(env); + const generated = resolveSystemdEnvironmentFilePath({ + stateDir: resolveStateDir({ ...env, ...environment }), + environment, + }); + return { unit, generated }; +} + +async function readStableFile( + file: string, + stat: Stats, + requireReplacement: boolean, +): Promise<{ contents: Buffer } | { sealed: true }> { + const handle = await fs.open( + file, + constants.O_RDONLY | constants.O_NOFOLLOW | (constants.O_NONBLOCK ?? 0), + ); + try { + const opened = await handle.stat(); + if (!opened.isFile() || identity(opened) !== identity(stat)) { + throw new Error("changed artifact"); + } + if (requireReplacement && process.platform === "linux") { + // fdinfo selects the opened file's actual mount, including stacked mounts. + // W_OK alone misses read-only mounts when DAC denies an otherwise replaceable 0400 file. + const fdinfo = await fs.readFile(`/proc/self/fdinfo/${handle.fd}`, "utf8"); + const mountId = /^mnt_id:\s+(\d+)$/m.exec(fdinfo)?.[1]; + const mount = (await fs.readFile("/proc/self/mountinfo", "utf8")) + .split("\n") + .find((line) => mountId && line.startsWith(`${mountId} `)) + ?.split(" "); + if (!mount?.[4] || !mount[5]) { + throw new Error("Cannot inspect the service artifact mount."); + } + if ( + mount[5].split(",").includes("ro") || + decodeMountInfoPath(mount[4]) === (await fs.realpath(file)) + ) { + return { sealed: true }; + } + } + return { contents: await handle.readFile() }; + } finally { + await handle.close(); + } +} + +async function inspect(env: GatewayServiceEnv, environment: GatewayServiceEnv, timeoutMs?: number) { + const { unit, generated } = resolveMutationTargets(env, environment); + const snapshots = new Map(); + const fingerprint = new Map(); + let shared = new Set(); + let sourcePath: string | undefined; + const result = (capability: ServiceDefinitionMutationCapability) => ({ + capability, + snapshots, + fingerprint, + shared, + sourcePath, + }); + let inspected = "the selected service"; + try { + const command = await readSystemdServiceExecStart(env, { + requireEffective: true, + timeoutMs, + }); + sourcePath = command?.sourcePath; + const targets = new Set([unit, generated, `${unit}.bak`]); + const definitions = new Set(command?.definitionPaths ?? []); + // Type-wide service.d defaults are shared read-only inputs. Selected fragments + // and unit-specific overrides still require authority; never shadow a sealed unit. + shared = new Set( + [...definitions].filter( + (file) => + file !== command?.sourcePath && + !targets.has(file) && + path.basename(path.dirname(file)) === "service.d", + ), + ); + const definitionParents = new Set( + [...definitions].filter((file) => !shared.has(file)).map(path.dirname), + ); + const parents = new Set([path.dirname(unit), path.dirname(generated), ...definitionParents]); + const artifacts = new Set([...parents, ...targets, ...definitions]); + for (const file of artifacts) { + const directory = parents.has(file) && !definitions.has(file); + const required = definitions.has(file) || definitionParents.has(file); + inspected = directory && !required ? ((await findExistingAncestor(file)) ?? file) : file; + const stat = await fs.lstat(inspected).catch((error: unknown) => { + if (required || !hasErrnoCode(error, "ENOENT")) { + throw error; + } + }); + if (!stat) { + fingerprint.set(file, "missing"); + continue; + } + // systemd retains lexical directory aliases; fingerprint both alias and target. + if (!directory && stat.isSymbolicLink()) { + return result({ + kind: "unknown", + detail: `Refusing to rewrite symlinked managed systemd file: ${file}`, + }); + } + const actual = directory && stat.isSymbolicLink() ? await fs.stat(inspected) : stat; + if (!shared.has(file) && actual.uid !== process.geteuid?.()) { + return result({ + kind: "sealed", + detail: `Service artifact ${inspected} belongs to another account.`, + }); + } + if ((directory && !actual.isDirectory()) || actual.mode & 0o022) { + throw new Error("unsafe service publication artifact"); + } + if (directory) { + await fs.access(inspected, constants.W_OK | constants.X_OK); + fingerprint.set(file, `${inspected}:${identity(stat)}:${identity(actual)}`); + continue; + } + const artifact = await readStableFile(file, stat, !shared.has(file)); + if ("sealed" in artifact) { + return result({ + kind: "sealed", + detail: `Service artifact ${file} cannot be replaced on its mount.`, + }); + } + const { contents } = artifact; + fingerprint.set(file, identity(stat, contents)); + if (targets.has(file)) { + snapshots.set(file, { contents, mode: stat.mode & 0o777 }); + } + } + return result({ kind: "writable" }); + } catch { + return result({ + kind: "unknown", + detail: `Cannot safely rewrite managed systemd artifact ${inspected}.`, + }); + } +} + +export async function readSystemdDefinitionMutationCapability( + env: GatewayServiceEnv, + options?: { environment?: GatewayServiceEnv; timeoutMs?: number }, +): Promise { + const selected = path.basename(resolveSystemdUnitPath(env)); + const names = + selected === "openclaw-gateway.service" ? [selected, "openclaw.service"] : [selected]; + const deadlineAt = options?.timeoutMs ? Date.now() + options.timeoutMs : undefined; + for (const name of names) { + try { + await assertNoSystemSystemdOwnership( + name, + deadlineAt === undefined ? undefined : Math.max(1, deadlineAt - Date.now()), + ); + } catch (error) { + return { + kind: + isSystemSystemdOwnershipError(error) && error.ownership.status !== "unverifiable" + ? "sealed" + : "unknown", + detail: `System service ${name} requires its privileged deployment owner.`, + }; + } + } + return (await inspect(env, options?.environment ?? env, options?.timeoutMs)).capability; +} + +export async function withSystemdDefinitionMutation( + env: GatewayServiceEnv, + environment: GatewayServiceEnv, + run: (mutation: SystemdDefinitionMutation) => Promise, +): Promise { + let initial = await inspect(env, environment); + assertServiceDefinitionWritable(initial.capability); + const { unit, generated } = resolveMutationTargets(env, environment); + // Group-writable umasks must not create directories that inspect() would reject. + await fs.mkdir(path.dirname(unit), { recursive: true, mode: 0o755 }); + await fs.mkdir(path.dirname(generated), { recursive: true, mode: 0o700 }); + const canonicalTargets = () => + Promise.all([unit, generated].map(canonicalPathFromExistingAncestor)); + const lockedTargets = await canonicalTargets(); + const targets = lockedTargets + .map((target) => path.join(path.dirname(target), `.openclaw-${sha256Hex(target)}`)) + .toSorted(); + const execute = async (): Promise => { + const refresh = async (unchanged = false, firstUnitPublication = false) => { + const current = await inspect(env, environment); + assertServiceDefinitionWritable(current.capability); + const expected = new Map(initial.fingerprint); + // LoadUnit can reveal shared defaults only after the first base publication. + // Admit only new shared inputs; every observed artifact must still match exactly. + if (firstUnitPublication && !initial.sourcePath && current.sourcePath === unit) { + for (const [file, fingerprint] of current.fingerprint) { + if (current.shared.has(file) && !expected.has(file)) { + expected.set(file, fingerprint); + } + } + } + if (unchanged && !isDeepStrictEqual(current.fingerprint, expected)) { + throw new Error("Managed service artifacts changed during publication."); + } + initial = current; + }; + await refresh(); + // Waiting may admit another writer's artifacts, never another directory's locks. + if (!isDeepStrictEqual(await canonicalTargets(), lockedTargets)) { + throw new Error("Managed service lock targets changed during acquisition."); + } + const allowed = new Set([unit, generated, `${unit}.bak`]); + const publications = new Map(); + const publish = async ( + file: string, + contents: string | Buffer, + mode: number, + rollback = true, + ) => { + if (!allowed.has(file)) { + throw new Error("Not a managed service publication target."); + } + await refresh(true); + const previous = initial.snapshots.get(file) ?? null; + const directory = await fs.realpath(path.dirname(file)); + const temporary = path.join(directory, `${path.basename(file)}.${randomUUID()}.tmp`); + try { + await fs.writeFile(temporary, contents, { flag: "wx", mode }); + const written = await fs.lstat(temporary); + await refresh(true); + // Locks coordinate OpenClaw writers, not external editors: POSIX rename + // has no expected-inode check. Quiesce administrative edits during installation. + await fs.rename(temporary, file); + // Re-read every artifact against this inode/payload. Canonical temp paths + // keep cleanup in the original directory even if the publication alias moves. + const published = identity(written, Buffer.from(contents)); + initial.fingerprint.set(file, published); + publications.set(file, published); + try { + await refresh(true, file === unit && previous === null); + } catch (error) { + // Roll back only our unchanged publication; a failing rollback must not recurse. + if (rollback) { + await restore(file, previous); + } + throw error; + } + } finally { + await fs.unlink(temporary).catch(() => undefined); + } + }; + const restore = async (file: string, snapshot: Snapshot) => { + if (!allowed.has(file) && snapshot) { + throw new Error("Not a managed service publication target."); + } + const published = publications.get(file); + if (published === undefined) { + return; + } + const current = await inspect(env, environment); + // A refreshed global snapshot never grants ownership of another artifact's edit. + if (current.capability.kind !== "writable" || current.fingerprint.get(file) !== published) { + return; + } + initial = current; + if (snapshot) { + await publish(file, snapshot.contents, snapshot.mode, false); + } else { + await refresh(true); + await fs.unlink(file); + initial.fingerprint.set(file, "missing"); + await refresh(true); + } + publications.delete(file); + }; + return await run({ snapshots: initial.snapshots, publish, restore }); + }; + const lockOptions = { + stale: 60_000, + retries: { retries: 100, factor: 1, minTimeout: 50, maxTimeout: 100 }, + }; + const acquire = async (index: number): Promise => + index === targets.length + ? execute() + : withFileLock(targets[index]!, lockOptions, () => acquire(index + 1)); + return await acquire(0); +} diff --git a/src/daemon/systemd-install.ts b/src/daemon/systemd-install.ts index d7856ec1aaaf..892805b9c5f8 100644 --- a/src/daemon/systemd-install.ts +++ b/src/daemon/systemd-install.ts @@ -1,12 +1,11 @@ /** systemd unit publication, installation, staging, and uninstall. */ -import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; -import path from "node:path"; import { resolveStateDir } from "../config/paths.js"; import { isUnresolvedShellReference, readStateDirDotEnvFromStateDir, } from "../config/state-dir-dotenv.js"; +import { hasErrnoCode } from "../infra/errno.js"; import { resolveGatewayServiceDescription } from "./constants.js"; import { formatLine, writeFormattedLines } from "./output.js"; import { @@ -26,6 +25,7 @@ import { type GatewayServiceInstallArgs, type GatewayServiceManageArgs, } from "./service-types.js"; +import { withSystemdDefinitionMutation } from "./systemd-definition-mutation.js"; import { assertSystemdAvailable, disableSystemdUserUnitForRemoval, @@ -56,9 +56,7 @@ function collectSystemdInlineManagedKeys(params: { environmentValueSources?: Record; }): Set { const keys = readManagedServiceEnvKeysFromEnvironment(params.environment); - for (const key of collectSystemdFileManagedKeys({ - environmentValueSources: params.environmentValueSources, - })) { + for (const key of collectSystemdFileManagedKeys(params.environmentValueSources)) { keys.delete(key); } for (const [rawKey, value] of Object.entries(params.environment ?? {})) { @@ -78,26 +76,20 @@ function collectSystemdInlineManagedKeys(params: { return keys; } -function collectSystemdFileManagedKeys(params: { - environmentValueSources?: Record; -}): Set { - const keys = new Set(); - for (const [rawKey, source] of Object.entries(params.environmentValueSources ?? {})) { - const key = normalizeServiceEnvKey(rawKey); - if (key && isEnvironmentFileOnlySource(source)) { - keys.add(key); - } - } - return keys; +function collectSystemdFileManagedKeys( + environmentValueSources?: Record, +): Set { + return normalizeServiceEnvKeys( + Object.entries(environmentValueSources ?? {}) + .filter(([, source]) => isEnvironmentFileOnlySource(source)) + .map(([key]) => key), + ); } function collectSystemdFileBackedEnvironment(params: { environment?: GatewayServiceEnv; fileManagedKeys: ReadonlySet; }): Record { - if (params.fileManagedKeys.size === 0) { - return {}; - } const environment: Record = {}; for (const [rawKey, rawValue] of Object.entries(params.environment ?? {})) { if (typeof rawValue !== "string" || !rawValue.trim()) { @@ -128,10 +120,6 @@ function sanitizeSystemdUnitBackupContent(params: { continue; } const assignments = parseSystemdEnvAssignments(line.slice("Environment=".length).trim()); - if (assignments.length === 0) { - sanitizedLines.push(rawLine); - continue; - } const keptAssignments = assignments.filter(({ key }) => { const normalizedKey = normalizeServiceEnvKey(key); return !normalizedKey || !params.fileManagedKeys.has(normalizedKey); @@ -165,305 +153,149 @@ async function writeSystemdUnit({ await assertNoSystemGatewayOwnership(env); const unitPath = resolveSystemdUnitPath(env); - const priorManagedKeys = readManagedServiceEnvKeysFromEnvironment( - resolveManagedGatewayServiceCommand(await readSystemdServiceExecStart(env))?.environment, - ); - await fs.mkdir(path.dirname(unitPath), { recursive: true }); - await assertSystemdManagedPathIsNotSymlink(unitPath); - const fileManagedKeys = collectSystemdFileManagedKeys({ - environmentValueSources, - }); - - // Preserve user customizations: back up existing unit file before overwriting. - let backedUp = false; - try { - const backupPath = `${unitPath}.bak`; - const existingUnit = await fs.readFile(unitPath, "utf8"); - const existingStat = await fs.stat(unitPath); - const backupMode = existingStat.mode & 0o777 || 0o600; - const backupUnit = sanitizeSystemdUnitBackupContent({ - content: existingUnit, - fileManagedKeys, - }); - await fs.writeFile(backupPath, backupUnit, { encoding: "utf8", mode: backupMode }); - await fs.chmod(backupPath, backupMode); - backedUp = true; - } catch { - // File does not exist yet — nothing to back up. - } - - const serviceDescription = resolveGatewayServiceDescription({ env, description }); - const stateDir = resolveStateDir(env as NodeJS.ProcessEnv); - const { entries: stateDirDotEnvEntries, skippedShellReferenceKeys } = - readStateDirDotEnvFromStateDir(stateDir); - const stateDirDotEnvVars = Object.fromEntries( - Object.entries(stateDirDotEnvEntries).filter(([key, value]) => { - const inlineValue = environment?.[key]; - if (typeof inlineValue !== "string") { - return true; - } - return inlineValue.trim() === value.trim(); - }), - ); - const inlineManagedKeys = collectSystemdInlineManagedKeys({ - environment, - environmentValueSources, - }); - const environmentFilePath = resolveSystemdEnvironmentFilePath({ - stateDir, - environment, - }); - const environmentFileSnapshot = isNodeSystemdEnvironment(env) - ? undefined - : await readSystemdFileSnapshot(environmentFilePath); - try { - const environmentFileResult = await writeSystemdGatewayEnvironmentFile({ - stateDir, - stateDirDotEnvKeys: Object.keys(stateDirDotEnvVars), - priorManagedKeys, - inlineManagedKeys, - fileManagedKeys, - skippedManagedKeys: skippedShellReferenceKeys, - fileBackedEnvironment: collectSystemdFileBackedEnvironment({ - environment, - fileManagedKeys, - }), - environment, - }); - const environmentSansDotEnvEntries = Object.fromEntries( - Object.entries(environment ?? {}).filter(([key, value]) => { - if (typeof value !== "string") { - return false; - } - const source = readEnvironmentValueSource(environmentValueSources, key); - if (hasEnvironmentFileSource(source) && isUnresolvedShellReference(value)) { - return false; - } - const normalizedKey = normalizeServiceEnvKey(key); - if ( - normalizedKey && - environmentFileResult.environmentKeys.has(normalizedKey) && - !inlineManagedKeys.has(normalizedKey) - ) { - return false; - } - const stateDirValue = stateDirDotEnvVars[key]; - if (typeof stateDirValue !== "string") { - return true; - } - return value.trim() !== stateDirValue.trim(); + return await withSystemdDefinitionMutation(env, environment ?? env, async (mutation) => { + const priorManagedKeys = readManagedServiceEnvKeysFromEnvironment( + resolveManagedGatewayServiceCommand(await readSystemdServiceExecStart(env))?.environment, + ); + const stateDir = resolveStateDir({ ...env, ...environment }); + const environmentFilePath = resolveSystemdEnvironmentFilePath({ stateDir, environment }); + const environmentFileSnapshot = isNodeSystemdEnvironment(env) + ? undefined + : (mutation.snapshots.get(environmentFilePath) ?? null); + const existingUnit = mutation.snapshots.get(unitPath) ?? null; + const { entries: stateDirDotEnvEntries, skippedShellReferenceKeys } = + readStateDirDotEnvFromStateDir(stateDir); + const stateDirDotEnvVars = new Map( + Object.entries(stateDirDotEnvEntries).filter(([key, value]) => { + const inlineValue = environment?.[key]; + return typeof inlineValue !== "string" || inlineValue.trim() === value.trim(); }), ); - const unit = buildSystemdUnit({ - description: serviceDescription, - programArguments, - workingDirectory, - environment: environmentSansDotEnvEntries, - environmentFiles: environmentFileResult.environmentFiles, + const inlineManagedKeys = collectSystemdInlineManagedKeys({ + environment, + environmentValueSources, }); - await publishSystemdUnit({ env, unitPath, contents: unit }); - } catch (error) { - if (environmentFileSnapshot !== undefined) { - try { - await restoreSystemdFileSnapshot(environmentFilePath, environmentFileSnapshot); - } catch (rollbackError) { - const failureDetail = error instanceof Error ? error.message : String(error); - throw new Error( - `${failureDetail}\nThe previous systemd environment file at ${environmentFilePath} could not be restored.`, - { cause: rollbackError }, - ); - } - } - throw error; - } - return { unitPath, backedUp }; -} + const fileManagedKeys = collectSystemdFileManagedKeys(environmentValueSources); + const existingEnvironment = await readSystemdGatewayEnvironmentFiles(stateDir, environment); -type SystemdFileSnapshot = { contents: Buffer; mode: number } | null; - -async function assertSystemdManagedPathIsNotSymlink(filePath: string): Promise { - try { - const stat = await fs.lstat(filePath); - if (stat.isSymbolicLink()) { - throw new Error(`Refusing to rewrite symlinked managed systemd file: ${filePath}`); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return; - } - throw error; - } -} - -async function readSystemdFileSnapshot(filePath: string): Promise { - try { - const stat = await fs.lstat(filePath); - if (stat.isSymbolicLink()) { - throw new Error(`Refusing to rewrite symlinked managed systemd file: ${filePath}`); - } - const contents = await fs.readFile(filePath); - return { contents, mode: stat.mode & 0o777 }; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return null; - } - throw error; - } -} - -async function restoreSystemdFileSnapshot( - filePath: string, - snapshot: SystemdFileSnapshot, -): Promise { - if (snapshot === null) { - await fs.rm(filePath, { force: true }); - return; - } - await fs.mkdir(path.dirname(filePath), { recursive: true }); - const rollbackPath = `${filePath}.openclaw-${randomUUID()}.rollback`; - try { - await fs.writeFile(rollbackPath, snapshot.contents, { - flag: "wx", - mode: snapshot.mode, - }); - await fs.rename(rollbackPath, filePath); - } finally { - await fs.unlink(rollbackPath).catch(() => undefined); - } -} - -async function publishSystemdUnit(params: { - env: GatewayServiceEnv; - unitPath: string; - contents: string; -}): Promise { - const previous = await readSystemdFileSnapshot(params.unitPath); - const temporaryPath = `${params.unitPath}.openclaw-${randomUUID()}.tmp`; - await fs.writeFile(temporaryPath, params.contents, { - encoding: "utf8", - flag: "wx", - mode: previous?.mode ?? 0o644, - }); - try { - // systemd ignores the temporary suffix, so this is the last ownership check - // before the canonical user unit becomes discoverable. - await assertNoSystemGatewayOwnership(params.env); - await fs.rename(temporaryPath, params.unitPath); - try { - await assertNoSystemGatewayOwnership(params.env); - } catch (ownershipError) { - try { - await restoreSystemdFileSnapshot(params.unitPath, previous); - } catch (rollbackError) { - const ownershipDetail = - ownershipError instanceof Error ? ownershipError.message : String(ownershipError); - throw new Error( - `${ownershipDetail}\nThe previous user systemd unit at ${params.unitPath} could not be restored.`, - { cause: rollbackError }, - ); - } - throw ownershipError; - } - } finally { - await fs.unlink(temporaryPath).catch(() => undefined); - } -} - -async function writeSystemdGatewayEnvironmentFile(params: { - stateDir: string; - /** Keys loaded by the Gateway directly from the state-dir .env. They must be removed from - * generated files so a supervisor restart cannot shadow a later .env edit. */ - stateDirDotEnvKeys?: Iterable; - /** Keys owned by the previously installed service. Preserve the prior ownership record so - * deleting a managed dotenv key cannot reclassify its stale file value as operator-owned. */ - priorManagedKeys?: Iterable; - /** OpenClaw-managed keys that must not be preserved from an old env file; stale file values - * would override fresh inline Environment= entries because EnvironmentFile takes precedence. */ - inlineManagedKeys?: ReadonlySet; - /** File-managed keys that should be written from current environment values or removed when absent. */ - fileManagedKeys?: ReadonlySet; - /** State-dir .env keys OpenClaw previously managed but is now skipping (unresolved shell - * references). A prior re-stage may have written a stale literal value for them; drop it so - * the regenerated env file no longer carries the obsolete reference. */ - skippedManagedKeys?: Iterable; - fileBackedEnvironment?: Record; - environment?: GatewayServiceEnv; -}): Promise<{ environmentFiles: string[]; environmentKeys: Set }> { - const incoming = { ...params.fileBackedEnvironment }; - for (const [key, value] of Object.entries(incoming)) { - if (/[\r\n]/.test(value)) { - throw new Error( - `state-dir .env contains a multiline value for ${key}; systemd EnvironmentFile values must be single-line`, + if (existingUnit) { + await mutation.publish( + `${unitPath}.bak`, + sanitizeSystemdUnitBackupContent({ + content: existingUnit.contents.toString("utf8"), + fileManagedKeys, + }), + existingUnit.mode || 0o600, ); } - } - const envFilePath = resolveSystemdEnvironmentFilePath({ - stateDir: params.stateDir, - environment: params.environment, + try { + const incoming = collectSystemdFileBackedEnvironment({ environment, fileManagedKeys }); + for (const [key, value] of Object.entries(incoming)) { + if (/[\r\n]/.test(value)) { + throw new Error( + `state-dir .env contains a multiline value for ${key}; systemd EnvironmentFile values must be single-line`, + ); + } + } + // Deleted managed values remain managed. Drop their stale file copies so + // EnvironmentFile precedence cannot shadow inline values or runtime .env edits. + const managedKeysToDrop = normalizeServiceEnvKeys([ + ...inlineManagedKeys, + ...fileManagedKeys, + ...priorManagedKeys, + ...stateDirDotEnvVars.keys(), + ...skippedShellReferenceKeys, + ]); + const { existing, literalShellReferenceKeys } = existingEnvironment; + const operatorOnly = Object.fromEntries( + Object.entries(existing).filter(([key, value]) => { + const normalized = normalizeServiceEnvKey(key); + if (normalized && managedKeysToDrop.has(normalized)) { + return false; + } + // Quoted/escaped $VAR is operator intent; bare references can be stale + // values copied from the state-dir dotenv file. + return literalShellReferenceKeys.has(key) || !isUnresolvedShellReference(value); + }), + ); + const merged = { ...operatorOnly, ...incoming }; + const hasGeneratedValues = Object.keys(merged).length > 0; + const environmentKeys = normalizeServiceEnvKeys(Object.keys(merged)); + // Keep an existing empty file readable until the manager drops its reference. + if (hasGeneratedValues || mutation.snapshots.has(environmentFilePath)) { + const content = hasGeneratedValues ? `${serializeSystemdEnvironmentFile(merged)}\n` : ""; + await mutation.publish(environmentFilePath, content, 0o600); + } + const environmentSansDotEnvEntries = Object.fromEntries( + Object.entries(environment ?? {}).filter(([key, value]) => { + if (typeof value !== "string") { + return false; + } + const source = readEnvironmentValueSource(environmentValueSources, key); + const normalized = normalizeServiceEnvKey(key); + const generated = + normalized && environmentKeys.has(normalized) && !inlineManagedKeys.has(normalized); + return ( + !(hasEnvironmentFileSource(source) && isUnresolvedShellReference(value)) && + !generated && + value.trim() !== stateDirDotEnvVars.get(key)?.trim() + ); + }), + ); + const unit = buildSystemdUnit({ + description: resolveGatewayServiceDescription({ env, description }), + programArguments, + workingDirectory, + environment: environmentSansDotEnvEntries, + environmentFiles: hasGeneratedValues ? [environmentFilePath] : [], + }); + await assertNoSystemGatewayOwnership(env); + await mutation.publish(unitPath, unit, existingUnit?.mode ?? 0o644); + try { + await assertNoSystemGatewayOwnership(env); + } catch (ownershipError) { + await mutation.restore(unitPath, existingUnit); + throw ownershipError; + } + } catch (error) { + if (environmentFileSnapshot !== undefined) { + await mutation.restore(environmentFilePath, environmentFileSnapshot); + } + throw error; + } + return { unitPath, backedUp: existingUnit !== null }; }); +} - // Read existing env files first so we can preserve operator-added secrets - // (e.g. provider API keys) across upgrades and re-stages. Node units used - // to share gateway.systemd.env, so migrate those entries into node.systemd.env. - // OpenClaw-managed keys (identified by inlineManagedKeys) are excluded: a stale - // file copy would override the fresh inline Environment= value because systemd's - // EnvironmentFile takes precedence over inline Environment= directives. +async function readSystemdGatewayEnvironmentFiles( + stateDir: string, + environment?: GatewayServiceEnv, +) { const existing: Record = {}; const literalShellReferenceKeys = new Set(); - const legacyNodeEnvFilePath = resolveLegacyNodeSystemdEnvironmentFilePath({ - stateDir: params.stateDir, - environment: params.environment, - }); - for (const sourceEnvFilePath of [legacyNodeEnvFilePath, envFilePath]) { - if (!sourceEnvFilePath) { + for (const sourcePath of [ + resolveLegacyNodeSystemdEnvironmentFilePath({ stateDir, environment }), + resolveSystemdEnvironmentFilePath({ stateDir, environment }), + ]) { + if (!sourcePath) { continue; } try { - const fromFile = await readSystemdEnvironmentFile(sourceEnvFilePath); + const fromFile = await readSystemdEnvironmentFile(sourcePath); for (const [key, value] of Object.entries(fromFile.environment)) { existing[key] = value; + literalShellReferenceKeys.delete(key); if (fromFile.literalShellReferenceKeys.has(key)) { literalShellReferenceKeys.add(key); - } else { - literalShellReferenceKeys.delete(key); } } - } catch { - // File does not exist yet — nothing to preserve. + } catch (error) { + if (!hasErrnoCode(error, "ENOENT")) { + throw error; + } } } - const managedKeysToDrop = normalizeServiceEnvKeys([ - ...(params.inlineManagedKeys ?? []), - ...(params.fileManagedKeys ?? []), - ...(params.priorManagedKeys ?? []), - ...(params.stateDirDotEnvKeys ?? []), - ...(params.skippedManagedKeys ?? []), - ]); - const operatorOnly = Object.fromEntries( - Object.entries(existing).filter(([key, value]) => { - const normalized = normalizeServiceEnvKey(key); - if (normalized && managedKeysToDrop.has(normalized)) { - return false; - } - // Quoting or escaping `$VAR` records operator intent; bare references can - // still be stale values copied from the state-dir dotenv file. - return literalShellReferenceKeys.has(key) || !isUnresolvedShellReference(value); - }), - ); - const merged = { ...operatorOnly, ...incoming }; - const environmentKeys = normalizeServiceEnvKeys(Object.keys(merged)); - - // If the merged result is empty there is nothing to write and no file needed. - if (Object.keys(merged).length === 0) { - await fs.rm(envFilePath, { force: true }).catch(() => undefined); - return { environmentFiles: [], environmentKeys }; - } - - const content = serializeSystemdEnvironmentFile(merged); - await fs.mkdir(path.dirname(envFilePath), { recursive: true }); - await fs.writeFile(envFilePath, `${content}\n`, { encoding: "utf8", mode: 0o600 }); - await fs.chmod(envFilePath, 0o600); - return { environmentFiles: [envFilePath], environmentKeys }; + return { existing, literalShellReferenceKeys }; } async function removeNodeSystemdManagedEnvironmentKeys(env: GatewayServiceEnv): Promise { @@ -500,78 +332,59 @@ async function removeNodeSystemdManagedEnvironmentKeys(env: GatewayServiceEnv): await fs.chmod(envFilePath, 0o600); } +function reportSystemdServicePublication( + stdout: NodeJS.WritableStream, + label: string, + unitPath: string, + backedUp: boolean, +): void { + const lines = [{ label, value: unitPath }]; + if (backedUp) { + lines.push({ label: "Previous unit backed up to", value: `${unitPath}.bak` }); + } + writeFormattedLines(stdout, lines, { leadingBlankLine: true }); +} + export async function stageSystemdService({ stdout, ...args }: GatewayServiceInstallArgs): Promise<{ unitPath: string }> { const { unitPath, backedUp } = await writeSystemdUnit(args); - writeFormattedLines( - stdout, - [ - { - label: "Staged systemd service", - value: unitPath, - }, - ...(backedUp - ? [ - { - label: "Previous unit backed up to", - value: `${unitPath}.bak`, - }, - ] - : []), - ], - { leadingBlankLine: true }, - ); + reportSystemdServicePublication(stdout, "Staged systemd service", unitPath, backedUp); return { unitPath }; } async function activateSystemdService(params: { env: GatewayServiceEnv }) { - const serviceName = resolveSystemdServiceName(params.env); - const unitName = `${serviceName}.service`; + const unitName = `${resolveSystemdServiceName(params.env)}.service`; // A system unit may appear after publication. Refuse before the user manager // can load a second supervisor for the same gateway name. await assertNoSystemGatewayOwnership(params.env); - const reloadSystemd = async () => await execSystemctlUser(params.env, ["daemon-reload"]); - const throwActivationFailure = ( + const runActivation = async ( action: "daemon-reload" | "enable" | "restart", - result: { stdout: string; stderr: string }, - ): never => { + retryMissing = true, + ): Promise => { + const args = action === "daemon-reload" ? [action] : [action, unitName]; + const result = await execSystemctlUser(params.env, args); + if (result.code === 0) { + return; + } const detail = readSystemctlDetail(result); + if ( + action !== "daemon-reload" && + retryMissing && + result.termination === "exit" && + isSystemdUnitMissingDetail(detail) + ) { + await runActivation("daemon-reload"); + return await runActivation(action, false); + } if (isSystemdUserScopeUnavailable(detail)) { throw new Error(`systemctl --user unavailable: ${detail || "unknown error"}`.trim()); } throw new Error(`systemctl ${action} failed: ${detail || "unknown error"}`.trim()); }; - const reload = await reloadSystemd(); - if (reload.code !== 0) { - throwActivationFailure("daemon-reload", reload); - } - - const runAfterReloadRetry = async (action: "enable" | "restart") => { - const result = await execSystemctlUser(params.env, [action, unitName]); - if ( - result.code === 0 || - result.termination !== "exit" || - !isSystemdUnitMissingDetail(readSystemctlDetail(result)) - ) { - return result; - } - const retryReload = await reloadSystemd(); - if (retryReload.code !== 0) { - throwActivationFailure("daemon-reload", retryReload); - } - return await execSystemctlUser(params.env, [action, unitName]); - }; - - const enable = await runAfterReloadRetry("enable"); - if (enable.code !== 0) { - throwActivationFailure("enable", enable); - } - - const restart = await runAfterReloadRetry("restart"); - if (restart.code !== 0) { - throwActivationFailure("restart", restart); + for (const action of ["daemon-reload", "enable", "restart"] as const) { + await runActivation(action); } } @@ -588,24 +401,7 @@ export async function installSystemdService( "Systemd drop-in overrides the managed service command or working directory; inspect, update, or remove the drop-in because reinstalling the base unit does not change the effective launcher.", ); } - writeFormattedLines( - args.stdout, - [ - { - label: "Installed systemd service", - value: unitPath, - }, - ...(backedUp - ? [ - { - label: "Previous unit backed up to", - value: `${unitPath}.bak`, - }, - ] - : []), - ], - { leadingBlankLine: true }, - ); + reportSystemdServicePublication(args.stdout, "Installed systemd service", unitPath, backedUp); return { unitPath }; } @@ -614,8 +410,7 @@ export async function uninstallSystemdService({ stdout, }: GatewayServiceManageArgs): Promise { await assertSystemdAvailable(env); - const serviceName = resolveSystemdServiceName(env); - const unitName = `${serviceName}.service`; + const unitName = `${resolveSystemdServiceName(env)}.service`; await disableSystemdUserUnitForRemoval(env, unitName); const unitPath = resolveSystemdUnitPath(env); diff --git a/src/daemon/systemd-service-files.ts b/src/daemon/systemd-service-files.ts index 872148a34f13..6f8c2fbb79b4 100644 --- a/src/daemon/systemd-service-files.ts +++ b/src/daemon/systemd-service-files.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { asOptionalRecord } from "@openclaw/normalization-core/record-coerce"; import { normalizeStringEntries } from "@openclaw/normalization-core/string-normalization"; import { isUnresolvedShellReference } from "../config/state-dir-dotenv.js"; +import { hasErrnoCode } from "../infra/errno.js"; import { splitArgsPreservingQuotes } from "./arg-split.js"; import { resolveGatewaySystemdServiceName } from "./constants.js"; import { normalizeWindowsPathSeparators } from "./output.js"; @@ -61,6 +62,7 @@ async function buildSystemdCommandSnapshot(params: { unsetEnvironment: string[]; env: GatewayServiceEnv; unitPath: string; + failOnUnavailable?: boolean; }): Promise { const fileEnvironment = await resolveSystemdEnvironmentFiles(params); const environment = { ...params.inlineEnvironment, ...fileEnvironment }; @@ -88,12 +90,13 @@ async function buildSystemdCommandSnapshot(params: { async function readSystemdManagerCommand( env: GatewayServiceEnv, - sourcePath: string, - managedDefinition: GatewayServiceCommandSnapshot, + localDefinition: GatewayServiceCommandSnapshot | null, managedUnsetEnvironment: string[], opts?: GatewayServiceReadOptions, ): Promise { const manager = "org.freedesktop.systemd1"; + const unitName = `${resolveSystemdServiceName(env)}.service`; + const unavailable = () => new Error("Effective systemd service command could not be inspected."); const timeoutMs = opts?.timeoutMs && opts.timeoutMs > 0 ? opts.timeoutMs : SYSTEMD_MANAGER_QUERY_TIMEOUT_MS; const deadlineAt = Date.now() + timeoutMs; @@ -106,57 +109,71 @@ async function readSystemdManagerCommand( Math.max(1, Math.floor((deadlineAt - Date.now()) / remainingCalls--)), ); if (result.code !== 0) { - return null; + if ( + args.includes("LoadUnit") && + result.stderr.trim() === `Call failed: Unit ${unitName} not found.` + ) { + return null; + } + throw unavailable(); } const properties = result.stdout .trim() .split(/\r?\n/) .map((line) => asOptionalRecord(JSON.parse(line))); - return properties.length === signatures.length && - properties.every((property, index) => property?.type === signatures[index]) - ? properties.map((property) => property?.data) - : null; + if ( + properties.length !== signatures.length || + !properties.every((property, index) => property?.type === signatures[index]) + ) { + throw unavailable(); + } + return properties.map((property) => property?.data); }; const loaded = await query( - [ - "call", - manager, - "/org/freedesktop/systemd1", - `${manager}.Manager`, - "LoadUnit", - "s", - `${resolveSystemdServiceName(env)}.service`, - ], + ["call", manager, "/org/freedesktop/systemd1", `${manager}.Manager`, "LoadUnit", "s", unitName], ["o"], ); - const loadedUnit = loaded?.[0]; + if (!loaded) { + return null; + } + const loadedUnit = loaded[0]; const unitPath = Array.isArray(loadedUnit) && loadedUnit.length === 1 ? loadedUnit[0] : null; if (typeof unitPath !== "string" || !unitPath) { - return null; + throw unavailable(); } - const properties = await query( - [ - "get-property", - manager, - unitPath, - `${manager}.Service`, - "ExecStart", - "WorkingDirectory", - "Environment", - "EnvironmentFiles", - "UnsetEnvironment", - ], - ["a(sasbttttuii)", "s", "as", "a(sb)", "as"], - ); - if (!properties) { - return null; - } - const [executions, workingDirectory, assignments, environmentFileSpecs, unsetEnvironment] = - properties; - const execution = Array.isArray(executions) && executions.length === 1 ? executions[0] : null; - const programArguments = Array.isArray(execution) ? execution[1] : null; + const readProperties = (scope: "Unit" | "Service", names: string[], signatures: string[]) => + query(["get-property", manager, unitPath, `${manager}.${scope}`, ...names], signatures); const isStringArray = (value: unknown): value is string[] => Array.isArray(value) && value.every((entry) => typeof entry === "string"); + const unitProperties = await readProperties( + "Unit", + ["FragmentPath", "DropInPaths", "NeedDaemonReload", "LoadState"], + ["s", "as", "b", "s"], + ); + const [sourcePath, dropInPaths, reloadPending, loadState] = unitProperties ?? []; + // LoadUnit also returns objects for missing units; only LoadState proves absence. + if (loadState === "not-found") { + return null; + } + if ( + loadState !== "loaded" || + typeof sourcePath !== "string" || + !sourcePath || + !isStringArray(dropInPaths) || + dropInPaths.some((pathname) => !pathname) || + typeof reloadPending !== "boolean" + ) { + throw unavailable(); + } + const properties = await readProperties( + "Service", + ["ExecStart", "WorkingDirectory", "Environment", "EnvironmentFiles", "UnsetEnvironment"], + ["a(sasbttttuii)", "s", "as", "a(sb)", "as"], + ); + const [executions, workingDirectory, assignments, environmentFileSpecs, unsetEnvironment] = + properties ?? []; + const execution = Array.isArray(executions) && executions.length === 1 ? executions[0] : null; + const programArguments = Array.isArray(execution) ? execution[1] : null; if ( !Array.isArray(execution) || execution.length !== 10 || @@ -180,43 +197,20 @@ async function readSystemdManagerCommand( !isStringArray(unsetEnvironment) || unsetEnvironment.some((assignment) => !assignment || assignment.startsWith("=")) ) { - return null; + throw unavailable(); } const inlineEnvironment: Record = {}; for (const assignment of assignments) { const separator = assignment.indexOf("="); if (separator <= 0) { - return null; + throw unavailable(); } inlineEnvironment[assignment.slice(0, separator)] = assignment.slice(separator + 1); } - const unitProperties = await query( - [ - "get-property", - manager, - unitPath, - `${manager}.Unit`, - "FragmentPath", - "DropInPaths", - "NeedDaemonReload", - ], - ["s", "as", "b"], - ); - const [fragmentPath, dropInPaths, reloadPending] = unitProperties ?? []; - if ( - !unitProperties || - typeof fragmentPath !== "string" || - !isStringArray(dropInPaths) || - dropInPaths.some((pathname) => !pathname) || - typeof reloadPending !== "boolean" - ) { - return null; - } + const managedDefinition = sourcePath === resolveSystemdUnitPath(env) ? localDefinition : null; const managedOverrides = - !reloadPending && - path.posix.normalize(normalizeWindowsPathSeparators(fragmentPath)) === - path.posix.normalize(normalizeWindowsPathSeparators(sourcePath)) + !reloadPending && managedDefinition ? await readSystemdDropInOverrides( dropInPaths, managedUnsetEnvironment, @@ -233,8 +227,11 @@ async function readSystemdManagerCommand( unsetEnvironment, env, unitPath: sourcePath, + failOnUnavailable: opts?.requireEffective, })), - ...(managedOverrides ? { managedDefinition, managedOverrides } : {}), + ...(managedDefinition && managedOverrides ? { managedDefinition, managedOverrides } : {}), + sourcePath, + definitionPaths: [sourcePath, ...dropInPaths], ...(reloadPending ? { reloadPending: true } : {}), }; } @@ -362,13 +359,21 @@ export async function readSystemdServiceExecStart( ): Promise { const unitPath = resolveSystemdUnitPath(env); try { - const content = await fs.readFile(unitPath, "utf8"); + const content = await fs.readFile(unitPath, "utf8").catch((error: unknown) => { + if (!hasErrnoCode(error, "ENOENT")) { + throw error; + } + return null; + }); + if (content === null && !opts?.requireEffective) { + return null; + } let execStart = ""; let workingDirectory = ""; let inlineEnvironment: Record = {}; const environmentFileSpecs: string[] = []; const unsetEnvironment: string[] = []; - for (const rawLine of content.split("\n")) { + for (const rawLine of (content ?? "").split("\n")) { const line = rawLine.trim(); if (!line || line.startsWith("#")) { continue; @@ -386,26 +391,22 @@ export async function readSystemdServiceExecStart( for (const parsed of parseSystemdEnvAssignments(raw)) { inlineEnvironment[parsed.key] = expandSystemdSpecifier(parsed.value, env); } - } else if (line.startsWith("EnvironmentFile=")) { - const raw = line.slice("EnvironmentFile=".length).trim(); - if (raw) { - environmentFileSpecs.push(raw); - } else { - environmentFileSpecs.length = 0; - } - } else if (line.startsWith("UnsetEnvironment=")) { - const raw = line.slice("UnsetEnvironment=".length).trim(); + } else if (line.startsWith("EnvironmentFile=") || line.startsWith("UnsetEnvironment=")) { + const file = line.startsWith("EnvironmentFile="); + const entries = file ? environmentFileSpecs : unsetEnvironment; + const raw = line.slice(line.indexOf("=") + 1).trim(); if (!raw) { - unsetEnvironment.length = 0; + entries.length = 0; } else { - unsetEnvironment.push(...splitSystemdEnvironmentWords(raw)); + entries.push(...(file ? [raw] : splitSystemdEnvironmentWords(raw))); } } } + // Only manager-effective EnvironmentFile entries are required; drop-ins can reset the base. const managedDefinition = await buildSystemdCommandSnapshot({ - programArguments: execStart - ? parseSystemdExecStart(execStart).map((argument) => expandSystemdSpecifier(argument, env)) - : [], + programArguments: parseSystemdExecStart(execStart).map((argument) => + expandSystemdSpecifier(argument, env), + ), workingDirectory, inlineEnvironment, environmentFileSpecs, @@ -413,23 +414,26 @@ export async function readSystemdServiceExecStart( env, unitPath, }); - const manager = await readSystemdManagerCommand( - env, - unitPath, - managedDefinition, - unsetEnvironment, - opts, - ).catch(() => null); - if (!manager && managedDefinition.programArguments.length === 0) { - return null; + const localDefinition = content === null ? null : managedDefinition; + const managerRead = readSystemdManagerCommand(env, localDefinition, unsetEnvironment, opts); + const manager = opts?.requireEffective + ? await managerRead + : await managerRead.catch(() => null); + if (manager || opts?.requireEffective) { + return manager; + } + return managedDefinition.programArguments.length + ? { + ...managedDefinition, + managedDefinition, + managedOverrides: UNKNOWN_SYSTEMD_OVERRIDES, + sourcePath: unitPath, + } + : null; + } catch (error) { + if (opts?.requireEffective) { + throw error; } - const command = manager ?? { - ...managedDefinition, - managedDefinition, - managedOverrides: UNKNOWN_SYSTEMD_OVERRIDES, - }; - return { ...command, sourcePath: unitPath }; - } catch { return null; } } @@ -646,8 +650,8 @@ async function resolveSystemdEnvironmentFiles(params: { const managerExpandedPath = typeof specRaw !== "string"; const tokens = managerExpandedPath ? [specRaw[0]] : parseEnvironmentFileSpecs(specRaw); for (const token of tokens) { - const optional = token.startsWith("-"); - const pathnameRaw = optional ? token.slice(1).trim() : token; + const optional = token.startsWith("-") || (typeof specRaw !== "string" && specRaw[1]); + const pathnameRaw = token.startsWith("-") ? token.slice(1).trim() : token; if (!pathnameRaw) { continue; } diff --git a/src/daemon/systemd-system.test.ts b/src/daemon/systemd-system.test.ts index 4b5fb5089c4c..0d935dcf5567 100644 --- a/src/daemon/systemd-system.test.ts +++ b/src/daemon/systemd-system.test.ts @@ -1,4 +1,5 @@ // System systemd ownership tests cover loaded, installed, and unverifiable states. +import fs from "node:fs/promises"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { ExecResult } from "./exec-file.js"; @@ -41,12 +42,17 @@ vi.mock("node:fs/promises", () => { }); const execFileUtf8 = vi.hoisted(() => - vi.fn(async (_command: string, args: string[]) => - args.includes("--property=UnitPath") ? state.managerUnitPath : state.systemctl, + vi.fn( + async ( + _command: string, + args: string[], + _options?: { timeout?: number; killSignal?: string; env?: NodeJS.ProcessEnv }, + ) => (args.includes("--property=UnitPath") ? state.managerUnitPath : state.systemctl), ), ); vi.mock("./exec-file.js", () => ({ execFileUtf8 })); +import { readSystemdDefinitionMutationCapability } from "./systemd-definition-mutation.js"; import { assertNoSystemSystemdOwnership } from "./systemd-system.js"; describe("system systemd ownership", () => { @@ -90,6 +96,55 @@ describe("system systemd ownership", () => { }); }); + it.each([ + { ownership: "loaded", kind: "sealed" }, + { ownership: "installed", kind: "sealed" }, + { ownership: "unverifiable", kind: "unknown" }, + { ownership: "manager absent", kind: "unknown" }, + { ownership: "unexpected error", kind: "unknown" }, + ])( + "fails closed for $ownership system ownership before user inspection", + async ({ ownership, kind }) => { + const unitName = "openclaw-owned.service"; + const systemUnitPath = `/etc/systemd/system/${unitName}`; + state.systemctl = { + code: ownership === "unverifiable" || ownership === "manager absent" ? 1 : 0, + termination: "exit", + stdout: ownership === "loaded" ? "loaded" : "not-found", + stderr: + ownership === "manager absent" ? "systemctl not available" : "manager-secret-canary", + }; + if (ownership === "installed") { + state.paths.add(systemUnitPath); + } else if (ownership === "unexpected error") { + execFileUtf8.mockRejectedValue(new Error("already owns manager-secret-canary")); + } + + const capability = await readSystemdDefinitionMutationCapability({ + HOME: "/home/openclaw-test", + OPENCLAW_STATE_DIR: "/state/openclaw-test", + OPENCLAW_SYSTEMD_UNIT: unitName, + OPENCLAW_SERVICE_KIND: "node", + }); + + expect(capability).toEqual({ + kind, + detail: `System service ${unitName} requires its privileged deployment owner.`, + }); + expect(JSON.stringify(capability)).not.toContain("manager-secret-canary"); + // Denial permits only system ownership probes, never user-manager or artifact reads. + expect(execFileUtf8.mock.calls.map(([command, args]) => [command, args])).toEqual([ + ["systemctl", ["show", "--property=LoadState", "--value", unitName]], + ...(ownership === "installed" + ? [["systemctl", ["show", "--property=UnitPath", "--value"]]] + : []), + ]); + expect(vi.mocked(fs.lstat).mock.calls).toEqual( + ownership === "installed" ? [[systemUnitPath]] : [], + ); + }, + ); + it.each([ "/etc/systemd/system/openclaw-gateway.service", "/run/systemd/system/openclaw-gateway.service", @@ -116,6 +171,38 @@ describe("system systemd ownership", () => { expect(execFileUtf8).toHaveBeenCalledTimes(3); }); + it("shares one timeout budget across system-manager ownership probes", async () => { + let now = 1_000; + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + execFileUtf8.mockImplementation(async (_command, args) => { + now += 20; + return args.includes("--property=UnitPath") ? state.managerUnitPath : state.systemctl; + }); + try { + await expect( + assertNoSystemSystemdOwnership("openclaw-gateway.service", 50), + ).resolves.toBeUndefined(); + expect( + execFileUtf8.mock.calls.map((call) => ({ + timeout: call[2]?.timeout, + killSignal: call[2]?.killSignal, + })), + ).toEqual([ + { timeout: 50, killSignal: "SIGKILL" }, + { timeout: 30, killSignal: "SIGKILL" }, + { timeout: 10, killSignal: "SIGKILL" }, + ]); + expect(execFileUtf8.mock.calls.every((call) => call[2]?.env === process.env)).toBe(true); + expect(execFileUtf8.mock.calls.map(([command, args]) => [command, args])).toEqual([ + ["systemctl", ["show", "--property=LoadState", "--value", "openclaw-gateway.service"]], + ["systemctl", ["show", "--property=UnitPath", "--value"]], + ["systemctl", ["show", "--property=LoadState", "--value", "openclaw-gateway.service"]], + ]); + } finally { + clock.mockRestore(); + } + }); + it("fails closed when the system manager cannot be queried", async () => { state.systemctl = { stdout: "", @@ -134,6 +221,24 @@ describe("system systemd ownership", () => { }); }); + it.each([ + "spawn systemctl ENOENT", + "systemctl not available", + "System has not been booted with systemd as init system", + ])("fails closed when manager absence cannot be proven: %s", async (detail) => { + state.systemctl = { stdout: "", stderr: detail, code: 1, termination: "exit" }; + + await expect(assertNoSystemSystemdOwnership("openclaw-gateway.service")).rejects.toMatchObject({ + ownership: { + status: "unverifiable", + unitName: "openclaw-gateway.service", + operation: "systemctl", + detail, + }, + }); + expect(fs.lstat).not.toHaveBeenCalled(); + }); + it("does not mistake a missing system bus for a missing unit", async () => { state.systemctl = { stdout: "", diff --git a/src/daemon/systemd-system.ts b/src/daemon/systemd-system.ts index 8133002b0ea6..fdfc78f064ef 100644 --- a/src/daemon/systemd-system.ts +++ b/src/daemon/systemd-system.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import { sanitizeForLog } from "../../packages/terminal-core/src/ansi.js"; import { isMissingPathError } from "../infra/errors.js"; -import { execFileUtf8 } from "./exec-file.js"; +import { execSystemctl, readSystemctlDetail } from "./systemd-exec.js"; type SystemSystemdOwnership = | { status: "absent"; unitName: string } @@ -28,13 +28,19 @@ function quotePosixArgument(value: string): string { return /^[A-Za-z0-9_@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`; } -async function querySystemManager(unitName: string): Promise { - const result = await execFileUtf8("systemctl", [ - "show", - "--property=LoadState", - "--value", - unitName, - ]); +function unverifiableSystemOwnership( + unitName: string, + detail: string, + operation: "systemctl" | "filesystem" = "systemctl", +): SystemSystemdOwnership { + return { status: "unverifiable", unitName, operation, detail }; +} + +async function querySystemManager( + unitName: string, + run = execSystemctl, +): Promise { + const result = await run(["show", "--property=LoadState", "--value", unitName]); const loadState = result.stdout.trim().toLowerCase(); if (result.code === 0) { if (loadState === "not-found") { @@ -43,14 +49,9 @@ async function querySystemManager(unitName: string): Promise { - const result = await execFileUtf8("systemctl", ["show", "--property=UnitPath", "--value"]); + run = execSystemctl, +): Promise { + const result = await run(["show", "--property=UnitPath", "--value"]); if (result.code !== 0) { - const detail = `${result.stderr} ${result.stdout}`.trim(); - return { - status: "unverifiable", - unitName, - operation: "systemctl", - detail: detail || `systemctl exited with code ${result.code}`, - }; + const detail = readSystemctlDetail(result) || `systemctl exited with code ${result.code}`; + return unverifiableSystemOwnership(unitName, detail); } - const paths = [ - ...new Set( - result.stdout - .split(/\s+/) - .map((entry) => entry.trim()) - .filter((entry) => path.posix.isAbsolute(entry)), - ), - ]; - if (paths.length === 0) { - return { - status: "unverifiable", + const loadPaths = [...new Set(result.stdout.split(/\s+/).filter(path.posix.isAbsolute))]; + if (loadPaths.length === 0) { + return unverifiableSystemOwnership( unitName, - operation: "systemctl", - detail: "systemctl returned no system manager unit load paths", - }; - } - return paths; -} - -async function findInstalledSystemUnit(unitName: string): Promise { - const loadPaths = await readSystemUnitLoadPaths(unitName); - if (!Array.isArray(loadPaths)) { - return loadPaths; + "systemctl returned no system manager unit load paths", + ); } for (const dir of loadPaths) { const unitPath = path.posix.join(dir, unitName); @@ -113,33 +88,35 @@ async function findInstalledSystemUnit(unitName: string): Promise { +async function inspectSystemSystemdOwnership( + unitName: string, + timeoutMs?: number, +): Promise { if (process.platform !== "linux") { return { status: "absent", unitName }; } - const initialQuery = await querySystemManager(unitName); + const deadlineAt = timeoutMs && timeoutMs > 0 ? Date.now() + timeoutMs : undefined; + const run = (args: string[]) => + execSystemctl(args, undefined, deadlineAt ? Math.max(1, deadlineAt - Date.now()) : undefined); + const initialQuery = await querySystemManager(unitName, run); if (initialQuery.status !== "absent") { return initialQuery; } - const installed = await findInstalledSystemUnit(unitName); + const installed = await findInstalledSystemUnit(unitName, run); if (installed.status !== "absent") { return installed; } // Close the manager-query-to-filesystem-snapshot race. Publication and // activation repeat the complete probe because root installers share no lock. - return await querySystemManager(unitName); + return await querySystemManager(unitName, run); } function isRunningAsRoot(): boolean { @@ -191,8 +168,17 @@ class SystemSystemdOwnershipError extends Error { } } -export async function assertNoSystemSystemdOwnership(unitName: string): Promise { - const ownership = await inspectSystemSystemdOwnership(unitName); +export function isSystemSystemdOwnershipError( + error: unknown, +): error is SystemSystemdOwnershipError { + return error instanceof SystemSystemdOwnershipError; +} + +export async function assertNoSystemSystemdOwnership( + unitName: string, + timeoutMs?: number, +): Promise { + const ownership = await inspectSystemSystemdOwnership(unitName, timeoutMs); if (ownership.status !== "absent") { throw new SystemSystemdOwnershipError(ownership); } diff --git a/src/daemon/systemd.test.ts b/src/daemon/systemd.test.ts index bccfb0c199a2..f0d1dd087af8 100644 --- a/src/daemon/systemd.test.ts +++ b/src/daemon/systemd.test.ts @@ -8,6 +8,11 @@ import { err as resultErr, ok } from "@openclaw/normalization-core/result"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { buildGatewayInstallPlan } from "../commands/daemon-install-helpers.js"; import type { ExecResult } from "./exec-file.js"; +import { + buildSystemdManagerPropertyOutput, + buildSystemdUnitPropertyOutput as serializeSystemdUnitProperties, + type SystemdManagerSnapshotFixture, +} from "./service.test-helpers.js"; type ExecFileError = Error & { stderr?: string; @@ -46,7 +51,8 @@ vi.mock("./inspect.js", () => ({ findSystemGatewayServices: () => findSystemGatewayServicesMock(), })); -vi.mock("./systemd-system.js", () => ({ +vi.mock("./systemd-system.js", async (importOriginal) => ({ + ...(await importOriginal()), assertNoSystemSystemdOwnership: (unitName: string) => assertNoSystemSystemdOwnershipMock(unitName), })); @@ -84,6 +90,7 @@ vi.mock("./exec-file.js", () => { }); import { splitArgsPreservingQuotes } from "./arg-split.js"; +import * as systemdExec from "./systemd-exec.js"; import { parseSystemdEnvAssignments, parseSystemdExecStart } from "./systemd-unit.js"; import { findInstalledSystemdGatewayScope, @@ -303,48 +310,24 @@ function mockReadGatewayServiceFile( }); } -type SystemdManagerSnapshotFixture = { - programArguments: string[]; - workingDirectory?: string; - environment?: string[]; - environmentFiles?: Array<[string, boolean]>; - unsetEnvironment?: string[]; - fragmentPath?: string; - dropInPaths?: string[]; - needDaemonReload?: boolean; -}; - -function buildSystemdManagerPropertyOutput(snapshot: SystemdManagerSnapshotFixture): string { - return [ - { - type: "a(sasbttttuii)", - data: [[snapshot.programArguments[0], snapshot.programArguments, false, 0, 0, 0, 0, 0, 0, 0]], - }, - { type: "s", data: snapshot.workingDirectory ?? "" }, - { type: "as", data: snapshot.environment ?? [] }, - { type: "a(sb)", data: snapshot.environmentFiles ?? [] }, - { type: "as", data: snapshot.unsetEnvironment ?? [] }, - ] - .map((property) => JSON.stringify(property)) - .join("\n"); -} - function buildSystemdUnitPropertyOutput( - params: Pick, + params: Pick< + SystemdManagerSnapshotFixture, + "fragmentPath" | "dropInPaths" | "needDaemonReload" | "loadState" + >, ): string { - const fragmentPath = - params.fragmentPath ?? `${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`; - return [ - JSON.stringify({ type: "s", data: fragmentPath }), - JSON.stringify({ type: "as", data: params.dropInPaths ?? [] }), - JSON.stringify({ type: "b", data: params.needDaemonReload ?? false }), - ].join("\n"); + return serializeSystemdUnitProperties({ + ...params, + fragmentPath: + params.fragmentPath ?? `${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`, + }); } function mockSystemdManagerProperties( output: string | Error, unitOutput: string | Error = buildSystemdUnitPropertyOutput({}), ): void { + vi.spyOn(systemdExec, "execBusctlUser").mockRestore(); execFileMock.mockReset(); execFileMock.mockImplementation((_command, args, _options, callback) => { const propertyOutput = args.includes("LoadUnit") @@ -1559,6 +1542,216 @@ describe("readSystemdServiceExecStart", () => { vi.restoreAllMocks(); }); + it("strictly distinguishes a missing base unit from an unreadable existing unit", async () => { + execFileMock.mockImplementation((_command, _args, _options, callback) => { + callback(createExecFileError(`Call failed: Unit ${GATEWAY_SERVICE} not found.`), "", ""); + }); + vi.spyOn(fs, "readFile").mockRejectedValueOnce( + Object.assign(new Error("missing service"), { code: "ENOENT" }), + ); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).resolves.toBeNull(); + expect(execFileMock).toHaveBeenCalledWith( + "busctl", + expect.arrayContaining(["LoadUnit", GATEWAY_SERVICE]), + expect.anything(), + expect.anything(), + ); + + vi.mocked(fs.readFile).mockRejectedValueOnce( + Object.assign(new Error("unreadable-service-secret-canary"), { code: "EACCES" }), + ); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).rejects.toThrow("unreadable-service-secret-canary"); + }); + + it.each([false, true])( + "reads a global user fragment without inventing a managed base (local=%s)", + async (local) => { + const fragmentPath = `/etc/systemd/user/${GATEWAY_SERVICE}`; + const dropInPaths = [`/etc/systemd/user/${GATEWAY_SERVICE}.d/10-operator.conf`]; + vi.spyOn(fs, "readFile").mockImplementation(async (file) => { + if (file === "/etc/systemd/user/gateway.env") { + return "OWNER=global\n"; + } + if (local) { + return "[Service]\nExecStart=/usr/bin/managed gateway\n"; + } + throw Object.assign(new Error("missing base"), { code: "ENOENT" }); + }); + mockSystemdManagerSnapshot({ + programArguments: ["/opt/operator/openclaw", "gateway", "run"], + fragmentPath, + dropInPaths, + environmentFiles: [["gateway.env", false]], + needDaemonReload: true, + }); + + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).resolves.toEqual({ + programArguments: ["/opt/operator/openclaw", "gateway", "run"], + environment: { OWNER: "global" }, + environmentValueSources: { OWNER: "file" }, + sourcePath: fragmentPath, + definitionPaths: [fragmentPath, ...dropInPaths], + reloadPending: true, + }); + }, + ); + + it.each([ + { name: "unavailable manager", output: new Error("manager-secret-canary") }, + { name: "malformed properties", output: "manager-secret-canary" }, + { name: "wrong property types", output: JSON.stringify({ type: "s", data: "bad" }) }, + ])("strictly rejects a missing local base with $name", async ({ output }) => { + vi.spyOn(fs, "readFile").mockRejectedValue( + Object.assign(new Error("missing base"), { code: "ENOENT" }), + ); + mockSystemdManagerProperties(output); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).rejects.toThrow(); + await expect(readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME })).resolves.toBeNull(); + }); + + it("requires manager-effective inspection for an existing unit only in strict mode", async () => { + mockReadGatewayServiceFile(["[Service]", "ExecStart=/usr/bin/openclaw gateway run"]); + mockSystemdManagerProperties(new Error("manager-effective-secret-canary")); + + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).rejects.toThrow(); + await expect(readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME })).resolves.toMatchObject({ + programArguments: ["/usr/bin/openclaw", "gateway", "run"], + }); + }); + + it.each([false, true])( + "accepts manager LoadState=not-found before inspecting empty service properties (local=%s)", + async (local) => { + if (local) { + mockReadGatewayServiceFile(["[Service]", "ExecStart=/usr/bin/openclaw gateway run"]); + } else { + vi.spyOn(fs, "readFile").mockRejectedValue( + Object.assign(new Error("missing base"), { code: "ENOENT" }), + ); + } + mockSystemdManagerProperties( + new Error("must not inspect a missing service"), + buildSystemdUnitPropertyOutput({ fragmentPath: "", loadState: "not-found" }), + ); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).resolves.toBeNull(); + expect(execFileMock).toHaveBeenCalledTimes(2); + }, + ); + + it.each([ + { name: "malformed LoadUnit", loaded: JSON.stringify({ type: "o", data: [] }) }, + { name: "failed LoadUnit", loaded: new Error("Call failed: Permission denied: secret-canary") }, + { name: "failed property", unit: new Error("Failed to get property LoadState: secret-canary") }, + { name: "empty fragment", unit: buildSystemdUnitPropertyOutput({ fragmentPath: "" }) }, + { name: "empty drop-in", unit: buildSystemdUnitPropertyOutput({ dropInPaths: [""] }) }, + { name: "invalid unit", unit: buildSystemdUnitPropertyOutput({ loadState: "error" }) }, + ])("strictly rejects $name with a missing local base", async ({ loaded, unit }) => { + vi.spyOn(fs, "readFile").mockRejectedValue( + Object.assign(new Error("missing base"), { code: "ENOENT" }), + ); + mockSystemdManagerProperties( + buildSystemdManagerPropertyOutput({ programArguments: ["/usr/bin/openclaw", "gateway"] }), + unit, + ); + if (loaded) { + execFileMock.mockImplementationOnce((_command, _args, _options, callback) => { + if (loaded instanceof Error) { + callback(createExecFileError(loaded.message), "", loaded.message); + } else { + callback(null, loaded, ""); + } + }); + } + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).rejects.toThrow(); + }); + + it("does not mistake an unreadable required environment file for a missing base unit", async () => { + const environmentFile = `${TEST_SERVICE_HOME}/.openclaw/effective.env`; + mockReadGatewayServiceFile([ + "[Service]", + "ExecStart=/usr/bin/openclaw gateway run", + `EnvironmentFile=${environmentFile}`, + ]); + + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).rejects.toThrow(); + await expect(readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME })).resolves.toMatchObject({ + programArguments: ["/usr/bin/openclaw", "gateway", "run"], + }); + }); + + it("requires manager-effective environment files while preserving optional manager files", async () => { + const environmentFile = `${TEST_SERVICE_HOME}/.openclaw/effective.env`; + mockReadGatewayServiceFile(["[Service]", "ExecStart=/usr/bin/openclaw gateway run"]); + mockSystemdManagerSnapshot({ + programArguments: ["/usr/bin/openclaw", "gateway", "run"], + environmentFiles: [[environmentFile, false]], + }); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).rejects.toThrow(); + + mockSystemdManagerSnapshot({ + programArguments: ["/usr/bin/openclaw", "gateway", "run"], + environmentFiles: [[environmentFile, true]], + }); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).resolves.toMatchObject({ programArguments: ["/usr/bin/openclaw", "gateway", "run"] }); + }); + + it.each(["ENOENT", "EACCES"])( + "enforces only active EnvironmentFile inputs (%s)", + async (code) => { + const inactive = `${TEST_SERVICE_HOME}/.openclaw/retired.env`; + const active = `${TEST_SERVICE_HOME}/.openclaw/current.env`; + const dropIn = `${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}.d/environment.conf`; + mockReadGatewayServiceFile( + ["[Service]", "ExecStart=/usr/bin/openclaw gateway run", `EnvironmentFile=${inactive}`], + { + [inactive]: Object.assign(new Error("retired environment unavailable"), { code }), + [active]: "ACTIVE_VALUE=current\n", + [dropIn]: `[Service]\nEnvironmentFile=\nEnvironmentFile=${active}\n`, + }, + ); + mockSystemdManagerSnapshot({ + programArguments: ["/usr/bin/openclaw", "gateway", "run"], + environmentFiles: [[active, false]], + dropInPaths: [dropIn], + }); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).resolves.toMatchObject({ + environment: { ACTIVE_VALUE: "current" }, + sourcePath: `${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`, + definitionPaths: [`${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`, dropIn], + managedOverrides: { environment: { keys: ["ACTIVE_VALUE"], resetFiles: true } }, + }); + mockSystemdManagerSnapshot({ + programArguments: ["/usr/bin/openclaw", "gateway", "run"], + environmentFiles: [[inactive, false]], + }); + await expect( + readSystemdServiceExecStart({ HOME: TEST_SERVICE_HOME }, { requireEffective: true }), + ).rejects.toThrow("retired environment unavailable"); + }, + ); + it("reports one manager-effective command snapshot while retaining the managed base definition", async () => { const effectiveArguments = ["/opt/operator/openclaw", "gateway", "run"]; const objectPath = "/org/freedesktop/systemd1/unit/openclaw_2dgateway_2eservice"; @@ -1654,7 +1847,7 @@ describe("readSystemdServiceExecStart", () => { expect(options.timeout).toBeLessThanOrEqual(1234); expect(options.killSignal).toBe("SIGKILL"); } - expect(execFileMock.mock.calls[2]?.[1]).toEqual([ + expect(execFileMock.mock.calls[1]?.[1]).toEqual([ "--user", "--json=short", "get-property", @@ -1664,6 +1857,7 @@ describe("readSystemdServiceExecStart", () => { "FragmentPath", "DropInPaths", "NeedDaemonReload", + "LoadState", ]); }); @@ -1748,6 +1942,7 @@ describe("readSystemdServiceExecStart", () => { environment: { OPENCLAW_HOME: `${TEST_SERVICE_HOME}/openclaw`, UNIT_NAME: GATEWAY_SERVICE }, environmentValueSources: { OPENCLAW_HOME: "inline", UNIT_NAME: "inline" }, sourcePath: `${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`, + definitionPaths: [`${TEST_SERVICE_HOME}/.config/systemd/user/${GATEWAY_SERVICE}`], }); }); @@ -2133,6 +2328,12 @@ describe("stageSystemdService", () => { beforeEach(() => { vi.restoreAllMocks(); execFileMock.mockReset(); + vi.spyOn(systemdExec, "execBusctlUser").mockImplementation(async (env) => ({ + code: 1, + termination: "exit", + stdout: "", + stderr: `Call failed: Unit ${env.OPENCLAW_SYSTEMD_UNIT ?? "openclaw-gateway-work"}.service not found.`, + })); assertNoSystemSystemdOwnershipMock.mockReset(); assertNoSystemSystemdOwnershipMock.mockResolvedValue(); }); @@ -2148,13 +2349,7 @@ describe("stageSystemdService", () => { ); await expect( - stageSystemdService({ - env, - stdout: createWritableStreamMock().stdout, - programArguments: ["/usr/bin/openclaw", "gateway", "run"], - workingDirectory: "/tmp", - environment: { OPENCLAW_GATEWAY_PORT: "18789" }, - }), + stageSystemdService(gatewayPortSystemdServiceFixture(env, "18789")), ).rejects.toThrow("system scope owns openclaw-gateway-stage-test.service"); await expect(fs.readFile(unitPath, "utf8")).resolves.toBe(previous); @@ -2166,55 +2361,6 @@ describe("stageSystemdService", () => { }); }); - it("refuses to rewrite a symlinked managed user unit", async () => { - await withStageFixture(async ({ env, unitPath }) => { - const targetPath = path.join(path.dirname(unitPath), "operator-gateway.service"); - const previous = "[Unit]\nDescription=Operator gateway\n"; - await fs.mkdir(path.dirname(unitPath), { recursive: true }); - await fs.writeFile(targetPath, previous, "utf8"); - await fs.symlink(targetPath, unitPath); - mockSystemctlStatusOk(); - - await expect( - stageSystemdService({ - env, - stdout: createWritableStreamMock().stdout, - programArguments: ["/usr/bin/openclaw", "gateway", "run"], - workingDirectory: "/tmp", - environment: { OPENCLAW_GATEWAY_PORT: "18789" }, - }), - ).rejects.toThrow(`Refusing to rewrite symlinked managed systemd file: ${unitPath}`); - - await expect(fs.lstat(unitPath)).resolves.toMatchObject({}); - await expect(fs.readlink(unitPath)).resolves.toBe(targetPath); - await expect(fs.readFile(targetPath, "utf8")).resolves.toBe(previous); - }); - }); - - it("refuses to rewrite a symlinked managed environment file", async () => { - await withStageFixture(async ({ env, envFilePath }) => { - const targetPath = path.join(path.dirname(envFilePath), "operator-gateway.env"); - const previous = "OPENCLAW_GATEWAY_TOKEN=operator-token\n"; - await fs.writeFile(targetPath, previous, "utf8"); - await fs.symlink(targetPath, envFilePath); - mockSystemctlStatusOk(); - - await expect( - stageSystemdService({ - env, - stdout: createWritableStreamMock().stdout, - programArguments: ["/usr/bin/openclaw", "gateway", "run"], - workingDirectory: "/tmp", - environment: { OPENCLAW_GATEWAY_TOKEN: "new-token" }, - environmentValueSources: { OPENCLAW_GATEWAY_TOKEN: "file" }, - }), - ).rejects.toThrow(`Refusing to rewrite symlinked managed systemd file: ${envFilePath}`); - - await expect(fs.readlink(envFilePath)).resolves.toBe(targetPath); - await expect(fs.readFile(targetPath, "utf8")).resolves.toBe(previous); - }); - }); - it("rolls back a new environment file when ownership appears before publication", async () => { await withStageFixture(async ({ env, unitPath, envFilePath }) => { mockSystemctlStatusOk(); @@ -2223,17 +2369,15 @@ describe("stageSystemdService", () => { .mockRejectedValueOnce(new Error("system ownership appeared")); await expect( - stageSystemdService({ - env, - stdout: createWritableStreamMock().stdout, - programArguments: ["/usr/bin/openclaw", "gateway", "run"], - workingDirectory: "/tmp", - environment: { - OPENCLAW_GATEWAY_PORT: "18789", - OPENCLAW_GATEWAY_TOKEN: "new-token", - }, - environmentValueSources: { OPENCLAW_GATEWAY_TOKEN: "file" }, - }), + stageSystemdService( + gatewaySystemdServiceFixture(env, { + environment: { + OPENCLAW_GATEWAY_PORT: "18789", + OPENCLAW_GATEWAY_TOKEN: "new-token", + }, + environmentValueSources: { OPENCLAW_GATEWAY_TOKEN: "file" }, + }), + ), ).rejects.toThrow("system ownership appeared"); await expect(fs.access(unitPath)).rejects.toMatchObject({ code: "ENOENT" }); @@ -2248,14 +2392,7 @@ describe("stageSystemdService", () => { await fs.mkdir(path.dirname(unitPath), { recursive: true }); await fs.writeFile(unitPath, previous, "utf8"); await fs.writeFile(envFilePath, previousEnv, "utf8"); - const originalLstat = fs.lstat.bind(fs); - vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { - const stat = await originalLstat(...args); - if (args[0] === unitPath || args[0] === envFilePath) { - Object.defineProperty(stat, "mode", { value: 0 }); - } - return stat; - }); + await Promise.all([fs.chmod(unitPath, 0o400), fs.chmod(envFilePath, 0o400)]); mockSystemctlStatusOk(); assertNoSystemSystemdOwnershipMock .mockResolvedValueOnce() @@ -2263,26 +2400,23 @@ describe("stageSystemdService", () => { .mockRejectedValueOnce(new Error("system ownership appeared")); await expect( - stageSystemdService({ - env, - stdout: createWritableStreamMock().stdout, - programArguments: ["/usr/bin/openclaw", "gateway", "run"], - workingDirectory: "/tmp", - environment: { - OPENCLAW_GATEWAY_PORT: "18789", - OPENCLAW_GATEWAY_TOKEN: "new-token", - }, - environmentValueSources: { OPENCLAW_GATEWAY_TOKEN: "file" }, - }), + stageSystemdService( + gatewaySystemdServiceFixture(env, { + environment: { + OPENCLAW_GATEWAY_PORT: "18789", + OPENCLAW_GATEWAY_TOKEN: "new-token", + }, + environmentValueSources: { OPENCLAW_GATEWAY_TOKEN: "file" }, + }), + ), ).rejects.toThrow("system ownership appeared"); const [unitStat, environmentStat] = await Promise.all([ fs.stat(unitPath), fs.stat(envFilePath), ]); - expect(unitStat.mode & 0o777).toBe(0); - expect(environmentStat.mode & 0o777).toBe(0); - await Promise.all([fs.chmod(unitPath, 0o600), fs.chmod(envFilePath, 0o600)]); + expect(unitStat.mode & 0o777).toBe(0o400); + expect(environmentStat.mode & 0o777).toBe(0o400); await expect(fs.readFile(unitPath, "utf8")).resolves.toBe(previous); await expect(fs.readFile(envFilePath, "utf8")).resolves.toBe(previousEnv); }); @@ -2316,13 +2450,7 @@ describe("stageSystemdService", () => { .mockRejectedValueOnce(new Error("system ownership appeared before activation")); await expect( - installSystemdService({ - env, - stdout: createWritableStreamMock().stdout, - programArguments: ["/usr/bin/openclaw", "gateway", "run"], - workingDirectory: "/tmp", - environment: { OPENCLAW_GATEWAY_PORT: "18789" }, - }), + installSystemdService(gatewayPortSystemdServiceFixture(env, "18789")), ).rejects.toThrow("system ownership appeared before activation"); await expect(fs.access(unitPath)).resolves.toBeUndefined(); @@ -2588,7 +2716,7 @@ describe("stageSystemdService", () => { const unit = await fs.readFile(unitPath, "utf8"); expect(unit).not.toContain("EnvironmentFile="); - await expect(fs.access(nodeEnvFilePath)).rejects.toThrow(); + await expect(fs.readFile(nodeEnvFilePath, "utf8")).resolves.toBe(""); await expect(fs.readFile(envFilePath, "utf8")).resolves.toBe( "OPENCLAW_GATEWAY_TOKEN=stale-token\n", ); @@ -2620,7 +2748,7 @@ describe("stageSystemdService", () => { expect(unit).not.toContain("EnvironmentFile="); expect(unit).not.toContain("LLM_API_KEY"); expect(unit).not.toContain("$SECRET_FROM_SHELL"); - await expect(fs.access(envFilePath)).rejects.toThrow(); + await expect(fs.readFile(envFilePath, "utf8")).resolves.toBe(""); }); }); @@ -2635,6 +2763,7 @@ describe("stageSystemdService", () => { "Environment=FOO=bar OPENCLAW_GATEWAY_TOKEN=inline-token BAZ=qux", "Environment=OPENCLAW_GATEWAY_TOKEN=token-only-line", "Environment='OPENCLAW_GATEWAY_TOKEN=single-quoted-token' FROM_SINGLE=kept", + "Environment=", "Environment=OPENCLAW_GATEWAY_PORT=18789", ].join("\n"), { encoding: "utf8", mode: 0o600 }, @@ -2666,8 +2795,10 @@ describe("stageSystemdService", () => { expect(backupUnit).not.toContain("Environment=OPENCLAW_GATEWAY_TOKEN=inline-token"); expect(backupUnit).not.toContain("Environment=OPENCLAW_GATEWAY_TOKEN=token-only-line"); expect(backupUnit).not.toContain("single-quoted-token"); + expect(backupUnit).toContain("[Service]"); + expect(backupUnit).toContain("ExecStart=/usr/bin/openclaw node run"); expect(backupUnit).toContain("Environment=FOO=bar BAZ=qux"); - expect(backupUnit).toContain("Environment=FROM_SINGLE=kept"); + expect(backupUnit).toContain("Environment=FROM_SINGLE=kept\nEnvironment=\n"); expect(backupUnit).toContain("Environment=OPENCLAW_GATEWAY_PORT=18789"); expect(backupStat.mode & 0o777).toBe(0o600); }); @@ -2677,7 +2808,11 @@ describe("stageSystemdService", () => { await withStageFixture(async ({ env, stateDir, unitPath, envFilePath }) => { await fs.writeFile( path.join(stateDir, ".env"), - ["OPENCLAW_GATEWAY_TOKEN=stale-token", "LLM_API_KEY=dotenv-key"].join("\n"), + [ + "OPENCLAW_GATEWAY_TOKEN=stale-token", + "LLM_API_KEY=dotenv-key", + "toString=dotenv-string", + ].join("\n"), "utf8", ); @@ -2688,6 +2823,8 @@ describe("stageSystemdService", () => { environment: { OPENCLAW_GATEWAY_TOKEN: "fresh-token", LLM_API_KEY: "dotenv-key", + constructor: "inline-constructor", + toString: "dotenv-string", }, }), ); @@ -2697,6 +2834,8 @@ describe("stageSystemdService", () => { expect(unit).not.toContain("EnvironmentFile="); expect(unit).toContain("Environment=OPENCLAW_GATEWAY_TOKEN=fresh-token"); expect(unit).not.toContain("Environment=LLM_API_KEY=dotenv-key"); + expect(unit).toContain("Environment=constructor=inline-constructor"); + expect(unit).not.toContain("Environment=toString=dotenv-string"); await expect(fs.access(envFilePath)).rejects.toMatchObject({ code: "ENOENT" }); }); }); @@ -2942,6 +3081,12 @@ describe("systemd service install and uninstall", () => { beforeEach(() => { vi.restoreAllMocks(); execFileMock.mockReset(); + vi.spyOn(systemdExec, "execBusctlUser").mockImplementation(async (env) => ({ + code: 1, + termination: "exit", + stdout: "", + stderr: `Call failed: Unit ${env.OPENCLAW_SYSTEMD_UNIT ?? "openclaw-gateway-work"}.service not found.`, + })); }); it("activates the OPENCLAW_SYSTEMD_UNIT override during install", async () => { @@ -2988,6 +3133,7 @@ describe("systemd service install and uninstall", () => { const dropInPath = path.join(`${unitPath}.d`, "operator.conf"); await fs.mkdir(path.dirname(dropInPath), { recursive: true }); await fs.writeFile(dropInPath, `[Service]\n${directive}\n`); + await fs.writeFile(unitPath, "[Service]\nExecStart=/usr/bin/openclaw node run\n"); mockSystemdManagerSnapshot({ programArguments: ["/usr/bin/openclaw", "node", "run"], workingDirectory: "/tmp", @@ -3117,7 +3263,7 @@ describe("systemd service install and uninstall", () => { it("uses the sudo-u target user for install activation machine-scope retry", async () => { await withNodeSystemdFixture(async ({ env }) => { - mockEffectiveUid(1000); + mockEffectiveUid(process.getuid?.() ?? 1000); const installEnv = { ...env, USER: "openclaw", SUDO_USER: "admin" }; mockNodeInstallNoMediumFailure("openclaw"); diff --git a/test/scripts/docker-build-helper.test.ts b/test/scripts/docker-build-helper.test.ts index afaaa8f12d3d..f0da13a82a69 100644 --- a/test/scripts/docker-build-helper.test.ts +++ b/test/scripts/docker-build-helper.test.ts @@ -126,10 +126,11 @@ function extractUpgradeSurvivorPayload(script: string) { const marker = " bash -lc "; const start = script.indexOf(marker); const quoted = script.slice(start + marker.length).trimEnd(); - if (start < 0 || !quoted.startsWith("'") || !quoted.endsWith("'")) { + const end = quoted.search(/\n'(?:\n|$)/u); + if (start < 0 || !quoted.startsWith("'") || end < 0) { throw new Error("upgrade survivor bash -lc payload not found"); } - return quoted.slice(1, -1).replaceAll(`'"'"'`, "'"); + return quoted.slice(1, end + 1).replaceAll(`'"'"'`, "'"); } // Prompt-driving scripts must consume public prompts in the order the CLI renders them. @@ -2986,7 +2987,9 @@ exec "$@" const exitPromise = new Promise<{ code: number | null; signal: NodeJS.Signals | null; - }>((resolve) => child.once("exit", (code, signal) => resolve({ code, signal }))); + }>((resolve) => { + child.once("exit", (code, signal) => resolve({ code, signal })); + }); try { for (let attempt = 0; attempt < 500 && !existsSync(markerPath); attempt += 1) { @@ -5279,7 +5282,7 @@ done expect(unitPath.stdout).toContain("/etc/systemd/system"); }); - it("reports the installed doctor switch unit through the systemd manager", () => { + it("reports the installed doctor switch unit through the systemd manager", async () => { const home = tempDirs.make("openclaw-doctor-busctl-shim-"); const serviceName = "openclaw-gateway.service"; const unitPath = join(home, ".config", "systemd", "user", serviceName); @@ -5363,13 +5366,32 @@ done "FragmentPath", "DropInPaths", "NeedDaemonReload", + "LoadState", ]), ).toEqual([ { type: "s", data: unitPath }, { type: "as", data: [] }, { type: "b", data: false }, + { type: "s", data: "loaded" }, ]); + const binDir = join(home, "bin"); + writeExecutables(binDir, { busctl: readFileSync(DOCTOR_SWITCH_BUSCTL_SHIM_PATH, "utf8") }); + const { readSystemdServiceExecStart } = + await import("../../src/daemon/systemd-service-files.js"); + expect( + await readSystemdServiceExecStart( + { HOME: home, PATH: `${binDir}:${process.env.PATH}`, OPENCLAW_SYSTEMD_UNIT: serviceName }, + { requireEffective: true }, + ), + ).toMatchObject({ + programArguments, + workingDirectory: "/opt/openclaw git", + sourcePath: unitPath, + definitionPaths: [unitPath], + environment: { GREETING: "hello world", OPENCLAW_PROFILE: "fixture" }, + }); + const unexpected = spawnSync( DOCTOR_SWITCH_BUSCTL_SHIM_PATH, ["--user", "--json=short", "list"], @@ -5382,6 +5404,75 @@ done expect(unexpected.stderr).toContain("unexpected invocation"); }); + it("distinguishes a missing named doctor switch unit from failed or unsupported inspection", async () => { + const home = tempDirs.make("openclaw-doctor-busctl-absence-"); + const binDir = join(home, "bin"); + const serviceName = "openclaw-gateway-fixture.service"; + const unitPath = join(home, ".config/systemd/user", serviceName); + writeExecutables(binDir, { busctl: readFileSync(DOCTOR_SWITCH_BUSCTL_SHIM_PATH, "utf8") }); + const env = { + HOME: home, + PATH: `${binDir}:${process.env.PATH}`, + OPENCLAW_SYSTEMD_UNIT: "openclaw-gateway-fixture", + }; + const { readSystemdServiceExecStart } = + await import("../../src/daemon/systemd-service-files.js"); + expect(await readSystemdServiceExecStart(env, { requireEffective: true })).toBeNull(); + const loadArgs = [ + "--user", + "--json=short", + "call", + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1", + "org.freedesktop.systemd1.Manager", + "LoadUnit", + "s", + serviceName, + ]; + const invoke = (args: string[]) => + spawnSync(join(binDir, "busctl"), args, { env, encoding: "utf8" }); + const missing = invoke(loadArgs); + expect(missing.status).toBe(1); + expect(missing.stderr.trim()).toBe(`Call failed: Unit ${serviceName} not found.`); + for (const args of [ + [...loadArgs, "extra"], + [...loadArgs.slice(0, -1), "../missing.service"], + [...loadArgs.slice(0, -1), "unrelated.service"], + ["--user", "--json=short", "list"], + ]) { + const unsupported = invoke(args); + expect(unsupported.status).toBe(1); + expect(unsupported.stderr).not.toContain("not found."); + } + mkdirSync(dirname(unitPath), { recursive: true }); + writeFileSync( + unitPath, + "[Service]\nExecStart=/usr/bin/node /opt/profile/openclaw.mjs gateway\nEnvironment=OLD=stale\nEnvironment=\nEnvironment=KEEP=current REMOVE=value\nUnsetEnvironment=KEEP\nUnsetEnvironment=\nUnsetEnvironment=REMOVE\nEnvironmentFile=/missing/required.env\nEnvironmentFile=\n", + ); + const command = await readSystemdServiceExecStart(env, { requireEffective: true }); + expect(command?.sourcePath).toBe(unitPath); + expect(command?.environment).toEqual({ KEEP: "current" }); + rmSync(unitPath); + mkdirSync(unitPath); + const unreadable = invoke(loadArgs); + expect(unreadable.status).toBe(1); + expect(unreadable.stderr).not.toContain("not found."); + const staleObject = invoke([ + "--user", + "--json=short", + "get-property", + "org.freedesktop.systemd1", + "/org/freedesktop/systemd1/unit/openclaw_2dgateway_2dfixture_2eservice", + "org.freedesktop.systemd1.Unit", + "FragmentPath", + "DropInPaths", + "NeedDaemonReload", + "LoadState", + ]); + expect(staleObject.status).toBe(1); + expect(staleObject.stdout).not.toContain('"loaded"'); + }); + it("routes doctor install switch commands through the E2E timeout helper", () => { const runner = readFileSync(DOCTOR_SWITCH_DOCKER_E2E_PATH, "utf8"); const scenario = readFileSync(DOCTOR_SWITCH_SCENARIO_PATH, "utf8"); diff --git a/test/scripts/install-cli.test.ts b/test/scripts/install-cli.test.ts index 21f120c7cc59..19e3e44cd6af 100644 --- a/test/scripts/install-cli.test.ts +++ b/test/scripts/install-cli.test.ts @@ -712,6 +712,73 @@ describe("install-cli.sh", () => { } }); + it.each([ + { error: "SERVICE_DEFINITION_SEALED: protected", args: "", stream: "stderr" }, + { error: "SERVICE_DEFINITION_SEALED: protected", args: "--json", stream: "stdout" }, + { error: "SERVICE_DEFINITION_UNKNOWN: inaccessible", args: "--json", stream: "stderr" }, + { error: "service manager unavailable", args: "--json", stream: "stderr" }, + ])("handles a traced $error refresh in $stream", ({ args, error, stream }) => { + const root = tempDirs.make("openclaw-install-cli-definition-"); + const prefix = join(root, "prefix"); + const openclaw = join(prefix, "bin", "openclaw"); + const secretCanary = "installer-cli-secret-canary-never-render"; + const commandLog = join(root, "commands.log"); + mkdirSync(join(prefix, "bin"), { recursive: true }); + writeFileSync( + openclaw, + [ + "#!/bin/bash", + 'printf "%s\\n" "$*" >> "$COMMAND_LOG"', + 'if [[ "$1" == "--version" ]]; then printf "OpenClaw 2026.8.25\\n"; exit 0; fi', + 'if [[ "$*" == "gateway install --force" ]]; then', + ' if [[ "$SERVICE_STREAM" == stdout ]]; then printf "%s\\n" "$SERVICE_ERROR"; else printf "%s\\n" "$SERVICE_ERROR" >&2; fi', + ' printf "%s\\n" "$SECRET_CANARY" >&2; exit 1', + "fi", + ].join("\n"), + ); + chmodSync(openclaw, 0o755); + + const result = runInstallCliShell( + [ + "set -euo pipefail", + `source ${JSON.stringify(SCRIPT_PATH)}`, + "install_node() { :; }; ensure_git() { :; }; install_openclaw() { :; }", + "is_gateway_daemon_loaded() { return 0; }", + "set -x", + `main ${args} --prefix ${JSON.stringify(prefix)}`, + ].join("\n"), + { + COMMAND_LOG: commandLog, + SECRET_CANARY: secretCanary, + SERVICE_ERROR: error, + SERVICE_STREAM: stream, + }, + ); + + const denied = error.startsWith("SERVICE_DEFINITION_"); + expect(readFileSync(commandLog, "utf8").split("\n")).not.toContain("gateway restart"); + expect(result.status).toBe(0); + expect(result.stderr).toContain("+ main"); + expect(result.stdout + result.stderr).not.toContain(secretCanary); + if (denied) { + expect(result.stderr).toContain("gateway service definition left unchanged"); + expect(result.stderr).toContain( + error.includes("SEALED") + ? "privileged deployment owner" + : "inspect service-definition access", + ); + if (args) { + expect(result.stdout).toContain('"event":"done"'); + expect(result.stdout).toContain('"reason":"definition-mutation-denied"'); + } else { + expect(result.stdout).toContain("OpenClaw installed (OpenClaw 2026.8.25)."); + } + } else { + expect(result.stdout).toContain('"reason":"install-failed"'); + expect(result.stdout).toContain('"event":"done"'); + } + }); + it.each([ { args: "--json", mode: "JSON" }, { args: "", mode: "human" }, diff --git a/test/scripts/install-sh.test.ts b/test/scripts/install-sh.test.ts index 0e5de8415aae..224ada4389f4 100644 --- a/test/scripts/install-sh.test.ts +++ b/test/scripts/install-sh.test.ts @@ -3576,6 +3576,71 @@ EOF } }); + it.each([ + { error: "SERVICE_DEFINITION_SEALED: protected", stream: "stderr" }, + { error: "SERVICE_DEFINITION_SEALED: protected", stream: "stdout" }, + { error: "SERVICE_DEFINITION_UNKNOWN: inaccessible", stream: "stderr" }, + { error: "service manager unavailable", stream: "stderr" }, + ])("handles a traced $error refresh in $stream", ({ error, stream }) => { + const root = mkdtempSync(join(tmpdir(), "openclaw-install-definition-")); + const openclaw = join(root, "openclaw"); + const secretCanary = "installer-sh-secret-canary-never-render"; + const commandLog = join(root, "commands.log"); + writeFileSync( + openclaw, + [ + "#!/bin/bash", + 'printf "%s\\n" "$*" >> "$COMMAND_LOG"', + 'if [[ "$*" == "gateway install --force" ]]; then', + ' if [[ "$SERVICE_STREAM" == stdout ]]; then printf "%s\\n" "$SERVICE_ERROR"; else printf "%s\\n" "$SERVICE_ERROR" >&2; fi', + ' printf "%s\\n" "$SECRET_CANARY" >&2; exit 1', + "fi", + ].join("\n"), + ); + chmodSync(openclaw, 0o755); + + try { + const result = runInstallShell( + [ + "set -euo pipefail", + `source ${JSON.stringify(SCRIPT_PATH)}`, + `OPENCLAW_BIN=${JSON.stringify(openclaw)}`, + "is_gateway_daemon_loaded() { return 0; }", + "set -x", + "refresh_gateway_service_if_loaded", + "printf 'INSTALL_COMPLETE\\n'", + ].join("\n"), + { + COMMAND_LOG: commandLog, + SECRET_CANARY: secretCanary, + SERVICE_ERROR: error, + SERVICE_STREAM: stream, + TERM: "dumb", + }, + ); + + const denied = error.startsWith("SERVICE_DEFINITION_"); + expect(readFileSync(commandLog, "utf8").split("\n")).not.toContain("gateway restart"); + expect(result.status).toBe(0); + expect(result.stderr).toContain("+ refresh_gateway_service_if_loaded"); + expect(result.stdout + result.stderr).not.toContain(secretCanary); + if (denied) { + expect(result.stdout).toContain("gateway service definition left unchanged"); + expect(result.stdout).toContain( + error.includes("SEALED") + ? "privileged deployment owner" + : "inspect service-definition access", + ); + expect(result.stdout).toContain("INSTALL_COMPLETE"); + } else { + expect(result.stdout).toContain("Gateway service refresh failed; continuing"); + expect(result.stdout).toContain("INSTALL_COMPLETE"); + } + } finally { + rmSync(root, { force: true, recursive: true }); + } + }); + it("refreshes the shell command cache after loading a persisted PATH update", () => { const result = runInstallShell(` set -euo pipefail diff --git a/test/scripts/upgrade-survivor-plugin-registry.test.ts b/test/scripts/upgrade-survivor-plugin-registry.test.ts index bc834ca769d4..7c478dd01849 100644 --- a/test/scripts/upgrade-survivor-plugin-registry.test.ts +++ b/test/scripts/upgrade-survivor-plugin-registry.test.ts @@ -10,6 +10,12 @@ const tempDirs = useAutoCleanupTempDirTracker(afterEach); const SOURCE_SHA = "a".repeat(40); const VERSION = "2026.8.1"; +function expectFinalFailure(stderr: string, exitCode: number) { + const summary = `[upgrade-survivor] FAILED (exit ${exitCode})`; + expect(stderr.trimEnd().split("\n").at(-1)).toBe(summary); + expect(stderr.split("\n").filter((line) => line === summary)).toHaveLength(1); +} + function registryManifest(): string { return `${JSON.stringify({ candidateVersion: VERSION, @@ -25,7 +31,7 @@ function writeExecutable(path: string, source: string): void { chmodSync(path, 0o755); } -function runSurvivor(overrides: NodeJS.ProcessEnv = {}) { +function runSurvivor(overrides: NodeJS.ProcessEnv = {}, shell = "bash") { const root = tempDirs.make("openclaw-upgrade-survivor-registry-"); const binDir = join(root, "bin"); const captureDir = join(root, "capture"); @@ -46,6 +52,7 @@ printf '%s|%s|%s\n' \ "\${OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS:-}" \ "$OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS" >>"$CAPTURE_DIR/node-env" mkdir -p "$OPENCLAW_DOCKER_ALL_LOG_DIR/prepublish-plugin-registry" +printf '%s' "$OPENCLAW_DOCKER_ALL_LOG_DIR" >"$CAPTURE_DIR/preparation-dir" printf '%s' "$REGISTRY_MANIFEST" \ >"$OPENCLAW_DOCKER_ALL_LOG_DIR/prepublish-plugin-registry/prepublish-plugin-registry.json" printf '{"dir":"%s"}\n' "$OPENCLAW_DOCKER_ALL_LOG_DIR/prepublish-plugin-registry" @@ -56,6 +63,12 @@ printf '{"dir":"%s"}\n' "$OPENCLAW_DOCKER_ALL_LOG_DIR/prepublish-plugin-registry `#!/usr/bin/env bash set -euo pipefail printf '%s\n' "$*" >>"$CAPTURE_DIR/docker-args" +if [ "\${1:-}" = run ]; then + printf '%s\\0' "$@" >"$CAPTURE_DIR/docker-run-args" + if [ -n "\${FIXTURE_PAYLOAD_SHELL:-}" ]; then + exec "$FIXTURE_PAYLOAD_SHELL" -c "\${!#}" + fi +fi previous="" for arg in "$@"; do if [ "$previous" = "--cidfile" ]; then @@ -63,10 +76,11 @@ for arg in "$@"; do fi previous="$arg" done +[ "\${1:-}" != run ] || exit "\${FIXTURE_RUN_EXIT:-0}" `, ); - const result = spawnSync("bash", [SCRIPT], { + const result = spawnSync(shell, [SCRIPT], { encoding: "utf8", env: { ...process.env, @@ -79,6 +93,8 @@ done OPENCLAW_SKIP_CHANNELS: "1", OPENCLAW_SKIP_PROVIDERS: "1", OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_DIR: join(root, "artifacts"), + OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT: join(root, "artifacts"), + OPENCLAW_UPGRADE_SURVIVOR_RUNTIME_ROOT: join(root, "runtime"), OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: "openclaw@2026.7.1-2", OPENCLAW_UPGRADE_SURVIVOR_E2E_SKIP_BUILD: "1", OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "1", @@ -92,35 +108,121 @@ done } describe("standalone upgrade survivor plugin registry", () => { - it("prepares and mounts the direct auto-auth planner registry", () => { - const { captureDir, result } = runSurvivor({ - OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: undefined, - OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "0", - OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE: "auto-auth", - }); + // macOS /bin/bash is 3.2; PATH may select a newer Bash. Exercise both owners. + describe.each(process.platform === "darwin" ? ["/bin/bash", "bash"] : ["bash"])( + "%s wrapper", + (shell) => { + it("reaches the direct child invocation with empty optional arguments", () => { + const { captureDir, result } = runSurvivor( + { + OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "0", + OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE: "auto-auth", + }, + shell, + ); + expect(result.status, result.stderr).toBe(0); + expect(result.stderr).not.toContain("unbound variable"); + expect(result.stderr).not.toContain("FAILED (exit"); + expect(readFileSync(join(captureDir, "node-env"), "utf8")).toBe( + "update-restart-auth||base\n", + ); + const args = readFileSync(join(captureDir, "docker-run-args"), "utf8") + .split("\0") + .slice(0, -1); + expect(args).toContain("run"); + expect(args).toContain("OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE=auto-auth"); + expect(args).not.toContain("--user"); + expect(args).not.toContain(""); + expect(args.at(-2)).toBe("-lc"); + }); - expect(result.status, result.stderr).toBe(0); - expect(readFileSync(join(captureDir, "node-env"), "utf8")).toBe("update-restart-auth||base\n"); - }); + it("rejects a nounset preflight failure even when Bash reports zero to EXIT", () => { + const prelude = join(tempDirs.make("survivor-preflight-fault-"), "bash-env"); + writeFileSync( + prelude, + `trap 'if [[ "$BASH_COMMAND" == docker_e2e_build_or_reuse* ]]; then : "$SURVIVOR_UNSET_PREFLIGHT"; fi' DEBUG\n`, + ); + const { captureDir, result } = runSurvivor( + { + BASH_ENV: prelude, + SURVIVOR_UNSET_PREFLIGHT: undefined, + OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "0", + }, + shell, + ); + expect(result.stderr).toContain("SURVIVOR_UNSET_PREFLIGHT"); + expect(result.status).toBe(1); + expectFinalFailure(result.stderr, 1); + expect(existsSync(join(captureDir, "docker-run-args"))).toBe(false); + expect(result.stdout).not.toContain("Docker E2E passed"); + }); - it("preserves an explicitly supplied direct registry", () => { - const registryDir = tempDirs.make("openclaw-direct-plugin-registry-"); - const manifestPath = join(registryDir, "prepublish-plugin-registry.json"); - writeFileSync(manifestPath, registryManifest()); + it("preserves child failure through cleanup", () => { + const { captureDir, result } = runSurvivor( + { + OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "0", + FIXTURE_RUN_EXIT: "42", + }, + shell, + ); + expect(existsSync(join(captureDir, "docker-run-args"))).toBe(true); + expect(result.status, result.stderr).toBe(42); + expectFinalFailure(result.stderr, 42); + expect(result.stdout).not.toContain("Docker E2E passed"); + expect(existsSync(readFileSync(join(captureDir, "preparation-dir"), "utf8"))).toBe(false); + }); - const { captureDir, result } = runSurvivor({ - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: registryDir, - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256: createHash("sha256") - .update(readFileSync(manifestPath)) - .digest("hex"), - OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: undefined, - OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "0", - OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE: "auto-auth", - }); + it("rejects an early zero exit from the actual scenario before any application work", () => { + const prelude = join(tempDirs.make("survivor-scenario-fault-"), "bash-env"); + writeFileSync( + prelude, + `trap 'if [[ "$BASH_COMMAND" == openclaw_e2e_eval_test_state_from_b64* ]]; then exit 0; fi' DEBUG\n`, + ); + const { captureDir, result } = runSurvivor( + { + BASH_ENV: prelude, + FIXTURE_PAYLOAD_SHELL: shell, + OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "0", + }, + shell, + ); + expect(existsSync(join(captureDir, "docker-run-args"))).toBe(true); + expect(result.status, result.stderr).toBe(1); + expectFinalFailure(result.stderr, 1); + expect(result.stderr).toContain("before all assertions completed"); + expect(result.stdout).not.toContain("Docker E2E passed"); + }); + }, + ); - expect(result.status, result.stderr).toBe(0); - expect(existsSync(join(captureDir, "node-args"))).toBe(false); - }); + it.each(["direct", "published"] as const)( + "preserves an explicitly supplied %s registry", + (mode) => { + const registryDir = tempDirs.make("openclaw-external-plugin-registry-"); + const manifestPath = join(registryDir, "prepublish-plugin-registry.json"); + writeFileSync(manifestPath, registryManifest()); + + const { captureDir, result } = runSurvivor({ + OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: registryDir, + OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256: createHash("sha256") + .update(readFileSync(manifestPath)) + .digest("hex"), + ...(mode === "direct" + ? { + OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC: undefined, + OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE: "0", + OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE: "auto-auth", + } + : { OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: "external-only-scenario" }), + }); + + expect(result.status, result.stderr).toBe(0); + expect(existsSync(join(captureDir, "node-args"))).toBe(false); + expect(readFileSync(join(captureDir, "docker-args"), "utf8")).toContain( + `${registryDir}:/tmp/openclaw-prepublish-plugin-registry:ro`, + ); + }, + ); it("prepares and mounts a planner-owned registry for the current candidate", () => { const { captureDir, result } = runSurvivor({ @@ -139,26 +241,6 @@ describe("standalone upgrade survivor plugin registry", () => { ); }); - it("preserves an explicitly supplied registry without preparing another", () => { - const registryDir = tempDirs.make("openclaw-external-plugin-registry-"); - const manifestPath = join(registryDir, "prepublish-plugin-registry.json"); - writeFileSync(manifestPath, registryManifest()); - - const { captureDir, result } = runSurvivor({ - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_DIR: registryDir, - OPENCLAW_PREPUBLISH_PLUGIN_REGISTRY_MANIFEST_SHA256: createHash("sha256") - .update(readFileSync(manifestPath)) - .digest("hex"), - OPENCLAW_UPGRADE_SURVIVOR_SCENARIO: "external-only-scenario", - }); - - expect(result.status, result.stderr).toBe(0); - expect(existsSync(join(captureDir, "node-args"))).toBe(false); - expect(readFileSync(join(captureDir, "docker-args"), "utf8")).toContain( - `${registryDir}:/tmp/openclaw-prepublish-plugin-registry:ro`, - ); - }); - it("does not prepare a registry for a published candidate", () => { const { captureDir, packageTarball, result } = runSurvivor({ OPENCLAW_CURRENT_PACKAGE_TGZ: undefined, @@ -183,6 +265,7 @@ describe("standalone upgrade survivor live OpenAI probe", () => { }); expect(result.status).toBe(2); + expectFinalFailure(result.stderr, 2); expect(result.stderr).toContain( "OPENCLAW_UPGRADE_SURVIVOR_LIVE_OPENAI=1 requires OPENAI_API_KEY", ); diff --git a/test/scripts/upgrade-survivor-systemd.test.ts b/test/scripts/upgrade-survivor-systemd.test.ts new file mode 100644 index 000000000000..236626ba1f94 --- /dev/null +++ b/test/scripts/upgrade-survivor-systemd.test.ts @@ -0,0 +1,213 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { + readSystemdServiceExecStart, + serializeSystemdEnvironmentFile, +} from "../../src/daemon/systemd-service-files.js"; +import { buildSystemdUnit } from "../../src/daemon/systemd-unit.js"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +const owner = resolve("scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh"); + +function fixture() { + const home = tempDirs.make("survivor-manager-"); + const env = { + HOME: home, + PATH: `${home}/bin:${process.env.PATH}`, + npm_config_prefix: home, + OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG: join(home, "systemctl.log"), + OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE: join(home, "gateway.pid"), + OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG: join(home, "gateway.log"), + }; + const shell = (script: string, args: string[] = []) => + spawnSync( + "bash", + ["-c", 'set -euo pipefail; source "$1"; shift; ' + script, "fixture", owner, ...args], + { + env, + encoding: "utf8", + timeout: 40_000, + }, + ); + const installed = shell("install_update_restart_systemctl_shim"); + expect(installed.status, installed.stderr).toBe(0); + const systemctl = (...args: string[]) => + spawnSync(join(home, "bin/systemctl"), ["--user", ...args], { + env, + encoding: "utf8", + timeout: 40_000, + }); + const unit = join(home, ".config/systemd/user/openclaw-gateway.service"); + mkdirSync(join(home, ".config/systemd/user"), { recursive: true }); + return { home, env, shell, systemctl, unit }; +} + +describe.skipIf(process.platform === "win32")("survivor manager fixture", () => { + it("distinguishes confirmed absence from unsupported inspection and reads the generated service", async () => { + const { home, env, systemctl, unit } = fixture(); + // First install must reach the same effective reader used by the guarded writer. + expect(await readSystemdServiceExecStart(env, { requireEffective: true })).toBeNull(); + expect(systemctl("is-enabled", "openclaw-gateway.service").status).not.toBe(0); + const environmentFile = join(home, "gateway.systemd.env"); + writeFileSync(environmentFile, 'FIXTURE_VALUE="from file"\n'); + const programArguments = [ + process.execPath, + join(home, "package root/openclaw.mjs"), + "gateway", + "--port", + "18817", + ]; + writeFileSync( + unit, + buildSystemdUnit({ + programArguments, + workingDirectory: home, + environment: { + OPENCLAW_STATE_DIR: join(home, "state"), + OPENCLAW_GATEWAY_PORT: "18817", + FIXTURE_VALUE: "inline", + }, + environmentFiles: [environmentFile], + }), + ); + const command = await readSystemdServiceExecStart(env, { requireEffective: true }); + expect(command).toMatchObject({ + programArguments, + workingDirectory: home, + sourcePath: unit, + definitionPaths: [unit], + environment: { + OPENCLAW_STATE_DIR: join(home, "state"), + OPENCLAW_GATEWAY_PORT: "18817", + FIXTURE_VALUE: "from file", + }, + environmentValueSources: { FIXTURE_VALUE: "inline-and-file" }, + }); + const invalid = spawnSync( + join(home, "bin/busctl"), + ["--user", "--json=short", "call", "unsupported"], + { env, encoding: "utf8" }, + ); + expect(invalid.status).not.toBe(0); + expect(invalid.stderr).not.toContain("not found"); + expect(systemctl("show", "openclaw-gateway.service", "--property=Unsupported").status).not.toBe( + 0, + ); + + writeFileSync( + unit, + buildSystemdUnit({ programArguments: [...programArguments.slice(0, -1), "18818"] }), + ); + expect(await readSystemdServiceExecStart(env, { requireEffective: true })).toMatchObject({ + programArguments, + reloadPending: true, + }); + expect(systemctl("daemon-reload").status).toBe(0); + expect( + (await readSystemdServiceExecStart(env, { requireEffective: true }))?.programArguments.at(-1), + ).toBe("18818"); + writeFileSync( + unit, + readFileSync(unit, "utf8").replace("[Service]", "[Service]\nExecStartPre=/bin/true"), + ); + await expect(readSystemdServiceExecStart(env, { requireEffective: true })).rejects.toThrow( + "could not be inspected", + ); + rmSync(unit); + expect(await readSystemdServiceExecStart(env, { requireEffective: true })).toBeNull(); + }); + + it("executes the inspected argv, cwd and file environment, rejects an old survivor, and drains restart children", async () => { + const { home, env, shell, systemctl, unit } = fixture(); + const record = join(home, "starts.jsonl"); + const program = join(home, "gateway fixture.mjs"); + const environmentFile = join(home, "gateway.systemd.env"); + const fileValue = 'file "quoted" \\ $literal `literal`'; + writeFileSync(environmentFile, serializeSystemdEnvironmentFile({ FIXTURE_VALUE: fileValue })); + writeFileSync( + program, + `import fs from "node:fs"; +fs.appendFileSync(${JSON.stringify(record)}, JSON.stringify({pid:process.pid, argv:process.argv.slice(2), cwd:process.cwd(), value:process.env.FIXTURE_VALUE, state:process.env.OPENCLAW_STATE_DIR, update:process.env.OPENCLAW_UPDATE_IN_PROGRESS}) + "\\n"); +process.on("SIGTERM", () => process.exit(0)); +setInterval(() => {}, 1000); +`, + ); + const programArguments = [ + process.execPath, + program, + "gateway", + "--port", + "18819", + "literal $notExpanded", + ]; + writeFileSync( + unit, + buildSystemdUnit({ + programArguments, + workingDirectory: home, + environment: { OPENCLAW_STATE_DIR: join(home, "state"), FIXTURE_VALUE: "inline" }, + environmentFiles: [environmentFile], + }), + ); + const records = (): Array<{ pid: number; argv: string[]; cwd: string; value: string }> => + existsSync(record) + ? readFileSync(record, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)) + : []; + const waitForStarts = async (count: number) => { + for (let attempt = 0; attempt < 200 && records().length < count; attempt++) { + await delay(10); + } + expect(records()).toHaveLength(count); + }; + try { + expect(systemctl("enable", "openclaw-gateway.service").status).toBe(0); + expect(systemctl("is-enabled", "openclaw-gateway.service").status).toBe(0); + expect( + shell("OPENCLAW_UPDATE_IN_PROGRESS=1 systemctl --user restart openclaw-gateway.service") + .status, + ).toBe(0); + await waitForStarts(1); + const inspected = await readSystemdServiceExecStart(env, { requireEffective: true }); + expect(records()[0]).toEqual({ + pid: expect.any(Number), + argv: inspected?.programArguments.slice(2), + cwd: inspected?.workingDirectory, + value: inspected?.environment?.FIXTURE_VALUE, + state: join(home, "state"), + }); + const previousPid = readFileSync( + env.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE, + "utf8", + ).trim(); + const previousLines = readFileSync(env.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG, "utf8") + .trim() + .split("\n").length; + const assertion = () => + shell('assert_update_restart_service_replaced "$1" "$2"', [ + previousPid, + String(previousLines), + ]); + expect(assertion().status).not.toBe(0); + expect(systemctl("restart", "openclaw-gateway.service").status).toBe(0); + await waitForStarts(2); + const proof = assertion(); + expect(proof.status, proof.stderr).toBe(0); + expect(records()[1]?.pid).not.toBe(records()[0]?.pid); + expect(() => process.kill(records()[0]!.pid, 0)).toThrow(); + } finally { + const stopped = systemctl("stop", "openclaw-gateway.service"); + expect(stopped.status, stopped.stderr).toBe(0); + for (const { pid } of records()) { + expect(() => process.kill(pid, 0)).toThrow(); + } + expect(existsSync(env.OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE)).toBe(false); + } + }); +}); diff --git a/test/tsconfig/tsconfig.core.test.plugins-platform.json b/test/tsconfig/tsconfig.core.test.plugins-platform.json index d50579d5729e..3a42ad4fae8b 100644 --- a/test/tsconfig/tsconfig.core.test.plugins-platform.json +++ b/test/tsconfig/tsconfig.core.test.plugins-platform.json @@ -15,8 +15,6 @@ "../../src/system-agent/**/*.test.ts", "../../src/system-agent/**/*.test.tsx", "../../src/hooks/**/*.test.ts", - "../../src/hooks/**/*.test.tsx", - "../../src/daemon/**/*.test.ts", - "../../src/daemon/**/*.test.tsx" + "../../src/hooks/**/*.test.tsx" ] } diff --git a/test/tsconfig/tsconfig.core.test.services.json b/test/tsconfig/tsconfig.core.test.services.json index e89d27e7b9a3..5a7e0dd44392 100644 --- a/test/tsconfig/tsconfig.core.test.services.json +++ b/test/tsconfig/tsconfig.core.test.services.json @@ -6,6 +6,8 @@ "include": [ "../../src/cron/**/*.test.ts", "../../src/cron/**/*.test.tsx", + "../../src/daemon/**/*.test.ts", + "../../src/daemon/**/*.test.tsx", "../../src/plugin-sdk/**/*.test.ts", "../../src/plugin-sdk/**/*.test.tsx", "../../src/skills/**/*.test.ts",