From bae12326a6e67709f5ff7c421392e9ceb21fa7ca Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 24 Sep 2026 08:38:14 -0700 Subject: [PATCH] fix(update): rebuild stale Git runtimes when source is already current (#156996) * fix(update): rebuild stale git runtimes through managed handoff * fix(update): decide Git no-ops from build provenance * fix(update): rebuild Git runtimes without provenance --- docs/cli/update/how-updates-run.md | 23 ++++- scripts/run-node.mts | 83 ++++++++++++++----- .../update-cli-state-snapshot.test-support.ts | 2 + src/cli/update-cli.test.ts | 9 +- .../update-command-execution.test.ts | 70 ++++++++++++++++ .../update-cli/update-command-execution.ts | 15 ++++ .../update-command-git-snapshot.test.ts | 25 ++++-- ...update-command-service-publication.test.ts | 62 ++++++++++++++ .../update-command-service-publication.ts | 18 ++-- src/commands/doctor-stale-runtime-build.ts | 2 +- src/infra/update-runner-git-candidate.test.ts | 51 ++++++++++++ src/infra/update-runner-git-preflight.ts | 12 ++- src/infra/update-runner-git.tags.test.ts | 14 +++- src/infra/update-runner-git.ts | 10 ++- test/scripts/build-all.test.ts | 15 +++- test/scripts/dist-artifact-ownership.test.ts | 2 + .../fixtures/native-runner-signals.mjs | 2 + .../fixtures/source-runner-service.mjs | 31 +++++++ test/scripts/run-node-lifecycle.test.ts | 9 +- test/scripts/run-node.test-support.ts | 20 ++++- 20 files changed, 421 insertions(+), 54 deletions(-) create mode 100644 test/scripts/fixtures/source-runner-service.mjs diff --git a/docs/cli/update/how-updates-run.md b/docs/cli/update/how-updates-run.md index a9ab8b936b18..111325b0f046 100644 --- a/docs/cli/update/how-updates-run.md +++ b/docs/cli/update/how-updates-run.md @@ -33,10 +33,31 @@ replacement. Choose an empty `OPENCLAW_GIT_DIR` and retry. If the resolved registry package version equals the installed version without changing the selected channel or installation method, or the Git target SHA equals -`HEAD`, plugin convergence still runs; if plugins and runtime artifacts remain unchanged, the run finishes `skipped` with reason `already-current`. Runtime maintenance can therefore succeed without changing the Git revision. A same-version +`HEAD` and the recorded build commit matches it, plugin convergence still runs; if plugins and runtime artifacts remain unchanged, the run finishes `skipped` with reason `already-current`. Runtime maintenance can therefore succeed without changing the Git revision. A same-version explicit `--channel` or installation-method change finishes successfully. Changed plugins restart a running managed Gateway unless `--no-restart` is set; retained exact pins produce the same advisories as a core update without requiring a restart. +If a Git checkout advanced without rebuilding or its runtime has no recorded +build commit, the matching source revision still needs an update. OpenClaw builds +and validates a separate candidate, then stops the managed Gateway before +replacing the runtime and restarting it. The new build records its commit, so +the next update can finish as already current. +`--no-restart` cannot replace runtime files used by a running Gateway in the same +installation; the update leaves those files intact and reports the process and +the stop/retry action. + +Source commands launched with `pnpm openclaw` also refuse an automatic rebuild +while that installation's Gateway is running. Use the installed `openclaw update` +or `node openclaw.mjs update` from the checkout to reach the updater's managed +handoff without the source wrapper rebuilding first. Manual `pnpm build` remains +an operator action: stop the Gateway before rebuilding its installation. + +This decision runs in the installed updater. An older updater that returns +`already-current` with `runtime-verification-failed` cannot obtain the fix from a +candidate it never builds. Stop the Gateway through its actual service manager, +rebuild the checkout, and start the Gateway through that manager before retrying. +Do not rebuild its installation while the old Gateway is still serving. + Linux updates also refresh outdated OpenClaw-managed systemd policy when the core is already current or `--no-restart` is set. This policy-only refresh confirms `daemon-reload` without stopping the Gateway and preserves operator drop-ins. diff --git a/scripts/run-node.mts b/scripts/run-node.mts index 9eefc75e5f7b..516901973bd6 100644 --- a/scripts/run-node.mts +++ b/scripts/run-node.mts @@ -12,7 +12,10 @@ import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; import { getCommandArgsWithRootOptions } from "../src/infra/cli-root-options.ts"; -import { withDistArtifactOwnership } from "./lib/dist-artifact-ownership.mts"; +import { + distArtifactEntryArgs, + withDistArtifactOwnership, +} from "./lib/dist-artifact-ownership.mts"; import { BUILD_STAMP_FILE, RUNTIME_POSTBUILD_STAMP_FILE, @@ -42,6 +45,7 @@ import { runNodeWatchedPaths, } from "./run-node-watch-paths.mts"; import { listCoreRuntimePostBuildOutputs, runRuntimePostBuild } from "./runtime-postbuild.mts"; +import { listTsdownOutputRoots } from "./tsdown-build.mts"; type RunNodeInjectedChild = { kill?: (signal?: NodeJS.Signals) => boolean | void; @@ -136,7 +140,6 @@ function asRunNodeChild(value: unknown): RunNodeChild { export { runNodeWatchedPaths }; -const runtimeBuildArgs = ["--import", "tsx", "scripts/build-all.mts", "qaRuntime"]; const RUN_NODE_DEFAULT_SHUTDOWN_GRACE_MS = 5_000; const RUN_NODE_MAX_SHUTDOWN_GRACE_MS = 5 * 60_000; const RUN_NODE_SHUTDOWN_GRACE_MESSAGE_TYPE = "openclaw:shutdown-grace"; @@ -1348,6 +1351,32 @@ const withRunNodeBuildLock = async (deps: RunNodeDeps, callback: () => Promi } }; +const withRunNodeRuntimePublication = async (deps: RunNodeDeps, publish: () => Promise) => { + const [{ withGatewayRuntimeArtifactPublication }, { parseCliProfileArgs, applyCliProfileEnv }] = + await Promise.all([ + import("../src/cli/update-cli/update-command-service-publication.ts"), + import("../src/cli/profile.ts"), + ]); + const selected = parseCliProfileArgs([deps.execPath, "openclaw.mjs", ...deps.args]); + if (!selected.ok) { + throw new Error(selected.error); + } + const env = { ...deps.env }; + if (selected.profile) { + applyCliProfileEnv({ profile: selected.profile, env }); + } + return await withGatewayRuntimeArtifactPublication( + { + root: deps.cwd, + env, + timeoutMs: 60_000, + outputPaths: listTsdownOutputRoots(), + assertCurrent() {}, + }, + publish, + ); +}; + const syncRuntimeArtifacts = async (deps: RunNodeDeps) => { try { await deps.runRuntimePostBuild({ cwd: deps.cwd, env: deps.env }); @@ -1376,11 +1405,13 @@ const syncRuntimeArtifactsAndStamp = async (deps: RunNodeDeps) => if (!resolveRuntimePostBuildRequirement(deps).shouldSync) { return true; } - const synced = await syncRuntimeArtifacts(deps); - if (synced) { - writeRuntimePostBuildStamp(deps); - } - return synced; + return await withRunNodeRuntimePublication(deps, async () => { + const synced = await syncRuntimeArtifacts(deps); + if (synced) { + writeRuntimePostBuildStamp(deps); + } + return synced; + }); }); const shouldSkipWatchRuntimeSync = (deps: RunNodeDeps, requirement: RuntimePostBuildRequirement) => @@ -1581,22 +1612,30 @@ export async function runNodeMain(params: RunNodeMainParams = {}): Promise { - const build = asRunNodeChild( - deps.spawn(deps.execPath, runtimeBuildArgs, { - cwd: deps.cwd, - detached: shouldUseRunNodeChildProcessGroup(deps), - env: { - ...deps.env, - [RUN_NODE_SKIP_DTS_BUILD_ENV]: deps.env[RUN_NODE_SKIP_DTS_BUILD_ENV] ?? "1", - }, - stdio: ["inherit", "pipe", "pipe"], + return await withDistArtifactOwnership(deps.cwd, () => + withRunNodeRuntimePublication(deps, () => + withRunNodeProgress(deps, "Building local CLI artifacts", async () => { + const build = asRunNodeChild( + deps.spawn( + deps.execPath, + distArtifactEntryArgs(path.join(deps.cwd, "scripts/build-all.mts"), ["qaRuntime"]), + { + cwd: deps.cwd, + detached: shouldUseRunNodeChildProcessGroup(deps), + env: { + ...deps.env, + [RUN_NODE_SKIP_DTS_BUILD_ENV]: deps.env[RUN_NODE_SKIP_DTS_BUILD_ENV] ?? "1", + }, + stdio: ["inherit", "pipe", "pipe"], + }, + ), + ); + pipeSpawnedOutput(build, deps, { stdoutTarget: "stderr" }); + const result = await waitForSpawnedProcess(build, deps); + return getInterruptedSpawnOutcome(result, deps.platform) ?? result.exitCode ?? 1; }), - ); - pipeSpawnedOutput(build, deps, { stdoutTarget: "stderr" }); - const result = await waitForSpawnedProcess(build, deps); - return getInterruptedSpawnOutcome(result, deps.platform) ?? result.exitCode ?? 1; - }); + ), + ); }); if (buildExitCode !== 0) { return await closeRunNodeOutputTee(deps, buildExitCode); diff --git a/src/cli/update-cli-state-snapshot.test-support.ts b/src/cli/update-cli-state-snapshot.test-support.ts index 1c435aa23b88..18c20426a390 100644 --- a/src/cli/update-cli-state-snapshot.test-support.ts +++ b/src/cli/update-cli-state-snapshot.test-support.ts @@ -23,6 +23,8 @@ export function createUpdateStateProfileInitializer( if (!fs.existsSync(databasePath) && fs.existsSync(preparedStateDatabase)) { fs.mkdirSync(path.dirname(databasePath), { recursive: true, mode: 0o700 }); fs.copyFileSync(preparedStateDatabase, databasePath); + fixtureStateDatabases.add(databasePath); + return; } const database = openOpenClawStateDatabase({ env }); fixtureStateDatabases.add(database.path); diff --git a/src/cli/update-cli.test.ts b/src/cli/update-cli.test.ts index d21804d8621a..d338c7022994 100644 --- a/src/cli/update-cli.test.ts +++ b/src/cli/update-cli.test.ts @@ -2140,7 +2140,6 @@ describe("update-cli", () => { ); it.each([ - { kind: "git", restart: false, capability: "sealed" }, { kind: "git", restart: true, capability: "sealed" }, { kind: "package", restart: false, capability: "sealed" }, { kind: "package", restart: true, capability: "sealed" }, @@ -12478,21 +12477,21 @@ describe("update-cli", () => { }, }, { - name: "skips service env refresh when --no-restart is set", + name: "refuses a running Git installation with --no-restart", run: async () => { mockGitUpdateAfterMutation(); serviceLoaded.mockResolvedValue(true); mockOwnedGitService(); - await updateCommand({ restart: false }); + await expect(updateCommand({ restart: false })).rejects.toEqual(new ExitError(1)); }, assert: () => { - expectNoSideEffects(runDaemonInstall, runDaemonRestart); + expectNoSideEffects(runDaemonInstall, runDaemonRestart, serviceStop); expect(freshRestartCalls()).toHaveLength(0); expect( gatewayCommandCall(path.join(process.cwd(), "dist", "index.js"), "install"), ).toBeUndefined(); - expect(getLogOutput()).toContain("Gateway: restart skipped (--no-restart)."); + expect(getErrorOutput()).toContain("still running"); }, }, { diff --git a/src/cli/update-cli/update-command-execution.test.ts b/src/cli/update-cli/update-command-execution.test.ts index efb75ffcc5dd..740719e99cf2 100644 --- a/src/cli/update-cli/update-command-execution.test.ts +++ b/src/cli/update-cli/update-command-execution.test.ts @@ -25,6 +25,76 @@ describe("mutable update execution", () => { registerExecutionTimeoutTests(); registerNativeAdmissionTests({ executionParams, mocks, successfulUpdate }); + it.each(["same", "alias", "disjoint"] as const)( + "preserves a serving Git runtime when activation did not stop it: %s", + async (destination) => { + await withTestDir({ prefix: "git-live-runtime-custody-" }, async (dir) => { + const servingRoot = path.join(dir, "serving"); + const targetRoot = destination === "same" ? servingRoot : path.join(dir, "target"); + await fs.mkdir(path.join(servingRoot, "dist"), { recursive: true }); + if (destination === "alias") { + await fs.symlink( + servingRoot, + targetRoot, + process.platform === "win32" ? "junction" : "dir", + ); + } else if (destination === "disjoint") { + await fs.mkdir(path.join(targetRoot, "dist"), { recursive: true }); + } + const artifact = path.join(servingRoot, "dist", "prepare.runtime.js"); + await fs.writeFile(artifact, "retained serving runtime"); + const target = { schemaVersions: { state: 15, agent: 19 } }; + mocks.maybeStopService.mockImplementation(async () => ({ + ...inspectOrStopService("inspect"), + servicePid: 23456, + serviceUpdateVerdict: { + kind: "owned", + root: servingRoot, + fingerprint: "serving-generation", + refreshDefinition: false, + }, + })); + vi.spyOn(readiness, "verifyPreviousGatewayForUpdate").mockResolvedValue(true); + mocks.runGitUpdate.mockImplementation( + async ( + options: Parameters[0], + ) => { + await options.inspectGitTarget?.(target); + await options.beforeGitMutation?.(target); + await fs.writeFile( + path.join(targetRoot, "dist", "prepare.runtime.js"), + "candidate runtime", + ); + return { ...successfulUpdate, mode: "git" }; + }, + ); + const execution = await executeMutableUpdate({ + ...executionParams("git"), + root: targetRoot, + shouldRestart: false, + opts: { json: true, restart: false }, + }); + expect(await fs.readFile(artifact, "utf8")).toBe("retained serving runtime"); + expect(mocks.serviceStopped).toBe(false); + if (destination === "disjoint") { + expect(execution?.result.status).toBe("ok"); + expect( + await fs.readFile(path.join(targetRoot, "dist", "prepare.runtime.js"), "utf8"), + ).toBe("candidate runtime"); + } else { + expect(execution).toMatchObject({ + mutationStarted: false, + result: { status: "error", reason: "runtime-artifact-publication" }, + }); + expect(execution?.result.steps).toEqual( + expect.arrayContaining([ + expect.objectContaining({ stderrTail: expect.stringContaining("23456") }), + ]), + ); + } + }); + }, + ); it("retains the live update run when stopped-service context capture fails", async () => { await withTestDir({ prefix: "partial-stop-recovery-owner-" }, async (dir) => { const control = path.join(dir, "leases"); diff --git a/src/cli/update-cli/update-command-execution.ts b/src/cli/update-cli/update-command-execution.ts index 41beeb1a485f..0a42536d4fb4 100644 --- a/src/cli/update-cli/update-command-execution.ts +++ b/src/cli/update-cli/update-command-execution.ts @@ -15,6 +15,7 @@ import { canResolveRegistryVersionForPackageTarget, verifyPackageUpdateRecovery, } from "../../infra/update-global.js"; +import { updateInstallRootsMatch } from "../../infra/update-install-root.js"; import { recordUpdateRunPhase } from "../../infra/update-run-ledger.js"; import { isFailedUpdateStep } from "../../infra/update-run-step.js"; import { readCurrentGitUpdateRecovery } from "../../infra/update-runner-git-recovery.js"; @@ -539,6 +540,20 @@ export async function executeMutableUpdate( await stopManagedServiceBeforeMutableUpdate(roots); await recheckSchemas(admittedTargetSchemaVersions); assertExecutionCurrent(); + const serving = preManagedServiceStop; + const servingVerdict = serving?.serviceUpdateVerdict; + if ( + params.updateInstallKind === "git" && + serving?.running && + !serving.stopped && + servingVerdict?.kind === "owned" && + roots.some((root) => updateInstallRootsMatch(root, servingVerdict.root)) + ) { + throw new UpdatePreMutationError( + "runtime-artifact-publication", + `Cannot replace Git runtime artifacts in ${servingVerdict.root}: its Gateway${serving.servicePid === undefined ? "" : ` (PID ${serving.servicePid})`} is still running and this update did not stop it. Stop that Gateway through its service manager, then rerun \`${formatCliCommand("openclaw update", serving.serviceEnv)}\` without \`--no-restart\`. The serving runtime was left unchanged.`, + ); + } // Both install paths enter mutation only after the post-stop schema/authority fence. preManagedServiceStop?.windowsTaskAutoStartRecovery?.beginMutation(); mutationStarted = true; diff --git a/src/cli/update-cli/update-command-git-snapshot.test.ts b/src/cli/update-cli/update-command-git-snapshot.test.ts index ea5c79bc5bb7..3ad39fcc2df4 100644 --- a/src/cli/update-cli/update-command-git-snapshot.test.ts +++ b/src/cli/update-cli/update-command-git-snapshot.test.ts @@ -19,9 +19,13 @@ async function git(root: string, ...args: string[]): Promise { return result.stdout.trim(); } -it.each([true, false])( - "checks snapshot space after the Git no-op decision (current=%s)", - async (current) => { +it.each([ + { current: true, recorded: true, noOp: true }, + { current: false, recorded: true, noOp: false }, + { current: true, recorded: false, noOp: false }, +])( + "checks snapshot space after the Git no-op decision (current=$current, recorded=$recorded)", + async ({ current, recorded, noOp }) => { await withTestDir({ prefix: "git-update-snapshot-" }, async (base) => { const root = path.join(base, "checkout"); const stateDir = path.join(base, "state"); @@ -46,6 +50,13 @@ it.each([true, false])( await git(root, "add", "."); await git(root, "commit", "-m", "fixture"); const before = await git(root, "rev-parse", "HEAD"); + if (recorded) { + await fs.mkdir(path.join(root, "dist")); + await fs.writeFile( + path.join(root, "dist", "build-info.json"), + JSON.stringify({ commit: before }), + ); + } let target = before; if (!current) { await fs.writeFile(path.join(root, "candidate.txt"), "candidate\n"); @@ -105,13 +116,13 @@ it.each([true, false])( : headCommandOptions?.timeoutMs, ).toBe(20 * 60_000); expect(result).toMatchObject( - current + noOp ? { status: "skipped", reason: "already-current" } : { status: "error", reason: "snapshot-capacity-insufficient" }, ); - expect(capacity.mock.calls.length === 0).toBe(current); - expect(getSnapshotSource).toHaveBeenCalledTimes(current ? 0 : 1); - if (!current) { + expect(capacity.mock.calls.length === 0).toBe(noOp); + expect(getSnapshotSource).toHaveBeenCalledTimes(noOp ? 0 : 1); + if (!noOp) { expect(result.steps).toContainEqual( expect.objectContaining({ name: "snapshot-space-preflight", diff --git a/src/cli/update-cli/update-command-service-publication.test.ts b/src/cli/update-cli/update-command-service-publication.test.ts index 93af5a162781..ca1713dd0a5c 100644 --- a/src/cli/update-cli/update-command-service-publication.test.ts +++ b/src/cli/update-cli/update-command-service-publication.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { runNodeMain } from "../../../scripts/run-node.mts"; import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; import { withGatewayServiceOperationLock } from "../../daemon/service-operation-lock.js"; import type { GatewayService } from "../../daemon/service.js"; @@ -97,6 +98,67 @@ async function withRuntimePublicationFixture( }); } +it.each([undefined, "stale-profile"])( + "preserves the serving installation when a source command requests an automatic rebuild (profile=%s)", + (profile) => + withRuntimePublicationFixture(async ({ root, env, service }) => { + vi.mocked(service.readRuntime).mockResolvedValue({ + status: "running", + pid: 23456, + systemd: { managerUid: 2001 }, + }); + const entry = path.join(root, "dist", "entry.js"); + const before = await fs.readFile(entry, "utf8"); + const spawn = vi.fn(() => { + throw new Error("Automatic build started under the serving Gateway"); + }); + await expect( + runNodeMain({ + cwd: root, + args: [...(profile ? ["--profile", profile] : []), "doctor"], + env: { ...env, OPENCLAW_RUNNER_LOG: "0" }, + spawn, + }), + ).rejects.toThrow(/affected Gateway.*running/); + expect(spawn).not.toHaveBeenCalled(); + expect( + vi + .mocked(service.readRuntime) + .mock.calls.some(([observedEnv]) => observedEnv?.OPENCLAW_PROFILE === profile), + ).toBe(true); + expect(await fs.readFile(entry, "utf8")).toBe(before); + }), +); + +it("holds Gateway startup custody until the automatic source build exits", () => + withRuntimePublicationFixture(async ({ root, env, coordinatorPath }) => { + let builds = 0; + const spawn = (_command: string, args: string[]) => { + if (args.some((arg) => arg.endsWith("scripts/build-all.mts"))) { + builds += 1; + const competingStartup = tryAcquireExclusiveSqliteCoordinator(coordinatorPath); + competingStartup?.release(); + expect(competingStartup).toBeNull(); + } + return { + on(event: string, listener: (code: number, signal: null) => void) { + if (event === "exit") { + queueMicrotask(() => listener(0, null)); + } + }, + }; + }; + expect( + await runNodeMain({ + cwd: root, + args: ["doctor"], + env: { ...env, OPENCLAW_RUNNER_LOG: "0" }, + spawn, + }), + ).toBe(0); + expect(builds).toBe(1); + })); + it.each([true, false])( "parks the foreground Gateway before source runtime publication only when artifacts change: %s", (changed) => diff --git a/src/cli/update-cli/update-command-service-publication.ts b/src/cli/update-cli/update-command-service-publication.ts index 5b5b4be0c461..50a1861e8f25 100644 --- a/src/cli/update-cli/update-command-service-publication.ts +++ b/src/cli/update-cli/update-command-service-publication.ts @@ -50,6 +50,7 @@ export async function withGatewayRuntimeArtifactPublication( env: NodeJS.ProcessEnv; timeoutMs: number; assertCurrent: () => void; + outputPaths?: readonly string[]; }, publish: (assertPublicationCurrent: () => Promise) => Promise, ): Promise { @@ -104,21 +105,24 @@ export async function withGatewayRuntimeArtifactPublication( const same = (a: PathIdentity, b: PathIdentity) => a.real === b.real || Boolean(a.stat && b.stat && a.stat.dev === b.stat.dev && a.stat.ino === b.stat.ino); - const outputPaths = [ + const outputPaths = params.outputPaths ?? [ "dist-runtime", path.join("dist", "extensions", "node_modules", "openclaw"), ]; + const parentPaths = new Set([""]); + for (const output of outputPaths) { + for (let parent = path.dirname(output); parent !== "."; parent = path.dirname(parent)) { + if (!outputPaths.some((replaced) => isPathInside(replaced, parent))) { + parentPaths.add(parent); + } + } + } const readInspection = async () => { assertCurrent(); // Parents are stable across publication; output roots themselves are renamed. // Record missing descendants too, so creating them cannot redirect a later effect. const parents = await Promise.all( - [ - "", - "dist", - path.join("dist", "extensions"), - path.join("dist", "extensions", "node_modules"), - ].map((relative) => + [...parentPaths].map((relative) => relative ? outputIdentity(path.join(params.root, relative)) : identity(params.root), ), ); diff --git a/src/commands/doctor-stale-runtime-build.ts b/src/commands/doctor-stale-runtime-build.ts index fa46f7ae7087..fb378517cf81 100644 --- a/src/commands/doctor-stale-runtime-build.ts +++ b/src/commands/doctor-stale-runtime-build.ts @@ -44,7 +44,7 @@ export async function collectStaleRuntimeBuildFindings( message: `Running build came from commit ${builtCommit.slice(0, 7)}, but the checkout is at ${checkoutCommit.slice(0, 7)}; the loaded runtime is older than its source.`, path: root, fixHint: - "Rebuild with `pnpm build` so the running runtime matches the checkout, then restart the Gateway.", + "Run `openclaw update` to rebuild and restart through the update lifecycle. For a manual build, stop the Gateway before running `pnpm build`, then start it again.", }, ]; } diff --git a/src/infra/update-runner-git-candidate.test.ts b/src/infra/update-runner-git-candidate.test.ts index f2165a2702ca..821ecffe1d6c 100644 --- a/src/infra/update-runner-git-candidate.test.ts +++ b/src/infra/update-runner-git-candidate.test.ts @@ -256,6 +256,57 @@ describe("Git candidate activation", () => { }, ); + it.each([ + { channel: "dev", recorded: true }, + { channel: "stable", recorded: true }, + { channel: "dev", recorded: false }, + { channel: "stable", recorded: false }, + ] as const)( + "rebuilds a source-current $channel checkout before activating its stale runtime (recorded=$recorded)", + async ({ channel, recorded }) => { + const builtSha = beforeSha; + const target = await advanceRemote(); + await git(remote, "tag", "v2026.9.1"); + await git(root, "pull", "--ff-only"); + beforeSha = target; + const buildInfoPath = path.join(root, "dist", "build-info.json"); + if (!recorded) { + await fs.writeFile(buildInfoPath, JSON.stringify({ buildId: builtSha })); + } + await expectRuntime(root, builtSha); + + const result = await update({ + channel, + beforeGitMutation: async () => { + expect(stopped).toBe(false); + await expectRuntime(root, builtSha); + stopped = true; + events.push("stop"); + }, + }); + + expect(result.status, JSON.stringify(result)).toBe("ok"); + expect(result.before).toMatchObject({ sha: target, buildId: builtSha }); + expect(result.after).toMatchObject({ sha: target, buildId: target }); + expect(events).toEqual(["build", "validate", "stop", "migrate"]); + await expectRuntime(root, target); + expect(JSON.parse(await fs.readFile(buildInfoPath, "utf8"))).toMatchObject({ + commit: target, + }); + + stopped = false; + events.length = 0; + expect(await update({ channel })).toMatchObject({ + status: "skipped", + reason: "already-current", + before: { sha: target, buildId: target }, + }); + expect(stopped).toBe(false); + expect(events).toEqual([]); + await expectNoRuntimeStagingPaths(); + }, + ); + it("keeps build and exposure source selection in the admitted candidate", async () => { vi.stubEnv("OPENCLAW_DEV_SOURCE_ROOT", root); await advanceRemote(); diff --git a/src/infra/update-runner-git-preflight.ts b/src/infra/update-runner-git-preflight.ts index 643e46ef826f..dfa0f03d3812 100644 --- a/src/infra/update-runner-git-preflight.ts +++ b/src/infra/update-runner-git-preflight.ts @@ -4,6 +4,7 @@ import { normalizeStringEntries } from "@openclaw/normalization-core/string-norm import { stripAnsi } from "../../packages/terminal-core/src/ansi.js"; import { resolveControlUiAssetHealth } from "./control-ui-assets.js"; import { hasErrnoCode } from "./errno.js"; +import { gitCommitPrefixesMatch } from "./git-commit.js"; import { DEV_BRANCH, resolveDevUpstreamRefs } from "./update-channels.js"; import { resolveDevUpdateTargetRevision, type DevUpdateTarget } from "./update-dev-target.js"; import { @@ -527,6 +528,7 @@ export async function runGitCandidatePreflight(params: { refreshedRemotes: readonly string[]; targetRevision?: string; beforeSha?: string | null; + beforeBuiltCommit: string | null; beforeGitStaging?: UpdateRunnerOptions["beforeGitStaging"]; validateCandidate: UpdateRunnerOptions["validateCandidate"]; prepareGitExposure?: UpdateRunnerOptions["prepareGitExposure"]; @@ -599,8 +601,12 @@ export async function runGitCandidatePreflight(params: { localDevBranchExists = upstream.localDevBranchExists; } - // A resolved no-op must not enter validation, stop the service, or rewrite its runtime. - if (!params.prepareGitExposure && preflightBaseSha === params.beforeSha) { + // A matching source revision cannot prove an unrecorded runtime is current. + const canSkipActivation = + !params.prepareGitExposure && + params.beforeBuiltCommit !== null && + gitCommitPrefixesMatch(params.beforeBuiltCommit, params.beforeSha ?? ""); + if (canSkipActivation && preflightBaseSha === params.beforeSha) { return { status: "skipped", reason: "already-current" }; } if (params.beforeGitStaging) { @@ -651,7 +657,7 @@ export async function runGitCandidatePreflight(params: { }; } for (const sha of candidates) { - if (!params.prepareGitExposure && sha === params.beforeSha) { + if (canSkipActivation && sha === params.beforeSha) { return { status: "skipped", reason: "already-current" }; } if (sha !== preflightBaseSha) { diff --git a/src/infra/update-runner-git.tags.test.ts b/src/infra/update-runner-git.tags.test.ts index 3d8ea77c734b..301aaa612270 100644 --- a/src/infra/update-runner-git.tags.test.ts +++ b/src/infra/update-runner-git.tags.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { writeFileSync } from "node:fs"; +import { mkdirSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -56,6 +56,7 @@ describe("Git updater release tag refresh", () => { path.join(seed, "package.json"), JSON.stringify({ name: "openclaw", version: "2026.9.1" }), ); + writeFileSync(path.join(seed, ".gitignore"), "dist/\n"); git(seed, "add", "."); git(seed, "commit", "-m", "original release"); const oldTag = git(seed, "rev-parse", "HEAD"); @@ -65,6 +66,17 @@ describe("Git updater release tag refresh", () => { git(seed, "push", "origin", "main", "--tags"); git(directory, "clone", "--origin", releaseRemote, upstream, root); git(root, "checkout", "--detach"); + const dist = path.join(root, "dist"); + mkdirSync(path.join(dist, "control-ui"), { recursive: true }); + writeFileSync(path.join(dist, "entry.js"), "export {};\n"); + writeFileSync( + path.join(dist, "build-info.json"), + JSON.stringify({ commit: release, buildId: release }), + ); + for (const stamp of [".buildstamp", ".runtime-postbuildstamp"]) { + writeFileSync(path.join(dist, stamp), JSON.stringify({ head: release })); + } + writeFileSync(path.join(dist, "control-ui", "index.html"), "ready"); git(root, "tag", "local-only", oldTag); if (forkRemote) { const fork = path.join(directory, "fork.git"); diff --git a/src/infra/update-runner-git.ts b/src/infra/update-runner-git.ts index 1a348a56c115..8ccd0de24992 100644 --- a/src/infra/update-runner-git.ts +++ b/src/infra/update-runner-git.ts @@ -4,7 +4,11 @@ import { readPackageVersion } from "./package-json.js"; import { DEV_BRANCH, type UpdateChannel } from "./update-channels.js"; import { getUpdateDoctorConfigFailureReason } from "./update-doctor-config.js"; import { createUpdateErrorFact } from "./update-failure-facts.js"; -import { readBuiltGatewayBuildId, verifyGitUpdateRecovery } from "./update-git-runtime.js"; +import { + readBuiltGatewayBuildId, + readBuiltRuntimeCommit, + verifyGitUpdateRecovery, +} from "./update-git-runtime.js"; import { UpdateRequesterRevokedError } from "./update-requester-authority.js"; import { isFailedUpdateStep } from "./update-run-step.js"; import { runStep } from "./update-runner-command.js"; @@ -65,9 +69,10 @@ export async function updateGitCheckout(params: { timeoutMs, }); const beforeSha = beforeShaResult.stdout.trim() || null; - const [beforeVersion, beforeBuildId] = await Promise.all([ + const [beforeVersion, beforeBuildId, beforeBuiltCommit] = await Promise.all([ readPackageVersion(gitRoot), readBuiltGatewayBuildId(gitRoot), + readBuiltRuntimeCommit(gitRoot), ]); const before = { sha: beforeSha, @@ -406,6 +411,7 @@ export async function updateGitCheckout(params: { devTarget, refreshedRemotes: fetched.refreshedRemotes, beforeSha, + beforeBuiltCommit, beforeGitStaging: opts.beforeGitStaging, needsCheckoutMain, timeoutMs, diff --git a/test/scripts/build-all.test.ts b/test/scripts/build-all.test.ts index 955e207e86ac..3dd3fbae95f7 100644 --- a/test/scripts/build-all.test.ts +++ b/test/scripts/build-all.test.ts @@ -38,6 +38,13 @@ import { import { resolveTestNodeExecPath } from "../../src/test-utils/node-process.js"; import { toolingProbeRuntimeEntrypoints } from "./tooling-probe-runtime.test-support.mts"; +vi.mock("../../src/cli/update-cli/update-command-service-publication.js", () => ({ + withGatewayRuntimeArtifactPublication: async ( + _params: unknown, + publish: () => Promise, + ) => publish(), +})); + const testNodeExecPath = resolveTestNodeExecPath(); const buildAllUrl = resolveRuntimeWorkerUrl(toolingProbeRuntimeEntrypoints.buildAll); const buildArtifactCacheUrl = resolveRuntimeWorkerUrl( @@ -884,7 +891,13 @@ describe("resolveBuildAllSteps", () => { }), ).toBe(0); expect(spawn.mock.calls.map(([, args]) => args)).toEqual([ - ["--import", "tsx", "scripts/build-all.mts", "qaRuntime"], + [ + "--import", + expect.stringMatching(/\/scripts\/tsx\.mjs$/), + expect.stringMatching(/[\\/]scripts[\\/]lib[\\/]dist-artifact-ownership\.mts$/), + expect.stringMatching(/\/scripts\/build-all\.mts$/), + "qaRuntime", + ], ["openclaw.mjs", "status"], ]); const env = spawn.mock.calls[0]![2].env!; diff --git a/test/scripts/dist-artifact-ownership.test.ts b/test/scripts/dist-artifact-ownership.test.ts index d760bf3e215b..42e8448f4a4a 100644 --- a/test/scripts/dist-artifact-ownership.test.ts +++ b/test/scripts/dist-artifact-ownership.test.ts @@ -527,6 +527,8 @@ describe.skipIf(process.platform === "win32")("dist artifact ownership", () => { import fs from 'node:fs'; import { createRequire } from 'node:module'; const require = createRequire(import.meta.url); + const { registerSourceRunnerServiceFixture } = await import(${JSON.stringify(path.join(sourceRoot, "test/scripts/fixtures/source-runner-service.mjs"))}); + registerSourceRunnerServiceFixture(${JSON.stringify(sourceRoot)}); const { runNodeMain } = await import(${JSON.stringify(path.join(sourceRoot, "scripts/run-node.mts"))}); process.exitCode = await runNodeMain({ cwd: process.cwd(), args: ['artifact-fixture'], diff --git a/test/scripts/fixtures/native-runner-signals.mjs b/test/scripts/fixtures/native-runner-signals.mjs index 3e0763950c2f..bf5cb7f8fe20 100644 --- a/test/scripts/fixtures/native-runner-signals.mjs +++ b/test/scripts/fixtures/native-runner-signals.mjs @@ -3,6 +3,7 @@ import fs from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { registerSourceRunnerServiceFixture } from "./source-runner-service.mjs"; const root = process.env.OPENCLAW_TEST_NATIVE_RUNNER_ROOT; const sourceRoot = process.env.OPENCLAW_TEST_NATIVE_RUNNER_SOURCE; @@ -10,6 +11,7 @@ const mode = process.env.OPENCLAW_TEST_NATIVE_RUNNER_MODE; if (!root || !sourceRoot || (mode !== "runner" && mode !== "watch")) { throw new Error("Native runner signal fixture is missing its private scope"); } +registerSourceRunnerServiceFixture(sourceRoot); const fixture = fileURLToPath(import.meta.url); const release = path.join(root, "release"); const terminate = path.join(root, "terminate"); diff --git a/test/scripts/fixtures/source-runner-service.mjs b/test/scripts/fixtures/source-runner-service.mjs new file mode 100644 index 000000000000..7b27e0ad94c0 --- /dev/null +++ b/test/scripts/fixtures/source-runner-service.mjs @@ -0,0 +1,31 @@ +import { registerHooks } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +// Compiler and process fixtures own no Gateway; its custody has separate boundary tests. +export function registerSourceRunnerServiceFixture(sourceRoot) { + const rootUrl = pathToFileURL(sourceRoot + path.sep).href; + const modules = new Map([ + [ + "src/cli/update-cli/update-command-service-publication", + "export async function withGatewayRuntimeArtifactPublication(_params, publish) { return publish(async () => {}); }", + ], + [ + "src/cli/profile", + "export function parseCliProfileArgs(argv) { return { ok: true, profile: null, argv }; } export function applyCliProfileEnv() { throw new Error('This fixture has no profile'); }", + ], + ]); + registerHooks({ + load(url, context, nextLoad) { + // The real wrappers load both source and prepared legacy-finalizer modules. + const source = url.startsWith(rootUrl) + ? [...modules].find( + ([name]) => url.endsWith(`/${name}.ts`) || url.endsWith(`/${name}.js`), + )?.[1] + : undefined; + return source === undefined + ? nextLoad(url, context) + : { format: "module", source, shortCircuit: true }; + }, + }); +} diff --git a/test/scripts/run-node-lifecycle.test.ts b/test/scripts/run-node-lifecycle.test.ts index 0f9c8f490666..cd15085e9a9e 100644 --- a/test/scripts/run-node-lifecycle.test.ts +++ b/test/scripts/run-node-lifecycle.test.ts @@ -239,7 +239,14 @@ it.runIf(process.platform !== "win32").each(["runner", "watch"] as const)( ); await runQaGatewayFixture( async () => { - const worker = await waitForPidFile(path.join(root, "worker.pid"), 5_000); + const worker = await Promise.race([ + waitForPidFile(path.join(root, "worker.pid"), 5_000), + command.then((result) => { + throw new Error( + `Native ${mode} exited before its worker started: ${formatShimResult(result)}`, + ); + }), + ]); expect(isProcessAlive(worker)).toBe(true); writeFileSync(path.join(root, "terminate"), "terminate"); const result = await command; diff --git a/test/scripts/run-node.test-support.ts b/test/scripts/run-node.test-support.ts index 7bd2232d625e..0e2a57b5b8dd 100644 --- a/test/scripts/run-node.test-support.ts +++ b/test/scripts/run-node.test-support.ts @@ -10,7 +10,7 @@ import { bundledPluginFile, bundledPluginRoot, } from "openclaw/plugin-sdk/test-fixtures"; -import { expect, it as baseIt } from "vitest"; +import { expect, it as baseIt, vi } from "vitest"; import { copyBundledPluginMetadata } from "../../scripts/copy-bundled-plugin-metadata.mts"; import { BUILD_STAMP_FILE, @@ -26,6 +26,13 @@ import { } from "../../scripts/lib/update-compat-chunks.mts"; import { runNodeMain } from "../../scripts/run-node.mts"; import { withTestDir } from "../../src/test-helpers/temp-dir.js"; +// These launcher fixtures have no service. Publication custody is covered at its owner. +vi.mock("../../src/cli/update-cli/update-command-service-publication.js", () => ({ + withGatewayRuntimeArtifactPublication: async ( + _params: unknown, + publish: () => Promise, + ) => publish(), +})); import { previousReleaseInventory, writeUpdateCompatibilityBuildFixture, @@ -210,7 +217,14 @@ export async function writeRuntimePostBuildScaffold(tmp: string): Promise } export function expectedBuildSpawn() { - return [process.execPath, "--import", "tsx", "scripts/build-all.mts", "qaRuntime"]; + return [ + process.execPath, + "--import", + expect.stringMatching(/\/scripts\/tsx\.mjs$/), + expect.stringMatching(/[\\/]scripts[\\/]lib[\\/]dist-artifact-ownership\.mts$/), + expect.stringMatching(/\/scripts\/build-all\.mts$/), + "qaRuntime", + ]; } export function statusCommandSpawn() { @@ -234,7 +248,7 @@ export function resolvePath(tmp: string, relativePath: string) { } export function isTsxScriptArgs(args: string[], scriptPath: string): boolean { - return args[0] === "--import" && args[1] === "tsx" && args[2] === scriptPath; + return args[0] === "--import" && args.some((arg) => arg.endsWith(scriptPath)); } export async function expectPathMissing(targetPath: string): Promise {