diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index e0a4a3f538a..7d0b8b1a7b5 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -168,6 +168,13 @@ openclaw doctor --lint --all --skip core/doctor/session-locks `openclaw doctor --post-upgrade` runs plugin compatibility probes for chaining after a build or upgrade. Findings go to stdout; exit code is 1 if any finding has `level: "error"`. Add `--json` for a machine-readable envelope (`{ probesRun, findings }`), suitable for CI, the community `fork-upgrade` skill, and other post-upgrade smoke tooling. If the installed plugin index is missing or malformed, JSON mode still emits the envelope with a `plugin.index_unavailable` error finding. +Container image startup is the exception to the usual "run doctor after +updating" flow. When `openclaw gateway run` starts on a new OpenClaw version, it +runs safe state and plugin repairs before reporting ready. If repair cannot +finish safely, startup exits and tells you to run the same image once with +`openclaw doctor --fix` against the same mounted state/config before restarting +the container normally. + ## Notes - In Nix mode (`OPENCLAW_NIX_MODE=1`), read-only doctor checks still work, but `doctor --fix`, `doctor --repair`, `doctor --yes`, and `doctor --generate-gateway-token` are disabled because `openclaw.json` is immutable. Edit the Nix source for this install instead; for nix-openclaw, use the agent-first [Quick Start](https://github.com/openclaw/nix-openclaw#quick-start). diff --git a/docs/docs_map.md b/docs/docs_map.md index 4f3661b1a5a..8008eae59a3 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -4200,6 +4200,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Prerequisites - H2: Containerized gateway - H3: Manual flow + - H3: Upgrading container images - H3: Environment variables - H3: Observability - H3: Health checks @@ -4468,6 +4469,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Podman and Tailscale - H2: Systemd (Quadlet, optional) - H2: Config, env, and storage + - H2: Upgrading images - H2: Useful commands - H2: Troubleshooting - H2: Related diff --git a/docs/gateway/doctor.md b/docs/gateway/doctor.md index 7fa16e1feb6..d2b3b630189 100644 --- a/docs/gateway/doctor.md +++ b/docs/gateway/doctor.md @@ -412,7 +412,9 @@ That stages grounded durable candidates into the short-term dreaming store while Doctor removes legacy OpenClaw-generated plugin dependency staging state in `openclaw doctor --fix` / `openclaw doctor --repair` mode: stale generated dependency roots, old install-stage directories, package-local debris from earlier bundled-plugin dependency repair code, and orphaned or recovered managed npm copies of bundled `@openclaw/*` plugins that can shadow the current bundled manifest. Doctor also relinks the host `openclaw` package into managed npm plugins that declare `peerDependencies.openclaw`, so package-local runtime imports such as `openclaw/plugin-sdk/*` keep resolving after updates or npm repairs. - Doctor can also reinstall missing downloadable plugins when config references them but the local plugin registry cannot find them (material `plugins.entries`, configured channel/provider/search settings, configured agent runtimes). During package updates, doctor avoids running package-manager plugin repair while the core package is being swapped; run `openclaw doctor --fix` again after the update if a configured plugin still needs recovery. Gateway startup and config reload do not run package managers; plugin installs remain explicit doctor/install/update work. + Doctor can also reinstall missing downloadable plugins when config references them but the local plugin registry cannot find them (material `plugins.entries`, configured channel/provider/search settings, configured agent runtimes). During package updates, doctor avoids reinstalling plugin packages while the core package is being swapped; run `openclaw doctor --fix` again after the update if a configured plugin still needs recovery. Outside the container image startup exception below, gateway startup and config reload do not run package repair; plugin installs remain explicit doctor/install/update work. + + Containerized gateway startup has a narrow upgrade exception: when `openclaw gateway run` starts on a new OpenClaw version, it runs safe state migrations and the existing post-core plugin convergence before readiness, then records a per-version checkpoint. This startup pass can clean stale bundled-plugin records, repair local plugin links, reinstall configured plugin packages when the convergence path requires it, and check active plugin payloads. If startup cannot repair safely, run the same image once with `openclaw doctor --fix` against the same mounted state/config before restarting the container normally. diff --git a/docs/install/docker.md b/docs/install/docker.md index c3530d49d1d..82897328935 100644 --- a/docs/install/docker.md +++ b/docs/install/docker.md @@ -115,6 +115,28 @@ docker compose up -d openclaw-gateway Run `docker compose` from the repo root. If you enabled `OPENCLAW_EXTRA_MOUNTS` or `OPENCLAW_HOME_VOLUME`, the setup script writes `docker-compose.extra.yml`; include it after any `docker-compose.override.yml` you maintain yourself, e.g. `-f docker-compose.yml -f docker-compose.override.yml -f docker-compose.extra.yml`. +### Upgrading container images + +When you replace the OpenClaw image but keep the same mounted state/config, the +new gateway runs startup-safe upgrade migrations and plugin convergence before +readiness. Routine image upgrades should not require a separate +`openclaw doctor --fix` pass. + +If startup cannot complete those repairs safely, the gateway exits instead of +reporting healthy. With a restart policy, Docker, Podman, or Kubernetes may show +the gateway container restarting. Keep the mounted state volume, then run the +same image once with `openclaw doctor --fix` as the container command, using the +same state/config mounts the gateway uses: + +```bash +docker run --rm -v :/home/node/.openclaw openclaw doctor --fix +podman run --rm -v :/home/node/.openclaw openclaw doctor --fix +``` + +After doctor finishes, restart the gateway container with its default command. +In Kubernetes, run the same command in a one-off Job or debug pod mounted to the +same PVC, then restart the Deployment or StatefulSet. + ### Environment variables Optional variables accepted by `scripts/docker/setup.sh` (and, for the gateway container, by `docker-compose.yml` directly): diff --git a/docs/install/podman.md b/docs/install/podman.md index 6886658522e..940288e8da0 100644 --- a/docs/install/podman.md +++ b/docs/install/podman.md @@ -153,6 +153,35 @@ Useful env vars for the manual launcher (persist these in `~/.openclaw/.env`; th If you use a non-default `OPENCLAW_CONFIG_DIR` or `OPENCLAW_WORKSPACE_DIR`, set the same variables for both `./scripts/podman/setup.sh` and later `./scripts/run-openclaw-podman.sh launch` commands -- the repo-local launcher does not persist custom path overrides across shells. +## Upgrading images + +After you rebuild or pull a new image, restart the container or Quadlet service. +On first startup for a new OpenClaw version, the gateway runs safe state and +plugin repairs before reporting ready. + +If the gateway exits instead of becoming ready, run the same image once with +`openclaw doctor --fix` against the same mounted state/config, then restart the +gateway normally: + +```bash +OPENCLAW_CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}" +OPENCLAW_WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$OPENCLAW_CONFIG_DIR/workspace}" +OPENCLAW_PODMAN_IMAGE="${OPENCLAW_PODMAN_IMAGE:-${OPENCLAW_IMAGE:-openclaw:local}}" + +podman run --rm -it \ + --userns=keep-id \ + --user "$(id -u):$(id -g)" \ + -e HOME=/home/node \ + -e NPM_CONFIG_CACHE=/home/node/.openclaw/.npm \ + -v "$OPENCLAW_CONFIG_DIR:/home/node/.openclaw:rw" \ + -v "$OPENCLAW_WORKSPACE_DIR:/home/node/.openclaw/workspace:rw" \ + "$OPENCLAW_PODMAN_IMAGE" \ + openclaw doctor --fix +``` + +On SELinux hosts, add `,Z` to both bind mounts if Podman blocks access to the +mounted state. + ## Useful commands - **Container logs:** `podman logs -f openclaw` @@ -165,6 +194,7 @@ If you use a non-default `OPENCLAW_CONFIG_DIR` or `OPENCLAW_WORKSPACE_DIR`, set - **Permission denied (EACCES) on config or workspace:** The container runs with `--userns=keep-id` and `--user :` by default. Ensure the host config/workspace paths are owned by your current user. - **Gateway start blocked (missing `gateway.mode=local`):** Ensure `~/.openclaw/openclaw.json` exists and sets `gateway.mode="local"`. `scripts/podman/setup.sh` creates this if missing. +- **Container restarts after an image update:** Run the one-off `openclaw doctor --fix` command in [Upgrading images](#upgrading-images), then start the gateway again. - **Container CLI commands hit the wrong target:** Use `openclaw --container ...` explicitly, or export `OPENCLAW_CONTAINER=` in your shell. - **`openclaw update` fails with `--container`:** Expected. Rebuild/pull the image, then restart the container or the Quadlet service. - **Quadlet service does not start:** Run `systemctl --user daemon-reload`, then `systemctl --user start openclaw.service`. On headless systems you may also need `sudo loginctl enable-linger "$(whoami)"`. diff --git a/docs/install/updating.md b/docs/install/updating.md index 1f65e3625cf..f4b588e2c98 100644 --- a/docs/install/updating.md +++ b/docs/install/updating.md @@ -8,6 +8,11 @@ title: "Updating" Keep OpenClaw up to date. +For Docker, Podman, and Kubernetes image replacements, see +[Upgrading container images](/install/docker#upgrading-container-images). The +gateway runs startup-safe upgrade work before readiness and exits if mounted +state needs manual repair. + ## Recommended: `openclaw update` Detects your install type (npm or git), fetches the latest version, runs `openclaw doctor`, and restarts the gateway. diff --git a/src/cli/program/config-guard.test.ts b/src/cli/program/config-guard.test.ts index a983c958d3a..17878a31ea5 100644 --- a/src/cli/program/config-guard.test.ts +++ b/src/cli/program/config-guard.test.ts @@ -206,6 +206,27 @@ describe("ensureConfigReady", () => { }); }); + it("requires a startup migration checkpoint for foreground gateway startup", async () => { + await runEnsureConfigReady(["gateway"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({ + migrateState: true, + migrateLegacyConfig: false, + invalidConfigNote: false, + requireStartupMigrationCheckpoint: true, + }); + }); + + it("does not require a startup migration checkpoint for gateway probes", async () => { + await runEnsureConfigReady(["gateway", "health"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({ + migrateState: true, + migrateLegacyConfig: false, + invalidConfigNote: false, + }); + }); + it("runs doctor flow for legacy sessions without task sidecars", async () => { const root = useTempOpenClawHome(); fs.mkdirSync(path.join(root, ".openclaw", "sessions"), { recursive: true }); diff --git a/src/cli/program/config-guard.ts b/src/cli/program/config-guard.ts index eaac25a6b0d..8d332d27b3f 100644 --- a/src/cli/program/config-guard.ts +++ b/src/cli/program/config-guard.ts @@ -10,14 +10,7 @@ import { resolveRequiredHomeDir } from "../../infra/home-dir.js"; import type { RuntimeEnv } from "../../runtime.js"; import { shouldMigrateStateFromPath } from "../argv.js"; -const ALLOWED_INVALID_COMMANDS = new Set([ - "audit", - "doctor", - "logs", - "health", - "help", - "status", -]); +const ALLOWED_INVALID_COMMANDS = new Set(["audit", "doctor", "logs", "health", "help", "status"]); const ALLOWED_INVALID_GATEWAY_SUBCOMMANDS = new Set([ "run", "status", @@ -177,6 +170,15 @@ function snapshotHasConfiguredSessionStore( return typeof store === "string" && store.trim().length > 0; } +function shouldRequireStartupMigrationCheckpoint(commandPath: string[]): boolean { + const commandName = commandPath[0]; + const subcommandName = commandPath[1]; + return ( + commandName === "gateway" && + (subcommandName === undefined || subcommandName === "run" || subcommandName.trim() === "") + ); +} + async function getConfigSnapshot() { // Tests often mutate config fixtures; caching can make those flaky. if (process.env.VITEST === "true") { @@ -212,6 +214,9 @@ export async function ensureConfigReady(params: { migrateState: true, migrateLegacyConfig: false, invalidConfigNote: false, + ...(shouldRequireStartupMigrationCheckpoint(commandPath) + ? { requireStartupMigrationCheckpoint: true } + : {}), ...(params.beforeStateMigrations ? { beforeStateMigrations: params.beforeStateMigrations } : {}), diff --git a/src/commands/doctor-config-preflight.state-migration.test.ts b/src/commands/doctor-config-preflight.state-migration.test.ts index 0cc094e991b..ae5c2da363f 100644 --- a/src/commands/doctor-config-preflight.state-migration.test.ts +++ b/src/commands/doctor-config-preflight.state-migration.test.ts @@ -1,26 +1,104 @@ // Doctor config preflight tests cover state migration preflight behavior before config repair. import { beforeEach, describe, expect, it, vi } from "vitest"; +type StateMigrationResult = { + migrated: boolean; + skipped: boolean; + changes: string[]; + warnings: string[]; +}; + +type StartupConvergenceWarning = { + message: string; + guidance: string[]; +}; + +type StartupConvergenceResult = { + changes: string[]; + notices?: StartupConvergenceWarning[]; + warnings: StartupConvergenceWarning[]; + errored: boolean; + smokeFailures: unknown[]; + installRecords: Record; +}; + const autoMigrateLegacyStateDir = vi.hoisted(() => - vi.fn(async () => ({ migrated: false, skipped: false, changes: [], warnings: [] })), + vi.fn( + async (): Promise => ({ + migrated: false, + skipped: false, + changes: [], + warnings: [], + }), + ), ); const autoMigrateLegacyState = vi.hoisted(() => - vi.fn(async () => ({ migrated: true, skipped: false, changes: ["imported"], warnings: [] })), + vi.fn( + async (): Promise => ({ + migrated: true, + skipped: false, + changes: ["imported"], + warnings: [], + }), + ), ); const autoMigrateLegacyPluginDoctorState = vi.hoisted(() => - vi.fn(async () => ({ - migrated: true, - skipped: false, - changes: ["plugin-imported"], - warnings: [], - })), + vi.fn( + async (): Promise => ({ + migrated: true, + skipped: false, + changes: ["plugin-imported"], + warnings: [], + }), + ), ); const autoMigrateLegacyTaskStateSidecars = vi.hoisted(() => - vi.fn(async () => ({ migrated: true, skipped: false, changes: ["task-imported"], warnings: [] })), + vi.fn( + async (): Promise => ({ + migrated: true, + skipped: false, + changes: ["task-imported"], + warnings: [], + }), + ), ); const repairLegacyCronStoreWithoutPrompt = vi.hoisted(() => vi.fn(async () => ({ changes: ["cron-imported"], warnings: [] })), ); +const needsStartupMigrationCheckpoint = vi.hoisted(() => vi.fn(() => false)); +const startupMigrationLeaseHeartbeat = vi.hoisted(() => vi.fn()); +const startupMigrationLeaseRelease = vi.hoisted(() => vi.fn()); +const startupMigrationLease = vi.hoisted(() => ({ + heartbeat: startupMigrationLeaseHeartbeat, + owner: "startup-test-owner", + release: startupMigrationLeaseRelease, +})); +const acquireStartupMigrationLease = vi.hoisted(() => vi.fn(() => startupMigrationLease)); +const recordSuccessfulStartupMigrations = vi.hoisted(() => vi.fn()); +const runPostCorePluginConvergence = vi.hoisted(() => + vi.fn( + async (): Promise => ({ + changes: [], + notices: [], + warnings: [], + errored: false, + smokeFailures: [], + installRecords: {}, + }), + ), +); +const makeStartupConvergenceResult = vi.hoisted( + () => + (overrides: Partial = {}): StartupConvergenceResult => ({ + changes: [], + notices: [], + warnings: [], + errored: false, + smokeFailures: [], + installRecords: {}, + ...overrides, + }), +); const readConfigFileSnapshot = vi.hoisted(() => vi.fn(async () => ({ exists: true, @@ -46,6 +124,16 @@ vi.mock("./doctor/cron/index.js", () => ({ repairLegacyCronStoreWithoutPrompt, })); +vi.mock("../infra/startup-migration-checkpoint.js", () => ({ + acquireStartupMigrationLease, + needsStartupMigrationCheckpoint, + recordSuccessfulStartupMigrations, +})); + +vi.mock("../cli/update-cli/post-core-plugin-convergence.js", () => ({ + runPostCorePluginConvergence, +})); + vi.mock("../config/io.js", () => ({ readConfigFileSnapshot, recoverConfigFromJsonRootSuffix: vi.fn(), @@ -59,6 +147,36 @@ const { runDoctorConfigPreflight } = await import("./doctor-config-preflight.js" describe("runDoctorConfigPreflight state migration", () => { beforeEach(() => { vi.clearAllMocks(); + needsStartupMigrationCheckpoint.mockReturnValue(false); + runPostCorePluginConvergence.mockResolvedValue(makeStartupConvergenceResult()); + autoMigrateLegacyStateDir.mockResolvedValue({ + migrated: false, + skipped: false, + changes: [], + warnings: [], + }); + autoMigrateLegacyState.mockResolvedValue({ + migrated: true, + skipped: false, + changes: ["imported"], + warnings: [], + }); + autoMigrateLegacyPluginDoctorState.mockResolvedValue({ + migrated: true, + skipped: false, + changes: ["plugin-imported"], + warnings: [], + }); + autoMigrateLegacyTaskStateSidecars.mockResolvedValue({ + migrated: true, + skipped: false, + changes: ["task-imported"], + warnings: [], + }); + repairLegacyCronStoreWithoutPrompt.mockResolvedValue({ + changes: ["cron-imported"], + warnings: [], + }); }); it("runs the startup guard immediately before the first state mutation", async () => { @@ -99,6 +217,23 @@ describe("runDoctorConfigPreflight state migration", () => { expect(readConfigFileSnapshot).toHaveBeenCalledOnce(); }); + it("does not touch the startup checkpoint before the startup guard accepts", async () => { + needsStartupMigrationCheckpoint.mockReturnValue(true); + + await expect( + runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + beforeStateMigrations: async () => false, + requireStartupMigrationCheckpoint: true, + }), + ).rejects.toThrow("selected config changed during startup"); + + expect(needsStartupMigrationCheckpoint).not.toHaveBeenCalled(); + expect(acquireStartupMigrationLease).not.toHaveBeenCalled(); + expect(readConfigFileSnapshot).not.toHaveBeenCalled(); + }); + it("skips config-dependent migrations when the fresh snapshot guard rejects", async () => { const beforeStateMigrations = vi .fn<(snapshot?: Record) => Promise>() @@ -138,6 +273,142 @@ describe("runDoctorConfigPreflight state migration", () => { expect(note).toHaveBeenCalledWith("- imported", "Doctor changes"); }); + it("records the startup migration checkpoint after clean startup migrations", async () => { + needsStartupMigrationCheckpoint.mockReturnValue(true); + + await runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + requireStartupMigrationCheckpoint: true, + }); + + expect(acquireStartupMigrationLease).toHaveBeenCalledWith({ env: process.env }); + expect(runPostCorePluginConvergence).toHaveBeenCalledWith({ + cfg: { gateway: { mode: "local", port: 19091 } }, + env: process.env, + }); + expect(recordSuccessfulStartupMigrations).toHaveBeenCalledWith({ + env: process.env, + lease: startupMigrationLease, + }); + expect(startupMigrationLeaseRelease).toHaveBeenCalledOnce(); + }); + + it("does not acquire the startup migration lease when the checkpoint is current", async () => { + await runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + requireStartupMigrationCheckpoint: true, + }); + + expect(acquireStartupMigrationLease).not.toHaveBeenCalled(); + expect(recordSuccessfulStartupMigrations).not.toHaveBeenCalled(); + }); + + it("blocks gateway readiness when startup migrations leave warnings", async () => { + needsStartupMigrationCheckpoint.mockReturnValue(true); + autoMigrateLegacyStateDir.mockResolvedValueOnce({ + migrated: false, + skipped: false, + changes: [], + warnings: ["Left legacy config health state in place."], + }); + + await expect( + runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + requireStartupMigrationCheckpoint: true, + }), + ).rejects.toThrow("refusing to report the gateway ready"); + + expect(recordSuccessfulStartupMigrations).not.toHaveBeenCalled(); + expect(startupMigrationLeaseRelease).toHaveBeenCalledOnce(); + }); + + it("blocks gateway readiness when plugin repair warnings remain", async () => { + needsStartupMigrationCheckpoint.mockReturnValue(true); + runPostCorePluginConvergence.mockResolvedValueOnce( + makeStartupConvergenceResult({ + warnings: [ + { + message: "Configured plugin discord is not installed.", + guidance: ["Run `openclaw update repair` to retry plugin repair."], + }, + ], + }), + ); + + await expect( + runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + requireStartupMigrationCheckpoint: true, + }), + ).rejects.toThrow("Configured plugin discord is not installed"); + + expect(recordSuccessfulStartupMigrations).not.toHaveBeenCalled(); + expect(note).toHaveBeenCalledWith( + "- Configured plugin discord is not installed. Run `openclaw update repair` to retry plugin repair.", + "Doctor warnings", + ); + expect(startupMigrationLeaseRelease).toHaveBeenCalledOnce(); + }); + + it("blocks gateway readiness when plugin convergence reports an error", async () => { + needsStartupMigrationCheckpoint.mockReturnValue(true); + runPostCorePluginConvergence.mockResolvedValueOnce( + makeStartupConvergenceResult({ + errored: true, + warnings: [ + { + message: 'Plugin "discord" failed post-core payload smoke check (missing): index.js', + guidance: [ + "Run `openclaw update repair` to retry plugin repair.", + "Run `openclaw plugins inspect discord --runtime --json` for details.", + ], + }, + ], + }), + ); + + await expect( + runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + requireStartupMigrationCheckpoint: true, + }), + ).rejects.toThrow("failed post-core payload smoke check"); + + expect(recordSuccessfulStartupMigrations).not.toHaveBeenCalled(); + expect(startupMigrationLeaseRelease).toHaveBeenCalledOnce(); + }); + + it("does not checkpoint startup migrations when the config snapshot is invalid", async () => { + needsStartupMigrationCheckpoint.mockReturnValue(true); + readConfigFileSnapshot.mockResolvedValueOnce({ + exists: true, + valid: false, + config: { gateway: { mode: "local", port: "bad" } }, + sourceConfig: { gateway: { mode: "local", port: "bad" } }, + parsed: { gateway: { mode: "local", port: "bad" } }, + legacyIssues: [], + warnings: [], + issues: [{ path: "gateway.port", message: "invalid" }], + }); + + await expect( + runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + requireStartupMigrationCheckpoint: true, + }), + ).rejects.toThrow("OpenClaw config is invalid"); + + expect(recordSuccessfulStartupMigrations).not.toHaveBeenCalled(); + expect(startupMigrationLeaseRelease).toHaveBeenCalledOnce(); + }); + it("passes explicit corrupt-target recovery to state migrations", async () => { await runDoctorConfigPreflight({ migrateLegacyConfig: false, diff --git a/src/commands/doctor-config-preflight.ts b/src/commands/doctor-config-preflight.ts index b9e8af62cbe..5720bf5930b 100644 --- a/src/commands/doctor-config-preflight.ts +++ b/src/commands/doctor-config-preflight.ts @@ -11,6 +11,7 @@ import { formatConfigIssueLines } from "../config/issue-format.js"; import type { ConfigFileSnapshot, LegacyConfigIssue } from "../config/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { isTruthyEnvValue } from "../infra/env.js"; +import type { StartupMigrationLease } from "../infra/startup-migration-checkpoint.js"; import { createLazyRuntimeModule } from "../shared/lazy-runtime.js"; import { resolveHomeDir } from "../utils.js"; import { noteIncludeConfinementWarning } from "./doctor-config-analysis.js"; @@ -108,6 +109,47 @@ function noteStateMigrationResult(result: { changes: string[]; warnings: string[ } } +async function runStartupUpgradeConvergence(params: { + cfg: OpenClawConfig; + env: NodeJS.ProcessEnv; +}): Promise { + const { runPostCorePluginConvergence } = + await import("../cli/update-cli/post-core-plugin-convergence.js"); + const convergence = await runPostCorePluginConvergence({ + cfg: params.cfg, + env: params.env, + }); + if (convergence.changes.length > 0) { + note(convergence.changes.map((entry) => `- ${entry}`).join("\n"), "Doctor changes"); + } + const notices = convergence.notices ?? []; + if (notices.length > 0) { + note( + notices.map((notice) => `- ${notice.message} ${notice.guidance.join(" ")}`.trim()).join("\n"), + "Doctor notices", + ); + } + const warnings = convergence.warnings.map((warning) => + `${warning.message} ${warning.guidance.join(" ")}`.trim(), + ); + if (warnings.length > 0) { + note(warnings.map((warning) => `- ${warning}`).join("\n"), "Doctor warnings"); + } + return warnings; +} + +function formatStartupMigrationFailure(params: { warnings: string[]; blockers: string[] }): string { + const details = [ + ...params.warnings.map((warning) => `- ${warning}`), + ...params.blockers.map((blocker) => `- ${blocker}`), + ]; + return [ + "OpenClaw startup migrations did not complete cleanly; refusing to report the gateway ready.", + ...details, + 'Run "openclaw doctor --fix" against the mounted state/config, then restart the container.', + ].join("\n"); +} + /** * Runs early doctor config checks before the main config repair flow. * @@ -122,110 +164,187 @@ export async function runDoctorConfigPreflight( recoverCorruptTargetStore?: boolean; invalidConfigNote?: string | false; beforeStateMigrations?: (snapshot?: ConfigFileSnapshot) => Promise; + requireStartupMigrationCheckpoint?: boolean; } = {}, ): Promise { const stateMigrations = options.migrateState !== false ? await loadDoctorStateMigrations() : undefined; - // The gateway uses this last-moment guard to ensure its prepared config did not change before - // any automatic migration mutates state. A rejected guard skips every state migration stage. - const stateMigrationsAllowed = - stateMigrations === undefined || - options.beforeStateMigrations === undefined || - (await options.beforeStateMigrations()); - if (stateMigrations && stateMigrationsAllowed) { - const { autoMigrateLegacyStateDir } = stateMigrations; - const stateDirResult = await autoMigrateLegacyStateDir({ env: process.env }); - noteStateMigrationResult(stateDirResult); - } - - if (options.migrateLegacyConfig !== false) { - const legacyConfigChanges = await maybeMigrateLegacyConfig(); - if (legacyConfigChanges.length > 0) { - note(legacyConfigChanges.map((entry) => `- ${entry}`).join("\n"), "Doctor changes"); - } - } - - const readOptions = { - skipPluginValidation: shouldSkipPluginValidationForDoctorConfigPreflight(), + const startupCheckpoint = + options.requireStartupMigrationCheckpoint === true + ? await import("../infra/startup-migration-checkpoint.js") + : undefined; + let shouldRecordStartupCheckpoint = false; + let startupMigrationLease: StartupMigrationLease | undefined; + let startupMigrationHeartbeat: ReturnType | undefined; + let startupMigrationHeartbeatError: unknown; + const startupMigrationWarnings: string[] = []; + const noteStartupStateMigrationResult = (result: { changes: string[]; warnings: string[] }) => { + startupMigrationWarnings.push(...result.warnings); + noteStateMigrationResult(result); }; - let snapshot = addDoctorLegacyIssues(await readConfigFileSnapshot(readOptions)); - if (options.repairPrefixedConfig === true && snapshot.exists && !snapshot.valid) { - if (await recoverConfigFromJsonRootSuffix(snapshot)) { - note("Removed non-JSON prefix from openclaw.json; original saved as .clobbered.*.", "Config"); - snapshot = addDoctorLegacyIssues(await readConfigFileSnapshot(readOptions)); - } else if ( - await recoverConfigFromLastKnownGood({ snapshot, reason: "doctor-invalid-config" }) - ) { - note( - "Restored openclaw.json from last-known-good; original saved as .clobbered.*.", - "Config", + try { + // The gateway uses this last-moment guard to ensure its prepared config did not change before + // any automatic migration mutates state. A rejected guard skips every state migration stage. + const stateMigrationsAllowed = + stateMigrations === undefined || + options.beforeStateMigrations === undefined || + (await options.beforeStateMigrations()); + if (startupCheckpoint && !stateMigrationsAllowed) { + throw new Error( + "OpenClaw startup migrations were skipped because the selected config changed during startup; refusing to report the gateway ready. Retry startup so the new config can be validated.", ); - snapshot = addDoctorLegacyIssues(await readConfigFileSnapshot(readOptions)); } - } - const invalidConfigNote = - options.invalidConfigNote ?? "Config invalid; doctor will run with best-effort config."; - if ( - invalidConfigNote && - snapshot.exists && - !snapshot.valid && - snapshot.legacyIssues.length === 0 - ) { - note(invalidConfigNote, "Config"); - noteIncludeConfinementWarning(snapshot); - } - - const warnings = snapshot.warnings ?? []; - if (warnings.length > 0) { - note(formatConfigIssueLines(warnings, "-").join("\n"), "Config warnings"); - } - - const baseConfig = snapshot.sourceConfig ?? snapshot.config ?? {}; - const stateMigrationInput = resolveStateMigrationConfigInput({ snapshot, baseConfig }); - const configStateMigrationsAllowed = - stateMigrations !== undefined && - stateMigrationsAllowed && - (options.beforeStateMigrations === undefined || - (await options.beforeStateMigrations(snapshot))); - if (stateMigrations && configStateMigrationsAllowed) { - const { - autoMigrateLegacyState, - autoMigrateLegacyPluginDoctorState, - autoMigrateLegacyTaskStateSidecars, - } = stateMigrations; - if (stateMigrationInput) { - if (stateMigrationInput.cfg) { - const { repairLegacyCronStoreWithoutPrompt } = await loadDoctorCron(); - const cronResult = await repairLegacyCronStoreWithoutPrompt({ - cfg: stateMigrationInput.cfg, - }); - noteStateMigrationResult(cronResult); - noteStateMigrationResult( - await autoMigrateLegacyState({ - cfg: stateMigrationInput.cfg, - ...(stateMigrationInput.pluginDoctorConfig - ? { pluginDoctorConfig: stateMigrationInput.pluginDoctorConfig } - : {}), - env: process.env, - recoverCorruptTargetStore: options.recoverCorruptTargetStore, - }), - ); - } else if (stateMigrationInput.pluginDoctorConfig) { - noteStateMigrationResult( - await autoMigrateLegacyPluginDoctorState({ - config: stateMigrationInput.pluginDoctorConfig, - env: process.env, - }), - ); - noteStateMigrationResult(await autoMigrateLegacyTaskStateSidecars({ env: process.env })); + if (startupCheckpoint) { + shouldRecordStartupCheckpoint = startupCheckpoint.needsStartupMigrationCheckpoint({ + env: process.env, + }); + startupMigrationLease = shouldRecordStartupCheckpoint + ? startupCheckpoint.acquireStartupMigrationLease({ env: process.env }) + : undefined; + if (startupMigrationLease) { + startupMigrationHeartbeat = setInterval(() => { + try { + startupMigrationLease?.heartbeat(); + } catch (error) { + startupMigrationHeartbeatError = error; + } + }, 60_000); + startupMigrationHeartbeat.unref?.(); } - } else { - noteStateMigrationResult(await autoMigrateLegacyTaskStateSidecars({ env: process.env })); } - } + if (stateMigrations && stateMigrationsAllowed) { + const { autoMigrateLegacyStateDir } = stateMigrations; + const stateDirResult = await autoMigrateLegacyStateDir({ env: process.env }); + noteStartupStateMigrationResult(stateDirResult); + } - return { - snapshot, - baseConfig, - }; + if (options.migrateLegacyConfig !== false) { + const legacyConfigChanges = await maybeMigrateLegacyConfig(); + if (legacyConfigChanges.length > 0) { + note(legacyConfigChanges.map((entry) => `- ${entry}`).join("\n"), "Doctor changes"); + } + } + + const readOptions = { + skipPluginValidation: shouldSkipPluginValidationForDoctorConfigPreflight(), + }; + let snapshot = addDoctorLegacyIssues(await readConfigFileSnapshot(readOptions)); + if (options.repairPrefixedConfig === true && snapshot.exists && !snapshot.valid) { + if (await recoverConfigFromJsonRootSuffix(snapshot)) { + note( + "Removed non-JSON prefix from openclaw.json; original saved as .clobbered.*.", + "Config", + ); + snapshot = addDoctorLegacyIssues(await readConfigFileSnapshot(readOptions)); + } else if ( + await recoverConfigFromLastKnownGood({ snapshot, reason: "doctor-invalid-config" }) + ) { + note( + "Restored openclaw.json from last-known-good; original saved as .clobbered.*.", + "Config", + ); + snapshot = addDoctorLegacyIssues(await readConfigFileSnapshot(readOptions)); + } + } + const invalidConfigNote = + options.invalidConfigNote ?? "Config invalid; doctor will run with best-effort config."; + if ( + invalidConfigNote && + snapshot.exists && + !snapshot.valid && + snapshot.legacyIssues.length === 0 + ) { + note(invalidConfigNote, "Config"); + noteIncludeConfinementWarning(snapshot); + } + + const warnings = snapshot.warnings ?? []; + if (warnings.length > 0) { + note(formatConfigIssueLines(warnings, "-").join("\n"), "Config warnings"); + } + + const baseConfig = snapshot.sourceConfig ?? snapshot.config ?? {}; + const stateMigrationInput = resolveStateMigrationConfigInput({ snapshot, baseConfig }); + const configStateMigrationsAllowed = + stateMigrations !== undefined && + stateMigrationsAllowed && + (options.beforeStateMigrations === undefined || + (await options.beforeStateMigrations(snapshot))); + if (stateMigrations && configStateMigrationsAllowed) { + const { + autoMigrateLegacyState, + autoMigrateLegacyPluginDoctorState, + autoMigrateLegacyTaskStateSidecars, + } = stateMigrations; + if (stateMigrationInput) { + if (stateMigrationInput.cfg) { + const { repairLegacyCronStoreWithoutPrompt } = await loadDoctorCron(); + const cronResult = await repairLegacyCronStoreWithoutPrompt({ + cfg: stateMigrationInput.cfg, + }); + noteStartupStateMigrationResult(cronResult); + noteStartupStateMigrationResult( + await autoMigrateLegacyState({ + cfg: stateMigrationInput.cfg, + ...(stateMigrationInput.pluginDoctorConfig + ? { pluginDoctorConfig: stateMigrationInput.pluginDoctorConfig } + : {}), + env: process.env, + recoverCorruptTargetStore: options.recoverCorruptTargetStore, + }), + ); + } else if (stateMigrationInput.pluginDoctorConfig) { + noteStartupStateMigrationResult( + await autoMigrateLegacyPluginDoctorState({ + config: stateMigrationInput.pluginDoctorConfig, + env: process.env, + }), + ); + noteStartupStateMigrationResult( + await autoMigrateLegacyTaskStateSidecars({ env: process.env }), + ); + } + } else { + noteStartupStateMigrationResult( + await autoMigrateLegacyTaskStateSidecars({ env: process.env }), + ); + } + } + + if (shouldRecordStartupCheckpoint) { + if (startupMigrationHeartbeatError) { + throw startupMigrationHeartbeatError instanceof Error + ? startupMigrationHeartbeatError + : new Error("OpenClaw startup migration lease heartbeat failed."); + } + const blockers = + startupMigrationWarnings.length > 0 + ? [] + : snapshot.valid + ? await runStartupUpgradeConvergence({ cfg: baseConfig, env: process.env }) + : ['OpenClaw config is invalid; run "openclaw doctor --fix" before startup.']; + if (startupMigrationWarnings.length > 0 || blockers.length > 0) { + throw new Error( + formatStartupMigrationFailure({ + warnings: startupMigrationWarnings, + blockers, + }), + ); + } + startupCheckpoint?.recordSuccessfulStartupMigrations({ + env: process.env, + lease: startupMigrationLease, + }); + } + + return { + snapshot, + baseConfig, + }; + } finally { + if (startupMigrationHeartbeat) { + clearInterval(startupMigrationHeartbeat); + } + startupMigrationLease?.release(); + } } diff --git a/src/infra/startup-migration-checkpoint.test.ts b/src/infra/startup-migration-checkpoint.test.ts new file mode 100644 index 00000000000..c814db93b77 --- /dev/null +++ b/src/infra/startup-migration-checkpoint.test.ts @@ -0,0 +1,135 @@ +// Startup migration checkpoint tests cover shared-state version records and leases. +import { mkdirSync } from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { requireNodeSqlite } from "./node-sqlite.js"; +import { + acquireStartupMigrationLease, + needsStartupMigrationCheckpoint, + readStartupMigrationVersion, + recordSuccessfulStartupMigrations, +} from "./startup-migration-checkpoint.js"; + +afterEach(() => { + closeOpenClawStateDatabaseForTest(); +}); + +const startupMigrationTempDirs = useAutoCleanupTempDirTracker(afterEach); + +describe("startup migration checkpoint", () => { + it("records the migrated OpenClaw version in shared state", () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + + expect(readStartupMigrationVersion(env)).toBeNull(); + expect(needsStartupMigrationCheckpoint({ env, version: "2026.7.1" })).toBe(true); + + recordSuccessfulStartupMigrations({ env, version: "2026.7.1", nowMs: 1234 }); + + expect(readStartupMigrationVersion(env)).toBe("2026.7.1"); + expect(needsStartupMigrationCheckpoint({ env, version: "2026.7.1" })).toBe(false); + expect(needsStartupMigrationCheckpoint({ env, version: "2026.7.2" })).toBe(true); + }); + + it("serializes startup migrations with an expiring shared-state lease", () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + const lease = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" }); + + expect(() => acquireStartupMigrationLease({ env, nowMs: 1001, owner: "second" })).toThrow( + "OpenClaw startup migrations are already running", + ); + + lease.release(); + + const next = acquireStartupMigrationLease({ env, nowMs: 1002, owner: "second" }); + next.release(); + }); + + it("renews startup migration leases while the owner is still running", () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + const lease = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" }); + + lease.heartbeat({ nowMs: 300_000 }); + + expect(() => acquireStartupMigrationLease({ env, nowMs: 301_001, owner: "second" })).toThrow( + "OpenClaw startup migrations are already running", + ); + + lease.release(); + }); + + it("does not checkpoint startup migrations after the lease is lost", () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + const first = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" }); + const second = acquireStartupMigrationLease({ env, nowMs: 400_000, owner: "second" }); + + expect(() => + recordSuccessfulStartupMigrations({ + env, + lease: first, + version: "2026.7.1", + nowMs: 400_001, + }), + ).toThrow("startup migration lease was lost"); + expect(readStartupMigrationVersion(env)).toBeNull(); + + second.release(); + }); + + it("reads the checkpoint without requiring the full state schema to be canonical", () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + const sqlite = requireNodeSqlite(); + const dbPath = resolveOpenClawStateSqlitePath(env); + mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new sqlite.DatabaseSync(dbPath); + db.exec(` + CREATE TABLE agent_databases ( + agent_id TEXT NOT NULL PRIMARY KEY, + path TEXT NOT NULL, + schema_version INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + size_bytes INTEGER + ); + `); + db.close(); + + expect(needsStartupMigrationCheckpoint({ env, version: "2026.7.1" })).toBe(true); + const lease = acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" }); + lease.release(); + }); + + it("refuses future-version state databases before creating checkpoint tables", () => { + const env = { + OPENCLAW_STATE_DIR: startupMigrationTempDirs.make("openclaw-startup-migration-"), + }; + const sqlite = requireNodeSqlite(); + const dbPath = resolveOpenClawStateSqlitePath(env); + mkdirSync(path.dirname(dbPath), { recursive: true }); + const db = new sqlite.DatabaseSync(dbPath); + db.exec("PRAGMA user_version = 2;"); + db.close(); + + expect(() => acquireStartupMigrationLease({ env, nowMs: 1000, owner: "first" })).toThrow( + "newer schema version 2", + ); + + const verify = new sqlite.DatabaseSync(dbPath, { readOnly: true }); + const row = verify + .prepare("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = 'state_leases'") + .get() as { ok?: unknown } | undefined; + verify.close(); + expect(row).toBeUndefined(); + }); +}); diff --git a/src/infra/startup-migration-checkpoint.ts b/src/infra/startup-migration-checkpoint.ts new file mode 100644 index 00000000000..a139f32ffff --- /dev/null +++ b/src/infra/startup-migration-checkpoint.ts @@ -0,0 +1,217 @@ +// Coordinates gateway startup migration version checkpoints in shared state. +import { randomUUID } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { withOpenClawStateStartupMigrationCheckpointDatabase } from "../state/openclaw-state-db.js"; +import { VERSION } from "../version.js"; +import { + executeSqliteQuerySync, + executeSqliteQueryTakeFirstSync, + getNodeSqliteKysely, +} from "./kysely-sync.js"; +import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js"; + +type StartupMigrationCheckpointDatabase = Pick< + OpenClawStateKyselyDatabase, + "schema_meta" | "state_leases" +>; + +const STARTUP_MIGRATION_META_KEY = "startup-migrations"; +const STARTUP_MIGRATION_LEASE_SCOPE = "startup-migrations"; +const STARTUP_MIGRATION_LEASE_KEY = "global"; +const STARTUP_MIGRATION_LEASE_TTL_MS = 5 * 60_000; + +export type StartupMigrationLease = { + heartbeat: (params?: { nowMs?: number }) => void; + release: () => void; + readonly owner: string; +}; + +function withStartupMigrationCheckpointDatabase( + env: NodeJS.ProcessEnv, + callback: (db: DatabaseSync) => T, +): T { + return withOpenClawStateStartupMigrationCheckpointDatabase(callback, { env }); +} + +function writeStartupMigrationCheckpointDatabase( + env: NodeJS.ProcessEnv, + callback: (db: DatabaseSync) => T, +): T { + return withStartupMigrationCheckpointDatabase(env, (db) => + runSqliteImmediateTransactionSync(db, () => callback(db)), + ); +} + +export function readStartupMigrationVersion(env: NodeJS.ProcessEnv = process.env): string | null { + return withStartupMigrationCheckpointDatabase(env, (db) => { + const stateDb = getNodeSqliteKysely(db); + const row = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("schema_meta") + .select("app_version as appVersion") + .where("meta_key", "=", STARTUP_MIGRATION_META_KEY), + ); + return row?.appVersion ?? null; + }); +} + +export function needsStartupMigrationCheckpoint( + params: { + env?: NodeJS.ProcessEnv; + version?: string; + } = {}, +): boolean { + return readStartupMigrationVersion(params.env) !== (params.version ?? VERSION); +} + +export function acquireStartupMigrationLease( + params: { + env?: NodeJS.ProcessEnv; + nowMs?: number; + owner?: string; + } = {}, +): StartupMigrationLease { + const env = params.env ?? process.env; + const nowMs = params.nowMs ?? Date.now(); + const owner = params.owner ?? randomUUID(); + const expiresAt = nowMs + STARTUP_MIGRATION_LEASE_TTL_MS; + + writeStartupMigrationCheckpointDatabase(env, (db) => { + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb + .deleteFrom("state_leases") + .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) + .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY) + .where("expires_at", "<=", nowMs), + ); + const existing = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("state_leases") + .select(["owner", "expires_at as expiresAt"]) + .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) + .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY), + ); + if (existing) { + throw new Error( + `OpenClaw startup migrations are already running for this state directory; retry after the other gateway finishes or after ${new Date(existing.expiresAt ?? expiresAt).toISOString()}.`, + ); + } + executeSqliteQuerySync( + db, + stateDb.insertInto("state_leases").values({ + scope: STARTUP_MIGRATION_LEASE_SCOPE, + lease_key: STARTUP_MIGRATION_LEASE_KEY, + owner, + expires_at: expiresAt, + heartbeat_at: nowMs, + payload_json: JSON.stringify({ version: VERSION }), + created_at: nowMs, + updated_at: nowMs, + }), + ); + }); + + return { + owner, + heartbeat: (heartbeatParams = {}) => { + const heartbeatNowMs = heartbeatParams.nowMs ?? Date.now(); + const heartbeatExpiresAt = heartbeatNowMs + STARTUP_MIGRATION_LEASE_TTL_MS; + writeStartupMigrationCheckpointDatabase(env, (db) => { + const stateDb = getNodeSqliteKysely(db); + const result = executeSqliteQuerySync( + db, + stateDb + .updateTable("state_leases") + .set({ + expires_at: heartbeatExpiresAt, + heartbeat_at: heartbeatNowMs, + updated_at: heartbeatNowMs, + }) + .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) + .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY) + .where("owner", "=", owner) + .where("expires_at", ">", heartbeatNowMs), + ); + if (result.numAffectedRows !== 1n) { + throw new Error( + "OpenClaw startup migration lease was lost before startup migrations completed; restart the gateway so migrations can run under a fresh lease.", + ); + } + }); + }, + release: () => { + writeStartupMigrationCheckpointDatabase(env, (db) => { + const stateDb = getNodeSqliteKysely(db); + executeSqliteQuerySync( + db, + stateDb + .deleteFrom("state_leases") + .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) + .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY) + .where("owner", "=", owner), + ); + }); + }, + }; +} + +export function recordSuccessfulStartupMigrations( + params: { + env?: NodeJS.ProcessEnv; + lease?: StartupMigrationLease; + version?: string; + nowMs?: number; + } = {}, +): void { + const env = params.env ?? process.env; + const version = params.version ?? VERSION; + const nowMs = params.nowMs ?? Date.now(); + writeStartupMigrationCheckpointDatabase(env, (db) => { + const stateDb = getNodeSqliteKysely(db); + if (params.lease) { + const activeLease = executeSqliteQueryTakeFirstSync( + db, + stateDb + .selectFrom("state_leases") + .select("owner") + .where("scope", "=", STARTUP_MIGRATION_LEASE_SCOPE) + .where("lease_key", "=", STARTUP_MIGRATION_LEASE_KEY) + .where("owner", "=", params.lease.owner) + .where("expires_at", ">", nowMs), + ); + if (!activeLease) { + throw new Error( + "OpenClaw startup migration lease was lost before checkpoint recording; restart the gateway so migrations can run under a fresh lease.", + ); + } + } + executeSqliteQuerySync( + db, + stateDb + .insertInto("schema_meta") + .values({ + meta_key: STARTUP_MIGRATION_META_KEY, + role: "global", + schema_version: 1, + agent_id: null, + app_version: version, + created_at: nowMs, + updated_at: nowMs, + }) + .onConflict((conflict) => + conflict.column("meta_key").doUpdateSet({ + role: "global", + schema_version: 1, + agent_id: null, + app_version: version, + updated_at: nowMs, + }), + ), + ); + }); +} diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 666bc03b655..57f5427d990 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -31,7 +31,7 @@ import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js" * tables, private file permissions, cached handles, and audit rows for * migrations/backups that operate on local state. */ -const OPENCLAW_STATE_SCHEMA_VERSION = 1; +export const OPENCLAW_STATE_SCHEMA_VERSION = 1; /** Shared timeout used by state and agent SQLite handles before surfacing busy errors. */ export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 30_000; const OPENCLAW_STATE_DIR_MODE = 0o700; @@ -86,7 +86,7 @@ function bestEffortChmodSync(target: string, mode: number): void { stateDbLog.warn(`skipped permission hardening for ${target}: ${String(result.error)}`); } -function ensureOpenClawStatePermissions(pathname: string, env: NodeJS.ProcessEnv): void { +export function ensureOpenClawStatePermissions(pathname: string, env: NodeJS.ProcessEnv): void { const dir = path.dirname(pathname); const defaultDir = resolveOpenClawStateSqliteDir(env); const isDefaultStateDatabase = @@ -307,6 +307,57 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase } } +function ensureStartupMigrationCheckpointSchema(db: DatabaseSync, pathname: string): void { + assertSupportedSchemaVersion(db, pathname); + db.exec(` + CREATE TABLE IF NOT EXISTS schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS state_leases ( + scope TEXT NOT NULL, + lease_key TEXT NOT NULL, + owner TEXT NOT NULL, + expires_at INTEGER, + heartbeat_at INTEGER, + payload_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope, lease_key) + ); + CREATE INDEX IF NOT EXISTS idx_state_leases_expiry + ON state_leases(expires_at, scope, lease_key) + WHERE expires_at IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_state_leases_owner + ON state_leases(owner, updated_at DESC); + `); + ensureColumn(db, "schema_meta", "app_version TEXT"); +} + +export function withOpenClawStateStartupMigrationCheckpointDatabase( + callback: (db: DatabaseSync) => T, + options: OpenClawStateDatabaseOptions = {}, +): T { + const env = options.env ?? process.env; + const pathname = resolveDatabasePath(options); + ensureOpenClawStatePermissions(pathname, env); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(pathname); + db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); + try { + ensureStartupMigrationCheckpointSchema(db, pathname); + return callback(db); + } finally { + db.close(); + ensureOpenClawStatePermissions(pathname, env); + } +} + function backfillCronRunLogEntryJson(db: DatabaseSync): void { if (!tableExists(db, "cron_run_logs") || !tableHasColumn(db, "cron_run_logs", "entry_json")) { return;