diff --git a/docs/concepts/managed-worktrees.md b/docs/concepts/managed-worktrees.md index 4329ea639634..ae87f8b0e258 100644 --- a/docs/concepts/managed-worktrees.md +++ b/docs/concepts/managed-worktrees.md @@ -278,6 +278,39 @@ Restore recreates `openclaw/` at the original pre-snapshot commit, reusing A branch at a shallow history boundary can still be snapshotted and restored. If a later depth-limited fetch makes the snapshot commit itself shallow, Git may no longer resolve its parent. Restore then preserves the snapshot and reports the repository and snapshot commit with `git fetch --unshallow` guidance. OpenClaw does not deepen automatically: origin may not contain the local snapshot, so even a successful fetch cannot guarantee recovery. If the snapshot remains shallow after fetching, recover its parent from the original repository before retrying. +## Retire an already removed snapshot early + +Use `openclaw worktrees retire-snapshot` only when the removed snapshot is redundant +with a retained local branch or remote-tracking commit. This local CLI operation +requires the exact worktree ID, snapshot ref and commit, recorded `removedAt` +milliseconds, and retained source ref and commit. Read those identities from the +removed record and repository before invoking it: + +```bash +openclaw worktrees retire-snapshot \ + --expected-ref refs/openclaw/snapshots/ --expected-oid \ + --removed-at \ + --retained-ref refs/heads/ --retained-oid --json +``` + +Retirement applies only to ordinary snapshots, not exact-state recovery snapshots or +their retained checkouts. It requires identical source trees and retained snapshot-parent history. +It refuses a live or reappeared checkout, Git registration, changed identity, +pending removal, run/removal consumer, unknown or nonempty provisioned-file +inventory, or any retained local workspace projection. Git equality does not prove +that accepted ignored projection data is redundant. It uses the existing allocation +lease and projection owner, then atomically verifies the retained ref and deletes +only the expected snapshot ref. It does not create a backup, delete the retained +source or PR outcome refs, run global garbage collection, or report reclaimed disk +bytes. Git object storage can remain shared after ref retirement. + +A successful JSON result is `{ "retired": true, "id": "" }`; the removed +registry entry is then absent and cannot be restored. A refusal is not cleanup +success. If an interruption occurs after the ref is retired, inspect the remaining +record and projection custody before further cleanup; this command does not infer +safe retirement from a missing ref. This operation is CLI-only, not a Gateway or +Control UI action. + ## Exact-state detached retirement Ordinary removal, forced removal, and `--if-lossless` still refuse a detached diff --git a/src/agents/worktrees/registry.ts b/src/agents/worktrees/registry.ts index cac43fb12cd3..1f0f02b9cae2 100644 --- a/src/agents/worktrees/registry.ts +++ b/src/agents/worktrees/registry.ts @@ -12,6 +12,7 @@ import { runOpenClawStateWriteTransaction, } from "../../state/openclaw-state-db.js"; import { + getRegistryWorktreeProvisionedStateInDatabase, listRegistryWorktreesInDatabase, rowToRecord, WORKTREE_RECORD_COLUMNS, @@ -356,15 +357,73 @@ export function updateRegistryWorktree( ); } +/** Retired snapshots cannot discard provisioned data or another lifecycle's custody. */ +function assertSnapshotRetirementInDatabase( + db: DatabaseSync, + observed: ManagedWorktreeRecord, +): void { + const row = executeSqliteQuerySync( + db, + kyselyFor(db).selectFrom("worktrees").selectAll().where("id", "=", observed.id), + ).rows[0]; + if ( + observed.removedAt === undefined || + !row || + JSON.stringify(rowToRecord(row)) !== JSON.stringify(observed) + ) { + throw new Error("Worktree snapshot retirement identity changed"); + } + const provisioned = getRegistryWorktreeProvisionedStateInDatabase(db, observed.id); + const chunk = executeSqliteQuerySync( + db, + kyselyProvisionedFor(db) + .selectFrom("worktree_provisioned_file_chunks") + .select("worktree_id") + .where("worktree_id", "=", observed.id) + .limit(1), + ).rows[0]; + if (provisioned === undefined || provisioned.length !== 0 || chunk) { + throw new Error("Worktree snapshot retains provisioned data; retain its custody"); + } + const leases = collectLiveRunLeases( + db, + kyselyLeaseFor(db), + worktreeRunLeaseScope(observed.id), + {}, + ); + if (leases.liveCount !== 0 || leases.removingToken !== undefined) { + throw new Error("Worktree snapshot has an active or unresolved run/removal consumer"); + } +} + +export function assertRegistrySnapshotRetirement( + env: NodeJS.ProcessEnv, + observed: ManagedWorktreeRecord, +): void { + runOpenClawStateWriteTransaction(({ db }) => assertSnapshotRetirementInDatabase(db, observed), { + env, + }); +} + export function deleteRegistryWorktree( env: NodeJS.ProcessEnv, id: string, - options: { assertCurrent?: () => void; removalToken?: string } = {}, + options: { + assertCurrent?: () => void; + removalToken?: string; + expectedRetired?: ManagedWorktreeRecord; + } = {}, ): void { runOpenClawStateWriteTransaction( ({ db }) => { // Validate before deleting either the record or its provisioned recovery chunks. options.assertCurrent?.(); + if (options.expectedRetired) { + if (options.expectedRetired.id !== id) { + throw new Error("Worktree snapshot retirement ID changed"); + } + assertSnapshotRetirementInDatabase(db, options.expectedRetired); + } assertRegistryMutationCustody(db, kyselyLeaseFor(db), id, options.removalToken); executeSqliteQuerySync( db, diff --git a/src/agents/worktrees/service.retire-snapshot.test.ts b/src/agents/worktrees/service.retire-snapshot.test.ts new file mode 100644 index 000000000000..93ef5c35d259 --- /dev/null +++ b/src/agents/worktrees/service.retire-snapshot.test.ts @@ -0,0 +1,573 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { promisify } from "node:util"; +import { Command } from "commander"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; +import { registerWorktreesCli } from "../../cli/worktrees-cli.js"; +import { localWorkspaceStore } from "../../gateway/worker-environments/local-workspace-store.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../../infra/kysely-sync.js"; +import { defaultRuntime } from "../../runtime.js"; +import type { DB } from "../../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabaseAsync, + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../../state/openclaw-state-db.js"; +import * as worktreeGit from "./git.js"; +import { + getRegistryWorktree, + getRegistryWorktreeProvisionedChunk, + insertRegistryWorktree, + insertRegistryWorktreeProvisionedChunk, + updateRegistryWorktree, +} from "./registry.js"; +import { resolveRepository } from "./service-preparation.js"; +import { useManagedWorktreeTestRepository } from "./service.test-support.js"; +import { retireManagedWorktreeSnapshotById } from "./snapshot-host.js"; +import type { ManagedWorktreeRecord } from "./types.js"; + +const execFileAsync = promisify(execFile); +const removedAt = 1_700_000_000_100; + +async function git(cwd: string, ...args: string[]): Promise { + const { stdout } = await execFileAsync("git", ["-C", cwd, ...args], { + encoding: "utf8", + env: { ...process.env, GIT_NO_LAZY_FETCH: "1", GIT_OPTIONAL_LOCKS: "0" }, + }); + return stdout.trim(); +} + +describe("Exact removed worktree snapshot retirement", () => { + const initializeRepository = useManagedWorktreeTestRepository(); + const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { + afterEach(async () => { + vi.restoreAllMocks(); + await closeOpenClawStateDatabaseAsync(); + closeOpenClawStateDatabaseForTest(); + vi.unstubAllEnvs(); + cleanup(); + }); + }); + let root: string; + let repo: string; + let env: NodeJS.ProcessEnv; + let record: ManagedWorktreeRecord; + let request: Parameters[0]; + let source: string; + let tree: string; + + beforeEach(async () => { + root = tempDirs.make("openclaw-snapshot-retirement-"); + repo = await initializeRepository(root); + const stateDir = path.join(root, "state"); + await fs.mkdir(stateDir); + expect(await fs.realpath(stateDir)).toBe(stateDir); + vi.stubEnv("OPENCLAW_STATE_DIR", stateDir); + vi.stubEnv("GIT_NO_LAZY_FETCH", "1"); + vi.stubEnv("GIT_OPTIONAL_LOCKS", "0"); + env = { ...process.env, OPENCLAW_STATE_DIR: stateDir }; + source = await git(repo, "rev-parse", "HEAD"); + tree = await git(repo, "rev-parse", "HEAD^{tree}"); + const repository = await resolveRepository(repo); + const id = "a0000000-0000-4000-8000-000000000001"; + record = { + id, + name: "retired", + repoFingerprint: repository.fingerprint, + repoRoot: repository.repoRoot, + path: path.join(root, "retired"), + branch: "openclaw/retired", + baseRef: "main", + ownerKind: "session", + ownerId: "agent:main:test:snapshot-retirement", + createdAt: removedAt - 100, + lastActiveAt: removedAt - 1, + removedAt, + snapshotRef: `refs/openclaw/snapshots/${id}`, + }; + const snapshot = await git(repo, "commit-tree", tree, "-p", source, "-m", "removed snapshot"); + await git(repo, "update-ref", record.snapshotRef!, snapshot); + const retainedSourceRef = "refs/heads/retained-source"; + await git(repo, "update-ref", retainedSourceRef, source); + insertRegistryWorktree(env, record, { provisionedPaths: [] }); + const database = openOpenClawStateDatabase({ env }); + expect(await fs.realpath(database.path)).toBe(path.join(stateDir, "state", "openclaw.sqlite")); + request = { + id, + expectedSnapshotRef: record.snapshotRef!, + expectedSnapshotOid: snapshot, + expectedRemovedAt: removedAt, + retainedSourceRef, + expectedRetainedSourceOid: source, + }; + }); + + async function expectPreserved(expected: ManagedWorktreeRecord | undefined) { + expect(getRegistryWorktree(env, record.id)).toEqual(expected); + expect(await git(repo, "rev-parse", "--verify", record.snapshotRef!)).toBe( + request.expectedSnapshotOid, + ); + expect(await git(repo, "rev-parse", request.retainedSourceRef)).toBe(source); + } + + function retirementCli() { + const program = new Command().name("openclaw").exitOverride(); + program.configureOutput({ writeErr: () => undefined }); + registerWorktreesCli(program); + const output = vi.spyOn(defaultRuntime, "writeJson").mockImplementation(() => undefined); + const argv = [ + "node", + "openclaw", + "worktrees", + "retire-snapshot", + record.id, + "--expected-ref", + request.expectedSnapshotRef, + "--expected-oid", + request.expectedSnapshotOid, + "--removed-at", + String(request.expectedRemovedAt), + "--retained-ref", + request.retainedSourceRef, + "--retained-oid", + request.expectedRetainedSourceOid, + "--json", + ]; + return { program, output, argv }; + } + + it("retires through the Commander entrypoint and reports the exact disposition as JSON", async () => { + const cli = retirementCli(); + await cli.program.parseAsync(cli.argv); + expect(cli.output).toHaveBeenCalledExactlyOnceWith({ retired: true, id: record.id }); + expect(getRegistryWorktree(env, record.id)).toBeUndefined(); + await expect(git(repo, "show-ref", "--verify", record.snapshotRef!)).rejects.toThrow(); + expect(await git(repo, "rev-parse", request.retainedSourceRef)).toBe(source); + }); + + it.each(["wrong timestamp", "missing required argument"])( + "preserves snapshot custody through the CLI with %s", + async (kind) => { + const cli = retirementCli(); + if (kind === "wrong timestamp") { + cli.argv[cli.argv.indexOf("--removed-at") + 1] = String(removedAt + 1); + } else { + cli.argv.splice(cli.argv.indexOf("--retained-oid"), 2); + } + await expect(cli.program.parseAsync(cli.argv)).rejects.toThrow( + kind === "wrong timestamp" ? /snapshot identity does not match/ : /required option/, + ); + expect(cli.output).not.toHaveBeenCalled(); + await expectPreserved(record); + }, + ); + + it("retires only the exact redundant snapshot and record, preserving foreign recovery and PR outcomes", async () => { + const foreign: ManagedWorktreeRecord = { + ...record, + id: "a0000000-0000-4000-8000-000000000002", + name: "foreign", + path: path.join(root, "foreign"), + branch: "openclaw/foreign", + snapshotRef: "refs/openclaw/snapshots/a0000000-0000-4000-8000-000000000002", + }; + insertRegistryWorktree(env, foreign, { provisionedPaths: [] }); + await git(repo, "update-ref", foreign.snapshotRef!, source); + const outcome = "refs/openclaw/pr-merge-outcomes/123"; + await git(repo, "update-ref", outcome, source); + + await expect(retireManagedWorktreeSnapshotById(request)).resolves.toEqual({ + retired: true, + id: record.id, + }); + + expect(getRegistryWorktree(env, record.id)).toBeUndefined(); + await expect(git(repo, "show-ref", "--verify", record.snapshotRef!)).rejects.toThrow(); + await expect(fs.lstat(record.path)).rejects.toMatchObject({ code: "ENOENT" }); + expect(getRegistryWorktree(env, foreign.id)).toEqual(foreign); + expect(await git(repo, "rev-parse", foreign.snapshotRef!)).toBe(source); + expect(await git(repo, "rev-parse", outcome)).toBe(source); + expect(await git(repo, "rev-parse", request.retainedSourceRef)).toBe(source); + expect(await git(repo, "show", "HEAD:README.md")).toBe("base"); + }); + + it.each([ + ["ID", { id: "a0000000-0000-4000-8000-000000000099" }], + ["snapshot namespace", { expectedSnapshotRef: "refs/heads/main" }], + ["removal generation", { expectedRemovedAt: removedAt + 1 }], + ["snapshot OID", { expectedSnapshotOid: "1".repeat(40) }], + ["retained source OID", { expectedRetainedSourceOid: "2".repeat(40) }], + ] as const)("preserves custody when the expected %s does not match", async (_label, patch) => { + await expect(retireManagedWorktreeSnapshotById({ ...request, ...patch })).rejects.toThrow( + /snapshot identity does not match|ref OID changed/, + ); + await expectPreserved(record); + }); + + it("preserves exact-state recovery instead of treating it as a redundant ordinary snapshot", async () => { + const exactRef = `refs/openclaw/snapshots/exact-v1/${record.id}`; + await git(repo, "update-ref", exactRef, request.expectedSnapshotOid); + updateRegistryWorktree(env, record.id, { snapshotRef: exactRef }); + const exactRecord = getRegistryWorktree(env, record.id); + const recovery = path.join(root, "exact-recovery"); + await git(repo, "worktree", "add", "--detach", recovery, source); + await fs.writeFile(path.join(recovery, "newer.txt"), "write after capture"); + const registrations = await git(repo, "worktree", "list", "--porcelain"); + + await expect( + retireManagedWorktreeSnapshotById({ ...request, expectedSnapshotRef: exactRef }), + ).rejects.toThrow(/Invalid exact snapshot retirement identity/); + + expect(getRegistryWorktree(env, record.id)).toEqual(exactRecord); + expect(await git(repo, "rev-parse", exactRef)).toBe(request.expectedSnapshotOid); + expect(await fs.readFile(path.join(recovery, "newer.txt"), "utf8")).toBe("write after capture"); + expect(await git(repo, "worktree", "list", "--porcelain")).toBe(registrations); + expect(await git(repo, "rev-parse", request.retainedSourceRef)).toBe(source); + }); + + it("refuses a live registry lifecycle", async () => { + updateRegistryWorktree(env, record.id, { removedAt: undefined }); + const live = getRegistryWorktree(env, record.id); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /snapshot identity does not match/, + ); + await expectPreserved(live); + }); + + it("refuses changed repository identity", async () => { + updateRegistryWorktree(env, record.id, { + repositoryIdentity: { repoRoot: repo, repoFingerprint: "foreign-fingerprint" }, + }); + const changed = getRegistryWorktree(env, record.id); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /repository identity changed/, + ); + await expectPreserved(changed); + }); + + it.each(["directory", "dangling symlink"])("refuses a reappeared checkout %s", async (kind) => { + if (kind === "directory") { + await fs.mkdir(record.path); + await fs.writeFile(path.join(record.path, "newer.txt"), "newer source"); + } else { + await fs.symlink(path.join(root, "missing-target"), record.path); + } + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /checkout or Git registration/, + ); + await expectPreserved(record); + if (kind === "directory") { + expect(await fs.readFile(path.join(record.path, "newer.txt"), "utf8")).toBe("newer source"); + } else { + expect(await fs.readlink(record.path)).toBe(path.join(root, "missing-target")); + } + }); + + it("refuses a lingering Git registration even when the checkout path is absent", async () => { + await git(repo, "worktree", "add", "--detach", record.path, source); + await fs.rm(record.path, { recursive: true }); + const registered = await git(repo, "worktree", "list", "--porcelain"); + expect(registered).toContain(record.path); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /checkout or Git registration/, + ); + await expectPreserved(record); + expect(await git(repo, "worktree", "list", "--porcelain")).toBe(registered); + }); + + it.each(["live run", "unknown run", "removal"])("preserves a %s consumer", async (kind) => { + runOpenClawStateWriteTransaction( + ({ db }) => { + const query = getNodeSqliteKysely>(db); + executeSqliteQuerySync( + db, + query.insertInto("state_leases").values({ + scope: `worktree-run:${record.id}`, + lease_key: kind === "removal" ? "__removing__" : "retained-run", + owner: "synthetic-consumer", + expires_at: null, + heartbeat_at: null, + payload_json: JSON.stringify(kind === "unknown run" ? {} : { pid: process.pid }), + created_at: removedAt, + updated_at: removedAt, + }), + ); + }, + { env }, + ); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /run\/removal consumer/, + ); + await expectPreserved(record); + const { db } = openOpenClawStateDatabase({ env }); + expect( + executeSqliteQuerySync( + db, + getNodeSqliteKysely>(db) + .selectFrom("state_leases") + .select("owner") + .where("scope", "=", `worktree-run:${record.id}`), + ).rows, + ).toEqual([{ owner: "synthetic-consumer" }]); + }); + + it.each(["missing ledger", "unknown ledger", "provisioned ledger", "orphan chunk"])( + "preserves %s instead of assuming the Git tree contains all recovery data", + async (kind) => { + const chunk = { worktreeId: record.id, path: "local.env", chunkIndex: 0 }; + const bytes = Buffer.from("private provisioned content"); + if (kind === "missing ledger") { + runOpenClawStateWriteTransaction( + ({ db }) => { + executeSqliteQuerySync( + db, + getNodeSqliteKysely>(db) + .updateTable("worktrees") + .set({ provisioned_paths_json: null }) + .where("id", "=", record.id), + ); + }, + { env }, + ); + } else if (kind === "unknown ledger") { + // A legacy path-only ledger does not prove an empty captured-file inventory. + updateRegistryWorktree(env, record.id, { provisionedPaths: [chunk.path] }); + } else if (kind === "provisioned ledger") { + updateRegistryWorktree(env, record.id, { + provisionedState: [{ path: chunk.path, mode: 0o600, chunks: 1 }], + }); + insertRegistryWorktreeProvisionedChunk(env, { ...chunk, data: bytes }); + } else { + insertRegistryWorktreeProvisionedChunk(env, { ...chunk, data: bytes }); + } + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /retains provisioned data/, + ); + await expectPreserved(record); + if (kind === "provisioned ledger" || kind === "orphan chunk") { + expect(Buffer.from((await getRegistryWorktreeProvisionedChunk(env, chunk))!)).toEqual( + bytes, + ); + } + }, + ); + + it("refuses a pending-removal pin without deleting it", async () => { + const pending = `refs/openclaw/removals/${record.id}`; + await git(repo, "update-ref", pending, source); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /pending removal custody/, + ); + await expectPreserved(record); + expect(await git(repo, "rev-parse", pending)).toBe(source); + }); + + it("preserves projection custody and ignored bytes that Git equality cannot cover", async () => { + const projection = path.join( + env.OPENCLAW_STATE_DIR!, + "worktree-projections", + record.id, + "workspace", + ); + await fs.mkdir(projection, { recursive: true }); + const payload = path.join(projection, "ignored-owned.txt"); + await fs.writeFile(payload, "projection-only content"); + const store = localWorkspaceStore(env); + const row = store.create( + { + worktree_id: record.id, + agent_id: "main", + session_key: record.ownerId!, + session_id: "b0000000-0000-4000-8000-000000000001", + lifecycle_revision: null, + projection_path: projection, + base_commit: source, + source_paths_json: JSON.stringify(["README.md"]), + baseline_json: null, + baseline_ref: null, + pending_ref: null, + pending_target: null, + journal_json: null, + journal_pack: null, + paused_runtimes_json: null, + created_at_ms: removedAt - 1, + }, + () => undefined, + ); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow(/projection custody/); + await expectPreserved(record); + expect(store.get(record.id)).toEqual(row); + expect(await fs.readFile(payload, "utf8")).toBe("projection-only content"); + }); + + it("preserves a symbolic snapshot ref and its foreign target", async () => { + await git(repo, "update-ref", "-d", record.snapshotRef!); + await git(repo, "symbolic-ref", record.snapshotRef!, request.retainedSourceRef); + await expect( + retireManagedWorktreeSnapshotById({ ...request, expectedSnapshotOid: source }), + ).rejects.toThrow(/direct, not symbolic, refs/); + expect(getRegistryWorktree(env, record.id)).toEqual(record); + expect(await git(repo, "symbolic-ref", record.snapshotRef!)).toBe(request.retainedSourceRef); + expect(await git(repo, "rev-parse", request.retainedSourceRef)).toBe(source); + }); + + it("preserves a snapshot with source content absent from the retained ref", async () => { + await fs.writeFile(path.join(repo, "README.md"), "unique snapshot bytes"); + await git(repo, "add", "README.md"); + const uniqueTree = await git(repo, "write-tree"); + const uniqueSnapshot = await git(repo, "commit-tree", uniqueTree, "-p", source, "-m", "unique"); + await git(repo, "update-ref", record.snapshotRef!, uniqueSnapshot); + await expect( + retireManagedWorktreeSnapshotById({ ...request, expectedSnapshotOid: uniqueSnapshot }), + ).rejects.toThrow(/source not covered by the retained commit/); + expect(getRegistryWorktree(env, record.id)).toEqual(record); + expect(await git(repo, "show", `${record.snapshotRef}:README.md`)).toBe( + "unique snapshot bytes", + ); + }); + + it("preserves same-tree snapshots whose parent history is not retained", async () => { + const unrelated = await git(repo, "commit-tree", tree, "-m", "unrelated history"); + const snapshot = await git( + repo, + "commit-tree", + tree, + "-p", + unrelated, + "-m", + "unretained parent", + ); + await git(repo, "update-ref", record.snapshotRef!, snapshot); + await expect( + retireManagedWorktreeSnapshotById({ ...request, expectedSnapshotOid: snapshot }), + ).rejects.toThrow(/merge-base/); + expect(getRegistryWorktree(env, record.id)).toEqual(record); + expect(await git(repo, "rev-parse", record.snapshotRef!)).toBe(snapshot); + }); + + it.each(["snapshot", "retained source", "pending removal"])( + "rejects changed %s custody at the Git deletion boundary", + async (kind) => { + const replacement = await git(repo, "commit-tree", tree, "-p", source, "-m", "new snapshot"); + const target = + kind === "snapshot" + ? record.snapshotRef! + : kind === "retained source" + ? request.retainedSourceRef + : `refs/openclaw/removals/${record.id}`; + const realGit = worktreeGit.requireGit; + let replaced = false; + vi.spyOn(worktreeGit, "requireGit").mockImplementation(async (cwd, args, options) => { + if ( + !replaced && + args[0] === "update-ref" && + args.includes("--stdin") && + String(options?.input).includes(`delete ${record.snapshotRef} `) + ) { + replaced = true; + await git(repo, "update-ref", target, replacement); + } + return await realGit(cwd, args, options); + }); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /cannot lock ref|reference already exists/, + ); + expect(replaced).toBe(true); + expect(getRegistryWorktree(env, record.id)).toEqual(record); + expect(await git(repo, "rev-parse", target)).toBe(replacement); + expect(await git(repo, "rev-parse", record.snapshotRef!)).toBe( + kind === "snapshot" ? replacement : request.expectedSnapshotOid, + ); + expect(await git(repo, "rev-parse", request.retainedSourceRef)).toBe( + kind === "retained source" ? replacement : source, + ); + }, + ); + + it("preserves a newer registry lifecycle observed at the deletion boundary", async () => { + const realGit = worktreeGit.requireGit; + let changed = false; + vi.spyOn(worktreeGit, "requireGit").mockImplementation(async (cwd, args, options) => { + if ( + !changed && + args[0] === "update-ref" && + args.includes("--stdin") && + String(options?.input).includes(`delete ${record.snapshotRef} `) + ) { + changed = true; + updateRegistryWorktree(env, record.id, { removedAt: removedAt + 1 }); + } + return await realGit(cwd, args, options); + }); + await expect(retireManagedWorktreeSnapshotById(request)).rejects.toThrow( + /retirement identity changed/, + ); + expect(changed).toBe(true); + await expectPreserved({ ...record, removedAt: removedAt + 1 }); + }); + + it("refuses a retained-source symref retargeted to the retiring snapshot", async () => { + await git(repo, "update-ref", request.retainedSourceRef, request.expectedSnapshotOid); + const realGit = worktreeGit.requireGit; + let retargeted = false; + vi.spyOn(worktreeGit, "requireGit").mockImplementation(async (cwd, args, options) => { + if ( + !retargeted && + args[0] === "update-ref" && + args.includes("--stdin") && + String(options?.input).includes(`delete ${record.snapshotRef} `) + ) { + retargeted = true; + await git(repo, "symbolic-ref", request.retainedSourceRef, record.snapshotRef!); + } + return await realGit(cwd, args, options); + }); + + await expect( + retireManagedWorktreeSnapshotById({ + ...request, + expectedRetainedSourceOid: request.expectedSnapshotOid, + }), + ).rejects.toThrow(/multiple updates|cannot lock ref/); + + expect(retargeted).toBe(true); + expect(getRegistryWorktree(env, record.id)).toEqual(record); + expect(await git(repo, "rev-parse", "--verify", record.snapshotRef!)).toBe( + request.expectedSnapshotOid, + ); + expect(await git(repo, "symbolic-ref", request.retainedSourceRef)).toBe(record.snapshotRef); + expect(await git(repo, "rev-parse", "--verify", request.retainedSourceRef)).toBe( + request.expectedSnapshotOid, + ); + }); + + it("stops before deletion when caller authority is revoked after inspection", async () => { + const realGit = worktreeGit.requireGit; + let revoked = false; + vi.spyOn(worktreeGit, "requireGit").mockImplementation(async (cwd, args, options) => { + if ( + args[0] === "update-ref" && + args.includes("--stdin") && + String(options?.input).includes(`delete ${record.snapshotRef} `) + ) { + revoked = true; + } + return await realGit(cwd, args, options); + }); + await expect( + retireManagedWorktreeSnapshotById({ + ...request, + commitGuard: () => { + if (revoked) { + throw new Error("caller authority revoked"); + } + }, + }), + ).rejects.toThrow("caller authority revoked"); + expect(revoked).toBe(true); + await expectPreserved(record); + }); +}); diff --git a/src/agents/worktrees/snapshot-host.ts b/src/agents/worktrees/snapshot-host.ts index 0924f0801c15..7ba853ae4e4b 100644 --- a/src/agents/worktrees/snapshot-host.ts +++ b/src/agents/worktrees/snapshot-host.ts @@ -3,6 +3,7 @@ import { lstatSync } from "node:fs"; import path from "node:path"; import { resolveStateDir } from "../../config/paths.js"; import { runGitWorkerOperation } from "../../infra/git-worker.js"; +import { withWorktreeAllocationLease } from "./allocation.js"; import { requireWorktreeDiskSpace } from "./capacity.js"; import type { WorktreeGitPolicy } from "./checkout-git-config.js"; import { removeUnusedEmptyWorktreeSource } from "./empty-source.js"; @@ -10,15 +11,17 @@ import { requireGit, worktreePathExists, commandError, listGitWorktrees, runGit import { snapshotProvisionedFiles } from "./provisioned-files.js"; import { deleteRegistryWorktree, + assertRegistrySnapshotRetirement, assertWorktreeRemovalClaim, getRegistryWorktree, } from "./registry.js"; import { assertExactStateSourceIdentity } from "./removal-git.js"; import { abortWorktreeRemoval, claimWorktreeRemoval } from "./run-lease.js"; +import { resolveRepository } from "./service-preparation.js"; import type { ExactStateRetirement } from "./snapshot-exact-state-contract.js"; import { readExactStateSnapshot } from "./snapshot-exact-state.js"; import { clearExactRestoreReceipt, readExactRestoreReceipt } from "./snapshot-restore-exact.js"; -import type { ManagedWorktreeRecord } from "./types.js"; +import type { ManagedWorktreeRecord, RetireManagedWorktreeSnapshotParams } from "./types.js"; /** Existing snapshot worker and effect owners, supplied with this operation's Git policy. */ export async function captureManagedWorktreeSnapshot(params: { @@ -158,14 +161,194 @@ export function assertExactSnapshotRecordCurrent( } } +/** Local CLI retirement shares the same allocation owner as removal and GC. */ +export async function retireManagedWorktreeSnapshotById( + params: RetireManagedWorktreeSnapshotParams, + env: NodeJS.ProcessEnv = process.env, +) { + return await withWorktreeAllocationLease({ ...params, env }, async (guard) => { + const record = getRegistryWorktree(env, params.id); + if ( + !record || + record.removedAt === undefined || + record.removedAt !== params.expectedRemovedAt || + record.snapshotRef !== params.expectedSnapshotRef + ) { + throw new Error("Expected removed worktree snapshot identity does not match"); + } + await retireManagedWorktreeSnapshot({ + record, + env, + signal: guard.signal, + assertCurrent: () => guard.commitGuard(), + expected: params, + }); + return { retired: true as const, id: record.id }; + }); +} + +/** Preparation remains under the allocation owner; Git commits exact ref custody atomically. */ +async function prepareExactSnapshotRetirement(params: { + record: ManagedWorktreeRecord; + expected: RetireManagedWorktreeSnapshotParams; + signal?: AbortSignal; + assertCurrent: () => void; +}) { + const { record, expected, signal, assertCurrent } = params; + const ref = `refs/openclaw/snapshots/${record.id}`; + if ( + expected.id !== record.id || + expected.expectedSnapshotRef !== ref || + record.snapshotRef !== ref || + record.removedAt !== expected.expectedRemovedAt || + !Number.isSafeInteger(expected.expectedRemovedAt) || + expected.expectedRemovedAt < 0 || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(expected.expectedSnapshotOid) || + !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/u.test(expected.expectedRetainedSourceOid) || + !/^refs\/(?:heads|remotes)\//u.test(expected.retainedSourceRef) + ) { + throw new Error("Invalid exact snapshot retirement identity or retained source ref"); + } + const options = { + signal, + beforeRun: assertCurrent, + env: { GIT_NO_LAZY_FETCH: "1", GIT_NO_REPLACE_OBJECTS: "1", GIT_OPTIONAL_LOCKS: "0" }, + }; + await requireGit(record.repoRoot, ["check-ref-format", ref], options); + await requireGit(record.repoRoot, ["check-ref-format", expected.retainedSourceRef], options); + const repository = await resolveRepository(record.repoRoot); + assertCurrent(); + if ( + repository.repoRoot !== record.repoRoot || + repository.fingerprint !== record.repoFingerprint + ) { + throw new Error("Worktree snapshot repository identity changed"); + } + if ( + (await worktreePathExists(record.path)) || + (await listGitWorktrees(record.repoRoot, options)).some( + (entry) => path.resolve(entry.path) === path.resolve(record.path), + ) + ) { + throw new Error("Worktree snapshot still has a checkout or Git registration"); + } + for (const [reference, oid] of [ + [ref, expected.expectedSnapshotOid], + [expected.retainedSourceRef, expected.expectedRetainedSourceOid], + ] as const) { + const symbolic = await runGit(record.repoRoot, ["symbolic-ref", "--quiet", reference], options); + if (symbolic.code !== 1) { + throw new Error("Snapshot retirement requires direct, not symbolic, refs"); + } + if ( + (await requireGit( + record.repoRoot, + ["show-ref", "--verify", "--hash", reference], + options, + )) !== oid + ) { + throw new Error("Snapshot retirement ref OID changed"); + } + } + const pendingRef = `refs/openclaw/removals/${record.id}`; + const pending = await runGit( + record.repoRoot, + ["show-ref", "--verify", "--quiet", pendingRef], + options, + ); + if (pending.code !== 1) { + throw new Error("Snapshot retirement has pending removal custody"); + } + const snapshotTree = await requireGit( + record.repoRoot, + ["rev-parse", `${expected.expectedSnapshotOid}^{tree}`], + options, + ); + const retainedTree = await requireGit( + record.repoRoot, + ["rev-parse", `${expected.expectedRetainedSourceOid}^{tree}`], + options, + ); + if (snapshotTree !== retainedTree) { + throw new Error("Snapshot contains source not covered by the retained commit"); + } + const parents = ( + await requireGit( + record.repoRoot, + ["rev-list", "--parents", "-n", "1", expected.expectedSnapshotOid], + options, + ) + ).split(" "); + if (parents.length !== 2) { + throw new Error("Snapshot parent custody is unavailable"); + } + await requireGit( + record.repoRoot, + ["merge-base", "--is-ancestor", parents[1]!, expected.expectedRetainedSourceOid], + options, + ); + assertCurrent(); + return async (assertProjectionCurrent: () => void) => { + const beforeRun = () => { + assertCurrent(); + assertProjectionCurrent(); + }; + if ( + (await worktreePathExists(record.path)) || + (await listGitWorktrees(record.repoRoot, { ...options, beforeRun })).some( + (entry) => path.resolve(entry.path) === path.resolve(record.path), + ) + ) { + throw new Error("Worktree snapshot checkout or Git registration reappeared"); + } + // One transaction verifies the retained source and absent removal pin while + // deleting only the observed snapshot. Verification must lock the retained + // ref chain; no-deref is scoped to the pending pin and snapshot deletion. + await requireGit(record.repoRoot, ["update-ref", "--stdin"], { + ...options, + beforeRun, + input: `start\nverify ${expected.retainedSourceRef} ${expected.expectedRetainedSourceOid}\noption no-deref\nverify ${pendingRef} ${"0".repeat(expected.expectedSnapshotOid.length)}\noption no-deref\ndelete ${ref} ${expected.expectedSnapshotOid}\nprepare\ncommit\n`, + }); + }; +} + /** Retire the restore entry point before releasing accepted projection custody. */ export async function retireManagedWorktreeSnapshot(params: { record: ManagedWorktreeRecord; env: NodeJS.ProcessEnv; signal?: AbortSignal; assertCurrent: () => void; + expected?: RetireManagedWorktreeSnapshotParams; }) { const { record, env, signal } = params; + if (params.expected) { + const { localWorkspaceStore } = + await import("../../gateway/worker-environments/local-workspace-store.js"); + const projectionStore = localWorkspaceStore(env); + const assertCurrent = () => { + params.assertCurrent(); + assertRegistrySnapshotRetirement(env, record); + if (projectionStore.get(record.id)) { + throw new Error( + "Snapshot retains local workspace projection custody; preserve its recovery data", + ); + } + }; + assertCurrent(); + const retireSnapshot = await prepareExactSnapshotRetirement({ + record, + expected: params.expected, + signal, + assertCurrent, + }); + const { expireLocalWorkspaceProjection } = + await import("../../gateway/worker-environments/local-workspace-projection.js"); + await expireLocalWorkspaceProjection({ worktree: record, env, assertCurrent, retireSnapshot }); + assertCurrent(); + deleteRegistryWorktree(env, record.id, { assertCurrent, expectedRetired: record }); + // Retained source refs remain owned, including an otherwise empty source repository. + return; + } const exactRecord = record.snapshotRef?.startsWith("refs/openclaw/snapshots/exact-"); const expirationToken = exactRecord ? randomUUID() : undefined; let claimed = false; diff --git a/src/agents/worktrees/types.ts b/src/agents/worktrees/types.ts index a40396b3eaa8..a4b5ff4f0bdd 100644 --- a/src/agents/worktrees/types.ts +++ b/src/agents/worktrees/types.ts @@ -129,3 +129,15 @@ export type ManagedWorktreeGcResult = { /** Null when incomplete inventory or size measurements prevent a conclusion. */ limitsSatisfied: boolean | null; }; + +/** Explicit early retirement only for a snapshot whose source remains retained. */ +export type RetireManagedWorktreeSnapshotParams = { + id: string; + expectedSnapshotRef: string; + expectedSnapshotOid: string; + expectedRemovedAt: number; + retainedSourceRef: string; + expectedRetainedSourceOid: string; + signal?: AbortSignal; + commitGuard?: () => void; +}; diff --git a/src/cli/worktrees-cli.ts b/src/cli/worktrees-cli.ts index c7db2f82cc5f..1028c6db80ca 100644 --- a/src/cli/worktrees-cli.ts +++ b/src/cli/worktrees-cli.ts @@ -160,6 +160,45 @@ export function registerWorktreesCli(program: Command): void { }, ); + worktrees + .command("retire-snapshot") + .description("Retire one removed snapshot whose source is retained elsewhere") + .argument("", "Removed managed worktree id") + .requiredOption("--expected-ref ", "Exact snapshot ref") + .requiredOption("--expected-oid ", "Exact snapshot commit") + .requiredOption("--removed-at ", "Exact recorded removal time") + .requiredOption("--retained-ref ", "Retained branch or remote-tracking source ref") + .requiredOption("--retained-oid ", "Exact retained source commit") + .option("--json", "Output JSON", false) + .action( + async ( + id: string, + opts: JsonOption & { + expectedRef: string; + expectedOid: string; + removedAt: string; + retainedRef: string; + retainedOid: string; + }, + ) => { + const { retireManagedWorktreeSnapshotById } = + await import("../agents/worktrees/snapshot-host.js"); + const result = await retireManagedWorktreeSnapshotById({ + id, + expectedSnapshotRef: opts.expectedRef, + expectedSnapshotOid: opts.expectedOid, + expectedRemovedAt: Number(opts.removedAt), + retainedSourceRef: opts.retainedRef, + expectedRetainedSourceOid: opts.retainedOid, + }); + if (opts.json) { + printJson(result); + } else { + defaultRuntime.log(`Retired snapshot ${id}.`); + } + }, + ); + worktrees .command("restore") .description("Restore a managed worktree from its snapshot") diff --git a/test/plugins/session-share.integration.test.ts b/test/plugins/session-share.integration.test.ts index f3beb98e7f1d..0f30d525ab6f 100644 --- a/test/plugins/session-share.integration.test.ts +++ b/test/plugins/session-share.integration.test.ts @@ -1,7 +1,10 @@ import fs from "node:fs"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { validateJsonSchemaValue } from "openclaw/plugin-sdk/json-schema-runtime"; -import type { OpenClawPluginNodeHostCommand } from "openclaw/plugin-sdk/plugin-entry"; +import type { + OpenClawPluginNodeHostCommand, + OpenClawPluginService, +} from "openclaw/plugin-sdk/plugin-entry"; import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime"; import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api"; import { createPluginRuntimeMock } from "openclaw/plugin-sdk/plugin-test-runtime"; @@ -37,23 +40,26 @@ afterEach(() => vi.restoreAllMocks()); function registerSessionShare(runtime: PluginRuntime, config: OpenClawConfig = {}) { const nodeCommands: OpenClawPluginNodeHostCommand[] = []; const catalogs: SessionCatalogProvider[] = []; - sessionSharePlugin.register( - createTestPluginApi({ - runtime, - config, - registerNodeHostCommand: (command) => { - nodeCommands.push(command); - }, - registerSessionCatalog: (catalog) => { - catalogs.push(catalog); - }, - }), - ); + const services: OpenClawPluginService[] = []; + const api = createTestPluginApi({ + runtime, + config, + registerNodeHostCommand: (command) => { + nodeCommands.push(command); + }, + registerSessionCatalog: (catalog) => { + catalogs.push(catalog); + }, + registerService: (service) => { + services.push(service); + }, + }); + sessionSharePlugin.register(api); const catalog = catalogs.find((entry) => entry.id === "openclaw"); if (!catalog) { throw new Error("Session Share did not register its catalog"); } - return { commands: nodeCommands, catalog }; + return { commands: nodeCommands, catalog, services, logger: api.logger }; } type SessionPage = { sessions: SessionCatalogSession[]; nextCursor?: string }; @@ -100,7 +106,7 @@ const remoteIdentity = { id: "4242", }; -function catalogFixture() { +function catalogFixture(stateDir: string) { let config: OpenClawConfig = {}; const list = vi.fn().mockResolvedValue({ nodes: [{ nodeId: "alpha", displayName: " Alpha ", connected: true, commands }], @@ -121,22 +127,51 @@ function catalogFixture() { config: { current: () => config }, nodes: { list, invoke }, }); - const catalog = registerSessionShare(runtime).catalog; + const { catalog, services, logger } = registerSessionShare(runtime); + const serviceContext = { config, stateDir, logger, invokeNode: invoke }; return { catalog, list, invoke, + start: async () => { + await Promise.all( + services.map(async (service) => { + await service.start(serviceContext); + }), + ); + }, + stop: async () => { + await Promise.all( + services.map(async (service) => { + await service.stop?.(serviceContext); + }), + ); + }, setConfig: (next: OpenClawConfig) => { config = next; }, }; } +async function withCatalogFixture( + run: (fixture: ReturnType) => Promise, +) { + await withOpenClawTestState({ scenario: "minimal" }, async (state) => { + const fixture = catalogFixture(state.stateDir); + try { + await fixture.start(); + await run(fixture); + } finally { + // Stop all publishers before the fixture closes its private databases and environment. + await fixture.stop(); + } + }); +} + describe("session-share node commands", () => { it("derives titles only for the requested page while preserving transcript-title search", async () => { - await withOpenClawTestState({ scenario: "minimal" }, async () => { + await withCatalogFixture(async (receiver) => { const source = commandFixture(); - const receiver = catalogFixture(); receiver.invoke.mockImplementation(async ({ command, params }) => { const handler = source.commands.find((candidate) => candidate.command === command)!; return { payloadJSON: await handler.handle(JSON.stringify(params)) }; @@ -736,12 +771,11 @@ describe("session-share receiver identity integration", () => { it.each(["alpha", "beta"])( "keeps %s claims remote by default and applies only explicit owner and numeric GitHub links", async (nodeId) => { - await withOpenClawTestState({ scenario: "minimal" }, async () => { + await withCatalogFixture(async (fixture) => { const profile = syncGitHubIdentity({ identity: { accountId: 4242, login: "catalog-person", name: "Catalog Person" }, authenticationAlias: { kind: "github-login", login: "catalog-person" }, }); - const fixture = catalogFixture(); fixture.list.mockResolvedValue({ nodes: [{ nodeId, connected: true, commands }] }); const hostId = `node:${nodeId}`; const namespacedIdentity = { ...remoteIdentity, domain: hostId }; diff --git a/test/vitest/vitest.database-worker-core-paths.mjs b/test/vitest/vitest.database-worker-core-paths.mjs index f6df06362704..59fcb2754a91 100644 --- a/test/vitest/vitest.database-worker-core-paths.mjs +++ b/test/vitest/vitest.database-worker-core-paths.mjs @@ -289,6 +289,7 @@ export const databaseWorkerCoreTestFiles = [ "src/agents/worktrees/service.refs.test.ts", "src/agents/worktrees/service.removal-safety.test.ts", "src/agents/worktrees/service.remove-lease.test.ts", + "src/agents/worktrees/service.retire-snapshot.test.ts", "src/agents/worktrees/service.run-end-cleanup.test.ts", "src/agents/worktrees/service.snapshot-index.test.ts", "src/agents/worktrees/service.exact-state.test.ts",