fix(crabbox): finish destroying workers when leases are absent (#160571)

Cloud worker environments could remain in destroying forever after their fixed Crabbox lease was never admitted. Share lease absence classification between inspect and stop, requiring a normal nonzero exit and matching coordinator read/release 404/not_found diagnostics without auth wording, or recognized direct-provider exit-4 absence.

Confirmed absence follows normal heartbeat settlement and warm-image allocation release. After a Gateway update, affected environments complete cleanup on the next reconcile without a state migration. Ambiguous output and failed releases remain errors.

Validation: 869 Crabbox tests passed, 13 existing Linux desktop tests skipped on macOS; extension production/test typechecks, targeted lint and formatting passed; independent Codex P2 review found no actionable issues. Live coordinator reconciliation remains a separate dev Gateway restart verification.

docs/gateway/cloud-workers/troubleshooting.md

@@ -24,6 +24,7 @@ Symptoms you may see when dispatching to or running on a cloud worker, and the c
 - **Archive fails** — the Control UI hides the session immediately and restores it only when the archive itself is rejected. Active session work must finish stopping; repeated requests cannot replace an operation already stopping that work. An already-failed worker can be archived while provider cleanup remains pending, with its worktree and recovery records preserved. Worktree cleanup errors after the archive is saved are logged for later cleanup. A client timeout does not cancel an accepted request or prove its outcome; check the session's current archive state before retrying.
 - **Crabbox setup cannot reach the lease** — check the selected backend's networking and setup-transport requirements in the [Crabbox provider reference](https://crabbox.sh/providers/index.html). Correct Crabbox's configuration and rerun `crabbox doctor --provider <backend> --json` before retrying.
 - **Crabbox inspection reports coordinator read retries** — Crabbox owns read retries within a one-minute budget. OpenClaw allows two minutes per lease inspection, including one minute for process startup and exit on a loaded host; Machine0 retains five minutes for readiness. Provisioning reads also respect the remaining overall deadline. If inspection still fails, check coordinator availability and the lease with `crabbox inspect --provider <backend> --id <lease> --json` before retrying dispatch.
+- **Crabbox worker stays in `destroying` after its lease was never admitted** — update the Gateway. On the next reconciliation, a normally exited stop reporting `404/not_found` for both the coordinator lease read and release completes teardown and releases warm-image allocation ownership. Both responses must name that lease. Recognized direct-provider exit-4 absence also completes teardown; authentication errors, timeouts, incomplete output, and failed releases remain errors.
 - **Session shows a reclaimed or suspended badge after being idle** — this is expected when its profile sets `suspendAfter`. The next message provisions a replacement worker, warm when an image exists.
 - **A warm image is unavailable** — a new allocation can select cold provisioning before its choice is recorded. An already admitted allocation keeps its original cold/checkpoint choice through retries. If its checkpoint cannot be forked, resolve the provider error or stop that allocation before starting a replacement; retry does not switch images silently.
 - **Warm-image migration or capacity blocks dispatch** — run `openclaw doctor --fix` for legacy state and follow its exact cleanup guidance. For capacity, stop outstanding workers or resolve pending image cleanup with `openclaw crabbox warm-images`; allocation choices and cleanup obligations are never evicted to make room.

extensions/crabbox/src/crabbox-worker-command.test.ts

@@ -0,0 +1,145 @@
+import type { SpawnResult } from "openclaw/plugin-sdk/process-runtime";
+import { describe, expect, it, vi } from "vitest";
+import { isUnrecognizedLease, stopCrabboxLease } from "./crabbox-worker-command.js";
+
+const LEASE_ID = "cbx_absent_fixture";
+const readError = `coordinator GET /v1/leases/${LEASE_ID}: http 404: {"error":"not_found"}`;
+const releaseError = `coordinator POST /v1/leases/${LEASE_ID}/release: http 404: {"error":"not_found"}`;
+const absentOutput = `warning: could not inspect lease before release: ${readError}\n${releaseError}`;
+const absentResult: SpawnResult = {
+  stdout: "",
+  stderr: absentOutput,
+  code: 1,
+  signal: null,
+  killed: false,
+  termination: "exit",
+};
+
+describe("Crabbox lease absence classification", () => {
+  it.each<{
+    name: string;
+    result?: Partial<SpawnResult>;
+    absent: boolean;
+    inspect?: boolean;
+  }>([
+    { name: "matching coordinator read and release 404/not_found", absent: true },
+    {
+      name: "absolute coordinator URLs",
+      result: { stderr: absentOutput.replaceAll("/v1/", "https://coordinator.example/v1/") },
+      absent: true,
+    },
+    {
+      name: "diagnostics split across streams",
+      result: { stderr: `${readError}\n`, stdout: releaseError },
+      absent: true,
+    },
+    {
+      name: "release 503 after a read 404",
+      result: { stderr: `${readError}\n${releaseError.replace("404", "503")}` },
+      absent: false,
+      inspect: true,
+    },
+    {
+      name: "read-only 404",
+      result: { stderr: readError },
+      absent: false,
+      inspect: true,
+    },
+    {
+      name: "release-only 404",
+      result: { stderr: releaseError },
+      absent: false,
+    },
+    {
+      name: "truncated release body",
+      result: { stderr: absentOutput.slice(0, -2) },
+      absent: false,
+      inspect: true,
+    },
+    {
+      name: "unexplained 404 response",
+      result: { stderr: absentOutput.replaceAll('"not_found"', '"route_missing"') },
+      absent: false,
+      inspect: true,
+    },
+    {
+      name: "missing requested identifier",
+      result: { stderr: absentOutput.replaceAll(LEASE_ID, "cbx_other") },
+      absent: false,
+    },
+    {
+      name: "release names a different lease",
+      result: { stderr: `${readError}\n${releaseError.replace(LEASE_ID, "cbx_other")}` },
+      absent: false,
+      inspect: true,
+    },
+    {
+      name: "requested identifier is only a prefix",
+      result: { stderr: absentOutput.replaceAll(LEASE_ID, `${LEASE_ID}_other`) },
+      absent: false,
+    },
+    {
+      name: "timeout despite complete absence output",
+      result: { termination: "timeout", code: null, killed: true },
+      absent: false,
+    },
+    {
+      name: "signal despite complete absence output",
+      result: { termination: "signal", code: null, signal: "SIGTERM" },
+      absent: false,
+    },
+    {
+      name: "unknown exit code",
+      result: { code: null },
+      absent: false,
+    },
+    ...["auth", "authentication", "authorization", "credentials", "permission", "token"].map(
+      (word) => ({
+        name: `${word} diagnostic`,
+        result: { stdout: `${word} failure for ${LEASE_ID}` },
+        absent: false,
+      }),
+    ),
+    ...[
+      `lease/server not found: ${LEASE_ID}`,
+      `unikraftcloud lease ${LEASE_ID} no longer exists`,
+      `unknown lease: ${LEASE_ID}`,
+    ].map((stderr) => ({ name: stderr, result: { code: 4, stderr }, absent: true })),
+    {
+      name: "direct provider absence with a different exit code",
+      result: { code: 1, stderr: `lease/server not found: ${LEASE_ID}` },
+      absent: false,
+    },
+    {
+      name: "coder recognition alone does not confirm stop",
+      result: { code: 5, stderr: `coder workspace "${LEASE_ID}" not found` },
+      absent: false,
+      inspect: true,
+    },
+  ])("$name", async ({ result: overrides, absent, inspect }) => {
+    const result = { ...absentResult, ...overrides };
+    expect(isUnrecognizedLease(result, LEASE_ID, "inspect")).toBe(inspect ?? absent);
+    expect(isUnrecognizedLease(result, LEASE_ID, "stop")).toBe(absent);
+    const warn = vi.fn();
+    const stop = stopCrabboxLease({
+      binary: "crabbox",
+      id: LEASE_ID,
+      provider: "hetzner",
+      runCommand: async () => result,
+      warn,
+    });
+    if (absent) {
+      await expect(stop).resolves.toBeUndefined();
+      expect(warn).toHaveBeenCalledExactlyOnceWith(
+        `Crabbox lease ${LEASE_ID} (provider hetzner) is absent; treating stop as already released`,
+      );
+    } else {
+      await expect(stop).rejects.toThrow(
+        result.termination === "exit"
+          ? `Crabbox stop failed with exit code ${result.code ?? "unknown"}`
+          : `Crabbox stop did not exit normally (${result.termination})`,
+      );
+      expect(warn).not.toHaveBeenCalled();
+    }
+  });
+});

extensions/crabbox/src/crabbox-worker-command.ts

@@ -1,6 +1,6 @@
 import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core";
 import type { SpawnResult } from "openclaw/plugin-sdk/process-runtime";
-import { sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
+import { escapeRegExp, sliceUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime";
 import { CRABBOX_STOP_TIMEOUT_MS } from "./crabbox-worker-timeouts.js";
 
 const MAX_OUTPUT_BYTES = 64 * 1024;
@@ -114,17 +114,56 @@ export function parseCrabboxJson(stdout: string, action: string): unknown {
   }
 }
 
-// Recognition failure does not prove resource absence; only the stop owner can confirm cleanup.
-export function isUnrecognizedLease(result: SpawnResult, identifier: string): boolean {
+export function isUnrecognizedLease(
+  result: SpawnResult,
+  identifier: string,
+  action: "inspect" | "stop",
+): boolean {
   const output = `${result.stderr}\n${result.stdout}`;
   if (
-    !output.includes(identifier) ||
-    /\b(?:access\s+denied|authentication|authorization|credentials?|forbidden|permission|token|unauthorized)\b/iu.test(
+    result.termination !== "exit" ||
+    result.code === null ||
+    result.code === 0 ||
+    !new RegExp(`(?:^|[^\\w-])${escapeRegExp(identifier)}(?=$|[^\\w-])`, "u").test(output) ||
+    /\b(?:access\s+denied|auth|authentication|authorization|credentials?|forbidden|permission|token|unauthorized)\b/iu.test(
       output,
     )
   ) {
     return false;
   }
+  if (/\bcoordinator\b/iu.test(output)) {
+    const responses = output
+      .trim()
+      .split(/[\r\n]+/u)
+      .map((line) =>
+        line.match(
+          /^(?:warning: could not inspect lease before release: )?coordinator (GET|POST) (?:https?:\/\/[^/\s]+)?\/v1\/leases\/([^/:\s]+)(\/release)?:[ \t]*http (\d{3})\b([^\r\n]*)$/iu,
+        ),
+      );
+    const hasRead = responses.some(
+      (response) =>
+        response?.[1] === "GET" &&
+        response[2] === identifier &&
+        !response[3] &&
+        response[4] === "404",
+    );
+    if (action === "inspect") {
+      return hasRead;
+    }
+    // A missing read alone cannot attest release. Accept only complete, matching
+    // read/release not_found diagnostics, with no other failure output.
+    return (
+      hasRead &&
+      responses.some((response) => response?.[1] === "POST" && response[3] === "/release") &&
+      responses.every(
+        (response) =>
+          response?.[2] === identifier &&
+          response[4] === "404" &&
+          (response[1] === "GET" ? !response[3] : response[3] === "/release") &&
+          /^:\s*(?:not_found|\{\s*"error"\s*:\s*"not_found"\s*\})\s*$/u.test(response[5] ?? ""),
+      )
+    );
+  }
   return (
     (result.code === 4 &&
       (/\b(?:was\s+)?not found\b/iu.test(output) ||
@@ -135,8 +174,9 @@ export function isUnrecognizedLease(result: SpawnResult, identifier: string): bo
         /\bis not claimed by Crabbox\b/iu.test(output) ||
         /\bwandb sandbox "[^"\r\n]+" has no matching local ownership claim\b/iu.test(output) ||
         /\bunknown lease(?:\s|:)/iu.test(output))) ||
-    (result.code === 5 && /\bcoder workspace "[^"\r\n]+" not found\b/iu.test(output)) ||
-    /\bcoordinator GET \S*\/v1\/leases\/\S+:\s*http 404\b/iu.test(output)
+    (action === "inspect" &&
+      result.code === 5 &&
+      /\bcoder workspace "[^"\r\n]+" not found\b/iu.test(output))
   );
 }
 
@@ -145,6 +185,7 @@ export async function stopCrabboxLease(params: {
   id: string;
   provider: string;
   runCommand: CrabboxCommandRunner;
+  warn: (message: string) => void;
 }): Promise<void> {
   const result = await runCrabboxCommand({
     action: "stop",
@@ -153,5 +194,11 @@ export async function stopCrabboxLease(params: {
     runCommand: params.runCommand,
     timeoutMs: CRABBOX_STOP_TIMEOUT_MS,
   });
+  if (isUnrecognizedLease(result, params.id, "stop")) {
+    params.warn(
+      `Crabbox lease ${params.id} (provider ${params.provider}) is absent; treating stop as already released`,
+    );
+    return;
+  }
   crabboxCommandOutput("stop", result);
 }

extensions/crabbox/src/crabbox-worker-provider.ts

@@ -169,10 +169,11 @@ export function createCrabboxWorkerProvider(
   });
   const stopLease = async (context: LeaseCommandContext): Promise<void> => {
     await heartbeats.stop(context.id);
-    // Cleanup has its own deadline. Only confirmed stop releases allocation/image ownership.
+    // Cleanup has its own deadline. Confirmed stop or absence releases allocation/image ownership.
     await stopCrabboxLease({
       ...context,
       runCommand,
+      warn,
     });
     await warmImages.release(context);
   };

extensions/crabbox/src/crabbox-worker-provision-commands.ts

@@ -121,7 +121,7 @@ export async function inspectWithContext(params: {
     }
     return { status: "found", inspect };
   }
-  if (result.termination === "exit" && isUnrecognizedLease(result, params.id)) {
+  if (isUnrecognizedLease(result, params.id, "inspect")) {
     return { status: "unknown" };
   }
   throw crabboxCommandError(action, result);

extensions/crabbox/src/crabbox-worker-stop.test.ts

@@ -1,6 +1,12 @@
-import { describe, expect, it, vi } from "vitest";
+import { describe, expect, it } from "vitest";
+import { openWarmImageStore } from "./crabbox-state.test-support.js";
 import { commandResult } from "./crabbox-worker-provider.test-support.js";
-import { createWarmProvider, LEASE_ID, PROFILE } from "./crabbox-worker-warm-image.test-support.js";
+import {
+  createWarmProvider,
+  LEASE_ID,
+  PROFILE,
+  provisionWarmProfile,
+} from "./crabbox-worker-warm-image.test-support.js";
 
 const lease = { leaseId: LEASE_ID, profile: { ...PROFILE, warmImage: false } };
 
@@ -10,7 +16,6 @@ describe("Crabbox worker stop confirmation", () => {
       code: 5,
       stderr: `warning: could not inspect lease before release: coordinator GET http://127.0.0.1/v1/leases/${LEASE_ID}: http 404: not_found\ncoordinator accepted release for ${LEASE_ID}, but remote cleanup reported a cleanup failure or scheduled retry`,
     },
-    { code: 4, stderr: `lease/server not found: ${LEASE_ID}` },
     { code: 4, stderr: `lease ${LEASE_ID} already stopped` },
   ])("rejects unproven stop despite misleading prose: $stderr", async ({ code, stderr }) => {
     const { provider, calls } = createWarmProvider(() => commandResult({ code, stderr }));
@@ -20,21 +25,41 @@ describe("Crabbox worker stop confirmation", () => {
     ]);
   });
 
-  it("keeps a dual-404 stop outcome unknown without a structured absence receipt", async () => {
-    const runCommand = vi.fn(async () =>
-      commandResult({
-        code: 1,
-        stderr:
-          `warning: could not inspect lease before release: coordinator GET /v1/leases/${LEASE_ID}: http 404: {"error":"not_found"}\n` +
-          `coordinator POST /v1/leases/${LEASE_ID}/release: http 404: {"error":"not_found"}`,
-      }),
+  it("completes destroy and releases warm allocation ownership for a never-admitted lease", async () => {
+    const inspectError = `coordinator GET /v1/leases/${LEASE_ID}: http 404: {"error":"not_found"}`;
+    const { provider, calls, warn } = createWarmProvider(({ argv }) => {
+      if (argv[1] === "warmup") {
+        return commandResult({ code: 1, stderr: "allocation not admitted" });
+      }
+      if (argv[1] === "inspect") {
+        return commandResult({ code: 1, stderr: inspectError });
+      }
+      if (argv[1] === "stop") {
+        // Crabbox 0.67.0 output, with the captured lease ID replaced by the fixture's ID.
+        return commandResult({
+          code: 1,
+          stderr:
+            `warning: could not inspect lease before release: ${inspectError}\n` +
+            `coordinator POST /v1/leases/${LEASE_ID}/release: http 404: {"error":"not_found"}`,
+        });
+      }
+      return undefined;
+    });
+    await expect(provisionWarmProfile(provider)).rejects.toThrow("warmup failed");
+    const store = openWarmImageStore();
+    const owner = store.entries()[0]!;
+    expect(owner.value.allocations[LEASE_ID]).toBeDefined();
+    await expect(provider.inspect(lease)).resolves.toEqual({ status: "unknown" });
+    await expect(provider.destroy({ ...lease, profile: PROFILE })).resolves.toBeUndefined();
+    expect(store.lookup(owner.key)?.allocations[LEASE_ID]).toBeUndefined();
+    expect(calls.filter(({ argv }) => argv[1] === "stop")).toHaveLength(1);
+    expect(calls.some(({ argv }) => argv[1] === "heartbeat")).toBe(false);
+    expect(warn).toHaveBeenCalledExactlyOnceWith(
+      `Crabbox lease ${LEASE_ID} (provider aws) is absent; treating stop as already released`,
     );
-    const { provider } = createWarmProvider(runCommand);
-    await expect(provider.destroy(lease)).rejects.toThrow("stop failed with exit code 1");
-    expect(runCommand).toHaveBeenCalledOnce();
   });
 
-  it("accepts producer-confirmed absence only after a normal successful stop", async () => {
+  it("accepts a normal successful stop repeatedly", async () => {
     const { provider, calls } = createWarmProvider(() => commandResult());
     await expect(provider.destroy(lease)).resolves.toBeUndefined();
     await expect(provider.destroy(lease)).resolves.toBeUndefined();

extensions/crabbox/src/crabbox-worker-warm-image.ts

@@ -578,7 +578,7 @@ export function createCrabboxWarmImageManager(dependencies: {
       openStore().notePreparedDemand(id, preparation),
 
     async release(context: LeaseContext) {
-      // Only confirmed stop releases this hold: enrollment success may itself be a lost response,
+      // Only confirmed stop or absence releases this hold: enrollment success may be a lost response,
       // and replay still needs the original checkpoint catalog entry and native artifact.
       const owner = await lookupLease(context.id);
       if (!owner) {