fix(update): bypass managed proxy for canary readiness (#147941)

* fix(update): bypass managed proxy for canary readiness

Register exact readiness URLs with the existing Gateway loopback bypass and preserve proxy/block policy. Record exhausted probe outcomes as actionable candidate advisories so healthy update preparation does not fall through to inference repair. Reported by @yoyo837 (#147860).

* chore: refresh canary readiness CI against current main

* chore: refresh canary readiness CI after main repair

docs/security/network-proxy.md

@@ -102,6 +102,10 @@ without activating it.
 
 Gateway control-plane bypass is limited to `localhost` and literal loopback IP URLs — use `ws://127.0.0.1:18789`, `ws://[::1]:18789`, or `ws://localhost:18789`. Other hostnames route like ordinary traffic.
 
+Update canary `/startupz` and `/readyz` probes use temporary exceptions for their exact loopback URLs under `gateway-only`. The updater releases each exception after polling. The `proxy` and `block` modes still apply. If a running canary never answers successfully within the validation budget, the update records the observed HTTP or transport failure as a warning, including the next troubleshooting step, and continues best effort.
+
+Environment-only HTTP proxy routing honors `no_proxy`/`NO_PROXY` (lowercase takes precedence). These environment bypass lists do not override managed proxy policy.
+
 ### Containers
 
 For `openclaw --container ...` commands, OpenClaw forwards `OPENCLAW_PROXY_URL` into the container-targeted child CLI when it is set. The URL must be reachable from inside the container — `127.0.0.1` there refers to the container itself, not the host. OpenClaw rejects loopback proxy URLs for container-targeted commands unless you set `OPENCLAW_CONTAINER_ALLOW_LOOPBACK_PROXY_URL=1` to explicitly override that check.

src/cli/update-cli/update-command-execution.test.ts

@@ -16,6 +16,11 @@ import * as portInspection from "../../infra/ports-inspect.js";
 import * as tempRoot from "../../infra/tmp-openclaw-dir.js";
 import { UpdateRequesterRevokedError } from "../../infra/update-requester-authority.js";
 import { createUpdateRun } from "../../infra/update-run-ledger.js";
+import {
+  updateRunStepsFromResultStep,
+  updateRunWarningMessages,
+} from "../../infra/update-run-step.js";
+import type { UpdateStepProgress, UpdateStepResult } from "../../infra/update-runner.js";
 import { withTestDir } from "../../test-helpers/temp-dir.js";
 import { withEnvAsync } from "../../test-utils/env.js";
 import { mockProcessPlatform } from "../../test-utils/vitest-spies.js";
@@ -33,6 +38,64 @@ const { executionParams, inspectOrStopService, mocks, schemaContext, successfulU
   await import("./update-command-execution.test-support.js");
 
 describe("mutable update execution", () => {
+  it.each(["package", "git"] as const)(
+    "continues the %s update with the recorded readiness warning instead of inference repair",
+    async (kind) => {
+      const message =
+        "Readiness probe http://127.0.0.1:18789/readyz failed: HTTP 502. Check the configured proxy.";
+      const step: UpdateStepResult = {
+        name: "candidate gateway canary",
+        command: "gateway run",
+        cwd: "/candidate",
+        durationMs: 1,
+        exitCode: null,
+        advisory: { kind: "candidate-runtime-unavailable", message },
+        failureFacts: [{ check: "readyz", code: "candidate-readiness-probe-failed", message }],
+      };
+      mocks.validateCanary.mockImplementation(async ({ onStep }) => {
+        onStep(step);
+        return {
+          status: "ok",
+          phase: "readiness",
+          steps: [step],
+          durationMs: 1,
+          logTail: [message],
+        };
+      });
+      const repair = await import("./update-command-repair.js");
+      const runRepair = vi.spyOn(repair, "runUpdateCommandRepair");
+      const accepted = vi.fn();
+      const runStagedUpdate = async ({
+        validateCandidate,
+      }: {
+        validateCandidate?: (root: string) => Promise<unknown>;
+      }) => {
+        expect(validateCandidate).toBeTypeOf("function");
+        await validateCandidate?.("/candidate");
+        accepted();
+        return successfulUpdate;
+      };
+      mocks.runPackageUpdate.mockImplementation(runStagedUpdate);
+      mocks.runGitUpdate.mockImplementation(runStagedUpdate);
+      const onStepComplete = vi.fn<NonNullable<UpdateStepProgress["onStepComplete"]>>();
+
+      const execution = await executeMutableUpdate({
+        ...executionParams(kind),
+        progress: { onStepComplete },
+      });
+
+      expect(execution?.result.status).toBe("ok");
+      expect(accepted).toHaveBeenCalledOnce();
+      expect(runRepair).not.toHaveBeenCalled();
+      expect(onStepComplete).toHaveBeenCalledWith(expect.objectContaining(step));
+      const recorded = onStepComplete.mock.calls.flatMap(([completed]) =>
+        updateRunStepsFromResultStep(completed),
+      );
+      expect(updateRunWarningMessages(recorded)).toEqual([message]);
+      expect(recorded.every((entry) => entry.status === "completed")).toBe(true);
+    },
+  );
+
   it.each(
     (["package", "git"] as const).flatMap((kind) =>
       [undefined, 30_000, 600_000].map((timeoutMs) => ({ kind, timeoutMs })),

src/infra/update-candidate-canary-readiness.test-support.ts

@@ -1,9 +1,166 @@
 import { once } from "node:events";
 import { createServer } from "node:http";
+import { getGlobalDispatcher, setGlobalDispatcher } from "undici";
 import { expect, it, vi } from "vitest";
+import { startProxy, stopProxy, type ProxyHandle } from "./net/proxy/proxy-lifecycle.js";
 import { validateUpdateCandidateCanary } from "./update-candidate-canary.js";
+import { prepareUpdateCandidateRehearsal } from "./update-candidate-rehearsal.js";
+import type { UpdateStepResult } from "./update-runner-types.js";
+
+export function expectCanaryReadinessWarning(
+  step: UpdateStepResult | undefined,
+  check: string,
+  status: number,
+) {
+  expect(step).toMatchObject({
+    name: "candidate gateway canary",
+    advisory: {
+      kind: "candidate-runtime-unavailable",
+      message: expect.stringContaining(`failed: HTTP ${status}`),
+    },
+    failureFacts: [
+      {
+        check,
+        code: "candidate-readiness-probe-failed",
+        message: expect.stringContaining(`failed: HTTP ${status}`),
+      },
+    ],
+  });
+}
 
 export function registerCanaryReadinessBudgetTests(root: () => string) {
+  it.each(["gateway-only", "proxy", "block"] as const)(
+    "probes the canary with managed proxy mode %s",
+    async (loopbackMode) => {
+      const rehearsal = await prepareUpdateCandidateRehearsal({
+        candidateRoot: root(),
+        stateDir: root(),
+        config: {},
+        env: {},
+      });
+      const requests: string[] = [];
+      const proxyRequests: string[] = [];
+      const server = createServer((request, response) => {
+        requests.push(request.url ?? "");
+        response.writeHead(200, { "content-type": "application/json" });
+        response.end(JSON.stringify({ status: "started", ready: true }));
+      });
+      const proxy = createServer((request, response) => {
+        proxyRequests.push(request.url ?? "");
+        response.writeHead(502);
+        response.end();
+      });
+      proxy.on("connect", (request, socket) => {
+        proxyRequests.push(`CONNECT ${request.url}`);
+        socket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n");
+      });
+      const dispatcher = getGlobalDispatcher();
+      let handle: ProxyHandle | null = null;
+      try {
+        server.listen(rehearsal.port, "127.0.0.1");
+        await once(server, "listening");
+        proxy.listen(0, "127.0.0.1");
+        await once(proxy, "listening");
+        const address = proxy.address();
+        if (!address || typeof address === "string") {
+          throw new Error("Expected a loopback proxy listener");
+        }
+        vi.stubEnv("no_proxy", "localhost,127.0.0.1,::1");
+        handle = await startProxy({
+          proxyUrl: `http://127.0.0.1:${address.port}`,
+          loopbackMode,
+        });
+        const result = await validateUpdateCandidateCanary({
+          root: root(),
+          stateDir: root(),
+          config: {},
+          env: {},
+          rehearsal,
+          timeoutMs: 1_000,
+        });
+        if (loopbackMode === "gateway-only") {
+          expect(
+            result,
+            JSON.stringify({ log: result.logTail, requests, proxyRequests }),
+          ).toMatchObject({
+            status: "ok",
+            phase: "readiness",
+          });
+          expect(requests).toEqual(["/startupz", "/readyz"]);
+          expect(proxyRequests).toEqual([]);
+          // The canary releases its exception; unrelated traffic still uses the proxy.
+          for (const url of [
+            `http://127.0.0.1:${rehearsal.port}/readyz`,
+            "http://external.example/",
+          ]) {
+            const response = await fetch(url);
+            expect(response.status).toBe(502);
+            await response.body?.cancel();
+          }
+        } else if (loopbackMode === "proxy") {
+          const message = `Readiness probe http://127.0.0.1:${rehearsal.port}/startupz failed: HTTP 502 (via proxy http://127.0.0.1:${address.port}). Check Gateway logs and proxy.loopbackMode; rerun openclaw update.`;
+          expectCanaryReadinessWarning(result.steps.at(-1), "startupz", 502);
+          expect(result.steps.at(-1)).toMatchObject({
+            advisory: { kind: "candidate-runtime-unavailable", message },
+          });
+          expect(result.status).toBe("ok");
+          expect(requests).toEqual([]);
+          expect(proxyRequests.length).toBeGreaterThan(0);
+        } else {
+          expect(result.status).toBe("error");
+          expect(result.logTail.join("\n")).toContain("blocked by proxy.loopbackMode");
+          expect(requests).toEqual([]);
+          expect(proxyRequests).toEqual([]);
+        }
+      } finally {
+        await stopProxy(handle);
+        setGlobalDispatcher(dispatcher);
+        vi.unstubAllEnvs();
+        for (const listener of [server, proxy]) {
+          listener.closeAllConnections();
+          if (listener.listening) {
+            await new Promise<void>((resolve, reject) => {
+              listener.close((error) => (error ? reject(error) : resolve()));
+            });
+          }
+        }
+        await rehearsal.cleanup();
+      }
+    },
+  );
+
+  it("records transport causes without turning cancellation into an advisory", async () => {
+    const controller = new AbortController();
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => {
+        throw new TypeError("fetch failed", { cause: new Error("connect ECONNREFUSED") });
+      }),
+    );
+    const params = { root: root(), stateDir: root(), config: {}, env: {}, timeoutMs: 250 };
+    const unavailable = await validateUpdateCandidateCanary(params);
+    expect(unavailable.status).toBe("ok");
+    expect(unavailable.steps.at(-1)?.advisory?.message).toContain("ECONNREFUSED");
+    expect(unavailable.steps.at(-1)?.failureFacts).toEqual([
+      {
+        check: "startupz",
+        code: "candidate-readiness-probe-failed",
+        message: expect.stringContaining("ECONNREFUSED"),
+      },
+    ]);
+    vi.stubGlobal(
+      "fetch",
+      vi.fn(async () => {
+        controller.abort(new Error("operator cancelled"));
+        return Response.json({ status: "started" });
+      }),
+    );
+    const cancelled = await validateUpdateCandidateCanary({ ...params, signal: controller.signal });
+    expect(cancelled.status).toBe("error");
+    expect(cancelled.steps.at(-1)?.advisory).toBeUndefined();
+    expect(cancelled.logTail.join("\n")).toContain("operator cancelled");
+  });
+
   it.each(
     ["startupz", "readyz"].flatMap((endpoint) =>
       ["headers", "body"].map((delay) => ({ endpoint, delay })),

src/infra/update-candidate-canary-readiness.ts

@@ -0,0 +1,107 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { isRecord } from "@openclaw/normalization-core/record-coerce";
+import {
+  redactSupportDiagnosticLine,
+  redactSupportString,
+  type SupportRedactionContext,
+} from "../logging/diagnostic-support-redaction.js";
+import { scheduleAbsoluteDeadline } from "../utils/absolute-deadline.js";
+import { formatErrorMessageWithCode } from "./errors.js";
+import { getActiveManagedProxyUrl } from "./net/proxy/active-proxy-state.js";
+import { registerManagedProxyGatewayLoopbackBypass } from "./net/proxy/proxy-lifecycle.js";
+import { createUpdateFailureFact, type UpdateFailureFact } from "./update-failure-facts.js";
+
+/** Poll candidate control-plane endpoints under the existing managed loopback policy. */
+export async function waitForUpdateCandidateReadiness(
+  params: SupportRedactionContext & {
+    port: number;
+    workDeadline: number;
+    started: number;
+    signal?: AbortSignal;
+    assertCurrent?: () => void;
+    hasExited: () => boolean;
+    onEndpoint: (endpoint: "startupz" | "readyz") => void;
+    capture: (message: string) => void;
+  },
+): Promise<{ fact: UpdateFailureFact; message: string } | undefined> {
+  const deadline = new AbortController();
+  const cancelDeadline = scheduleAbsoluteDeadline(params.workDeadline, () => deadline.abort());
+  const signal = AbortSignal.any([deadline.signal, ...(params.signal ? [params.signal] : [])]);
+  const assertRunning = () => {
+    params.signal?.throwIfAborted();
+    params.assertCurrent?.();
+    if (params.hasExited()) {
+      throw new Error("Candidate gateway exited before readiness");
+    }
+  };
+  try {
+    for (const endpoint of ["startupz", "readyz"] as const) {
+      params.onEndpoint(endpoint);
+      const url = `http://127.0.0.1:${params.port}/${endpoint}`;
+      const releaseBypass = registerManagedProxyGatewayLoopbackBypass(url);
+      const proxy = releaseBypass ? undefined : getActiveManagedProxyUrl();
+      let failure: { fact: UpdateFailureFact; message: string } | undefined;
+      try {
+        while (true) {
+          assertRunning();
+          if (Date.now() >= params.workDeadline) {
+            if (!failure) {
+              throw new Error("Candidate validation deadline exceeded");
+            }
+            params.capture(failure.message);
+            return failure;
+          }
+          let outcome = "";
+          let ready = false;
+          try {
+            const response = await fetch(url, { signal });
+            outcome = `HTTP ${response.status}`;
+            if (response.status === 200) {
+              const payload: unknown = await response.json();
+              ready = endpoint === "readyz" || (isRecord(payload) && payload.status === "started");
+              outcome += " (startup response not ready within the validation budget)";
+            } else {
+              await response.body?.cancel();
+            }
+          } catch (error) {
+            outcome = `${outcome ? `${outcome}: ` : ""}${formatErrorMessageWithCode(error)}`;
+          }
+          assertRunning();
+          if (ready && Date.now() < params.workDeadline) {
+            params.capture(
+              `${endpoint}: ${endpoint === "startupz" ? "started" : "ready"} (${Date.now() - params.started}ms)`,
+            );
+            break;
+          }
+          // Keep the last observed cause when the common deadline aborts a later poll.
+          if (!deadline.signal.aborted || !failure) {
+            const detail = redactSupportDiagnosticLine(outcome, params);
+            const nextStep = "Check Gateway logs and proxy.loopbackMode; rerun openclaw update.";
+            failure = {
+              message: redactSupportString(
+                `Readiness probe ${url} failed: ${detail}${proxy ? ` (via proxy ${proxy.origin})` : ""}. ${nextStep}`,
+                params,
+              ),
+              fact: createUpdateFailureFact(
+                {
+                  check: endpoint,
+                  code: "candidate-readiness-probe-failed",
+                  message: `Readiness probe ${endpoint} failed: ${detail}. ${nextStep}`,
+                },
+                params.env,
+              ),
+            };
+          }
+          await sleep(Math.min(100, Math.max(1, params.workDeadline - Date.now())), undefined, {
+            signal: params.signal,
+          });
+        }
+      } finally {
+        releaseBypass?.();
+      }
+    }
+    return undefined;
+  } finally {
+    cancelDeadline();
+  }
+}

src/infra/update-candidate-canary.test.ts

@@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
 import type { OpenClawConfig } from "../config/types.openclaw.js";
 import * as diskSpace from "./disk-space.js";
-import { registerCanaryReadinessBudgetTests } from "./update-candidate-canary-readiness.test-support.js";
+import * as readiness from "./update-candidate-canary-readiness.test-support.js";
 import { validateUpdateCandidateCanary } from "./update-candidate-canary.js";
 import {
   completeCanaryCommand,
@@ -108,7 +108,7 @@ afterEach(() => {
 });
 
 describe("update candidate canary", () => {
-  registerCanaryReadinessBudgetTests(() => root);
+  readiness.registerCanaryReadinessBudgetTests(() => root);
   it("records a typed capacity refusal before notifying the snapshot failure", async () => {
     const capacity = vi.spyOn(diskSpace, "tryReadDiskSpace").mockImplementation((targetPath) => ({
       targetPath,
@@ -798,7 +798,7 @@ describe("update candidate canary", () => {
   });
 
   it.each(["snapshot", "doctor", "plugins", "runtime", "readiness"] as const)(
-    "records a failed %s step and cleans private state",
+    "records the %s outcome and cleans private state",
     async (failure) => {
       pluginErrors = failure === "plugins";
       runtimeError = failure === "runtime";
@@ -842,10 +842,10 @@ describe("update candidate canary", () => {
         env: {},
         timeoutMs: 250,
       });
-      expect(result.status).toBe("error");
+      expect(result.status).toBe(failure === "readiness" ? "ok" : "error");
       expect(result.phase).toBe(failure);
       if (failure === "readiness") {
-        expect(result.steps.at(-1)?.name).toBe("candidate gateway canary");
+        readiness.expectCanaryReadinessWarning(result.steps.at(-1), "readyz", 503);
       }
       expect(result.steps.some((step) => step.exitCode !== 0)).toBe(true);
       expect(result.logTail.length).toBeLessThanOrEqual(40);

src/infra/update-candidate-canary.ts

@@ -1,7 +1,6 @@
 import { spawn, type ChildProcess } from "node:child_process";
 import fs from "node:fs/promises";
 import path from "node:path";
-import { setTimeout as sleep } from "node:timers/promises";
 import { isDeepStrictEqual } from "node:util";
 import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion";
 import { isRecord } from "@openclaw/normalization-core/record-coerce";
@@ -17,11 +16,11 @@ import {
   parseOpenClawSchemaVersions,
   type OpenClawSchemaVersions,
 } from "../state/openclaw-schema-versions.js";
-import { scheduleAbsoluteDeadline } from "../utils/absolute-deadline.js";
 import { hasErrnoCode } from "./errors.js";
 import { readPackageVersion } from "./package-json.js";
 import { runtimeProcessEntrypoints } from "./runtime-process-entrypoints.js";
 import { resolveSqliteInspectionBudget } from "./sqlite-readonly-worker.js";
+import { waitForUpdateCandidateReadiness } from "./update-candidate-canary-readiness.js";
 import {
   prepareUpdateCandidateRehearsal,
   type UpdateCandidateRehearsal,
@@ -51,6 +50,7 @@ type CanaryPhase =
   | "runtime"
   | "startup"
   | "readiness";
+
 type CanaryResult = {
   phase: CanaryPhase;
   durationMs: number;
@@ -571,51 +571,37 @@ export async function validateUpdateCandidateCanary(params: {
       "--port",
       String(port),
     ]);
-    const probeDeadline = new AbortController();
-    const cancelProbeDeadline = scheduleAbsoluteDeadline(workDeadline, () => probeDeadline.abort());
     try {
-      for (const endpoint of ["startupz", "readyz"] as const) {
-        phase = endpoint === "startupz" ? "startup" : "readiness";
-        while (true) {
-          remaining();
-          if (running.hasExited()) {
-            throw new Error("Candidate gateway exited before readiness");
-          }
-          try {
-            const response = await fetch(`http://127.0.0.1:${port}/${endpoint}`, {
-              signal: AbortSignal.any([
-                probeDeadline.signal,
-                ...(params.signal ? [params.signal] : []),
-              ]),
-            });
-            const payload: unknown = await response.json();
-            remaining();
-            if (
-              response.status === 200 &&
-              (endpoint === "readyz" || (isRecord(payload) && payload.status === "started"))
-            ) {
-              capture(
-                `${endpoint}: ${endpoint === "startupz" ? "started" : "ready"} (${Date.now() - started}ms)`,
-              );
-              break;
-            }
-          } catch {
-            // The listener may not exist yet; only the common deadline permits another probe.
-          }
-          await sleep(Math.min(100, remaining()), undefined, { signal: params.signal });
-        }
-      }
+      const probeFailure = await waitForUpdateCandidateReadiness({
+        port,
+        workDeadline,
+        started,
+        signal: params.signal,
+        assertCurrent: params.assertCurrent,
+        hasExited: running.hasExited,
+        env,
+        stateDir: params.stateDir,
+        onEndpoint: (endpoint) => {
+          phase = endpoint === "startupz" ? "startup" : "readiness";
+        },
+        capture,
+      });
       const step: UpdateStepResult = {
         name: "candidate gateway canary",
         command: "gateway run",
         cwd: params.root,
         durationMs: Date.now() - gatewayStart,
-        exitCode: 0,
+        exitCode: probeFailure ? null : 0,
+        ...(probeFailure
+          ? {
+              advisory: { kind: "candidate-runtime-unavailable", message: probeFailure.message },
+              failureFacts: [probeFailure.fact],
+            }
+          : {}),
       };
       steps.push(step);
       params.onStep?.(step);
     } finally {
-      cancelProbeDeadline();
       await terminateCanary(running.child, running.closed, deadline);
     }
     return {

src/infra/update-failure-public-identifiers.ts

@@ -55,6 +55,7 @@ const PUBLIC_CODES = new Set<string>([
   ...updateRecoverySchema.options[1].shape.reason.options,
   ...GATEWAY_RESTART_WAIT_OUTCOMES,
   ...CANARY_CHECKS.map((phase) => `candidate-${phase}-failed`),
+  "candidate-readiness-probe-failed",
   "Error",
   "TypeError",
   "SyntaxError",

src/infra/update-failure-report-prepare.test.ts

@@ -14,6 +14,40 @@ function prepareDiagnosticReport(reason: string) {
 }
 
 describe("update report diagnostic command boundary", () => {
+  it.each(["startupz", "readyz"])(
+    "preserves the %s readiness probe failure identifier",
+    async (check) => {
+      const report = await prepareUpdateFailureReport(
+        {
+          attemptId: "candidate-readiness-probe",
+          result: {
+            status: "error",
+            mode: "npm",
+            durationMs: 1,
+            steps: [
+              {
+                name: "candidate gateway canary",
+                command: "gateway run",
+                cwd: "/candidate",
+                durationMs: 1,
+                exitCode: 1,
+                failureFacts: [
+                  {
+                    check,
+                    code: "candidate-readiness-probe-failed",
+                    message: "Readiness probe failed: HTTP 502. Check the configured proxy.",
+                  },
+                ],
+              },
+            ],
+          },
+        },
+        context,
+      );
+      expect(report.body).toContain(`Failing check ${check} (candidate-readiness-probe-failed)`);
+    },
+  );
+
   it.each([true, false])("uses only matching finalization facts (matches=%s)", async (matches) => {
     const message =
       "Doctor could not enter maintenance. Error: The update parent owns Gateway activation.";