diff --git a/docs/security/network-proxy.md b/docs/security/network-proxy.md index f389560c54a1..c2dfd206c07e 100644 --- a/docs/security/network-proxy.md +++ b/docs/security/network-proxy.md @@ -102,6 +102,10 @@ without activating it. Gateway control-plane bypass is limited to `localhost` and literal loopback IP URLs — use `ws://127.0.0.1:18789`, `ws://[::1]:18789`, or `ws://localhost:18789`. Other hostnames route like ordinary traffic. +Update canary `/startupz` and `/readyz` probes use temporary exceptions for their exact loopback URLs under `gateway-only`. The updater releases each exception after polling. The `proxy` and `block` modes still apply. If a running canary never answers successfully within the validation budget, the update records the observed HTTP or transport failure as a warning, including the next troubleshooting step, and continues best effort. + +Environment-only HTTP proxy routing honors `no_proxy`/`NO_PROXY` (lowercase takes precedence). These environment bypass lists do not override managed proxy policy. + ### Containers For `openclaw --container ...` commands, OpenClaw forwards `OPENCLAW_PROXY_URL` into the container-targeted child CLI when it is set. The URL must be reachable from inside the container — `127.0.0.1` there refers to the container itself, not the host. OpenClaw rejects loopback proxy URLs for container-targeted commands unless you set `OPENCLAW_CONTAINER_ALLOW_LOOPBACK_PROXY_URL=1` to explicitly override that check. diff --git a/src/cli/update-cli/update-command-execution.test.ts b/src/cli/update-cli/update-command-execution.test.ts index a8adf123e245..cc75b3fd946a 100644 --- a/src/cli/update-cli/update-command-execution.test.ts +++ b/src/cli/update-cli/update-command-execution.test.ts @@ -16,6 +16,11 @@ import * as portInspection from "../../infra/ports-inspect.js"; import * as tempRoot from "../../infra/tmp-openclaw-dir.js"; import { UpdateRequesterRevokedError } from "../../infra/update-requester-authority.js"; import { createUpdateRun } from "../../infra/update-run-ledger.js"; +import { + updateRunStepsFromResultStep, + updateRunWarningMessages, +} from "../../infra/update-run-step.js"; +import type { UpdateStepProgress, UpdateStepResult } from "../../infra/update-runner.js"; import { withTestDir } from "../../test-helpers/temp-dir.js"; import { withEnvAsync } from "../../test-utils/env.js"; import { mockProcessPlatform } from "../../test-utils/vitest-spies.js"; @@ -33,6 +38,64 @@ const { executionParams, inspectOrStopService, mocks, schemaContext, successfulU await import("./update-command-execution.test-support.js"); describe("mutable update execution", () => { + it.each(["package", "git"] as const)( + "continues the %s update with the recorded readiness warning instead of inference repair", + async (kind) => { + const message = + "Readiness probe http://127.0.0.1:18789/readyz failed: HTTP 502. Check the configured proxy."; + const step: UpdateStepResult = { + name: "candidate gateway canary", + command: "gateway run", + cwd: "/candidate", + durationMs: 1, + exitCode: null, + advisory: { kind: "candidate-runtime-unavailable", message }, + failureFacts: [{ check: "readyz", code: "candidate-readiness-probe-failed", message }], + }; + mocks.validateCanary.mockImplementation(async ({ onStep }) => { + onStep(step); + return { + status: "ok", + phase: "readiness", + steps: [step], + durationMs: 1, + logTail: [message], + }; + }); + const repair = await import("./update-command-repair.js"); + const runRepair = vi.spyOn(repair, "runUpdateCommandRepair"); + const accepted = vi.fn(); + const runStagedUpdate = async ({ + validateCandidate, + }: { + validateCandidate?: (root: string) => Promise; + }) => { + expect(validateCandidate).toBeTypeOf("function"); + await validateCandidate?.("/candidate"); + accepted(); + return successfulUpdate; + }; + mocks.runPackageUpdate.mockImplementation(runStagedUpdate); + mocks.runGitUpdate.mockImplementation(runStagedUpdate); + const onStepComplete = vi.fn>(); + + const execution = await executeMutableUpdate({ + ...executionParams(kind), + progress: { onStepComplete }, + }); + + expect(execution?.result.status).toBe("ok"); + expect(accepted).toHaveBeenCalledOnce(); + expect(runRepair).not.toHaveBeenCalled(); + expect(onStepComplete).toHaveBeenCalledWith(expect.objectContaining(step)); + const recorded = onStepComplete.mock.calls.flatMap(([completed]) => + updateRunStepsFromResultStep(completed), + ); + expect(updateRunWarningMessages(recorded)).toEqual([message]); + expect(recorded.every((entry) => entry.status === "completed")).toBe(true); + }, + ); + it.each( (["package", "git"] as const).flatMap((kind) => [undefined, 30_000, 600_000].map((timeoutMs) => ({ kind, timeoutMs })), diff --git a/src/infra/update-candidate-canary-readiness.test-support.ts b/src/infra/update-candidate-canary-readiness.test-support.ts index 054f214d58a1..4bc3dc51d95f 100644 --- a/src/infra/update-candidate-canary-readiness.test-support.ts +++ b/src/infra/update-candidate-canary-readiness.test-support.ts @@ -1,9 +1,166 @@ import { once } from "node:events"; import { createServer } from "node:http"; +import { getGlobalDispatcher, setGlobalDispatcher } from "undici"; import { expect, it, vi } from "vitest"; +import { startProxy, stopProxy, type ProxyHandle } from "./net/proxy/proxy-lifecycle.js"; import { validateUpdateCandidateCanary } from "./update-candidate-canary.js"; +import { prepareUpdateCandidateRehearsal } from "./update-candidate-rehearsal.js"; +import type { UpdateStepResult } from "./update-runner-types.js"; + +export function expectCanaryReadinessWarning( + step: UpdateStepResult | undefined, + check: string, + status: number, +) { + expect(step).toMatchObject({ + name: "candidate gateway canary", + advisory: { + kind: "candidate-runtime-unavailable", + message: expect.stringContaining(`failed: HTTP ${status}`), + }, + failureFacts: [ + { + check, + code: "candidate-readiness-probe-failed", + message: expect.stringContaining(`failed: HTTP ${status}`), + }, + ], + }); +} export function registerCanaryReadinessBudgetTests(root: () => string) { + it.each(["gateway-only", "proxy", "block"] as const)( + "probes the canary with managed proxy mode %s", + async (loopbackMode) => { + const rehearsal = await prepareUpdateCandidateRehearsal({ + candidateRoot: root(), + stateDir: root(), + config: {}, + env: {}, + }); + const requests: string[] = []; + const proxyRequests: string[] = []; + const server = createServer((request, response) => { + requests.push(request.url ?? ""); + response.writeHead(200, { "content-type": "application/json" }); + response.end(JSON.stringify({ status: "started", ready: true })); + }); + const proxy = createServer((request, response) => { + proxyRequests.push(request.url ?? ""); + response.writeHead(502); + response.end(); + }); + proxy.on("connect", (request, socket) => { + proxyRequests.push(`CONNECT ${request.url}`); + socket.end("HTTP/1.1 502 Bad Gateway\r\nConnection: close\r\n\r\n"); + }); + const dispatcher = getGlobalDispatcher(); + let handle: ProxyHandle | null = null; + try { + server.listen(rehearsal.port, "127.0.0.1"); + await once(server, "listening"); + proxy.listen(0, "127.0.0.1"); + await once(proxy, "listening"); + const address = proxy.address(); + if (!address || typeof address === "string") { + throw new Error("Expected a loopback proxy listener"); + } + vi.stubEnv("no_proxy", "localhost,127.0.0.1,::1"); + handle = await startProxy({ + proxyUrl: `http://127.0.0.1:${address.port}`, + loopbackMode, + }); + const result = await validateUpdateCandidateCanary({ + root: root(), + stateDir: root(), + config: {}, + env: {}, + rehearsal, + timeoutMs: 1_000, + }); + if (loopbackMode === "gateway-only") { + expect( + result, + JSON.stringify({ log: result.logTail, requests, proxyRequests }), + ).toMatchObject({ + status: "ok", + phase: "readiness", + }); + expect(requests).toEqual(["/startupz", "/readyz"]); + expect(proxyRequests).toEqual([]); + // The canary releases its exception; unrelated traffic still uses the proxy. + for (const url of [ + `http://127.0.0.1:${rehearsal.port}/readyz`, + "http://external.example/", + ]) { + const response = await fetch(url); + expect(response.status).toBe(502); + await response.body?.cancel(); + } + } else if (loopbackMode === "proxy") { + const message = `Readiness probe http://127.0.0.1:${rehearsal.port}/startupz failed: HTTP 502 (via proxy http://127.0.0.1:${address.port}). Check Gateway logs and proxy.loopbackMode; rerun openclaw update.`; + expectCanaryReadinessWarning(result.steps.at(-1), "startupz", 502); + expect(result.steps.at(-1)).toMatchObject({ + advisory: { kind: "candidate-runtime-unavailable", message }, + }); + expect(result.status).toBe("ok"); + expect(requests).toEqual([]); + expect(proxyRequests.length).toBeGreaterThan(0); + } else { + expect(result.status).toBe("error"); + expect(result.logTail.join("\n")).toContain("blocked by proxy.loopbackMode"); + expect(requests).toEqual([]); + expect(proxyRequests).toEqual([]); + } + } finally { + await stopProxy(handle); + setGlobalDispatcher(dispatcher); + vi.unstubAllEnvs(); + for (const listener of [server, proxy]) { + listener.closeAllConnections(); + if (listener.listening) { + await new Promise((resolve, reject) => { + listener.close((error) => (error ? reject(error) : resolve())); + }); + } + } + await rehearsal.cleanup(); + } + }, + ); + + it("records transport causes without turning cancellation into an advisory", async () => { + const controller = new AbortController(); + vi.stubGlobal( + "fetch", + vi.fn(async () => { + throw new TypeError("fetch failed", { cause: new Error("connect ECONNREFUSED") }); + }), + ); + const params = { root: root(), stateDir: root(), config: {}, env: {}, timeoutMs: 250 }; + const unavailable = await validateUpdateCandidateCanary(params); + expect(unavailable.status).toBe("ok"); + expect(unavailable.steps.at(-1)?.advisory?.message).toContain("ECONNREFUSED"); + expect(unavailable.steps.at(-1)?.failureFacts).toEqual([ + { + check: "startupz", + code: "candidate-readiness-probe-failed", + message: expect.stringContaining("ECONNREFUSED"), + }, + ]); + vi.stubGlobal( + "fetch", + vi.fn(async () => { + controller.abort(new Error("operator cancelled")); + return Response.json({ status: "started" }); + }), + ); + const cancelled = await validateUpdateCandidateCanary({ ...params, signal: controller.signal }); + expect(cancelled.status).toBe("error"); + expect(cancelled.steps.at(-1)?.advisory).toBeUndefined(); + expect(cancelled.logTail.join("\n")).toContain("operator cancelled"); + }); + it.each( ["startupz", "readyz"].flatMap((endpoint) => ["headers", "body"].map((delay) => ({ endpoint, delay })), diff --git a/src/infra/update-candidate-canary-readiness.ts b/src/infra/update-candidate-canary-readiness.ts new file mode 100644 index 000000000000..9fa62f38fe53 --- /dev/null +++ b/src/infra/update-candidate-canary-readiness.ts @@ -0,0 +1,107 @@ +import { setTimeout as sleep } from "node:timers/promises"; +import { isRecord } from "@openclaw/normalization-core/record-coerce"; +import { + redactSupportDiagnosticLine, + redactSupportString, + type SupportRedactionContext, +} from "../logging/diagnostic-support-redaction.js"; +import { scheduleAbsoluteDeadline } from "../utils/absolute-deadline.js"; +import { formatErrorMessageWithCode } from "./errors.js"; +import { getActiveManagedProxyUrl } from "./net/proxy/active-proxy-state.js"; +import { registerManagedProxyGatewayLoopbackBypass } from "./net/proxy/proxy-lifecycle.js"; +import { createUpdateFailureFact, type UpdateFailureFact } from "./update-failure-facts.js"; + +/** Poll candidate control-plane endpoints under the existing managed loopback policy. */ +export async function waitForUpdateCandidateReadiness( + params: SupportRedactionContext & { + port: number; + workDeadline: number; + started: number; + signal?: AbortSignal; + assertCurrent?: () => void; + hasExited: () => boolean; + onEndpoint: (endpoint: "startupz" | "readyz") => void; + capture: (message: string) => void; + }, +): Promise<{ fact: UpdateFailureFact; message: string } | undefined> { + const deadline = new AbortController(); + const cancelDeadline = scheduleAbsoluteDeadline(params.workDeadline, () => deadline.abort()); + const signal = AbortSignal.any([deadline.signal, ...(params.signal ? [params.signal] : [])]); + const assertRunning = () => { + params.signal?.throwIfAborted(); + params.assertCurrent?.(); + if (params.hasExited()) { + throw new Error("Candidate gateway exited before readiness"); + } + }; + try { + for (const endpoint of ["startupz", "readyz"] as const) { + params.onEndpoint(endpoint); + const url = `http://127.0.0.1:${params.port}/${endpoint}`; + const releaseBypass = registerManagedProxyGatewayLoopbackBypass(url); + const proxy = releaseBypass ? undefined : getActiveManagedProxyUrl(); + let failure: { fact: UpdateFailureFact; message: string } | undefined; + try { + while (true) { + assertRunning(); + if (Date.now() >= params.workDeadline) { + if (!failure) { + throw new Error("Candidate validation deadline exceeded"); + } + params.capture(failure.message); + return failure; + } + let outcome = ""; + let ready = false; + try { + const response = await fetch(url, { signal }); + outcome = `HTTP ${response.status}`; + if (response.status === 200) { + const payload: unknown = await response.json(); + ready = endpoint === "readyz" || (isRecord(payload) && payload.status === "started"); + outcome += " (startup response not ready within the validation budget)"; + } else { + await response.body?.cancel(); + } + } catch (error) { + outcome = `${outcome ? `${outcome}: ` : ""}${formatErrorMessageWithCode(error)}`; + } + assertRunning(); + if (ready && Date.now() < params.workDeadline) { + params.capture( + `${endpoint}: ${endpoint === "startupz" ? "started" : "ready"} (${Date.now() - params.started}ms)`, + ); + break; + } + // Keep the last observed cause when the common deadline aborts a later poll. + if (!deadline.signal.aborted || !failure) { + const detail = redactSupportDiagnosticLine(outcome, params); + const nextStep = "Check Gateway logs and proxy.loopbackMode; rerun openclaw update."; + failure = { + message: redactSupportString( + `Readiness probe ${url} failed: ${detail}${proxy ? ` (via proxy ${proxy.origin})` : ""}. ${nextStep}`, + params, + ), + fact: createUpdateFailureFact( + { + check: endpoint, + code: "candidate-readiness-probe-failed", + message: `Readiness probe ${endpoint} failed: ${detail}. ${nextStep}`, + }, + params.env, + ), + }; + } + await sleep(Math.min(100, Math.max(1, params.workDeadline - Date.now())), undefined, { + signal: params.signal, + }); + } + } finally { + releaseBypass?.(); + } + } + return undefined; + } finally { + cancelDeadline(); + } +} diff --git a/src/infra/update-candidate-canary.test.ts b/src/infra/update-candidate-canary.test.ts index c13c9eafce4e..9083b61c07ef 100644 --- a/src/infra/update-candidate-canary.test.ts +++ b/src/infra/update-candidate-canary.test.ts @@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import * as diskSpace from "./disk-space.js"; -import { registerCanaryReadinessBudgetTests } from "./update-candidate-canary-readiness.test-support.js"; +import * as readiness from "./update-candidate-canary-readiness.test-support.js"; import { validateUpdateCandidateCanary } from "./update-candidate-canary.js"; import { completeCanaryCommand, @@ -108,7 +108,7 @@ afterEach(() => { }); describe("update candidate canary", () => { - registerCanaryReadinessBudgetTests(() => root); + readiness.registerCanaryReadinessBudgetTests(() => root); it("records a typed capacity refusal before notifying the snapshot failure", async () => { const capacity = vi.spyOn(diskSpace, "tryReadDiskSpace").mockImplementation((targetPath) => ({ targetPath, @@ -798,7 +798,7 @@ describe("update candidate canary", () => { }); it.each(["snapshot", "doctor", "plugins", "runtime", "readiness"] as const)( - "records a failed %s step and cleans private state", + "records the %s outcome and cleans private state", async (failure) => { pluginErrors = failure === "plugins"; runtimeError = failure === "runtime"; @@ -842,10 +842,10 @@ describe("update candidate canary", () => { env: {}, timeoutMs: 250, }); - expect(result.status).toBe("error"); + expect(result.status).toBe(failure === "readiness" ? "ok" : "error"); expect(result.phase).toBe(failure); if (failure === "readiness") { - expect(result.steps.at(-1)?.name).toBe("candidate gateway canary"); + readiness.expectCanaryReadinessWarning(result.steps.at(-1), "readyz", 503); } expect(result.steps.some((step) => step.exitCode !== 0)).toBe(true); expect(result.logTail.length).toBeLessThanOrEqual(40); diff --git a/src/infra/update-candidate-canary.ts b/src/infra/update-candidate-canary.ts index 6c9c11dafcf1..972eae6433d4 100644 --- a/src/infra/update-candidate-canary.ts +++ b/src/infra/update-candidate-canary.ts @@ -1,7 +1,6 @@ import { spawn, type ChildProcess } from "node:child_process"; import fs from "node:fs/promises"; import path from "node:path"; -import { setTimeout as sleep } from "node:timers/promises"; import { isDeepStrictEqual } from "node:util"; import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; @@ -17,11 +16,11 @@ import { parseOpenClawSchemaVersions, type OpenClawSchemaVersions, } from "../state/openclaw-schema-versions.js"; -import { scheduleAbsoluteDeadline } from "../utils/absolute-deadline.js"; import { hasErrnoCode } from "./errors.js"; import { readPackageVersion } from "./package-json.js"; import { runtimeProcessEntrypoints } from "./runtime-process-entrypoints.js"; import { resolveSqliteInspectionBudget } from "./sqlite-readonly-worker.js"; +import { waitForUpdateCandidateReadiness } from "./update-candidate-canary-readiness.js"; import { prepareUpdateCandidateRehearsal, type UpdateCandidateRehearsal, @@ -51,6 +50,7 @@ type CanaryPhase = | "runtime" | "startup" | "readiness"; + type CanaryResult = { phase: CanaryPhase; durationMs: number; @@ -571,51 +571,37 @@ export async function validateUpdateCandidateCanary(params: { "--port", String(port), ]); - const probeDeadline = new AbortController(); - const cancelProbeDeadline = scheduleAbsoluteDeadline(workDeadline, () => probeDeadline.abort()); try { - for (const endpoint of ["startupz", "readyz"] as const) { - phase = endpoint === "startupz" ? "startup" : "readiness"; - while (true) { - remaining(); - if (running.hasExited()) { - throw new Error("Candidate gateway exited before readiness"); - } - try { - const response = await fetch(`http://127.0.0.1:${port}/${endpoint}`, { - signal: AbortSignal.any([ - probeDeadline.signal, - ...(params.signal ? [params.signal] : []), - ]), - }); - const payload: unknown = await response.json(); - remaining(); - if ( - response.status === 200 && - (endpoint === "readyz" || (isRecord(payload) && payload.status === "started")) - ) { - capture( - `${endpoint}: ${endpoint === "startupz" ? "started" : "ready"} (${Date.now() - started}ms)`, - ); - break; - } - } catch { - // The listener may not exist yet; only the common deadline permits another probe. - } - await sleep(Math.min(100, remaining()), undefined, { signal: params.signal }); - } - } + const probeFailure = await waitForUpdateCandidateReadiness({ + port, + workDeadline, + started, + signal: params.signal, + assertCurrent: params.assertCurrent, + hasExited: running.hasExited, + env, + stateDir: params.stateDir, + onEndpoint: (endpoint) => { + phase = endpoint === "startupz" ? "startup" : "readiness"; + }, + capture, + }); const step: UpdateStepResult = { name: "candidate gateway canary", command: "gateway run", cwd: params.root, durationMs: Date.now() - gatewayStart, - exitCode: 0, + exitCode: probeFailure ? null : 0, + ...(probeFailure + ? { + advisory: { kind: "candidate-runtime-unavailable", message: probeFailure.message }, + failureFacts: [probeFailure.fact], + } + : {}), }; steps.push(step); params.onStep?.(step); } finally { - cancelProbeDeadline(); await terminateCanary(running.child, running.closed, deadline); } return { diff --git a/src/infra/update-failure-public-identifiers.ts b/src/infra/update-failure-public-identifiers.ts index 394cd81b2910..7160a5dcd016 100644 --- a/src/infra/update-failure-public-identifiers.ts +++ b/src/infra/update-failure-public-identifiers.ts @@ -55,6 +55,7 @@ const PUBLIC_CODES = new Set([ ...updateRecoverySchema.options[1].shape.reason.options, ...GATEWAY_RESTART_WAIT_OUTCOMES, ...CANARY_CHECKS.map((phase) => `candidate-${phase}-failed`), + "candidate-readiness-probe-failed", "Error", "TypeError", "SyntaxError", diff --git a/src/infra/update-failure-report-prepare.test.ts b/src/infra/update-failure-report-prepare.test.ts index 4908709eb4b2..616ffd86817b 100644 --- a/src/infra/update-failure-report-prepare.test.ts +++ b/src/infra/update-failure-report-prepare.test.ts @@ -14,6 +14,40 @@ function prepareDiagnosticReport(reason: string) { } describe("update report diagnostic command boundary", () => { + it.each(["startupz", "readyz"])( + "preserves the %s readiness probe failure identifier", + async (check) => { + const report = await prepareUpdateFailureReport( + { + attemptId: "candidate-readiness-probe", + result: { + status: "error", + mode: "npm", + durationMs: 1, + steps: [ + { + name: "candidate gateway canary", + command: "gateway run", + cwd: "/candidate", + durationMs: 1, + exitCode: 1, + failureFacts: [ + { + check, + code: "candidate-readiness-probe-failed", + message: "Readiness probe failed: HTTP 502. Check the configured proxy.", + }, + ], + }, + ], + }, + }, + context, + ); + expect(report.body).toContain(`Failing check ${check} (candidate-readiness-probe-failed)`); + }, + ); + it.each([true, false])("uses only matching finalization facts (matches=%s)", async (matches) => { const message = "Doctor could not enter maintenance. Error: The update parent owns Gateway activation.";