fix(status): warn when the running Gateway Node path is gone (#157190)

* fix(status): warn when the running Gateway Node path is gone

A Homebrew upgrade can delete the Cellar Node binary while the Gateway stays up. Deep status and doctor now report that retained path and say to restart.

* fix(status): keep stale runtime diagnostics on deep scans

* test(gateway): include live runtime in exact health expectations

---------

Co-authored-by: Ayaan Zaidi <hi@obviy.us>
This commit is contained in:
Olli 2026-09-28 17:49:12 +02:00 • committed by GitHub
parent fa63d9bb84
commit 92c86fefcd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 298 additions and 13 deletions

View file

@ -47,6 +47,14 @@ Channels without a probe, such as WhatsApp, report lifecycle health instead.
In the Health table, `healthy` is `OK`; degraded lifecycle states and failed
probes remain `WARN`. A lifecycle `OK` does not mean a live probe ran.
`--deep` also asks the running Gateway whether the Node executable it still holds can be started. A Homebrew upgrade can delete that Cellar path while the LaunchAgent plist still points at a valid symlink and the Gateway port stays reachable. Status then warns:
```text
Gateway runtime is stale after Node upgrade: child workers are using <path>, which no longer exists. Restart the Gateway.
```
The check does not restart the Gateway. Run `openclaw gateway restart` after the warning.
`--deep` and `--all` also show delivery queue warnings for dead-lettered messages
and pressured inbound lanes. These warnings include pending, claimed, and blocked
message counts even when a channel connection is healthy. See

View file

@ -177,6 +177,14 @@ const HealthSnapshotSchema = closedObject({
hotReloadStatus: Type.Union([Type.Literal("active"), Type.Literal("disabled")]),
}),
),
// The running process reports the Node binary it will use for child workers.
// A deleted Homebrew Cellar path stays reachable at the Gateway port.
childRuntime: Type.Optional(
closedObject({
execPath: Type.String(),
available: Type.Boolean(),
}),
),
// Channel plugins own their nested account/probe summaries, so this is the
// one provider-contributed bag that deliberately remains unknown.
channels: Type.Optional(Type.Record(Type.String(), Type.Unknown())),

View file

@ -281,6 +281,21 @@ describe("checkGatewayHealth", () => {
},
);
it("reports a deleted Gateway Node path without marking the gateway unhealthy", async () => {
const execPath = "/opt/homebrew/Cellar/node@24/24.20.0/bin/node";
callGateway
.mockResolvedValueOnce({
childRuntime: { execPath, available: false },
})
.mockResolvedValue({});
const runtime = { log: vi.fn(), error: vi.fn(), exit: vi.fn() };
await expect(checkGatewayHealth({ runtime, cfg })).resolves.toMatchObject({ healthOk: true });
expect(note).toHaveBeenCalledWith(
`Gateway runtime is stale after Node upgrade: child workers are using ${execPath}, which no longer exists. Restart the Gateway.`,
"Gateway runtime",
);
});
it.each([
["startupMigrationWarning", "Startup migration warnings"],
["startupRecoveryWarning", "Startup session recovery"],

View file

@ -30,6 +30,7 @@ import type {
DoctorMemoryStatusPayload,
} from "../gateway/server-methods/doctor.js";
import { collectChannelStatusIssues } from "../infra/channels-status-issues.js";
import { formatMissingChildRuntimeWarning } from "../infra/child-runtime-viability.js";
import { formatErrorMessage } from "../infra/errors.js";
import { formatDurationSeconds } from "../infra/format-time/format-duration.js";
import { readGatewayLastInstallationReplacement } from "../infra/gateway-boot-lifecycle.js";
@ -342,6 +343,12 @@ export async function checkGatewayHealth(params: {
if (status.startupRecoveryWarning) {
note(sanitizeTerminalText(status.startupRecoveryWarning), "Startup session recovery");
}
const childRuntimeWarning = status.childRuntime
? formatMissingChildRuntimeWarning(status.childRuntime)
: undefined;
if (childRuntimeWarning) {
note(sanitizeTerminalText(childRuntimeWarning), "Gateway runtime");
}
if (status.installationReplacementWarning) {
note(sanitizeTerminalText(status.installationReplacementWarning), "Installation replaced");
}

View file

@ -221,6 +221,16 @@ describe("status-overview-rows", () => {
expect(findRowValue(rows, label)).toContain(params.summary[field]);
});
it("surfaces a deleted Gateway Node path in the overview", () => {
const execPath = "/opt/homebrew/Cellar/node@24/24.20.0/bin/node";
const params = createStatusCommandOverviewRowsParams();
params.summary.childRuntime = { execPath, available: false };
const rows = buildStatusCommandOverviewRows(params);
expect(findRowValue(rows, "Gateway runtime")).toBe(
`warn(Gateway runtime is stale after Node upgrade: child workers are using ${execPath}, which no longer exists. Restart the Gateway.)`,
);
});
it("builds status-all overview rows from the shared surface", () => {
const summary = createStatusCommandOverviewRowsParams().summary;
const rows = buildStatusAllOverviewRows({

View file

@ -3,6 +3,7 @@
import { formatCliCommand } from "../cli/command-format.js";
import { resolveIsNixMode } from "../config/paths.js";
import { formatMissingChildRuntimeWarning } from "../infra/child-runtime-viability.js";
import { isTruthyEnvValue } from "../infra/env.js";
import type { HeartbeatEventPayload } from "../infra/heartbeat-events.js";
import type { PluginCompatibilityNotice } from "../plugins/status.js";
@ -40,6 +41,7 @@ type StatusDegradationSummary = Pick<
| "startupMigrationWarning"
| "startupRecoveryWarning"
| "installationReplacementWarning"
| "childRuntime"
| "secretEgressProxy"
>;
@ -54,6 +56,12 @@ function buildStatusDegradationRows(
if (summary.startupRecoveryWarning) {
rows.push({ Item: "Session recovery", Value: decorate(summary.startupRecoveryWarning) });
}
const childRuntimeWarning = summary.childRuntime
? formatMissingChildRuntimeWarning(summary.childRuntime)
: undefined;
if (childRuntimeWarning) {
rows.push({ Item: "Gateway runtime", Value: decorate(childRuntimeWarning) });
}
if (summary.installationReplacementWarning) {
rows.push({
Item: "Installation replaced",

View file

@ -293,6 +293,27 @@ describe("status.command-sections", () => {
]);
});
it("warns when deep health says the retained Node executable is gone", () => {
const execPath = "/opt/homebrew/Cellar/node@24/24.20.0/bin/node";
const rows = buildStatusHealthRows({
health: {
durationMs: 42,
childRuntime: { execPath, available: false },
} as HealthSummary,
formatHealthChannelLines: () => ["Discord: OK"],
ok: (value) => `ok(${value})`,
warn: (value) => `warn(${value})`,
muted: (value) => `muted(${value})`,
});
expect(rows[0]).toEqual({ Item: "Gateway", Status: "ok(reachable)", Detail: "42ms" });
expect(rows[1]).toEqual({
Item: "Gateway runtime",
Status: "warn(WARN)",
Detail: `Gateway runtime is stale after Node upgrade: child workers are using ${execPath}, which no longer exists. Restart the Gateway.`,
});
});
it.each([
{ account: {}, status: "ok(OK)", detail: "healthy" },
{

View file

@ -6,6 +6,7 @@ import {
} from "../../packages/gateway-protocol/src/connect-error-details.js";
import type { TableColumn } from "../../packages/terminal-core/src/table.js";
import { areRuntimeModelRefsEquivalent } from "../agents/model-runtime-aliases.js";
import { formatMissingChildRuntimeWarning } from "../infra/child-runtime-viability.js";
import { formatDurationCompact } from "../infra/format-time/format-duration.js";
import type { HeartbeatEventPayload } from "../infra/heartbeat-events.js";
import type { Tone } from "../memory-host-sdk/status.js";
@ -252,6 +253,16 @@ export function buildStatusHealthRows(params: {
Detail: `${params.health.durationMs}ms`,
},
];
const childRuntimeWarning = params.health.childRuntime
? formatMissingChildRuntimeWarning(params.health.childRuntime)
: undefined;
if (childRuntimeWarning) {
rows.push({
Item: "Gateway runtime",
Status: params.warn("WARN"),
Detail: childRuntimeWarning,
});
}
const sqliteWalWarning = formatSqliteWalHealthWarning(params.sqliteWal);
if (sqliteWalWarning) {
rows.push({ Item: "SQLite WAL", Status: params.warn("WARN"), Detail: sqliteWalWarning });

View file

@ -164,6 +164,10 @@ describe("collectStatusScanOverview", () => {
degradedPlugins: [],
startupMigrationWarning: "Retained legacy state; run openclaw doctor --fix.",
installationReplacementWarning: "Installation replaced; draining before handoff.",
childRuntime: {
execPath: "/opt/homebrew/Cellar/node@24/24.20.0/bin/node",
available: false,
},
sqliteWal,
}
: { channelAccounts: {} },
@ -175,7 +179,7 @@ describe("collectStatusScanOverview", () => {
it("uses gateway fallback overrides for channels.status when requested", async () => {
const result = await collectStatusScanOverview({
commandName: "status --all",
opts: createStatusGatewayProbeBudget(1234),
opts: { ...createStatusGatewayProbeBudget(1234), deep: true },
showSecrets: false,
useGatewayCallOverridesForChannelsStatus: true,
});
@ -202,6 +206,10 @@ describe("collectStatusScanOverview", () => {
expect(result.runtimeDegradation?.installationReplacementWarning).toBe(
"Installation replaced; draining before handoff.",
);
expect(result.runtimeDegradation?.childRuntime).toEqual({
execPath: "/opt/homebrew/Cellar/node@24/24.20.0/bin/node",
available: false,
});
});
it("can keep channel overview on metadata-only status paths", async () => {
@ -222,6 +230,7 @@ describe("collectStatusScanOverview", () => {
expect(channelTableCall?.[1]?.showSecrets).toBe(false);
expect(channelTableCall?.[1]?.sourceConfig).toStrictEqual({ session: { raw: true } });
expect(result.channelIssues).toStrictEqual([]);
expect(result.runtimeDegradation).not.toHaveProperty("childRuntime");
});
it("skips channels.status when the gateway is unreachable", async () => {

View file

@ -104,6 +104,7 @@ export type StatusScanOverviewResult = {
| "degradedPlugins"
| "startupMigrationWarning"
| "installationReplacementWarning"
| "childRuntime"
| "secretEgressProxy"
| "sqliteWal"
> &
@ -120,7 +121,7 @@ export type StatusScanOverviewResult = {
export async function collectStatusScanOverview(params: {
env?: NodeJS.ProcessEnv;
commandName: string;
opts: StatusGatewayProbeBudget & { all?: boolean };
opts: StatusGatewayProbeBudget & { all?: boolean; deep?: boolean };
showSecrets: boolean;
runtime?: RuntimeEnv;
allowMissingConfigFastPath?: boolean;
@ -304,6 +305,7 @@ export async function collectStatusScanOverview(params: {
degradedPlugins: status.degradedPlugins ?? [],
startupMigrationWarning: status.startupMigrationWarning,
installationReplacementWarning: status.installationReplacementWarning,
...(params.opts.deep && status.childRuntime ? { childRuntime: status.childRuntime } : {}),
secretEgressProxy: status.secretEgressProxy,
sqliteWal: status.sqliteWal,
// The Gateway owns route readiness; CLI channel runtimes stay unloaded.

View file

@ -9,6 +9,7 @@ import {
getAgentEventLifecycleGeneration,
rotateAgentEventLifecycleGeneration,
} from "../../infra/agent-events.js";
import * as childRuntime from "../../infra/child-runtime-viability.js";
import { beginLifecycleWriteCustody } from "../../infra/lifecycle-write-custody.js";
import { recordStartupMigrationWarnings } from "../../infra/state-migrations.messages.js";
import { withStateDirEnv } from "../../test-helpers/state-dir-env.js";
@ -211,6 +212,46 @@ describe("Gateway status owner routing", () => {
},
);
it("reports a deleted child runtime executable without storing it on cached health", async () => {
const removed = "/opt/homebrew/Cellar/node@24/24.20.0/bin/node";
const read = vi.spyOn(childRuntime, "readChildRuntimeViability").mockReturnValue({
execPath: removed,
available: false,
});
const cached: HealthSummary = {
ok: true,
ts: Date.now(),
durationMs: 1,
channels: {},
channelOrder: [],
channelLabels: {},
heartbeatSeconds: 0,
agents: [],
sessions: { path: "/tmp/sessions.json", count: 0, recent: [] },
};
const respond = vi.fn();
await healthHandlers.health!({
req: {} as never,
params: {},
respond: respond as never,
context: {
getHealthCache: () => cached,
refreshHealthSnapshot: vi.fn(async () => cached),
getRuntimeSnapshot: () => ({ channels: {}, channelAccounts: {} }),
logHealth: { error: vi.fn() },
} as never,
client: { connect: { role: "operator", scopes: ["operator.read"] } } as never,
isWebchatConnect: () => false,
});
expect(read).toHaveBeenCalled();
expect(respond.mock.calls[0]?.[1].childRuntime).toEqual({
execPath: removed,
available: false,
});
expect(cached).not.toHaveProperty("childRuntime");
expect(respond.mock.calls[0]?.[3]).toEqual({ cached: true });
});
it("projects requested CLI facts without choosing a fleet owner or widening read scopes", async () => {
await withStateDirEnv("openclaw-gateway-cli-status-", async ({ stateDir }) => {
const config = {

View file

@ -3,6 +3,7 @@
import { isFutureDateTimestampMs } from "@openclaw/normalization-core/number-coercion";
import { getPreparedModelRuntimeStartupStatus } from "../../agents/prepared-model-runtime.startup-status.js";
import type { ChannelAccountSnapshot } from "../../channels/plugins/types.public.js";
import { readChildRuntimeViability } from "../../infra/child-runtime-viability.js";
import { formatErrorMessage as formatError } from "../../infra/errors.js";
import { readGatewayMaintenanceWork } from "../../infra/gateway-active-work.js";
import { getStatusSummary } from "../../status/summary.js";
@ -150,11 +151,15 @@ export const healthHandlers: GatewayRequestHandlers = {
) {
respond(
true,
await mergeCachedHealthRuntimeState({
cached,
getEventLoopHealth: context.getEventLoopHealth,
configReloadHotReloadStatus: context.getConfigReloaderHotReloadStatus?.(),
}),
{
...(await mergeCachedHealthRuntimeState({
cached,
getEventLoopHealth: context.getEventLoopHealth,
configReloadHotReloadStatus: context.getConfigReloaderHotReloadStatus?.(),
})),
// Live check. The cache must not keep a path that disappeared after it was stored.
childRuntime: readChildRuntimeViability(),
},
undefined,
{ cached: true },
);
@ -167,7 +172,15 @@ export const healthHandlers: GatewayRequestHandlers = {
}
await respondUnavailableOnThrow(respond, async () => {
const snap = await refreshHealthSnapshot({ probe: wantsProbe, includeSensitive });
respond(true, { ...snap, modelRuntime: getPreparedModelRuntimeStartupStatus() }, undefined);
respond(
true,
{
...snap,
modelRuntime: getPreparedModelRuntimeStartupStatus(),
childRuntime: readChildRuntimeViability(),
},
undefined,
);
});
},
status: async ({ respond, client, params, context }) => {
@ -202,6 +215,7 @@ export const healthHandlers: GatewayRequestHandlers = {
workerPools,
pid: process.pid,
shutdownBudget: shutdownStatus,
childRuntime: readChildRuntimeViability(),
},
undefined,
);

View file

@ -30,6 +30,7 @@ import {
resetContextEngineRuntimeQuarantineForTests,
} from "../../context-engine/registry.test-support.js";
import { emitAgentEvent } from "../../infra/agent-events.js";
import * as childRuntime from "../../infra/child-runtime-viability.js";
import {
buildSystemRunApprovalBinding,
buildSystemRunApprovalEnvBinding,
@ -4138,6 +4139,8 @@ describe("gateway healthHandlers.health cache freshness", () => {
let healthHandlers: typeof import("./health.js").healthHandlers;
let restoreContextEngineRegistryState: () => void;
const contextEngineTestOwner = "plugin:health-test";
const healthyChildRuntime = { execPath: "/test/node", available: true };
let restoreChildRuntime: () => void;
function createHealthSnapshot<T extends Record<string, unknown>>(overrides: T) {
return {
@ -4235,12 +4238,17 @@ describe("gateway healthHandlers.health cache freshness", () => {
});
beforeEach(() => {
const runtimeSpy = vi
.spyOn(childRuntime, "readChildRuntimeViability")
.mockReturnValue(healthyChildRuntime);
restoreChildRuntime = () => runtimeSpy.mockRestore();
restoreContextEngineRegistryState = captureContextEngineRegistryStateForTests();
registerLegacyContextEngine();
resetContextEngineRuntimeQuarantineForTests();
});
afterEach(() => {
restoreChildRuntime();
vi.useRealTimers();
restoreContextEngineRegistryState();
});
@ -4284,7 +4292,11 @@ describe("gateway healthHandlers.health cache freshness", () => {
const { respond, refreshHealthSnapshot } = await requestHealthSnapshot({ cached, fresh });
expect(refreshHealthSnapshot).toHaveBeenCalledOnce();
expect(respond).toHaveBeenCalledWith(true, fresh, undefined);
expect(respond).toHaveBeenCalledWith(
true,
{ ...fresh, childRuntime: healthyChildRuntime },
undefined,
);
});
it("restarts request-driven health refreshes when the clock moves backward", async () => {
@ -4316,7 +4328,11 @@ describe("gateway healthHandlers.health cache freshness", () => {
probe: true,
includeSensitive: true,
});
expect(respond).toHaveBeenCalledWith(true, fresh, undefined);
expect(respond).toHaveBeenCalledWith(
true,
{ ...fresh, childRuntime: healthyChildRuntime },
undefined,
);
});
it("maps health collection failures to UNAVAILABLE", async () => {
@ -4378,7 +4394,11 @@ describe("gateway healthHandlers.health cache freshness", () => {
probe: false,
includeSensitive: false,
});
expect(respond).toHaveBeenCalledWith(true, fresh, undefined);
expect(respond).toHaveBeenCalledWith(
true,
{ ...fresh, childRuntime: healthyChildRuntime },
undefined,
);
});
it("refreshes cached health when runtime channel lifecycle has changed", async () => {
@ -4412,7 +4432,11 @@ describe("gateway healthHandlers.health cache freshness", () => {
probe: false,
includeSensitive: false,
});
expect(respond).toHaveBeenCalledWith(true, fresh, undefined);
expect(respond).toHaveBeenCalledWith(
true,
{ ...fresh, childRuntime: healthyChildRuntime },
undefined,
);
});
it("refreshes cached health when recorded lifecycle changes without socket churn", async () => {
@ -4738,7 +4762,11 @@ describe("gateway healthHandlers.health cache freshness", () => {
probe: false,
includeSensitive: false,
});
expect(respond).toHaveBeenCalledWith(true, fresh, undefined);
expect(respond).toHaveBeenCalledWith(
true,
{ ...fresh, childRuntime: healthyChildRuntime },
undefined,
);
},
);
});

View file

@ -0,0 +1,45 @@
import { describe, expect, it } from "vitest";
import {
formatMissingChildRuntimeWarning,
readChildRuntimeViability,
} from "./child-runtime-viability.ts";
const removedCellarPath = "/opt/homebrew/Cellar/node@24/24.20.0/bin/node";
describe("child runtime viability", () => {
it("treats a missing executable as a stale runtime", () => {
const viability = readChildRuntimeViability({
execPath: removedCellarPath,
access: () => {
throw Object.assign(new Error("spawn ENOENT"), { code: "ENOENT" });
},
});
expect(viability).toEqual({ execPath: removedCellarPath, available: false });
expect(formatMissingChildRuntimeWarning(viability)).toBe(
`Gateway runtime is stale after Node upgrade: child workers are using ${removedCellarPath}, which no longer exists. Restart the Gateway.`,
);
});
it("stays quiet when the retained executable can still be started", () => {
const viability = readChildRuntimeViability({
execPath: process.execPath,
access: () => undefined,
});
expect(viability).toEqual({ execPath: process.execPath, available: true });
expect(formatMissingChildRuntimeWarning(viability)).toBeUndefined();
});
it("does not call a permission error a deleted Node path", () => {
const viability = readChildRuntimeViability({
execPath: removedCellarPath,
access: () => {
throw Object.assign(new Error("permission denied"), { code: "EACCES" });
},
});
expect(viability.available).toBe(true);
expect(formatMissingChildRuntimeWarning(viability)).toBeUndefined();
});
});

View file

@ -0,0 +1,54 @@
// The running Gateway keeps the Node path it was launched with.
// A Homebrew upgrade can delete that Cellar binary while the process stays up.
import { accessSync, constants } from "node:fs";
import { sanitizeTerminalText } from "../../packages/terminal-core/src/safe-text.js";
export type ChildRuntimeViability = {
execPath: string;
available: boolean;
};
function errnoCode(error: unknown): string | undefined {
if (!(error instanceof Error) || !("code" in error)) {
return undefined;
}
const code = error.code;
return typeof code === "string" ? code : undefined;
}
function accessExecutable(execPath: string): void {
accessSync(execPath, constants.X_OK);
}
/** Reports whether this process can still spawn children with its own Node binary. */
export function readChildRuntimeViability(params?: {
execPath?: string;
access?: (execPath: string) => void;
}): ChildRuntimeViability {
const execPath = params?.execPath ?? process.execPath;
const access = params?.access ?? accessExecutable;
try {
access(execPath);
return { execPath, available: true };
} catch (error) {
const code = errnoCode(error);
// Only a removed path matches the Homebrew Cellar failure. Other access
// errors are not this diagnostic.
if (code === "ENOENT" || code === "ENOTDIR") {
return { execPath, available: false };
}
return { execPath, available: true };
}
}
/** Operator text for a Gateway whose retained Node binary is gone. */
export function formatMissingChildRuntimeWarning(
viability: ChildRuntimeViability,
): string | undefined {
if (viability.available) {
return undefined;
}
const execPath = sanitizeTerminalText(viability.execPath);
return `Gateway runtime is stale after Node upgrade: child workers are using ${execPath}, which no longer exists. Restart the Gateway.`;
}

View file

@ -574,6 +574,10 @@ export type StatusSummary = Omit<
"heartbeat" | "channelSummary" | "queuedSystemEvents" | "sessions"
> & {
runtimeVersion?: string | null;
childRuntime?: {
execPath: string;
available: boolean;
};
eventLoop?: NonNullable<SystemInfoResult["eventLoop"]>;
processMemory?: NonNullable<SystemInfoResult["processMemory"]>;
degradedSecretOwners?: Array<