vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-09 15:59:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

fix(gateway): keep N-1 nodes manageable during upgrades (#101109)

Browse source 
* fix(gateway): allow N-1 node protocol maintenance

* docs: refresh generated map for node upgrade guide
This commit is contained in:
Peter Steinberger 2026-07-06 20:01:20 +01:00 • committed by GitHub
parent 2f89de8165
commit 912af0a56f
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 263 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

1
docs/docs_map.md
View file

@ -4676,6 +4676,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
- Route: /nodes
- Headings:
- H2: Pairing + status
- H2: Version skew and upgrade order
- H2: Remote node host (system.run)
- H3: Start a node host (foreground)
- H3: Remote gateway via SSH tunnel (loopback bind)

20
docs/gateway/protocol.md
View file

@ -720,12 +720,18 @@ context.
## Versioning
- `PROTOCOL_VERSION` and `MIN_CLIENT_PROTOCOL_VERSION` live in
`packages/gateway-protocol/src/version.ts`. Both are currently `4`.
- Clients send `minProtocol` + `maxProtocol`; the gateway accepts a connect
when `maxProtocol >= PROTOCOL_VERSION && minProtocol <= PROTOCOL_VERSION`
(`src/gateway/server/ws-connection/message-handler.ts`). Current clients and
servers both run protocol v4.
- `PROTOCOL_VERSION`, `MIN_CLIENT_PROTOCOL_VERSION`,
`MIN_NODE_PROTOCOL_VERSION`, and `MIN_PROBE_PROTOCOL_VERSION` live in
`packages/gateway-protocol/src/version.ts`.
- Clients send `minProtocol` + `maxProtocol`. Operator and UI clients must
include the current protocol in that range; current clients and servers run
protocol v4.
- Authenticated clients with both `role: "node"` and `client.mode: "node"`
may use the N-1 node protocol (currently v3). Lightweight restart probes use
the same N-1 window. Device auth, pairing, scopes, command policy, and exec
approvals are unchanged by this compatibility window. Plugin-owned node
capabilities and commands are withheld until the node upgrades to the current
protocol because their hosted surfaces are not part of the N-1 contract.
- Schemas and models are generated from TypeBox definitions:
- `pnpm protocol:gen`
- `pnpm protocol:gen:swift`
@ -742,6 +748,8 @@ third-party clients.
| ----------------------------------------- | ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
| `PROTOCOL_VERSION` | `4` | `packages/gateway-protocol/src/version.ts` |
| `MIN_CLIENT_PROTOCOL_VERSION` | `4` | `packages/gateway-protocol/src/version.ts` |
| `MIN_NODE_PROTOCOL_VERSION` | `3` | `packages/gateway-protocol/src/version.ts` |
| `MIN_PROBE_PROTOCOL_VERSION` | `3` | `packages/gateway-protocol/src/version.ts` |
| Request timeout (per RPC) | `30_000` ms | `packages/gateway-client/src/client.ts` (`requestTimeoutMs`) |
| Preauth / connect-challenge timeout | `15_000` ms | `packages/gateway-client/src/timeouts.ts` (`OPENCLAW_HANDSHAKE_TIMEOUT_MS` env can raise the paired server/client budget) |
| Initial reconnect backoff | `1_000` ms | `packages/gateway-client/src/client.ts` (`backoffMs`) |

15
docs/nodes/index.md
View file

@ -40,6 +40,21 @@ Pending pairing requests expire 5 minutes after the device's last retry — a de
- non-exec node commands: `operator.pairing` + `operator.write`
- `system.run` / `system.run.prepare` / `system.which`: `operator.pairing` + `operator.admin`
## Version skew and upgrade order
The Gateway accepts authenticated node clients across an N-1 protocol window.
The current v4 Gateway therefore accepts v3 nodes when the connection declares
both `role: "node"` and `client.mode: "node"`. Operator and UI sessions must
still use the current protocol.
For staged fleet upgrades, upgrade the Gateway first, then upgrade each node.
An N-1 node remains visible and manageable while it is upgraded; the Gateway
logs `legacy node protocol accepted` with an upgrade recommendation. Pairing,
device authentication, command allowlists, and exec approvals still apply.
Plugin-owned capabilities and commands stay hidden until the node upgrades to
the current protocol. Nodes older than N-1 require an out-of-band upgrade before
reconnecting.
## Remote node host (system.run)
Use a **node host** when your Gateway runs on one machine and you want commands to execute on another. The model still talks to the **gateway**; the gateway forwards `exec` calls to the **node host** when `host=node` is selected.

2
packages/gateway-protocol/src/index.ts
View file

@ -356,6 +356,7 @@ import {
type PollParams,
PollParamsSchema,
MIN_CLIENT_PROTOCOL_VERSION,
MIN_NODE_PROTOCOL_VERSION,
MIN_PROBE_PROTOCOL_VERSION,
PROTOCOL_VERSION,
type PushTestParams,
@ -1396,6 +1397,7 @@ export {
WorktreesGcResultSchema,
ProtocolSchemas,
MIN_CLIENT_PROTOCOL_VERSION,
MIN_NODE_PROTOCOL_VERSION,
MIN_PROBE_PROTOCOL_VERSION,
PROTOCOL_VERSION,
ErrorCodes,

15
packages/gateway-protocol/src/native-protocol-levels.guard.test.ts
View file

@ -3,7 +3,12 @@ import fs from "node:fs/promises";
import path from "node:path";
import { describe, it } from "vitest";
import { ProtocolSchemas } from "./schema/protocol-schemas.js";
import { MIN_CLIENT_PROTOCOL_VERSION, PROTOCOL_VERSION } from "./version.js";
import {
MIN_CLIENT_PROTOCOL_VERSION,
MIN_NODE_PROTOCOL_VERSION,
MIN_PROBE_PROTOCOL_VERSION,
PROTOCOL_VERSION,
} from "./version.js";
/**
* Cross-language guard for Gateway protocol version constants.
@ -99,6 +104,14 @@ describe("native Gateway protocol levels", () => {
`packages/gateway-protocol/src/version.ts: MIN_CLIENT_PROTOCOL_VERSION (${MIN_CLIENT_PROTOCOL_VERSION}) must not exceed PROTOCOL_VERSION (${PROTOCOL_VERSION}).`,
);
}
if (
MIN_NODE_PROTOCOL_VERSION !== PROTOCOL_VERSION - 1 ||
MIN_PROBE_PROTOCOL_VERSION !== PROTOCOL_VERSION - 1
) {
throw new Error(
"packages/gateway-protocol/src/version.ts: node and probe compatibility must remain exactly N-1.",
);
}
const swiftGeneratedPath =
"apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift";

1
packages/gateway-protocol/src/schema/protocol-schemas.ts
View file

@ -704,6 +704,7 @@ export const ProtocolSchemas = {
export {
MIN_CLIENT_PROTOCOL_VERSION,
MIN_NODE_PROTOCOL_VERSION,
MIN_PROBE_PROTOCOL_VERSION,
PROTOCOL_VERSION,
} from "../version.js";

6
packages/gateway-protocol/src/version.ts
View file

@ -1,6 +1,8 @@
/** Current gateway protocol version emitted by modern clients and servers. */
export const PROTOCOL_VERSION = 4 as const;
/** Lowest client protocol version accepted by the gateway. */
/** Lowest general client protocol version accepted by the gateway. */
export const MIN_CLIENT_PROTOCOL_VERSION = 4 as const;
/** Lowest authenticated node protocol version accepted by the gateway. */
export const MIN_NODE_PROTOCOL_VERSION = 3 as const;
/** Lowest lightweight probe protocol version accepted by the gateway. */
export const MIN_PROBE_PROTOCOL_VERSION = 4 as const;
export const MIN_PROBE_PROTOCOL_VERSION = 3 as const;

16
src/gateway/node-command-policy.test.ts
View file

@ -14,6 +14,7 @@ import {
setActivePluginRegistry,
} from "../plugins/runtime.js";
import {
filterLegacyNodeProtocolFeatures,
isForegroundRestrictedPluginNodeCommand,
isNodeCommandAllowed,
normalizeDeclaredNodeCommands,
@ -115,6 +116,21 @@ describe("gateway/node-command-policy", () => {
expect(allowlist.has("canvas.present")).toBe(true);
});
it("suppresses plugin-owned features for legacy protocol nodes", () => {
installCanvasPluginDefaults();
expect(
filterLegacyNodeProtocolFeatures({
caps: ["canvas", "device"],
commands: ["canvas.snapshot", "device.info"],
pluginSurfaces: ["canvas"],
}),
).toEqual({
caps: ["device"],
commands: ["device.info"],
});
});
it("keeps plugin node defaults from the pinned Gateway registry", () => {
const startupRegistry = installCanvasPluginDefaults();
pinActivePluginChannelRegistry(startupRegistry);

26
src/gateway/node-command-policy.ts
View file

@ -270,6 +270,32 @@ export function isForegroundRestrictedPluginNodeCommand(command: string): boolea
);
}
export function filterLegacyNodeProtocolFeatures(params: {
caps: readonly string[];
commands: readonly string[];
pluginSurfaces: readonly string[];
}): { caps: string[]; commands: string[] } {
// N-1 nodes predate plugin-hosted surfaces. Preserve their durable pairing
// declarations elsewhere, but hide unusable plugin features from this session.
const registry = getActivePluginGatewayNodePolicyRegistry();
if (!registry) {
return { caps: [...params.caps], commands: [...params.commands] };
}
const pluginIds = new Set([
...registry.nodeHostCommands.map((entry) => entry.pluginId),
...registry.nodeInvokePolicies.map((entry) => entry.pluginId),
]);
const pluginCaps = new Set([...params.pluginSurfaces, ...pluginIds]);
const pluginCommands = new Set([
...registry.nodeHostCommands.map((entry) => entry.command.command),
...registry.nodeInvokePolicies.flatMap((entry) => entry.policy.commands),
]);
return {
caps: params.caps.filter((cap) => !pluginCaps.has(cap)),
commands: params.commands.filter((command) => !pluginCommands.has(command)),
};
}
type NodeCommandPolicyNode = Pick<NodeSession, "platform" | "deviceFamily"> &
Partial<Pick<NodeSession, "caps" | "commands" | "connId" | "nodeId">> & {
approvedCommands?: readonly string[];

27
src/gateway/node-registry.test.ts
View file

@ -26,6 +26,8 @@ function makeClient(
declaredCaps?: string[];
declaredCommands?: string[];
declaredPermissions?: Record<string, boolean>;
sessionCapsCeiling?: string[];
sessionCommandsCeiling?: string[];
socket?: GatewayWsClient["socket"];
} = {},
): GatewayWsClient {
@ -63,6 +65,8 @@ function makeClient(
declaredCaps: opts.declaredCaps,
declaredCommands: opts.declaredCommands,
declaredPermissions: opts.declaredPermissions,
sessionCapsCeiling: opts.sessionCapsCeiling,
sessionCommandsCeiling: opts.sessionCommandsCeiling,
} as unknown as GatewayWsClient["connect"],
};
}
@ -619,4 +623,27 @@ describe("gateway/node-registry", () => {
(client.connect as { permissions?: Record<string, boolean> }).permissions,
).toBeUndefined();
});
it("preserves a legacy session feature ceiling across surface approvals", () => {
const registry = new NodeRegistry();
const client = makeClient("conn-1", "node-1", [], {
caps: [],
commands: [],
declaredCaps: ["canvas", "device"],
declaredCommands: ["canvas.snapshot", "device.info"],
sessionCapsCeiling: ["device"],
sessionCommandsCeiling: ["device.info"],
});
registry.register(client, {});
const updated = registry.updateSurface("node-1", {
caps: ["canvas", "device"],
commands: ["canvas.snapshot", "device.info"],
});
expect(updated?.declaredCaps).toEqual(["canvas", "device"]);
expect(updated?.declaredCommands).toEqual(["canvas.snapshot", "device.info"]);
expect(updated?.caps).toEqual(["device"]);
expect(updated?.commands).toEqual(["device.info"]);
});
});

26
src/gateway/node-registry.ts
View file

@ -27,8 +27,10 @@ export type NodeSession = {
modelIdentifier?: string;
remoteIp?: string;
declaredCaps: string[];
sessionCapsCeiling?: string[];
caps: string[];
declaredCommands: string[];
sessionCommandsCeiling?: string[];
commands: string[];
declaredPermissions?: Record<string, boolean>;
permissions?: Record<string, boolean>;
@ -194,6 +196,18 @@ export class NodeRegistry {
)
? ((connect as { declaredCommands?: string[] }).declaredCommands ?? [])
: commands;
// Session ceilings preserve protocol compatibility across later pairing
// approvals while declared* retains the durable approval surface.
const sessionCapsCeiling = Array.isArray(
(connect as { sessionCapsCeiling?: string[] }).sessionCapsCeiling,
)
? ((connect as { sessionCapsCeiling?: string[] }).sessionCapsCeiling ?? [])
: declaredCaps;
const sessionCommandsCeiling = Array.isArray(
(connect as { sessionCommandsCeiling?: string[] }).sessionCommandsCeiling,
)
? ((connect as { sessionCommandsCeiling?: string[] }).sessionCommandsCeiling ?? [])
: declaredCommands;
const permissions =
typeof (connect as { permissions?: Record<string, boolean> }).permissions === "object"
? ((connect as { permissions?: Record<string, boolean> }).permissions ?? undefined)
@ -223,8 +237,10 @@ export class NodeRegistry {
modelIdentifier: connect.client.modelIdentifier,
remoteIp: opts.remoteIp,
declaredCaps,
sessionCapsCeiling,
caps,
declaredCommands,
sessionCommandsCeiling,
commands,
declaredPermissions,
permissions,
@ -375,14 +391,16 @@ export class NodeRegistry {
}
// Runtime approvals can only narrow capabilities/commands/permissions declared at connect.
const declaredCommands = new Set(node.declaredCommands);
const nextCommands = surface.commands.filter((command) => declaredCommands.has(command));
const sessionCommandsCeiling = new Set(node.sessionCommandsCeiling ?? node.declaredCommands);
const nextCommands = surface.commands.filter((command) => sessionCommandsCeiling.has(command));
node.commands = nextCommands;
(node.client.connect as { commands?: string[] }).commands = nextCommands;
if ("caps" in surface) {
const declaredCaps = new Set(node.declaredCaps);
const nextCaps = (surface.caps ?? []).filter((capability) => declaredCaps.has(capability));
const sessionCapsCeiling = new Set(node.sessionCapsCeiling ?? node.declaredCaps);
const nextCaps = (surface.caps ?? []).filter((capability) =>
sessionCapsCeiling.has(capability),
);
node.caps = nextCaps;
(node.client.connect as { caps?: string[] }).caps = nextCaps;
}

79
src/gateway/server.auth.default-token.suite.ts
View file

@ -2,7 +2,10 @@
// protocol version checks, and token-backed operator/node clients.
import { afterAll, beforeAll, describe, expect, test, vi } from "vitest";
import { WebSocket } from "ws";
import { GATEWAY_SERVER_CAPS } from "../../packages/gateway-protocol/src/index.js";
import {
GATEWAY_SERVER_CAPS,
MIN_NODE_PROTOCOL_VERSION,
} from "../../packages/gateway-protocol/src/index.js";
import {
connectReq,
ConnectErrorDetailCodes,
@ -462,12 +465,82 @@ export function registerDefaultAuthTokenSuite(): void {
ws.close();
});
test("allows authenticated previous-protocol nodes to register for maintenance", async () => {
const nodeWs = await openWs(port);
const operatorWs = await openWs(port);
try {
const legacyVersion = "2026.5.7";
const nodeRes = await connectReq(nodeWs, {
minProtocol: MIN_NODE_PROTOCOL_VERSION,
maxProtocol: MIN_NODE_PROTOCOL_VERSION,
role: "node",
client: { ...NODE_CLIENT, version: legacyVersion },
});
expect(nodeRes.ok).toBe(true);
const operatorRes = await connectReq(operatorWs);
expect(operatorRes.ok).toBe(true);
const listRes = await rpcReq<{ nodes?: Array<{ connected?: boolean; version?: string }> }>(
operatorWs,
"node.list",
{},
);
expect(listRes.ok).toBe(true);
expect(
listRes.payload?.nodes?.some(
(node) => node.connected === true && node.version === legacyVersion,
),
).toBe(true);
} finally {
nodeWs.close();
operatorWs.close();
}
});
test("keeps previous-protocol node connections behind gateway auth", async () => {
const ws = await openWs(port);
try {
const res = await connectReq(ws, {
minProtocol: MIN_NODE_PROTOCOL_VERSION,
maxProtocol: MIN_NODE_PROTOCOL_VERSION,
role: "node",
client: NODE_CLIENT,
token: "invalid-token",
});
expect(res.ok).toBe(false);
expect((res.error?.details as { code?: unknown } | undefined)?.code).toBe(
ConnectErrorDetailCodes.AUTH_TOKEN_MISMATCH,
);
} finally {
ws.close();
}
});
test("rejects node protocols older than the N-1 window", async () => {
const ws = await openWs(port);
try {
const unsupportedProtocol = MIN_NODE_PROTOCOL_VERSION - 1;
const res = await connectReq(ws, {
minProtocol: unsupportedProtocol,
maxProtocol: unsupportedProtocol,
role: "node",
client: NODE_CLIENT,
});
expect(res.ok).toBe(false);
expect((res.error?.details as { code?: unknown } | undefined)?.code).toBe(
ConnectErrorDetailCodes.PROTOCOL_MISMATCH,
);
} finally {
ws.close();
}
});
test("keeps previous protocol rejected for non-probe clients", async () => {
const ws = await openWs(port);
try {
const res = await connectReq(ws, {
minProtocol: MIN_PROBE_PROTOCOL_VERSION,
maxProtocol: MIN_PROBE_PROTOCOL_VERSION,
minProtocol: MIN_NODE_PROTOCOL_VERSION,
maxProtocol: MIN_NODE_PROTOCOL_VERSION,
});
expect(res.ok).toBe(false);
} catch {

49
src/gateway/server/ws-connection/message-handler.ts
View file

@ -28,6 +28,7 @@ import {
errorShape,
formatValidationErrors,
GATEWAY_SERVER_CAPS,
MIN_NODE_PROTOCOL_VERSION,
MIN_PROBE_PROTOCOL_VERSION,
PROTOCOL_VERSION,
validateConnectParams,
@ -120,6 +121,7 @@ import {
isTrustedProxyAddress,
resolveClientIp,
} from "../../net.js";
import { filterLegacyNodeProtocolFeatures } from "../../node-command-policy.js";
import { reconcileNodePairingOnConnect } from "../../node-connect-reconcile.js";
import {
resolveNodePairingClientIpSource,
@ -814,7 +816,19 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
connectParams.client.mode === GATEWAY_CLIENT_MODES.PROBE &&
maxProtocol >= MIN_PROBE_PROTOCOL_VERSION &&
minProtocol <= PROTOCOL_VERSION;
if (!supportsCurrentProtocol && !supportsProbeRestartProtocol) {
// Protocol v4 changed chat deltas, not node RPC frames. Keep N-1 limited to
// the node role+mode so stale operator/UI clients cannot enter the v4 surface.
const supportsPreviousNodeProtocol =
connectParams.role === "node" &&
connectParams.client.mode === GATEWAY_CLIENT_MODES.NODE &&
maxProtocol >= MIN_NODE_PROTOCOL_VERSION &&
minProtocol <= MIN_NODE_PROTOCOL_VERSION;
const usesLegacyNodeProtocol = !supportsCurrentProtocol && supportsPreviousNodeProtocol;
if (
!supportsCurrentProtocol &&
!supportsProbeRestartProtocol &&
!supportsPreviousNodeProtocol
) {
markHandshakeFailure("protocol-mismatch", {
minProtocol,
maxProtocol,
@ -1842,12 +1856,34 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
declaredCaps?: string[];
declaredCommands?: string[];
declaredPermissions?: Record<string, boolean>;
sessionCapsCeiling?: string[];
sessionCommandsCeiling?: string[];
};
nodeConnectParams.declaredCaps = reconciliation.declaredCaps;
nodeConnectParams.declaredCommands = reconciliation.declaredCommands;
nodeConnectParams.declaredPermissions = reconciliation.declaredPermissions;
connectParams.caps = reconciliation.effectiveCaps;
connectParams.commands = reconciliation.effectiveCommands;
const pluginSurfaces = pluginNodeCapabilities.map((surface) => surface.surface);
if (usesLegacyNodeProtocol) {
const sessionCeiling = filterLegacyNodeProtocolFeatures({
caps: reconciliation.declaredCaps,
commands: reconciliation.declaredCommands,
pluginSurfaces,
});
nodeConnectParams.sessionCapsCeiling = sessionCeiling.caps;
nodeConnectParams.sessionCommandsCeiling = sessionCeiling.commands;
}
const effectiveFeatures = usesLegacyNodeProtocol
? filterLegacyNodeProtocolFeatures({
caps: reconciliation.effectiveCaps,
commands: reconciliation.effectiveCommands,
pluginSurfaces,
})
: {
caps: reconciliation.effectiveCaps,
commands: reconciliation.effectiveCommands,
};
connectParams.caps = effectiveFeatures.caps;
connectParams.commands = effectiveFeatures.commands;
connectParams.permissions = reconciliation.effectivePermissions;
}
@ -1873,7 +1909,7 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
capability: string;
expiresAtMs: number;
}> = [];
if (pluginSurfaceBaseUrl) {
if (pluginSurfaceBaseUrl && !usesLegacyNodeProtocol) {
for (const pluginCapabilitySurface of Object.values(pluginNodeCapabilitySurfaces)) {
const capability = mintPluginNodeCapabilityToken();
const expiresAtMs = resolvePluginNodeCapabilityExpiresAtMs(pluginCapabilitySurface);
@ -1940,6 +1976,11 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar
: {}),
}
: undefined;
if (usesLegacyNodeProtocol) {
logWsControl.warn(
`legacy node protocol accepted conn=${connId} client=${formatForLog(clientLabel)} v${formatForLog(connectParams.client.version)} min=${minProtocol} max=${maxProtocol} current=${PROTOCOL_VERSION}; upgrade recommended`,
);
}
clearHandshakeTimer();
const nextClient: GatewayWsClient = {
socket,