diff --git a/docs/reference/test/runner-internals.md b/docs/reference/test/runner-internals.md index ea3717953975..f81ec1cb603b 100644 --- a/docs/reference/test/runner-internals.md +++ b/docs/reference/test/runner-internals.md @@ -187,6 +187,12 @@ Vitest namespaces, found through their explicit resource owners. Parallel invoca therefore share port ownership while a fixture hands its reserved socket to a child; removing one invocation's files cannot remove another fixture's port claim. +A fixture that binds a Gateway, in-process or spawned, on a shared pool port holds +that port's claim from selection until the Gateway closes. A Gateway retries a busy +port while starting, so an unclaimed fixture can take another fixture's port during +its handoff. `getDeterministicFreePortBlock` is a probe, not a lease; in-process +Gateway E2E fixtures use `acquireGatewayE2ePortBlock` with `startClaimedGateway`. + Live-aware setup still loads the original profile and stages live state when requested. A bounded invocation artifact carries the original home to that setup; it does not grant live access, and hermetic setup never consults it. Known diff --git a/src/agents/bash-tools.exec-gateway-approval.e2e.test.ts b/src/agents/bash-tools.exec-gateway-approval.e2e.test.ts index 10d42778df76..9b62d5cc4577 100644 --- a/src/agents/bash-tools.exec-gateway-approval.e2e.test.ts +++ b/src/agents/bash-tools.exec-gateway-approval.e2e.test.ts @@ -16,12 +16,12 @@ import { clearSessionStoreCacheForTest } from "../config/sessions/store-writer-s import type { OpenClawConfig } from "../config/types.openclaw.js"; import { ADMIN_SCOPE, APPROVALS_SCOPE } from "../gateway/method-scopes.js"; import { startGatewayServer } from "../gateway/server.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "../gateway/test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "../gateway/test-helpers.e2e.js"; import { GATEWAY_STARTUP_MUTATED_ENV_KEYS } from "../gateway/test-helpers.env.js"; +import { + acquireGatewayE2ePortBlock, + startClaimedGateway, +} from "../gateway/test-helpers.listener.js"; import { loadOrCreateDeviceIdentity } from "../infra/device-identity.js"; import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js"; @@ -85,10 +85,10 @@ describe("gateway-hosted exec approvals", () => { const workspaceDir = path.join(tempHome, "workspace"); await fs.mkdir(workspaceDir, { recursive: true }); - const port = await getGatewayE2ePortBlock(); const token = "exec-approval-e2e-token"; const configPath = path.join(stateDir, "openclaw.json"); await fs.mkdir(stateDir, { recursive: true }); + const claim = await acquireGatewayE2ePortBlock(); const config = { agents: { ownership: "explicit", @@ -96,7 +96,7 @@ describe("gateway-hosted exec approvals", () => { list: [{ id: "main", tools: { exec: { cleanupMs: 180_000 } } }, { id: "helper" }], }, gateway: { - port, + port: claim.port, auth: { mode: "token", token }, }, tools: { @@ -108,34 +108,36 @@ describe("gateway-hosted exec approvals", () => { }, }, } satisfies OpenClawConfig; - await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); + const server = await startClaimedGateway(claim, async () => { + await fs.writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); - setTestEnvValue("HOME", tempHome); - setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); - setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); - setTestEnvValue("OPENCLAW_GATEWAY_TOKEN", token); - setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(port)); - setTestEnvValue("OPENCLAW_SKIP_CHANNELS", "1"); - setTestEnvValue("OPENCLAW_SKIP_GMAIL_WATCHER", "1"); - setTestEnvValue("OPENCLAW_SKIP_CRON", "1"); - setTestEnvValue("OPENCLAW_SKIP_CANVAS_HOST", "1"); - setTestEnvValue("OPENCLAW_SKIP_BROWSER_CONTROL_SERVER", "1"); - setTestEnvValue("OPENCLAW_SKIP_PROVIDERS", "1"); - setTestEnvValue("OPENCLAW_TEST_MINIMAL_GATEWAY", "1"); - clearRuntimeConfigSnapshot(); - clearConfigCache(); - clearSessionStoreCacheForTest(); + setTestEnvValue("HOME", tempHome); + setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); + setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); + setTestEnvValue("OPENCLAW_GATEWAY_TOKEN", token); + setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(claim.port)); + setTestEnvValue("OPENCLAW_SKIP_CHANNELS", "1"); + setTestEnvValue("OPENCLAW_SKIP_GMAIL_WATCHER", "1"); + setTestEnvValue("OPENCLAW_SKIP_CRON", "1"); + setTestEnvValue("OPENCLAW_SKIP_CANVAS_HOST", "1"); + setTestEnvValue("OPENCLAW_SKIP_BROWSER_CONTROL_SERVER", "1"); + setTestEnvValue("OPENCLAW_SKIP_PROVIDERS", "1"); + setTestEnvValue("OPENCLAW_TEST_MINIMAL_GATEWAY", "1"); + clearRuntimeConfigSnapshot(); + clearConfigCache(); + clearSessionStoreCacheForTest(); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - sidecarStartup: "defer", + return await startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + sidecarStartup: "defer", + }); }); cleanup.push(() => server.close()); const operator = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, clientName: GATEWAY_CLIENT_NAMES.TEST, clientDisplayName: "approval operator", @@ -154,7 +156,7 @@ describe("gateway-hosted exec approvals", () => { path: path.join(stateDir, "test-device-identities", "other-reviewer.sqlite"), }); const originReviewer = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, clientName: GATEWAY_CLIENT_NAMES.TEST, clientDisplayName: "origin approval reviewer", @@ -167,7 +169,7 @@ describe("gateway-hosted exec approvals", () => { }); cleanup.push(() => disconnectGatewayClient(originReviewer)); const otherReviewer = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, clientName: GATEWAY_CLIENT_NAMES.TEST, clientDisplayName: "other approval reviewer", diff --git a/src/cron/session-reaper.worker-placement.e2e.test.ts b/src/cron/session-reaper.worker-placement.e2e.test.ts index 1a6b44900638..76c5f8f04065 100644 --- a/src/cron/session-reaper.worker-placement.e2e.test.ts +++ b/src/cron/session-reaper.worker-placement.e2e.test.ts @@ -21,7 +21,10 @@ import { } from "../gateway/gateway.test-support.js"; import { startGatewayServer } from "../gateway/server.js"; import type { SessionsListResult } from "../gateway/session-utils.types.js"; -import { getGatewayE2ePortBlock } from "../gateway/test-helpers.e2e.js"; +import { + acquireGatewayE2ePortBlock, + startClaimedGateway, +} from "../gateway/test-helpers.listener.js"; import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { deleteTestEnvValue, setTestEnvValue } from "../test-utils/env.js"; @@ -124,13 +127,13 @@ describe("scheduled cron session retirement through the Gateway", () => { }, };`, ); - const port = await getGatewayE2ePortBlock(); const token = "synthetic-cron-retirement-token"; const configPath = await createGatewayConfigPath(setup.tempHome); + const claim = await acquireGatewayE2ePortBlock(); const config: OpenClawConfig = { gateway: { mode: "local", - port, + port: claim.port, bind: "loopback", auth: { mode: "token", token }, controlUi: { enabled: false }, @@ -151,13 +154,15 @@ describe("scheduled cron session retirement through the Gateway", () => { }, cron: { enabled: true }, }; - await fs.writeFile(configPath, JSON.stringify(config)); - setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); - setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(port)); - setTestEnvValue("OPENCLAW_GATEWAY_TOKEN", token); stateDatabasePath = path.join(setup.tempHome, ".openclaw", "state", "openclaw.sqlite"); - sessionStorePath = resolveDefaultSessionStorePath("main"); - server = await startGatewayServer(port, { controlUiEnabled: false }); + server = await startClaimedGateway(claim, async () => { + await fs.writeFile(configPath, JSON.stringify(config)); + setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); + setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(claim.port)); + setTestEnvValue("OPENCLAW_GATEWAY_TOKEN", token); + sessionStorePath = resolveDefaultSessionStorePath("main"); + return await startGatewayServer(claim.port, { controlUiEnabled: false }); + }); }, 120_000); afterAll(async () => { diff --git a/src/gateway/gateway-wizard-cancel.e2e.test.ts b/src/gateway/gateway-wizard-cancel.e2e.test.ts index e95c884525a4..b9a16dbab9fe 100644 --- a/src/gateway/gateway-wizard-cancel.e2e.test.ts +++ b/src/gateway/gateway-wizard-cancel.e2e.test.ts @@ -22,12 +22,9 @@ import { import { captureEnv, deleteTestEnvValue, setTestEnvValue } from "../test-utils/env.js"; import type { SetupWizardRunner } from "./server-methods/wizard.js"; import { startGatewayServer } from "./server.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "./test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "./test-helpers.e2e.js"; import { GATEWAY_STARTUP_MUTATED_ENV_KEYS } from "./test-helpers.env.js"; +import { acquireGatewayE2ePortBlock, startClaimedGateway } from "./test-helpers.listener.js"; const GATEWAY_E2E_TIMEOUT_MS = 90_000; const ENV_KEYS = [ @@ -91,19 +88,21 @@ async function withWizardGateway( setTestEnvValue("OPENCLAW_BUNDLED_PLUGINS_DIR", bundledPluginsDir); setTestEnvValue("OPENCLAW_DISABLE_BUNDLED_PLUGINS", "1"); setTestEnvValue("OPENCLAW_TEST_MINIMAL_GATEWAY", "1"); - const port = await getGatewayE2ePortBlock(); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - wizardRunner, - }); + const claim = await acquireGatewayE2ePortBlock(); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + wizardRunner, + }), + ); try { await run({ server, connect: async () => { const client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, clientDisplayName: "vitest-wizard-cancel", }); diff --git a/src/gateway/gateway-wizard.e2e.test.ts b/src/gateway/gateway-wizard.e2e.test.ts index f3ce9f9ffb58..fb695610ff8c 100644 --- a/src/gateway/gateway-wizard.e2e.test.ts +++ b/src/gateway/gateway-wizard.e2e.test.ts @@ -10,11 +10,8 @@ import { setupGatewayTempHome, } from "./gateway.test-support.js"; import { startGatewayServer } from "./server.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "./test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "./test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock, startClaimedGateway } from "./test-helpers.listener.js"; const GATEWAY_E2E_TIMEOUT_MS = 90_000; @@ -31,17 +28,21 @@ describe("gateway wizard e2e", () => { minimalGateway: true, }); const token = nextGatewayId("wizard-consent"); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); const confirmations: boolean[] = []; - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - wizardRunner: async (_opts, _runtime, prompter) => { - confirmations.push(await prompter.confirm({ message: "Continue?", initialValue: false })); - }, - }); - const client = await connectGatewayClient({ url: `ws://127.0.0.1:${port}`, token }); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + wizardRunner: async (_opts, _runtime, prompter) => { + confirmations.push( + await prompter.confirm({ message: "Continue?", initialValue: false }), + ); + }, + }), + ); + const client = await connectGatewayClient({ url: `ws://127.0.0.1:${claim.port}`, token }); try { for (const { value, expected } of [ @@ -74,22 +75,24 @@ describe("gateway wizard e2e", () => { }); const wizardToken = nextGatewayId("wiz-contained-exit"); let exitCode = 0; - const port = await getGatewayE2ePortBlock(); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token: wizardToken }, - controlUiEnabled: false, - wizardRunner: async (_opts, runtime, prompter) => { - await prompter.outro("wizard complete"); - runtime.exit(exitCode); - }, - channelWizardRunner: async (_opts, runtime, prompter) => { - await prompter.outro("channel wizard complete"); - runtime.exit(exitCode); - }, - }); + const claim = await acquireGatewayE2ePortBlock(); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token: wizardToken }, + controlUiEnabled: false, + wizardRunner: async (_opts, runtime, prompter) => { + await prompter.outro("wizard complete"); + runtime.exit(exitCode); + }, + channelWizardRunner: async (_opts, runtime, prompter) => { + await prompter.outro("channel wizard complete"); + runtime.exit(exitCode); + }, + }), + ); const client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: wizardToken, }); // Intercept an actual host exit so the fail-first Gateway test cannot @@ -140,29 +143,31 @@ describe("gateway wizard e2e", () => { minimalGateway: true, }); const wizAuth = nextGatewayId("wiz-chan"); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); const channelRuns: Array = []; - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token: wizAuth }, - controlUiEnabled: false, - wizardRunner: async () => { - throw new Error("setup wizard runner must not run for flow channels"); - }, - channelWizardRunner: async (opts, _runtime, prompter) => { - channelRuns.push(opts.channel); - await prompter.intro("Channel setup"); - const choice = await prompter.select({ - message: "channel", - options: [{ value: opts.channel ?? "none", label: opts.channel ?? "none" }], - }); - opts.onConfigured?.([{ channel: choice, accountId: "default" }]); - await prompter.outro(`configured ${choice}`); - }, - }); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token: wizAuth }, + controlUiEnabled: false, + wizardRunner: async () => { + throw new Error("setup wizard runner must not run for flow channels"); + }, + channelWizardRunner: async (opts, _runtime, prompter) => { + channelRuns.push(opts.channel); + await prompter.intro("Channel setup"); + const choice = await prompter.select({ + message: "channel", + options: [{ value: opts.channel ?? "none", label: opts.channel ?? "none" }], + }); + opts.onConfigured?.([{ channel: choice, accountId: "default" }]); + await prompter.outro(`configured ${choice}`); + }, + }), + ); const client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: wizAuth, clientDisplayName: "vitest-wizard-channels", }); diff --git a/src/gateway/gateway.test.ts b/src/gateway/gateway.test.ts index 7581486ff81c..534efe7b3c8d 100644 --- a/src/gateway/gateway.test.ts +++ b/src/gateway/gateway.test.ts @@ -34,9 +34,9 @@ import { connectDeviceAuthReq, disconnectGatewayClient, connectGatewayClient, - getGatewayE2ePortBlock, startGatewayWithClient, } from "./test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock, startClaimedGateway } from "./test-helpers.listener.js"; import { installOpenAiResponsesMock } from "./test-helpers.openai-mock.js"; import { buildMockOpenAiResponsesProvider } from "./test-openai-responses-model.js"; @@ -44,14 +44,16 @@ let createConfigIO: typeof import("../config/config.js").createConfigIO; const GATEWAY_E2E_TIMEOUT_MS = 90_000; async function startLoopbackTokenGateway(token: string) { - const port = await getGatewayE2ePortBlock(); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); - return { port, server }; + const claim = await acquireGatewayE2ePortBlock(); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); + return { port: claim.port, server }; } async function writeWorkspacePlugin(params: { @@ -112,17 +114,19 @@ describe("gateway e2e", () => { logging: { level: "info" }, }; await createConfigIO({ configPath }).writeConfigFile(initialConfig); - const port = await getGatewayE2ePortBlock(); - server = await startGatewayServer(port, { - bind: "loopback", - controlUiEnabled: false, - sidecarStartup: "defer", - }); + const claim = await acquireGatewayE2ePortBlock(); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); await expect( callGateway({ config: initialConfig, - localPortOverride: port, + localPortOverride: claim.port, method: "health", timeoutMs: 5_000, }), @@ -227,18 +231,20 @@ describe("gateway e2e", () => { authSource === "explicit-override" ? { mode: "off" as const, preserveFunnel: true } : undefined; - const port = await getGatewayE2ePortBlock(); - server = await startGatewayServer(port, { - bind: "loopback", - ...(callerAuthOverride ? { auth: callerAuthOverride } : {}), - ...(callerTailscaleOverride ? { tailscale: callerTailscaleOverride } : {}), - controlUiEnabled: false, - }); + const claim = await acquireGatewayE2ePortBlock(); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + ...(callerAuthOverride ? { auth: callerAuthOverride } : {}), + ...(callerTailscaleOverride ? { tailscale: callerTailscaleOverride } : {}), + controlUiEnabled: false, + }), + ); const expectedToken = authSource === "generated" ? getRuntimeConfig().gateway?.auth?.token : overrideToken; expect(typeof expectedToken).toBe("string"); client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: expectedToken as string, clientDisplayName: "vitest-direct-reload", }); @@ -308,7 +314,7 @@ describe("gateway e2e", () => { } const reconnected = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: expectedToken as string, clientDisplayName: "vitest-direct-reload-reconnect", }); @@ -350,19 +356,21 @@ describe("gateway e2e", () => { await createConfigIO({ configPath }).writeConfigFile(initialConfig, { allowedAgentRosterRemovals: ["main"], }); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); const hotReloadRecovery = vi.fn(() => ({ status: "emitted" as const })); - server = await startGatewayServer(port, { - bind: "loopback", - controlUiEnabled: false, - hotReloadRecovery, - }); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + controlUiEnabled: false, + hotReloadRecovery, + }), + ); await expect .poll( async () => { const health = await callGateway({ config: initialConfig, - localPortOverride: port, + localPortOverride: claim.port, method: "health", timeoutMs: 5_000, }); @@ -379,7 +387,7 @@ describe("gateway e2e", () => { }; const postAgentHook = async (agentId: string) => { - const response = await fetch(`http://127.0.0.1:${port}/hooks/agent`, { + const response = await fetch(`http://127.0.0.1:${claim.port}/hooks/agent`, { method: "POST", headers: { authorization: `Bearer ${hookToken}`, @@ -460,21 +468,23 @@ describe("gateway e2e", () => { logging: { level: "info" }, }); setTestEnvValue("OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", oldToken); - const port = await getGatewayE2ePortBlock(); - server = await startGatewayServer(port, { - bind: "loopback", - auth: { - mode: "token", - token: { - source: "env", - provider: "default", - id: "OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", + const claim = await acquireGatewayE2ePortBlock(); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { + mode: "token", + token: { + source: "env", + provider: "default", + id: "OPENCLAW_TEST_GATEWAY_OVERRIDE_TOKEN", + }, }, - }, - controlUiEnabled: false, - }); + controlUiEnabled: false, + }), + ); oldClient = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: oldToken, clientDisplayName: "vitest-startup-auth-ref-old", }); @@ -487,7 +497,7 @@ describe("gateway e2e", () => { expect(reload.ok).toBe(true); } const newClient = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: newToken, clientDisplayName: "vitest-startup-auth-ref-new", }); @@ -528,11 +538,13 @@ describe("gateway e2e", () => { logging: { level: "info" }, }; await configIO.writeConfigFile(initialConfig); - const port = await getGatewayE2ePortBlock(); - const server = await startGatewayServer(port, { - bind: "lan", - controlUiEnabled: false, - }); + const claim = await acquireGatewayE2ePortBlock(); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "lan", + controlUiEnabled: false, + }), + ); try { const seededOrigins = getRuntimeConfig().gateway?.controlUi?.allowedOrigins; @@ -775,24 +787,26 @@ module.exports = { clearConfigCache(); const wizardToken = nextGatewayId("wiz-token"); - const port = await getGatewayE2ePortBlock(); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token: wizardToken }, - controlUiEnabled: false, - wizardRunner: async (_opts, _runtime, prompter) => { - await prompter.intro("Wizard E2E"); - await prompter.note("write token"); - const token = await prompter.text({ message: "token" }); - await createConfigIO({ configPath }).writeConfigFile({ - gateway: { auth: { mode: "token", token } }, - }); - await prompter.outro("ok"); - }, - }); + const claim = await acquireGatewayE2ePortBlock(); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token: wizardToken }, + controlUiEnabled: false, + wizardRunner: async (_opts, _runtime, prompter) => { + await prompter.intro("Wizard E2E"); + await prompter.note("write token"); + const token = await prompter.text({ message: "token" }); + await createConfigIO({ configPath }).writeConfigFile({ + gateway: { auth: { mode: "token", token } }, + }); + await prompter.outro("ok"); + }, + }), + ); const client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: wizardToken, clientDisplayName: "vitest-wizard", }); @@ -847,20 +861,22 @@ module.exports = { await server.close({ reason: "wizard e2e complete" }); } - const port2 = await getGatewayE2ePortBlock(); - const server2 = await startGatewayServer(port2, { - bind: "loopback", - controlUiEnabled: false, - }); + const claim2 = await acquireGatewayE2ePortBlock(); + const server2 = await startClaimedGateway(claim2, () => + startGatewayServer(claim2.port, { + bind: "loopback", + controlUiEnabled: false, + }), + ); try { const resNoToken = await connectDeviceAuthReq({ - url: `ws://127.0.0.1:${port2}`, + url: `ws://127.0.0.1:${claim2.port}`, }); expect(resNoToken.ok).toBe(false); expect(resNoToken.error?.message ?? "").toContain("unauthorized"); const resToken = await connectDeviceAuthReq({ - url: `ws://127.0.0.1:${port2}`, + url: `ws://127.0.0.1:${claim2.port}`, token: wizardToken, }); expect(resToken.ok).toBe(true); diff --git a/src/gateway/operator-approvals-client.e2e.test.ts b/src/gateway/operator-approvals-client.e2e.test.ts index 509f94fdc7ca..45cea665b811 100644 --- a/src/gateway/operator-approvals-client.e2e.test.ts +++ b/src/gateway/operator-approvals-client.e2e.test.ts @@ -19,11 +19,8 @@ import { captureEnv, deleteTestEnvValue, setTestEnvValue } from "../test-utils/e import { ADMIN_SCOPE, APPROVALS_SCOPE, READ_SCOPE } from "./method-scopes.js"; import { withOperatorApprovalsGatewayClient } from "./operator-approvals-client.js"; import { startGatewayServer } from "./server.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "./test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "./test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock, startClaimedGateway } from "./test-helpers.listener.js"; import { configureManualGatewayBackgroundEnv, MANUAL_GATEWAY_ENV_KEYS, @@ -93,17 +90,19 @@ describe("operator approval gateway client e2e", () => { setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); configureManualGatewayBackgroundEnv(tempHome); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); const token = "approval-client-e2e-token"; - const url = `ws://127.0.0.1:${port}`; - setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(port)); + const url = `ws://127.0.0.1:${claim.port}`; + setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(claim.port)); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); cleanup.push(() => server.close()); const admin = await connectGatewayClient({ @@ -126,7 +125,7 @@ describe("operator approval gateway client e2e", () => { const localConfig = { gateway: { - port, + port: claim.port, auth: { mode: "token", token }, }, } satisfies OpenClawConfig; @@ -209,17 +208,19 @@ describe("operator approval gateway client e2e", () => { }); expect(requesterIdentity.deviceId).not.toBe(reviewerIdentity.deviceId); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); const token = "approval-surfaces-e2e-token"; - const url = `ws://127.0.0.1:${port}`; - setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(port)); + const url = `ws://127.0.0.1:${claim.port}`; + setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(claim.port)); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); cleanup.push(() => server.close()); const requester = await connectGatewayClient({ diff --git a/src/gateway/server-methods/plugin-approval-turn-source-routing.e2e.test.ts b/src/gateway/server-methods/plugin-approval-turn-source-routing.e2e.test.ts index dbb6f4a957da..9d66927cbd4b 100644 --- a/src/gateway/server-methods/plugin-approval-turn-source-routing.e2e.test.ts +++ b/src/gateway/server-methods/plugin-approval-turn-source-routing.e2e.test.ts @@ -17,11 +17,8 @@ import { clearSessionStoreCacheForTest } from "../../config/sessions/store-write import { captureEnv, deleteTestEnvValue, setTestEnvValue } from "../../test-utils/env.js"; import { APPROVALS_SCOPE } from "../method-scopes.js"; import { startGatewayServer } from "../server.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "../test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "../test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock, startClaimedGateway } from "../test-helpers.listener.js"; import { configureManualGatewayBackgroundEnv, MANUAL_GATEWAY_ENV_KEYS, @@ -61,17 +58,19 @@ describe("plugin.approval.request delivery routing (real gateway)", () => { setTestEnvValue("OPENCLAW_STATE_DIR", stateDir); configureManualGatewayBackgroundEnv(tempHome); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); const token = "plugin-approval-turn-source-e2e-token"; - const url = `ws://127.0.0.1:${port}`; - setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(port)); + const url = `ws://127.0.0.1:${claim.port}`; + setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(claim.port)); - server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); // No operator approval client; only a requester with APPROVALS_SCOPE. // This is the state that triggers the no-route expiry in the unfixed code. diff --git a/src/gateway/server-network-runtime.e2e.test.ts b/src/gateway/server-network-runtime.e2e.test.ts index d936ef2e0b00..004a95adb66a 100644 --- a/src/gateway/server-network-runtime.e2e.test.ts +++ b/src/gateway/server-network-runtime.e2e.test.ts @@ -9,8 +9,8 @@ import { clearSessionStoreCacheForTest } from "../config/sessions/store-writer-s import { resetAgentEventsForTest } from "../infra/agent-events.js"; import { captureEnv, deleteTestEnvValue, setTestEnvValue } from "../test-utils/env.js"; import { startGatewayServer } from "./server.js"; -import { getGatewayE2ePortBlock } from "./test-helpers.e2e.js"; import { GATEWAY_STARTUP_MUTATED_ENV_KEYS } from "./test-helpers.env.js"; +import { acquireGatewayE2ePortBlock, startClaimedGateway } from "./test-helpers.listener.js"; const NETWORK_GATEWAY_ENV_KEYS = [ "HOME", @@ -102,11 +102,14 @@ describe("gateway network runtime", () => { ); setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); - server = await startGatewayServer(await getGatewayE2ePortBlock(), { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - }); + const claim = await acquireGatewayE2ePortBlock(); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + }), + ); expect(isEnvHttpProxyDispatcher(getGlobalDispatcher())).toBe(true); } finally { @@ -158,9 +161,12 @@ describe("gateway network runtime", () => { await fs.writeFile(configPath, raw, { mode: 0o600 }); setTestEnvValue("OPENCLAW_CONFIG_PATH", configPath); - server = await startGatewayServer(await getGatewayE2ePortBlock(), { - controlUiEnabled: false, - }); + const claim = await acquireGatewayE2ePortBlock(); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + controlUiEnabled: false, + }), + ); await expect(fs.readFile(configPath, "utf-8")).resolves.toBe(raw); } finally { diff --git a/src/gateway/server-startup-node-capabilities.test.ts b/src/gateway/server-startup-node-capabilities.test.ts index a2efc4435ba2..8bb85845a1f9 100644 --- a/src/gateway/server-startup-node-capabilities.test.ts +++ b/src/gateway/server-startup-node-capabilities.test.ts @@ -9,14 +9,14 @@ import { stageActivePluginRegistry, } from "../plugins/runtime.js"; import { createOpenClawTestState } from "../test-utils/openclaw-test-state.js"; -import { getDeterministicFreePortBlock } from "../test-utils/ports.js"; +import { acquireTestPortBlock } from "../test-utils/port-claims.js"; import { createGatewayKernel } from "./server-kernel.js"; import type { GatewayServer } from "./server-public.js"; import type { GatewayWsClient } from "./server/ws-types.js"; +import { startClaimedGateway } from "./test-helpers.listener.js"; describe("Gateway startup node capabilities", () => { it("reconnects affected nodes when plugins attach after their handshake", async () => { - const port = await getDeterministicFreePortBlock({ offsets: [0] }); const state = await createOpenClawTestState({ label: "gateway-startup-node-capabilities", layout: "home", @@ -108,17 +108,24 @@ describe("Gateway startup node capabilities", () => { }); try { const token = "startup-node-capability-token"; - await state.writeConfig({ - gateway: { auth: { mode: "token", token }, controlUi: { enabled: false }, port }, - }); - state.applyEnv(); - stageActivePluginRegistry(createEmptyPluginRegistry(), null, "default"); - const { startGatewayServerCore } = await import("./server-start.js"); - server = await startGatewayServerCore(port, { - auth: { mode: "token", token }, - bind: "loopback", - controlUiEnabled: false, - sidecarStartup: "defer", + const claim = await acquireTestPortBlock({ offsets: [0] }); + server = await startClaimedGateway(claim, async () => { + await state.writeConfig({ + gateway: { + auth: { mode: "token", token }, + controlUi: { enabled: false }, + port: claim.port, + }, + }); + state.applyEnv(); + stageActivePluginRegistry(createEmptyPluginRegistry(), null, "default"); + const { startGatewayServerCore } = await import("./server-start.js"); + return await startGatewayServerCore(claim.port, { + auth: { mode: "token", token }, + bind: "loopback", + controlUiEnabled: false, + sidecarStartup: "defer", + }); }); expect(affected.client.invalidated).toBe(true); await expect(affected.closed).resolves.toEqual({ diff --git a/src/gateway/server.agent-artifact-apis.test.ts b/src/gateway/server.agent-artifact-apis.test.ts index a1f787b34cd9..d89834e244f9 100644 --- a/src/gateway/server.agent-artifact-apis.test.ts +++ b/src/gateway/server.agent-artifact-apis.test.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; +import { collectNestedErrorCandidates } from "@openclaw/normalization-core/error-coercion"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { clearConfigCache, clearRuntimeConfigSnapshot } from "../config/config.js"; @@ -24,13 +25,11 @@ import { } from "./managed-image-attachments.js"; import { listManagedImageRecordEntries } from "./managed-image-record-store.js"; import { ADMIN_SCOPE, READ_SCOPE } from "./method-scopes.js"; +import { GatewayStartupCleanupError } from "./server-shutdown.js"; import { startGatewayServer } from "./server.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "./test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "./test-helpers.e2e.js"; import { GATEWAY_STARTUP_MUTATED_ENV_KEYS } from "./test-helpers.env.js"; +import { acquireGatewayE2ePortBlock } from "./test-helpers.listener.js"; import type { WorkerEnvironmentServiceRecord } from "./worker-environments/service-contract.js"; const injectedWorkerService = vi.hoisted(() => { @@ -196,17 +195,35 @@ describe("Gateway agent and artifact APIs", () => { clearConfigCache(); clearSessionStoreCacheForTest(); - const port = await getGatewayE2ePortBlock(); - setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(port)); - let server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, + const claim = await acquireGatewayE2ePortBlock(); + // Restarts reuse the port, so the claim outlives each server. + let server: Awaited> | undefined; + let releaseClaim: (() => Promise) | undefined = claim.release; + cleanup.push(async () => { + await server?.close(); + await releaseClaim?.(); }); - cleanup.push(() => server.close()); + const startServer = () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + }).catch((error: unknown) => { + // Incomplete startup rollback can leave the listener bound; keep the port claimed. + if ( + collectNestedErrorCandidates(error).some( + (cause) => cause instanceof GatewayStartupCleanupError, + ) + ) { + releaseClaim = undefined; + } + throw error; + }); + setTestEnvValue("OPENCLAW_GATEWAY_PORT", String(claim.port)); + server = await startServer(); let client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, clientDisplayName: "gateway agent artifact APIs", scopes: [ADMIN_SCOPE, READ_SCOPE], @@ -215,16 +232,13 @@ describe("Gateway agent and artifact APIs", () => { cleanup.push(() => disconnectGatewayClient(client)); const restartGateway = async (clientDisplayName: string) => { await disconnectGatewayClient(client); - await server.close(); + await server?.close(); + server = undefined; clearRuntimeConfigSnapshot(); clearConfigCache(); - server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - }); + server = await startServer(); client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, clientDisplayName, scopes: [ADMIN_SCOPE, READ_SCOPE], @@ -404,7 +418,7 @@ describe("Gateway agent and artifact APIs", () => { await disconnectGatewayClient(client); client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, clientDisplayName: "gateway artifact APIs after reload", scopes: [ADMIN_SCOPE, READ_SCOPE], @@ -461,7 +475,7 @@ describe("Gateway agent and artifact APIs", () => { ); expect(download.url).toContain("mediaTicket="); expect(download.expiresAt).toBeTruthy(); - const downloadUrl = `http://127.0.0.1:${port}${download.url}`; + const downloadUrl = `http://127.0.0.1:${claim.port}${download.url}`; const response = await fetch(downloadUrl); expect(response.status).toBe(200); expect(Buffer.from(await response.arrayBuffer())).toEqual(fixture.body); diff --git a/src/gateway/server.visible-spawn-fallback.product-proof.test.ts b/src/gateway/server.visible-spawn-fallback.product-proof.test.ts index 1603348cc952..2e7b837c41e2 100644 --- a/src/gateway/server.visible-spawn-fallback.product-proof.test.ts +++ b/src/gateway/server.visible-spawn-fallback.product-proof.test.ts @@ -59,11 +59,8 @@ import { resolveMcpRequestContext } from "./mcp-http.request.js"; import { resolveMcpLoopbackScopedTools } from "./mcp-http.runtime.js"; import { buildMcpToolSchema } from "./mcp-http.schema.js"; import type { SessionsListResult } from "./session-utils.types.js"; -import { - disconnectGatewayClient, - getGatewayE2ePortBlock, - startGatewayWithClient, -} from "./test-helpers.e2e.js"; +import { disconnectGatewayClient, startGatewayWithClient } from "./test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "./test-helpers.listener.js"; const PRIMARY = "proof-primary/primary"; const BACKUP = "proof-backup/backup"; @@ -381,14 +378,14 @@ describe("sessions_spawn model fallback through the Gateway", () => { }, }); } - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); let onSessionChanged: (payload: unknown) => void = () => {}; gateway = await startGatewayWithClient({ cfg, - port, + portClaim: claim, clientName: GATEWAY_CLIENT_NAMES.CONTROL_UI, mode: GATEWAY_CLIENT_MODES.WEBCHAT, - origin: `http://127.0.0.1:${port}`, + origin: `http://127.0.0.1:${claim.port}`, configPath: await createGatewayConfigPath(home.tempHome), token, onEvent: ({ event, payload }) => { @@ -717,7 +714,7 @@ describe("CLI model inheritance through MCP", () => { async () => { provider = await startProvider({ name: "CLI model inheritance", directAgent: true }); const token = randomUUID(); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); setTestEnvValue("OPENCLAW_GATEWAY_TOKEN", token); const cfg: OpenClawConfig = { agents: { @@ -741,12 +738,12 @@ describe("CLI model inheritance through MCP", () => { }, }, tools: { profile: "coding" }, - gateway: { port, auth: { mode: "token", token } }, + gateway: { port: claim.port, auth: { mode: "token", token } }, hooks: { enabled: false }, }; gateway = await startGatewayWithClient({ cfg, - port, + portClaim: claim, token, configPath: await createGatewayConfigPath(home.tempHome), }); diff --git a/src/gateway/sessions-patch-catalog-queue.e2e.test.ts b/src/gateway/sessions-patch-catalog-queue.e2e.test.ts index 11aacee0d4dd..22eefd3da976 100644 --- a/src/gateway/sessions-patch-catalog-queue.e2e.test.ts +++ b/src/gateway/sessions-patch-catalog-queue.e2e.test.ts @@ -23,11 +23,8 @@ import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { ADMIN_SCOPE } from "./method-scopes.js"; import * as modelCatalog from "./server-model-catalog.js"; import { startGatewayServer } from "./server.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "./test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "./test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock, startClaimedGateway } from "./test-helpers.listener.js"; import { configureManualGatewayBackgroundEnv, MANUAL_GATEWAY_ENV_KEYS, @@ -46,16 +43,18 @@ test("an authenticated metadata patch completes while another session awaits cat agents: { defaults: { workspace: state.workspaceDir } }, diagnostics: { enabled: true }, }); - const port = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); const token = "catalog-queue-synthetic-token"; - server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); client = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token, scopes: [ADMIN_SCOPE], clientDisplayName: "catalog queue proof", diff --git a/src/gateway/test-helpers.e2e.ts b/src/gateway/test-helpers.e2e.ts index fc805d1f92ad..d7715780eee5 100644 --- a/src/gateway/test-helpers.e2e.ts +++ b/src/gateway/test-helpers.e2e.ts @@ -31,7 +31,6 @@ import { } from "../infra/device-identity.js"; import { captureEnv } from "../test-utils/env.js"; import type { TestPortClaim } from "../test-utils/port-claims.js"; -import { getDeterministicFreePortBlock } from "../test-utils/ports.js"; import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES, @@ -45,11 +44,6 @@ import { startGatewayServer, type GatewayServerOptions } from "./server.js"; import { GATEWAY_STARTUP_MUTATED_ENV_KEYS } from "./test-helpers.env.js"; import { reserveGatewayTestListener } from "./test-helpers.listener.js"; -/** Reserve a deterministic free port block for Gateway E2E tests. */ -export async function getGatewayE2ePortBlock(): Promise { - return await getDeterministicFreePortBlock({ offsets: [0, 1, 2, 3, 4] }); -} - /** Connect a GatewayClient with test defaults and resolve after hello-ok. */ export async function connectGatewayClient(params: { url: string; diff --git a/src/gateway/test-helpers.listener.ts b/src/gateway/test-helpers.listener.ts index 21cef36fca6b..d45252975b08 100644 --- a/src/gateway/test-helpers.listener.ts +++ b/src/gateway/test-helpers.listener.ts @@ -12,6 +12,11 @@ import { getDeterministicFreePortBlock } from "../test-utils/ports.js"; import { GatewayStartupCleanupError } from "./server-shutdown.js"; import type { GatewayServer } from "./server.js"; +/** Claim a Gateway E2E port block; release it only after the Gateway bound to it has closed. */ +export async function acquireGatewayE2ePortBlock(): Promise { + return await acquireTestPortBlock({ offsets: [0, 1, 2, 3, 4] }); +} + export async function getGatewayTestPort(): Promise { return await getDeterministicFreePortBlock({ offsets: [0, 1, 2, 3, 4] }); } diff --git a/src/gateway/test-helpers.server-env.test.ts b/src/gateway/test-helpers.server-env.test.ts index a8ec4d964cc9..19d111da7d2a 100644 --- a/src/gateway/test-helpers.server-env.test.ts +++ b/src/gateway/test-helpers.server-env.test.ts @@ -16,9 +16,9 @@ import { createGatewayConfigOverrides } from "./test-helpers.config-runtime.js"; import { connectGatewayClient, disconnectGatewayClient, - getGatewayE2ePortBlock, startGatewayWithClient, } from "./test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "./test-helpers.listener.js"; import { testState } from "./test-helpers.runtime-state.js"; import { installGatewayTestHooks, @@ -80,7 +80,7 @@ describe("Gateway test environment lifecycle", () => { }); const token = "retained-listener-token"; const acquisition = startGatewayWithClient({ - port: await getGatewayE2ePortBlock(), + portClaim: await acquireGatewayE2ePortBlock(), cfg: { gateway: { auth: { mode: "token", token } } }, configPath, token, diff --git a/test/cron-message-read.integration.test.ts b/test/cron-message-read.integration.test.ts index cb8630c78881..f989cee1fb58 100644 --- a/test/cron-message-read.integration.test.ts +++ b/test/cron-message-read.integration.test.ts @@ -27,9 +27,9 @@ import { closeMcpLoopbackServer, ensureMcpLoopbackServer } from "../src/gateway/ import { getActiveMcpLoopbackRuntime } from "../src/gateway/mcp-http.loopback-runtime.js"; import { disconnectGatewayClient, - getGatewayE2ePortBlock, startGatewayWithClient, } from "../src/gateway/test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "../src/gateway/test-helpers.listener.js"; import { buildMockOpenAiResponsesProvider } from "../src/gateway/test-openai-responses-model.js"; import { formatErrorMessage } from "../src/infra/errors.js"; import { redactToolPayloadText } from "../src/logging/redact.js"; @@ -358,7 +358,11 @@ describe("scheduled message actions", () => { vi.stubEnv("OPENCLAW_SKIP_PROVIDERS", undefined); vi.stubEnv("OPENCLAW_GATEWAY_URL", undefined); vi.stubEnv("OPENCLAW_GATEWAY_TOKEN", undefined); - const gatewayPort = await getGatewayE2ePortBlock(); + const gatewayPortClaim = await acquireGatewayE2ePortBlock(); + const gatewayPort = gatewayPortClaim.port; + // Released here until Gateway startup owns the claim. + let unstartedGatewayPortClaim: typeof gatewayPortClaim | undefined = gatewayPortClaim; + cleanup.push(() => unstartedGatewayPortClaim?.release()); const gatewayToken = "synthetic-scheduled-read-gateway-token"; vi.stubEnv("OPENCLAW_SCHEDULED_READ_ARGUMENTS", JSON.stringify(actionParams)); vi.stubEnv("OPENCLAW_SCHEDULED_CREATE_JOB", undefined); @@ -647,8 +651,9 @@ describe("scheduled message actions", () => { cleanup.push(() => resetPreparedModelRuntimeSnapshotsForTest()); const finished = createDeferred>(); const scheduledJob: { id?: string } = {}; + unstartedGatewayPortClaim = undefined; const gateway = await startGatewayWithClient({ - port: gatewayPort, + portClaim: gatewayPortClaim, cfg, configPath, token: gatewayToken, diff --git a/test/e2e/qa-lab/plugins/canvas-agent-node.e2e.test.ts b/test/e2e/qa-lab/plugins/canvas-agent-node.e2e.test.ts index 091851dee678..856f68857f1c 100644 --- a/test/e2e/qa-lab/plugins/canvas-agent-node.e2e.test.ts +++ b/test/e2e/qa-lab/plugins/canvas-agent-node.e2e.test.ts @@ -13,8 +13,11 @@ import { startGatewayServer } from "../../../../src/gateway/server.js"; import { connectGatewayClient, disconnectGatewayClient, - getGatewayE2ePortBlock, } from "../../../../src/gateway/test-helpers.e2e.js"; +import { + acquireGatewayE2ePortBlock, + startClaimedGateway, +} from "../../../../src/gateway/test-helpers.listener.js"; import { loadOrCreateDeviceIdentity } from "../../../../src/infra/device-identity.js"; import { captureActivePluginRegistrySnapshot, @@ -105,7 +108,10 @@ describe("Canvas agent tool over a paired macOS node", () => { OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", }, }); - const port = await getGatewayE2ePortBlock(); + const portClaim = await acquireGatewayE2ePortBlock(); + const { port } = portClaim; + // Released here until the started Gateway owns the claim. + let unstartedPortClaim: typeof portClaim | undefined = portClaim; const gatewayToken = "qa-canvas-agent-node-token"; const config: OpenClawConfig = { gateway: { @@ -162,12 +168,15 @@ describe("Canvas agent tool over a paired macOS node", () => { "default", state.workspaceDir, ); - gateway = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token: gatewayToken }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); + unstartedPortClaim = undefined; + gateway = await startClaimedGateway(portClaim, () => + startGatewayServer(port, { + bind: "loopback", + auth: { mode: "token", token: gatewayToken }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); const operatorIdentity = loadOrCreateDeviceIdentity({ path: path.join(state.home, "canvas-operator.sqlite"), }); @@ -295,6 +304,7 @@ describe("Canvas agent tool over a paired macOS node", () => { if (gateway) { await gateway.close({ reason: "canvas agent node proof complete" }); } + await unstartedPortClaim?.release(); restoreActivePluginRegistrySnapshot(previousPluginRegistry); await state.cleanup(); } diff --git a/test/e2e/qa-lab/plugins/file-transfer-exact-approval.e2e.test.ts b/test/e2e/qa-lab/plugins/file-transfer-exact-approval.e2e.test.ts index 90b4cd850b0d..6b68c1df90c4 100644 --- a/test/e2e/qa-lab/plugins/file-transfer-exact-approval.e2e.test.ts +++ b/test/e2e/qa-lab/plugins/file-transfer-exact-approval.e2e.test.ts @@ -15,8 +15,11 @@ import { startGatewayServer } from "../../../../src/gateway/server.js"; import { connectGatewayClient, disconnectGatewayClient, - getGatewayE2ePortBlock, } from "../../../../src/gateway/test-helpers.e2e.js"; +import { + acquireGatewayE2ePortBlock, + startClaimedGateway, +} from "../../../../src/gateway/test-helpers.listener.js"; import { loadOrCreateDeviceIdentity } from "../../../../src/infra/device-identity.js"; import { captureActivePluginRegistrySnapshot, @@ -60,7 +63,10 @@ describe("file-transfer exact approval transport", () => { OPENCLAW_TEST_MINIMAL_GATEWAY: "1", }, }); - const port = await getGatewayE2ePortBlock(); + const portClaim = await acquireGatewayE2ePortBlock(); + const { port } = portClaim; + // Released here until the started Gateway owns the claim. + let unstartedPortClaim: typeof portClaim | undefined = portClaim; const gatewayToken = "qa-file-transfer-exact-approval-token"; const target = path.join(state.home, "report.txt"); const approvedObject = path.join(state.home, "approved-object.txt"); @@ -162,12 +168,15 @@ describe("file-transfer exact approval transport", () => { "default", state.workspaceDir, ); - gateway = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token: gatewayToken }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); + unstartedPortClaim = undefined; + gateway = await startClaimedGateway(portClaim, () => + startGatewayServer(port, { + bind: "loopback", + auth: { mode: "token", token: gatewayToken }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); operator = await connectGatewayClient({ url: `ws://127.0.0.1:${port}`, token: gatewayToken, @@ -267,6 +276,7 @@ describe("file-transfer exact approval transport", () => { if (gateway) { await gateway.close({ reason: "file-transfer exact approval proof complete" }); } + await unstartedPortClaim?.release(); restoreActivePluginRegistrySnapshot(previousPluginRegistry); await state.cleanup(); } diff --git a/test/e2e/qa-lab/runtime/gateway-hosted-web.e2e.test.ts b/test/e2e/qa-lab/runtime/gateway-hosted-web.e2e.test.ts index ee0e4d4b267e..932a9a4d9e67 100644 --- a/test/e2e/qa-lab/runtime/gateway-hosted-web.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/gateway-hosted-web.e2e.test.ts @@ -11,9 +11,12 @@ import { startGatewayServer } from "../../../../src/gateway/server.js"; import { connectGatewayClient, disconnectGatewayClient, - getGatewayE2ePortBlock, } from "../../../../src/gateway/test-helpers.e2e.js"; import { snapshotGatewayStartupEnv } from "../../../../src/gateway/test-helpers.env.js"; +import { + acquireGatewayE2ePortBlock, + startClaimedGateway, +} from "../../../../src/gateway/test-helpers.listener.js"; import { registerPluginHttpRoute, withPluginHttpRouteRegistry, @@ -94,13 +97,15 @@ describe("Gateway hosted web surfaces", () => { async () => { clearConfigCache(); clearRuntimeConfigSnapshot(); - const port = await getGatewayE2ePortBlock(); - const server = await startGatewayServer(port, { - auth: { mode: "token", token: TOKEN }, - bind: "loopback", - controlUiEnabled: true, - sidecarStartup: "defer", - }); + const claim = await acquireGatewayE2ePortBlock(); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + auth: { mode: "token", token: TOKEN }, + bind: "loopback", + controlUiEnabled: true, + sidecarStartup: "defer", + }), + ); // Deferred startup replaces the bootstrap registry; register routes only after the // server publishes the settled runtime so requests do not target a retired registry. await server.startupSettled; @@ -159,7 +164,7 @@ describe("Gateway hosted web surfaces", () => { expect.arrayContaining(["/api/v1/admin/rpc", "/__openclaw__/a2ui"]), ); - const origin = `http://127.0.0.1:${port}`; + const origin = `http://127.0.0.1:${claim.port}`; const controlUi = await fetch(`${origin}/`, { headers: { authorization: `Bearer ${TOKEN}` }, }); @@ -202,7 +207,7 @@ describe("Gateway hosted web surfaces", () => { vi.setSystemTime(capabilityIssuedAtMs); let helloCanvasUrl: string | undefined; operator = await connectGatewayClient({ - url: `ws://127.0.0.1:${port}`, + url: `ws://127.0.0.1:${claim.port}`, token: TOKEN, role: "operator", scopes: OPERATOR_SCOPES, diff --git a/test/e2e/qa-lab/runtime/gateway-tls-pinning.ts b/test/e2e/qa-lab/runtime/gateway-tls-pinning.ts index 8d0b096ab909..839619ebfa99 100644 --- a/test/e2e/qa-lab/runtime/gateway-tls-pinning.ts +++ b/test/e2e/qa-lab/runtime/gateway-tls-pinning.ts @@ -16,13 +16,15 @@ import { clearConfigCache, clearRuntimeConfigSnapshot } from "../../../../src/co import { createConfiguredGatewayLocalProbe } from "../../../../src/gateway/local-http-probe.js"; import { startGatewayServer } from "../../../../src/gateway/server.js"; import { GATEWAY_STARTUP_MUTATED_ENV_KEYS } from "../../../../src/gateway/test-helpers.env.js"; +import { startClaimedGateway } from "../../../../src/gateway/test-helpers.listener.js"; import { resolveGatewayConnectionTlsFingerprint } from "../../../../src/gateway/tls-fingerprint.js"; import { formatErrorMessage } from "../../../../src/infra/errors.js"; import { loadGatewayTlsServerRuntime } from "../../../../src/infra/tls/gateway.js"; import { flushLogger, resetLogger } from "../../../../src/logging/logger.js"; import { closeOpenClawStateDatabaseByPathAsync } from "../../../../src/state/openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "../../../../src/state/openclaw-state-db.paths.js"; -import { getDeterministicFreePortBlock, getFreePort } from "../../../../src/test-utils/ports.js"; +import { acquireTestPortBlock } from "../../../../src/test-utils/port-claims.js"; +import { getFreePort } from "../../../../src/test-utils/ports.js"; import { createDeferred, withinTest } from "../../../helpers/promise.js"; import { runQaGatewayFixture } from "../../../helpers/qa-gateway-cleanup.js"; import { createQaScriptEvidenceWriter } from "./script-evidence.js"; @@ -410,14 +412,16 @@ export async function runGatewayTlsPinningProof( clearConfigCache(); clearRuntimeConfigSnapshot(); - const port = await getDeterministicFreePortBlock({ offsets: [0, 1] }); - server = await startGatewayServer(port, { - auth: { mode: "none" }, - bind: "loopback", - controlUiEnabled: false, - sidecarStartup: "defer", - }); - const url = `wss://127.0.0.1:${port}`; + const claim = await acquireTestPortBlock({ offsets: [0, 1] }); + server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + auth: { mode: "none" }, + bind: "loopback", + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); + const url = `wss://127.0.0.1:${claim.port}`; const probeConfig = { gateway: { tls: { enabled: true, certPath, keyPath } }, }; @@ -426,7 +430,7 @@ export async function runGatewayTlsPinningProof( const probeHealth = () => localProbe.requestHttp({ host: "127.0.0.1", - port, + port: claim.port, pathname: "/healthz", timeoutMs: 1000, }); @@ -435,7 +439,7 @@ export async function runGatewayTlsPinningProof( if ((await probeHealth())?.statusCode !== 200) { probeFailures.push(`${stage}: HTTP probe rejected the healthy accepted listener`); } - const target = await localProbe.resolveWebSocketTarget(port); + const target = await localProbe.resolveWebSocketTarget(claim.port); if (target?.tlsFingerprint !== expectedFingerprint) { probeFailures.push(`${stage}: WebSocket probe selected an unaccepted certificate pin`); } else { @@ -448,13 +452,13 @@ export async function runGatewayTlsPinningProof( async () => ((await probeHealth())?.statusCode === 200 ? true : undefined), "the initial accepted local TLS health listener", ); - const initialTarget = await missedRenewalProbe.resolveWebSocketTarget(port); + const initialTarget = await missedRenewalProbe.resolveWebSocketTarget(claim.port); if (initialTarget?.tlsFingerprint !== preparedTls.fingerprintSha256) { throw new Error("A WebSocket-first probe did not verify the initial listener pin"); } await (signal ? withinTest(advertisementReady.promise, signal) : advertisementReady.promise); const advertisedFingerprint = await readAdvertisedFingerprint(advertisementPath); - const peerFingerprint = await waitForPeerFingerprint(port); + const peerFingerprint = await waitForPeerFingerprint(claim.port); if (peerFingerprint !== advertisedFingerprint) { throw new Error("Gateway advertised TLS fingerprint did not match the live peer certificate"); } @@ -484,7 +488,7 @@ export async function runGatewayTlsPinningProof( const log = await fs.readFile(gatewayLogPath, "utf8"); return log.includes("TLS renewal failed; keeping accepted material") ? true : undefined; }, "rejection of the incomplete cert/key replacement"); - for (const listener of [port, existing.portalPort]) { + for (const listener of [claim.port, existing.portalPort]) { if ((await waitForPeerFingerprint(listener)) !== advertisedFingerprint) { throw new Error("An incomplete renewal changed an accepted TLS listener"); } @@ -500,11 +504,11 @@ export async function runGatewayTlsPinningProof( if ( (await coldProbe.requestHttp({ host: "127.0.0.1", - port, + port: claim.port, pathname: "/healthz", timeoutMs: 1000, })) !== null || - (await coldProbe.resolveWebSocketTarget(port)) !== null + (await coldProbe.resolveWebSocketTarget(claim.port)) !== null ) { throw new Error( "A cold probe trusted a serving certificate absent from its configured file", @@ -512,7 +516,7 @@ export async function runGatewayTlsPinningProof( } await fs.writeFile(keyPath, nextKey); const renewedFingerprint = await waitForRenewalFact(async () => { - const fingerprint = await waitForPeerFingerprint(port); + const fingerprint = await waitForPeerFingerprint(claim.port); return fingerprint === replacement.fingerprintSha256 ? fingerprint : undefined; }, "the renewed listener certificate"); await waitForRenewalFact( @@ -575,24 +579,24 @@ export async function runGatewayTlsPinningProof( const laterLog = (await fs.readFile(gatewayLogPath)).subarray(pausedLogSize).toString(); return laterLog.includes("TLS renewal deferred") ? true : undefined; }, "the paused owner's certificate observation"); - if ((await waitForPeerFingerprint(port)) !== renewedFingerprint) { + if ((await waitForPeerFingerprint(claim.port)) !== renewedFingerprint) { throw new Error("TLS certificate changed while automatic reload was off"); } await verifyRetainedProbe("reload off", renewedFingerprint); if ( (await missedRenewalProbe.requestHttp({ host: "127.0.0.1", - port, + port: claim.port, pathname: "/healthz", timeoutMs: 1000, })) !== null || - (await missedRenewalProbe.resolveWebSocketTarget(port)) !== null + (await missedRenewalProbe.resolveWebSocketTarget(claim.port)) !== null ) { throw new Error("A probe trusted an intermediate serving certificate it never verified"); } await setReloadMode("hybrid"); const resumedFingerprint = await waitForRenewalFact(async () => { - const fingerprint = await waitForPeerFingerprint(port); + const fingerprint = await waitForPeerFingerprint(claim.port); return fingerprint === nextPair.fingerprintSha256 ? fingerprint : undefined; }, "renewal on re-enable without another certificate write"); for (const listener of [existing.portalPort, later.portalPort, later.sandboxPort]) { @@ -624,7 +628,7 @@ export async function runGatewayTlsPinningProof( await fs.rename(`${certPath}.next`, certPath); await fs.rename(`${keyPath}.next`, keyPath); finalFingerprint = await waitForRenewalFact(async () => { - const fingerprint = await waitForPeerFingerprint(port); + const fingerprint = await waitForPeerFingerprint(claim.port); return fingerprint === retargeted.fingerprintSha256 ? fingerprint : undefined; }, "the certificate after atomic symlink retargeting"); for (const listener of [existing.portalPort, later.portalPort, later.sandboxPort]) { @@ -641,7 +645,9 @@ export async function runGatewayTlsPinningProof( ); } const health = await client.request("health", {}); - if ((await localProbe.resolveWebSocketTarget(port))?.tlsFingerprint !== finalFingerprint) { + if ( + (await localProbe.resolveWebSocketTarget(claim.port))?.tlsFingerprint !== finalFingerprint + ) { throw new Error("Retained local probe returned its startup certificate pin after renewal"); } if ((await probeHealth())?.statusCode !== 200) { @@ -674,7 +680,7 @@ export async function runGatewayTlsPinningProof( await withExactPin(url, configuredPin, async (client) => { await client.request("health", {}); }); - const cleartextMismatch = await proveCleartextMismatch(port, advertisedFingerprint); + const cleartextMismatch = await proveCleartextMismatch(claim.port, advertisedFingerprint); if (probeFailures.length > 0) { throw new Error(probeFailures.join("; ")); } diff --git a/test/e2e/qa-lab/runtime/gateway-usage-memory-apis.e2e.test.ts b/test/e2e/qa-lab/runtime/gateway-usage-memory-apis.e2e.test.ts index 136da84b69de..6ca41290a2c4 100644 --- a/test/e2e/qa-lab/runtime/gateway-usage-memory-apis.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/gateway-usage-memory-apis.e2e.test.ts @@ -18,8 +18,11 @@ import { loadGatewaySessionEntryReadOnly } from "../../../../src/gateway/session import { connectGatewayClient, disconnectGatewayClient, - getGatewayE2ePortBlock, } from "../../../../src/gateway/test-helpers.e2e.js"; +import { + acquireGatewayE2ePortBlock, + startClaimedGateway, +} from "../../../../src/gateway/test-helpers.listener.js"; import type { UsageSummary } from "../../../../src/infra/provider-usage.types.js"; import { refreshCostUsageCacheForAgent } from "../../../../src/infra/session-cost-usage-aggregation.js"; import { readSessionCostUsageRollupRows } from "../../../../src/infra/session-cost-usage-cache.test-support.js"; @@ -150,7 +153,10 @@ describe("gateway usage and memory APIs", () => { "projects deterministic usage and explicit memory readiness over authenticated WebSocket RPCs", { timeout: TEST_TIMEOUT_MS }, async () => { - const port = await getGatewayE2ePortBlock(); + const portClaim = await acquireGatewayE2ePortBlock(); + const { port } = portClaim; + // Released here until the started Gateway owns the claim. + let unstartedPortClaim: typeof portClaim | undefined = portClaim; const token = `gateway-usage-memory-${process.pid}-${process.env.VITEST_POOL_ID ?? "0"}`; const state = await createOpenClawTestState({ label: "gateway-usage-memory-apis", @@ -210,12 +216,15 @@ describe("gateway usage and memory APIs", () => { }); expect(storedSession.entry).not.toHaveProperty("sessionFile"); - server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - sidecarStartup: "defer", - }); + unstartedPortClaim = undefined; + server = await startClaimedGateway(portClaim, () => + startGatewayServer(port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); client = await connectGatewayClient({ url: `ws://127.0.0.1:${port}`, token, @@ -332,6 +341,7 @@ describe("gateway usage and memory APIs", () => { await disconnectGatewayClient(client); } await server?.close({ reason: "gateway usage and memory QA complete" }); + await unstartedPortClaim?.release(); clearModelAuthStatusUsageCache(); await state.cleanup(); } diff --git a/test/e2e/qa-lab/runtime/native-cancellation-authz.e2e.test.ts b/test/e2e/qa-lab/runtime/native-cancellation-authz.e2e.test.ts index d10ae79d9a5b..b1eaf810bd99 100644 --- a/test/e2e/qa-lab/runtime/native-cancellation-authz.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/native-cancellation-authz.e2e.test.ts @@ -29,8 +29,11 @@ import { resolveSessionStorePathCore } from "../../../../src/config/sessions/pat import { replaceSessionEntrySync } from "../../../../src/config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../../../../src/config/types.openclaw.js"; import { startGatewayServer } from "../../../../src/gateway/server.js"; -import { getGatewayE2ePortBlock } from "../../../../src/gateway/test-helpers.e2e.js"; import { snapshotGatewayStartupEnv } from "../../../../src/gateway/test-helpers.env.js"; +import { + acquireGatewayE2ePortBlock, + startClaimedGateway, +} from "../../../../src/gateway/test-helpers.listener.js"; import { resetPluginRuntimeStateForTest } from "../../../../src/plugins/runtime.js"; import { withEnvAsync } from "../../../../src/test-utils/env.js"; import { createDeferred, withinTest } from "../../../helpers/promise.js"; @@ -144,13 +147,15 @@ describe("native child cancellation authority", () => { async () => { clearConfigCache(); clearRuntimeConfigSnapshot(); - const port = await getGatewayE2ePortBlock(); - const server = await startGatewayServer(port, { - auth: { mode: "token", token: TOKEN }, - bind: "loopback", - controlUiEnabled: false, - sidecarStartup: "defer", - }); + const claim = await acquireGatewayE2ePortBlock(); + const server = await startClaimedGateway(claim, () => + startGatewayServer(claim.port, { + auth: { mode: "token", token: TOKEN }, + bind: "loopback", + controlUiEnabled: false, + sidecarStartup: "defer", + }), + ); await server.startupSettled; const acpxServices: OpenClawPluginService[] = []; const acpxRuntime = createPluginRuntimeMock({ diff --git a/test/gateway-cold-agent-abort.e2e.test.ts b/test/gateway-cold-agent-abort.e2e.test.ts index 2d5a62f8f53c..3c12109f02fd 100644 --- a/test/gateway-cold-agent-abort.e2e.test.ts +++ b/test/gateway-cold-agent-abort.e2e.test.ts @@ -95,9 +95,13 @@ it( }, }); await instance.startGateway(); + const started = instance; client = await connectGatewayClient({ - url: instance.url, - token: instance.gatewayToken, + url: started.url, + token: started.gatewayToken, + }).catch(async (error: unknown) => { + // Capture the owned child before cleanup retires it. + throw new Error(`${String(error)}\n${await started.diagnose()}`, { cause: error }); }); final = client.request( "agent", diff --git a/test/helpers/gateway-client.ts b/test/helpers/gateway-client.ts index 18077814bdeb..d4f50388e082 100644 --- a/test/helpers/gateway-client.ts +++ b/test/helpers/gateway-client.ts @@ -62,7 +62,14 @@ export async function acquireGatewayTestClient( }, onConnectError: (error) => settle({ error }), onClose: (code, reason, info) => { - settle({ error: new Error(`${wait.closeMessage} (${code}): ${reason}`) }); + // Name the handshake stage: an abnormal close before the connect frame is a peer + // drop, while one after it is a refused or interrupted handshake. + const stage = info + ? ` [${info.phase} socketOpened=${String(info.socketOpened)} connectRequestSent=${String( + info.connectRequestSent ?? false, + )}]` + : ""; + settle({ error: new Error(`${wait.closeMessage} (${code}): ${reason}${stage}`) }); options.onClose?.(code, reason, info); }, }); diff --git a/test/helpers/openclaw-test-instance.port-collision.test.ts b/test/helpers/openclaw-test-instance.port-collision.test.ts index 6bae338694bb..7963bf5e8947 100644 --- a/test/helpers/openclaw-test-instance.port-collision.test.ts +++ b/test/helpers/openclaw-test-instance.port-collision.test.ts @@ -2,6 +2,7 @@ import { AsyncLocalStorage } from "node:async_hooks"; import fs from "node:fs/promises"; import net from "node:net"; import { expect, it, vi } from "vitest"; +import { acquireGatewayE2ePortBlock } from "../../src/gateway/test-helpers.listener.js"; import * as ports from "../../src/test-utils/ports.js"; import { createOpenClawTestInstance } from "./openclaw-test-instance.js"; import { createDeferred } from "./promise.js"; @@ -74,3 +75,19 @@ it("keeps overlapping fixture allocations exclusive before their reservation bin await Promise.all(instances.map((instance) => instance.cleanup())); } }); + +it("keeps an in-process Gateway E2E port out of fixture instance allocation", async () => { + const owner = await acquireGatewayE2ePortBlock(); + // Same-shard workers try the same first candidate. + const pickerSpy = vi + .spyOn(ports, "getDeterministicFreePortBlock") + .mockResolvedValueOnce(owner.port); + const instance = await createOpenClawTestInstance({ name: "in-process-port-owner" }); + try { + expect(instance.port).not.toBe(owner.port); + } finally { + pickerSpy.mockRestore(); + await instance.cleanup(); + await owner.release(); + } +}); diff --git a/test/helpers/openclaw-test-instance.ts b/test/helpers/openclaw-test-instance.ts index 44790fc61140..dc84981b19d0 100644 --- a/test/helpers/openclaw-test-instance.ts +++ b/test/helpers/openclaw-test-instance.ts @@ -108,6 +108,8 @@ export type OpenClawTestInstance = { startGateway: () => Promise; stopGateway: () => Promise; logs: () => string; + /** Bounded owner/readiness/listener facts for a failure after startup. */ + diagnose: () => Promise; cleanup: () => Promise; }; @@ -119,6 +121,8 @@ type ReadinessProbe = { ready?: boolean; failing?: string[]; omittedFailing?: number; + /** Responder's reported server uptime; binds the answer to a process started after spawn. */ + uptimeMs?: number; error?: "timeout" | "child-exit" | "fetch-failed" | "invalid-json" | "body-failed" | "aborted"; }; @@ -425,6 +429,9 @@ async function waitForGatewayReady( if (typeof readiness.ready === "boolean") { probe.ready = readiness.ready; } + if (typeof readiness.uptimeMs === "number" && Number.isFinite(readiness.uptimeMs)) { + probe.uptimeMs = readiness.uptimeMs; + } if (Array.isArray(readiness.failing)) { // Channel IDs and arbitrary startup reasons are private; retain only core categories. probe.failing = readiness.failing.slice(0, 8).map((reason) => { @@ -696,10 +703,10 @@ function mergeConfig( return result; } -function formatLogs(stdout: string[], stderr: string[]): string { +function formatLogs(stdout: string[], stderr: string[], maxBytes = LOG_TAIL_MAX_BYTES): string { const diagnosticTail = (log: string[]): string => { const tail = createBoundedStringLog( - Math.min((log as BoundedStringLog).maxBytes ?? LOG_TAIL_MAX_BYTES, LOG_TAIL_MAX_BYTES), + Math.min((log as BoundedStringLog).maxBytes ?? LOG_TAIL_MAX_BYTES, maxBytes), ) as BoundedStringLog; for (const chunk of log) { appendLogChunk(tail, chunk); @@ -710,6 +717,63 @@ function formatLogs(stdout: string[], stderr: string[]): string { return `--- stdout ---\n${diagnosticTail(stdout)}\n--- stderr ---\n${diagnosticTail(stderr)}`; } +type PortListenerOwner = { pid: number; pgid: number | null; comm: string | null }; +type PortListenerScan = { owners: PortListenerOwner[]; complete: boolean } | "unsupported"; + +// Failure diagnostics must not consume the test deadline before teardown. +const PORT_LISTENER_SCAN_BUDGET_MS = 1_000; + +/** Linux only: which processes hold a LISTEN socket on the loopback port. */ +async function inspectPortListeners(port: number): Promise { + if (process.platform !== "linux") { + return "unsupported"; + } + const deadline = Date.now() + PORT_LISTENER_SCAN_BUDGET_MS; + const inodes = new Set(); + for (const table of ["/proc/net/tcp", "/proc/net/tcp6"]) { + const rows = await fs.readFile(table, "utf8").catch(() => ""); + for (const row of rows.split("\n").slice(1)) { + // sl local_address rem_address st ... inode; state 0A is LISTEN. + const fields = row.trim().split(/\s+/u); + const localPort = Number.parseInt(fields[1]?.split(":").at(-1) ?? "", 16); + if (localPort === port && fields[3] === "0A" && fields[9]) { + inodes.add(fields[9]); + } + } + } + const owners: PortListenerOwner[] = []; + if (inodes.size === 0) { + return { owners, complete: true }; + } + for (const pid of await fs.readdir("/proc")) { + if (owners.length >= 8 || Date.now() > deadline) { + return { owners, complete: false }; + } + if (!/^\d+$/u.test(pid)) { + continue; + } + const fds = await fs.readdir(`/proc/${pid}/fd`).catch(() => []); + for (const fd of fds) { + if (Date.now() > deadline) { + return { owners, complete: false }; + } + const target = await fs.readlink(`/proc/${pid}/fd/${fd}`).catch(() => ""); + if (inodes.has(/^socket:\[(\d+)\]$/u.exec(target)?.[1] ?? "")) { + const stat = await fs.readFile(`/proc/${pid}/stat`, "utf8").catch(() => ""); + // comm may contain spaces; pgrp is the third field after its closing paren. + const pgid = Number(stat.slice(stat.lastIndexOf(")") + 2).split(" ")[2]); + owners.push({ + pid: Number(pid), + pgid: Number.isInteger(pgid) ? pgid : null, + comm: /\((.*)\)/su.exec(stat)?.[1]?.slice(0, 32) ?? null, + }); + break; + } + } + } + return { owners, complete: true }; +} + function createInstanceEnv(params: { stateEnv: NodeJS.ProcessEnv; extraEnv: Record; @@ -838,7 +902,7 @@ export async function createOpenClawTestInstance( stateEnv: state.env, extraEnv: options.env ?? {}, }); - let child: { process: OpenClawTestProcess; ready: boolean } | undefined; + let child: { process: OpenClawTestProcess; ready: boolean; spawnedAtMs: number } | undefined; const commands = new Set>(); const reserveIdlePort = async () => { if ( @@ -1030,7 +1094,7 @@ export async function createOpenClawTestInstance( }, reserveIdlePort); return; } - const owner = { process: attempt, ready: false }; + const owner = { process: attempt, ready: false, spawnedAtMs: Date.now() }; child = owner; try { await waitForGatewayReady( @@ -1127,6 +1191,41 @@ export async function createOpenClawTestInstance( }, stopGateway: () => enqueue("stop", stopGatewayChild), logs: () => formatLogs(stdout, stderr), + diagnose: async () => { + const owner = child; + const ready = readiness.at(-1); + const readyProbe = ready?.probes.at(-1); + const facts = { + port, + child: owner + ? { + pid: owner.process.pid ?? null, + exitCode: owner.process.exitCode, + signalCode: owner.process.signalCode, + ready: owner.ready, + ageMs: Date.now() - owner.spawnedAtMs, + } + : null, + readiness: ready + ? { + outcome: ready.outcome, + attempts: ready.attempts, + elapsedMs: ready.elapsedMs, + lastProbe: ready.lastProbe, + // A responder older than the child cannot be the child. + childAgeAtLastProbeMs: readyProbe + ? readyProbe.startedAtMs - (owner?.spawnedAtMs ?? ready.startedAtMs) + : null, + } + : null, + listeners: await inspectPortListeners(port), + }; + return `[openclaw-test-instance] gateway ${JSON.stringify(facts)}\n${formatLogs( + stdout, + stderr, + 64 * 1024, + )}`; + }, cleanup: () => { acceptingWork = false; signal?.removeEventListener("abort", closeAdmission); diff --git a/test/plugin-cron-registry-owner.e2e.test.ts b/test/plugin-cron-registry-owner.e2e.test.ts index e981fbeb5d33..a5df462b2a14 100644 --- a/test/plugin-cron-registry-owner.e2e.test.ts +++ b/test/plugin-cron-registry-owner.e2e.test.ts @@ -5,13 +5,11 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../src/config/types.openclaw.js"; -import { - connectGatewayClient, - disconnectGatewayClient, - getGatewayE2ePortBlock, -} from "../src/gateway/test-helpers.e2e.js"; +import { connectGatewayClient, disconnectGatewayClient } from "../src/gateway/test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "../src/gateway/test-helpers.listener.js"; import { upsertSessionEntry } from "../src/plugin-sdk/session-store-runtime.js"; import { closeOpenClawAgentDatabasesForTest } from "../src/plugin-sdk/sqlite-runtime-testing.js"; +import type { TestPortClaim } from "../src/test-utils/port-claims.js"; import { writeOpenAiResponsesSse } from "./helpers/openai-responses-sse.js"; import { createOpenClawTestInstance, @@ -72,11 +70,13 @@ type CronListPage = { }; const instances: OpenClawTestInstance[] = []; +const portClaims: TestPortClaim[] = []; const cleanupDirs: string[] = []; const modelServers: MockModelServer[] = []; afterEach(async () => { await Promise.all(instances.splice(0).map((instance) => instance.cleanup())); + await Promise.all(portClaims.splice(0).map((claim) => claim.release())); await Promise.all(modelServers.splice(0).map((server) => server.stop())); await Promise.all(cleanupDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); @@ -548,10 +548,11 @@ describe("plugin cron registry ownership e2e", () => { }, }, } satisfies OpenClawConfig; - const customPort = await getGatewayE2ePortBlock(); + const claim = await acquireGatewayE2ePortBlock(); + portClaims.push(claim); const instance = await createOpenClawTestInstance({ name: "plugin-cron-registry-owner", - port: customPort, + port: claim.port, config, env: { OPENCLAW_BUNDLED_PLUGINS_DIR: bundledRoot, @@ -564,7 +565,7 @@ describe("plugin cron registry ownership e2e", () => { }); instances.push(instance); await instance.startGateway(); - expect(instance.port).toBe(customPort); + expect(instance.port).toBe(claim.port); const client = await connectGatewayClient({ url: instance.url, diff --git a/ui/src/e2e/board-a2ui.e2e.test.ts b/ui/src/e2e/board-a2ui.e2e.test.ts index 67252715d7e0..042c56bab273 100644 --- a/ui/src/e2e/board-a2ui.e2e.test.ts +++ b/ui/src/e2e/board-a2ui.e2e.test.ts @@ -8,7 +8,8 @@ import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { buildWidgetDocument } from "../../../src/canvas/wrap.js"; import { buildBoardWidgetSandboxPath } from "../../../src/gateway/board-sandbox.js"; import { createSandboxHostHttpServer } from "../../../src/gateway/mcp-app-sandbox-http.js"; -import { getGatewayE2ePortBlock } from "../../../src/gateway/test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "../../../src/gateway/test-helpers.listener.js"; +import type { TestPortClaim } from "../../../src/test-utils/port-claims.js"; import { createControlUiE2eArtifactDir } from "../test-helpers/control-ui-e2e-artifacts.ts"; import { clickBoardWidgetControl } from "../test-helpers/control-ui-e2e-widget.ts"; import { @@ -34,6 +35,7 @@ let browser: Browser; let controlUi: ControlUiE2eServer; let sandboxServer: HttpServer; let sandboxPort: number; +let sandboxPortClaim: TestPortClaim | undefined; let rendererServer: HttpServer; let rendererOrigin: string; let rendererBundle: Buffer; @@ -83,7 +85,8 @@ describeControlUiE2e("Control UI dashboard A2UI", () => { } rendererOrigin = `http://127.0.0.1:${rendererAddress.port}`; controlUi = await startControlUiE2eServer(); - sandboxPort = await getGatewayE2ePortBlock(); + sandboxPortClaim = await acquireGatewayE2ePortBlock(); + sandboxPort = sandboxPortClaim.port; sandboxServer = createSandboxHostHttpServer(); await new Promise((resolve) => { sandboxServer.listen(sandboxPort, "127.0.0.1", resolve); @@ -104,6 +107,7 @@ describeControlUiE2e("Control UI dashboard A2UI", () => { sandboxServer.close(() => resolve()); }); } + await sandboxPortClaim?.release(); if (rendererServer) { await new Promise((resolve) => { rendererServer.close(() => resolve()); diff --git a/ui/src/e2e/board-mcp-app.e2e.test.ts b/ui/src/e2e/board-mcp-app.e2e.test.ts index e772379a441a..5d266a392b91 100644 --- a/ui/src/e2e/board-mcp-app.e2e.test.ts +++ b/ui/src/e2e/board-mcp-app.e2e.test.ts @@ -5,7 +5,8 @@ import { LATEST_PROTOCOL_VERSION } from "@modelcontextprotocol/ext-apps/app-brid import { chromium, type Browser, type BrowserContext, type Page } from "playwright"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { createSandboxHostHttpServer } from "../../../src/gateway/mcp-app-sandbox-http.js"; -import { getGatewayE2ePortBlock } from "../../../src/gateway/test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "../../../src/gateway/test-helpers.listener.js"; +import type { TestPortClaim } from "../../../src/test-utils/port-claims.js"; import { createControlUiE2eArtifactDir } from "../test-helpers/control-ui-e2e-artifacts.ts"; import { takeControlUiViewportScreenshot } from "../test-helpers/control-ui-e2e-screenshot.ts"; import { clickBoardWidgetControl } from "../test-helpers/control-ui-e2e-widget.ts"; @@ -31,6 +32,7 @@ let browser: Browser; let controlUi: ControlUiE2eServer; let sandboxServer: HttpServer; let sandboxPort: number; +let sandboxPortClaim: TestPortClaim | undefined; const contexts = new Set(); function widget(index: number) { @@ -176,7 +178,8 @@ async function expectRetainedBoardPresentation( describeControlUiE2e("Control UI dashboard MCP Apps", () => { beforeAll(async () => { controlUi = await startControlUiE2eServer(); - sandboxPort = await getGatewayE2ePortBlock(); + sandboxPortClaim = await acquireGatewayE2ePortBlock(); + sandboxPort = sandboxPortClaim.port; sandboxServer = createSandboxHostHttpServer(); await new Promise((resolve) => { sandboxServer.listen(sandboxPort, "127.0.0.1", resolve); @@ -195,6 +198,7 @@ describeControlUiE2e("Control UI dashboard MCP Apps", () => { sandboxServer.close(() => resolve()); }); } + await sandboxPortClaim?.release(); await controlUi?.close(); }); diff --git a/ui/src/e2e/chat-flow.catalog-bootstrap.e2e.test.ts b/ui/src/e2e/chat-flow.catalog-bootstrap.e2e.test.ts index f0b7dfbfddbf..539b4bf0d4b6 100644 --- a/ui/src/e2e/chat-flow.catalog-bootstrap.e2e.test.ts +++ b/ui/src/e2e/chat-flow.catalog-bootstrap.e2e.test.ts @@ -5,9 +5,9 @@ import { expect, it } from "vitest"; import { upsertSessionEntryCore } from "../../../src/config/sessions/session-accessor.js"; import { disconnectGatewayClient, - getGatewayE2ePortBlock, startGatewayWithClient, } from "../../../src/gateway/test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "../../../src/gateway/test-helpers.listener.js"; import { createOpenClawTestState, type OpenClawTestState, @@ -65,10 +65,14 @@ const suite = createControlUiE2eSuite({ }, "alpha", ); - const port = await getGatewayE2ePortBlock(); - signal.throwIfAborted(); + const portClaim = await acquireGatewayE2ePortBlock(); + if (signal.aborted) { + // Gateway startup has not taken ownership of the claim yet. + await portClaim.release(); + signal.throwIfAborted(); + } gatewayStartup = startGatewayWithClient({ - port, + portClaim, configPath: state.configPath, token, scopes: ["operator.admin"], diff --git a/ui/src/e2e/mcp-app-conformance.e2e.test.ts b/ui/src/e2e/mcp-app-conformance.e2e.test.ts index 8cda1a37c92e..ba7c30ea728a 100644 --- a/ui/src/e2e/mcp-app-conformance.e2e.test.ts +++ b/ui/src/e2e/mcp-app-conformance.e2e.test.ts @@ -17,12 +17,13 @@ import { getMcpAppViewLease } from "../../../src/agents/mcp-ui-resource.js"; import { readConfigFileSnapshotWithPluginMetadata } from "../../../src/config/config.js"; import type { OpenClawConfig } from "../../../src/config/types.openclaw.js"; import { startGatewayServer } from "../../../src/gateway/server.js"; -import { getGatewayE2ePortBlock } from "../../../src/gateway/test-helpers.e2e.js"; +import { acquireGatewayE2ePortBlock } from "../../../src/gateway/test-helpers.listener.js"; import { createTestGatewayScheduler } from "../../../src/test-utils/gateway-scheduler-clock.js"; import { createOpenClawTestState, type OpenClawTestState, } from "../../../src/test-utils/openclaw-test-state.ts"; +import type { TestPortClaim } from "../../../src/test-utils/port-claims.js"; import { createControlUiE2eArtifactDir } from "../test-helpers/control-ui-e2e-artifacts.ts"; import { startControlUiE2eServer } from "../test-helpers/control-ui-e2e.ts"; import { @@ -60,6 +61,8 @@ let runtimeStartup: ReturnType | undefined; let mcpScheduler: ReturnType | undefined; let gatewayPort: number; let sandboxPort: number; +// Each claim is held until the listener bound to its port has closed. +const portClaims: Partial> = {}; let tempRoot: string; let viewId: string; let appAssetServer: HttpServer | undefined; @@ -71,11 +74,13 @@ let fixture: ReturnType; let showFixture: (callId: string) => Promise; const failures: Array<{ step: string; error: string }> = []; -async function settleCleanup(step: string, cleanup: () => Promise) { +async function settleCleanup(step: string, cleanup: () => Promise): Promise { try { await cleanup(); + return true; } catch (error) { failures.push({ step, error: String(error) }); + return false; } } async function recordCleanup() { @@ -127,7 +132,8 @@ const suite = createControlUiE2eSuite({ state.envVars.OPENCLAW_BUNDLED_PLUGINS_DIR = bundledPluginsDir; const appEntryPath = require.resolve("@modelcontextprotocol/ext-apps/app-with-deps"); const appModuleSource = await fs.readFile(appEntryPath, "utf8"); - const appAssetPort = await getGatewayE2ePortBlock(); + portClaims.appAsset = await acquireGatewayE2ePortBlock(); + const appAssetPort = portClaims.appAsset.port; signal.throwIfAborted(); const fixtureAssetServer = createHttpServer((request, response) => { if (request.url === "/history-away") { @@ -167,11 +173,12 @@ const suite = createControlUiE2eSuite({ fixtureControlPath, fixtureEventsPath, ); - gatewayPort = await getGatewayE2ePortBlock(); - do { - signal.throwIfAborted(); - sandboxPort = await getGatewayE2ePortBlock(); - } while (sandboxPort === gatewayPort); + portClaims.gateway = await acquireGatewayE2ePortBlock(); + gatewayPort = portClaims.gateway.port; + signal.throwIfAborted(); + // A held claim keeps the sandbox block distinct from the Gateway block. + portClaims.sandbox = await acquireGatewayE2ePortBlock(); + sandboxPort = portClaims.sandbox.port; const cfg: OpenClawConfig = { gateway: { auth: { mode: "token", token: authValue }, @@ -243,26 +250,35 @@ const suite = createControlUiE2eSuite({ signal.throwIfAborted(); }, close: async () => { - await settleCleanup("gateway", async () => { + const gatewayClosed = await settleCleanup("gateway", async () => { const gateway = await gatewayStartup; await gateway?.close({ reason: "MCP App conformance complete" }); }); - await settleCleanup("MCP startup", async () => { + const mcpStartupSettled = await settleCleanup("MCP startup", async () => { await runtimeStartup; }); - await settleCleanup("MCP runtimes", () => disposeAllSessionMcpRuntimes()); + const mcpClosed = await settleCleanup("MCP runtimes", () => disposeAllSessionMcpRuntimes()); await settleCleanup("MCP scheduler", async () => { await mcpScheduler?.stop(); }); - if (appAssetServer) { - await settleCleanup( - "asset server", - () => - new Promise((resolve, reject) => { - appAssetServer?.close((error) => (error ? reject(error) : resolve())); - }), - ); - } + const assetServerClosed = appAssetServer + ? await settleCleanup( + "asset server", + () => + new Promise((resolve, reject) => { + appAssetServer?.close((error) => (error ? reject(error) : resolve())); + }), + ) + : true; + // Incomplete cleanup can leave a listener bound; keep its port claimed. + const releasable = [ + gatewayClosed && portClaims.gateway, + gatewayClosed && mcpStartupSettled && mcpClosed && portClaims.sandbox, + assetServerClosed && portClaims.appAsset, + ].filter((claim): claim is TestPortClaim => Boolean(claim)); + await settleCleanup("port claims", () => + Promise.all(releasable.map((claim) => claim.release())), + ); if (tempRoot) { await settleCleanup("archive fixture events", () => fs.copyFile(fixtureEventsPath, path.join(proofDir, "fixture-events.jsonl")),