From 63144fe0c4d170902aa50155daa36460af6e8569 Mon Sep 17 00:00:00 2001 From: Josh Avant <830519+joshavant@users.noreply.github.com> Date: Wed, 30 Sep 2026 01:48:56 -0500 Subject: [PATCH] fix(qa): Inspector collection fails when a pairing link is reused (#161540) * fix(qa): pair Inspector pages with fresh dashboard links Acquire a fresh single-use dashboard handoff for every Inspector page. Cover real pairing, replay rejection, identity and receipt selection, reload, and Chat draft preservation in a release-only browser fixture. Real Gateway proof: 81.29s wall including prerequisites, 17.84s test body. * fix(ci): complete Inspector pairing fixture inventories Align private-server discovery and non-release counts with the Inspector fixture, and include it in the frozen-target fallback command. Shorten nearby comments to preserve the existing workflow size limit. Workflow guards: 137 passed, 8 skipped; 85.65s wall. Focused routing: 6 passed; 11.77s wall. * fix(ci): include preflight manifest in trusted checkout --- .github/workflows/ci.yml | 1 + docs/reference/test/lanes.md | 8 + scripts/lib/ci-node-test-plan.mts | 1 + test/scripts/ci-git-owner.test.ts | 65 +++++- test/scripts/ci-node-test-plan.test.ts | 1 + .../ci-ui-e2e-ownership.test-support.ts | 1 + test/scripts/ci-workflow-guards.test.ts | 2 +- test/vitest-ui-e2e-config.test.ts | 3 +- test/vitest/vitest.ui-e2e.config.ts | 1 + test/vitest/vitest.ui-paths.mjs | 2 + ...ity-run-inspector.real-gateway.e2e.test.ts | 220 ++++++++++++++++++ .../control-ui-browser-pairing.ts | 17 ++ .../test-helpers/control-ui-run-inspector.ts | 2 +- 13 files changed, 320 insertions(+), 4 deletions(-) create mode 100644 ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts create mode 100644 ui/src/test-helpers/control-ui-browser-pairing.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd1f55142555..6983711bef1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2436,6 +2436,7 @@ jobs: ui/src/e2e/profile-page.real-gateway.e2e.test.ts \ ui/src/e2e/quota-reset-status.real-gateway.e2e.test.ts \ ui/src/e2e/logs-lifecycle.e2e.test.ts \ + ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts \ ui/src/e2e/agent-file-lifecycle.real-gateway.e2e.test.ts \ ui/src/e2e/chat-collaborator-scroll.real-gateway.e2e.test.ts \ ui/src/e2e/chat-composer-websearch-kill-switch.real-gateway.e2e.test.ts \ diff --git a/docs/reference/test/lanes.md b/docs/reference/test/lanes.md index b53f30d447d2..e3efa95bceb1 100644 --- a/docs/reference/test/lanes.md +++ b/docs/reference/test/lanes.md @@ -30,6 +30,14 @@ read_when: ### Real-Gateway Control UI fixture lifetimes +Use `pairControlUiPage` from `ui/src/test-helpers/control-ui-browser-pairing.ts` +with the isolated Gateway's CLI runner to authenticate each new page. It obtains +and consumes a fresh `dashboard --json` browser handoff and waits for the Gateway +handshake. Dashboard pairing links are single-use; do not reuse a captured URL +for another tab or browser context. Pass this operation as `preparePage` to +`withControlUiRunInspector` so collection authenticates its own page while leaving +the caller's Chat and draft in place. Reload uses the browser's paired credential. + Use `createControlUiE2eSuite` from `ui/src/e2e/control-ui-e2e-suite.test-support.ts` for real-Gateway browser fixtures. `suite.define(...)` owns the native hooks. Each native `it` passes its test context diff --git a/scripts/lib/ci-node-test-plan.mts b/scripts/lib/ci-node-test-plan.mts index 4c6027506093..f423451dc7bd 100644 --- a/scripts/lib/ci-node-test-plan.mts +++ b/scripts/lib/ci-node-test-plan.mts @@ -1569,6 +1569,7 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([ const RELEASE_ONLY_PLUGIN_SHARDS = new Set(["agentic-plugins"]); const RELEASE_ONLY_TOOLING_SHARDS = new Set(["core-tooling"]); const RELEASE_ONLY_UI_TEST_FILES = new Set([ + "ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts", "ui/src/e2e/board-fixture.e2e.test.ts", "ui/src/e2e/chat-attachment-menu.e2e.test.ts", "ui/src/e2e/chat-mobile-bubble-margin.e2e.test.ts", diff --git a/test/scripts/ci-git-owner.test.ts b/test/scripts/ci-git-owner.test.ts index 151c85967781..768cc67c7884 100644 --- a/test/scripts/ci-git-owner.test.ts +++ b/test/scripts/ci-git-owner.test.ts @@ -1,7 +1,7 @@ import { spawnSync } from "node:child_process"; import { chmodSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { EOL, tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { pathToFileURL } from "node:url"; import { expectDefined } from "@openclaw/normalization-core/expect"; import { beforeAll, expect, vi } from "vitest"; @@ -9,6 +9,7 @@ import { parse } from "yaml"; import { createCommandTest } from "../helpers/command-fixture.js"; import { readCiCheckoutStep, renderGitTestClock } from "./ci-checkout.test-support.js"; import { runCiGitStep, type FetchResult } from "./ci-git-owner.test-support.js"; +import { runDependencyFreePreflight } from "./ci-preflight-dependencies.test-support.js"; // Each case owns its checkout and process trees. Overlap their real timeout and // drain waits, but keep subprocess pressure bounded on the four-core CI runner. @@ -572,6 +573,68 @@ releasePolicyIt("returns 124 when the release ancestry total budget is exhausted expect(report.commands).toEqual([]); }); +it("materializes an executable preflight manifest from the workflow revision", async ({ + command, +}) => { + const root = command.createTempDir("ci-preflight-harness-"); + const origin = join(root, "origin"); + const workspace = join(root, "checkout"); + mkdirSync(origin); + mkdirSync(workspace); + fixtureGit(origin, ["init", "--quiet"]); + for (const file of [ + ".github/actions/setup-node-env/action.yml", + ".github/actions/git-owner/test-prerequisites.mjs", + ".github/actions/git-owner/test-prerequisites.json", + "scripts/ci-build-manifest.mjs", + "scripts/lib/release-context.mjs", + "scripts/lib/release-version.mjs", + ]) { + const destination = join(origin, file); + mkdirSync(dirname(destination), { recursive: true }); + writeFileSync(destination, readFileSync(file)); + } + fixtureGit(origin, ["add", "."]); + const tree = fixtureGit(origin, ["write-tree"]); + const revision = fixtureCommit(join(origin, ".git"), tree, undefined, "workflow fixture"); + fixtureGit(origin, ["update-ref", "HEAD", revision]); + const gitConfig = join(root, "gitconfig"); + writeFileSync(gitConfig, ""); + const checkout = await command.run( + process.platform === "win32" ? "python" : "python3", + ["-I", "-S", gitOwnerPath], + { + cwd: workspace, + env: { + ...process.env, + CHECKOUT_KIND: "preflight", + CHECKOUT_REPO: "fixture/preflight", + CHECKOUT_TOKEN: "", + CHECKOUT_REF: revision, + CHECKOUT_FALLBACK_REF: revision, + WORKFLOW_SHA: revision, + GITHUB_WORKSPACE: workspace, + GITHUB_EVENT_NAME: "pull_request", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: gitConfig, + GIT_CONFIG_COUNT: "1", + GIT_CONFIG_KEY_0: `url.${pathToFileURL(origin).href}.insteadOf`, + GIT_CONFIG_VALUE_0: "https://github.com/fixture/preflight.git", + }, + }, + ); + expect(checkout.status, `${checkout.stdout}\n${checkout.stderr}`).toBe(0); + // Consume the exported trusted entrypoint against the real target planners. + const { result, manifest } = runDependencyFreePreflight( + pathToFileURL(join(workspace, ".ci-harness/scripts/ci-build-manifest.mjs")), + root, + process.execPath, + ); + expect(result.status, result.stderr).toBe(0); + expect(manifest).toContain("run_windows=true\n"); + expect(fixtureGit(workspace, ["status", "--porcelain"])).toBe(""); +}); + // Ask Bash to decode the source independently of the generator and fixture codec. it("keeps exactly one byte-identical generated CI owner", () => { const workflow = readFileSync(".github/workflows/ci.yml", "utf8"); diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index bf1a8109f5f1..0089dbffb421 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -105,6 +105,7 @@ describe("Control UI release-only inventories", () => { const automationManagement = "extensions/qa-lab/src/control-ui-automation-management.real-gateway.e2e.test.ts"; const releaseOnlyRealGateway = new Set([ + "ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts", "ui/src/e2e/cron-duration-save.real-gateway.e2e.test.ts", "ui/src/e2e/desktop-resize.real-gateway.e2e.test.ts", automationManagement, diff --git a/test/scripts/ci-ui-e2e-ownership.test-support.ts b/test/scripts/ci-ui-e2e-ownership.test-support.ts index 3bad7bd1849d..dd9d010c9251 100644 --- a/test/scripts/ci-ui-e2e-ownership.test-support.ts +++ b/test/scripts/ci-ui-e2e-ownership.test-support.ts @@ -107,6 +107,7 @@ export function assertControlUiE2eOwnership( expect(privateServerFiles).toEqual(uiE2ePrivateServerTestFiles); expect(helperPrivateServerFiles.toSorted()).toEqual([ + "ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts", "ui/src/e2e/agent-file-lifecycle.real-gateway.e2e.test.ts", "ui/src/e2e/agent-switch-roster.e2e.test.ts", "ui/src/e2e/chat-agent-avatar.real-gateway.e2e.test.ts", diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 197efe52af2f..4b612200080b 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -5371,7 +5371,7 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}" .toSorted(), ); if (releaseTier === false) { - expect(selectedFiles).toHaveLength(uiE2eRealGatewayTestFiles.length - 11); + expect(selectedFiles).toHaveLength(uiE2eRealGatewayTestFiles.length - 12); expect(selectedFiles).not.toContain( "ui/src/e2e/cron-duration-save.real-gateway.e2e.test.ts", ); diff --git a/test/vitest-ui-e2e-config.test.ts b/test/vitest-ui-e2e-config.test.ts index 22a440ec260e..60fdd9ae44bf 100644 --- a/test/vitest-ui-e2e-config.test.ts +++ b/test/vitest-ui-e2e-config.test.ts @@ -111,6 +111,7 @@ const qaLabFiles = [ "extensions/qa-lab/src/control-ui-openclaw-delegation.real-gateway.e2e.test.ts", ] as const; const realGatewayFiles = [ + "activity-run-inspector.real-gateway", "agent-file-lifecycle.real-gateway", "chat-agent-avatar.real-gateway", "chat-collaborator-scroll.real-gateway", @@ -669,7 +670,7 @@ describe("Control UI E2E resource ownership", () => { }, ]); const parallel = result.files.filter((entry) => entry.phase === 2); - expect(parallel).toHaveLength(26); + expect(parallel).toHaveLength(27); expect(parallel.every((entry) => entry.fileParallelism)).toBe(true); expect(parallel.every((entry) => entry.workers === result.rootWorkers)).toBe(true); for (const entry of parallel) { diff --git a/test/vitest/vitest.ui-e2e.config.ts b/test/vitest/vitest.ui-e2e.config.ts index 5710db2b6cca..016f31c26edc 100644 --- a/test/vitest/vitest.ui-e2e.config.ts +++ b/test/vitest/vitest.ui-e2e.config.ts @@ -20,6 +20,7 @@ const uiE2eIncludePatterns = [ // These files own their server instead of leasing the global production bundle. // Keep any shared source-module optimizer cache under one worker. export const uiE2ePrivateServerTestFiles = [ + "ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts", "ui/src/e2e/agent-file-lifecycle.real-gateway.e2e.test.ts", "ui/src/e2e/agent-switch-roster.e2e.test.ts", "ui/src/e2e/approval-bootstrap.e2e.test.ts", diff --git a/test/vitest/vitest.ui-paths.mjs b/test/vitest/vitest.ui-paths.mjs index 70db8d3a95d2..b9bbcf6a805a 100644 --- a/test/vitest/vitest.ui-paths.mjs +++ b/test/vitest/vitest.ui-paths.mjs @@ -59,6 +59,7 @@ export function isUiTestTarget(relative) { } export const uiE2eRealGatewayTestFiles = [ + "ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts", "ui/src/e2e/session-roster-request-rate.real-gateway.e2e.test.ts", "ui/src/e2e/quota-reset-status.real-gateway.e2e.test.ts", "ui/src/e2e/model-api-keys.real-gateway.e2e.test.ts", @@ -100,6 +101,7 @@ export const uiE2eRealGatewayTestFiles = [ // Listed fixtures own their HOME, state, ports, and cleanup; UI bytes are either // borrowed from the invocation preview or read by their prepared Gateway child. export const uiE2ePrebuiltParallelTestFiles = [ + "ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts", "ui/src/e2e/agent-file-lifecycle.real-gateway.e2e.test.ts", "ui/src/e2e/chat-agent-avatar.real-gateway.e2e.test.ts", "ui/src/e2e/chat-composer-websearch-kill-switch.real-gateway.e2e.test.ts", diff --git a/ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts b/ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts new file mode 100644 index 000000000000..e001658d9776 --- /dev/null +++ b/ui/src/e2e/activity-run-inspector.real-gateway.e2e.test.ts @@ -0,0 +1,220 @@ +import { randomUUID } from "node:crypto"; +import { createServer } from "node:http"; +import { expect, it } from "vitest"; +import type { AuditRunInspectResult } from "../../../packages/gateway-protocol/src/schema/audit-run.js"; +import type { + ErrorShape, + GatewayFrame, +} from "../../../packages/gateway-protocol/src/schema/frames.js"; +import { reserveTestPortListener } from "../../../src/test-utils/port-claims.js"; +import { writeOpenAiResponsesText } from "../../../test/helpers/openai-responses-sse.ts"; +import { + createOpenClawTestInstance, + type OpenClawTestInstance, +} from "../../../test/helpers/openclaw-test-instance.ts"; +import { runQaGatewayFixture } from "../../../test/helpers/qa-gateway-cleanup.ts"; +import { pairControlUiPage } from "../test-helpers/control-ui-browser-pairing.ts"; +import { waitForControlUiGatewayReady } from "../test-helpers/control-ui-e2e-readiness.ts"; +import { controlUiSessionUrl } from "../test-helpers/control-ui-e2e.ts"; +import { withControlUiRunInspector } from "../test-helpers/control-ui-run-inspector.ts"; +import { + createControlUiE2eContextOptions, + createControlUiE2eSuite, +} from "./control-ui-e2e-suite.test-support.ts"; + +const replyText = "Inspector pairing fixture completed."; +let instance: OpenClawTestInstance; +const suite = createControlUiE2eSuite({ + name: "Run Inspector with real browser pairing", + startServerBeforeBrowser: true, + async startServer() { + const provider = await reserveTestPortListener({ + offsets: [0], + createListener: () => + createServer((request, response) => { + request.resume().once("end", () => { + if (request.method !== "POST" || request.url !== "/v1/responses") { + response.writeHead(404).end(); + return; + } + writeOpenAiResponsesText(response, { + text: replyText, + messageId: "inspector-message", + responseId: "inspector-response", + }); + }); + }), + }); + const cleanup = () => + runQaGatewayFixture( + () => instance?.cleanup(), + async () => { + provider.listener.closeAllConnections(); + await provider.releaseListener(); + }, + () => provider.claim.release(), + ); + try { + instance = await createOpenClawTestInstance({ + name: "inspector-pairing", + env: { OPENCLAW_TEST_MINIMAL_GATEWAY: undefined, VITEST: undefined }, + config: { + gateway: { controlUi: { enabled: true } }, + logging: { audit: { executionIdentity: true } }, + cron: { enabled: false }, + agents: { + ownership: "explicit", + defaults: { + model: "inspector-fixture/echo", + modelPolicy: { allow: ["inspector-fixture/*"] }, + }, + entries: { main: { identity: { name: "Inspector fixture" } } }, + }, + models: { + catalogRefresh: { enabled: false }, + providers: { + "inspector-fixture": { + api: "openai-responses", + apiKey: "synthetic-unused-key", + baseUrl: `http://127.0.0.1:${provider.claim.port}/v1`, + models: [{ id: "echo", name: "Echo" }], + }, + }, + }, + plugins: { allow: [] }, + }, + }); + await instance.startGateway(); + return { baseUrl: `http://127.0.0.1:${instance.port}/`, close: cleanup }; + } catch (error) { + return runQaGatewayFixture(async () => { + throw error; + }, cleanup); + } + }, +}); + +async function runCli(args: string[]): Promise { + const result = await instance.cli(args); + expect(result.code, result.stderr).toBe(0); + return result.stdout; +} + +async function rpc(method: string, params: Record) { + return JSON.parse( + await runCli(["gateway", "call", method, "--params", JSON.stringify(params), "--json"]), + ); +} + +suite.define(() => { + it("pairs each Inspector page independently and retains identity across reload", async (test) => { + await suite.runScenario(test, { + retainedState: () => instance.stateDir, + run: async () => { + const sessionKey = "agent:main:inspector-pairing"; + await rpc("sessions.create", { key: sessionKey, agentId: "main" }); + const runId = randomUUID(); + await rpc("chat.send", { + sessionKey, + message: "Complete the fixture.", + idempotencyKey: runId, + }); + expect(await rpc("agent.wait", { runId, timeoutMs: 30_000 })).toMatchObject({ + status: "ok", + terminalReply: { disposition: "visible", text: replyText }, + }); + const inspected: AuditRunInspectResult = await rpc("audit.run.inspect", { runId }); + if (inspected.identity.state !== "present") { + throw new Error(`Fixture identity is ${inspected.identity.state}`); + } + const { executionId } = inspected.identity.context; + const receipt = inspected.decisionDisplays.find( + (display) => + display.provenance.state === "verified" && + display.provenance.producer === "run-admission", + ); + if (!receipt) { + throw new Error("Fixture run admission receipt is missing"); + } + const { browserUrl }: { browserUrl: string } = JSON.parse( + await runCli(["dashboard", "--json"]), + ); + const context = await suite.newBrowserContext(createControlUiE2eContextOptions()); + const freshContext = await suite.newBrowserContext(createControlUiE2eContextOptions()); + for (const browserContext of [context, freshContext]) { + await browserContext.addInitScript(() => { + localStorage.setItem("openclaw:control-ui:community-invite", "dismissed"); + }); + } + const chat = await context.newPage(); + // Keep this handoff only for the explicit replay rejection below. + await chat.goto(browserUrl); + await waitForControlUiGatewayReady(chat); + await chat.goto(controlUiSessionUrl(suite.server.baseUrl, sessionKey)); + await chat.getByText(replyText, { exact: true }).waitFor(); + const composer = chat.getByRole("textbox", { name: "Chat composer", exact: true }); + const draft = "Keep this unsent Inspector draft"; + await composer.fill(draft); + const chatUrl = chat.url(); + + // Explicit replay must fail even when this context already has a paired device. + const replay = await context.newPage(); + const rejections: ErrorShape[] = []; + replay.on("websocket", (socket) => { + socket.on("framereceived", ({ payload }) => { + const frame: GatewayFrame = JSON.parse(String(payload)); + if (frame.type === "res" && !frame.ok && frame.error) { + rejections.push(frame.error); + } + }); + }); + await replay.goto(browserUrl); + await replay.getByText("Pairing link is no longer valid", { exact: false }).waitFor(); + expect(rejections).toContainEqual( + expect.objectContaining({ + details: expect.objectContaining({ code: "AUTH_BOOTSTRAP_TOKEN_INVALID" }), + }), + ); + expect(await replay.locator("#activity-run-panel").count()).toBe(0); + await replay.close(); + + for (const inspectorContext of [context, freshContext]) { + await withControlUiRunInspector( + inspectorContext, + { + baseUrl: suite.server.baseUrl, + selector: { kind: "execution", id: executionId }, + receipt: { id: receipt.selectorId }, + preparePage: (page) => pairControlUiPage(page, runCli), + }, + async (page, inspector) => { + const assertIdentity = async () => { + await inspector.waitFor({ state: "visible" }); + await page.getByRole("heading", { name: "Identity and authority" }).waitFor(); + expect(await inspector.getAttribute("data-execution-id")).toBe(executionId); + expect(await inspector.getAttribute("data-run-id")).toBe(runId); + const detail = inspector.locator( + '[aria-labelledby="run-inspector-receipt-detail"]', + ); + await detail.waitFor({ state: "visible" }); + expect(await detail.getAttribute("data-receipt-selector-id")).toBe( + receipt.selectorId, + ); + expect(await detail.textContent()).toContain(receipt.decision.reasonCode); + }; + await assertIdentity(); + await page.reload(); + await assertIdentity(); + }, + ); + expect(context.pages()).toEqual([chat]); + expect(freshContext.pages()).toEqual([]); + expect(chat.url()).toBe(chatUrl); + expect(await composer.inputValue()).toBe(draft); + } + await chat.reload(); + await chat.getByText(replyText, { exact: true }).waitFor(); + }, + }); + }, 90_000); +}); diff --git a/ui/src/test-helpers/control-ui-browser-pairing.ts b/ui/src/test-helpers/control-ui-browser-pairing.ts new file mode 100644 index 000000000000..b6fb73934b8e --- /dev/null +++ b/ui/src/test-helpers/control-ui-browser-pairing.ts @@ -0,0 +1,17 @@ +import type { Page } from "playwright"; +import { waitForControlUiGatewayReady } from "./control-ui-e2e-readiness.ts"; + +/** Acquire and consume a fresh handoff for this page; never cache a single-use dashboard URL. */ +export async function pairControlUiPage( + page: Page, + runCli: (args: string[]) => Promise, +): Promise { + const handoff: { ok: boolean; browserUrl?: string; reason?: string } = JSON.parse( + await runCli(["dashboard", "--json"]), + ); + if (!handoff.ok || !handoff.browserUrl) { + throw new Error(handoff.reason ?? "Dashboard did not issue a browser pairing link"); + } + await page.goto(handoff.browserUrl); + await waitForControlUiGatewayReady(page); +} diff --git a/ui/src/test-helpers/control-ui-run-inspector.ts b/ui/src/test-helpers/control-ui-run-inspector.ts index 25e37e4eecca..771aa67d741e 100644 --- a/ui/src/test-helpers/control-ui-run-inspector.ts +++ b/ui/src/test-helpers/control-ui-run-inspector.ts @@ -17,7 +17,7 @@ export async function withControlUiRunInspector( baseUrl: string; selector: RunInspectorSelector; receipt?: Parameters[2]; - /** Install the mock Gateway or campaign-owned per-tab auth before navigation. */ + /** Install a mock Gateway or use pairControlUiPage with the isolated Gateway's CLI. */ preparePage?: (page: Page) => Promise; }, collect: (page: Page, inspector: Locator) => Promise,