From 5fdaa04ecaa84ad9422db74c848c7e8863e66b82 Mon Sep 17 00:00:00 2001 From: Kevin Lin Date: Sun, 5 Jul 2026 14:47:18 -0700 Subject: [PATCH] feat(qa-lab): add Codex Slack approval scenarios (#91519) * feat(qa-lab): add codex slack approval scenarios * fix(qa-lab): harden Codex Slack approval proof * fix(qa-lab): validate Codex approval model * test(qa-lab): verify approved Codex operation * fix(qa-lab): normalize Slack QA account routing * fix(qa-lab): read Codex tool result transcripts * fix(qa-lab): harden Codex approval cleanup * fix(qa-lab): support Codex Mantis checkpoints * fix(qa-lab): repair Mantis pnpm bootstrap * fix(qa-lab): normalize Codex tool result text * fix(qa-lab): scope Mantis pnpm fallback * fix(qa-lab): bootstrap pinned pnpm without npm * fix(qa-lab): terminate pnpm metadata record * fix(qa-lab): bootstrap official node for mantis * fix(qa-lab): make mantis pnpm shim executable --------- Co-authored-by: Peter Steinberger --- .../workflows/mantis-slack-desktop-smoke.yml | 13 +- docs/concepts/qa-e2e-automation.md | 25 + .../src/app-server/transport-stdio.test.ts | 17 + .../codex/src/app-server/transport-stdio.ts | 11 +- extensions/qa-lab/src/gateway-child.test.ts | 23 + extensions/qa-lab/src/gateway-child.ts | 11 +- .../qa-lab/src/gateway-log-sentinel.test.ts | 8 + extensions/qa-lab/src/gateway-log-sentinel.ts | 6 +- .../shared/live-gateway.runtime.test.ts | 40 + .../shared/live-gateway.runtime.ts | 17 +- .../slack/slack-live.runtime.test.ts | 374 ++++++- .../slack/slack-live.runtime.ts | 914 ++++++++++++++++-- .../slack-desktop-smoke.runtime.test.ts | 96 ++ .../src/mantis/slack-desktop-smoke.runtime.ts | 137 ++- .../package-acceptance-workflow.test.ts | 11 + 15 files changed, 1619 insertions(+), 84 deletions(-) diff --git a/.github/workflows/mantis-slack-desktop-smoke.yml b/.github/workflows/mantis-slack-desktop-smoke.yml index be6af638c14..e813f83e220 100644 --- a/.github/workflows/mantis-slack-desktop-smoke.yml +++ b/.github/workflows/mantis-slack-desktop-smoke.yml @@ -364,7 +364,6 @@ jobs: checkpoint_required=false if [[ "$APPROVAL_CHECKPOINTS" == "true" ]]; then evidence_summary="Mantis ran Slack native approval QA inside a Crabbox Linux VNC desktop, rendered pending/resolved approval checkpoints from the Slack API messages, and stored Slack QA artifacts." - expected_result="Slack native exec and plugin approval checkpoints pass" screenshot_required=false desktop_capture_inline=false if [[ "$status" == "pass" ]]; then @@ -373,8 +372,10 @@ jobs: checkpoint_scenarios=() if [[ "$scenario_label" == "approval-checkpoints" ]]; then checkpoint_scenarios=("slack-approval-exec-native" "slack-approval-plugin-native") + expected_result="Slack native exec and plugin approval checkpoints pass" else checkpoint_scenarios=("$scenario_label") + expected_result="Slack approval checkpoint passes for $scenario_label" fi checkpoint_scenarios_json="$(printf '%s\n' "${checkpoint_scenarios[@]}" | jq -R . | jq -s .)" checkpoint_artifacts="$( @@ -383,12 +384,16 @@ jobs: --argjson scenario_ids "$checkpoint_scenarios_json" \ ' def scenario_kind($id): - if $id == "slack-approval-exec-native" then "exec" - elif $id == "slack-approval-plugin-native" then "plugin" + if ($id | endswith("-exec-native")) then "exec" + elif ($id | endswith("-plugin-native")) then "plugin" else error("unsupported approval checkpoint scenario: \($id)") end; def scenario_title($id): - if scenario_kind($id) == "exec" then "Exec" else "Plugin" end; + if ($id | startswith("slack-codex-")) then + "Codex \(scenario_kind($id))" + elif scenario_kind($id) == "exec" then "Exec" + else "Plugin" + end; [ $scenario_ids[] as $id | ["pending", "resolved"][] as $state diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 34f9114a534..79d710d1b46 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -288,6 +288,14 @@ empty. Cold CI leases may still show Slack sign-in in `slack-desktop-smoke.png`; the approval checkpoint images are the visual proof for this lane. +The default checkpoint run keeps the two standard Slack approval scenarios. +To capture either opt-in Codex approval route, select it explicitly with +`--scenario slack-codex-approval-exec-native` or +`--scenario slack-codex-approval-plugin-native`; Mantis accepts both and emits +the same pending/resolved screenshot pair. The runner expands its checkpoint +and remote-command deadlines for each selected Codex route so the full +approval, agent completion, and resolved-update sequence can finish. + The operator checklist, GitHub workflow dispatch command, evidence-comment contract, hydrate-mode decision table, timing interpretation, and failure handling steps live in @@ -611,6 +619,23 @@ Scenarios (`extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts`): scenario. Enables exec and plugin approval forwarding together so plugin events are not suppressed by exec approval routing, then verifies the same pending/resolved native Slack UI path. +- `slack-codex-approval-exec-native` - opt-in Codex Guardian command approval + scenario. Enables the Codex plugin in Guardian mode, routes a + Slack-originated Gateway agent turn through the Codex app-server harness, + waits for the native Slack plugin approval prompt for + `openclaw-codex-app-server`, resolves it, and verifies the Codex turn + finishes with the expected command-output and assistant markers. +- `slack-codex-approval-plugin-native` - opt-in Codex Guardian file approval + scenario. Uses an outside-workspace `apply_patch` instruction so Codex emits + the app-server file-change approval route, then verifies the same native + Slack pending/resolved approval path, final assistant marker, and exact file + contents before cleanup. + +The Codex approval scenarios require an `openai/*` or `codex/*` `--model`, the +normal live model credentials, and Codex auth or API-key auth accepted by the Codex plugin. +The Slack report includes the Codex app-server method, selected Codex model key, +final Codex turn status, and operation-marker verification alongside the +redacted Slack approval metadata. Output artifacts: diff --git a/extensions/codex/src/app-server/transport-stdio.test.ts b/extensions/codex/src/app-server/transport-stdio.test.ts index 0a523de09ca..3a72a2d5136 100644 --- a/extensions/codex/src/app-server/transport-stdio.test.ts +++ b/extensions/codex/src/app-server/transport-stdio.test.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { CodexAppServerStartOptions } from "./config.js"; import { + resolveCodexAppServerDetachedMode, resolveCodexAppServerSpawnEnv, resolveCodexAppServerSpawnInvocation, } from "./transport-stdio.js"; @@ -185,3 +186,19 @@ describe("resolveCodexAppServerSpawnEnv", () => { expect(Object.hasOwn(env, "prototype")).toBe(false); }); }); + +describe("resolveCodexAppServerDetachedMode", () => { + it("detaches normal POSIX app-server processes", () => { + expect(resolveCodexAppServerDetachedMode({}, "darwin")).toBe(true); + }); + + it("keeps QA app-server processes in the gateway process group", () => { + expect(resolveCodexAppServerDetachedMode({ OPENCLAW_QA_PARENT_PID: "12345" }, "linux")).toBe( + false, + ); + }); + + it("does not detach Windows app-server processes", () => { + expect(resolveCodexAppServerDetachedMode({}, "win32")).toBe(false); + }); +}); diff --git a/extensions/codex/src/app-server/transport-stdio.ts b/extensions/codex/src/app-server/transport-stdio.ts index c3623df8147..08988ef5b4a 100644 --- a/extensions/codex/src/app-server/transport-stdio.ts +++ b/extensions/codex/src/app-server/transport-stdio.ts @@ -11,6 +11,7 @@ import type { CodexAppServerStartOptions } from "./config.js"; import type { CodexAppServerTransport } from "./transport.js"; const UNSAFE_ENVIRONMENT_KEYS = new Set(["__proto__", "constructor", "prototype"]); +const QA_PARENT_PID_ENV = "OPENCLAW_QA_PARENT_PID"; type CodexAppServerSpawnRuntime = { platform: NodeJS.Platform; @@ -73,6 +74,14 @@ export function resolveCodexAppServerSpawnEnv( return env; } +/** Keeps QA-owned app-server processes inside the gateway process-group cleanup boundary. */ +export function resolveCodexAppServerDetachedMode( + env: NodeJS.ProcessEnv, + platform: NodeJS.Platform = process.platform, +): boolean { + return platform !== "win32" && !env[QA_PARENT_PID_ENV]?.trim(); +} + function normalizedEnvironmentKeys(rawKeys: readonly string[]): string[] { const keys: string[] = []; for (const rawKey of rawKeys) { @@ -106,7 +115,7 @@ export function createStdioTransport(options: CodexAppServerStartOptions): Codex }); return spawn(invocation.command, invocation.args, { env, - detached: process.platform !== "win32", + detached: resolveCodexAppServerDetachedMode(env), shell: invocation.shell, stdio: ["pipe", "pipe", "pipe"], windowsHide: invocation.windowsHide, diff --git a/extensions/qa-lab/src/gateway-child.test.ts b/extensions/qa-lab/src/gateway-child.test.ts index 6ce2e7c1a46..c14d304f5cf 100644 --- a/extensions/qa-lab/src/gateway-child.test.ts +++ b/extensions/qa-lab/src/gateway-child.test.ts @@ -989,6 +989,9 @@ describe("buildQaRuntimeEnv", () => { child.signalCode = "SIGKILL"; queueMicrotask(() => child.emit("exit")); } + if (signal === 0 && child.signalCode) { + throw Object.assign(new Error("no such process"), { code: "ESRCH" }); + } return true; }); @@ -1010,6 +1013,26 @@ describe("buildQaRuntimeEnv", () => { expect([child.exitCode, child.signalCode]).not.toEqual([null, null]); }); + it.runIf(process.platform !== "win32")( + "fails closed when forced gateway process-group shutdown times out", + async () => { + const child = Object.assign(new EventEmitter(), { + pid: 12345, + exitCode: null as number | null, + signalCode: null as string | null, + kill: vi.fn(() => true), + }); + vi.spyOn(process, "kill").mockImplementation(() => true); + + await expect( + testing.stopQaGatewayChildProcessTree(child as never, { + gracefulTimeoutMs: 1, + forceTimeoutMs: 1, + }), + ).rejects.toThrow("qa gateway process tree remained alive after forced shutdown"); + }, + ); + it("force-kills Windows gateway process trees when graceful taskkill fails", () => { const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform"); const originalSystemRoot = process.env.SystemRoot; diff --git a/extensions/qa-lab/src/gateway-child.ts b/extensions/qa-lab/src/gateway-child.ts index 6e726709067..34c66f339f6 100644 --- a/extensions/qa-lab/src/gateway-child.ts +++ b/extensions/qa-lab/src/gateway-child.ts @@ -413,11 +413,11 @@ function isQaGatewayChildProcessTreeAlive(child: ChildProcess) { process.kill(-child.pid, 0); return true; } catch (error) { - if (isProcessAlreadyExitedError(error)) { - return false; + if (!isProcessAlreadyExitedError(error) && !hasChildExited(child)) { + return true; } - return !hasChildExited(child); } + return false; } type QaGatewayTaskkillRunner = typeof spawnSync; @@ -498,7 +498,10 @@ async function stopQaGatewayChildProcessTree( return; } signalQaGatewayChildProcessTree(child, "SIGKILL"); - await waitForQaGatewayChildExit(child, opts?.forceTimeoutMs ?? 2_000); + const stopped = await waitForQaGatewayChildExit(child, opts?.forceTimeoutMs ?? 2_000); + if (!stopped) { + throw new Error("qa gateway process tree remained alive after forced shutdown"); + } } function isQaModelProviderConfig(value: unknown): value is ModelProviderConfig { diff --git a/extensions/qa-lab/src/gateway-log-sentinel.test.ts b/extensions/qa-lab/src/gateway-log-sentinel.test.ts index 08c47eec28b..4442b438835 100644 --- a/extensions/qa-lab/src/gateway-log-sentinel.test.ts +++ b/extensions/qa-lab/src/gateway-log-sentinel.test.ts @@ -2,12 +2,20 @@ import { describe, expect, it } from "vitest"; import { assertNoGatewayLogSentinels, + extractGatewayMessageText, formatGatewayLogSentinelSummary, scanDirectReplyTranscriptSentinels, scanGatewayLogSentinels, } from "./gateway-log-sentinel.js"; describe("gateway log sentinels", () => { + it.each([ + [{ content: [{ type: "toolResult", content: "codex output" }] }, "codex output"], + [{ content: [{ type: "text", text: "standard output" }] }, "standard output"], + ])("extracts message text from tool result shapes", (message, expected) => { + expect(extractGatewayMessageText(message)).toBe(expected); + }); + it("classifies May 13 beta.5 operational failure signatures", () => { const findings = scanGatewayLogSentinels( [ diff --git a/extensions/qa-lab/src/gateway-log-sentinel.ts b/extensions/qa-lab/src/gateway-log-sentinel.ts index 9b3426e59b3..822268c0512 100644 --- a/extensions/qa-lab/src/gateway-log-sentinel.ts +++ b/extensions/qa-lab/src/gateway-log-sentinel.ts @@ -169,9 +169,13 @@ export function extractGatewayMessageText(message: Record) { continue; } const nestedText = readNonEmptyString(block.content); + const normalizedType = readNonEmptyString(block.type)?.toLowerCase().replace(/_/g, ""); if ( nestedText && - (block.type === "output_text" || block.type === "text" || block.type === "message") + (normalizedType === "outputtext" || + normalizedType === "text" || + normalizedType === "message" || + normalizedType === "toolresult") ) { parts.push(nestedText); } diff --git a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts index 9c5919ead23..7690f4d67ba 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts @@ -268,4 +268,44 @@ describe("startQaLiveLaneGateway", () => { "failed to stop QA live lane resources:\ngateway stop failed: gateway down\nmock provider stop failed: mock down", ); }); + + it("retries only mock cleanup after gateway preservation succeeds", async () => { + mockStop.mockRejectedValueOnce(new Error("mock down")); + const harness = await startQaLiveLaneGateway({ + repoRoot: "/tmp/openclaw-repo", + transport: createStubTransport(), + transportBaseUrl: "http://127.0.0.1:43123", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + alternateModel: "mock-openai/gpt-5.5-alt", + controlUiEnabled: false, + }); + const stopOptions = { preserveToDir: ".artifacts/qa-e2e/debug" }; + + await expect(harness.stop(stopOptions)).rejects.toThrow("mock provider stop failed: mock down"); + await expect(harness.stop(stopOptions)).resolves.toBeUndefined(); + + expect(gatewayStop).toHaveBeenCalledTimes(1); + expect(gatewayStop).toHaveBeenCalledWith(stopOptions); + expect(mockStop).toHaveBeenCalledTimes(2); + }); + + it("retries only gateway cleanup after mock shutdown succeeds", async () => { + gatewayStop.mockRejectedValueOnce(new Error("gateway down")); + const harness = await startQaLiveLaneGateway({ + repoRoot: "/tmp/openclaw-repo", + transport: createStubTransport(), + transportBaseUrl: "http://127.0.0.1:43123", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + alternateModel: "mock-openai/gpt-5.5-alt", + controlUiEnabled: false, + }); + + await expect(harness.stop()).rejects.toThrow("gateway stop failed: gateway down"); + await expect(harness.stop()).resolves.toBeUndefined(); + + expect(gatewayStop).toHaveBeenCalledTimes(2); + expect(mockStop).toHaveBeenCalledTimes(1); + }); }); diff --git a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts index d0a6044d01e..7dfc9f11986 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts @@ -12,20 +12,24 @@ import { appendQaLiveLaneIssue as appendLiveLaneIssue } from "./live-artifacts.j async function stopQaLiveLaneResources( resources: { - gateway: Awaited>; + gateway: Awaited> | null; mock: { baseUrl: string; stop(): Promise } | null; }, opts?: { keepTemp?: boolean; preserveToDir?: string }, ) { const errors: string[] = []; - try { - await resources.gateway.stop(opts); - } catch (error) { - appendLiveLaneIssue(errors, "gateway stop failed", error); + if (resources.gateway) { + try { + await resources.gateway.stop(opts); + resources.gateway = null; + } catch (error) { + appendLiveLaneIssue(errors, "gateway stop failed", error); + } } if (resources.mock) { try { await resources.mock.stop(); + resources.mock = null; } catch (error) { appendLiveLaneIssue(errors, "mock provider stop failed", error); } @@ -122,11 +126,12 @@ export async function startQaLiveLaneGateway(params: { mutateConfig: (cfg) => prepareLiveTransportGatewayConfig(params.mutateConfig ? params.mutateConfig(cfg) : cfg), }); + const resources = { gateway, mock }; return { gateway, mock, async stop(opts?: { keepTemp?: boolean; preserveToDir?: string }) { - await stopQaLiveLaneResources({ gateway, mock }, opts); + await stopQaLiveLaneResources(resources, opts); }, }; } catch (error) { diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index 3e61fdd0bb4..f091e19d5c2 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -38,6 +38,11 @@ describe("Slack live QA runtime helpers", () => { ).toThrow("OPENCLAW_QA_SLACK channelId must be a Slack id like C123 or U123."); }); + it("canonicalizes the SUT account before config and approval routing", () => { + expect(testing.resolveSlackQaSutAccountId(" QA-SUT ")).toBe("qa-sut"); + expect(testing.resolveSlackQaSutAccountId()).toBe("sut"); + }); + it("parses Convex credential payloads", () => { expect( testing.parseSlackQaCredentialPayload({ @@ -75,10 +80,47 @@ describe("Slack live QA runtime helpers", () => { it("selects native approval scenarios by id without changing standard scenario coverage", () => { expect( testing - .findScenario(["slack-approval-exec-native", "slack-approval-plugin-native"]) + .findScenario([ + "slack-approval-exec-native", + "slack-approval-plugin-native", + "slack-codex-approval-exec-native", + "slack-codex-approval-plugin-native", + ]) .map((scenario) => scenario.id), - ).toEqual(["slack-approval-exec-native", "slack-approval-plugin-native"]); + ).toEqual([ + "slack-approval-exec-native", + "slack-approval-plugin-native", + "slack-codex-approval-exec-native", + "slack-codex-approval-plugin-native", + ]); expect(testing.SLACK_QA_STANDARD_SCENARIO_IDS).not.toContain("slack-approval-exec-native"); + expect(testing.SLACK_QA_STANDARD_SCENARIO_IDS).not.toContain( + "slack-codex-approval-exec-native", + ); + }); + + it("accepts only Codex harness providers for Codex approval scenarios", () => { + expect(() => testing.assertSlackCodexApprovalModelSupported("openai/gpt-5.5")).not.toThrow(); + expect(() => testing.assertSlackCodexApprovalModelSupported("codex/gpt-5.5")).not.toThrow(); + expect(() => + testing.assertSlackCodexApprovalModelSupported("anthropic/claude-sonnet-4-6"), + ).toThrow( + 'Slack Codex approval scenarios require an openai/* or codex/* model; received "anthropic/claude-sonnet-4-6".', + ); + }); + + it("rejects an incompatible Codex approval model before credential acquisition", async () => { + const outputDir = await fs.mkdtemp(path.join(tmpdir(), "openclaw-slack-codex-model-")); + await expect( + runSlackQaLive({ + credentialSource: "convex", + outputDir, + primaryModel: "anthropic/claude-sonnet-4-6", + scenarioIds: ["slack-codex-approval-exec-native"], + }), + ).rejects.toThrow( + 'Slack Codex approval scenarios require an openai/* or codex/* model; received "anthropic/claude-sonnet-4-6".', + ); }); it("enables Slack native exec and plugin approval delivery for approval scenarios", () => { @@ -112,6 +154,58 @@ describe("Slack live QA runtime helpers", () => { expect(account?.channels?.C123456789?.users).toEqual(["U999999999"]); }); + it("enables Codex guardian runtime and native plugin approval delivery for Codex approval scenarios", () => { + const cfg = testing.buildSlackQaConfig( + { + agents: { + defaults: {}, + list: [ + { + id: "qa", + model: { primary: "openai/gpt-5.5" }, + }, + ], + }, + }, + { + channelId: "C123456789", + driverBotUserId: "U999999999", + overrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + codexApproval: true, + }, + primaryModel: "openai/gpt-5.5", + sutAccountId: "sut", + sutAppToken: "xapp-sut", + sutBotToken: "xoxb-sut", + }, + ); + + expect(cfg.plugins?.allow).toEqual(["slack", "codex"]); + expect(cfg.plugins?.entries?.codex).toEqual({ + enabled: true, + config: { + appServer: { + mode: "guardian", + }, + }, + }); + expect(cfg.tools?.exec?.mode).toBe("ask"); + expect(cfg.agents?.defaults?.models?.["openai/gpt-5.5"]?.agentRuntime).toEqual({ + id: "codex", + }); + expect(cfg.approvals?.plugin).toEqual({ enabled: true, mode: "session" }); + expect(cfg.channels?.slack?.accounts?.sut?.execApprovals).toEqual({ + enabled: true, + approvers: ["U999999999"], + target: "channel", + }); + }); + it("overrides both owner and channel allowlists for block scenarios", () => { const cfg = testing.buildSlackQaConfig( {}, @@ -150,6 +244,230 @@ describe("Slack live QA runtime helpers", () => { ).toEqual(["/approve plugin:abc allow-once"]); }); + it("extracts plugin approval ids from native Slack approval action values", () => { + expect( + testing.extractSlackNativeApprovalId({ + actionValues: ["/approve plugin:abc123 allow-once", "/approve plugin:abc123 deny"], + decision: "allow-once", + }), + ).toBe("plugin:abc123"); + }); + + it("builds Codex approval instructions for command and file-change routes", () => { + expect( + testing.buildCodexApprovalInstruction({ + appServerMethod: "item/commandExecution/requestApproval", + token: "SLACK_QA_CODEX_EXEC_APPROVAL_ABC123", + }), + ).toContain("Use the shell tool exactly once"); + expect( + testing.buildCodexApprovalInstruction({ + appServerMethod: "item/fileChange/requestApproval", + token: "SLACK_QA_CODEX_FILE_APPROVAL_ABC123", + }), + ).toContain("Do not ask for approval in chat"); + expect(testing.resolveCodexFileApprovalTargetPath("MARKER")).toMatch( + /\.openclaw-qa-codex-file-approval-marker\.txt$/u, + ); + }); + + it("reads the accepted asynchronous Gateway agent run id", () => { + expect( + testing.readAcceptedAgentRunId({ + runId: "run-123", + status: "accepted", + }), + ).toBe("run-123"); + expect(() => + testing.readAcceptedAgentRunId({ + runId: "run-123", + status: "started", + }), + ).toThrow("instead of accepted"); + }); + + it("requires the Codex command transcript to prove the approved operation", () => { + const run = { + approvalKind: "plugin" as const, + appServerMethod: "item/commandExecution/requestApproval" as const, + decision: "allow-once" as const, + kind: "codex-approval" as const, + token: "SLACK_QA_CODEX_EXEC_APPROVAL_ABC123", + }; + expect(() => + testing.assertCodexApprovalTranscriptSucceeded( + [ + { + role: "toolResult", + isError: false, + content: [ + { + type: "toolResult", + content: "SLACK_QA_CODEX_EXEC_APPROVAL_ABC123", + }, + ], + }, + { + role: "assistant", + content: [{ type: "text", text: "SLACK_QA_CODEX_EXEC_APPROVAL_ABC123" }], + }, + ], + run, + ), + ).not.toThrow(); + expect(() => + testing.assertCodexApprovalTranscriptSucceeded( + [ + { + role: "assistant", + content: [{ type: "text", text: "SLACK_QA_CODEX_EXEC_APPROVAL_ABC123" }], + }, + ], + run, + ), + ).toThrow("Codex command result did not contain marker"); + }); + + it("aborts, awaits terminal cleanup, and stops the gateway process tree before cleanup", async () => { + const call = vi + .fn() + .mockResolvedValueOnce({ aborted: true, runIds: ["run-123"] }) + .mockResolvedValueOnce({ endedAt: 123, runId: "run-123", status: "ok" }); + const stopGateway = vi.fn(); + + await testing.quiesceCodexApprovalAgentRun({ + context: { gateway: { call } } as never, + preserveDebugArtifacts: false, + runId: "run-123", + sessionKey: "agent:qa:approval", + stopGateway, + }); + + expect(call).toHaveBeenNthCalledWith( + 1, + "chat.abort", + { runId: "run-123", sessionKey: "agent:qa:approval" }, + { timeoutMs: 10_000 }, + ); + expect(call).toHaveBeenNthCalledWith( + 2, + "agent.wait", + { runId: "run-123", timeoutMs: 10_000 }, + { timeoutMs: 15_000 }, + ); + expect(stopGateway).toHaveBeenCalledWith(false); + }); + + it("preserves debug artifacts when abort and terminal acknowledgements fail", async () => { + const call = vi.fn().mockRejectedValue(new Error("gateway unavailable")); + const stopGateway = vi.fn(); + + await testing.quiesceCodexApprovalAgentRun({ + context: { gateway: { call } } as never, + preserveDebugArtifacts: true, + runId: "run-123", + sessionKey: "agent:qa:approval", + stopGateway, + }); + + expect(stopGateway).toHaveBeenCalledWith(true); + }); + + it("matches pending Codex plugin approvals by id, route, and Slack turn source", () => { + expect( + testing.findPendingCodexPluginApprovalRecord({ + approvalId: "plugin:abc123", + appServerMethod: "item/fileChange/requestApproval", + channelId: "C123456789", + records: [ + { + id: "plugin:abc123", + request: { + pluginId: "openclaw-codex-app-server", + title: "Codex app-server file approval", + toolName: "codex_file_approval", + sessionKey: "agent:qa:slack-codex-approval-plugin-native-token", + turnSourceChannel: "slack", + turnSourceTo: "channel:C123456789", + turnSourceAccountId: "sut", + }, + }, + ], + sessionKey: "agent:qa:slack-codex-approval-plugin-native-token", + sutAccountId: "sut", + }), + ).toBeDefined(); + expect( + testing.findPendingCodexPluginApprovalRecord({ + approvalId: "plugin:abc123", + appServerMethod: "item/commandExecution/requestApproval", + channelId: "C123456789", + records: [ + { + id: "plugin:abc123", + request: { + pluginId: "openclaw-codex-app-server", + title: "Codex app-server file approval", + toolName: "codex_file_approval", + sessionKey: "agent:qa:slack-codex-approval-plugin-native-token", + turnSourceChannel: "slack", + turnSourceTo: "channel:C123456789", + turnSourceAccountId: "sut", + }, + }, + ], + sessionKey: "agent:qa:slack-codex-approval-plugin-native-token", + sutAccountId: "sut", + }), + ).toBeUndefined(); + }); + + it("matches resolved Codex approvals without pending-only marker text", () => { + expect( + testing.matchesSlackApprovalResolvedUpdate({ + actionValues: [], + approvalKind: "plugin", + decision: "allow-once", + extraTextMatches: ["openclaw-codex-app-server", "Codex app-server file approval"], + text: [ + "Plugin approval: Allowed once", + "Codex app-server file approval", + "Plugin: openclaw-codex-app-server", + ].join("\n"), + }), + ).toBe(true); + expect( + testing.matchesSlackApprovalResolvedUpdate({ + actionValues: ["/approve plugin:abc allow-once"], + approvalKind: "plugin", + decision: "allow-once", + extraTextMatches: ["Codex app-server file approval"], + text: "Plugin approval: Allowed once\nCodex app-server file approval", + }), + ).toBe(false); + }); + + it("matches pending Codex approvals by stable renderer fields without marker text", () => { + expect( + testing.matchesSlackApprovalPromptText({ + approvalKind: "plugin", + extraTextMatches: ["openclaw-codex-app-server", "Codex app-server command approval"], + text: [ + "Plugin approval required", + "Codex app-server command approval", + "Plugin: openclaw-codex-app-server", + ].join("\n"), + }), + ).toBe(true); + expect( + testing.matchesSlackApprovalPromptText({ + approvalKind: "plugin", + extraTextMatches: ["Codex app-server file approval"], + text: "Plugin approval required\nCodex app-server command approval", + }), + ).toBe(false); + }); + it("builds approval checkpoint message evidence from Slack blocks", () => { expect( testing.buildSlackApprovalCheckpointMessage({ @@ -340,8 +658,12 @@ describe("Slack live QA runtime helpers", () => { ).toEqual({ approvalId: "", approvalKind: "plugin", + appServerMethod: undefined, channelId: undefined, + codexModelKey: undefined, decision: "allow-once", + finalCodexTurnStatus: undefined, + operationVerified: undefined, pendingActionValues: undefined, pendingCheckpointPath: undefined, pendingMessageTs: undefined, @@ -356,6 +678,54 @@ describe("Slack live QA runtime helpers", () => { }); }); + it("keeps Codex approval route metadata while redacting Slack metadata", () => { + expect( + testing.toSlackQaScenarioArtifactResults({ + includeContent: false, + redactMetadata: true, + scenarios: [ + { + approval: { + approvalId: "plugin:abc", + approvalKind: "plugin", + appServerMethod: "item/fileChange/requestApproval", + channelId: "C123456789", + codexModelKey: "openai/gpt-5.5", + decision: "allow-once", + finalCodexTurnStatus: "ok", + operationVerified: true, + pendingActionValues: ["/approve plugin:abc allow-once"], + pendingMessageTs: "1.000000", + pendingText: "Plugin approval required", + resolvedActionValues: [], + resolvedMessageTs: "1.000000", + resolvedText: "Plugin approval: Allowed once", + threadTs: "1.000000", + }, + details: "codex plugin approval resolved", + id: "slack-codex-approval-plugin-native", + status: "pass", + title: "Slack native Codex file approval prompt resolves", + }, + ], + })[0]?.approval, + ).toMatchObject({ + approvalId: "", + appServerMethod: "item/fileChange/requestApproval", + channelId: undefined, + codexModelKey: "openai/gpt-5.5", + finalCodexTurnStatus: "ok", + operationVerified: true, + pendingActionValues: undefined, + pendingMessageTs: undefined, + pendingText: undefined, + resolvedActionValues: undefined, + resolvedMessageTs: undefined, + resolvedText: undefined, + threadTs: undefined, + }); + }); + it("ignores delayed unrelated SUT replies during mention-gating", async () => { const observedMessages: Array = []; await expect( diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts index c23bbd2b1aa..e23dc1c05ba 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts @@ -1,9 +1,11 @@ // Qa Lab plugin module implements slack live behavior. import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { createSlackWebClient, createSlackWriteClient } from "@openclaw/slack/api.js"; import type { WebClient } from "@slack/web-api"; +import { normalizeAccountId } from "openclaw/plugin-sdk/account-id"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime"; @@ -12,7 +14,9 @@ import { z } from "zod"; import { createQaArtifactRunId } from "../../artifact-run-id.js"; import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js"; import { startQaGatewayChild } from "../../gateway-child.js"; +import { extractGatewayMessageText } from "../../gateway-log-sentinel.js"; import { isTruthyOptIn } from "../../mantis-options.runtime.js"; +import { splitQaModelRef } from "../../model-selection.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js"; import { defaultQaModelForMode, @@ -64,12 +68,16 @@ const SLACK_QA_GATEWAY_STOP_SETTLE_MS = 3_000; const SLACK_QA_RETRYABLE_SCENARIO_ATTEMPTS = 2; const SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS = 30_000; const SLACK_QA_APPROVAL_CHECKPOINT_DEFAULT_TIMEOUT_MS = 120_000; +// These scenarios force the Codex harness, whose default provider set is intentionally narrow. +const SLACK_QA_CODEX_PROVIDER_IDS = new Set(["codex", "openai"]); type SlackQaScenarioId = | "slack-allowlist-block" | "slack-approval-exec-native" | "slack-approval-plugin-native" | "slack-canary" + | "slack-codex-approval-exec-native" + | "slack-codex-approval-plugin-native" | "slack-mention-gating" | "slack-restart-resume" | "slack-thread-follow-up" @@ -78,6 +86,23 @@ type SlackQaScenarioId = type SlackQaApprovalKind = "exec" | "plugin"; type SlackQaApprovalDecision = "allow-always" | "allow-once" | "deny"; +type SlackQaCodexApprovalMethod = + | "item/commandExecution/requestApproval" + | "item/fileChange/requestApproval"; + +function assertSlackCodexApprovalModelSupported(modelRef: string) { + const provider = splitQaModelRef(modelRef)?.provider.trim().toLowerCase(); + if (provider && SLACK_QA_CODEX_PROVIDER_IDS.has(provider)) { + return; + } + throw new Error( + `Slack Codex approval scenarios require an openai/* or codex/* model; received "${modelRef}".`, + ); +} + +function resolveSlackQaSutAccountId(value?: string) { + return normalizeAccountId(value?.trim() || "sut"); +} type SlackQaMessageScenarioRun = { kind?: "message"; @@ -96,7 +121,18 @@ type SlackQaApprovalScenarioRun = { token: string; }; -type SlackQaScenarioRun = SlackQaApprovalScenarioRun | SlackQaMessageScenarioRun; +type SlackQaCodexApprovalScenarioRun = { + approvalKind: "plugin"; + appServerMethod: SlackQaCodexApprovalMethod; + decision: "allow-once"; + kind: "codex-approval"; + token: string; +}; + +type SlackQaScenarioRun = + | SlackQaApprovalScenarioRun + | SlackQaCodexApprovalScenarioRun + | SlackQaMessageScenarioRun; type SlackQaBeforeRunResult = | string @@ -113,6 +149,7 @@ type SlackQaConfigOverrides = { plugin?: boolean; target?: "both" | "channel" | "dm"; }; + codexApproval?: boolean; replyToMode?: "all" | "off"; users?: string[]; }; @@ -181,8 +218,12 @@ type SlackObservedMessageArtifact = { type SlackApprovalArtifact = { approvalId: string; approvalKind: SlackQaApprovalKind; + appServerMethod?: SlackQaCodexApprovalMethod; channelId?: string; + codexModelKey?: string; decision: SlackQaApprovalDecision; + finalCodexTurnStatus?: string; + operationVerified?: boolean; pendingActionValues?: string[]; pendingCheckpointPath?: string; pendingMessageTs?: string; @@ -396,6 +437,46 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [ token: `SLACK_QA_PLUGIN_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, }), }, + { + id: "slack-codex-approval-exec-native", + title: "Slack native Codex command approval prompt resolves", + timeoutMs: 180_000, + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + codexApproval: true, + }, + buildRun: () => ({ + approvalKind: "plugin", + appServerMethod: "item/commandExecution/requestApproval", + decision: "allow-once", + kind: "codex-approval", + token: `SLACK_QA_CODEX_EXEC_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, + { + id: "slack-codex-approval-plugin-native", + title: "Slack native Codex file approval prompt resolves", + timeoutMs: 180_000, + configOverrides: { + approvals: { + exec: true, + plugin: true, + target: "channel", + }, + codexApproval: true, + }, + buildRun: () => ({ + approvalKind: "plugin", + appServerMethod: "item/fileChange/requestApproval", + decision: "allow-once", + kind: "codex-approval", + token: `SLACK_QA_CODEX_FILE_APPROVAL_${randomUUID().slice(0, 8).toUpperCase()}`, + }), + }, { id: "slack-restart-resume", standardId: "restart-resume", @@ -558,19 +639,35 @@ function findScenario(ids?: string[]) { }); } +function asPlainRecord(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + function buildSlackQaConfig( baseCfg: OpenClawConfig, params: { channelId: string; driverBotUserId: string; overrides?: SlackQaConfigOverrides; + primaryModel?: string; sutAccountId: string; sutAppToken: string; sutBotToken: string; }, ): OpenClawConfig { - const pluginAllow = uniqueStrings([...(baseCfg.plugins?.allow ?? []), "slack"]); + const codexApprovalConfig = params.overrides?.codexApproval === true; + const primaryModel = params.primaryModel; + const pluginAllow = uniqueStrings([ + ...(baseCfg.plugins?.allow ?? []), + "slack", + ...(codexApprovalConfig ? ["codex"] : []), + ]); const approvalOverrides = params.overrides?.approvals; + const codexEntry = baseCfg.plugins?.entries?.codex; + const codexEntryConfig = asPlainRecord(codexEntry?.config); + const codexAppServerConfig = asPlainRecord(codexEntryConfig.appServer); const approvalForwardingConfig = approvalOverrides?.exec || approvalOverrides?.plugin ? { @@ -597,6 +694,19 @@ function buildSlackQaConfig( }, } : {}; + const codexAgentDefaults = + codexApprovalConfig && primaryModel + ? { + ...baseCfg.agents?.defaults, + models: { + ...baseCfg.agents?.defaults?.models, + [primaryModel]: { + ...baseCfg.agents?.defaults?.models?.[primaryModel], + agentRuntime: { id: "codex" as const }, + }, + }, + } + : baseCfg.agents?.defaults; const execApprovalsConfig = approvalOverrides ? { enabled: true, @@ -613,8 +723,38 @@ function buildSlackQaConfig( entries: { ...baseCfg.plugins?.entries, slack: { enabled: true }, + ...(codexApprovalConfig + ? { + codex: { + ...codexEntry, + enabled: true, + config: { + ...codexEntryConfig, + appServer: { + ...codexAppServerConfig, + mode: "guardian" as const, + }, + }, + }, + } + : {}), }, }, + ...(codexApprovalConfig + ? { + agents: { + ...baseCfg.agents, + ...(codexAgentDefaults ? { defaults: codexAgentDefaults } : {}), + }, + tools: { + ...baseCfg.tools, + exec: { + ...baseCfg.tools?.exec, + mode: "ask" as const, + }, + }, + } + : {}), messages: { ...baseCfg.messages, groupChat: { @@ -796,17 +936,33 @@ function buildSlackApprovalCheckpointMessage( function hasSlackNativeApprovalActions(params: { actionValues: string[]; - approvalId: string; + approvalId?: string; decision: SlackQaApprovalDecision; }) { return params.actionValues.some( (value) => value.includes("/approve") && - value.includes(params.approvalId) && + (!params.approvalId || value.includes(params.approvalId)) && value.includes(params.decision), ); } +function extractSlackNativeApprovalId(params: { + actionValues: string[]; + decision: SlackQaApprovalDecision; +}) { + for (const value of params.actionValues) { + if (!value.includes("/approve") || !value.includes(params.decision)) { + continue; + } + const match = value.match(/\b((?:exec|plugin):[^\s]+)/); + if (match?.[1]) { + return match[1]; + } + } + return undefined; +} + function isSutSlackMessage(message: SlackMessage, sutIdentity: SlackAuthIdentity) { return ( (message.user !== undefined && message.user === sutIdentity.userId) || @@ -1012,7 +1168,7 @@ function pushObservedApprovalMessage(params: { } async function waitForSlackApprovalPrompt(params: { - approvalId: string; + approvalId?: string; approvalKind: SlackQaApprovalKind; channelId: string; client: WebClient; @@ -1023,7 +1179,8 @@ async function waitForSlackApprovalPrompt(params: { scenarioTitle: string; sutIdentity: SlackAuthIdentity; timeoutMs: number; - token: string; + token?: string; + extraTextMatches?: string[]; }) { const startedAt = Date.now(); const seenObservedMessages = new Set(); @@ -1040,17 +1197,19 @@ async function waitForSlackApprovalPrompt(params: { } const text = getSlackMessageSearchText(message); const actionValues = collectSlackActionValues(message.blocks); - const hasHeading = text.includes( - resolveApprovalHeading({ approvalKind: params.approvalKind, state: "pending" }), - ); - const hasToken = text.includes(params.token); + const matchedScenario = matchesSlackApprovalPromptText({ + approvalKind: params.approvalKind, + extraTextMatches: params.extraTextMatches, + text, + token: params.token, + }); const observedKey = `${message.ts}:${message.text ?? ""}:${actionValues.join("|")}`; - if (hasHeading || hasToken || hasSlackNativeApprovalActions({ ...params, actionValues })) { + if (matchedScenario || hasSlackNativeApprovalActions({ ...params, actionValues })) { if (!seenObservedMessages.has(observedKey)) { seenObservedMessages.add(observedKey); pushObservedApprovalMessage({ channelId: params.channelId, - matchedScenario: hasHeading && hasToken, + matchedScenario, message, observedMessages: params.observedMessages, scenarioId: params.scenarioId, @@ -1058,7 +1217,7 @@ async function waitForSlackApprovalPrompt(params: { }); } } - if (!hasHeading || !hasToken) { + if (!matchedScenario) { continue; } if ( @@ -1073,6 +1232,12 @@ async function waitForSlackApprovalPrompt(params: { } return { actionValues, + approvalId: + params.approvalId ?? + extractSlackNativeApprovalId({ + actionValues, + decision: params.decision, + }), message, observedAt: new Date().toISOString(), }; @@ -1091,6 +1256,21 @@ async function waitForSlackApprovalPrompt(params: { ); } +function matchesSlackApprovalPromptText(params: { + approvalKind: SlackQaApprovalKind; + extraTextMatches?: string[]; + text: string; + token?: string; +}) { + return ( + params.text.includes( + resolveApprovalHeading({ approvalKind: params.approvalKind, state: "pending" }), + ) && + (!params.token || params.text.includes(params.token)) && + (params.extraTextMatches ?? []).every((match) => params.text.includes(match)) + ); +} + async function waitForSlackApprovalResolvedUpdate(params: { approvalKind: SlackQaApprovalKind; channelId: string; @@ -1103,7 +1283,8 @@ async function waitForSlackApprovalResolvedUpdate(params: { scenarioTitle: string; sutIdentity: SlackAuthIdentity; timeoutMs: number; - token: string; + token?: string; + extraTextMatches?: string[]; }) { const startedAt = Date.now(); const seenObservedMessages = new Set(); @@ -1117,29 +1298,27 @@ async function waitForSlackApprovalResolvedUpdate(params: { if (message && isSutSlackMessage(message, params.sutIdentity)) { const text = getSlackMessageSearchText(message); const actionValues = collectSlackActionValues(message.blocks); + const matchedScenario = matchesSlackApprovalResolvedUpdate({ + actionValues, + approvalKind: params.approvalKind, + decision: params.decision, + extraTextMatches: params.extraTextMatches, + text, + token: params.token, + }); const observedKey = `${message.ts}:${message.text ?? ""}:${actionValues.join("|")}`; if (!seenObservedMessages.has(observedKey)) { seenObservedMessages.add(observedKey); pushObservedApprovalMessage({ channelId: params.channelId, - matchedScenario: text.includes(params.token), + matchedScenario, message, observedMessages: params.observedMessages, scenarioId: params.scenarioId, scenarioTitle: params.scenarioTitle, }); } - if ( - text.includes( - resolveApprovalHeading({ - approvalKind: params.approvalKind, - decision: params.decision, - state: "resolved", - }), - ) && - text.includes(params.token) && - !actionValues.some((value) => value.includes("/approve")) - ) { + if (matchedScenario) { return { actionValues, message, @@ -1156,6 +1335,28 @@ async function waitForSlackApprovalResolvedUpdate(params: { ); } +function matchesSlackApprovalResolvedUpdate(params: { + actionValues: string[]; + approvalKind: SlackQaApprovalKind; + decision: SlackQaApprovalDecision; + extraTextMatches?: string[]; + text: string; + token?: string; +}) { + return ( + params.text.includes( + resolveApprovalHeading({ + approvalKind: params.approvalKind, + decision: params.decision, + state: "resolved", + }), + ) && + (!params.token || params.text.includes(params.token)) && + (params.extraTextMatches ?? []).every((match) => params.text.includes(match)) && + !params.actionValues.some((value) => value.includes("/approve")) + ); +} + function resolveSlackApprovalCheckpointConfig(env: NodeJS.ProcessEnv = process.env) { const checkpointDir = env[SLACK_QA_APPROVAL_CHECKPOINT_DIR_ENV]?.trim(); if (!checkpointDir) { @@ -1458,6 +1659,470 @@ async function runSlackApprovalScenario(params: { }; } +function buildCodexApprovalInstruction(params: { + appServerMethod: SlackQaCodexApprovalMethod; + token: string; +}) { + if (params.appServerMethod === "item/commandExecution/requestApproval") { + return [ + "Slack QA Codex command approval scenario.", + `Marker: ${params.token}`, + "Use the shell tool exactly once and request approval before running it.", + `Run exactly: printf '${params.token}\\n'`, + `After the command succeeds, reply exactly ${params.token} and nothing else.`, + ].join("\n"); + } + const targetPath = resolveCodexFileApprovalTargetPath(params.token); + return [ + "Slack QA Codex file approval scenario.", + `Marker: ${params.token}`, + "Use apply_patch exactly once to create this file outside the workspace.", + `Target file: ${targetPath}`, + `File contents: ${params.token}`, + "Invoke apply_patch now. Do not ask for approval in chat; the harness will resolve the native tool approval.", + `After the file change succeeds, reply exactly ${params.token} and nothing else.`, + ].join("\n"); +} + +function readAcceptedAgentRunId(result: unknown) { + const started = + typeof result === "object" && result !== null + ? (result as { runId?: unknown; status?: unknown }) + : null; + if (started?.status !== "accepted") { + throw new Error( + `Codex agent run status was ${formatApprovalResultValue(started?.status)} instead of accepted`, + ); + } + if (typeof started.runId !== "string" || started.runId.trim().length === 0) { + throw new Error(`Codex agent run id was ${formatApprovalResultValue(started.runId)}`); + } + return started.runId; +} + +function readAgentWaitStatus(result: unknown) { + if (typeof result !== "object" || result === null) { + return "unknown"; + } + const status = (result as { status?: unknown }).status; + return typeof status === "string" && status.trim() ? status : "unknown"; +} + +function assertCodexApprovalTranscriptSucceeded( + messages: unknown, + run: SlackQaCodexApprovalScenarioRun, +) { + const records = Array.isArray(messages) ? messages.map(asPlainRecord) : []; + const assistantReply = records + .toReversed() + .find((message) => message.role === "assistant" && extractGatewayMessageText(message)); + if (!assistantReply || extractGatewayMessageText(assistantReply) !== run.token) { + throw new Error(`Codex approval run did not finish with assistant marker ${run.token}`); + } + if (run.appServerMethod !== "item/commandExecution/requestApproval") { + return; + } + const commandSucceeded = records.some((message) => { + if (message.role !== "toolResult" || message.isError === true) { + return false; + } + return extractGatewayMessageText(message) + .split(/\r?\n/u) + .some((line) => line.trim() === run.token); + }); + if (!commandSucceeded) { + throw new Error(`Codex command result did not contain marker ${run.token}`); + } +} + +async function assertCodexApprovalOperationSucceeded(params: { + context: Omit; + run: SlackQaCodexApprovalScenarioRun; + sessionKey: string; +}) { + const history = asPlainRecord( + await params.context.gateway.call( + "chat.history", + { sessionKey: params.sessionKey, limit: 24 }, + { timeoutMs: 10_000 }, + ), + ); + assertCodexApprovalTranscriptSucceeded(history.messages, params.run); + if (params.run.appServerMethod !== "item/fileChange/requestApproval") { + return; + } + const targetPath = resolveCodexFileApprovalTargetPath(params.run.token); + const contents = await fs.readFile(targetPath, "utf8"); + if (contents.trim() !== params.run.token) { + throw new Error(`Codex file result at ${targetPath} did not contain the expected marker`); + } +} + +function findPendingCodexPluginApprovalRecord(params: { + approvalId: string; + appServerMethod: SlackQaCodexApprovalMethod; + channelId: string; + records: unknown; + sessionKey: string; + sutAccountId: string; +}) { + const list = Array.isArray(params.records) ? params.records : []; + const expectedTitle = + params.appServerMethod === "item/commandExecution/requestApproval" + ? "Codex app-server command approval" + : "Codex app-server file approval"; + const expectedToolName = + params.appServerMethod === "item/commandExecution/requestApproval" + ? "codex_command_approval" + : "codex_file_approval"; + for (const entry of list) { + const record = asPlainRecord(entry); + if (record.id !== params.approvalId) { + continue; + } + const request = asPlainRecord(record.request); + if ( + request.pluginId === "openclaw-codex-app-server" && + request.title === expectedTitle && + request.toolName === expectedToolName && + request.sessionKey === params.sessionKey && + request.turnSourceChannel === "slack" && + request.turnSourceTo === `channel:${params.channelId}` && + request.turnSourceAccountId === params.sutAccountId + ) { + return record; + } + } + return undefined; +} + +async function assertPendingCodexPluginApproval(params: { + approvalId: string; + appServerMethod: SlackQaCodexApprovalMethod; + channelId: string; + context: Omit; + sessionKey: string; + sutAccountId: string; +}) { + const records = await params.context.gateway.call( + "plugin.approval.list", + {}, + { + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS, + }, + ); + const record = findPendingCodexPluginApprovalRecord({ + approvalId: params.approvalId, + appServerMethod: params.appServerMethod, + channelId: params.channelId, + records, + sessionKey: params.sessionKey, + sutAccountId: params.sutAccountId, + }); + if (!record) { + throw new Error( + `Pending Codex plugin approval ${params.approvalId} did not match the expected app-server route and Slack turn source.`, + ); + } +} + +async function startCodexApprovalAgentRun(params: { + channelId: string; + context: Omit; + primaryModel: string; + run: SlackQaCodexApprovalScenarioRun; + runId: string; + scenario: SlackQaScenarioDefinition; + sessionKey: string; + sutAccountId: string; +}) { + const result = await params.context.gateway.call( + "agent", + { + accountId: params.sutAccountId, + agentId: "qa", + channel: "slack", + cleanupBundleMcpOnRunEnd: true, + deliver: false, + idempotencyKey: params.runId, + message: buildCodexApprovalInstruction({ + appServerMethod: params.run.appServerMethod, + token: params.run.token, + }), + model: params.primaryModel, + sessionKey: params.sessionKey, + thinking: "low", + timeout: Math.ceil(params.scenario.timeoutMs / 1_000), + to: `channel:${params.channelId}`, + }, + { + timeoutMs: SLACK_QA_APPROVAL_DECISION_TIMEOUT_MS + 5_000, + }, + ); + const acceptedRunId = readAcceptedAgentRunId(result); + if (acceptedRunId !== params.runId) { + throw new Error(`Codex agent run id was ${acceptedRunId} instead of ${params.runId}`); + } +} + +function buildCodexApprovalSessionKey(params: { + scenario: SlackQaScenarioDefinition; + token: string; +}) { + return `agent:qa:${params.scenario.id}-${params.token.toLowerCase()}`; +} + +async function waitForCodexApprovalAgentRun(params: { + context: Omit; + runId: string; + timeoutMs: number; +}) { + const result = await params.context.gateway.call( + "agent.wait", + { + runId: params.runId, + timeoutMs: params.timeoutMs, + }, + { + timeoutMs: params.timeoutMs + 5_000, + }, + ); + return readAgentWaitStatus(result); +} + +async function quiesceCodexApprovalAgentRun(params: { + context: Omit; + preserveDebugArtifacts: boolean; + runId: string; + sessionKey: string; + stopGateway: (preserveDebugArtifacts: boolean) => Promise; +}) { + try { + await params.context.gateway.call( + "chat.abort", + { runId: params.runId, sessionKey: params.sessionKey }, + { timeoutMs: 10_000 }, + ); + } catch { + // The bounded terminal wait and gateway process-group teardown do not depend on this ack. + } + try { + await params.context.gateway.call( + "agent.wait", + { runId: params.runId, timeoutMs: 10_000 }, + { timeoutMs: 15_000 }, + ); + } catch { + // QA-owned Codex app-server processes inherit the gateway cleanup process group. + } + await params.stopGateway(params.preserveDebugArtifacts); +} + +async function runSlackCodexApprovalScenario(params: { + channelId: string; + context: Omit; + observedMessages: SlackObservedMessage[]; + primaryModel: string; + run: SlackQaCodexApprovalScenarioRun; + scenario: SlackQaScenarioDefinition; + stopGateway: (preserveDebugArtifacts: boolean) => Promise; + sutAccountId: string; +}) { + const codexRun = { + runId: `slack-qa-codex-approval-${randomUUID()}`, + sessionKey: buildCodexApprovalSessionKey({ + scenario: params.scenario, + token: params.run.token, + }), + }; + const targetPath = + params.run.appServerMethod === "item/fileChange/requestApproval" + ? resolveCodexFileApprovalTargetPath(params.run.token) + : undefined; + if (targetPath) { + await fs.rm(targetPath, { force: true }); + } + const outcome = await runSlackCodexApprovalScenarioInner({ ...params, codexRun }).then( + (result) => ({ kind: "success", result }) as const, + (error: unknown) => ({ error, kind: "failure" }) as const, + ); + // Kill the gateway process tree before deleting the probe. Agent completion + // does not prove the native Codex turn has stopped writing after an interrupt. + const cleanupErrors: unknown[] = []; + try { + await quiesceCodexApprovalAgentRun({ + context: params.context, + preserveDebugArtifacts: outcome.kind === "failure", + stopGateway: params.stopGateway, + ...codexRun, + }); + } catch (error) { + cleanupErrors.push(error); + } + if (cleanupErrors.length === 0 && targetPath) { + try { + await fs.rm(targetPath, { force: true }); + } catch (error) { + cleanupErrors.push(error); + } + } + if (cleanupErrors.length > 0) { + const cleanupSummary = cleanupErrors.map(formatErrorMessage).join("; "); + if (outcome.kind === "failure") { + throw new AggregateError( + [outcome.error, ...cleanupErrors], + `Codex approval scenario failed: ${formatErrorMessage(outcome.error)}; cleanup also failed: ${cleanupSummary}`, + { cause: outcome.error }, + ); + } + throw new AggregateError(cleanupErrors, `Codex approval cleanup failed: ${cleanupSummary}`); + } + if (outcome.kind === "failure") { + throw outcome.error; + } + return outcome.result; +} + +function resolveCodexFileApprovalTargetPath(token: string) { + return path.join(os.homedir(), `.openclaw-qa-codex-file-approval-${token.toLowerCase()}.txt`); +} + +async function runSlackCodexApprovalScenarioInner(params: { + channelId: string; + codexRun: { runId: string; sessionKey: string }; + context: Omit; + observedMessages: SlackObservedMessage[]; + primaryModel: string; + run: SlackQaCodexApprovalScenarioRun; + scenario: SlackQaScenarioDefinition; + sutAccountId: string; +}) { + const requestStartedAt = new Date(); + const oldestTs = ((requestStartedAt.getTime() - 5_000) / 1_000).toFixed(6); + await startCodexApprovalAgentRun({ + channelId: params.channelId, + context: params.context, + primaryModel: params.primaryModel, + run: params.run, + runId: params.codexRun.runId, + scenario: params.scenario, + sessionKey: params.codexRun.sessionKey, + sutAccountId: params.sutAccountId, + }); + const expectedTitle = + params.run.appServerMethod === "item/commandExecution/requestApproval" + ? "Codex app-server command approval" + : "Codex app-server file approval"; + const pending = await waitForSlackApprovalPrompt({ + approvalKind: params.run.approvalKind, + channelId: params.channelId, + client: params.context.sutReadClient, + decision: params.run.decision, + extraTextMatches: ["openclaw-codex-app-server", expectedTitle], + observedMessages: params.observedMessages, + oldestTs, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + sutIdentity: params.context.sutIdentity, + timeoutMs: params.scenario.timeoutMs, + }); + const approvalId = pending.approvalId; + if (!approvalId) { + throw new Error( + "Codex Slack approval prompt exposed native actions but no plugin approval id.", + ); + } + await assertPendingCodexPluginApproval({ + approvalId, + appServerMethod: params.run.appServerMethod, + channelId: params.channelId, + context: params.context, + sessionKey: params.codexRun.sessionKey, + sutAccountId: params.sutAccountId, + }); + const pendingCheckpoint = await writeSlackApprovalCheckpoint({ + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + message: pending.message, + observedAt: pending.observedAt, + scenarioId: params.scenario.id, + state: "pending", + }); + await resolveApprovalDecision({ + approvalId, + context: params.context, + decision: params.run.decision, + kind: params.run.approvalKind, + }); + const finalCodexTurnStatus = await waitForCodexApprovalAgentRun({ + context: params.context, + runId: params.codexRun.runId, + timeoutMs: params.scenario.timeoutMs, + }); + if (finalCodexTurnStatus !== "ok") { + throw new Error( + `Codex approval run ${params.codexRun.runId} finished with status ${finalCodexTurnStatus}`, + ); + } + await assertCodexApprovalOperationSucceeded({ + context: params.context, + run: params.run, + sessionKey: params.codexRun.sessionKey, + }); + const resolved = await waitForSlackApprovalResolvedUpdate({ + approvalKind: params.run.approvalKind, + channelId: params.channelId, + client: params.context.sutReadClient, + decision: params.run.decision, + messageTs: pending.message.ts, + observedMessages: params.observedMessages, + oldestTs, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + sutIdentity: params.context.sutIdentity, + timeoutMs: params.scenario.timeoutMs, + extraTextMatches: ["openclaw-codex-app-server", expectedTitle], + }); + const resolvedCheckpoint = await writeSlackApprovalCheckpoint({ + approvalId, + approvalKind: params.run.approvalKind, + channelId: params.channelId, + decision: params.run.decision, + message: resolved.message, + observedAt: resolved.observedAt, + scenarioId: params.scenario.id, + state: "resolved", + }); + const responseObservedAt = new Date(resolved.observedAt); + return { + artifact: { + approvalId, + approvalKind: params.run.approvalKind, + appServerMethod: params.run.appServerMethod, + channelId: params.channelId, + codexModelKey: params.primaryModel, + decision: params.run.decision, + finalCodexTurnStatus, + operationVerified: true, + pendingActionValues: pending.actionValues, + pendingCheckpointPath: pendingCheckpoint?.checkpointPath, + pendingMessageTs: pending.message.ts, + pendingScreenshotPath: pendingCheckpoint?.screenshotPath, + pendingText: pending.message.text, + resolvedActionValues: resolved.actionValues, + resolvedCheckpointPath: resolvedCheckpoint?.checkpointPath, + resolvedMessageTs: resolved.message.ts, + resolvedScreenshotPath: resolvedCheckpoint?.screenshotPath, + resolvedText: resolved.message.text, + threadTs: pending.message.thread_ts, + } satisfies SlackApprovalArtifact, + requestStartedAt, + responseObservedAt, + rttMs: responseObservedAt.getTime() - requestStartedAt.getTime(), + }; +} + async function waitForSlackChannelRunning( gateway: Awaited>, accountId: string, @@ -1619,8 +2284,12 @@ function toSlackQaScenarioArtifactResults(params: { approval: { approvalId: params.redactMetadata ? "" : approval.approvalId, approvalKind: approval.approvalKind, + appServerMethod: approval.appServerMethod, channelId: params.redactMetadata ? undefined : approval.channelId, + codexModelKey: approval.codexModelKey, decision: approval.decision, + finalCodexTurnStatus: approval.finalCodexTurnStatus, + operationVerified: approval.operationVerified, pendingActionValues: params.includeContent ? approval.pendingActionValues : undefined, pendingCheckpointPath: approval.pendingCheckpointPath, pendingMessageTs: params.redactMetadata ? undefined : approval.pendingMessageTs, @@ -1675,6 +2344,18 @@ function renderSlackQaMarkdown(params: { } if (scenario.approval) { lines.push(`- Approval kind: ${scenario.approval.approvalKind}`); + if (scenario.approval.appServerMethod) { + lines.push(`- Codex app-server method: \`${scenario.approval.appServerMethod}\``); + } + if (scenario.approval.codexModelKey) { + lines.push(`- Codex model: \`${scenario.approval.codexModelKey}\``); + } + if (scenario.approval.finalCodexTurnStatus) { + lines.push(`- Codex turn status: ${scenario.approval.finalCodexTurnStatus}`); + } + if (scenario.approval.operationVerified) { + lines.push("- Codex operation marker: verified"); + } lines.push(`- Approval ID: \`${scenario.approval.approvalId}\``); lines.push(`- Decision: ${scenario.approval.decision}`); if (scenario.approval.pendingScreenshotPath) { @@ -1694,11 +2375,13 @@ async function preserveSlackGatewayDebugArtifacts(params: { gatewayDebugDirPath: string; gatewayHarness: SlackQaGatewayHarness; }) { - await params.gatewayHarness - .stop({ preserveToDir: params.gatewayDebugDirPath }) - .catch((error: unknown) => { - appendLiveLaneIssue(params.cleanupIssues, "gateway debug preservation failed", error); - }); + try { + await params.gatewayHarness.stop({ preserveToDir: params.gatewayDebugDirPath }); + return true; + } catch (error) { + appendLiveLaneIssue(params.cleanupIssues, "gateway debug preservation failed", error); + return false; + } } export async function runSlackQaLive(params: { @@ -1724,8 +2407,11 @@ export async function runSlackQaLive(params: { ); const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); - const sutAccountId = params.sutAccountId?.trim() || "sut"; + const sutAccountId = resolveSlackQaSutAccountId(params.sutAccountId); const scenarios = findScenario(params.scenarioIds); + if (scenarios.some((scenario) => scenario.configOverrides?.codexApproval === true)) { + assertSlackCodexApprovalModelSupported(primaryModel); + } const requestedCredentialSource = inferSlackCredentialSource(params.credentialSource); const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]); const includeObservedMessageContent = isTruthyOptIn(process.env[SLACK_QA_CAPTURE_CONTENT_ENV]); @@ -1772,6 +2458,9 @@ export async function runSlackQaLive(params: { let scenarioAttempt = 1; while (true) { let gatewayHarness: SlackQaGatewayHarness | undefined; + let codexProbeCleanupPath: string | undefined; + let preserveAttemptGatewayDebug = false; + let retryScenario = false; try { assertLeaseHealthy(); gatewayHarness = await startQaLiveLaneGateway({ @@ -1791,6 +2480,7 @@ export async function runSlackQaLive(params: { channelId: activeRuntimeEnv.channelId, driverBotUserId: driverIdentity.userId, overrides: scenario.configOverrides, + primaryModel, sutAccountId, sutAppToken: activeRuntimeEnv.sutAppToken, sutBotToken: activeRuntimeEnv.sutBotToken, @@ -1798,8 +2488,16 @@ export async function runSlackQaLive(params: { }); const activeGatewayHarness = gatewayHarness; const scenarioRun = scenario.buildRun(sutIdentity.userId); + if ( + scenarioRun.kind === "codex-approval" && + scenarioRun.appServerMethod === "item/fileChange/requestApproval" + ) { + codexProbeCleanupPath = resolveCodexFileApprovalTargetPath(scenarioRun.token); + } const readinessMode: SlackChannelReadinessMode = - scenarioRun.kind === "approval" ? "started" : "connected"; + scenarioRun.kind === "approval" || scenarioRun.kind === "codex-approval" + ? "started" + : "connected"; await waitForSlackChannelStable( activeGatewayHarness.gateway, sutAccountId, @@ -1858,6 +2556,51 @@ export async function runSlackQaLive(params: { }); break; } + if (scenarioRun.kind === "codex-approval") { + const approval = await runSlackCodexApprovalScenario({ + channelId: activeRuntimeEnv.channelId, + context: baseScenarioContext, + observedMessages, + primaryModel, + run: scenarioRun, + scenario, + stopGateway: async (preserveDebugArtifacts) => { + await activeGatewayHarness.stop( + preserveDebugArtifacts ? { preserveToDir: gatewayDebugDirPath } : undefined, + ); + await new Promise((resolve) => { + setTimeout(resolve, SLACK_QA_GATEWAY_STOP_SETTLE_MS); + }); + gatewayHarness = undefined; + if (preserveDebugArtifacts) { + preservedGatewayDebugArtifacts = true; + } + }, + sutAccountId, + }); + scenarioResults.push({ + approval: approval.artifact, + id: scenario.id, + title: scenario.title, + status: "pass", + details: [ + `Codex ${scenarioRun.appServerMethod} approval resolved ${scenarioRun.decision} in ${approval.rttMs}ms`, + scenarioAttempt > 1 ? `retried ${scenarioAttempt - 1}x` : undefined, + ] + .filter(Boolean) + .join("; "), + rttMs: approval.rttMs, + requestStartedAt: approval.requestStartedAt.toISOString(), + responseObservedAt: approval.responseObservedAt.toISOString(), + rttMeasurement: { + finalMatchedReplyRttMs: approval.rttMs, + requestStartedAt: approval.requestStartedAt.toISOString(), + responseObservedAt: approval.responseObservedAt.toISOString(), + source: "approval-request-to-resolution", + }, + }); + break; + } const beforeRunResult = await scenarioRun.beforeRun?.(baseScenarioContext); const beforeRunDetails = typeof beforeRunResult === "string" ? beforeRunResult : beforeRunResult?.details; @@ -1943,40 +2686,80 @@ export async function runSlackQaLive(params: { isRetryableSlackQaScenarioError(error) ) { scenarioAttempt += 1; - continue; - } - scenarioResults.push({ - id: scenario.id, - title: scenario.title, - standardId: scenario.standardId, - status: "fail", - details: - scenarioAttempt > 1 - ? `${formatErrorMessage(error)}; retried ${scenarioAttempt - 1}x` - : formatErrorMessage(error), - }); - preservedGatewayDebugArtifacts = true; - if (gatewayHarness) { - await preserveSlackGatewayDebugArtifacts({ - cleanupIssues, - gatewayDebugDirPath, - gatewayHarness, + retryScenario = true; + } else { + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + standardId: scenario.standardId, + status: "fail", + details: + scenarioAttempt > 1 + ? `${formatErrorMessage(error)}; retried ${scenarioAttempt - 1}x` + : formatErrorMessage(error), }); + preserveAttemptGatewayDebug = true; + preservedGatewayDebugArtifacts = true; + if (gatewayHarness) { + const stopped = await preserveSlackGatewayDebugArtifacts({ + cleanupIssues, + gatewayDebugDirPath, + gatewayHarness, + }); + if (stopped) { + gatewayHarness = undefined; + } + } } - break; } finally { - if (!preservedGatewayDebugArtifacts && gatewayHarness) { - await gatewayHarness.stop().catch((error: unknown) => { - appendLiveLaneIssue(cleanupIssues, "gateway stop failed", error); - }); - await new Promise((resolve) => { - setTimeout(resolve, SLACK_QA_GATEWAY_STOP_SETTLE_MS); + if (gatewayHarness) { + await gatewayHarness + .stop( + preserveAttemptGatewayDebug ? { preserveToDir: gatewayDebugDirPath } : undefined, + ) + .then(() => { + gatewayHarness = undefined; + if (preserveAttemptGatewayDebug) { + preservedGatewayDebugArtifacts = true; + } + }) + .catch((error: unknown) => { + appendLiveLaneIssue(cleanupIssues, "gateway stop failed", error); + retryScenario = false; + const details = `gateway stop failed: ${formatErrorMessage(error)}`; + const currentResult = scenarioResults.at(-1); + if (currentResult?.id === scenario.id) { + scenarioResults[scenarioResults.length - 1] = { + ...currentResult, + status: "fail", + details: `${currentResult.details}; ${details}`, + }; + } else { + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + standardId: scenario.standardId, + status: "fail", + details, + }); + } + }); + if (!gatewayHarness) { + await new Promise((resolve) => { + setTimeout(resolve, SLACK_QA_GATEWAY_STOP_SETTLE_MS); + }); + } + } + if (!gatewayHarness && codexProbeCleanupPath) { + await fs.rm(codexProbeCleanupPath, { force: true }).catch((error: unknown) => { + appendLiveLaneIssue(cleanupIssues, "Codex approval probe cleanup failed", error); }); } } - if (scenarioResults.at(-1)?.id === scenario.id) { - break; + if (retryScenario) { + continue; } + break; } if (scenarioResults.at(-1)?.status === "fail") { break; @@ -2081,19 +2864,30 @@ export async function runSlackQaLive(params: { } export const testing = { + assertSlackCodexApprovalModelSupported, + assertCodexApprovalTranscriptSucceeded, + buildCodexApprovalInstruction, buildSlackApprovalCheckpointMessage, buildSlackQaConfig, collectSlackActionValues, collectSlackButtonLabels, collectSlackBlockText, + extractSlackNativeApprovalId, + findPendingCodexPluginApprovalRecord, findScenario, isSlackChannelReadyForQa, + matchesSlackApprovalResolvedUpdate, + matchesSlackApprovalPromptText, parseSlackQaCredentialPayload, preserveSlackGatewayDebugArtifacts, + quiesceCodexApprovalAgentRun, + readAcceptedAgentRunId, + resolveCodexFileApprovalTargetPath, resolveSlackChannelReadySince, resolveSlackQaReadyTimeoutMs, resolveSlackApprovalCheckpointConfig, resolveApprovalDecision, + resolveSlackQaSutAccountId, resolveSlackQaRuntimeEnv, SLACK_QA_STANDARD_SCENARIO_IDS, toSlackQaScenarioArtifactResults, diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts index 048dfcd575c..5fb20b87162 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts @@ -198,6 +198,21 @@ describe("mantis Slack desktop smoke runtime", () => { expect(remoteScript).toContain("${CHROME_BIN:-}"); expect(remoteScript).toContain("PNPM_STORE_DIR"); expect(remoteScript).toContain("build-essential python3"); + expect(remoteScript).toContain("node_supports_type_stripping"); + expect(remoteScript).toContain("scripts/crabbox-untrusted-bootstrap.sh"); + expect(remoteScript).toContain("https://nodejs.org/dist/v$node_version"); + expect(remoteScript).toContain('grep " $node_archive$" SHASUMS256.txt | sha256sum -c -'); + expect(remoteScript).toContain('export PATH="$node_root/bin:$PATH"'); + expect(remoteScript).toContain("packageManager ??"); + expect(remoteScript).toContain("[0-9a-f]{128}"); + expect(remoteScript).toContain('console.log(match[1] + " " + match[2])'); + expect(remoteScript).toContain('active_pnpm_version="$(pnpm --version 2>/dev/null || true)"'); + expect(remoteScript).toContain("https://registry.npmjs.org/pnpm/-/pnpm-$pnpm_version.tgz"); + expect(remoteScript).toContain('sha512sum "$pnpm_archive"'); + expect(remoteScript).toContain('chmod +x "$pnpm_cli"'); + expect(remoteScript).toContain('ln -sfn "$pnpm_cli" "$pnpm_bin_dir/pnpm"'); + expect(remoteScript).toContain('export PATH="$pnpm_bin_dir:$PATH"'); + expect(remoteScript).toContain("Expected pnpm $pnpm_version, got $active_pnpm_version."); expect(remoteScript).toContain("pnpm install --frozen-lockfile --prefer-offline"); expect(remoteScript).toContain("pnpm build"); expect(remoteScript).toContain("ffmpeg"); @@ -370,6 +385,87 @@ describe("mantis Slack desktop smoke runtime", () => { ).rejects.toThrow("--approval-checkpoints only supports approval checkpoint scenarios"); }); + it.each([ + { + expectedScenarioIds: ["slack-codex-approval-exec-native"], + label: "command", + remoteTimeoutSeconds: 1_250, + requestedScenarioIds: ["slack-codex-approval-exec-native"], + }, + { + expectedScenarioIds: ["slack-codex-approval-plugin-native"], + label: "file", + remoteTimeoutSeconds: 1_250, + requestedScenarioIds: ["slack-codex-approval-plugin-native"], + }, + { + expectedScenarioIds: [ + "slack-codex-approval-exec-native", + "slack-codex-approval-plugin-native", + ], + label: "command and file", + remoteTimeoutSeconds: 1_900, + requestedScenarioIds: [ + "slack-codex-approval-plugin-native", + "slack-codex-approval-exec-native", + "slack-codex-approval-plugin-native", + ], + }, + ])( + "accepts the opt-in Codex $label checkpoint scenarios", + async ({ expectedScenarioIds, remoteTimeoutSeconds, requestedScenarioIds }) => { + const commands: { args: readonly string[]; command: string }[] = []; + const runner = vi.fn(async (command: string, args: readonly string[]) => { + commands.push({ command, args }); + if (command === "/tmp/crabbox" && args[0] === "warmup") { + return { stdout: "ready lease cbx_123abc\n", stderr: "" }; + } + if (command === "/tmp/crabbox" && args[0] === "inspect") { + return { + stdout: `${JSON.stringify({ + host: "203.0.113.10", + id: "cbx_123abc", + provider: "hetzner", + sshKey: "/tmp/key", + sshPort: "2222", + sshUser: "crabbox", + state: "active", + })}\n`, + stderr: "", + }; + } + if (command === "/tmp/crabbox" && args[0] === "run") { + throw new Error("stop after remote script capture"); + } + return { stdout: "", stderr: "" }; + }); + + const result = await runMantisSlackDesktopSmoke({ + approvalCheckpoints: true, + commandRunner: runner, + crabboxBin: "/tmp/crabbox", + outputDir: `.artifacts/qa-e2e/mantis/${requestedScenarioIds.join("-")}`, + repoRoot, + scenarioIds: requestedScenarioIds, + }); + + expect(result.status).toBe("fail"); + const remoteScript = commands + .find((entry) => entry.command === "/tmp/crabbox" && entry.args[0] === "run") + ?.args.at(-1); + for (const scenarioId of expectedScenarioIds) { + expect(remoteScript?.split(`--scenario '${scenarioId}'`)).toHaveLength(3); + } + expect(remoteScript).toContain( + expectedScenarioIds.map((scenarioId) => `--scenario '${scenarioId}'`).join(" "), + ); + expect(remoteScript).toContain("OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS:-470000"); + expect(remoteScript).toContain( + `OPENCLAW_MANTIS_REMOTE_COMMAND_TIMEOUT_SECONDS:-${remoteTimeoutSeconds}`, + ); + }, + ); + it("fails approval checkpoint mode when ack metadata does not match the expected state", async () => { const expectedScenarios = ["slack-approval-exec-native", "slack-approval-plugin-native"]; const runner = vi.fn(async (command: string, args: readonly string[]) => { diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts index 179202c586b..c9240ff9951 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts @@ -8,6 +8,7 @@ import { acquireQaCredentialLease, startQaCredentialLeaseHeartbeat, } from "../live-transports/shared/credential-lease.runtime.js"; +import { listSlackQaScenarioCatalog } from "../live-transports/slack/slack-live.runtime.js"; import { isTruthyOptIn, trimToValue } from "../mantis-options.runtime.js"; import { createPhaseTimer, type MantisPhaseTimings } from "../mantis-phase-timer.runtime.js"; import { @@ -142,6 +143,27 @@ const DEFAULT_APPROVAL_CHECKPOINT_SCENARIOS = [ "slack-approval-exec-native", "slack-approval-plugin-native", ] as const; +const SUPPORTED_APPROVAL_CHECKPOINT_SCENARIOS = [ + ...DEFAULT_APPROVAL_CHECKPOINT_SCENARIOS, + "slack-codex-approval-exec-native", + "slack-codex-approval-plugin-native", +] as const; +const DEFAULT_APPROVAL_CHECKPOINT_TIMEOUT_MS = 120_000; +const CODEX_APPROVAL_PENDING_TIMEOUT_MS = 180_000; +const CODEX_APPROVAL_RESOLVE_TIMEOUT_MS = 35_000; +const CODEX_APPROVAL_AGENT_WAIT_TIMEOUT_MS = 185_000; +const CODEX_APPROVAL_HISTORY_TIMEOUT_MS = 10_000; +const CODEX_APPROVAL_RESOLVED_UPDATE_TIMEOUT_MS = 180_000; +const CODEX_APPROVAL_CAPTURE_HEADROOM_MS = 60_000; +const CODEX_APPROVAL_POST_PENDING_BUDGET_MS = + CODEX_APPROVAL_RESOLVE_TIMEOUT_MS + + CODEX_APPROVAL_AGENT_WAIT_TIMEOUT_MS + + CODEX_APPROVAL_HISTORY_TIMEOUT_MS + + CODEX_APPROVAL_RESOLVED_UPDATE_TIMEOUT_MS + + CODEX_APPROVAL_CAPTURE_HEADROOM_MS; +const CODEX_APPROVAL_SCENARIO_BUDGET_MS = + CODEX_APPROVAL_PENDING_TIMEOUT_MS + CODEX_APPROVAL_POST_PENDING_BUDGET_MS; +const DEFAULT_REMOTE_COMMAND_TIMEOUT_SECONDS = 600; const CRABBOX_BIN_ENV = "OPENCLAW_MANTIS_CRABBOX_BIN"; const CRABBOX_PROVIDER_ENV = "OPENCLAW_MANTIS_CRABBOX_PROVIDER"; const CRABBOX_CLASS_ENV = "OPENCLAW_MANTIS_CRABBOX_CLASS"; @@ -183,15 +205,21 @@ function resolveScenarioIds(params: { ? [...DEFAULT_APPROVAL_CHECKPOINT_SCENARIOS] : []; if (params.approvalCheckpoints) { - const allowed = new Set(DEFAULT_APPROVAL_CHECKPOINT_SCENARIOS); + const allowed = new Set(SUPPORTED_APPROVAL_CHECKPOINT_SCENARIOS); const unsupported = scenarioIds.filter((scenarioId) => !allowed.has(scenarioId)); if (unsupported.length > 0) { throw new Error( `--approval-checkpoints only supports approval checkpoint scenarios: ${[ - ...DEFAULT_APPROVAL_CHECKPOINT_SCENARIOS, + ...SUPPORTED_APPROVAL_CHECKPOINT_SCENARIOS, ].join(", ")}. Unsupported: ${unsupported.join(", ")}.`, ); } + const requested = new Set(scenarioIds); + // Slack selects scenarios from catalog order, not CLI order. The watcher + // must mirror that order or both sides can block on different checkpoints. + return listSlackQaScenarioCatalog() + .map((scenario) => scenario.id) + .filter((scenarioId) => requested.has(scenarioId)); } return scenarioIds; } @@ -518,6 +546,20 @@ function renderRemoteScript(params: { const slackChannelId = shellQuote(params.slackChannelId); const scenarioArgs = params.scenarioIds.flatMap((id) => ["--scenario", shellQuote(id)]).join(" "); const checkpointScenarioJson = shellQuote(JSON.stringify(params.scenarioIds)); + const codexScenarioCount = params.scenarioIds.filter((id) => + id.startsWith("slack-codex-approval-"), + ).length; + // The watcher starts before gateway setup, then waits for pending and resolved + // sequentially. The resolved phase includes approval, agent, history, and Slack waits. + const approvalCheckpointTimeoutMs = + codexScenarioCount > 0 + ? CODEX_APPROVAL_POST_PENDING_BUDGET_MS + : DEFAULT_APPROVAL_CHECKPOINT_TIMEOUT_MS; + // Preserve the existing hydration/startup budget, then add every selected + // Codex scenario's complete sequential approval budget. + const remoteCommandTimeoutSeconds = + DEFAULT_REMOTE_COMMAND_TIMEOUT_SECONDS + + Math.ceil((codexScenarioCount * CODEX_APPROVAL_SCENARIO_BUDGET_MS) / 1_000); return `set -euo pipefail out=${shellOutputDir} slack_url_override=${slackUrl} @@ -532,7 +574,7 @@ setup_gateway=${setupGateway} approval_checkpoints=${approvalCheckpoints} slack_channel_id=${slackChannelId} approval_checkpoint_scenarios_json=${checkpointScenarioJson} -remote_command_timeout_seconds="\${OPENCLAW_MANTIS_REMOTE_COMMAND_TIMEOUT_SECONDS:-600}" +remote_command_timeout_seconds="\${OPENCLAW_MANTIS_REMOTE_COMMAND_TIMEOUT_SECONDS:-${remoteCommandTimeoutSeconds}}" if [ -z "\${OPENCLAW_QA_SLACK_CHANNEL_ID:-}" ] && [ -n "$slack_channel_id" ]; then export OPENCLAW_QA_SLACK_CHANNEL_ID="$slack_channel_id" fi @@ -652,7 +694,90 @@ qa_status=0 run_mantis_remote_body() { set -e echo "remote pwd: $(pwd)" - sudo corepack enable || sudo npm install -g pnpm@11 + node_supports_type_stripping() { + node_probe="$(mktemp --suffix=.ts)" + printf 'const value: number = 1;\nif (value !== 1) process.exit(1);\n' >"$node_probe" + node --experimental-strip-types "$node_probe" >/dev/null 2>&1 + probe_status=$? + rm -f "$node_probe" + return "$probe_status" + } + if ! node_supports_type_stripping; then + # Distro Node builds can satisfy the version range while omitting native + # TypeScript stripping, which the repository build requires. + node_version="$(sed -n 's/^node_version="\\([0-9][0-9.]*\\)"$/\\1/p' scripts/crabbox-untrusted-bootstrap.sh)" + case "$node_version" in + ''|*[!0-9.]*) + echo "Could not resolve the trusted Crabbox Node version." >&2 + exit 3 + ;; + esac + case "$(uname -m)" in + x86_64) node_arch=x64 ;; + aarch64|arm64) node_arch=arm64 ;; + *) + echo "Unsupported Node bootstrap architecture: $(uname -m)" >&2 + exit 3 + ;; + esac + node_root="$HOME/.cache/openclaw-mantis/node-v$node_version-linux-$node_arch" + if [ ! -x "$node_root/bin/node" ]; then + node_tmp="$(mktemp -d)" + node_archive="node-v$node_version-linux-$node_arch.tar.xz" + node_base_url="https://nodejs.org/dist/v$node_version" + curl -fsSL --retry 3 --retry-all-errors "$node_base_url/SHASUMS256.txt" \ + -o "$node_tmp/SHASUMS256.txt" + curl -fsSL --retry 3 --retry-all-errors "$node_base_url/$node_archive" \ + -o "$node_tmp/$node_archive" + (cd "$node_tmp" && grep " $node_archive$" SHASUMS256.txt | sha256sum -c -) + rm -rf "$node_root" + mkdir -p "$node_root" + tar -xJf "$node_tmp/$node_archive" -C "$node_root" --strip-components=1 + rm -rf "$node_tmp" + fi + export PATH="$node_root/bin:$PATH" + node_supports_type_stripping || { + echo "Official Node $node_version lacks required TypeScript stripping." >&2 + exit 3 + } + fi + node --version + read -r pnpm_version pnpm_sha512 < <(node -e ' +const value = require("./package.json").packageManager ?? ""; +const match = /^pnpm@([0-9]+\\.[0-9]+\\.[0-9]+)\\+sha512\\.([0-9a-f]{128})$/.exec(value); +if (!match) process.exit(1); +console.log(match[1] + " " + match[2]); +') + active_pnpm_version="$(pnpm --version 2>/dev/null || true)" + if [ "$active_pnpm_version" != "$pnpm_version" ]; then + # Some desktop images ship an old distro Corepack that enables its shim but + # cannot execute current pnpm and may omit npm. Download the exact package, + # verify the repository-pinned digest, and run its bundled CLI with Node. + pnpm_root="$out/pnpm-$pnpm_version" + pnpm_archive="$pnpm_root/pnpm.tgz" + mkdir -p "$pnpm_root" + curl -fsSL --retry 3 --retry-all-errors \ + "https://registry.npmjs.org/pnpm/-/pnpm-$pnpm_version.tgz" \ + -o "$pnpm_archive" + downloaded_pnpm_sha512="$(sha512sum "$pnpm_archive" | awk '{print $1}')" + if [ "$downloaded_pnpm_sha512" != "$pnpm_sha512" ]; then + echo "pnpm $pnpm_version SHA-512 mismatch." >&2 + exit 3 + fi + tar -xzf "$pnpm_archive" -C "$pnpm_root" + pnpm_cli="$pnpm_root/package/bin/pnpm.cjs" + chmod +x "$pnpm_cli" + pnpm_bin_dir="$pnpm_root/bin" + mkdir -p "$pnpm_bin_dir" + ln -sfn "$pnpm_cli" "$pnpm_bin_dir/pnpm" + export PATH="$pnpm_bin_dir:$PATH" + hash -r + active_pnpm_version="$(pnpm --version)" + fi + if [ "$active_pnpm_version" != "$pnpm_version" ]; then + echo "Expected pnpm $pnpm_version, got $active_pnpm_version." >&2 + exit 3 + fi if [ "$hydrate_mode" = "source" ]; then if ! command -v make >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then sudo apt-get update -y >>"$out/apt.log" 2>&1 || true @@ -739,7 +864,7 @@ MANTIS_SLACK_PATCH checkpoint_dir="$out/approval-checkpoints" mkdir -p "$checkpoint_dir" export OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_DIR="$checkpoint_dir" - export OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS="\${OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS:-120000}" + export OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS="\${OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS:-${approvalCheckpointTimeoutMs}}" export OPENCLAW_MANTIS_APPROVAL_CHECKPOINT_SCENARIOS_JSON="$approval_checkpoint_scenarios_json" export OPENCLAW_MANTIS_APPROVAL_BROWSER_BIN="$browser_bin" cat >"$out/approval-checkpoint-watcher.mjs" <<'MANTIS_APPROVAL_WATCHER' @@ -749,7 +874,7 @@ MANTIS_SLACK_PATCH const checkpointDir = process.env.OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_DIR; const timeoutMs = Number.parseInt( - process.env.OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS || "120000", + process.env.OPENCLAW_QA_SLACK_APPROVAL_CHECKPOINT_TIMEOUT_MS || "${approvalCheckpointTimeoutMs}", 10, ); const scenarioIds = JSON.parse( diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 7f74387dcbc..a42ce638ed1 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -1550,6 +1550,17 @@ describe("package artifact reuse", () => { } }); + it("maps every supported Slack approval checkpoint scenario family", () => { + const workflow = readFileSync(MANTIS_SLACK_DESKTOP_SMOKE_WORKFLOW, "utf8"); + + expectTextToIncludeAll(workflow, [ + 'endswith("-exec-native")', + 'endswith("-plugin-native")', + 'startswith("slack-codex-")', + 'expected_result="Slack approval checkpoint passes for $scenario_label"', + ]); + }); + it("fails Docker E2E release lanes when summary artifacts are missing", () => { const cases = [ {