From e75223286d2000bdf7df4a6aeda9291789c1b7ef Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 6 Jul 2026 21:22:56 +0200 Subject: [PATCH] fix(qa): require live channel driver for access-gate scenarios --- extensions/qa-lab/src/coverage-report.test.ts | 15 ++++ extensions/qa-lab/src/coverage-report.ts | 21 ++++- .../qa-lab/src/scenario-catalog.test.ts | 18 ++++ extensions/qa-lab/src/suite-planning.test.ts | 15 +++- extensions/qa-lab/src/suite-planning.ts | 88 ++++++++++++------- .../channels/matrix-allowlist-hot-reload.yaml | 1 + .../whatsapp-access-control-dm-disabled.yaml | 1 + .../whatsapp-access-control-dm-open.yaml | 1 + ...hatsapp-access-control-group-disabled.yaml | 1 + .../whatsapp-access-control-group-open.yaml | 1 + .../channels/whatsapp-pairing-block.yaml | 1 + taxonomy.yaml | 6 +- 12 files changed, 132 insertions(+), 37 deletions(-) diff --git a/extensions/qa-lab/src/coverage-report.test.ts b/extensions/qa-lab/src/coverage-report.test.ts index bf8b4914b49..7faac93ad07 100644 --- a/extensions/qa-lab/src/coverage-report.test.ts +++ b/extensions/qa-lab/src/coverage-report.test.ts @@ -304,6 +304,21 @@ describe("qa coverage report", () => { expect(report).not.toContain("Native test refs"); }); + it("includes required channel driver flags in scenario match commands", () => { + const matches = findQaScenarioMatches( + readQaScenarioPack().scenarios, + "whatsapp-access-control-group-disabled", + ); + const report = renderQaScenarioMatchesMarkdownReport({ + query: "whatsapp-access-control-group-disabled", + matches, + }); + + expect(report).toContain( + "- Suite command: `pnpm openclaw qa suite --channel-driver live --channel whatsapp --scenario whatsapp-access-control-group-disabled`", + ); + }); + it("splits qa suite targets when matches mix execution kinds", () => { const playwrightExecutionPath = "ui/src/e2e/chat-flow.e2e.test.ts"; const flowScenario = scenarioWithCoverage({ diff --git a/extensions/qa-lab/src/coverage-report.ts b/extensions/qa-lab/src/coverage-report.ts index 77082a138fc..d2f99d81993 100644 --- a/extensions/qa-lab/src/coverage-report.ts +++ b/extensions/qa-lab/src/coverage-report.ts @@ -20,6 +20,7 @@ type QaCoverageScenarioSummary = { }; type QaScenarioSearchMatch = QaCoverageScenarioSummary & { + channel?: string; coverageIds: string[]; docsRefs: string[]; codeRefs: string[]; @@ -163,6 +164,7 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce docsRefs: [...(scenario.docsRefs ?? [])], codeRefs: [...(scenario.codeRefs ?? [])], executionKind: scenario.execution.kind, + channel: scenario.execution.channel, ...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}), runtimeParityTier: scenario.runtimeParityTier, requiredProviderMode: stringifyConfigValue(config.requiredProviderMode), @@ -481,7 +483,24 @@ function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) { function formatSuiteCommand(matches: readonly QaScenarioSearchMatch[]) { const scenarioArgs = matches.map((match) => `--scenario ${match.id}`).join(" "); - return `pnpm openclaw qa suite ${scenarioArgs}`; + const requiredDrivers = [ + ...new Set( + matches.map((match) => match.requiredChannelDriver).filter((driver): driver is string => + Boolean(driver), + ), + ), + ]; + const driverArg = + requiredDrivers.length === 1 && requiredDrivers[0] !== "qa-channel" + ? ` --channel-driver ${requiredDrivers[0]}` + : ""; + const channels = [ + ...new Set(matches.map((match) => match.channel).filter((channel): channel is string => + Boolean(channel), + )), + ]; + const channelArg = driverArg && channels.length === 1 ? ` --channel ${channels[0]}` : ""; + return `pnpm openclaw qa suite${driverArg}${channelArg} ${scenarioArgs}`; } function scenarioMatchCommandGroups(matches: readonly QaScenarioSearchMatch[]) { diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 056c24b1f1b..aeaa4eca622 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -799,6 +799,24 @@ describe("qa scenario catalog", () => { expect(config?.requiredProviderMode).toBe("mock-openai"); }); + it("marks channel-owned access gates with their required channel driver", () => { + const liveScenarioIds = [ + "whatsapp-access-control-dm-disabled", + "whatsapp-access-control-dm-open", + "whatsapp-access-control-group-disabled", + "whatsapp-access-control-group-open", + "whatsapp-pairing-block", + "matrix-allowlist-hot-reload", + ]; + + for (const scenarioId of liveScenarioIds) { + const config = readQaScenarioExecutionConfig(scenarioId) as + | { requiredChannelDriver?: string } + | undefined; + expect(config?.requiredChannelDriver, scenarioId).toBe("live"); + } + }); + it("adds a dreaming shadow trial report scenario", () => { const scenario = readQaScenarioById("dreaming-shadow-trial-report"); const config = readQaScenarioExecutionConfig("dreaming-shadow-trial-report") as diff --git a/extensions/qa-lab/src/suite-planning.test.ts b/extensions/qa-lab/src/suite-planning.test.ts index 0f2bee883e8..ecd72ddf592 100644 --- a/extensions/qa-lab/src/suite-planning.test.ts +++ b/extensions/qa-lab/src/suite-planning.test.ts @@ -593,7 +593,7 @@ describe("qa suite planning helpers", () => { ).toEqual(["generic", "crabline-only"]); }); - it("keeps explicitly requested channel-driver-specific scenarios", () => { + it("rejects explicitly requested scenarios that do not match the current lane", () => { const scenarios = [ makeQaSuiteTestScenario("generic"), makeQaSuiteTestScenario("qa-channel-only", { @@ -601,13 +601,24 @@ describe("qa suite planning helpers", () => { }), ]; - expect( + expect(() => selectQaFlowSuiteScenarios({ scenarios, scenarioIds: ["qa-channel-only"], providerMode: "mock-openai", primaryModel: "mock-openai/gpt-5.5", channelDriver: "crabline", + }), + ).toThrow( + "selected QA scenario(s) do not match the current QA lane: qa-channel-only (channelDriver=qa-channel)", + ); + + expect( + selectQaFlowSuiteScenarios({ + scenarios, + scenarioIds: ["qa-channel-only"], + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", }).map((scenario) => scenario.id), ).toEqual(["qa-channel-only"]); }); diff --git a/extensions/qa-lab/src/suite-planning.ts b/extensions/qa-lab/src/suite-planning.ts index e985b0f816f..0e59bc049f4 100644 --- a/extensions/qa-lab/src/suite-planning.ts +++ b/extensions/qa-lab/src/suite-planning.ts @@ -26,6 +26,47 @@ function normalizeQaConfigString(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function describeQaProviderLaneMismatches(params: { + scenario: QaSeedScenario; + primaryModel: string; + providerMode: QaProviderMode; + channelDriver?: QaScorecardChannelDriver | null; + claudeCliAuthMode?: QaCliBackendAuthMode; +}) { + const mismatches: string[] = []; + const provider = getQaProvider(params.providerMode); + if (params.scenario.runtimeParityTier === "live-only" && provider.kind !== "live") { + mismatches.push("live provider mode"); + } + const config = params.scenario.execution.config ?? {}; + const requiredProviderMode = normalizeQaConfigString(config.requiredProviderMode); + if (requiredProviderMode && params.providerMode !== requiredProviderMode) { + mismatches.push(`providerMode=${requiredProviderMode}`); + } + const requiredChannelDriver = normalizeQaConfigString(config.requiredChannelDriver); + const effectiveChannelDriver = params.channelDriver ?? "qa-channel"; + if (requiredChannelDriver && effectiveChannelDriver !== requiredChannelDriver) { + mismatches.push(`channelDriver=${requiredChannelDriver}`); + } + if (provider.kind !== "live") { + return mismatches; + } + const selected = splitModelRef(params.primaryModel); + const requiredProvider = normalizeQaConfigString(config.requiredProvider); + if (requiredProvider && selected?.provider !== requiredProvider) { + mismatches.push(`provider=${requiredProvider}`); + } + const requiredModel = normalizeQaConfigString(config.requiredModel); + if (requiredModel && selected?.model !== requiredModel) { + mismatches.push(`model=${requiredModel}`); + } + const requiredAuthMode = normalizeQaConfigString(config.authMode); + if (requiredAuthMode && params.claudeCliAuthMode !== requiredAuthMode) { + mismatches.push(`authMode=${requiredAuthMode}`); + } + return mismatches; +} + function scenarioMatchesQaProviderLane(params: { scenario: QaSeedScenario; primaryModel: string; @@ -33,37 +74,7 @@ function scenarioMatchesQaProviderLane(params: { channelDriver?: QaScorecardChannelDriver | null; claudeCliAuthMode?: QaCliBackendAuthMode; }) { - const provider = getQaProvider(params.providerMode); - if (params.scenario.runtimeParityTier === "live-only" && provider.kind !== "live") { - return false; - } - const config = params.scenario.execution.config ?? {}; - const requiredProviderMode = normalizeQaConfigString(config.requiredProviderMode); - if (requiredProviderMode && params.providerMode !== requiredProviderMode) { - return false; - } - const requiredChannelDriver = normalizeQaConfigString(config.requiredChannelDriver); - const effectiveChannelDriver = params.channelDriver ?? "qa-channel"; - if (requiredChannelDriver && effectiveChannelDriver !== requiredChannelDriver) { - return false; - } - if (provider.kind !== "live") { - return true; - } - const selected = splitModelRef(params.primaryModel); - const requiredProvider = normalizeQaConfigString(config.requiredProvider); - if (requiredProvider && selected?.provider !== requiredProvider) { - return false; - } - const requiredModel = normalizeQaConfigString(config.requiredModel); - if (requiredModel && selected?.model !== requiredModel) { - return false; - } - const requiredAuthMode = normalizeQaConfigString(config.authMode); - if (requiredAuthMode && params.claudeCliAuthMode !== requiredAuthMode) { - return false; - } - return true; + return describeQaProviderLaneMismatches(params).length === 0; } function selectQaFlowSuiteScenarios(params: { @@ -98,6 +109,21 @@ function selectQaFlowSuiteScenarios(params: { `flow execution requires execution.kind: flow; unsupported scenario(s): ${scenarioList}`, ); } + const channelDriverMismatches = selectedScenarios.flatMap((scenario) => { + const mismatches = describeQaProviderLaneMismatches({ + scenario, + providerMode: params.providerMode, + primaryModel: params.primaryModel, + channelDriver: params.channelDriver, + claudeCliAuthMode: params.claudeCliAuthMode, + }).filter((mismatch) => mismatch.startsWith("channelDriver=")); + return mismatches.length > 0 ? [`${scenario.id} (${mismatches.join(", ")})`] : []; + }); + if (channelDriverMismatches.length > 0) { + throw new Error( + `selected QA scenario(s) do not match the current QA lane: ${channelDriverMismatches.join(", ")}`, + ); + } return selectedScenarios; } return params.scenarios.filter( diff --git a/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml b/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml index 4dcb3eaa2b8..39f1e877521 100644 --- a/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml +++ b/qa/scenarios/channels/matrix-allowlist-hot-reload.yaml @@ -20,6 +20,7 @@ scenario: suiteIsolation: isolated summary: Exercise Matrix observer-role allowlist hot reload through normalized transport state. config: + requiredChannelDriver: live accountId: sut flow: diff --git a/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml b/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml index fee1c20b1bc..c2fc6390ddb 100644 --- a/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-dm-disabled.yaml @@ -19,6 +19,7 @@ scenario: suiteIsolation: isolated summary: Verify WhatsApp dmPolicy disabled behavior through the canonical adapter. config: + requiredChannelDriver: live policyKey: dmPolicy policyValue: disabled conversationKind: direct diff --git a/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml b/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml index 046909fcd2c..ca3a96c46b0 100644 --- a/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-dm-open.yaml @@ -19,6 +19,7 @@ scenario: suiteIsolation: isolated summary: Verify WhatsApp dmPolicy open behavior through the canonical adapter. config: + requiredChannelDriver: live policyKey: dmPolicy policyValue: open conversationKind: direct diff --git a/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml b/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml index 7e46050e7c3..14976407fb4 100644 --- a/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-group-disabled.yaml @@ -19,6 +19,7 @@ scenario: suiteIsolation: isolated summary: Verify WhatsApp groupPolicy disabled behavior through the canonical adapter. config: + requiredChannelDriver: live policyKey: groupPolicy policyValue: disabled conversationKind: group diff --git a/qa/scenarios/channels/whatsapp-access-control-group-open.yaml b/qa/scenarios/channels/whatsapp-access-control-group-open.yaml index 75ad0ee84ef..5567784d275 100644 --- a/qa/scenarios/channels/whatsapp-access-control-group-open.yaml +++ b/qa/scenarios/channels/whatsapp-access-control-group-open.yaml @@ -19,6 +19,7 @@ scenario: suiteIsolation: isolated summary: Verify WhatsApp groupPolicy open behavior through the canonical adapter. config: + requiredChannelDriver: live policyKey: groupPolicy policyValue: open conversationKind: group diff --git a/qa/scenarios/channels/whatsapp-pairing-block.yaml b/qa/scenarios/channels/whatsapp-pairing-block.yaml index 804f2d72fbe..b2dca31097c 100644 --- a/qa/scenarios/channels/whatsapp-pairing-block.yaml +++ b/qa/scenarios/channels/whatsapp-pairing-block.yaml @@ -20,6 +20,7 @@ scenario: suiteIsolation: isolated summary: Verify pairing gate output through the canonical WhatsApp adapter. config: + requiredChannelDriver: live accountId: sut conversationId: 15550000001@s.whatsapp.net senderId: 15550000002@s.whatsapp.net diff --git a/taxonomy.yaml b/taxonomy.yaml index 46c8ae8c811..621d699b232 100644 --- a/taxonomy.yaml +++ b/taxonomy.yaml @@ -1874,10 +1874,10 @@ surfaces: id: inbound-access-and-identity-gates features: - name: DM pairing - coverageIds: [security.dm-pairing] + coverageIds: [security.dm-pairing, channels.pairing-gate] description: DM pairing and allowFrom sender controls - name: Group/channel allowlists - coverageIds: [channels.group-channel-allowlists] + coverageIds: [channels.access-policy, channels.allowlist-hot-reload, channels.group-channel-allowlists] description: Group/channel allowlists and sender allowlists - name: Access group expansion coverageIds: [channels.access-group-expansion] @@ -1930,7 +1930,7 @@ surfaces: id: outbound-delivery-and-reply-pipeline features: - name: Automatic final reply delivery - coverageIds: [agents.subagents, channels.dedup, channels.direct-visible-replies, channels.dm, channels.group-visible-replies, channels.qa-channel, channels.reconnect, channels.streaming, channels.threads, commitments.heartbeat-target-none, commitments.scope, personal.channel-replies, runtime.delivery, runtime.fallback-delivery, runtime.gateway-restart, runtime.restart-recovery, tools.message] + coverageIds: [agents.subagents, channels.dedup, channels.direct-visible-replies, channels.dm, channels.group-visible-replies, channels.qa-channel, channels.reconnect, channels.restart-resume, channels.streaming, channels.threads, commitments.heartbeat-target-none, commitments.scope, personal.channel-replies, runtime.delivery, runtime.fallback-delivery, runtime.gateway-restart, runtime.restart-recovery, tools.message] description: Automatic final reply delivery and strict message-tool-only visible delivery - name: Durable outbound send orchestration coverageIds: [channels.dedup, channels.reconnect, runtime.delivery]