fix(qa): gate smoke profile scenarios by channel driver

2026-07-10 00:11:19 +00:00 · 2026-06-23 08:36:07 +08:00 · 2026-06-23 08:36:07 +08:00 · def4b51485
commit def4b51485
parent d84a8b1506
29 changed files with 167 additions and 0 deletions
--- a/extensions/qa-lab/src/cli.runtime.test.ts
+++ b/extensions/qa-lab/src/cli.runtime.test.ts
@ -514,6 +514,51 @@ describe("qa cli runtime", () => {
    expect(suiteArgs.channelDriverSelection).toBeUndefined();
  });

+  it("filters QA-channel-pinned scenarios from the Crabline smoke profile", async () => {
+    runQaSuite.mockImplementationOnce(async () => {
+      await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8");
+      return flowSuiteRuntimeResult({
+        reportPath: suiteReportPath,
+        summaryPath: suiteSummaryPath,
+      });
+    });
+
+    await runQaProfileCommand({
+      repoRoot: "/tmp/openclaw-repo",
+      profile: "smoke-ci",
+    });
+
+    const suiteArgs = mockFirstObjectArg(runQaSuite);
+    expect(suiteArgs.channelDriver).toBe("crabline");
+    expect(suiteArgs.scenarioIds).toEqual(expect.arrayContaining(["dm-chat-baseline"]));
+    expect(suiteArgs.scenarioIds).not.toEqual(
+      expect.arrayContaining([
+        "instruction-followthrough-repo-contract",
+        "subagent-forked-context",
+        "subagent-handoff",
+        "group-message-tool-unavailable-fallback",
+        "qa-channel-reconnect-dedupe",
+        "reaction-edit-delete",
+        "thread-follow-up",
+        "image-generation-roundtrip",
+        "image-understanding-attachment",
+        "native-image-generation",
+        "active-memory-preprompt-recall",
+        "memory-recall",
+        "session-memory-ranking",
+        "thread-memory-isolation",
+        "personal-channel-thread-reply",
+        "personal-memory-preference-recall",
+        "personal-reminder-roundtrip",
+        "cron-natural-fire-no-duplicate",
+        "cron-one-minute-ping",
+        "cron-single-run-no-duplicate",
+        "control-ui-qa-channel-image-roundtrip",
+        "config-apply-restart-wakeup",
+      ]),
+    );
+  });
+
  it("rejects qa profile runs that do not match taxonomy categories", async () => {
    await expect(
      runQaProfileCommand({
--- a/extensions/qa-lab/src/cli.runtime.ts
+++ b/extensions/qa-lab/src/cli.runtime.ts
@ -736,6 +736,7 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
      scenario,
      providerMode: normalizedProviderMode,
      primaryModel,
+      channelDriver: profileReport.channelDriver,
    }),
  );
  if (scenarios.length === 0) {
@ -786,6 +787,7 @@ function selectQaScenarioDefinitionsForChannelResolution(params: {
  scenarioIds: string[];
  providerMode: QaProviderMode;
  primaryModel: string;
+  channelDriver?: QaScorecardChannelDriver | null;
  claudeCliAuthMode?: QaCliBackendAuthMode;
 }) {
  const scenarios = readQaScenarioPack().scenarios;
@ -801,6 +803,7 @@ function selectQaScenarioDefinitionsForChannelResolution(params: {
      scenario,
      providerMode: params.providerMode,
      primaryModel: params.primaryModel,
+      channelDriver: params.channelDriver,
      claudeCliAuthMode: params.claudeCliAuthMode,
    }),
  );
@ -915,6 +918,7 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
              scenarioIds,
              providerMode,
              primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
+              channelDriver,
              claudeCliAuthMode,
            }),
          }),
--- a/extensions/qa-lab/src/coverage-report.ts
+++ b/extensions/qa-lab/src/coverage-report.ts
@ -27,6 +27,7 @@ type QaScenarioSearchMatch = QaCoverageScenarioSummary & {
  executionPath?: string;
  runtimeParityTier?: string;
  requiredProviderMode?: string;
+  requiredChannelDriver?: string;
  requiredProvider?: string;
  requiredModel?: string;
 };
@ -144,6 +145,7 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce
    ...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}),
    runtimeParityTier: scenario.runtimeParityTier,
    requiredProviderMode: stringifyConfigValue(config.requiredProviderMode),
+    requiredChannelDriver: stringifyConfigValue(config.requiredChannelDriver),
    requiredProvider: stringifyConfigValue(config.requiredProvider),
    requiredModel: stringifyConfigValue(config.requiredModel),
  };
@ -447,6 +449,7 @@ function formatOptionalScenarioMetadata(match: QaScenarioSearchMatch) {
  const metadata = [
    match.runtimeParityTier ? `runtimeParityTier=${match.runtimeParityTier}` : "",
    match.requiredProviderMode ? `providerMode=${match.requiredProviderMode}` : "",
+    match.requiredChannelDriver ? `channelDriver=${match.requiredChannelDriver}` : "",
    match.requiredProvider ? `provider=${match.requiredProvider}` : "",
    match.requiredModel ? `model=${match.requiredModel}` : "",
  ].filter(Boolean);
--- a/extensions/qa-lab/src/scenario-catalog.test.ts
+++ b/extensions/qa-lab/src/scenario-catalog.test.ts
@ -639,6 +639,7 @@ describe("qa scenario catalog", () => {
      | {
          workspaceFiles?: Record<string, string>;
          prompt?: string;
+          requiredChannelDriver?: string;
          expectedReplyAll?: string[];
          expectedArtifactAll?: string[];
          expectedArtifactAny?: string[];
@ -651,12 +652,46 @@ describe("qa scenario catalog", () => {
      "Mission: prove you followed the repo contract.",
    );
    expect(config?.prompt).toContain("Repo contract followthrough check.");
+    expect(config?.requiredChannelDriver).toBe("qa-channel");
    expect(config?.expectedReplyAll).toEqual(["read:", "wrote:", "status:"]);
    expect(config?.expectedArtifactAll).toEqual(["repo contract"]);
    expect(config?.expectedArtifactAny).toContain("evidence path");
    expect(scenario.title).toBe("Instruction followthrough repo contract");
  });

+  it("keeps native QA-channel fixtures out of Crabline profile selection", () => {
+    const scenarioIds = [
+      "subagent-forked-context",
+      "subagent-handoff",
+      "group-message-tool-unavailable-fallback",
+      "qa-channel-reconnect-dedupe",
+      "reaction-edit-delete",
+      "thread-follow-up",
+      "image-generation-roundtrip",
+      "image-understanding-attachment",
+      "native-image-generation",
+      "active-memory-preprompt-recall",
+      "memory-recall",
+      "session-memory-ranking",
+      "thread-memory-isolation",
+      "personal-channel-thread-reply",
+      "personal-memory-preference-recall",
+      "personal-reminder-roundtrip",
+      "cron-natural-fire-no-duplicate",
+      "cron-one-minute-ping",
+      "cron-single-run-no-duplicate",
+      "control-ui-qa-channel-image-roundtrip",
+      "config-apply-restart-wakeup",
+    ];
+
+    for (const scenarioId of scenarioIds) {
+      const config = readQaScenarioExecutionConfig(scenarioId) as
+        | { requiredChannelDriver?: string }
+        | undefined;
+      expect(config?.requiredChannelDriver, scenarioId).toBe("qa-channel");
+    }
+  });
+
  it("adds a dreaming shadow trial report scenario", () => {
    const scenario = readQaScenarioById("dreaming-shadow-trial-report");
    const config = readQaScenarioExecutionConfig("dreaming-shadow-trial-report") as
--- a/extensions/qa-lab/src/suite-planning.test.ts
+++ b/extensions/qa-lab/src/suite-planning.test.ts
@ -528,6 +528,54 @@ describe("qa suite planning helpers", () => {
    ).toEqual(["generic", "live-only"]);
  });

+  it("filters channel-driver-specific scenarios from implicit suite selections", () => {
+    const scenarios = [
+      makeQaSuiteTestScenario("generic"),
+      makeQaSuiteTestScenario("qa-channel-only", {
+        config: { requiredChannelDriver: "qa-channel" },
+      }),
+      makeQaSuiteTestScenario("crabline-only", {
+        config: { requiredChannelDriver: "crabline" },
+      }),
+    ];
+
+    expect(
+      selectQaFlowSuiteScenarios({
+        scenarios,
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.5",
+      }).map((scenario) => scenario.id),
+    ).toEqual(["generic", "qa-channel-only"]);
+
+    expect(
+      selectQaFlowSuiteScenarios({
+        scenarios,
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.5",
+        channelDriver: "crabline",
+      }).map((scenario) => scenario.id),
+    ).toEqual(["generic", "crabline-only"]);
+  });
+
+  it("keeps explicitly requested channel-driver-specific scenarios", () => {
+    const scenarios = [
+      makeQaSuiteTestScenario("generic"),
+      makeQaSuiteTestScenario("qa-channel-only", {
+        config: { requiredChannelDriver: "qa-channel" },
+      }),
+    ];
+
+    expect(
+      selectQaFlowSuiteScenarios({
+        scenarios,
+        scenarioIds: ["qa-channel-only"],
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.5",
+        channelDriver: "crabline",
+      }).map((scenario) => scenario.id),
+    ).toEqual(["qa-channel-only"]);
+  });
+
  it("keeps live-only runtime parity scenarios out of implicit mock selections", () => {
    const scenarios = [
      makeQaSuiteTestScenario("generic"),
--- a/extensions/qa-lab/src/suite-planning.ts
+++ b/extensions/qa-lab/src/suite-planning.ts
@ -7,6 +7,7 @@ import type { QaCliBackendAuthMode } from "./gateway-child.js";
 import { splitQaModelRef as splitModelRef, type QaProviderMode } from "./model-selection.js";
 import { getQaProvider } from "./providers/index.js";
 import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
+import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js";
 import { applyQaMergePatch, isQaMergePatchObject } from "./suite-merge-patch.js";

 const DEFAULT_QA_SUITE_CONCURRENCY = 64;
@ -22,6 +23,7 @@ function scenarioMatchesQaProviderLane(params: {
  scenario: QaSeedScenario;
  primaryModel: string;
  providerMode: QaProviderMode;
+  channelDriver?: QaScorecardChannelDriver | null;
  claudeCliAuthMode?: QaCliBackendAuthMode;
 }) {
  const provider = getQaProvider(params.providerMode);
@ -33,6 +35,11 @@ function scenarioMatchesQaProviderLane(params: {
  if (requiredProviderMode && params.providerMode !== requiredProviderMode) {
    return false;
  }
+  const requiredChannelDriver = normalizeQaConfigString(config.requiredChannelDriver);
+  const effectiveChannelDriver = params.channelDriver ?? "qa-channel";
+  if (requiredChannelDriver && effectiveChannelDriver !== requiredChannelDriver) {
+    return false;
+  }
  if (provider.kind !== "live") {
    return true;
  }
@ -57,6 +64,7 @@ function selectQaFlowSuiteScenarios(params: {
  scenarioIds?: string[];
  providerMode: QaProviderMode;
  primaryModel: string;
+  channelDriver?: QaScorecardChannelDriver | null;
  claudeCliAuthMode?: QaCliBackendAuthMode;
 }) {
  const requestedScenarioIds =
@ -92,6 +100,7 @@ function selectQaFlowSuiteScenarios(params: {
        scenario,
        providerMode: params.providerMode,
        primaryModel: params.primaryModel,
+        channelDriver: params.channelDriver,
        claudeCliAuthMode: params.claudeCliAuthMode,
      }),
  );
--- a/extensions/qa-lab/src/suite.ts
+++ b/extensions/qa-lab/src/suite.ts
@ -1146,6 +1146,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
    scenarioIds: params?.scenarioIds,
    providerMode,
    primaryModel,
+    channelDriver: params?.channelDriver ?? params?.channelDriverSelection?.channelDriver,
    claudeCliAuthMode: params?.claudeCliAuthMode,
  });
  const enabledPluginIds = [
--- a/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml
+++ b/qa/scenarios/agents/instruction-followthrough-repo-contract.yaml
@ -25,6 +25,7 @@ scenario:
    kind: flow
    summary: Verify the agent reads repo instructions first, then completes the first bounded followthrough task without stalling.
    config:
+      requiredChannelDriver: qa-channel
      workspaceFiles:
        AGENT.md: |-
          # Repo contract
--- a/qa/scenarios/agents/subagent-forked-context.yaml
+++ b/qa/scenarios/agents/subagent-forked-context.yaml
@ -22,6 +22,7 @@ scenario:
    kind: flow
    summary: Ask the agent to delegate work that depends on the current transcript and assert sessions_spawn carries context=fork.
    config:
+      requiredChannelDriver: qa-channel
      contextNeedle: FORKED-CONTEXT-ALPHA
      prompt: "Forked subagent context QA check. The visible code in this current conversation is FORKED-CONTEXT-ALPHA. Delegate to a native subagent to report the visible code from the requester transcript. Do not include the visible code in the child task text; the child must recover it from forked transcript context. Use forked context if the child needs the current transcript; otherwise it will not know the code. A spawn-accepted result is not the answer. Wait for the child completion, then make sure user-visible output includes the visible code."

--- a/qa/scenarios/agents/subagent-handoff.yaml
+++ b/qa/scenarios/agents/subagent-handoff.yaml
@ -21,6 +21,7 @@ scenario:
    kind: flow
    summary: Verify the agent can delegate a bounded task to a subagent and fold the result back into the main thread.
    config:
+      requiredChannelDriver: qa-channel
      prompt: "Delegate one bounded QA task to a subagent. Wait for the subagent to finish. Then reply with three labeled sections exactly once: Delegated task, Result, Evidence. Include the child result itself, not 'waiting'."

 flow:
--- a/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml
+++ b/qa/scenarios/channels/group-message-tool-unavailable-fallback.yaml
@ -35,6 +35,7 @@ scenario:
    kind: flow
    summary: Verify message_tool visible replies degrade to automatic delivery when the active group policy removes message.
    config:
+      requiredChannelDriver: qa-channel
      conversationId: qa-fallback-room
      promptSnippet: qa group message unavailable fallback check
      prompt: "@openclaw qa group message unavailable fallback check. exact marker: `QA-GROUP-FALLBACK-OK`"
--- a/qa/scenarios/channels/qa-channel-reconnect-dedupe.yaml
+++ b/qa/scenarios/channels/qa-channel-reconnect-dedupe.yaml
@ -25,6 +25,7 @@ scenario:
    kind: flow
    summary: Verify qa-channel readiness recovery does not duplicate old outbound delivery.
    config:
+      requiredChannelDriver: qa-channel
      firstPrompt: "@openclaw Reconnect dedupe setup marker. Reply exactly: RECONNECT-FIRST-OK"
      secondPrompt: "@openclaw Reconnect dedupe follow-up marker. Reply exactly: RECONNECT-SECOND-OK"
      firstMarker: RECONNECT-FIRST-OK
--- a/qa/scenarios/channels/reaction-edit-delete.yaml
+++ b/qa/scenarios/channels/reaction-edit-delete.yaml
@ -22,6 +22,7 @@ scenario:
    kind: flow
    summary: Verify the agent can use channel-owned message actions and that the QA transcript reflects them.
    config:
+      requiredChannelDriver: qa-channel
      target: "channel:qa-room"
      seedText: "seed message"
      editedText: "seed message (edited)"
--- a/qa/scenarios/channels/thread-follow-up.yaml
+++ b/qa/scenarios/channels/thread-follow-up.yaml
@ -24,6 +24,7 @@ scenario:
    kind: flow
    summary: Verify the agent can keep follow-up work inside a thread and not leak context into the root channel.
    config:
+      requiredChannelDriver: qa-channel
      prompt: "@openclaw reply in one short sentence inside this thread only. Do not use ACP or any external runtime. Confirm you stayed in-thread."

 flow:
--- a/qa/scenarios/config/config-apply-restart-wakeup.yaml
+++ b/qa/scenarios/config/config-apply-restart-wakeup.yaml
@ -23,6 +23,7 @@ scenario:
    kind: flow
    summary: Verify a restart-required config.apply restarts cleanly and delivers the post-restart wake message back into the QA channel.
    config:
+      requiredChannelDriver: qa-channel
      channelId: qa-room
      announcePrompt: "Acknowledge restart wake-up setup in qa-room."

--- a/qa/scenarios/media/image-generation-roundtrip.yaml
+++ b/qa/scenarios/media/image-generation-roundtrip.yaml
@ -25,6 +25,7 @@ scenario:
    kind: flow
    summary: Verify a generated image is saved as media, reattached on the next turn, and described correctly through the vision path.
    config:
+      requiredChannelDriver: qa-channel
      generatePrompt: "Image generation check: generate a QA lighthouse image and summarize it in one short sentence."
      generatePromptSnippet: "Image generation check"
      inspectPrompt: "Roundtrip image inspection check: describe the generated lighthouse attachment in one short sentence."
--- a/qa/scenarios/media/image-understanding-attachment.yaml
+++ b/qa/scenarios/media/image-understanding-attachment.yaml
@ -24,6 +24,7 @@ scenario:
    kind: flow
    summary: Verify an attached image reaches the agent model and the agent can describe what it sees.
    config:
+      requiredChannelDriver: qa-channel
      prompt: "Image understanding check: describe the top and bottom colors in the attached image in one short sentence."
      requiredColorGroups:
        - [red, scarlet, crimson]
--- a/qa/scenarios/media/native-image-generation.yaml
+++ b/qa/scenarios/media/native-image-generation.yaml
@ -23,6 +23,7 @@ scenario:
    kind: flow
    summary: Verify image_generate appears when configured and returns a real saved media artifact.
    config:
+      requiredChannelDriver: qa-channel
      prompt: "Image generation check: generate a QA lighthouse image and summarize it in one short sentence."
      promptSnippet: "Image generation check"
      generatedNeedle: "QA lighthouse"
--- a/qa/scenarios/memory/active-memory-preprompt-recall.yaml
+++ b/qa/scenarios/memory/active-memory-preprompt-recall.yaml
@ -43,6 +43,7 @@ scenario:
    kind: flow
    summary: Verify Active Memory stays off when session-toggled off, runs memory search/get when enabled, and helps a live model answer with the recalled preference in the first visible reply.
    config:
+      requiredChannelDriver: qa-channel
      baselineConversationId: qa-active-memory-off
      activeConversationId: qa-active-memory-on
      memoryFact: "Stable QA movie night usual favorite snack preference: lemon pepper wings with blue cheese."
--- a/qa/scenarios/memory/memory-recall.yaml
+++ b/qa/scenarios/memory/memory-recall.yaml
@ -48,6 +48,7 @@ scenario:
    kind: flow
    summary: Verify the agent can store a fact, switch topics, then recall the fact accurately later.
    config:
+      requiredChannelDriver: qa-channel
      resetDurableMemory: true
      rememberPrompt: "Please remember this fact for later: the QA canary code is ALPHA-7. Use your normal memory mechanism, avoid manual repo cleanup, and reply exactly `Remembered ALPHA-7.` once stored."
      rememberAckAny:
--- a/qa/scenarios/memory/session-memory-ranking.yaml
+++ b/qa/scenarios/memory/session-memory-ranking.yaml
@ -24,6 +24,7 @@ scenario:
    kind: flow
    summary: Verify session-transcript memory can outrank stale durable notes and drive the final answer toward the newer fact.
    config:
+      requiredChannelDriver: qa-channel
      staleFact: ORBIT-9
      currentFact: ORBIT-10
      transcriptId: qa-session-memory-ranking
--- a/qa/scenarios/memory/thread-memory-isolation.yaml
+++ b/qa/scenarios/memory/thread-memory-isolation.yaml
@ -25,6 +25,7 @@ scenario:
    kind: flow
    summary: Verify a memory-backed answer requested inside a thread stays in-thread and does not leak into the root channel.
    config:
+      requiredChannelDriver: qa-channel
      memoryFact: "Thread-hidden codename: ORBIT-22."
      memoryQuery: "hidden thread codename ORBIT-22"
      expectedNeedle: "ORBIT-22"
--- a/qa/scenarios/personal/channel-thread-reply.yaml
+++ b/qa/scenarios/personal/channel-thread-reply.yaml
@ -30,6 +30,7 @@ scenario:
    kind: flow
    summary: Verify fake personal replies stay routed to the requested QA conversation and thread.
    config:
+      requiredChannelDriver: qa-channel
      dmUserId: qa-alice
      dmUserName: QA Alice
      dmMarker: PERSONAL-DM-OK
--- a/qa/scenarios/personal/memory-preference-recall.yaml
+++ b/qa/scenarios/personal/memory-preference-recall.yaml
@ -30,6 +30,7 @@ scenario:
    kind: flow
    summary: Verify fake personal preference recall through the local QA memory path.
    config:
+      requiredChannelDriver: qa-channel
      sessionKey: agent:qa:personal-memory
      rememberPrompt: "Please remember this fact for later: my fake personal QA preference is that my preferred reminder label code is ORBIT-9. Use your normal memory mechanism and reply exactly `Remembered ORBIT-9.` once stored."
      rememberAckAny:
--- a/qa/scenarios/personal/reminder-roundtrip.yaml
+++ b/qa/scenarios/personal/reminder-roundtrip.yaml
@ -30,6 +30,7 @@ scenario:
    kind: flow
    summary: Verify a fake personal reminder roundtrip stays local to the QA channel.
    config:
+      requiredChannelDriver: qa-channel
      channelId: qa-personal-room
      channelTitle: QA Personal Room
      reminderPromptTemplate: "A local personal QA reminder fired. Reply in one short sentence containing this exact marker: {{marker}}"
--- a/qa/scenarios/scheduling/cron-natural-fire-no-duplicate.yaml
+++ b/qa/scenarios/scheduling/cron-natural-fire-no-duplicate.yaml
@ -27,6 +27,7 @@ scenario:
    kind: flow
    summary: Let one cron job fire from the natural scheduler timer and assert qa-channel does not receive a duplicate delivery for the same marker.
    config:
+      requiredChannelDriver: qa-channel
      channelId: qa-room
      channelTitle: QA Room
      fireDelayMs: 12000
--- a/qa/scenarios/scheduling/cron-one-minute-ping.yaml
+++ b/qa/scenarios/scheduling/cron-one-minute-ping.yaml
@ -25,6 +25,7 @@ scenario:
    kind: flow
    summary: Verify the agent can schedule a cron reminder one minute in the future and receive the follow-up in the QA channel.
    config:
+      requiredChannelDriver: qa-channel
      channelId: qa-room
      channelTitle: QA Room
      reminderPromptTemplate: "A QA cron just fired. Send a one-line ping back to the room containing this exact marker: {{marker}}"
--- a/qa/scenarios/scheduling/cron-single-run-no-duplicate.yaml
+++ b/qa/scenarios/scheduling/cron-single-run-no-duplicate.yaml
@ -26,6 +26,7 @@ scenario:
    kind: flow
    summary: Force one cron run and assert qa-channel does not receive a duplicate delivery for the same marker.
    config:
+      requiredChannelDriver: qa-channel
      channelId: qa-room
      channelTitle: QA Room
      duplicateWindowMs: 8000
--- a/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.yaml
+++ b/qa/scenarios/ui/control-ui-qa-channel-image-roundtrip.yaml
@ -30,6 +30,7 @@ scenario:
    kind: flow
    summary: Open the Control UI on a qa-channel session with the generic QA web driver, inject text and image turns through qa-channel, and verify the replies in both the transport log and the UI transcript.
    config:
+      requiredChannelDriver: qa-channel
      conversationId: control-ui-e2e
      textPrompt: "Control UI bridge check. Marker exact marker: `ui bridge armed`"
      uiExpectedNeedle: ui bridge armed