diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 1a830aa7536..f70d8016255 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -10,6 +10,7 @@ const { runQaSuite, runQaCharacterEval, runQaMultipass, + runCanonicalLiveScenarios, listLiveTransportQaAdapterFactories, listTelegramQaScenarioCatalog, runTelegramQaLive, @@ -24,6 +25,7 @@ const { runQaSuite: vi.fn(), runQaCharacterEval: vi.fn(), runQaMultipass: vi.fn(), + runCanonicalLiveScenarios: vi.fn(), listLiveTransportQaAdapterFactories: vi.fn(), listTelegramQaScenarioCatalog: vi.fn(), runTelegramQaLive: vi.fn(), @@ -56,6 +58,15 @@ vi.mock("./live-transports/cli.js", () => ({ listLiveTransportQaAdapterFactories, })); +vi.mock("./live-transports/shared/canonical-scenarios.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCanonicalLiveScenarios, +})); + +vi.mock("./live-transports/telegram/adapter.runtime.js", () => ({ + createTelegramQaTransportAdapter: vi.fn(), +})); + vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({ listTelegramQaScenarioCatalog, runTelegramQaLive, @@ -228,6 +239,7 @@ describe("qa cli runtime", () => { runQaCharacterEval.mockReset(); runQaManualLane.mockReset(); runQaMultipass.mockReset(); + runCanonicalLiveScenarios.mockReset(); listLiveTransportQaAdapterFactories.mockReset(); listTelegramQaScenarioCatalog.mockReset(); runTelegramQaLive.mockReset(); @@ -280,13 +292,19 @@ describe("qa cli runtime", () => { observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"), scenarios: [], }); + runCanonicalLiveScenarios.mockResolvedValue({ + outputDir: telegramArtifactsDir, + reportPath: path.join(telegramArtifactsDir, "canonical-report.md"), + summaryPath: telegramSummaryPath, + scenarios: [], + }); listTelegramQaScenarioCatalog.mockReturnValue([ { - id: "telegram-status-command", - title: "Telegram status command reply", - defaultEnabled: true, - rationale: "status rationale", - regressionRefs: ["openclaw/openclaw#74698"], + id: "telegram-stream-final-single-message", + title: "Telegram streamed final reply", + defaultEnabled: false, + rationale: "stream rationale", + regressionRefs: [], }, ]); listLiveTransportQaAdapterFactories.mockReturnValue([ @@ -909,17 +927,22 @@ describe("qa cli runtime", () => { sutAccountId: "sut-live", }); - expect(runTelegramQaLive).toHaveBeenCalledWith({ - repoRoot: path.resolve("/tmp/openclaw-repo"), - outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"), - providerMode: "live-frontier", - primaryModel: "openai/gpt-5.5", - alternateModel: "openai/gpt-5.5", - fastMode: true, - allowFailures: undefined, - scenarioIds: ["telegram-help-command"], - sutAccountId: "sut-live", - }); + expect(runCanonicalLiveScenarios).toHaveBeenCalledWith( + expect.objectContaining({ + channelId: "telegram", + options: expect.objectContaining({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"), + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.5", + alternateModel: "openai/gpt-5.5", + fastMode: true, + sutAccountId: "sut-live", + }), + scenarioIds: ["telegram-help-command"], + }), + ); + expect(runTelegramQaLive).not.toHaveBeenCalled(); }); it("rejects output dirs that escape the repo root", () => { @@ -937,11 +960,40 @@ describe("qa cli runtime", () => { scenarioIds: ["telegram-help-command"], }); - expectFields(mockFirstObjectArg(runTelegramQaLive), { - repoRoot: path.resolve("/tmp/openclaw-repo"), - providerMode: "live-frontier", - allowFailures: undefined, + expect(runCanonicalLiveScenarios).toHaveBeenCalledWith( + expect.objectContaining({ + options: expect.objectContaining({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + providerMode: "live-frontier", + allowFailures: undefined, + }), + }), + ); + expect(runTelegramQaLive).not.toHaveBeenCalled(); + }); + + it("keeps remaining Telegram defaults when Commander supplies an empty scenario list", async () => { + await runQaTelegramCommand({ + repoRoot: "/tmp/openclaw-repo", + scenarioIds: [], }); + + expect(runCanonicalLiveScenarios).toHaveBeenCalled(); + expect(runTelegramQaLive).toHaveBeenCalledWith( + expect.objectContaining({ scenarioIds: undefined }), + ); + }); + + it("rejects unknown mixed Telegram selections before starting canonical scenarios", async () => { + await expect( + runQaTelegramCommand({ + repoRoot: "/tmp/openclaw-repo", + scenarioIds: ["telegram-help-command", "missing-telegram-scenario"], + }), + ).rejects.toThrow("unknown Telegram QA scenario id(s): missing-telegram-scenario"); + + expect(runCanonicalLiveScenarios).not.toHaveBeenCalled(); + expect(runTelegramQaLive).not.toHaveBeenCalled(); }); it("prints telegram scenario catalog without starting the live lane", async () => { @@ -952,10 +1004,11 @@ describe("qa cli runtime", () => { }); expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai"); + expect(runCanonicalLiveScenarios).not.toHaveBeenCalled(); expect(runTelegramQaLive).not.toHaveBeenCalled(); expectWriteContains( stdoutWrite, - "telegram-status-command\tdefault\tTelegram status command reply\tstatus rationale refs=openclaw/openclaw#74698", + "telegram-status-command\tdefault\tTelegram status command reply\tVerify Telegram status returns model, session, and activation details. refs=openclaw/openclaw#74698", ); }); diff --git a/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.test.ts new file mode 100644 index 00000000000..5c8994a91b5 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.test.ts @@ -0,0 +1,111 @@ +// Qa Lab tests cover canonical live-transport scenario delegation. +import { describe, expect, it, vi } from "vitest"; + +const { runQaFlowSuiteFromRuntime } = vi.hoisted(() => ({ + runQaFlowSuiteFromRuntime: vi.fn(), +})); + +vi.mock("../../suite-launch.runtime.js", () => ({ + runQaFlowSuiteFromRuntime, +})); +import { + assertKnownScenarioIds, + partitionCanonicalScenarioIds, + TELEGRAM_CANONICAL_SCENARIO_IDS, + TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, + WHATSAPP_CANONICAL_SCENARIO_IDS, + WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS, + whatsappDefaultCanonicalScenarioIds, + listCanonicalScenarios, + runCanonicalLiveScenarios, +} from "./canonical-scenarios.js"; +import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js"; + +describe("canonical live-transport scenarios", () => { + it("loads every migrated command and session-context scenario from YAML", () => { + const telegram = listCanonicalScenarios({ + ids: TELEGRAM_CANONICAL_SCENARIO_IDS, + defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, + }); + const whatsapp = listCanonicalScenarios({ + ids: WHATSAPP_CANONICAL_SCENARIO_IDS, + defaultIds: WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS, + }); + + expect(telegram.map(({ id }) => id).toSorted()).toEqual( + [...TELEGRAM_CANONICAL_SCENARIO_IDS].toSorted(), + ); + expect(whatsapp.map(({ id }) => id).toSorted()).toEqual( + [...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted(), + ); + expect(telegram.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual( + expect.arrayContaining([...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS]), + ); + expect(whatsapp.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual( + expect.arrayContaining([...WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS]), + ); + expect(whatsappDefaultCanonicalScenarioIds("live-frontier")).toEqual(["whatsapp-help-command"]); + expect(telegram.find(({ id }) => id === "telegram-status-command")?.regressionRefs).toEqual([ + "openclaw/openclaw#74698", + ]); + }); + + it("partitions canonical aliases from remaining imperative scenarios", () => { + expect( + partitionCanonicalScenarioIds( + ["telegram-help-command", "telegram-mentioned-message-reply"], + TELEGRAM_CANONICAL_SCENARIO_IDS, + ), + ).toEqual({ + canonical: ["telegram-help-command"], + legacy: ["telegram-mentioned-message-reply"], + }); + }); + + it("rejects unknown legacy ids before either live runner starts", () => { + expect(() => + assertKnownScenarioIds({ + ids: ["known", "missing"], + knownIds: ["known"], + laneLabel: "Demo", + }), + ).toThrow("unknown Demo QA scenario id(s): missing"); + }); + + it("runs canonical live aliases through the runtime lab launcher", async () => { + runQaFlowSuiteFromRuntime.mockResolvedValueOnce({ summaryPath: "/tmp/summary.json" }); + + await runCanonicalLiveScenarios({ + channelId: "telegram", + factory: { + id: "telegram", + matches: () => true, + create: vi.fn(), + }, + options: { + providerMode: "mock-openai", + repoRoot: "/tmp/openclaw-repo", + }, + scenarioIds: ["telegram-help-command"], + }); + + expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith( + expect.objectContaining({ + channelDriver: "live", + channelId: "telegram", + scenarioIds: ["telegram-help-command"], + }), + ); + }); + + it("removes migrated ids from non-YAML scenario ownership", async () => { + const nonYamlIds = new Set((await loadNonYamlScenarioRefs()).map(({ id }) => id)); + + for (const scenarioId of [ + ...TELEGRAM_CANONICAL_SCENARIO_IDS, + ...WHATSAPP_CANONICAL_SCENARIO_IDS, + ]) { + expect(nonYamlIds.has(scenarioId), scenarioId).toBe(false); + } + }); +}); diff --git a/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts new file mode 100644 index 00000000000..dbb29cef796 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/canonical-scenarios.ts @@ -0,0 +1,146 @@ +// Qa Lab plugin module defines canonical live-transport scenario delegation. +import path from "node:path"; +import type { QaTransportAdapterFactory } from "../../qa-transport-registry.js"; +import { readQaScenarioPack } from "../../scenario-catalog.js"; +import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js"; +import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js"; + +export const TELEGRAM_CANONICAL_SCENARIO_IDS = [ + "telegram-help-command", + "telegram-commands-command", + "telegram-tools-compact-command", + "telegram-whoami-command", + "telegram-status-command", + "telegram-repeated-command-authorization", + "telegram-context-command", + "telegram-current-session-status-tool", + "telegram-tool-only-usage-footer", + "telegram-reply-chain-exact-marker", +] as const; + +export const TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS = [ + "telegram-help-command", + "telegram-commands-command", + "telegram-tools-compact-command", + "telegram-whoami-command", + "telegram-status-command", + "telegram-repeated-command-authorization", + "telegram-context-command", +] as const; + +export const WHATSAPP_CANONICAL_SCENARIO_IDS = [ + "whatsapp-help-command", + "whatsapp-status-command", + "whatsapp-commands-command", + "whatsapp-tools-compact-command", + "whatsapp-whoami-command", + "whatsapp-context-command", + "whatsapp-tool-only-usage-footer", + "whatsapp-native-new-command", +] as const; + +export const WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS = ["whatsapp-help-command"] as const; + +export const WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS = [ + "whatsapp-help-command", + "whatsapp-commands-command", + "whatsapp-tools-compact-command", + "whatsapp-whoami-command", + "whatsapp-context-command", + "whatsapp-tool-only-usage-footer", + "whatsapp-native-new-command", +] as const; + +export function whatsappDefaultCanonicalScenarioIds(providerMode: string) { + return providerMode === "mock-openai" + ? [...WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS] + : [...WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS]; +} + +export type CanonicalScenarioPartition = { + canonical: string[]; + legacy: string[]; +}; + +export function assertKnownScenarioIds(params: { + ids: readonly string[]; + knownIds: readonly string[]; + laneLabel: string; +}) { + const knownIds = new Set(params.knownIds); + const missingIds = params.ids.filter((id) => !knownIds.has(id)); + if (missingIds.length > 0) { + throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`); + } +} + +export function partitionCanonicalScenarioIds( + scenarioIds: readonly string[] | undefined, + canonicalIds: readonly string[], +): CanonicalScenarioPartition { + const canonicalSet = new Set(canonicalIds); + const canonical: string[] = []; + const legacy: string[] = []; + for (const scenarioId of scenarioIds ?? []) { + (canonicalSet.has(scenarioId) ? canonical : legacy).push(scenarioId); + } + return { canonical, legacy }; +} + +export function listCanonicalScenarios(params: { + ids: readonly string[]; + defaultIds: readonly string[]; +}) { + const requestedIds = new Set(params.ids); + const defaultIds = new Set(params.defaultIds); + return readQaScenarioPack() + .scenarios.filter((scenario) => requestedIds.has(scenario.id)) + .map((scenario) => ({ + id: scenario.id, + title: scenario.title, + rationale: scenario.objective, + regressionRefs: scenario.regressionRefs ?? [], + defaultEnabled: defaultIds.has(scenario.id), + })); +} + +export async function runCanonicalLiveScenarios(params: { + channelId: string; + factory: QaTransportAdapterFactory; + options: LiveTransportQaCommandOptions & { + providerMode: "mock-openai" | "aimock" | "live-frontier"; + repoRoot: string; + }; + scenarioIds: string[]; +}) { + return await runQaFlowSuiteFromRuntime({ + adapterFactories: [params.factory], + adapterOptions: { + repoRoot: params.options.repoRoot, + ...(params.options.credentialRole ? { credentialRole: params.options.credentialRole } : {}), + ...(params.options.credentialSource + ? { credentialSource: params.options.credentialSource } + : {}), + ...(params.options.sutAccountId ? { sutAccountId: params.options.sutAccountId } : {}), + }, + ...(params.options.alternateModel ? { alternateModel: params.options.alternateModel } : {}), + channelDriver: "live", + channelId: params.channelId, + concurrency: 1, + ...(params.options.fastMode !== undefined ? { fastMode: params.options.fastMode } : {}), + ...(params.options.outputDir ? { outputDir: params.options.outputDir } : {}), + ...(params.options.primaryModel ? { primaryModel: params.options.primaryModel } : {}), + providerMode: params.options.providerMode, + repoRoot: params.options.repoRoot, + scenarioIds: params.scenarioIds, + }); +} + +export function canonicalScenarioOutputDir( + options: LiveTransportQaCommandOptions, + includesLegacyRun: boolean, +) { + return includesLegacyRun && options.outputDir + ? path.join(options.outputDir, "canonical") + : options.outputDir; +} diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts index 095bc7cbb3f..55d441ad931 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-scenarios.test.ts @@ -151,10 +151,16 @@ describe("live transport scenario helpers", () => { expect.arrayContaining(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS), ); expect(lanes.get("telegram")).toEqual( - expect.arrayContaining(telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS), + expect.arrayContaining([ + "help-command", + ...telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS, + ]), ); expect(lanes.get("whatsapp")).toEqual( - expect.arrayContaining(whatsAppTesting.WHATSAPP_QA_STANDARD_SCENARIO_IDS), + expect.arrayContaining([ + "help-command", + ...whatsAppTesting.WHATSAPP_QA_STANDARD_SCENARIO_IDS, + ]), ); }); }); diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts index eaf9513fcd1..81f6c76b1ac 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts @@ -1,14 +1,31 @@ import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js"; +import { + assertKnownScenarioIds, + canonicalScenarioOutputDir, + listCanonicalScenarios, + partitionCanonicalScenarioIds, + runCanonicalLiveScenarios, + TELEGRAM_CANONICAL_SCENARIO_IDS, + TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, +} from "../shared/canonical-scenarios.js"; // Qa Lab plugin module implements cli behavior. import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js"; import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; +import { createTelegramQaTransportAdapter } from "./adapter.runtime.js"; import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js"; export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) { const runOptions = resolveLiveTransportQaRunOptions(opts); if (runOptions.listScenarios) { - for (const scenario of listTelegramQaScenarioCatalog(runOptions.providerMode)) { + const scenarios = [ + ...listCanonicalScenarios({ + ids: TELEGRAM_CANONICAL_SCENARIO_IDS, + defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS, + }), + ...listTelegramQaScenarioCatalog(runOptions.providerMode), + ]; + for (const scenario of scenarios) { const defaultLabel = scenario.defaultEnabled ? "default" : "optional"; const refs = scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : ""; @@ -18,7 +35,56 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) } return; } - const result = await runTelegramQaLive(runOptions); + const selected = partitionCanonicalScenarioIds( + runOptions.scenarioIds, + TELEGRAM_CANONICAL_SCENARIO_IDS, + ); + const hasExplicitScenarioIds = (runOptions.scenarioIds?.length ?? 0) > 0; + if (hasExplicitScenarioIds) { + assertKnownScenarioIds({ + ids: selected.legacy, + knownIds: listTelegramQaScenarioCatalog(runOptions.providerMode).map(({ id }) => id), + laneLabel: "Telegram", + }); + } + const canonicalScenarioIds = hasExplicitScenarioIds + ? selected.canonical + : [...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS]; + const runsLegacyScenarios = !hasExplicitScenarioIds || selected.legacy.length > 0; + if (canonicalScenarioIds.length > 0) { + const canonical = await runCanonicalLiveScenarios({ + channelId: "telegram", + factory: { + id: "telegram", + matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram", + create: createTelegramQaTransportAdapter, + }, + options: { + ...runOptions, + outputDir: canonicalScenarioOutputDir(runOptions, runsLegacyScenarios), + }, + scenarioIds: canonicalScenarioIds, + }); + printLiveTransportQaArtifacts("Telegram canonical QA", { + report: canonical.reportPath, + summary: canonical.summaryPath, + }); + if (!runOptions.allowFailures) { + const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile( + canonical.summaryPath, + ); + if (failedScenarioCount > 0) { + process.exitCode = 1; + } + } + } + if (!runsLegacyScenarios) { + return; + } + const result = await runTelegramQaLive({ + ...runOptions, + scenarioIds: hasExplicitScenarioIds ? selected.legacy : undefined, + }); printLiveTransportQaArtifacts("Telegram QA", { report: result.reportPath, summary: result.summaryPath, diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index d20ff2615a4..fdbf9ab1bc1 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -1,3 +1,4 @@ +import { TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS } from "../shared/canonical-scenarios.js"; // Qa Lab plugin module implements cli behavior. import { createLazyCliRuntimeLoader, @@ -24,7 +25,7 @@ export const telegramQaAdapterFactory: NonNullable< LiveTransportQaCliRegistration["adapterFactory"] > = { id: "telegram", - scenarioIds: ["channel-chat-baseline"], + scenarioIds: ["channel-chat-baseline", ...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS], matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram", async create(context) { return await (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context); diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts index ff5fa8a87ac..2bdcd563d70 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts @@ -622,9 +622,9 @@ describe("telegram live qa runtime", () => { }); it("fails when any requested Telegram scenario id is unknown", () => { - expect(() => testing.findScenario(["telegram-help-command", "typo-scenario"])).toThrow( - "unknown Telegram QA scenario id(s): typo-scenario", - ); + expect(() => + testing.findScenario(["telegram-mentioned-message-reply", "typo-scenario"]), + ).toThrow("unknown Telegram QA scenario id(s): typo-scenario"); }); it("recognizes Telegram observation timeouts with retry details", () => { @@ -646,98 +646,26 @@ describe("telegram live qa runtime", () => { it("includes mention gating in the Telegram live scenario catalog", () => { const scenarios = testing.findScenario([ - "telegram-help-command", - "telegram-commands-command", - "telegram-tools-compact-command", - "telegram-whoami-command", - "telegram-status-command", - "telegram-repeated-command-authorization", "telegram-other-bot-command-gating", - "telegram-context-command", - "telegram-current-session-status-tool", - "telegram-tool-only-usage-footer", "telegram-mentioned-message-reply", - "telegram-reply-chain-exact-marker", "telegram-stream-final-single-message", "telegram-long-final-reuses-preview", "telegram-long-final-three-chunks", "telegram-mention-gating", ]); expect(scenarios.map((scenario) => scenario.id)).toEqual([ - "telegram-help-command", - "telegram-commands-command", - "telegram-tools-compact-command", - "telegram-whoami-command", - "telegram-status-command", - "telegram-repeated-command-authorization", "telegram-other-bot-command-gating", - "telegram-context-command", - "telegram-current-session-status-tool", - "telegram-tool-only-usage-footer", "telegram-mentioned-message-reply", - "telegram-reply-chain-exact-marker", "telegram-stream-final-single-message", "telegram-long-final-reuses-preview", "telegram-long-final-three-chunks", "telegram-mention-gating", ]); - expect( - scenarios.find((scenario) => scenario.id === "telegram-status-command")?.buildRun("sut_bot") - .steps[0].input, - ).toBe("/status@sut_bot"); - expect( - scenarios.find((scenario) => scenario.id === "telegram-status-command")?.buildRun("sut_bot") - .steps[0].expectedTextIncludes, - ).toEqual(["OpenClaw", "Model:", "Session:", "Activation:"]); - expect( - scenarios - .find((scenario) => scenario.id === "telegram-repeated-command-authorization") - ?.buildRun("sut_bot").steps, - ).toHaveLength(4); - const repeatedSteps = requireScenario( - scenarios, - "telegram-repeated-command-authorization", - ).buildRun("sut_bot").steps; - expect(repeatedSteps[0]?.driverGroupAuthorization).toBe("deny"); - expect(repeatedSteps[0]?.input).toBe("/status@sut_bot"); - expect(repeatedSteps[0]?.expectReply).toBe(false); - expect(repeatedSteps[1]?.driverGroupAuthorization).toBe("allow"); - expect(repeatedSteps[1]?.input).toBe("/status@sut_bot"); - expect(repeatedSteps[1]?.expectReply).toBe(true); - expect(repeatedSteps[2]?.input).toBe("/help@sut_bot"); - expect(repeatedSteps[2]?.expectReply).toBe(true); - expect(repeatedSteps[3]?.input).toBe("/commands@sut_bot"); - expect(repeatedSteps[3]?.expectReply).toBe(true); - expect(repeatedSteps[3]?.expectedTextIncludes).toEqual(["Commands (1/", "/session", "/stop"]); - const commandsStep = requireScenario(scenarios, "telegram-commands-command").buildRun("sut_bot") - .steps[0]; - expect(commandsStep?.expectedTextIncludes).toEqual(["Commands (1/", "/session", "/stop"]); const otherBotStep = requireScenario(scenarios, "telegram-other-bot-command-gating").buildRun( "sut_bot", ).steps[0]; expect(otherBotStep?.expectReply).toBe(false); expect(otherBotStep?.input).toBe("/status@OpenClawQaOtherBot"); - const contextStep = requireScenario(scenarios, "telegram-context-command").buildRun("sut_bot") - .steps[0]; - expect(contextStep?.matchText).toBe("/context list"); - const statusToolStep = requireScenario( - scenarios, - "telegram-current-session-status-tool", - ).buildRun("sut_bot").steps[0]; - expect(statusToolStep?.expectedTextIncludes).toEqual([ - "QA-TELEGRAM-CURRENT-SESSION-OK", - ":telegram:group:", - ]); - expect(statusToolStep?.replyToLatestSutMessage).toBe(true); - const usageFooterSteps = requireScenario(scenarios, "telegram-tool-only-usage-footer").buildRun( - "sut_bot", - ).steps; - expect(usageFooterSteps).toHaveLength(2); - expect(usageFooterSteps[0]?.input).toBe("/usage@sut_bot tokens"); - expect(usageFooterSteps[0]?.expectedTextIncludes).toEqual(["Usage", "tokens"]); - expect(usageFooterSteps[1]?.expectedTextIncludes?.at(-1)).toBe("Usage:"); - expect(usageFooterSteps[1]?.expectedSutMessageCount).toBe(2); - expect(usageFooterSteps[1]?.replyToLatestSutMessage).toBe(true); expect( scenarios .find((scenario) => scenario.id === "telegram-mentioned-message-reply") @@ -747,12 +675,6 @@ describe("telegram live qa runtime", () => { scenarios.find((scenario) => scenario.id === "telegram-mentioned-message-reply") ?.evidenceCoverageIds, ).toEqual(["channels.telegram.mention-gating"]); - const replyChainStep = requireScenario(scenarios, "telegram-reply-chain-exact-marker").buildRun( - "sut_bot", - ).steps[0]; - expect(replyChainStep?.expectedJoinedSutTextIncludes).toEqual(["QA-TELEGRAM-REPLY-CHAIN-OK"]); - expect(replyChainStep?.expectedSutMessageCount).toBe(1); - expect(replyChainStep?.replyToLatestSutMessage).toBeUndefined(); const streamSingleStep = requireScenario( scenarios, "telegram-stream-final-single-message", @@ -787,14 +709,7 @@ describe("telegram live qa runtime", () => { it("keeps mock-scripted Telegram checks out of the default live-frontier set", () => { expect(testing.findScenario(undefined, "live-frontier").map((scenario) => scenario.id)).toEqual( [ - "telegram-help-command", - "telegram-commands-command", - "telegram-tools-compact-command", - "telegram-whoami-command", - "telegram-status-command", - "telegram-repeated-command-authorization", "telegram-other-bot-command-gating", - "telegram-context-command", "telegram-mentioned-message-reply", "telegram-mention-gating", ], @@ -803,45 +718,22 @@ describe("telegram live qa runtime", () => { it("adds deterministic model-scripted checks to the default mock-openai set", () => { expect(testing.findScenario(undefined, "mock-openai").map((scenario) => scenario.id)).toEqual([ - "telegram-help-command", - "telegram-commands-command", - "telegram-tools-compact-command", - "telegram-whoami-command", - "telegram-status-command", - "telegram-repeated-command-authorization", "telegram-other-bot-command-gating", - "telegram-context-command", "telegram-mentioned-message-reply", "telegram-long-final-reuses-preview", "telegram-mention-gating", ]); }); - it("lists default status and regression refs in the Telegram scenario catalog", () => { + it("lists remaining optional Telegram scenario metadata", () => { const catalog = testing.listTelegramQaScenarioCatalog("mock-openai"); - const status = requireScenario(catalog, "telegram-status-command"); - expect(status.defaultEnabled).toBe(true); - expect(status.regressionRefs).toEqual(["openclaw/openclaw#74698"]); - expect(requireScenario(catalog, "telegram-current-session-status-tool").defaultEnabled).toBe( - false, - ); - const usageFooter = requireScenario(catalog, "telegram-tool-only-usage-footer"); - expect(usageFooter.defaultEnabled).toBe(false); - expect(usageFooter.regressionRefs).toEqual(["openclaw/openclaw#87392"]); const streamSingle = requireScenario(catalog, "telegram-stream-final-single-message"); expect(streamSingle.defaultEnabled).toBe(false); expect(streamSingle.regressionRefs).toEqual(["openclaw/openclaw#39905"]); - expect(requireScenario(catalog, "telegram-reply-chain-exact-marker").defaultEnabled).toBe( - false, - ); }); it("tracks Telegram live coverage against the shared transport contract", () => { - expect(testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS).toEqual([ - "canary", - "help-command", - "mention-gating", - ]); + expect(testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]); expect( findMissingLiveTransportStandardScenarios({ coveredStandardScenarioIds: testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS, @@ -1317,8 +1209,8 @@ describe("telegram live qa runtime", () => { initialOffset: 7, timeoutMs: 5_000, observedMessages, - observationScenarioId: "telegram-whoami-command", - observationScenarioTitle: "Telegram whoami reply", + observationScenarioId: "telegram-mentioned-message-reply", + observationScenarioTitle: "Telegram mentioned message gets a reply", predicate: (message) => testing.matchesTelegramScenarioReply({ groupId: "-100123", @@ -1334,7 +1226,7 @@ describe("telegram live qa runtime", () => { expect(observedMessages).toHaveLength(1); expect(observedMessages[0]?.matchedScenario).toBe(true); expect(observedMessages[0]?.messageId).toBe(99); - expect(observedMessages[0]?.scenarioId).toBe("telegram-whoami-command"); + expect(observedMessages[0]?.scenarioId).toBe("telegram-mentioned-message-reply"); }); it("prints Telegram scenario RTT in the Markdown report", () => { diff --git a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts index a190c24dab4..ca79166b47b 100644 --- a/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts @@ -70,26 +70,15 @@ type TelegramBotIdentity = { }; type TelegramQaScenarioId = - | "telegram-help-command" - | "telegram-commands-command" - | "telegram-tools-compact-command" - | "telegram-whoami-command" - | "telegram-status-command" - | "telegram-repeated-command-authorization" | "telegram-other-bot-command-gating" - | "telegram-context-command" - | "telegram-current-session-status-tool" - | "telegram-tool-only-usage-footer" | "telegram-stream-final-single-message" | "telegram-long-final-three-chunks" | "telegram-long-final-reuses-preview" - | "telegram-reply-chain-exact-marker" | "telegram-mentioned-message-reply" | "telegram-mention-gating"; type TelegramQaScenarioStep = { allowAnySutReply?: boolean; - driverGroupAuthorization?: "allow" | "deny"; expectReply: boolean; input: string; expectedTextIncludes?: string[]; @@ -221,18 +210,18 @@ type TelegramRichMessage = { type TelegramMessage = Pick & Partial> & { - audio?: unknown; - chat: { id: number }; - document?: unknown; - from?: Pick, "id" | "is_bot" | "username">; - photo?: unknown[]; - rich_message?: TelegramRichMessage; - reply_markup?: TelegramReplyMarkup; - reply_to_message?: { message_id?: number }; - sticker?: unknown; - video?: unknown; - voice?: unknown; -}; + audio?: unknown; + chat: { id: number }; + document?: unknown; + from?: Pick, "id" | "is_bot" | "username">; + photo?: unknown[]; + rich_message?: TelegramRichMessage; + reply_markup?: TelegramReplyMarkup; + reply_to_message?: { message_id?: number }; + sticker?: unknown; + video?: unknown; + voice?: unknown; + }; type TelegramUpdate = Pick & { edited_message?: TelegramMessage; @@ -251,102 +240,6 @@ function telegramQaStepRun(step: TelegramQaScenarioStep): TelegramQaScenarioRun } const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [ - { - id: "telegram-help-command", - standardId: "help-command", - title: "Telegram help command reply", - rationale: "Canary-grade native command reply path.", - timeoutMs: 45_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `/help@${sutUsername}`, - expectedTextIncludes: ["/new", "/commands for full list"], - }), - }, - { - id: "telegram-commands-command", - title: "Telegram commands list reply", - rationale: "Native command catalog must render in Telegram group replies.", - timeoutMs: 45_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `/commands@${sutUsername}`, - expectedTextIncludes: ["Commands (1/", "/session", "/stop"], - }), - }, - { - id: "telegram-tools-compact-command", - title: "Telegram tools compact reply", - rationale: "Tool catalog rendering catches command dispatch plus model-tool inventory drift.", - timeoutMs: 45_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `/tools@${sutUsername} compact`, - expectedTextIncludes: ["exec", "Use /tools verbose for descriptions."], - }), - }, - { - id: "telegram-whoami-command", - title: "Telegram whoami reply", - rationale: "Identity command proves Telegram channel context is attached to native commands.", - timeoutMs: 45_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `/whoami@${sutUsername}`, - expectedTextIncludes: ["🧭 Identity", "Channel: telegram"], - }), - }, - { - id: "telegram-status-command", - title: "Telegram status command reply", - rationale: "Recent Telegram group regressions broke /status while normal chat still worked.", - regressionRefs: ["openclaw/openclaw#74698"], - timeoutMs: 45_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `/status@${sutUsername}`, - expectedTextIncludes: ["OpenClaw", "Model:", "Session:", "Activation:"], - }), - }, - { - id: "telegram-repeated-command-authorization", - title: "Telegram repeated command authorization", - rationale: - "Allowlisted bot-to-bot operators should not hit a fresh auth gate for each native slash command.", - timeoutMs: 45_000, - buildRun: (sutUsername) => { - const steps = [ - { - driverGroupAuthorization: "deny", - expectReply: false, - input: `/status@${sutUsername}`, - timeoutMs: 8_000, - }, - { - driverGroupAuthorization: "allow", - expectReply: true, - input: `/status@${sutUsername}`, - expectedTextIncludes: ["OpenClaw", "Session:"], - }, - { - expectReply: true, - input: `/help@${sutUsername}`, - expectedTextIncludes: ["/new", "/commands for full list"], - }, - { - expectReply: true, - input: `/commands@${sutUsername}`, - expectedTextIncludes: ["Commands (1/", "/session", "/stop"], - }, - ] satisfies TelegramQaScenarioStep[]; - return { steps }; - }, - }, { id: "telegram-other-bot-command-gating", title: "Telegram command addressed to another bot is ignored", @@ -358,65 +251,6 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [ input: "/status@OpenClawQaOtherBot", }), }, - { - id: "telegram-context-command", - title: "Telegram context reply", - rationale: "Context command exercises native command routing into Telegram-specific help text.", - timeoutMs: 45_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `/context@${sutUsername}`, - matchText: "/context list", - expectedTextIncludes: ["/context list", "Inline shortcut"], - }), - }, - { - id: "telegram-current-session-status-tool", - title: "Telegram current session_status tool call", - defaultEnabled: false, - rationale: - "Opt-in threaded probe for current Telegram group session resolution through model tools.", - timeoutMs: 60_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `@${sutUsername} Telegram current session_status QA check. Call session_status with sessionKey set to current, then reply with the exact QA marker and resolved session key.`, - expectedTextIncludes: ["QA-TELEGRAM-CURRENT-SESSION-OK", ":telegram:group:"], - replyToLatestSutMessage: true, - }), - }, - { - id: "telegram-tool-only-usage-footer", - title: "Telegram tool-only reply includes usage footer", - defaultEnabled: false, - rationale: - "Opt-in real Telegram proof that /usage tokens decorates message-tool-only visible replies.", - regressionRefs: ["openclaw/openclaw#87392"], - timeoutMs: 90_000, - buildRun: (sutUsername) => { - const marker = `QA-TELEGRAM-USAGE-FOOTER-${randomUUID().slice(0, 8).toUpperCase()}`; - return { - steps: [ - { - expectReply: true, - input: `/usage@${sutUsername} tokens`, - expectedTextIncludes: ["Usage", "tokens"], - }, - { - allowAnySutReply: true, - expectReply: true, - input: `@${sutUsername} Telegram usage footer QA. Reply exactly: ${marker}`, - expectedTextIncludes: [marker, "Usage:"], - expectedJoinedSutTextIncludes: [marker, "Usage:"], - expectedSutMessageCount: 2, - replyToLatestSutMessage: true, - settleMs: 4_000, - }, - ], - }; - }, - }, { id: "telegram-mentioned-message-reply", title: "Telegram mentioned message gets a reply", @@ -440,24 +274,6 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [ }); }, }, - { - id: "telegram-reply-chain-exact-marker", - title: "Telegram reply-chain exact marker", - defaultEnabled: false, - defaultProviderModes: ["mock-openai"], - rationale: - "Opt-in mock-backed exact-marker check for Telegram final text through reply handling.", - timeoutMs: 75_000, - buildRun: (sutUsername) => - telegramQaStepRun({ - expectReply: true, - input: `@${sutUsername} Telegram reply-chain marker QA. Reply exactly: QA-TELEGRAM-REPLY-CHAIN-OK`, - expectedTextIncludes: ["QA-TELEGRAM-REPLY-CHAIN-OK"], - expectedJoinedSutTextIncludes: ["QA-TELEGRAM-REPLY-CHAIN-OK"], - expectedSutMessageCount: 1, - settleMs: 4_000, - }), - }, { id: "telegram-stream-final-single-message", title: "Telegram streamed final stays one message", @@ -564,14 +380,6 @@ function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof TELEGRAM_QA_ENV_KE return value; } -function readConfigRecord(root: Record, key: string): Record { - const value = root[key]; - if (!isRecord(value)) { - throw new Error(`Telegram QA config missing object at ${key}`); - } - return value; -} - function shouldLogTelegramQaLiveProgress(env: NodeJS.ProcessEnv = process.env) { const override = parseTelegramQaProgressBooleanEnv(env[QA_SUITE_PROGRESS_ENV]); if (override !== undefined) { @@ -1280,34 +1088,6 @@ async function waitForTelegramChannelRunning( throw new Error(`telegram account "${accountId}" did not become ready${details}`); } -async function setTelegramQaDriverGroupAuthorization(params: { - driverBotId: number; - env: NodeJS.ProcessEnv; - gateway: Awaited>; - groupId: string; - sutAccountId: string; - authorized: boolean; -}) { - await params.gateway.restartAfterStateMutation(async ({ configPath }) => { - const parsed: unknown = JSON.parse(await fs.readFile(configPath, "utf8")); - if (!isRecord(parsed)) { - throw new Error("Telegram QA config root must be an object"); - } - const channels = readConfigRecord(parsed, "channels"); - const telegram = readConfigRecord(channels, "telegram"); - const accounts = readConfigRecord(telegram, "accounts"); - const account = readConfigRecord(accounts, params.sutAccountId); - const groups = readConfigRecord(account, "groups"); - const group = readConfigRecord(groups, params.groupId); - group.allowFrom = params.authorized ? [String(params.driverBotId)] : []; - await fs.writeFile(configPath, `${JSON.stringify(parsed, null, 2)}\n`, { - encoding: "utf8", - mode: 0o600, - }); - }); - await waitForTelegramChannelRunning(params.gateway, params.sutAccountId, { env: params.env }); -} - function renderTelegramQaMarkdown(params: { cleanupIssues: string[]; credentialSource: "convex" | "env"; @@ -1975,17 +1755,6 @@ export async function runTelegramQaLive(params: { let lastMatched: Awaited> | undefined; let lastSentMessageId: number | undefined; for (const step of scenarioSteps) { - if (step.driverGroupAuthorization) { - await setTelegramQaDriverGroupAuthorization({ - driverBotId: driverIdentity.id, - env, - gateway: gatewayHarness.gateway, - groupId: runtimeEnv.groupId, - sutAccountId, - authorized: step.driverGroupAuthorization === "allow", - }); - driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken); - } driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken); const stepResult = await runTelegramQaScenarioStep({ driverOffset, diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts index be43be164c2..290c7af929b 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts @@ -2,16 +2,26 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { QA_EVIDENCE_FILENAME } from "../../evidence-summary.js"; import { runQaWhatsAppCommand } from "./cli.runtime.js"; -const runWhatsAppQaLiveMock = vi.hoisted(() => vi.fn()); +const { listWhatsAppQaScenarioCatalogMock, runCanonicalLiveScenariosMock, runWhatsAppQaLiveMock } = + vi.hoisted(() => ({ + listWhatsAppQaScenarioCatalogMock: vi.fn(), + runCanonicalLiveScenariosMock: vi.fn(), + runWhatsAppQaLiveMock: vi.fn(), + })); vi.mock("../shared/live-artifacts.js", () => ({ printLiveTransportQaArtifacts: vi.fn(), })); +vi.mock("../shared/canonical-scenarios.js", async (importOriginal) => ({ + ...(await importOriginal()), + runCanonicalLiveScenarios: runCanonicalLiveScenariosMock, +})); + vi.mock("../shared/live-transport-cli.runtime.js", () => ({ resolveLiveTransportQaRunOptions: (opts: Record) => ({ outputDir: opts.repoRoot, @@ -22,15 +32,26 @@ vi.mock("../shared/live-transport-cli.runtime.js", () => ({ })); vi.mock("./whatsapp-live.runtime.js", () => ({ + listWhatsAppQaScenarioCatalog: listWhatsAppQaScenarioCatalogMock, runWhatsAppQaLive: runWhatsAppQaLiveMock, })); +vi.mock("./adapter.runtime.js", () => ({ + createWhatsAppQaTransportAdapter: vi.fn(), +})); + const tempDirs: string[] = []; let originalExitCode: typeof process.exitCode; +beforeEach(() => { + listWhatsAppQaScenarioCatalogMock.mockReturnValue([{ id: "whatsapp-canary" }]); +}); + afterEach(async () => { process.exitCode = originalExitCode; + runCanonicalLiveScenariosMock.mockReset(); runWhatsAppQaLiveMock.mockReset(); + listWhatsAppQaScenarioCatalogMock.mockReset(); for (const dir of tempDirs.splice(0)) { await fs.rm(dir, { recursive: true, force: true }); } @@ -88,6 +109,12 @@ describe("WhatsApp QA CLI runtime", () => { scenarios: [], summaryPath, }); + runCanonicalLiveScenariosMock.mockResolvedValueOnce({ + outputDir, + reportPath: path.join(outputDir, "canonical-report.md"), + scenarios: [], + summaryPath, + }); await runQaWhatsAppCommand({ repoRoot: outputDir }); @@ -104,9 +131,75 @@ describe("WhatsApp QA CLI runtime", () => { scenarios: [], summaryPath, }); + runCanonicalLiveScenariosMock.mockResolvedValueOnce({ + outputDir, + reportPath: path.join(outputDir, "canonical-report.md"), + scenarios: [], + summaryPath, + }); await runQaWhatsAppCommand({ allowFailures: true, repoRoot: outputDir }); expect(process.exitCode).toBeUndefined(); }); + + it("delegates canonical WhatsApp scenario ids without starting the legacy runner", async () => { + originalExitCode = process.exitCode; + process.exitCode = undefined; + const { outputDir, summaryPath } = await writeSummary(makeEvidenceSummary("pass")); + runCanonicalLiveScenariosMock.mockResolvedValueOnce({ + outputDir, + reportPath: path.join(outputDir, "canonical-report.md"), + scenarios: [], + summaryPath, + }); + + await runQaWhatsAppCommand({ + repoRoot: outputDir, + scenarioIds: ["whatsapp-help-command"], + }); + + expect(runCanonicalLiveScenariosMock).toHaveBeenCalledWith( + expect.objectContaining({ + channelId: "whatsapp", + scenarioIds: ["whatsapp-help-command"], + }), + ); + expect(runWhatsAppQaLiveMock).not.toHaveBeenCalled(); + }); + + it("keeps remaining WhatsApp defaults when Commander supplies an empty scenario list", async () => { + const { outputDir, summaryPath } = await writeSummary(makeEvidenceSummary("pass")); + runWhatsAppQaLiveMock.mockResolvedValueOnce({ + observedMessagesPath: path.join(outputDir, "observed.json"), + reportPath: path.join(outputDir, "report.md"), + scenarios: [], + summaryPath, + }); + runCanonicalLiveScenariosMock.mockResolvedValueOnce({ + outputDir, + reportPath: path.join(outputDir, "canonical-report.md"), + scenarios: [], + summaryPath, + }); + + await runQaWhatsAppCommand({ repoRoot: outputDir, scenarioIds: [] }); + + expect(runCanonicalLiveScenariosMock).toHaveBeenCalled(); + expect(runWhatsAppQaLiveMock).toHaveBeenCalledWith( + expect.objectContaining({ scenarioIds: undefined }), + ); + }); + + it("rejects unknown mixed WhatsApp selections before starting canonical scenarios", async () => { + await expect( + runQaWhatsAppCommand({ + repoRoot: "/tmp/openclaw-repo", + scenarioIds: ["whatsapp-help-command", "missing-whatsapp-scenario"], + }), + ).rejects.toThrow("unknown WhatsApp QA scenario id(s): missing-whatsapp-scenario"); + + expect(runCanonicalLiveScenariosMock).not.toHaveBeenCalled(); + expect(runWhatsAppQaLiveMock).not.toHaveBeenCalled(); + }); }); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts index 457e234f7e8..53288223d28 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts @@ -1,13 +1,71 @@ import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "../../suite-summary.js"; +import { + assertKnownScenarioIds, + canonicalScenarioOutputDir, + partitionCanonicalScenarioIds, + runCanonicalLiveScenarios, + WHATSAPP_CANONICAL_SCENARIO_IDS, + whatsappDefaultCanonicalScenarioIds, +} from "../shared/canonical-scenarios.js"; // Qa Lab plugin module implements cli behavior. import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js"; import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; -import { runWhatsAppQaLive } from "./whatsapp-live.runtime.js"; +import { createWhatsAppQaTransportAdapter } from "./adapter.runtime.js"; +import { listWhatsAppQaScenarioCatalog, runWhatsAppQaLive } from "./whatsapp-live.runtime.js"; export async function runQaWhatsAppCommand(opts: LiveTransportQaCommandOptions) { const runOptions = resolveLiveTransportQaRunOptions(opts); - const result = await runWhatsAppQaLive(runOptions); + const selected = partitionCanonicalScenarioIds( + runOptions.scenarioIds, + WHATSAPP_CANONICAL_SCENARIO_IDS, + ); + const hasExplicitScenarioIds = (runOptions.scenarioIds?.length ?? 0) > 0; + if (hasExplicitScenarioIds) { + assertKnownScenarioIds({ + ids: selected.legacy, + knownIds: listWhatsAppQaScenarioCatalog().map(({ id }) => id), + laneLabel: "WhatsApp", + }); + } + const canonicalScenarioIds = hasExplicitScenarioIds + ? selected.canonical + : whatsappDefaultCanonicalScenarioIds(runOptions.providerMode); + const runsLegacyScenarios = !hasExplicitScenarioIds || selected.legacy.length > 0; + if (canonicalScenarioIds.length > 0) { + const canonical = await runCanonicalLiveScenarios({ + channelId: "whatsapp", + factory: { + id: "whatsapp", + matches: ({ channelId, driver }) => driver === "live" && channelId === "whatsapp", + create: createWhatsAppQaTransportAdapter, + }, + options: { + ...runOptions, + outputDir: canonicalScenarioOutputDir(runOptions, runsLegacyScenarios), + }, + scenarioIds: canonicalScenarioIds, + }); + printLiveTransportQaArtifacts("WhatsApp canonical QA", { + report: canonical.reportPath, + summary: canonical.summaryPath, + }); + if (!runOptions.allowFailures) { + const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile( + canonical.summaryPath, + ); + if (blockingScenarioCount > 0) { + process.exitCode = 1; + } + } + } + if (!runsLegacyScenarios) { + return; + } + const result = await runWhatsAppQaLive({ + ...runOptions, + scenarioIds: hasExplicitScenarioIds ? selected.legacy : undefined, + }); printLiveTransportQaArtifacts("WhatsApp QA", { report: result.reportPath, summary: result.summaryPath, diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts index a28b4cf4a93..1c2fd03b1f6 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts @@ -1,3 +1,4 @@ +import { WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS } from "../shared/canonical-scenarios.js"; // Qa Lab plugin module implements cli behavior. import { createLazyCliRuntimeLoader, @@ -24,7 +25,7 @@ export const whatsappQaAdapterFactory: NonNullable< LiveTransportQaCliRegistration["adapterFactory"] > = { id: "whatsapp", - scenarioIds: ["dm-chat-baseline"], + scenarioIds: ["dm-chat-baseline", ...WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS], matches: ({ channelId, driver }) => driver === "live" && channelId === "whatsapp", async create(context) { return await (await loadWhatsAppQaAdapterRuntime()).createWhatsAppQaTransportAdapter(context); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts index ac4d3ef49ef..a40ca178cbf 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts @@ -1321,7 +1321,6 @@ describe("WhatsApp QA live runtime", () => { "canary", "mention-gating", "top-level-reply-shape", - "help-command", "quote-reply", "reaction-observation", "allowlist-block", @@ -1578,7 +1577,6 @@ describe("WhatsApp QA live runtime", () => { "whatsapp-canary", "whatsapp-mention-gating", "whatsapp-top-level-reply-shape", - "whatsapp-help-command", "whatsapp-reply-to-message", "whatsapp-group-reply-to-message", "whatsapp-status-reactions", @@ -1602,12 +1600,6 @@ describe("WhatsApp QA live runtime", () => { "whatsapp-group-activation-always", "whatsapp-group-reply-to-bot-triggers", "whatsapp-top-level-reply-shape", - "whatsapp-help-command", - "whatsapp-commands-command", - "whatsapp-tools-compact-command", - "whatsapp-whoami-command", - "whatsapp-context-command", - "whatsapp-tool-only-usage-footer", "whatsapp-reply-to-message", "whatsapp-group-reply-to-message", "whatsapp-reply-to-mode-batched", @@ -1630,7 +1622,6 @@ describe("WhatsApp QA live runtime", () => { "whatsapp-group-audio-gating", "whatsapp-reply-delivery-shape", "whatsapp-stream-final-message-accounting", - "whatsapp-native-new-command", "whatsapp-status-reactions", "whatsapp-status-reaction-lifecycle", "whatsapp-group-allowlist-block", @@ -2274,75 +2265,6 @@ describe("WhatsApp QA live runtime", () => { ).toBe("safe local diagnostic"); }); - it("adds WhatsApp command UX parity scenarios to the mock-backed selection", () => { - const scenarios = testing.findScenarios([ - "whatsapp-commands-command", - "whatsapp-tools-compact-command", - "whatsapp-whoami-command", - "whatsapp-context-command", - "whatsapp-tool-only-usage-footer", - ]); - - expect( - scenarios.map((scenario) => { - const run = scenario.buildRun(); - if (run.kind === "approval") { - throw new Error(`${scenario.id} unexpectedly built an approval run`); - } - return [ - scenario.id, - run.input, - String(run.matchText), - run.expectedJoinedSutTextIncludes, - run.expectedSutMessageCountRange, - ] as const; - }), - ).toEqual([ - [ - "whatsapp-commands-command", - "/commands", - "/Commands \\(|\\/session|\\/verbose/iu", - ["/session", "/verbose"], - undefined, - ], - [ - "whatsapp-tools-compact-command", - "/tools compact", - "/Available tools|exec|Use \\/tools verbose for descriptions/iu", - ["exec", "Use /tools verbose for descriptions"], - undefined, - ], - [ - "whatsapp-whoami-command", - "/whoami", - "/(?=.*Identity)(?=.*Channel: whatsapp)(?=.*AllowFrom:)/isu", - undefined, - undefined, - ], - [ - "whatsapp-context-command", - "/context list", - "/(?=.*Context breakdown)(?=.*Workspace:)(?=.*Tool schemas)/isu", - undefined, - undefined, - ], - [ - "whatsapp-tool-only-usage-footer", - "/usage tokens", - "/Usage footer: tokens/iu", - undefined, - undefined, - ], - ]); - expect(scenarios.map((scenario) => scenario.defaultProviderModes)).toEqual([ - ["mock-openai"], - ["mock-openai"], - ["mock-openai"], - ["mock-openai"], - ["mock-openai"], - ]); - }); - it("defines WhatsApp final-message accounting as a settled two-chunk assertion", () => { const [scenario] = testing.findScenarios(["whatsapp-stream-final-message-accounting"]); const run = scenario.buildRun(); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts index eb33a03da57..a9576bb8361 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts @@ -68,8 +68,6 @@ type WhatsAppQaScenarioId = | "whatsapp-audio-preflight" | "whatsapp-broadcast-group-fanout" | "whatsapp-canary" - | "whatsapp-commands-command" - | "whatsapp-context-command" | "whatsapp-group-allowlist-block" | "whatsapp-group-activation-always" | "whatsapp-group-agent-message-action-react" @@ -82,11 +80,9 @@ type WhatsAppQaScenarioId = | "whatsapp-group-reply-to-bot-triggers" | "whatsapp-group-reply-to-message" | "whatsapp-inbound-reaction-no-trigger" - | "whatsapp-help-command" | "whatsapp-inbound-image-caption" | "whatsapp-inbound-structured-messages" | "whatsapp-message-actions" - | "whatsapp-native-new-command" | "whatsapp-outbound-document-preserves-filename" | "whatsapp-outbound-media-matrix" | "whatsapp-outbound-poll" @@ -97,13 +93,9 @@ type WhatsAppQaScenarioId = | "whatsapp-reply-to-message" | "whatsapp-reply-to-mode-batched" | "whatsapp-stream-final-message-accounting" - | "whatsapp-status-command" | "whatsapp-status-reaction-lifecycle" | "whatsapp-status-reactions" | "whatsapp-top-level-reply-shape" - | "whatsapp-tools-compact-command" - | "whatsapp-tool-only-usage-footer" - | "whatsapp-whoami-command" | "whatsapp-approval-exec-native" | "whatsapp-approval-plugin-native"; @@ -127,8 +119,6 @@ const WHATSAPP_QA_SCENARIO_POSTURES = { "whatsapp-audio-preflight": "user-path", "whatsapp-broadcast-group-fanout": "user-path", "whatsapp-canary": "user-path", - "whatsapp-commands-command": "user-path", - "whatsapp-context-command": "user-path", "whatsapp-group-activation-always": "user-path", "whatsapp-group-allowlist-block": "user-path", "whatsapp-group-agent-message-action-react": "user-path", @@ -140,13 +130,11 @@ const WHATSAPP_QA_SCENARIO_POSTURES = { "whatsapp-group-pending-history-context": "user-path", "whatsapp-group-reply-to-bot-triggers": "user-path", "whatsapp-group-reply-to-message": "user-path", - "whatsapp-help-command": "user-path", "whatsapp-inbound-image-caption": "user-path", "whatsapp-inbound-reaction-no-trigger": "user-path", "whatsapp-inbound-structured-messages": "user-path", "whatsapp-mention-gating": "user-path", "whatsapp-message-actions": "direct-gateway", - "whatsapp-native-new-command": "user-path", "whatsapp-outbound-document-preserves-filename": "direct-gateway", "whatsapp-outbound-media-matrix": "direct-gateway", "whatsapp-outbound-poll": "direct-gateway", @@ -155,14 +143,10 @@ const WHATSAPP_QA_SCENARIO_POSTURES = { "whatsapp-reply-delivery-shape": "direct-gateway", "whatsapp-reply-to-message": "user-path", "whatsapp-reply-to-mode-batched": "user-path", - "whatsapp-status-command": "user-path", "whatsapp-status-reaction-lifecycle": "user-path", "whatsapp-status-reactions": "user-path", "whatsapp-stream-final-message-accounting": "user-path", - "whatsapp-tool-only-usage-footer": "user-path", - "whatsapp-tools-compact-command": "user-path", "whatsapp-top-level-reply-shape": "user-path", - "whatsapp-whoami-command": "user-path", } satisfies Record; type WhatsAppQaMessageSendMode = @@ -881,120 +865,6 @@ const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ }; }, }, - { - id: "whatsapp-help-command", - standardId: "help-command", - title: "WhatsApp help command replies", - timeoutMs: 60_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - input: "/help", - matchText: /OpenClaw|commands|status|\/new/iu, - target: "dm", - }), - }, - { - id: "whatsapp-status-command", - title: "WhatsApp status command replies", - timeoutMs: 60_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - input: "/status", - matchText: /OpenClaw|status|session|agent/iu, - target: "dm", - }), - }, - { - id: "whatsapp-commands-command", - title: "WhatsApp commands list replies", - defaultProviderModes: ["mock-openai"], - timeoutMs: 60_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - expectedJoinedSutTextIncludes: ["/session", "/verbose"], - input: "/commands", - matchText: /Commands \(|\/session|\/verbose/iu, - settleMs: 4_000, - target: "dm", - }), - }, - { - id: "whatsapp-tools-compact-command", - title: "WhatsApp tools compact reply", - defaultProviderModes: ["mock-openai"], - timeoutMs: 60_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - expectedJoinedSutTextIncludes: ["exec", "Use /tools verbose for descriptions"], - input: "/tools compact", - matchText: /Available tools|exec|Use \/tools verbose for descriptions/iu, - settleMs: 4_000, - target: "dm", - }), - }, - { - id: "whatsapp-whoami-command", - title: "WhatsApp whoami reply", - defaultProviderModes: ["mock-openai"], - timeoutMs: 60_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - input: "/whoami", - matchText: /(?=.*Identity)(?=.*Channel: whatsapp)(?=.*AllowFrom:)/isu, - target: "dm", - }), - }, - { - id: "whatsapp-context-command", - title: "WhatsApp context list reply", - defaultProviderModes: ["mock-openai"], - timeoutMs: 60_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - input: "/context list", - matchText: /(?=.*Context breakdown)(?=.*Workspace:)(?=.*Tool schemas)/isu, - target: "dm", - }), - }, - { - id: "whatsapp-tool-only-usage-footer", - title: "WhatsApp tool-only reply includes usage footer", - defaultProviderModes: ["mock-openai"], - timeoutMs: 120_000, - buildRun: () => { - const token = `WHATSAPP_QA_USAGE_FOOTER_${randomUUID().slice(0, 8).toUpperCase()}`; - return { - afterReply: async (_reply, context) => { - const usageStartedAt = new Date(); - await context.driver.sendText( - context.target, - `Reply with only this exact marker after usage footer setup: ${token}`, - ); - const usageReply = await context.driver.waitForMessage({ - observedAfter: usageStartedAt, - timeoutMs: 60_000, - match: (message) => - message.fromPhoneE164 === context.sutPhoneE164 && - message.text.includes(token) && - message.text.includes("Usage:"), - }); - context.recordObservedMessage(usageReply); - return "model reply included visible usage footer"; - }, - configMode: "allowlist", - expectReply: true, - input: "/usage tokens", - matchText: /Usage footer: tokens/iu, - target: "dm", - }; - }, - }, { id: "whatsapp-reply-to-message", standardId: "quote-reply", @@ -1923,19 +1793,6 @@ const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [ target: "dm", }), }, - { - id: "whatsapp-native-new-command", - title: "WhatsApp /new command starts a new session", - defaultProviderModes: ["mock-openai"], - timeoutMs: 60_000, - buildRun: () => ({ - configMode: "allowlist", - expectReply: true, - input: "/new", - matchText: /new session|session/i, - target: "dm", - }), - }, { id: "whatsapp-approval-exec-deny-native", title: "WhatsApp native exec approval prompt denies", diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 611a87ea01e..056c24b1f1b 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -201,6 +201,7 @@ describe("qa scenario catalog", () => { "matrix-restart-resume", "slack-restart-resume", "subagent-stale-child-links", + "telegram-repeated-command-authorization", "whatsapp-restart-resume", ]); expect( diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index 31cea031b1e..c352a43d7bd 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -275,6 +275,7 @@ const qaSeedScenarioBodySchema = z.object({ plugins: z.array(z.string().trim().min(1)).optional(), gatewayConfigPatch: z.record(z.string(), z.unknown()).optional(), gatewayRuntime: qaScenarioGatewayRuntimeSchema.optional(), + regressionRefs: z.array(z.string().trim().min(1)).optional(), docsRefs: z.array(z.string().trim().min(1)).optional(), codeRefs: z.array(z.string().trim().min(1)).optional(), execution: qaScenarioExecutionSchema.optional(), diff --git a/qa/scenarios/channels/telegram-commands-command.yaml b/qa/scenarios/channels/telegram-commands-command.yaml new file mode 100644 index 00000000000..7e6f542fb25 --- /dev/null +++ b/qa/scenarios/channels/telegram-commands-command.yaml @@ -0,0 +1,50 @@ +title: Telegram commands list reply + +scenario: + id: telegram-commands-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify Telegram renders the full native command catalog. + successCriteria: + - The command list is returned through the Telegram group command path. + - The joined reply includes session and stop commands. + codeRefs: + - src/auto-reply/reply/commands-core.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Request the Telegram command catalog and verify representative entries. + config: + expectedAny: [/session, /stop] + +flow: + steps: + - name: commands renders the full catalog + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? '/commands@openclaw' : '/commands'" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + timeoutMs: 60000 + - call: sleep + args: [1000] + - set: joinedReply + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')" + - assert: + expr: "config.expectedAny.every((needle) => joinedReply.includes(needle)) && (transport.id === 'qa-channel' ? joinedReply.includes('Slash commands') : joinedReply.includes('Commands (1/'))" + message: + expr: "`commands reply missing expected text: ${joinedReply}`" + detailsExpr: joinedReply diff --git a/qa/scenarios/channels/telegram-context-command.yaml b/qa/scenarios/channels/telegram-context-command.yaml new file mode 100644 index 00000000000..560a8ca6d31 --- /dev/null +++ b/qa/scenarios/channels/telegram-context-command.yaml @@ -0,0 +1,47 @@ +title: Telegram context command reply + +scenario: + id: telegram-context-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify Telegram context help reaches native command routing. + successCriteria: + - The context command returns the context list shortcut. + - The reply explains the inline shortcut. + codeRefs: + - src/auto-reply/reply/commands-context.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Run Telegram context help and verify the list shortcut. + config: + expectedAny: [/context list, Inline shortcut] + +flow: + steps: + - name: context returns shortcut help + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? '/context@openclaw' : '/context'" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: /context list + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "config.expectedAny.every((needle) => reply.text.includes(needle))" + message: + expr: "`context reply missing expected text: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/telegram-current-session-status-tool.yaml b/qa/scenarios/channels/telegram-current-session-status-tool.yaml new file mode 100644 index 00000000000..d75846b008e --- /dev/null +++ b/qa/scenarios/channels/telegram-current-session-status-tool.yaml @@ -0,0 +1,44 @@ +title: Telegram current-session status tool context + +scenario: + id: telegram-current-session-status-tool + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [tools.session-status] + secondary: [memory.session-key-construction] + objective: Verify the session_status tool resolves current to the active Telegram group session. + successCriteria: + - The model calls session_status with sessionKey current. + - The final reply contains the Telegram group session key and success marker. + codeRefs: + - src/agents/tools/session-status-tool.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Ask the model to resolve the current Telegram group session through session_status. + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + +flow: + steps: + - name: session status resolves current Telegram group + actions: + - resetTransport: true + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: "@openclaw Telegram current session_status QA check. Call session_status with sessionKey set to current, then reply with the exact QA marker and resolved session key." + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + textIncludes: QA-TELEGRAM-CURRENT-SESSION-OK + timeoutMs: 90000 + saveAs: reply + - assert: + expr: "reply.text.includes(':telegram:group:')" + message: + expr: "`current-session reply missing Telegram group key: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/telegram-help-command.yaml b/qa/scenarios/channels/telegram-help-command.yaml new file mode 100644 index 00000000000..1475a474071 --- /dev/null +++ b/qa/scenarios/channels/telegram-help-command.yaml @@ -0,0 +1,51 @@ +title: Telegram help command reply + +scenario: + id: telegram-help-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: + - channels.native-commands + objective: Verify the Telegram help command returns the concise command guide. + successCriteria: + - The help command is accepted in the Telegram group command path. + - The reply advertises new-session and full command-list commands. + codeRefs: + - src/auto-reply/reply/commands-core.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Send Telegram help and verify the concise native command guide. + config: + command: help + expectedAny: + - /new + - /commands for full list + +flow: + steps: + - name: help returns the concise command guide + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? `/${config.command}@openclaw` : `/${config.command}`" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: { expr: "config.expectedAny[0]" } + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "config.expectedAny.every((needle) => reply.text.includes(needle))" + message: + expr: "`help reply missing expected text: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/telegram-repeated-command-authorization.yaml b/qa/scenarios/channels/telegram-repeated-command-authorization.yaml new file mode 100644 index 00000000000..e0a34d72a64 --- /dev/null +++ b/qa/scenarios/channels/telegram-repeated-command-authorization.yaml @@ -0,0 +1,95 @@ +title: Telegram repeated command authorization + +scenario: + id: telegram-repeated-command-authorization + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify an allowlisted Telegram bot operator remains authorized across repeated native commands. + successCriteria: + - The same operator is blocked while removed from the group allowlist. + - Restoring the original allowlist authorizes status, help, and commands without another gate. + codeRefs: + - extensions/telegram/src/bot-message-context.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + suiteIsolation: isolated + isolationReason: Mutates the live Telegram group allowlist and restarts the QA gateway. + summary: Remove and restore the Telegram operator allowlist, then run repeated commands. + config: + requiredChannelDriver: live + +flow: + steps: + - name: repeated commands reuse restored authorization + actions: + - assert: + expr: "transport.id === 'telegram'" + message: repeated authorization requires the live Telegram adapter + - set: authorizationState + value: + expr: "({ allowFrom: [] })" + - try: + actions: + - call: env.gateway.restartAfterStateMutation + args: + - lambda: + async: true + params: [ctx] + expr: "fs.readFile(ctx.configPath, 'utf8').then((raw) => { const cfg = JSON.parse(raw); const account = cfg.channels.telegram.accounts[transport.accountId]; const group = account.groups[Object.keys(account.groups)[0]]; authorizationState.allowFrom = [...(group.allowFrom ?? [])]; group.allowFrom = []; return fs.writeFile(ctx.configPath, `${JSON.stringify(cfg, null, 2)}\n`, 'utf8'); })" + - call: waitForTransportReady + args: [{ ref: env }, 60000] + - resetTransport: true + - set: deniedStart + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: /status@openclaw + - waitForNoOutbound: + quietMs: 8000 + sinceIndex: { ref: deniedStart } + finally: + - call: env.gateway.restartAfterStateMutation + args: + - lambda: + async: true + params: [ctx] + expr: "fs.readFile(ctx.configPath, 'utf8').then((raw) => { const cfg = JSON.parse(raw); const account = cfg.channels.telegram.accounts[transport.accountId]; const group = account.groups[Object.keys(account.groups)[0]]; group.allowFrom = authorizationState.allowFrom; return fs.writeFile(ctx.configPath, `${JSON.stringify(cfg, null, 2)}\n`, 'utf8'); })" + - call: waitForTransportReady + args: [{ ref: env }, 60000] + - resetTransport: true + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: /status@openclaw + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + textIncludes: "Session:" + timeoutMs: 60000 + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: /help@openclaw + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + textIncludes: /commands for full list + timeoutMs: 60000 + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: /commands@openclaw + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + textIncludes: /stop + timeoutMs: 60000 + saveAs: reply + detailsExpr: reply.text diff --git a/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml b/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml new file mode 100644 index 00000000000..8a71a722185 --- /dev/null +++ b/qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml @@ -0,0 +1,53 @@ +title: Telegram reply-chain exact marker + +scenario: + id: telegram-reply-chain-exact-marker + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [runtime.delivery] + objective: Verify an exact-marker Telegram reply is delivered once without extra visible text. + successCriteria: + - The model returns the requested exact marker through the reply path. + - Exactly one outbound message contains the marker. + codeRefs: + - src/auto-reply/reply/reply-payload.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Request one exact Telegram marker and verify single-message delivery. + config: + requiredProviderMode: mock-openai + marker: QA-TELEGRAM-REPLY-CHAIN-OK + +flow: + steps: + - name: exact marker is delivered once + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "`@openclaw Telegram reply-chain marker QA. Reply exactly: ${config.marker}`" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: { ref: config.marker } + timeoutMs: 90000 + saveAs: reply + - call: sleep + args: [1000] + - set: markerReplies + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).filter((message) => message.text.includes(config.marker))" + - assert: + expr: "markerReplies.length === 1 && reply.text.trim() === config.marker" + message: + expr: "`expected one exact marker reply; saw ${markerReplies.length}: ${markerReplies.map((message) => message.text).join(' | ')}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/telegram-status-command.yaml b/qa/scenarios/channels/telegram-status-command.yaml new file mode 100644 index 00000000000..9425b338e7e --- /dev/null +++ b/qa/scenarios/channels/telegram-status-command.yaml @@ -0,0 +1,49 @@ +title: Telegram status command reply + +scenario: + id: telegram-status-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + regressionRefs: + - openclaw/openclaw#74698 + objective: Verify Telegram status returns model, session, and activation details. + successCriteria: + - The status command succeeds in the Telegram group path. + - The reply includes model, session, and activation fields. + codeRefs: + - src/auto-reply/reply/commands-status.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Run Telegram status and verify representative status fields. + config: + expectedAny: [OpenClaw, "Model:", "Session:", "Activation:"] + +flow: + steps: + - name: status includes session details + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? '/status@openclaw' : '/status'" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: OpenClaw + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "config.expectedAny.every((needle) => reply.text.includes(needle))" + message: + expr: "`status reply missing expected text: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml b/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml new file mode 100644 index 00000000000..f68bf885822 --- /dev/null +++ b/qa/scenarios/channels/telegram-tool-only-usage-footer.yaml @@ -0,0 +1,78 @@ +title: Telegram tool-only usage footer + +scenario: + id: telegram-tool-only-usage-footer + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [telemetry.model-usage] + secondary: [channels.native-commands] + regressionRefs: + - openclaw/openclaw#87392 + objective: Verify Telegram usage mode decorates a message-tool-only visible reply. + successCriteria: + - The usage command enables token footer mode. + - The next exact-marker reply includes the marker and a visible Usage footer. + codeRefs: + - src/auto-reply/reply/commands-session.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Enable token usage mode, then verify the next Telegram reply footer. + config: + requiredChannelDriver: live + marker: QA-TELEGRAM-USAGE-FOOTER-OK + +flow: + steps: + - name: next visible reply includes usage footer + actions: + - resetTransport: true + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? '/usage@openclaw tokens' : '/usage tokens'" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + textIncludes: tokens + timeoutMs: 60000 + - try: + actions: + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "`@openclaw Telegram usage footer QA. Reply exactly: ${config.marker}`" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: { ref: config.marker } + timeoutMs: 90000 + - call: sleep + args: [1000] + - set: joinedReply + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')" + - assert: + expr: "joinedReply.includes(config.marker) && joinedReply.includes('Usage:')" + message: + expr: "`usage footer reply missing marker or footer: ${joinedReply}`" + finally: + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? '/usage@openclaw off' : '/usage off'" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + textIncludes: "Usage footer: off" + timeoutMs: 60000 + detailsExpr: joinedReply diff --git a/qa/scenarios/channels/telegram-tools-compact-command.yaml b/qa/scenarios/channels/telegram-tools-compact-command.yaml new file mode 100644 index 00000000000..6bfb8e6ae0a --- /dev/null +++ b/qa/scenarios/channels/telegram-tools-compact-command.yaml @@ -0,0 +1,51 @@ +title: Telegram compact tools command reply + +scenario: + id: telegram-tools-compact-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify Telegram renders the compact model-tool inventory. + successCriteria: + - The compact tools command reaches native command dispatch. + - The joined reply includes exec and the verbose-mode hint. + codeRefs: + - src/auto-reply/reply/commands-tools.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Request compact tools through Telegram and verify inventory text. + config: + expectedAny: [exec, Use /tools verbose for descriptions.] + +flow: + steps: + - name: compact tools renders inventory + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? '/tools@openclaw compact' : '/tools compact'" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: exec + timeoutMs: 60000 + - call: sleep + args: [1000] + - set: joinedReply + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')" + - assert: + expr: "config.expectedAny.every((needle) => joinedReply.includes(needle))" + message: + expr: "`tools reply missing expected text: ${joinedReply}`" + detailsExpr: joinedReply diff --git a/qa/scenarios/channels/telegram-whoami-command.yaml b/qa/scenarios/channels/telegram-whoami-command.yaml new file mode 100644 index 00000000000..1acf3f398c2 --- /dev/null +++ b/qa/scenarios/channels/telegram-whoami-command.yaml @@ -0,0 +1,45 @@ +title: Telegram whoami command reply + +scenario: + id: telegram-whoami-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify Telegram command identity includes the active channel context. + successCriteria: + - The whoami command returns an identity block. + - The reply names Telegram as the active channel on Telegram-backed transports. + codeRefs: + - src/auto-reply/reply/commands-whoami.ts + - extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts + execution: + kind: flow + channel: telegram + summary: Run whoami and verify channel identity context. + +flow: + steps: + - name: whoami includes channel identity + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: telegram-command-room, kind: channel } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "transport.id === 'telegram' ? '/whoami@openclaw' : '/whoami'" + - waitForOutbound: + conversation: { id: telegram-command-room, kind: channel } + sinceIndex: { ref: startIndex } + textIncludes: Identity + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "reply.text.includes(`Channel: ${transport.id === 'qa-channel' ? 'qa-channel' : 'telegram'}`)" + message: + expr: "`whoami reply missing channel identity: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/whatsapp-commands-command.yaml b/qa/scenarios/channels/whatsapp-commands-command.yaml new file mode 100644 index 00000000000..8b2b04890af --- /dev/null +++ b/qa/scenarios/channels/whatsapp-commands-command.yaml @@ -0,0 +1,51 @@ +title: WhatsApp commands list reply + +scenario: + id: whatsapp-commands-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify WhatsApp renders the full native command catalog. + successCriteria: + - The command catalog is returned in a WhatsApp DM. + - The joined reply includes session and verbose commands. + codeRefs: + - src/auto-reply/reply/commands-core.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + expectedAny: [/session, /verbose] + summary: Request the WhatsApp command catalog and verify representative entries. + +flow: + steps: + - name: commands renders the full catalog + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /commands + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + sinceIndex: { ref: startIndex } + textIncludes: /session + timeoutMs: 60000 + - call: sleep + args: [1000] + - set: joinedReply + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')" + - assert: + expr: "config.expectedAny.every((needle) => joinedReply.includes(needle))" + message: + expr: "`commands reply missing expected text: ${joinedReply}`" + detailsExpr: joinedReply diff --git a/qa/scenarios/channels/whatsapp-context-command.yaml b/qa/scenarios/channels/whatsapp-context-command.yaml new file mode 100644 index 00000000000..0299f482dce --- /dev/null +++ b/qa/scenarios/channels/whatsapp-context-command.yaml @@ -0,0 +1,44 @@ +title: WhatsApp context command reply + +scenario: + id: whatsapp-context-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify WhatsApp returns the current context breakdown. + successCriteria: + - The context list command succeeds in a WhatsApp DM. + - The reply includes workspace and tool-schema context sections. + codeRefs: + - src/auto-reply/reply/commands-context.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + expectedAny: [Context breakdown, "Workspace:", Tool schemas] + summary: Run WhatsApp context list and verify the context breakdown. + +flow: + steps: + - name: context list returns breakdown + actions: + - resetTransport: true + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /context list + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + textIncludes: Context breakdown + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "config.expectedAny.every((needle) => reply.text.includes(needle))" + message: + expr: "`context reply missing expected text: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/whatsapp-help-command.yaml b/qa/scenarios/channels/whatsapp-help-command.yaml new file mode 100644 index 00000000000..53ffdcbed16 --- /dev/null +++ b/qa/scenarios/channels/whatsapp-help-command.yaml @@ -0,0 +1,38 @@ +title: WhatsApp help command reply + +scenario: + id: whatsapp-help-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify WhatsApp help returns the concise command guide. + successCriteria: + - The help command is accepted in a WhatsApp DM. + - The reply advertises new-session or command/status guidance. + codeRefs: + - src/auto-reply/reply/commands-core.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + summary: Send WhatsApp help and verify command guidance. + +flow: + steps: + - name: help returns command guidance + actions: + - resetTransport: true + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /help + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + textIncludes: /new + timeoutMs: 60000 + saveAs: reply + detailsExpr: reply.text diff --git a/qa/scenarios/channels/whatsapp-native-new-command.yaml b/qa/scenarios/channels/whatsapp-native-new-command.yaml new file mode 100644 index 00000000000..f5b78b399b3 --- /dev/null +++ b/qa/scenarios/channels/whatsapp-native-new-command.yaml @@ -0,0 +1,43 @@ +title: WhatsApp new-session command + +scenario: + id: whatsapp-native-new-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify WhatsApp `/new` starts a new session through native command dispatch. + successCriteria: + - The `/new` command is accepted in a WhatsApp DM. + - The reply confirms a new session. + codeRefs: + - src/auto-reply/reply/commands-core.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + summary: Run `/new` through WhatsApp and verify the new-session acknowledgement. + +flow: + steps: + - name: new starts a session + actions: + - resetTransport: true + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /new + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + textIncludes: session + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "reply.text.toLowerCase().includes('new')" + message: + expr: "`new-session reply missing confirmation: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/whatsapp-status-command.yaml b/qa/scenarios/channels/whatsapp-status-command.yaml new file mode 100644 index 00000000000..96530940a7b --- /dev/null +++ b/qa/scenarios/channels/whatsapp-status-command.yaml @@ -0,0 +1,42 @@ +title: WhatsApp status command reply + +scenario: + id: whatsapp-status-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify WhatsApp status returns the active session summary. + successCriteria: + - The status command succeeds in a WhatsApp DM. + - The reply includes OpenClaw and session information. + codeRefs: + - src/auto-reply/reply/commands-status.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + summary: Run WhatsApp status and verify session text. + +flow: + steps: + - name: status includes session information + actions: + - resetTransport: true + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /status + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + textIncludes: OpenClaw + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "reply.text.toLowerCase().includes('session')" + message: + expr: "`status reply missing session text: ${reply.text}`" + detailsExpr: reply.text diff --git a/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml b/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml new file mode 100644 index 00000000000..d044940014e --- /dev/null +++ b/qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml @@ -0,0 +1,75 @@ +title: WhatsApp tool-only usage footer + +scenario: + id: whatsapp-tool-only-usage-footer + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [telemetry.model-usage] + secondary: [channels.native-commands] + objective: Verify WhatsApp usage mode decorates a message-tool-only visible reply. + successCriteria: + - The usage command enables token footer mode. + - The next exact-marker reply includes the marker and visible Usage footer. + codeRefs: + - src/auto-reply/reply/commands-session.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + marker: WHATSAPP-QA-USAGE-FOOTER-OK + summary: Enable token usage mode, then verify the next WhatsApp reply footer. + +flow: + steps: + - name: next visible reply includes usage footer + actions: + - resetTransport: true + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /usage tokens + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + textIncludes: "Usage footer: tokens" + timeoutMs: 60000 + - try: + actions: + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: + expr: "`Reply with only this exact marker after usage footer setup: ${config.marker}`" + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + sinceIndex: { ref: startIndex } + textIncludes: { ref: config.marker } + timeoutMs: 90000 + - call: sleep + args: [1000] + - set: joinedReply + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')" + - assert: + expr: "joinedReply.includes(config.marker) && joinedReply.includes('Usage:')" + message: + expr: "`usage footer reply missing marker or footer: ${joinedReply}`" + finally: + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /usage off + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + textIncludes: "Usage footer: off" + timeoutMs: 60000 + detailsExpr: joinedReply diff --git a/qa/scenarios/channels/whatsapp-tools-compact-command.yaml b/qa/scenarios/channels/whatsapp-tools-compact-command.yaml new file mode 100644 index 00000000000..b4850cc77a2 --- /dev/null +++ b/qa/scenarios/channels/whatsapp-tools-compact-command.yaml @@ -0,0 +1,52 @@ +title: WhatsApp compact tools command reply + +scenario: + id: whatsapp-tools-compact-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify WhatsApp renders the compact model-tool inventory. + successCriteria: + - The compact tools command reaches native command dispatch. + - The joined reply includes exec and the verbose-mode hint. + codeRefs: + - src/auto-reply/reply/commands-tools.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + expectedAny: [exec, Use /tools verbose for descriptions] + summary: Request compact tools through WhatsApp and verify inventory text. + +flow: + steps: + - name: compact tools renders inventory + actions: + - resetTransport: true + - set: startIndex + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length" + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /tools compact + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + sinceIndex: { ref: startIndex } + textIncludes: exec + timeoutMs: 60000 + - call: sleep + args: [1000] + - set: joinedReply + value: + expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')" + - assert: + expr: "config.expectedAny.every((needle) => joinedReply.includes(needle))" + message: + expr: "`tools reply missing expected text: ${joinedReply}`" + detailsExpr: joinedReply diff --git a/qa/scenarios/channels/whatsapp-whoami-command.yaml b/qa/scenarios/channels/whatsapp-whoami-command.yaml new file mode 100644 index 00000000000..46b4928c288 --- /dev/null +++ b/qa/scenarios/channels/whatsapp-whoami-command.yaml @@ -0,0 +1,44 @@ +title: WhatsApp whoami command reply + +scenario: + id: whatsapp-whoami-command + surface: channel-framework + category: channel-framework.channel-actions-commands-and-approvals + coverage: + primary: [channels.native-commands] + objective: Verify WhatsApp command identity includes channel and allowlist context. + successCriteria: + - The whoami command returns an identity block. + - The reply names WhatsApp and includes AllowFrom context. + codeRefs: + - src/auto-reply/reply/commands-whoami.ts + - extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts + execution: + kind: flow + channel: whatsapp + config: + requiredChannelDriver: live + requiredProviderMode: mock-openai + expectedAny: [Identity, "Channel: whatsapp", "AllowFrom:"] + summary: Run WhatsApp whoami and verify channel identity context. + +flow: + steps: + - name: whoami includes WhatsApp identity + actions: + - resetTransport: true + - sendInbound: + conversation: { id: whatsapp-command-dm, kind: direct } + senderId: qa-command-operator + senderName: QA Command Operator + text: /whoami + - waitForOutbound: + conversation: { id: whatsapp-command-dm, kind: direct } + textIncludes: Identity + timeoutMs: 60000 + saveAs: reply + - assert: + expr: "config.expectedAny.every((needle) => reply.text.includes(needle))" + message: + expr: "`whoami reply missing expected text: ${reply.text}`" + detailsExpr: reply.text