mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 15:59:30 +00:00
refactor(qa): canonicalize channel command scenarios (#101080)
* test(qa): migrate channel command scenarios * test(qa): distinguish portable command headings * test(qa): preserve canonical scenario ownership
This commit is contained in:
Dallin Romney
GitHub
parent
db38127c22
commit
e3671e0980
33 changed files with 1540 additions and 611 deletions
|
|
@ -10,6 +10,7 @@ const {
|
|||
runQaSuite,
|
||||
runQaCharacterEval,
|
||||
runQaMultipass,
|
||||
runCanonicalLiveScenarios,
|
||||
listLiveTransportQaAdapterFactories,
|
||||
listTelegramQaScenarioCatalog,
|
||||
runTelegramQaLive,
|
||||
|
|
@ -24,6 +25,7 @@ const {
|
|||
runQaSuite: vi.fn(),
|
||||
runQaCharacterEval: vi.fn(),
|
||||
runQaMultipass: vi.fn(),
|
||||
runCanonicalLiveScenarios: vi.fn(),
|
||||
listLiveTransportQaAdapterFactories: vi.fn(),
|
||||
listTelegramQaScenarioCatalog: vi.fn(),
|
||||
runTelegramQaLive: vi.fn(),
|
||||
|
|
@ -56,6 +58,15 @@ vi.mock("./live-transports/cli.js", () => ({
|
|||
listLiveTransportQaAdapterFactories,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/shared/canonical-scenarios.js", async (importOriginal) => ({
|
||||
...(await importOriginal()),
|
||||
runCanonicalLiveScenarios,
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/telegram/adapter.runtime.js", () => ({
|
||||
createTelegramQaTransportAdapter: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
|
||||
listTelegramQaScenarioCatalog,
|
||||
runTelegramQaLive,
|
||||
|
|
@ -228,6 +239,7 @@ describe("qa cli runtime", () => {
|
|||
runQaCharacterEval.mockReset();
|
||||
runQaManualLane.mockReset();
|
||||
runQaMultipass.mockReset();
|
||||
runCanonicalLiveScenarios.mockReset();
|
||||
listLiveTransportQaAdapterFactories.mockReset();
|
||||
listTelegramQaScenarioCatalog.mockReset();
|
||||
runTelegramQaLive.mockReset();
|
||||
|
|
@ -280,13 +292,19 @@ describe("qa cli runtime", () => {
|
|||
observedMessagesPath: path.join(telegramArtifactsDir, "observed.json"),
|
||||
scenarios: [],
|
||||
});
|
||||
runCanonicalLiveScenarios.mockResolvedValue({
|
||||
outputDir: telegramArtifactsDir,
|
||||
reportPath: path.join(telegramArtifactsDir, "canonical-report.md"),
|
||||
summaryPath: telegramSummaryPath,
|
||||
scenarios: [],
|
||||
});
|
||||
listTelegramQaScenarioCatalog.mockReturnValue([
|
||||
{
|
||||
id: "telegram-status-command",
|
||||
title: "Telegram status command reply",
|
||||
defaultEnabled: true,
|
||||
rationale: "status rationale",
|
||||
regressionRefs: ["openclaw/openclaw#74698"],
|
||||
id: "telegram-stream-final-single-message",
|
||||
title: "Telegram streamed final reply",
|
||||
defaultEnabled: false,
|
||||
rationale: "stream rationale",
|
||||
regressionRefs: [],
|
||||
},
|
||||
]);
|
||||
listLiveTransportQaAdapterFactories.mockReturnValue([
|
||||
|
|
@ -909,17 +927,22 @@ describe("qa cli runtime", () => {
|
|||
sutAccountId: "sut-live",
|
||||
});
|
||||
|
||||
expect(runTelegramQaLive).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
alternateModel: "openai/gpt-5.5",
|
||||
fastMode: true,
|
||||
allowFailures: undefined,
|
||||
scenarioIds: ["telegram-help-command"],
|
||||
sutAccountId: "sut-live",
|
||||
});
|
||||
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
channelId: "telegram",
|
||||
options: expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa/telegram"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.5",
|
||||
alternateModel: "openai/gpt-5.5",
|
||||
fastMode: true,
|
||||
sutAccountId: "sut-live",
|
||||
}),
|
||||
scenarioIds: ["telegram-help-command"],
|
||||
}),
|
||||
);
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects output dirs that escape the repo root", () => {
|
||||
|
|
@ -937,11 +960,40 @@ describe("qa cli runtime", () => {
|
|||
scenarioIds: ["telegram-help-command"],
|
||||
});
|
||||
|
||||
expectFields(mockFirstObjectArg(runTelegramQaLive), {
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
allowFailures: undefined,
|
||||
expect(runCanonicalLiveScenarios).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
options: expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
allowFailures: undefined,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps remaining Telegram defaults when Commander supplies an empty scenario list", async () => {
|
||||
await runQaTelegramCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
scenarioIds: [],
|
||||
});
|
||||
|
||||
expect(runCanonicalLiveScenarios).toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ scenarioIds: undefined }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects unknown mixed Telegram selections before starting canonical scenarios", async () => {
|
||||
await expect(
|
||||
runQaTelegramCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
scenarioIds: ["telegram-help-command", "missing-telegram-scenario"],
|
||||
}),
|
||||
).rejects.toThrow("unknown Telegram QA scenario id(s): missing-telegram-scenario");
|
||||
|
||||
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("prints telegram scenario catalog without starting the live lane", async () => {
|
||||
|
|
@ -952,10 +1004,11 @@ describe("qa cli runtime", () => {
|
|||
});
|
||||
|
||||
expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai");
|
||||
expect(runCanonicalLiveScenarios).not.toHaveBeenCalled();
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
expectWriteContains(
|
||||
stdoutWrite,
|
||||
"telegram-status-command\tdefault\tTelegram status command reply\tstatus rationale refs=openclaw/openclaw#74698",
|
||||
"telegram-status-command\tdefault\tTelegram status command reply\tVerify Telegram status returns model, session, and activation details. refs=openclaw/openclaw#74698",
|
||||
);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
// Qa Lab tests cover canonical live-transport scenario delegation.
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { runQaFlowSuiteFromRuntime } = vi.hoisted(() => ({
|
||||
runQaFlowSuiteFromRuntime: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../suite-launch.runtime.js", () => ({
|
||||
runQaFlowSuiteFromRuntime,
|
||||
}));
|
||||
import {
|
||||
assertKnownScenarioIds,
|
||||
partitionCanonicalScenarioIds,
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
whatsappDefaultCanonicalScenarioIds,
|
||||
listCanonicalScenarios,
|
||||
runCanonicalLiveScenarios,
|
||||
} from "./canonical-scenarios.js";
|
||||
import { loadNonYamlScenarioRefs } from "./live-transport-scenarios.js";
|
||||
|
||||
describe("canonical live-transport scenarios", () => {
|
||||
it("loads every migrated command and session-context scenario from YAML", () => {
|
||||
const telegram = listCanonicalScenarios({
|
||||
ids: TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
});
|
||||
const whatsapp = listCanonicalScenarios({
|
||||
ids: WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
defaultIds: WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
});
|
||||
|
||||
expect(telegram.map(({ id }) => id).toSorted()).toEqual(
|
||||
[...TELEGRAM_CANONICAL_SCENARIO_IDS].toSorted(),
|
||||
);
|
||||
expect(whatsapp.map(({ id }) => id).toSorted()).toEqual(
|
||||
[...WHATSAPP_CANONICAL_SCENARIO_IDS].toSorted(),
|
||||
);
|
||||
expect(telegram.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual(
|
||||
expect.arrayContaining([...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS]),
|
||||
);
|
||||
expect(whatsapp.filter(({ defaultEnabled }) => defaultEnabled).map(({ id }) => id)).toEqual(
|
||||
expect.arrayContaining([...WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS]),
|
||||
);
|
||||
expect(whatsappDefaultCanonicalScenarioIds("live-frontier")).toEqual(["whatsapp-help-command"]);
|
||||
expect(telegram.find(({ id }) => id === "telegram-status-command")?.regressionRefs).toEqual([
|
||||
"openclaw/openclaw#74698",
|
||||
]);
|
||||
});
|
||||
|
||||
it("partitions canonical aliases from remaining imperative scenarios", () => {
|
||||
expect(
|
||||
partitionCanonicalScenarioIds(
|
||||
["telegram-help-command", "telegram-mentioned-message-reply"],
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
),
|
||||
).toEqual({
|
||||
canonical: ["telegram-help-command"],
|
||||
legacy: ["telegram-mentioned-message-reply"],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects unknown legacy ids before either live runner starts", () => {
|
||||
expect(() =>
|
||||
assertKnownScenarioIds({
|
||||
ids: ["known", "missing"],
|
||||
knownIds: ["known"],
|
||||
laneLabel: "Demo",
|
||||
}),
|
||||
).toThrow("unknown Demo QA scenario id(s): missing");
|
||||
});
|
||||
|
||||
it("runs canonical live aliases through the runtime lab launcher", async () => {
|
||||
runQaFlowSuiteFromRuntime.mockResolvedValueOnce({ summaryPath: "/tmp/summary.json" });
|
||||
|
||||
await runCanonicalLiveScenarios({
|
||||
channelId: "telegram",
|
||||
factory: {
|
||||
id: "telegram",
|
||||
matches: () => true,
|
||||
create: vi.fn(),
|
||||
},
|
||||
options: {
|
||||
providerMode: "mock-openai",
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
},
|
||||
scenarioIds: ["telegram-help-command"],
|
||||
});
|
||||
|
||||
expect(runQaFlowSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
channelDriver: "live",
|
||||
channelId: "telegram",
|
||||
scenarioIds: ["telegram-help-command"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("removes migrated ids from non-YAML scenario ownership", async () => {
|
||||
const nonYamlIds = new Set((await loadNonYamlScenarioRefs()).map(({ id }) => id));
|
||||
|
||||
for (const scenarioId of [
|
||||
...TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
...WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
]) {
|
||||
expect(nonYamlIds.has(scenarioId), scenarioId).toBe(false);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,146 @@
|
|||
// Qa Lab plugin module defines canonical live-transport scenario delegation.
|
||||
import path from "node:path";
|
||||
import type { QaTransportAdapterFactory } from "../../qa-transport-registry.js";
|
||||
import { readQaScenarioPack } from "../../scenario-catalog.js";
|
||||
import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js";
|
||||
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
|
||||
|
||||
export const TELEGRAM_CANONICAL_SCENARIO_IDS = [
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-context-command",
|
||||
"telegram-current-session-status-tool",
|
||||
"telegram-tool-only-usage-footer",
|
||||
"telegram-reply-chain-exact-marker",
|
||||
] as const;
|
||||
|
||||
export const TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS = [
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-context-command",
|
||||
] as const;
|
||||
|
||||
export const WHATSAPP_CANONICAL_SCENARIO_IDS = [
|
||||
"whatsapp-help-command",
|
||||
"whatsapp-status-command",
|
||||
"whatsapp-commands-command",
|
||||
"whatsapp-tools-compact-command",
|
||||
"whatsapp-whoami-command",
|
||||
"whatsapp-context-command",
|
||||
"whatsapp-tool-only-usage-footer",
|
||||
"whatsapp-native-new-command",
|
||||
] as const;
|
||||
|
||||
export const WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS = ["whatsapp-help-command"] as const;
|
||||
|
||||
export const WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS = [
|
||||
"whatsapp-help-command",
|
||||
"whatsapp-commands-command",
|
||||
"whatsapp-tools-compact-command",
|
||||
"whatsapp-whoami-command",
|
||||
"whatsapp-context-command",
|
||||
"whatsapp-tool-only-usage-footer",
|
||||
"whatsapp-native-new-command",
|
||||
] as const;
|
||||
|
||||
export function whatsappDefaultCanonicalScenarioIds(providerMode: string) {
|
||||
return providerMode === "mock-openai"
|
||||
? [...WHATSAPP_MOCK_DEFAULT_CANONICAL_SCENARIO_IDS]
|
||||
: [...WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS];
|
||||
}
|
||||
|
||||
export type CanonicalScenarioPartition = {
|
||||
canonical: string[];
|
||||
legacy: string[];
|
||||
};
|
||||
|
||||
export function assertKnownScenarioIds(params: {
|
||||
ids: readonly string[];
|
||||
knownIds: readonly string[];
|
||||
laneLabel: string;
|
||||
}) {
|
||||
const knownIds = new Set(params.knownIds);
|
||||
const missingIds = params.ids.filter((id) => !knownIds.has(id));
|
||||
if (missingIds.length > 0) {
|
||||
throw new Error(`unknown ${params.laneLabel} QA scenario id(s): ${missingIds.join(", ")}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function partitionCanonicalScenarioIds(
|
||||
scenarioIds: readonly string[] | undefined,
|
||||
canonicalIds: readonly string[],
|
||||
): CanonicalScenarioPartition {
|
||||
const canonicalSet = new Set(canonicalIds);
|
||||
const canonical: string[] = [];
|
||||
const legacy: string[] = [];
|
||||
for (const scenarioId of scenarioIds ?? []) {
|
||||
(canonicalSet.has(scenarioId) ? canonical : legacy).push(scenarioId);
|
||||
}
|
||||
return { canonical, legacy };
|
||||
}
|
||||
|
||||
export function listCanonicalScenarios(params: {
|
||||
ids: readonly string[];
|
||||
defaultIds: readonly string[];
|
||||
}) {
|
||||
const requestedIds = new Set(params.ids);
|
||||
const defaultIds = new Set(params.defaultIds);
|
||||
return readQaScenarioPack()
|
||||
.scenarios.filter((scenario) => requestedIds.has(scenario.id))
|
||||
.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
rationale: scenario.objective,
|
||||
regressionRefs: scenario.regressionRefs ?? [],
|
||||
defaultEnabled: defaultIds.has(scenario.id),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function runCanonicalLiveScenarios(params: {
|
||||
channelId: string;
|
||||
factory: QaTransportAdapterFactory;
|
||||
options: LiveTransportQaCommandOptions & {
|
||||
providerMode: "mock-openai" | "aimock" | "live-frontier";
|
||||
repoRoot: string;
|
||||
};
|
||||
scenarioIds: string[];
|
||||
}) {
|
||||
return await runQaFlowSuiteFromRuntime({
|
||||
adapterFactories: [params.factory],
|
||||
adapterOptions: {
|
||||
repoRoot: params.options.repoRoot,
|
||||
...(params.options.credentialRole ? { credentialRole: params.options.credentialRole } : {}),
|
||||
...(params.options.credentialSource
|
||||
? { credentialSource: params.options.credentialSource }
|
||||
: {}),
|
||||
...(params.options.sutAccountId ? { sutAccountId: params.options.sutAccountId } : {}),
|
||||
},
|
||||
...(params.options.alternateModel ? { alternateModel: params.options.alternateModel } : {}),
|
||||
channelDriver: "live",
|
||||
channelId: params.channelId,
|
||||
concurrency: 1,
|
||||
...(params.options.fastMode !== undefined ? { fastMode: params.options.fastMode } : {}),
|
||||
...(params.options.outputDir ? { outputDir: params.options.outputDir } : {}),
|
||||
...(params.options.primaryModel ? { primaryModel: params.options.primaryModel } : {}),
|
||||
providerMode: params.options.providerMode,
|
||||
repoRoot: params.options.repoRoot,
|
||||
scenarioIds: params.scenarioIds,
|
||||
});
|
||||
}
|
||||
|
||||
export function canonicalScenarioOutputDir(
|
||||
options: LiveTransportQaCommandOptions,
|
||||
includesLegacyRun: boolean,
|
||||
) {
|
||||
return includesLegacyRun && options.outputDir
|
||||
? path.join(options.outputDir, "canonical")
|
||||
: options.outputDir;
|
||||
}
|
||||
|
|
@ -151,10 +151,16 @@ describe("live transport scenario helpers", () => {
|
|||
expect.arrayContaining(slackTesting.SLACK_QA_STANDARD_SCENARIO_IDS),
|
||||
);
|
||||
expect(lanes.get("telegram")).toEqual(
|
||||
expect.arrayContaining(telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS),
|
||||
expect.arrayContaining([
|
||||
"help-command",
|
||||
...telegramTesting.TELEGRAM_QA_STANDARD_SCENARIO_IDS,
|
||||
]),
|
||||
);
|
||||
expect(lanes.get("whatsapp")).toEqual(
|
||||
expect.arrayContaining(whatsAppTesting.WHATSAPP_QA_STANDARD_SCENARIO_IDS),
|
||||
expect.arrayContaining([
|
||||
"help-command",
|
||||
...whatsAppTesting.WHATSAPP_QA_STANDARD_SCENARIO_IDS,
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,14 +1,31 @@
|
|||
import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js";
|
||||
import {
|
||||
assertKnownScenarioIds,
|
||||
canonicalScenarioOutputDir,
|
||||
listCanonicalScenarios,
|
||||
partitionCanonicalScenarioIds,
|
||||
runCanonicalLiveScenarios,
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
} from "../shared/canonical-scenarios.js";
|
||||
// Qa Lab plugin module implements cli behavior.
|
||||
import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js";
|
||||
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
|
||||
import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js";
|
||||
import { createTelegramQaTransportAdapter } from "./adapter.runtime.js";
|
||||
import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js";
|
||||
|
||||
export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
if (runOptions.listScenarios) {
|
||||
for (const scenario of listTelegramQaScenarioCatalog(runOptions.providerMode)) {
|
||||
const scenarios = [
|
||||
...listCanonicalScenarios({
|
||||
ids: TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
defaultIds: TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS,
|
||||
}),
|
||||
...listTelegramQaScenarioCatalog(runOptions.providerMode),
|
||||
];
|
||||
for (const scenario of scenarios) {
|
||||
const defaultLabel = scenario.defaultEnabled ? "default" : "optional";
|
||||
const refs =
|
||||
scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : "";
|
||||
|
|
@ -18,7 +35,56 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions)
|
|||
}
|
||||
return;
|
||||
}
|
||||
const result = await runTelegramQaLive(runOptions);
|
||||
const selected = partitionCanonicalScenarioIds(
|
||||
runOptions.scenarioIds,
|
||||
TELEGRAM_CANONICAL_SCENARIO_IDS,
|
||||
);
|
||||
const hasExplicitScenarioIds = (runOptions.scenarioIds?.length ?? 0) > 0;
|
||||
if (hasExplicitScenarioIds) {
|
||||
assertKnownScenarioIds({
|
||||
ids: selected.legacy,
|
||||
knownIds: listTelegramQaScenarioCatalog(runOptions.providerMode).map(({ id }) => id),
|
||||
laneLabel: "Telegram",
|
||||
});
|
||||
}
|
||||
const canonicalScenarioIds = hasExplicitScenarioIds
|
||||
? selected.canonical
|
||||
: [...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS];
|
||||
const runsLegacyScenarios = !hasExplicitScenarioIds || selected.legacy.length > 0;
|
||||
if (canonicalScenarioIds.length > 0) {
|
||||
const canonical = await runCanonicalLiveScenarios({
|
||||
channelId: "telegram",
|
||||
factory: {
|
||||
id: "telegram",
|
||||
matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram",
|
||||
create: createTelegramQaTransportAdapter,
|
||||
},
|
||||
options: {
|
||||
...runOptions,
|
||||
outputDir: canonicalScenarioOutputDir(runOptions, runsLegacyScenarios),
|
||||
},
|
||||
scenarioIds: canonicalScenarioIds,
|
||||
});
|
||||
printLiveTransportQaArtifacts("Telegram canonical QA", {
|
||||
report: canonical.reportPath,
|
||||
summary: canonical.summaryPath,
|
||||
});
|
||||
if (!runOptions.allowFailures) {
|
||||
const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(
|
||||
canonical.summaryPath,
|
||||
);
|
||||
if (failedScenarioCount > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!runsLegacyScenarios) {
|
||||
return;
|
||||
}
|
||||
const result = await runTelegramQaLive({
|
||||
...runOptions,
|
||||
scenarioIds: hasExplicitScenarioIds ? selected.legacy : undefined,
|
||||
});
|
||||
printLiveTransportQaArtifacts("Telegram QA", {
|
||||
report: result.reportPath,
|
||||
summary: result.summaryPath,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS } from "../shared/canonical-scenarios.js";
|
||||
// Qa Lab plugin module implements cli behavior.
|
||||
import {
|
||||
createLazyCliRuntimeLoader,
|
||||
|
|
@ -24,7 +25,7 @@ export const telegramQaAdapterFactory: NonNullable<
|
|||
LiveTransportQaCliRegistration["adapterFactory"]
|
||||
> = {
|
||||
id: "telegram",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
scenarioIds: ["channel-chat-baseline", ...TELEGRAM_DEFAULT_CANONICAL_SCENARIO_IDS],
|
||||
matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram",
|
||||
async create(context) {
|
||||
return await (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context);
|
||||
|
|
|
|||
|
|
@ -622,9 +622,9 @@ describe("telegram live qa runtime", () => {
|
|||
});
|
||||
|
||||
it("fails when any requested Telegram scenario id is unknown", () => {
|
||||
expect(() => testing.findScenario(["telegram-help-command", "typo-scenario"])).toThrow(
|
||||
"unknown Telegram QA scenario id(s): typo-scenario",
|
||||
);
|
||||
expect(() =>
|
||||
testing.findScenario(["telegram-mentioned-message-reply", "typo-scenario"]),
|
||||
).toThrow("unknown Telegram QA scenario id(s): typo-scenario");
|
||||
});
|
||||
|
||||
it("recognizes Telegram observation timeouts with retry details", () => {
|
||||
|
|
@ -646,98 +646,26 @@ describe("telegram live qa runtime", () => {
|
|||
|
||||
it("includes mention gating in the Telegram live scenario catalog", () => {
|
||||
const scenarios = testing.findScenario([
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-other-bot-command-gating",
|
||||
"telegram-context-command",
|
||||
"telegram-current-session-status-tool",
|
||||
"telegram-tool-only-usage-footer",
|
||||
"telegram-mentioned-message-reply",
|
||||
"telegram-reply-chain-exact-marker",
|
||||
"telegram-stream-final-single-message",
|
||||
"telegram-long-final-reuses-preview",
|
||||
"telegram-long-final-three-chunks",
|
||||
"telegram-mention-gating",
|
||||
]);
|
||||
expect(scenarios.map((scenario) => scenario.id)).toEqual([
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-other-bot-command-gating",
|
||||
"telegram-context-command",
|
||||
"telegram-current-session-status-tool",
|
||||
"telegram-tool-only-usage-footer",
|
||||
"telegram-mentioned-message-reply",
|
||||
"telegram-reply-chain-exact-marker",
|
||||
"telegram-stream-final-single-message",
|
||||
"telegram-long-final-reuses-preview",
|
||||
"telegram-long-final-three-chunks",
|
||||
"telegram-mention-gating",
|
||||
]);
|
||||
expect(
|
||||
scenarios.find((scenario) => scenario.id === "telegram-status-command")?.buildRun("sut_bot")
|
||||
.steps[0].input,
|
||||
).toBe("/status@sut_bot");
|
||||
expect(
|
||||
scenarios.find((scenario) => scenario.id === "telegram-status-command")?.buildRun("sut_bot")
|
||||
.steps[0].expectedTextIncludes,
|
||||
).toEqual(["OpenClaw", "Model:", "Session:", "Activation:"]);
|
||||
expect(
|
||||
scenarios
|
||||
.find((scenario) => scenario.id === "telegram-repeated-command-authorization")
|
||||
?.buildRun("sut_bot").steps,
|
||||
).toHaveLength(4);
|
||||
const repeatedSteps = requireScenario(
|
||||
scenarios,
|
||||
"telegram-repeated-command-authorization",
|
||||
).buildRun("sut_bot").steps;
|
||||
expect(repeatedSteps[0]?.driverGroupAuthorization).toBe("deny");
|
||||
expect(repeatedSteps[0]?.input).toBe("/status@sut_bot");
|
||||
expect(repeatedSteps[0]?.expectReply).toBe(false);
|
||||
expect(repeatedSteps[1]?.driverGroupAuthorization).toBe("allow");
|
||||
expect(repeatedSteps[1]?.input).toBe("/status@sut_bot");
|
||||
expect(repeatedSteps[1]?.expectReply).toBe(true);
|
||||
expect(repeatedSteps[2]?.input).toBe("/help@sut_bot");
|
||||
expect(repeatedSteps[2]?.expectReply).toBe(true);
|
||||
expect(repeatedSteps[3]?.input).toBe("/commands@sut_bot");
|
||||
expect(repeatedSteps[3]?.expectReply).toBe(true);
|
||||
expect(repeatedSteps[3]?.expectedTextIncludes).toEqual(["Commands (1/", "/session", "/stop"]);
|
||||
const commandsStep = requireScenario(scenarios, "telegram-commands-command").buildRun("sut_bot")
|
||||
.steps[0];
|
||||
expect(commandsStep?.expectedTextIncludes).toEqual(["Commands (1/", "/session", "/stop"]);
|
||||
const otherBotStep = requireScenario(scenarios, "telegram-other-bot-command-gating").buildRun(
|
||||
"sut_bot",
|
||||
).steps[0];
|
||||
expect(otherBotStep?.expectReply).toBe(false);
|
||||
expect(otherBotStep?.input).toBe("/status@OpenClawQaOtherBot");
|
||||
const contextStep = requireScenario(scenarios, "telegram-context-command").buildRun("sut_bot")
|
||||
.steps[0];
|
||||
expect(contextStep?.matchText).toBe("/context list");
|
||||
const statusToolStep = requireScenario(
|
||||
scenarios,
|
||||
"telegram-current-session-status-tool",
|
||||
).buildRun("sut_bot").steps[0];
|
||||
expect(statusToolStep?.expectedTextIncludes).toEqual([
|
||||
"QA-TELEGRAM-CURRENT-SESSION-OK",
|
||||
":telegram:group:",
|
||||
]);
|
||||
expect(statusToolStep?.replyToLatestSutMessage).toBe(true);
|
||||
const usageFooterSteps = requireScenario(scenarios, "telegram-tool-only-usage-footer").buildRun(
|
||||
"sut_bot",
|
||||
).steps;
|
||||
expect(usageFooterSteps).toHaveLength(2);
|
||||
expect(usageFooterSteps[0]?.input).toBe("/usage@sut_bot tokens");
|
||||
expect(usageFooterSteps[0]?.expectedTextIncludes).toEqual(["Usage", "tokens"]);
|
||||
expect(usageFooterSteps[1]?.expectedTextIncludes?.at(-1)).toBe("Usage:");
|
||||
expect(usageFooterSteps[1]?.expectedSutMessageCount).toBe(2);
|
||||
expect(usageFooterSteps[1]?.replyToLatestSutMessage).toBe(true);
|
||||
expect(
|
||||
scenarios
|
||||
.find((scenario) => scenario.id === "telegram-mentioned-message-reply")
|
||||
|
|
@ -747,12 +675,6 @@ describe("telegram live qa runtime", () => {
|
|||
scenarios.find((scenario) => scenario.id === "telegram-mentioned-message-reply")
|
||||
?.evidenceCoverageIds,
|
||||
).toEqual(["channels.telegram.mention-gating"]);
|
||||
const replyChainStep = requireScenario(scenarios, "telegram-reply-chain-exact-marker").buildRun(
|
||||
"sut_bot",
|
||||
).steps[0];
|
||||
expect(replyChainStep?.expectedJoinedSutTextIncludes).toEqual(["QA-TELEGRAM-REPLY-CHAIN-OK"]);
|
||||
expect(replyChainStep?.expectedSutMessageCount).toBe(1);
|
||||
expect(replyChainStep?.replyToLatestSutMessage).toBeUndefined();
|
||||
const streamSingleStep = requireScenario(
|
||||
scenarios,
|
||||
"telegram-stream-final-single-message",
|
||||
|
|
@ -787,14 +709,7 @@ describe("telegram live qa runtime", () => {
|
|||
it("keeps mock-scripted Telegram checks out of the default live-frontier set", () => {
|
||||
expect(testing.findScenario(undefined, "live-frontier").map((scenario) => scenario.id)).toEqual(
|
||||
[
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-other-bot-command-gating",
|
||||
"telegram-context-command",
|
||||
"telegram-mentioned-message-reply",
|
||||
"telegram-mention-gating",
|
||||
],
|
||||
|
|
@ -803,45 +718,22 @@ describe("telegram live qa runtime", () => {
|
|||
|
||||
it("adds deterministic model-scripted checks to the default mock-openai set", () => {
|
||||
expect(testing.findScenario(undefined, "mock-openai").map((scenario) => scenario.id)).toEqual([
|
||||
"telegram-help-command",
|
||||
"telegram-commands-command",
|
||||
"telegram-tools-compact-command",
|
||||
"telegram-whoami-command",
|
||||
"telegram-status-command",
|
||||
"telegram-repeated-command-authorization",
|
||||
"telegram-other-bot-command-gating",
|
||||
"telegram-context-command",
|
||||
"telegram-mentioned-message-reply",
|
||||
"telegram-long-final-reuses-preview",
|
||||
"telegram-mention-gating",
|
||||
]);
|
||||
});
|
||||
|
||||
it("lists default status and regression refs in the Telegram scenario catalog", () => {
|
||||
it("lists remaining optional Telegram scenario metadata", () => {
|
||||
const catalog = testing.listTelegramQaScenarioCatalog("mock-openai");
|
||||
const status = requireScenario(catalog, "telegram-status-command");
|
||||
expect(status.defaultEnabled).toBe(true);
|
||||
expect(status.regressionRefs).toEqual(["openclaw/openclaw#74698"]);
|
||||
expect(requireScenario(catalog, "telegram-current-session-status-tool").defaultEnabled).toBe(
|
||||
false,
|
||||
);
|
||||
const usageFooter = requireScenario(catalog, "telegram-tool-only-usage-footer");
|
||||
expect(usageFooter.defaultEnabled).toBe(false);
|
||||
expect(usageFooter.regressionRefs).toEqual(["openclaw/openclaw#87392"]);
|
||||
const streamSingle = requireScenario(catalog, "telegram-stream-final-single-message");
|
||||
expect(streamSingle.defaultEnabled).toBe(false);
|
||||
expect(streamSingle.regressionRefs).toEqual(["openclaw/openclaw#39905"]);
|
||||
expect(requireScenario(catalog, "telegram-reply-chain-exact-marker").defaultEnabled).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it("tracks Telegram live coverage against the shared transport contract", () => {
|
||||
expect(testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS).toEqual([
|
||||
"canary",
|
||||
"help-command",
|
||||
"mention-gating",
|
||||
]);
|
||||
expect(testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]);
|
||||
expect(
|
||||
findMissingLiveTransportStandardScenarios({
|
||||
coveredStandardScenarioIds: testing.TELEGRAM_QA_STANDARD_SCENARIO_IDS,
|
||||
|
|
@ -1317,8 +1209,8 @@ describe("telegram live qa runtime", () => {
|
|||
initialOffset: 7,
|
||||
timeoutMs: 5_000,
|
||||
observedMessages,
|
||||
observationScenarioId: "telegram-whoami-command",
|
||||
observationScenarioTitle: "Telegram whoami reply",
|
||||
observationScenarioId: "telegram-mentioned-message-reply",
|
||||
observationScenarioTitle: "Telegram mentioned message gets a reply",
|
||||
predicate: (message) =>
|
||||
testing.matchesTelegramScenarioReply({
|
||||
groupId: "-100123",
|
||||
|
|
@ -1334,7 +1226,7 @@ describe("telegram live qa runtime", () => {
|
|||
expect(observedMessages).toHaveLength(1);
|
||||
expect(observedMessages[0]?.matchedScenario).toBe(true);
|
||||
expect(observedMessages[0]?.messageId).toBe(99);
|
||||
expect(observedMessages[0]?.scenarioId).toBe("telegram-whoami-command");
|
||||
expect(observedMessages[0]?.scenarioId).toBe("telegram-mentioned-message-reply");
|
||||
});
|
||||
|
||||
it("prints Telegram scenario RTT in the Markdown report", () => {
|
||||
|
|
|
|||
|
|
@ -70,26 +70,15 @@ type TelegramBotIdentity = {
|
|||
};
|
||||
|
||||
type TelegramQaScenarioId =
|
||||
| "telegram-help-command"
|
||||
| "telegram-commands-command"
|
||||
| "telegram-tools-compact-command"
|
||||
| "telegram-whoami-command"
|
||||
| "telegram-status-command"
|
||||
| "telegram-repeated-command-authorization"
|
||||
| "telegram-other-bot-command-gating"
|
||||
| "telegram-context-command"
|
||||
| "telegram-current-session-status-tool"
|
||||
| "telegram-tool-only-usage-footer"
|
||||
| "telegram-stream-final-single-message"
|
||||
| "telegram-long-final-three-chunks"
|
||||
| "telegram-long-final-reuses-preview"
|
||||
| "telegram-reply-chain-exact-marker"
|
||||
| "telegram-mentioned-message-reply"
|
||||
| "telegram-mention-gating";
|
||||
|
||||
type TelegramQaScenarioStep = {
|
||||
allowAnySutReply?: boolean;
|
||||
driverGroupAuthorization?: "allow" | "deny";
|
||||
expectReply: boolean;
|
||||
input: string;
|
||||
expectedTextIncludes?: string[];
|
||||
|
|
@ -221,18 +210,18 @@ type TelegramRichMessage = {
|
|||
|
||||
type TelegramMessage = Pick<TelegramBotMessage, "date" | "message_id"> &
|
||||
Partial<Pick<TelegramBotMessage, "caption" | "text">> & {
|
||||
audio?: unknown;
|
||||
chat: { id: number };
|
||||
document?: unknown;
|
||||
from?: Pick<NonNullable<TelegramBotMessage["from"]>, "id" | "is_bot" | "username">;
|
||||
photo?: unknown[];
|
||||
rich_message?: TelegramRichMessage;
|
||||
reply_markup?: TelegramReplyMarkup;
|
||||
reply_to_message?: { message_id?: number };
|
||||
sticker?: unknown;
|
||||
video?: unknown;
|
||||
voice?: unknown;
|
||||
};
|
||||
audio?: unknown;
|
||||
chat: { id: number };
|
||||
document?: unknown;
|
||||
from?: Pick<NonNullable<TelegramBotMessage["from"]>, "id" | "is_bot" | "username">;
|
||||
photo?: unknown[];
|
||||
rich_message?: TelegramRichMessage;
|
||||
reply_markup?: TelegramReplyMarkup;
|
||||
reply_to_message?: { message_id?: number };
|
||||
sticker?: unknown;
|
||||
video?: unknown;
|
||||
voice?: unknown;
|
||||
};
|
||||
|
||||
type TelegramUpdate = Pick<TelegramBotUpdate, "update_id"> & {
|
||||
edited_message?: TelegramMessage;
|
||||
|
|
@ -251,102 +240,6 @@ function telegramQaStepRun(step: TelegramQaScenarioStep): TelegramQaScenarioRun
|
|||
}
|
||||
|
||||
const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [
|
||||
{
|
||||
id: "telegram-help-command",
|
||||
standardId: "help-command",
|
||||
title: "Telegram help command reply",
|
||||
rationale: "Canary-grade native command reply path.",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `/help@${sutUsername}`,
|
||||
expectedTextIncludes: ["/new", "/commands for full list"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-commands-command",
|
||||
title: "Telegram commands list reply",
|
||||
rationale: "Native command catalog must render in Telegram group replies.",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `/commands@${sutUsername}`,
|
||||
expectedTextIncludes: ["Commands (1/", "/session", "/stop"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-tools-compact-command",
|
||||
title: "Telegram tools compact reply",
|
||||
rationale: "Tool catalog rendering catches command dispatch plus model-tool inventory drift.",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `/tools@${sutUsername} compact`,
|
||||
expectedTextIncludes: ["exec", "Use /tools verbose for descriptions."],
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-whoami-command",
|
||||
title: "Telegram whoami reply",
|
||||
rationale: "Identity command proves Telegram channel context is attached to native commands.",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `/whoami@${sutUsername}`,
|
||||
expectedTextIncludes: ["🧭 Identity", "Channel: telegram"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-status-command",
|
||||
title: "Telegram status command reply",
|
||||
rationale: "Recent Telegram group regressions broke /status while normal chat still worked.",
|
||||
regressionRefs: ["openclaw/openclaw#74698"],
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `/status@${sutUsername}`,
|
||||
expectedTextIncludes: ["OpenClaw", "Model:", "Session:", "Activation:"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-repeated-command-authorization",
|
||||
title: "Telegram repeated command authorization",
|
||||
rationale:
|
||||
"Allowlisted bot-to-bot operators should not hit a fresh auth gate for each native slash command.",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUsername) => {
|
||||
const steps = [
|
||||
{
|
||||
driverGroupAuthorization: "deny",
|
||||
expectReply: false,
|
||||
input: `/status@${sutUsername}`,
|
||||
timeoutMs: 8_000,
|
||||
},
|
||||
{
|
||||
driverGroupAuthorization: "allow",
|
||||
expectReply: true,
|
||||
input: `/status@${sutUsername}`,
|
||||
expectedTextIncludes: ["OpenClaw", "Session:"],
|
||||
},
|
||||
{
|
||||
expectReply: true,
|
||||
input: `/help@${sutUsername}`,
|
||||
expectedTextIncludes: ["/new", "/commands for full list"],
|
||||
},
|
||||
{
|
||||
expectReply: true,
|
||||
input: `/commands@${sutUsername}`,
|
||||
expectedTextIncludes: ["Commands (1/", "/session", "/stop"],
|
||||
},
|
||||
] satisfies TelegramQaScenarioStep[];
|
||||
return { steps };
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "telegram-other-bot-command-gating",
|
||||
title: "Telegram command addressed to another bot is ignored",
|
||||
|
|
@ -358,65 +251,6 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [
|
|||
input: "/status@OpenClawQaOtherBot",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-context-command",
|
||||
title: "Telegram context reply",
|
||||
rationale: "Context command exercises native command routing into Telegram-specific help text.",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `/context@${sutUsername}`,
|
||||
matchText: "/context list",
|
||||
expectedTextIncludes: ["/context list", "Inline shortcut"],
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-current-session-status-tool",
|
||||
title: "Telegram current session_status tool call",
|
||||
defaultEnabled: false,
|
||||
rationale:
|
||||
"Opt-in threaded probe for current Telegram group session resolution through model tools.",
|
||||
timeoutMs: 60_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `@${sutUsername} Telegram current session_status QA check. Call session_status with sessionKey set to current, then reply with the exact QA marker and resolved session key.`,
|
||||
expectedTextIncludes: ["QA-TELEGRAM-CURRENT-SESSION-OK", ":telegram:group:"],
|
||||
replyToLatestSutMessage: true,
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-tool-only-usage-footer",
|
||||
title: "Telegram tool-only reply includes usage footer",
|
||||
defaultEnabled: false,
|
||||
rationale:
|
||||
"Opt-in real Telegram proof that /usage tokens decorates message-tool-only visible replies.",
|
||||
regressionRefs: ["openclaw/openclaw#87392"],
|
||||
timeoutMs: 90_000,
|
||||
buildRun: (sutUsername) => {
|
||||
const marker = `QA-TELEGRAM-USAGE-FOOTER-${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return {
|
||||
steps: [
|
||||
{
|
||||
expectReply: true,
|
||||
input: `/usage@${sutUsername} tokens`,
|
||||
expectedTextIncludes: ["Usage", "tokens"],
|
||||
},
|
||||
{
|
||||
allowAnySutReply: true,
|
||||
expectReply: true,
|
||||
input: `@${sutUsername} Telegram usage footer QA. Reply exactly: ${marker}`,
|
||||
expectedTextIncludes: [marker, "Usage:"],
|
||||
expectedJoinedSutTextIncludes: [marker, "Usage:"],
|
||||
expectedSutMessageCount: 2,
|
||||
replyToLatestSutMessage: true,
|
||||
settleMs: 4_000,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "telegram-mentioned-message-reply",
|
||||
title: "Telegram mentioned message gets a reply",
|
||||
|
|
@ -440,24 +274,6 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [
|
|||
});
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "telegram-reply-chain-exact-marker",
|
||||
title: "Telegram reply-chain exact marker",
|
||||
defaultEnabled: false,
|
||||
defaultProviderModes: ["mock-openai"],
|
||||
rationale:
|
||||
"Opt-in mock-backed exact-marker check for Telegram final text through reply handling.",
|
||||
timeoutMs: 75_000,
|
||||
buildRun: (sutUsername) =>
|
||||
telegramQaStepRun({
|
||||
expectReply: true,
|
||||
input: `@${sutUsername} Telegram reply-chain marker QA. Reply exactly: QA-TELEGRAM-REPLY-CHAIN-OK`,
|
||||
expectedTextIncludes: ["QA-TELEGRAM-REPLY-CHAIN-OK"],
|
||||
expectedJoinedSutTextIncludes: ["QA-TELEGRAM-REPLY-CHAIN-OK"],
|
||||
expectedSutMessageCount: 1,
|
||||
settleMs: 4_000,
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "telegram-stream-final-single-message",
|
||||
title: "Telegram streamed final stays one message",
|
||||
|
|
@ -564,14 +380,6 @@ function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof TELEGRAM_QA_ENV_KE
|
|||
return value;
|
||||
}
|
||||
|
||||
function readConfigRecord(root: Record<string, unknown>, key: string): Record<string, unknown> {
|
||||
const value = root[key];
|
||||
if (!isRecord(value)) {
|
||||
throw new Error(`Telegram QA config missing object at ${key}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function shouldLogTelegramQaLiveProgress(env: NodeJS.ProcessEnv = process.env) {
|
||||
const override = parseTelegramQaProgressBooleanEnv(env[QA_SUITE_PROGRESS_ENV]);
|
||||
if (override !== undefined) {
|
||||
|
|
@ -1280,34 +1088,6 @@ async function waitForTelegramChannelRunning(
|
|||
throw new Error(`telegram account "${accountId}" did not become ready${details}`);
|
||||
}
|
||||
|
||||
async function setTelegramQaDriverGroupAuthorization(params: {
|
||||
driverBotId: number;
|
||||
env: NodeJS.ProcessEnv;
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
|
||||
groupId: string;
|
||||
sutAccountId: string;
|
||||
authorized: boolean;
|
||||
}) {
|
||||
await params.gateway.restartAfterStateMutation(async ({ configPath }) => {
|
||||
const parsed: unknown = JSON.parse(await fs.readFile(configPath, "utf8"));
|
||||
if (!isRecord(parsed)) {
|
||||
throw new Error("Telegram QA config root must be an object");
|
||||
}
|
||||
const channels = readConfigRecord(parsed, "channels");
|
||||
const telegram = readConfigRecord(channels, "telegram");
|
||||
const accounts = readConfigRecord(telegram, "accounts");
|
||||
const account = readConfigRecord(accounts, params.sutAccountId);
|
||||
const groups = readConfigRecord(account, "groups");
|
||||
const group = readConfigRecord(groups, params.groupId);
|
||||
group.allowFrom = params.authorized ? [String(params.driverBotId)] : [];
|
||||
await fs.writeFile(configPath, `${JSON.stringify(parsed, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
});
|
||||
await waitForTelegramChannelRunning(params.gateway, params.sutAccountId, { env: params.env });
|
||||
}
|
||||
|
||||
function renderTelegramQaMarkdown(params: {
|
||||
cleanupIssues: string[];
|
||||
credentialSource: "convex" | "env";
|
||||
|
|
@ -1975,17 +1755,6 @@ export async function runTelegramQaLive(params: {
|
|||
let lastMatched: Awaited<ReturnType<typeof waitForObservedMessage>> | undefined;
|
||||
let lastSentMessageId: number | undefined;
|
||||
for (const step of scenarioSteps) {
|
||||
if (step.driverGroupAuthorization) {
|
||||
await setTelegramQaDriverGroupAuthorization({
|
||||
driverBotId: driverIdentity.id,
|
||||
env,
|
||||
gateway: gatewayHarness.gateway,
|
||||
groupId: runtimeEnv.groupId,
|
||||
sutAccountId,
|
||||
authorized: step.driverGroupAuthorization === "allow",
|
||||
});
|
||||
driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken);
|
||||
}
|
||||
driverOffset = await flushTelegramUpdates(runtimeEnv.driverToken);
|
||||
const stepResult = await runTelegramQaScenarioStep({
|
||||
driverOffset,
|
||||
|
|
|
|||
|
|
@ -2,16 +2,26 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { QA_EVIDENCE_FILENAME } from "../../evidence-summary.js";
|
||||
import { runQaWhatsAppCommand } from "./cli.runtime.js";
|
||||
|
||||
const runWhatsAppQaLiveMock = vi.hoisted(() => vi.fn());
|
||||
const { listWhatsAppQaScenarioCatalogMock, runCanonicalLiveScenariosMock, runWhatsAppQaLiveMock } =
|
||||
vi.hoisted(() => ({
|
||||
listWhatsAppQaScenarioCatalogMock: vi.fn(),
|
||||
runCanonicalLiveScenariosMock: vi.fn(),
|
||||
runWhatsAppQaLiveMock: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../shared/live-artifacts.js", () => ({
|
||||
printLiveTransportQaArtifacts: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../shared/canonical-scenarios.js", async (importOriginal) => ({
|
||||
...(await importOriginal()),
|
||||
runCanonicalLiveScenarios: runCanonicalLiveScenariosMock,
|
||||
}));
|
||||
|
||||
vi.mock("../shared/live-transport-cli.runtime.js", () => ({
|
||||
resolveLiveTransportQaRunOptions: (opts: Record<string, unknown>) => ({
|
||||
outputDir: opts.repoRoot,
|
||||
|
|
@ -22,15 +32,26 @@ vi.mock("../shared/live-transport-cli.runtime.js", () => ({
|
|||
}));
|
||||
|
||||
vi.mock("./whatsapp-live.runtime.js", () => ({
|
||||
listWhatsAppQaScenarioCatalog: listWhatsAppQaScenarioCatalogMock,
|
||||
runWhatsAppQaLive: runWhatsAppQaLiveMock,
|
||||
}));
|
||||
|
||||
vi.mock("./adapter.runtime.js", () => ({
|
||||
createWhatsAppQaTransportAdapter: vi.fn(),
|
||||
}));
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
let originalExitCode: typeof process.exitCode;
|
||||
|
||||
beforeEach(() => {
|
||||
listWhatsAppQaScenarioCatalogMock.mockReturnValue([{ id: "whatsapp-canary" }]);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
process.exitCode = originalExitCode;
|
||||
runCanonicalLiveScenariosMock.mockReset();
|
||||
runWhatsAppQaLiveMock.mockReset();
|
||||
listWhatsAppQaScenarioCatalogMock.mockReset();
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
await fs.rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
|
@ -88,6 +109,12 @@ describe("WhatsApp QA CLI runtime", () => {
|
|||
scenarios: [],
|
||||
summaryPath,
|
||||
});
|
||||
runCanonicalLiveScenariosMock.mockResolvedValueOnce({
|
||||
outputDir,
|
||||
reportPath: path.join(outputDir, "canonical-report.md"),
|
||||
scenarios: [],
|
||||
summaryPath,
|
||||
});
|
||||
|
||||
await runQaWhatsAppCommand({ repoRoot: outputDir });
|
||||
|
||||
|
|
@ -104,9 +131,75 @@ describe("WhatsApp QA CLI runtime", () => {
|
|||
scenarios: [],
|
||||
summaryPath,
|
||||
});
|
||||
runCanonicalLiveScenariosMock.mockResolvedValueOnce({
|
||||
outputDir,
|
||||
reportPath: path.join(outputDir, "canonical-report.md"),
|
||||
scenarios: [],
|
||||
summaryPath,
|
||||
});
|
||||
|
||||
await runQaWhatsAppCommand({ allowFailures: true, repoRoot: outputDir });
|
||||
|
||||
expect(process.exitCode).toBeUndefined();
|
||||
});
|
||||
|
||||
it("delegates canonical WhatsApp scenario ids without starting the legacy runner", async () => {
|
||||
originalExitCode = process.exitCode;
|
||||
process.exitCode = undefined;
|
||||
const { outputDir, summaryPath } = await writeSummary(makeEvidenceSummary("pass"));
|
||||
runCanonicalLiveScenariosMock.mockResolvedValueOnce({
|
||||
outputDir,
|
||||
reportPath: path.join(outputDir, "canonical-report.md"),
|
||||
scenarios: [],
|
||||
summaryPath,
|
||||
});
|
||||
|
||||
await runQaWhatsAppCommand({
|
||||
repoRoot: outputDir,
|
||||
scenarioIds: ["whatsapp-help-command"],
|
||||
});
|
||||
|
||||
expect(runCanonicalLiveScenariosMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
channelId: "whatsapp",
|
||||
scenarioIds: ["whatsapp-help-command"],
|
||||
}),
|
||||
);
|
||||
expect(runWhatsAppQaLiveMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps remaining WhatsApp defaults when Commander supplies an empty scenario list", async () => {
|
||||
const { outputDir, summaryPath } = await writeSummary(makeEvidenceSummary("pass"));
|
||||
runWhatsAppQaLiveMock.mockResolvedValueOnce({
|
||||
observedMessagesPath: path.join(outputDir, "observed.json"),
|
||||
reportPath: path.join(outputDir, "report.md"),
|
||||
scenarios: [],
|
||||
summaryPath,
|
||||
});
|
||||
runCanonicalLiveScenariosMock.mockResolvedValueOnce({
|
||||
outputDir,
|
||||
reportPath: path.join(outputDir, "canonical-report.md"),
|
||||
scenarios: [],
|
||||
summaryPath,
|
||||
});
|
||||
|
||||
await runQaWhatsAppCommand({ repoRoot: outputDir, scenarioIds: [] });
|
||||
|
||||
expect(runCanonicalLiveScenariosMock).toHaveBeenCalled();
|
||||
expect(runWhatsAppQaLiveMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ scenarioIds: undefined }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects unknown mixed WhatsApp selections before starting canonical scenarios", async () => {
|
||||
await expect(
|
||||
runQaWhatsAppCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
scenarioIds: ["whatsapp-help-command", "missing-whatsapp-scenario"],
|
||||
}),
|
||||
).rejects.toThrow("unknown WhatsApp QA scenario id(s): missing-whatsapp-scenario");
|
||||
|
||||
expect(runCanonicalLiveScenariosMock).not.toHaveBeenCalled();
|
||||
expect(runWhatsAppQaLiveMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,13 +1,71 @@
|
|||
import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "../../suite-summary.js";
|
||||
import {
|
||||
assertKnownScenarioIds,
|
||||
canonicalScenarioOutputDir,
|
||||
partitionCanonicalScenarioIds,
|
||||
runCanonicalLiveScenarios,
|
||||
WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
whatsappDefaultCanonicalScenarioIds,
|
||||
} from "../shared/canonical-scenarios.js";
|
||||
// Qa Lab plugin module implements cli behavior.
|
||||
import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js";
|
||||
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
|
||||
import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js";
|
||||
import { runWhatsAppQaLive } from "./whatsapp-live.runtime.js";
|
||||
import { createWhatsAppQaTransportAdapter } from "./adapter.runtime.js";
|
||||
import { listWhatsAppQaScenarioCatalog, runWhatsAppQaLive } from "./whatsapp-live.runtime.js";
|
||||
|
||||
export async function runQaWhatsAppCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
const result = await runWhatsAppQaLive(runOptions);
|
||||
const selected = partitionCanonicalScenarioIds(
|
||||
runOptions.scenarioIds,
|
||||
WHATSAPP_CANONICAL_SCENARIO_IDS,
|
||||
);
|
||||
const hasExplicitScenarioIds = (runOptions.scenarioIds?.length ?? 0) > 0;
|
||||
if (hasExplicitScenarioIds) {
|
||||
assertKnownScenarioIds({
|
||||
ids: selected.legacy,
|
||||
knownIds: listWhatsAppQaScenarioCatalog().map(({ id }) => id),
|
||||
laneLabel: "WhatsApp",
|
||||
});
|
||||
}
|
||||
const canonicalScenarioIds = hasExplicitScenarioIds
|
||||
? selected.canonical
|
||||
: whatsappDefaultCanonicalScenarioIds(runOptions.providerMode);
|
||||
const runsLegacyScenarios = !hasExplicitScenarioIds || selected.legacy.length > 0;
|
||||
if (canonicalScenarioIds.length > 0) {
|
||||
const canonical = await runCanonicalLiveScenarios({
|
||||
channelId: "whatsapp",
|
||||
factory: {
|
||||
id: "whatsapp",
|
||||
matches: ({ channelId, driver }) => driver === "live" && channelId === "whatsapp",
|
||||
create: createWhatsAppQaTransportAdapter,
|
||||
},
|
||||
options: {
|
||||
...runOptions,
|
||||
outputDir: canonicalScenarioOutputDir(runOptions, runsLegacyScenarios),
|
||||
},
|
||||
scenarioIds: canonicalScenarioIds,
|
||||
});
|
||||
printLiveTransportQaArtifacts("WhatsApp canonical QA", {
|
||||
report: canonical.reportPath,
|
||||
summary: canonical.summaryPath,
|
||||
});
|
||||
if (!runOptions.allowFailures) {
|
||||
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
||||
canonical.summaryPath,
|
||||
);
|
||||
if (blockingScenarioCount > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!runsLegacyScenarios) {
|
||||
return;
|
||||
}
|
||||
const result = await runWhatsAppQaLive({
|
||||
...runOptions,
|
||||
scenarioIds: hasExplicitScenarioIds ? selected.legacy : undefined,
|
||||
});
|
||||
printLiveTransportQaArtifacts("WhatsApp QA", {
|
||||
report: result.reportPath,
|
||||
summary: result.summaryPath,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import { WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS } from "../shared/canonical-scenarios.js";
|
||||
// Qa Lab plugin module implements cli behavior.
|
||||
import {
|
||||
createLazyCliRuntimeLoader,
|
||||
|
|
@ -24,7 +25,7 @@ export const whatsappQaAdapterFactory: NonNullable<
|
|||
LiveTransportQaCliRegistration["adapterFactory"]
|
||||
> = {
|
||||
id: "whatsapp",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
scenarioIds: ["dm-chat-baseline", ...WHATSAPP_LIVE_DEFAULT_CANONICAL_SCENARIO_IDS],
|
||||
matches: ({ channelId, driver }) => driver === "live" && channelId === "whatsapp",
|
||||
async create(context) {
|
||||
return await (await loadWhatsAppQaAdapterRuntime()).createWhatsAppQaTransportAdapter(context);
|
||||
|
|
|
|||
|
|
@ -1321,7 +1321,6 @@ describe("WhatsApp QA live runtime", () => {
|
|||
"canary",
|
||||
"mention-gating",
|
||||
"top-level-reply-shape",
|
||||
"help-command",
|
||||
"quote-reply",
|
||||
"reaction-observation",
|
||||
"allowlist-block",
|
||||
|
|
@ -1578,7 +1577,6 @@ describe("WhatsApp QA live runtime", () => {
|
|||
"whatsapp-canary",
|
||||
"whatsapp-mention-gating",
|
||||
"whatsapp-top-level-reply-shape",
|
||||
"whatsapp-help-command",
|
||||
"whatsapp-reply-to-message",
|
||||
"whatsapp-group-reply-to-message",
|
||||
"whatsapp-status-reactions",
|
||||
|
|
@ -1602,12 +1600,6 @@ describe("WhatsApp QA live runtime", () => {
|
|||
"whatsapp-group-activation-always",
|
||||
"whatsapp-group-reply-to-bot-triggers",
|
||||
"whatsapp-top-level-reply-shape",
|
||||
"whatsapp-help-command",
|
||||
"whatsapp-commands-command",
|
||||
"whatsapp-tools-compact-command",
|
||||
"whatsapp-whoami-command",
|
||||
"whatsapp-context-command",
|
||||
"whatsapp-tool-only-usage-footer",
|
||||
"whatsapp-reply-to-message",
|
||||
"whatsapp-group-reply-to-message",
|
||||
"whatsapp-reply-to-mode-batched",
|
||||
|
|
@ -1630,7 +1622,6 @@ describe("WhatsApp QA live runtime", () => {
|
|||
"whatsapp-group-audio-gating",
|
||||
"whatsapp-reply-delivery-shape",
|
||||
"whatsapp-stream-final-message-accounting",
|
||||
"whatsapp-native-new-command",
|
||||
"whatsapp-status-reactions",
|
||||
"whatsapp-status-reaction-lifecycle",
|
||||
"whatsapp-group-allowlist-block",
|
||||
|
|
@ -2274,75 +2265,6 @@ describe("WhatsApp QA live runtime", () => {
|
|||
).toBe("safe local diagnostic");
|
||||
});
|
||||
|
||||
it("adds WhatsApp command UX parity scenarios to the mock-backed selection", () => {
|
||||
const scenarios = testing.findScenarios([
|
||||
"whatsapp-commands-command",
|
||||
"whatsapp-tools-compact-command",
|
||||
"whatsapp-whoami-command",
|
||||
"whatsapp-context-command",
|
||||
"whatsapp-tool-only-usage-footer",
|
||||
]);
|
||||
|
||||
expect(
|
||||
scenarios.map((scenario) => {
|
||||
const run = scenario.buildRun();
|
||||
if (run.kind === "approval") {
|
||||
throw new Error(`${scenario.id} unexpectedly built an approval run`);
|
||||
}
|
||||
return [
|
||||
scenario.id,
|
||||
run.input,
|
||||
String(run.matchText),
|
||||
run.expectedJoinedSutTextIncludes,
|
||||
run.expectedSutMessageCountRange,
|
||||
] as const;
|
||||
}),
|
||||
).toEqual([
|
||||
[
|
||||
"whatsapp-commands-command",
|
||||
"/commands",
|
||||
"/Commands \\(|\\/session|\\/verbose/iu",
|
||||
["/session", "/verbose"],
|
||||
undefined,
|
||||
],
|
||||
[
|
||||
"whatsapp-tools-compact-command",
|
||||
"/tools compact",
|
||||
"/Available tools|exec|Use \\/tools verbose for descriptions/iu",
|
||||
["exec", "Use /tools verbose for descriptions"],
|
||||
undefined,
|
||||
],
|
||||
[
|
||||
"whatsapp-whoami-command",
|
||||
"/whoami",
|
||||
"/(?=.*Identity)(?=.*Channel: whatsapp)(?=.*AllowFrom:)/isu",
|
||||
undefined,
|
||||
undefined,
|
||||
],
|
||||
[
|
||||
"whatsapp-context-command",
|
||||
"/context list",
|
||||
"/(?=.*Context breakdown)(?=.*Workspace:)(?=.*Tool schemas)/isu",
|
||||
undefined,
|
||||
undefined,
|
||||
],
|
||||
[
|
||||
"whatsapp-tool-only-usage-footer",
|
||||
"/usage tokens",
|
||||
"/Usage footer: tokens/iu",
|
||||
undefined,
|
||||
undefined,
|
||||
],
|
||||
]);
|
||||
expect(scenarios.map((scenario) => scenario.defaultProviderModes)).toEqual([
|
||||
["mock-openai"],
|
||||
["mock-openai"],
|
||||
["mock-openai"],
|
||||
["mock-openai"],
|
||||
["mock-openai"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("defines WhatsApp final-message accounting as a settled two-chunk assertion", () => {
|
||||
const [scenario] = testing.findScenarios(["whatsapp-stream-final-message-accounting"]);
|
||||
const run = scenario.buildRun();
|
||||
|
|
|
|||
|
|
@ -68,8 +68,6 @@ type WhatsAppQaScenarioId =
|
|||
| "whatsapp-audio-preflight"
|
||||
| "whatsapp-broadcast-group-fanout"
|
||||
| "whatsapp-canary"
|
||||
| "whatsapp-commands-command"
|
||||
| "whatsapp-context-command"
|
||||
| "whatsapp-group-allowlist-block"
|
||||
| "whatsapp-group-activation-always"
|
||||
| "whatsapp-group-agent-message-action-react"
|
||||
|
|
@ -82,11 +80,9 @@ type WhatsAppQaScenarioId =
|
|||
| "whatsapp-group-reply-to-bot-triggers"
|
||||
| "whatsapp-group-reply-to-message"
|
||||
| "whatsapp-inbound-reaction-no-trigger"
|
||||
| "whatsapp-help-command"
|
||||
| "whatsapp-inbound-image-caption"
|
||||
| "whatsapp-inbound-structured-messages"
|
||||
| "whatsapp-message-actions"
|
||||
| "whatsapp-native-new-command"
|
||||
| "whatsapp-outbound-document-preserves-filename"
|
||||
| "whatsapp-outbound-media-matrix"
|
||||
| "whatsapp-outbound-poll"
|
||||
|
|
@ -97,13 +93,9 @@ type WhatsAppQaScenarioId =
|
|||
| "whatsapp-reply-to-message"
|
||||
| "whatsapp-reply-to-mode-batched"
|
||||
| "whatsapp-stream-final-message-accounting"
|
||||
| "whatsapp-status-command"
|
||||
| "whatsapp-status-reaction-lifecycle"
|
||||
| "whatsapp-status-reactions"
|
||||
| "whatsapp-top-level-reply-shape"
|
||||
| "whatsapp-tools-compact-command"
|
||||
| "whatsapp-tool-only-usage-footer"
|
||||
| "whatsapp-whoami-command"
|
||||
| "whatsapp-approval-exec-native"
|
||||
| "whatsapp-approval-plugin-native";
|
||||
|
||||
|
|
@ -127,8 +119,6 @@ const WHATSAPP_QA_SCENARIO_POSTURES = {
|
|||
"whatsapp-audio-preflight": "user-path",
|
||||
"whatsapp-broadcast-group-fanout": "user-path",
|
||||
"whatsapp-canary": "user-path",
|
||||
"whatsapp-commands-command": "user-path",
|
||||
"whatsapp-context-command": "user-path",
|
||||
"whatsapp-group-activation-always": "user-path",
|
||||
"whatsapp-group-allowlist-block": "user-path",
|
||||
"whatsapp-group-agent-message-action-react": "user-path",
|
||||
|
|
@ -140,13 +130,11 @@ const WHATSAPP_QA_SCENARIO_POSTURES = {
|
|||
"whatsapp-group-pending-history-context": "user-path",
|
||||
"whatsapp-group-reply-to-bot-triggers": "user-path",
|
||||
"whatsapp-group-reply-to-message": "user-path",
|
||||
"whatsapp-help-command": "user-path",
|
||||
"whatsapp-inbound-image-caption": "user-path",
|
||||
"whatsapp-inbound-reaction-no-trigger": "user-path",
|
||||
"whatsapp-inbound-structured-messages": "user-path",
|
||||
"whatsapp-mention-gating": "user-path",
|
||||
"whatsapp-message-actions": "direct-gateway",
|
||||
"whatsapp-native-new-command": "user-path",
|
||||
"whatsapp-outbound-document-preserves-filename": "direct-gateway",
|
||||
"whatsapp-outbound-media-matrix": "direct-gateway",
|
||||
"whatsapp-outbound-poll": "direct-gateway",
|
||||
|
|
@ -155,14 +143,10 @@ const WHATSAPP_QA_SCENARIO_POSTURES = {
|
|||
"whatsapp-reply-delivery-shape": "direct-gateway",
|
||||
"whatsapp-reply-to-message": "user-path",
|
||||
"whatsapp-reply-to-mode-batched": "user-path",
|
||||
"whatsapp-status-command": "user-path",
|
||||
"whatsapp-status-reaction-lifecycle": "user-path",
|
||||
"whatsapp-status-reactions": "user-path",
|
||||
"whatsapp-stream-final-message-accounting": "user-path",
|
||||
"whatsapp-tool-only-usage-footer": "user-path",
|
||||
"whatsapp-tools-compact-command": "user-path",
|
||||
"whatsapp-top-level-reply-shape": "user-path",
|
||||
"whatsapp-whoami-command": "user-path",
|
||||
} satisfies Record<WhatsAppQaScenarioId, WhatsAppQaScenarioPosture>;
|
||||
|
||||
type WhatsAppQaMessageSendMode =
|
||||
|
|
@ -881,120 +865,6 @@ const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [
|
|||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "whatsapp-help-command",
|
||||
standardId: "help-command",
|
||||
title: "WhatsApp help command replies",
|
||||
timeoutMs: 60_000,
|
||||
buildRun: () => ({
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
input: "/help",
|
||||
matchText: /OpenClaw|commands|status|\/new/iu,
|
||||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-status-command",
|
||||
title: "WhatsApp status command replies",
|
||||
timeoutMs: 60_000,
|
||||
buildRun: () => ({
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
input: "/status",
|
||||
matchText: /OpenClaw|status|session|agent/iu,
|
||||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-commands-command",
|
||||
title: "WhatsApp commands list replies",
|
||||
defaultProviderModes: ["mock-openai"],
|
||||
timeoutMs: 60_000,
|
||||
buildRun: () => ({
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
expectedJoinedSutTextIncludes: ["/session", "/verbose"],
|
||||
input: "/commands",
|
||||
matchText: /Commands \(|\/session|\/verbose/iu,
|
||||
settleMs: 4_000,
|
||||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-tools-compact-command",
|
||||
title: "WhatsApp tools compact reply",
|
||||
defaultProviderModes: ["mock-openai"],
|
||||
timeoutMs: 60_000,
|
||||
buildRun: () => ({
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
expectedJoinedSutTextIncludes: ["exec", "Use /tools verbose for descriptions"],
|
||||
input: "/tools compact",
|
||||
matchText: /Available tools|exec|Use \/tools verbose for descriptions/iu,
|
||||
settleMs: 4_000,
|
||||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-whoami-command",
|
||||
title: "WhatsApp whoami reply",
|
||||
defaultProviderModes: ["mock-openai"],
|
||||
timeoutMs: 60_000,
|
||||
buildRun: () => ({
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
input: "/whoami",
|
||||
matchText: /(?=.*Identity)(?=.*Channel: whatsapp)(?=.*AllowFrom:)/isu,
|
||||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-context-command",
|
||||
title: "WhatsApp context list reply",
|
||||
defaultProviderModes: ["mock-openai"],
|
||||
timeoutMs: 60_000,
|
||||
buildRun: () => ({
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
input: "/context list",
|
||||
matchText: /(?=.*Context breakdown)(?=.*Workspace:)(?=.*Tool schemas)/isu,
|
||||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-tool-only-usage-footer",
|
||||
title: "WhatsApp tool-only reply includes usage footer",
|
||||
defaultProviderModes: ["mock-openai"],
|
||||
timeoutMs: 120_000,
|
||||
buildRun: () => {
|
||||
const token = `WHATSAPP_QA_USAGE_FOOTER_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return {
|
||||
afterReply: async (_reply, context) => {
|
||||
const usageStartedAt = new Date();
|
||||
await context.driver.sendText(
|
||||
context.target,
|
||||
`Reply with only this exact marker after usage footer setup: ${token}`,
|
||||
);
|
||||
const usageReply = await context.driver.waitForMessage({
|
||||
observedAfter: usageStartedAt,
|
||||
timeoutMs: 60_000,
|
||||
match: (message) =>
|
||||
message.fromPhoneE164 === context.sutPhoneE164 &&
|
||||
message.text.includes(token) &&
|
||||
message.text.includes("Usage:"),
|
||||
});
|
||||
context.recordObservedMessage(usageReply);
|
||||
return "model reply included visible usage footer";
|
||||
},
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
input: "/usage tokens",
|
||||
matchText: /Usage footer: tokens/iu,
|
||||
target: "dm",
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "whatsapp-reply-to-message",
|
||||
standardId: "quote-reply",
|
||||
|
|
@ -1923,19 +1793,6 @@ const WHATSAPP_QA_SCENARIOS: WhatsAppQaScenarioDefinition[] = [
|
|||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-native-new-command",
|
||||
title: "WhatsApp /new command starts a new session",
|
||||
defaultProviderModes: ["mock-openai"],
|
||||
timeoutMs: 60_000,
|
||||
buildRun: () => ({
|
||||
configMode: "allowlist",
|
||||
expectReply: true,
|
||||
input: "/new",
|
||||
matchText: /new session|session/i,
|
||||
target: "dm",
|
||||
}),
|
||||
},
|
||||
{
|
||||
id: "whatsapp-approval-exec-deny-native",
|
||||
title: "WhatsApp native exec approval prompt denies",
|
||||
|
|
|
|||
|
|
@ -201,6 +201,7 @@ describe("qa scenario catalog", () => {
|
|||
"matrix-restart-resume",
|
||||
"slack-restart-resume",
|
||||
"subagent-stale-child-links",
|
||||
"telegram-repeated-command-authorization",
|
||||
"whatsapp-restart-resume",
|
||||
]);
|
||||
expect(
|
||||
|
|
|
|||
|
|
@ -275,6 +275,7 @@ const qaSeedScenarioBodySchema = z.object({
|
|||
plugins: z.array(z.string().trim().min(1)).optional(),
|
||||
gatewayConfigPatch: z.record(z.string(), z.unknown()).optional(),
|
||||
gatewayRuntime: qaScenarioGatewayRuntimeSchema.optional(),
|
||||
regressionRefs: z.array(z.string().trim().min(1)).optional(),
|
||||
docsRefs: z.array(z.string().trim().min(1)).optional(),
|
||||
codeRefs: z.array(z.string().trim().min(1)).optional(),
|
||||
execution: qaScenarioExecutionSchema.optional(),
|
||||
|
|
|
|||
50
qa/scenarios/channels/telegram-commands-command.yaml
Normal file
50
qa/scenarios/channels/telegram-commands-command.yaml
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
title: Telegram commands list reply
|
||||
|
||||
scenario:
|
||||
id: telegram-commands-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify Telegram renders the full native command catalog.
|
||||
successCriteria:
|
||||
- The command list is returned through the Telegram group command path.
|
||||
- The joined reply includes session and stop commands.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-core.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Request the Telegram command catalog and verify representative entries.
|
||||
config:
|
||||
expectedAny: [/session, /stop]
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: commands renders the full catalog
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? '/commands@openclaw' : '/commands'"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
timeoutMs: 60000
|
||||
- call: sleep
|
||||
args: [1000]
|
||||
- set: joinedReply
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')"
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => joinedReply.includes(needle)) && (transport.id === 'qa-channel' ? joinedReply.includes('Slash commands') : joinedReply.includes('Commands (1/'))"
|
||||
message:
|
||||
expr: "`commands reply missing expected text: ${joinedReply}`"
|
||||
detailsExpr: joinedReply
|
||||
47
qa/scenarios/channels/telegram-context-command.yaml
Normal file
47
qa/scenarios/channels/telegram-context-command.yaml
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
title: Telegram context command reply
|
||||
|
||||
scenario:
|
||||
id: telegram-context-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify Telegram context help reaches native command routing.
|
||||
successCriteria:
|
||||
- The context command returns the context list shortcut.
|
||||
- The reply explains the inline shortcut.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-context.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Run Telegram context help and verify the list shortcut.
|
||||
config:
|
||||
expectedAny: [/context list, Inline shortcut]
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: context returns shortcut help
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? '/context@openclaw' : '/context'"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: /context list
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => reply.text.includes(needle))"
|
||||
message:
|
||||
expr: "`context reply missing expected text: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
title: Telegram current-session status tool context
|
||||
|
||||
scenario:
|
||||
id: telegram-current-session-status-tool
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [tools.session-status]
|
||||
secondary: [memory.session-key-construction]
|
||||
objective: Verify the session_status tool resolves current to the active Telegram group session.
|
||||
successCriteria:
|
||||
- The model calls session_status with sessionKey current.
|
||||
- The final reply contains the Telegram group session key and success marker.
|
||||
codeRefs:
|
||||
- src/agents/tools/session-status-tool.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Ask the model to resolve the current Telegram group session through session_status.
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: session status resolves current Telegram group
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: "@openclaw Telegram current session_status QA check. Call session_status with sessionKey set to current, then reply with the exact QA marker and resolved session key."
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
textIncludes: QA-TELEGRAM-CURRENT-SESSION-OK
|
||||
timeoutMs: 90000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "reply.text.includes(':telegram:group:')"
|
||||
message:
|
||||
expr: "`current-session reply missing Telegram group key: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
51
qa/scenarios/channels/telegram-help-command.yaml
Normal file
51
qa/scenarios/channels/telegram-help-command.yaml
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
title: Telegram help command reply
|
||||
|
||||
scenario:
|
||||
id: telegram-help-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary:
|
||||
- channels.native-commands
|
||||
objective: Verify the Telegram help command returns the concise command guide.
|
||||
successCriteria:
|
||||
- The help command is accepted in the Telegram group command path.
|
||||
- The reply advertises new-session and full command-list commands.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-core.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Send Telegram help and verify the concise native command guide.
|
||||
config:
|
||||
command: help
|
||||
expectedAny:
|
||||
- /new
|
||||
- /commands for full list
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: help returns the concise command guide
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? `/${config.command}@openclaw` : `/${config.command}`"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: { expr: "config.expectedAny[0]" }
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => reply.text.includes(needle))"
|
||||
message:
|
||||
expr: "`help reply missing expected text: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
title: Telegram repeated command authorization
|
||||
|
||||
scenario:
|
||||
id: telegram-repeated-command-authorization
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify an allowlisted Telegram bot operator remains authorized across repeated native commands.
|
||||
successCriteria:
|
||||
- The same operator is blocked while removed from the group allowlist.
|
||||
- Restoring the original allowlist authorizes status, help, and commands without another gate.
|
||||
codeRefs:
|
||||
- extensions/telegram/src/bot-message-context.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
suiteIsolation: isolated
|
||||
isolationReason: Mutates the live Telegram group allowlist and restarts the QA gateway.
|
||||
summary: Remove and restore the Telegram operator allowlist, then run repeated commands.
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: repeated commands reuse restored authorization
|
||||
actions:
|
||||
- assert:
|
||||
expr: "transport.id === 'telegram'"
|
||||
message: repeated authorization requires the live Telegram adapter
|
||||
- set: authorizationState
|
||||
value:
|
||||
expr: "({ allowFrom: [] })"
|
||||
- try:
|
||||
actions:
|
||||
- call: env.gateway.restartAfterStateMutation
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
params: [ctx]
|
||||
expr: "fs.readFile(ctx.configPath, 'utf8').then((raw) => { const cfg = JSON.parse(raw); const account = cfg.channels.telegram.accounts[transport.accountId]; const group = account.groups[Object.keys(account.groups)[0]]; authorizationState.allowFrom = [...(group.allowFrom ?? [])]; group.allowFrom = []; return fs.writeFile(ctx.configPath, `${JSON.stringify(cfg, null, 2)}\n`, 'utf8'); })"
|
||||
- call: waitForTransportReady
|
||||
args: [{ ref: env }, 60000]
|
||||
- resetTransport: true
|
||||
- set: deniedStart
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /status@openclaw
|
||||
- waitForNoOutbound:
|
||||
quietMs: 8000
|
||||
sinceIndex: { ref: deniedStart }
|
||||
finally:
|
||||
- call: env.gateway.restartAfterStateMutation
|
||||
args:
|
||||
- lambda:
|
||||
async: true
|
||||
params: [ctx]
|
||||
expr: "fs.readFile(ctx.configPath, 'utf8').then((raw) => { const cfg = JSON.parse(raw); const account = cfg.channels.telegram.accounts[transport.accountId]; const group = account.groups[Object.keys(account.groups)[0]]; group.allowFrom = authorizationState.allowFrom; return fs.writeFile(ctx.configPath, `${JSON.stringify(cfg, null, 2)}\n`, 'utf8'); })"
|
||||
- call: waitForTransportReady
|
||||
args: [{ ref: env }, 60000]
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /status@openclaw
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
textIncludes: "Session:"
|
||||
timeoutMs: 60000
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /help@openclaw
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
textIncludes: /commands for full list
|
||||
timeoutMs: 60000
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /commands@openclaw
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
textIncludes: /stop
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
detailsExpr: reply.text
|
||||
53
qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml
Normal file
53
qa/scenarios/channels/telegram-reply-chain-exact-marker.yaml
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
title: Telegram reply-chain exact marker
|
||||
|
||||
scenario:
|
||||
id: telegram-reply-chain-exact-marker
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [runtime.delivery]
|
||||
objective: Verify an exact-marker Telegram reply is delivered once without extra visible text.
|
||||
successCriteria:
|
||||
- The model returns the requested exact marker through the reply path.
|
||||
- Exactly one outbound message contains the marker.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/reply-payload.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Request one exact Telegram marker and verify single-message delivery.
|
||||
config:
|
||||
requiredProviderMode: mock-openai
|
||||
marker: QA-TELEGRAM-REPLY-CHAIN-OK
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: exact marker is delivered once
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "`@openclaw Telegram reply-chain marker QA. Reply exactly: ${config.marker}`"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: { ref: config.marker }
|
||||
timeoutMs: 90000
|
||||
saveAs: reply
|
||||
- call: sleep
|
||||
args: [1000]
|
||||
- set: markerReplies
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).filter((message) => message.text.includes(config.marker))"
|
||||
- assert:
|
||||
expr: "markerReplies.length === 1 && reply.text.trim() === config.marker"
|
||||
message:
|
||||
expr: "`expected one exact marker reply; saw ${markerReplies.length}: ${markerReplies.map((message) => message.text).join(' | ')}`"
|
||||
detailsExpr: reply.text
|
||||
49
qa/scenarios/channels/telegram-status-command.yaml
Normal file
49
qa/scenarios/channels/telegram-status-command.yaml
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
title: Telegram status command reply
|
||||
|
||||
scenario:
|
||||
id: telegram-status-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
regressionRefs:
|
||||
- openclaw/openclaw#74698
|
||||
objective: Verify Telegram status returns model, session, and activation details.
|
||||
successCriteria:
|
||||
- The status command succeeds in the Telegram group path.
|
||||
- The reply includes model, session, and activation fields.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-status.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Run Telegram status and verify representative status fields.
|
||||
config:
|
||||
expectedAny: [OpenClaw, "Model:", "Session:", "Activation:"]
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: status includes session details
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? '/status@openclaw' : '/status'"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: OpenClaw
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => reply.text.includes(needle))"
|
||||
message:
|
||||
expr: "`status reply missing expected text: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
78
qa/scenarios/channels/telegram-tool-only-usage-footer.yaml
Normal file
78
qa/scenarios/channels/telegram-tool-only-usage-footer.yaml
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
title: Telegram tool-only usage footer
|
||||
|
||||
scenario:
|
||||
id: telegram-tool-only-usage-footer
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [telemetry.model-usage]
|
||||
secondary: [channels.native-commands]
|
||||
regressionRefs:
|
||||
- openclaw/openclaw#87392
|
||||
objective: Verify Telegram usage mode decorates a message-tool-only visible reply.
|
||||
successCriteria:
|
||||
- The usage command enables token footer mode.
|
||||
- The next exact-marker reply includes the marker and a visible Usage footer.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-session.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Enable token usage mode, then verify the next Telegram reply footer.
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
marker: QA-TELEGRAM-USAGE-FOOTER-OK
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: next visible reply includes usage footer
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? '/usage@openclaw tokens' : '/usage tokens'"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
textIncludes: tokens
|
||||
timeoutMs: 60000
|
||||
- try:
|
||||
actions:
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "`@openclaw Telegram usage footer QA. Reply exactly: ${config.marker}`"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: { ref: config.marker }
|
||||
timeoutMs: 90000
|
||||
- call: sleep
|
||||
args: [1000]
|
||||
- set: joinedReply
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')"
|
||||
- assert:
|
||||
expr: "joinedReply.includes(config.marker) && joinedReply.includes('Usage:')"
|
||||
message:
|
||||
expr: "`usage footer reply missing marker or footer: ${joinedReply}`"
|
||||
finally:
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? '/usage@openclaw off' : '/usage off'"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
textIncludes: "Usage footer: off"
|
||||
timeoutMs: 60000
|
||||
detailsExpr: joinedReply
|
||||
51
qa/scenarios/channels/telegram-tools-compact-command.yaml
Normal file
51
qa/scenarios/channels/telegram-tools-compact-command.yaml
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
title: Telegram compact tools command reply
|
||||
|
||||
scenario:
|
||||
id: telegram-tools-compact-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify Telegram renders the compact model-tool inventory.
|
||||
successCriteria:
|
||||
- The compact tools command reaches native command dispatch.
|
||||
- The joined reply includes exec and the verbose-mode hint.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-tools.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Request compact tools through Telegram and verify inventory text.
|
||||
config:
|
||||
expectedAny: [exec, Use /tools verbose for descriptions.]
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: compact tools renders inventory
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? '/tools@openclaw compact' : '/tools compact'"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: exec
|
||||
timeoutMs: 60000
|
||||
- call: sleep
|
||||
args: [1000]
|
||||
- set: joinedReply
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')"
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => joinedReply.includes(needle))"
|
||||
message:
|
||||
expr: "`tools reply missing expected text: ${joinedReply}`"
|
||||
detailsExpr: joinedReply
|
||||
45
qa/scenarios/channels/telegram-whoami-command.yaml
Normal file
45
qa/scenarios/channels/telegram-whoami-command.yaml
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
title: Telegram whoami command reply
|
||||
|
||||
scenario:
|
||||
id: telegram-whoami-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify Telegram command identity includes the active channel context.
|
||||
successCriteria:
|
||||
- The whoami command returns an identity block.
|
||||
- The reply names Telegram as the active channel on Telegram-backed transports.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-whoami.ts
|
||||
- extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: telegram
|
||||
summary: Run whoami and verify channel identity context.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: whoami includes channel identity
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "transport.id === 'telegram' ? '/whoami@openclaw' : '/whoami'"
|
||||
- waitForOutbound:
|
||||
conversation: { id: telegram-command-room, kind: channel }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: Identity
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "reply.text.includes(`Channel: ${transport.id === 'qa-channel' ? 'qa-channel' : 'telegram'}`)"
|
||||
message:
|
||||
expr: "`whoami reply missing channel identity: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
51
qa/scenarios/channels/whatsapp-commands-command.yaml
Normal file
51
qa/scenarios/channels/whatsapp-commands-command.yaml
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
title: WhatsApp commands list reply
|
||||
|
||||
scenario:
|
||||
id: whatsapp-commands-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify WhatsApp renders the full native command catalog.
|
||||
successCriteria:
|
||||
- The command catalog is returned in a WhatsApp DM.
|
||||
- The joined reply includes session and verbose commands.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-core.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
expectedAny: [/session, /verbose]
|
||||
summary: Request the WhatsApp command catalog and verify representative entries.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: commands renders the full catalog
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /commands
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: /session
|
||||
timeoutMs: 60000
|
||||
- call: sleep
|
||||
args: [1000]
|
||||
- set: joinedReply
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')"
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => joinedReply.includes(needle))"
|
||||
message:
|
||||
expr: "`commands reply missing expected text: ${joinedReply}`"
|
||||
detailsExpr: joinedReply
|
||||
44
qa/scenarios/channels/whatsapp-context-command.yaml
Normal file
44
qa/scenarios/channels/whatsapp-context-command.yaml
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
title: WhatsApp context command reply
|
||||
|
||||
scenario:
|
||||
id: whatsapp-context-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify WhatsApp returns the current context breakdown.
|
||||
successCriteria:
|
||||
- The context list command succeeds in a WhatsApp DM.
|
||||
- The reply includes workspace and tool-schema context sections.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-context.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
expectedAny: [Context breakdown, "Workspace:", Tool schemas]
|
||||
summary: Run WhatsApp context list and verify the context breakdown.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: context list returns breakdown
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /context list
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
textIncludes: Context breakdown
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => reply.text.includes(needle))"
|
||||
message:
|
||||
expr: "`context reply missing expected text: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
38
qa/scenarios/channels/whatsapp-help-command.yaml
Normal file
38
qa/scenarios/channels/whatsapp-help-command.yaml
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
title: WhatsApp help command reply
|
||||
|
||||
scenario:
|
||||
id: whatsapp-help-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify WhatsApp help returns the concise command guide.
|
||||
successCriteria:
|
||||
- The help command is accepted in a WhatsApp DM.
|
||||
- The reply advertises new-session or command/status guidance.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-core.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
summary: Send WhatsApp help and verify command guidance.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: help returns command guidance
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /help
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
textIncludes: /new
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
detailsExpr: reply.text
|
||||
43
qa/scenarios/channels/whatsapp-native-new-command.yaml
Normal file
43
qa/scenarios/channels/whatsapp-native-new-command.yaml
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
title: WhatsApp new-session command
|
||||
|
||||
scenario:
|
||||
id: whatsapp-native-new-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify WhatsApp `/new` starts a new session through native command dispatch.
|
||||
successCriteria:
|
||||
- The `/new` command is accepted in a WhatsApp DM.
|
||||
- The reply confirms a new session.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-core.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
summary: Run `/new` through WhatsApp and verify the new-session acknowledgement.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: new starts a session
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /new
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
textIncludes: session
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "reply.text.toLowerCase().includes('new')"
|
||||
message:
|
||||
expr: "`new-session reply missing confirmation: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
42
qa/scenarios/channels/whatsapp-status-command.yaml
Normal file
42
qa/scenarios/channels/whatsapp-status-command.yaml
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
title: WhatsApp status command reply
|
||||
|
||||
scenario:
|
||||
id: whatsapp-status-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify WhatsApp status returns the active session summary.
|
||||
successCriteria:
|
||||
- The status command succeeds in a WhatsApp DM.
|
||||
- The reply includes OpenClaw and session information.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-status.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
summary: Run WhatsApp status and verify session text.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: status includes session information
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /status
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
textIncludes: OpenClaw
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "reply.text.toLowerCase().includes('session')"
|
||||
message:
|
||||
expr: "`status reply missing session text: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
75
qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml
Normal file
75
qa/scenarios/channels/whatsapp-tool-only-usage-footer.yaml
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
title: WhatsApp tool-only usage footer
|
||||
|
||||
scenario:
|
||||
id: whatsapp-tool-only-usage-footer
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [telemetry.model-usage]
|
||||
secondary: [channels.native-commands]
|
||||
objective: Verify WhatsApp usage mode decorates a message-tool-only visible reply.
|
||||
successCriteria:
|
||||
- The usage command enables token footer mode.
|
||||
- The next exact-marker reply includes the marker and visible Usage footer.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-session.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
marker: WHATSAPP-QA-USAGE-FOOTER-OK
|
||||
summary: Enable token usage mode, then verify the next WhatsApp reply footer.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: next visible reply includes usage footer
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /usage tokens
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
textIncludes: "Usage footer: tokens"
|
||||
timeoutMs: 60000
|
||||
- try:
|
||||
actions:
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text:
|
||||
expr: "`Reply with only this exact marker after usage footer setup: ${config.marker}`"
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: { ref: config.marker }
|
||||
timeoutMs: 90000
|
||||
- call: sleep
|
||||
args: [1000]
|
||||
- set: joinedReply
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')"
|
||||
- assert:
|
||||
expr: "joinedReply.includes(config.marker) && joinedReply.includes('Usage:')"
|
||||
message:
|
||||
expr: "`usage footer reply missing marker or footer: ${joinedReply}`"
|
||||
finally:
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /usage off
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
textIncludes: "Usage footer: off"
|
||||
timeoutMs: 60000
|
||||
detailsExpr: joinedReply
|
||||
52
qa/scenarios/channels/whatsapp-tools-compact-command.yaml
Normal file
52
qa/scenarios/channels/whatsapp-tools-compact-command.yaml
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
title: WhatsApp compact tools command reply
|
||||
|
||||
scenario:
|
||||
id: whatsapp-tools-compact-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify WhatsApp renders the compact model-tool inventory.
|
||||
successCriteria:
|
||||
- The compact tools command reaches native command dispatch.
|
||||
- The joined reply includes exec and the verbose-mode hint.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-tools.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
expectedAny: [exec, Use /tools verbose for descriptions]
|
||||
summary: Request compact tools through WhatsApp and verify inventory text.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: compact tools renders inventory
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- set: startIndex
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').length"
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /tools compact
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
sinceIndex: { ref: startIndex }
|
||||
textIncludes: exec
|
||||
timeoutMs: 60000
|
||||
- call: sleep
|
||||
args: [1000]
|
||||
- set: joinedReply
|
||||
value:
|
||||
expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound').slice(startIndex).map((message) => message.text).join('\\n')"
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => joinedReply.includes(needle))"
|
||||
message:
|
||||
expr: "`tools reply missing expected text: ${joinedReply}`"
|
||||
detailsExpr: joinedReply
|
||||
44
qa/scenarios/channels/whatsapp-whoami-command.yaml
Normal file
44
qa/scenarios/channels/whatsapp-whoami-command.yaml
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
title: WhatsApp whoami command reply
|
||||
|
||||
scenario:
|
||||
id: whatsapp-whoami-command
|
||||
surface: channel-framework
|
||||
category: channel-framework.channel-actions-commands-and-approvals
|
||||
coverage:
|
||||
primary: [channels.native-commands]
|
||||
objective: Verify WhatsApp command identity includes channel and allowlist context.
|
||||
successCriteria:
|
||||
- The whoami command returns an identity block.
|
||||
- The reply names WhatsApp and includes AllowFrom context.
|
||||
codeRefs:
|
||||
- src/auto-reply/reply/commands-whoami.ts
|
||||
- extensions/qa-lab/src/live-transports/whatsapp/adapter.runtime.ts
|
||||
execution:
|
||||
kind: flow
|
||||
channel: whatsapp
|
||||
config:
|
||||
requiredChannelDriver: live
|
||||
requiredProviderMode: mock-openai
|
||||
expectedAny: [Identity, "Channel: whatsapp", "AllowFrom:"]
|
||||
summary: Run WhatsApp whoami and verify channel identity context.
|
||||
|
||||
flow:
|
||||
steps:
|
||||
- name: whoami includes WhatsApp identity
|
||||
actions:
|
||||
- resetTransport: true
|
||||
- sendInbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
senderId: qa-command-operator
|
||||
senderName: QA Command Operator
|
||||
text: /whoami
|
||||
- waitForOutbound:
|
||||
conversation: { id: whatsapp-command-dm, kind: direct }
|
||||
textIncludes: Identity
|
||||
timeoutMs: 60000
|
||||
saveAs: reply
|
||||
- assert:
|
||||
expr: "config.expectedAny.every((needle) => reply.text.includes(needle))"
|
||||
message:
|
||||
expr: "`whoami reply missing expected text: ${reply.text}`"
|
||||
detailsExpr: reply.text
|
||||
Loading…
Add table
Add a link
Reference in a new issue