diff --git a/extensions/qa-lab/src/scenario-flow-runner.stranded-reply.test.ts b/extensions/qa-lab/src/scenario-flow-runner.stranded-reply.test.ts new file mode 100644 index 000000000000..859df1f25456 --- /dev/null +++ b/extensions/qa-lab/src/scenario-flow-runner.stranded-reply.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { createQaBusState } from "./bus-state.js"; +import type { QaBusInboundMessageInput, QaBusOutboundMessageInput } from "./runtime-api.js"; +import { runLoadedScenarioFlow } from "./scenario-flow-runner.test-support.js"; +import { waitForOutboundMessage } from "./suite-runtime-transport.js"; + +const diagnostic = "I generated a reply but could not deliver it to this chat. Please try again."; +const rawMarker = "QA-STRANDED-RETRY-FAIL-RAW"; +const retryRequest = { allInputText: `${rawMarker}: you did not call message(action=send)` }; + +async function runStrandedRetryFailureFlow( + options: { + outbound?: Partial; + duplicate?: "immediate" | "settled"; + leakRaw?: "immediate" | "settled"; + extraRetry?: "immediate" | "settled"; + } = {}, +) { + const state = createQaBusState(); + const outbound = { + accountId: "default", + to: "dm:qa-stranded-retry-failure-dm", + isError: true, + text: diagnostic, + ...options.outbound, + }; + const requests = [{ allInputText: rawMarker }, retryRequest]; + if (options.extraRetry === "immediate") { + requests.push(retryRequest); + } + let settled = false; + return await runLoadedScenarioFlow("message-tool-stranded-final-retry-failure", { + state, + api: { + env: { + providerMode: "mock-openai", + mock: { baseUrl: "http://qa.invalid" }, + transport: { accountId: "default" }, + }, + transport: { + sendInbound: async (input: QaBusInboundMessageInput) => { + const inbound = state.addInboundMessage(input); + state.addOutboundMessage(outbound); + if (options.duplicate === "immediate") { + state.addOutboundMessage(outbound); + } + if (options.leakRaw === "immediate") { + state.addOutboundMessage({ ...outbound, text: rawMarker }); + } + return inbound; + }, + }, + // The loaded scenario must observe its expected error without weakening success-only waits. + waitForOutboundMessage, + fetchJson: async (url: string) => + new URL(url).pathname === "/debug/request-cursor" + ? { cursor: 0 } + : settled && options.extraRetry === "settled" + ? [...requests, retryRequest] + : requests, + sleep: async () => { + settled = true; + if (options.duplicate === "settled") { + state.addOutboundMessage(outbound); + } + if (options.leakRaw === "settled") { + state.addOutboundMessage({ ...outbound, text: rawMarker }); + } + }, + }, + }); +} + +describe("stranded-final retry failure scenario", () => { + it("accepts the classified diagnostic and completes its retry and privacy assertions", async () => { + const result = await runStrandedRetryFailureFlow(); + expect(result).toMatchObject({ + status: "pass", + steps: [{ status: "pass", details: "diagnostic=1; rawOutbound=0; retryRequests=1" }], + }); + }); + + it.each([ + { label: "unclassified", outbound: { isError: false } }, + { label: "foreign account", outbound: { accountId: "other" } }, + { label: "foreign conversation", outbound: { to: "dm:other" } }, + { label: "wrong conversation kind", outbound: { to: "channel:qa-stranded-retry-failure-dm" } }, + { label: "extra text", outbound: { text: `${diagnostic} Unexpected extra text.` } }, + ])("rejects a $label diagnostic", async ({ outbound }) => { + await expect(runStrandedRetryFailureFlow({ outbound })).rejects.toThrow( + "expected one classified delivery failure on the original account and conversation", + ); + }); + + it.each([ + { + label: "immediate duplicate", + options: { duplicate: "immediate" as const }, + failure: "expected exactly one sanitized diagnostic, saw 2", + }, + { + label: "late duplicate", + options: { duplicate: "settled" as const }, + failure: "expected exactly one sanitized diagnostic, saw 2", + }, + { + label: "immediate private text leak", + options: { leakRaw: "immediate" as const }, + failure: "raw stranded final text must not be delivered", + }, + { + label: "late private text leak", + options: { leakRaw: "settled" as const }, + failure: "raw stranded final text must not be delivered", + }, + { + label: "immediate second retry", + options: { extraRetry: "immediate" as const }, + failure: "expected exactly one stranded-reply retry request, saw 2", + }, + { + label: "late second retry", + options: { extraRetry: "settled" as const }, + failure: + "recovery must stop after retry failure: expected one retry request after settling, saw 2", + }, + ])("rejects $label evidence", async ({ options, failure }) => { + await expect(runStrandedRetryFailureFlow(options)).rejects.toThrow(failure); + }); +}); diff --git a/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml b/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml index 24c4344436af..dd1c6c58a01c 100644 --- a/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml +++ b/qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml @@ -65,14 +65,18 @@ flow: senderName: Alice text: expr: config.prompt - - call: waitForOutboundMessage + - call: state.waitFor saveAs: outbound args: - - ref: state - - lambda: - params: [candidate] - expr: "candidate.conversation.id === config.conversationId && candidate.conversation.kind === 'direct' && String(candidate.text ?? '') === config.expectedDiagnostic" - - expr: liveTurnTimeoutMs(env, 180000) + - kind: message-text + direction: outbound + textIncludes: + ref: config.expectedDiagnostic + timeoutMs: + expr: liveTurnTimeoutMs(env, 180000) + - assert: + expr: "!outbound.deleted && outbound.accountId === env.transport.accountId && outbound.conversation.id === config.conversationId && outbound.conversation.kind === 'direct' && outbound.isError === true && outbound.text === config.expectedDiagnostic" + message: expected one classified delivery failure on the original account and conversation - set: scenarioRequests value: expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []" @@ -91,6 +95,9 @@ flow: expr: "!env.mock || retryRequests.length === 1" message: expr: "`expected exactly one stranded-reply retry request, saw ${retryRequests.length}`" + - call: sleep + args: + - expr: liveTurnTimeoutMs(env, 8000) - set: diagnosticOutbound value: expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && message.conversation.id === config.conversationId && String(message.text ?? '') === config.expectedDiagnostic)" @@ -105,9 +112,6 @@ flow: expr: rawOutbound.length === 0 message: expr: "`raw stranded final text must not be delivered, saw ${rawOutbound.length} raw outbound messages`" - - call: sleep - args: - - expr: liveTurnTimeoutMs(env, 8000) - set: settledRequests value: expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []"