fix(qa): observe expected stranded-reply errors (#151889)

extensions/qa-lab/src/scenario-flow-runner.stranded-reply.test.ts

@@ -0,0 +1,130 @@
+import { describe, expect, it } from "vitest";
+import { createQaBusState } from "./bus-state.js";
+import type { QaBusInboundMessageInput, QaBusOutboundMessageInput } from "./runtime-api.js";
+import { runLoadedScenarioFlow } from "./scenario-flow-runner.test-support.js";
+import { waitForOutboundMessage } from "./suite-runtime-transport.js";
+
+const diagnostic = "I generated a reply but could not deliver it to this chat. Please try again.";
+const rawMarker = "QA-STRANDED-RETRY-FAIL-RAW";
+const retryRequest = { allInputText: `${rawMarker}: you did not call message(action=send)` };
+
+async function runStrandedRetryFailureFlow(
+  options: {
+    outbound?: Partial<QaBusOutboundMessageInput>;
+    duplicate?: "immediate" | "settled";
+    leakRaw?: "immediate" | "settled";
+    extraRetry?: "immediate" | "settled";
+  } = {},
+) {
+  const state = createQaBusState();
+  const outbound = {
+    accountId: "default",
+    to: "dm:qa-stranded-retry-failure-dm",
+    isError: true,
+    text: diagnostic,
+    ...options.outbound,
+  };
+  const requests = [{ allInputText: rawMarker }, retryRequest];
+  if (options.extraRetry === "immediate") {
+    requests.push(retryRequest);
+  }
+  let settled = false;
+  return await runLoadedScenarioFlow("message-tool-stranded-final-retry-failure", {
+    state,
+    api: {
+      env: {
+        providerMode: "mock-openai",
+        mock: { baseUrl: "http://qa.invalid" },
+        transport: { accountId: "default" },
+      },
+      transport: {
+        sendInbound: async (input: QaBusInboundMessageInput) => {
+          const inbound = state.addInboundMessage(input);
+          state.addOutboundMessage(outbound);
+          if (options.duplicate === "immediate") {
+            state.addOutboundMessage(outbound);
+          }
+          if (options.leakRaw === "immediate") {
+            state.addOutboundMessage({ ...outbound, text: rawMarker });
+          }
+          return inbound;
+        },
+      },
+      // The loaded scenario must observe its expected error without weakening success-only waits.
+      waitForOutboundMessage,
+      fetchJson: async (url: string) =>
+        new URL(url).pathname === "/debug/request-cursor"
+          ? { cursor: 0 }
+          : settled && options.extraRetry === "settled"
+            ? [...requests, retryRequest]
+            : requests,
+      sleep: async () => {
+        settled = true;
+        if (options.duplicate === "settled") {
+          state.addOutboundMessage(outbound);
+        }
+        if (options.leakRaw === "settled") {
+          state.addOutboundMessage({ ...outbound, text: rawMarker });
+        }
+      },
+    },
+  });
+}
+
+describe("stranded-final retry failure scenario", () => {
+  it("accepts the classified diagnostic and completes its retry and privacy assertions", async () => {
+    const result = await runStrandedRetryFailureFlow();
+    expect(result).toMatchObject({
+      status: "pass",
+      steps: [{ status: "pass", details: "diagnostic=1; rawOutbound=0; retryRequests=1" }],
+    });
+  });
+
+  it.each([
+    { label: "unclassified", outbound: { isError: false } },
+    { label: "foreign account", outbound: { accountId: "other" } },
+    { label: "foreign conversation", outbound: { to: "dm:other" } },
+    { label: "wrong conversation kind", outbound: { to: "channel:qa-stranded-retry-failure-dm" } },
+    { label: "extra text", outbound: { text: `${diagnostic} Unexpected extra text.` } },
+  ])("rejects a $label diagnostic", async ({ outbound }) => {
+    await expect(runStrandedRetryFailureFlow({ outbound })).rejects.toThrow(
+      "expected one classified delivery failure on the original account and conversation",
+    );
+  });
+
+  it.each([
+    {
+      label: "immediate duplicate",
+      options: { duplicate: "immediate" as const },
+      failure: "expected exactly one sanitized diagnostic, saw 2",
+    },
+    {
+      label: "late duplicate",
+      options: { duplicate: "settled" as const },
+      failure: "expected exactly one sanitized diagnostic, saw 2",
+    },
+    {
+      label: "immediate private text leak",
+      options: { leakRaw: "immediate" as const },
+      failure: "raw stranded final text must not be delivered",
+    },
+    {
+      label: "late private text leak",
+      options: { leakRaw: "settled" as const },
+      failure: "raw stranded final text must not be delivered",
+    },
+    {
+      label: "immediate second retry",
+      options: { extraRetry: "immediate" as const },
+      failure: "expected exactly one stranded-reply retry request, saw 2",
+    },
+    {
+      label: "late second retry",
+      options: { extraRetry: "settled" as const },
+      failure:
+        "recovery must stop after retry failure: expected one retry request after settling, saw 2",
+    },
+  ])("rejects $label evidence", async ({ options, failure }) => {
+    await expect(runStrandedRetryFailureFlow(options)).rejects.toThrow(failure);
+  });
+});

qa/scenarios/channels/message-tool-stranded-final-retry-failure.yaml

@@ -65,14 +65,18 @@ flow:
             senderName: Alice
             text:
               expr: config.prompt
-        - call: waitForOutboundMessage
+        - call: state.waitFor
           saveAs: outbound
           args:
-            - ref: state
-            - lambda:
-                params: [candidate]
-                expr: "candidate.conversation.id === config.conversationId && candidate.conversation.kind === 'direct' && String(candidate.text ?? '') === config.expectedDiagnostic"
-            - expr: liveTurnTimeoutMs(env, 180000)
+            - kind: message-text
+              direction: outbound
+              textIncludes:
+                ref: config.expectedDiagnostic
+              timeoutMs:
+                expr: liveTurnTimeoutMs(env, 180000)
+        - assert:
+            expr: "!outbound.deleted && outbound.accountId === env.transport.accountId && outbound.conversation.id === config.conversationId && outbound.conversation.kind === 'direct' && outbound.isError === true && outbound.text === config.expectedDiagnostic"
+            message: expected one classified delivery failure on the original account and conversation
         - set: scenarioRequests
           value:
             expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []"
@@ -91,6 +95,9 @@ flow:
             expr: "!env.mock || retryRequests.length === 1"
             message:
               expr: "`expected exactly one stranded-reply retry request, saw ${retryRequests.length}`"
+        - call: sleep
+          args:
+            - expr: liveTurnTimeoutMs(env, 8000)
         - set: diagnosticOutbound
           value:
             expr: "state.getSnapshot().messages.filter((message) => message.direction === 'outbound' && message.conversation.id === config.conversationId && String(message.text ?? '') === config.expectedDiagnostic)"
@@ -105,9 +112,6 @@ flow:
             expr: rawOutbound.length === 0
             message:
               expr: "`raw stranded final text must not be delivered, saw ${rawOutbound.length} raw outbound messages`"
-        - call: sleep
-          args:
-            - expr: liveTurnTimeoutMs(env, 8000)
         - set: settledRequests
           value:
             expr: "env.mock ? (await fetchJson(`${env.mock.baseUrl}/debug/requests?after=${requestCursorBefore}`)).filter((request) => String(request.allInputText ?? '').includes(config.rawMarker)) : []"