diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts index bd68bc2f546..84da472cd36 100644 --- a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts +++ b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts @@ -872,6 +872,7 @@ async function runMatrixToolProgressScenario( finalText: string; allowFinalOnly?: boolean; allowFinalBeforeProgress?: boolean; + allowFinalReplacementAsCompletion?: boolean; allowTopLevelFinalWithProgress?: boolean; label: string; allowGenericProgressLine?: boolean; @@ -927,6 +928,13 @@ async function runMatrixToolProgressScenario( event.relatesTo?.relType === "m.replace" && event.relatesTo.eventId === previewRootEventId && matchesExpectedProgress(event.body); + const isFinalReplacement = (event: MatrixQaObservedEvent, previewRootEventId: string) => + event.roomId === context.roomId && + event.sender === context.sutUserId && + isMatrixQaMessageLikeKind(event.kind) && + event.relatesTo?.relType === "m.replace" && + event.relatesTo.eventId === previewRootEventId && + doesMatrixQaReplyBodyMatchToken(event, params.finalText); const throwProgressTimeout = (err: unknown, previewEventId: string): never => { throw new Error( buildMatrixQaToolProgressTimeoutMessage({ @@ -1056,6 +1064,7 @@ async function runMatrixToolProgressScenario( isProgressReplacement(event, previewRootEventId) || (params.allowTopLevelFinalWithProgress === true && isProgressProofEvent(event)); let topLevelFinalBeforeProgress: typeof preview | undefined; + let finalReplacementBeforeProgress: typeof preview | undefined; let progress = preview; if (!matchesExpectedProgress(preview.event.body)) { const progressOrFinal = await client @@ -1063,13 +1072,21 @@ async function runMatrixToolProgressScenario( observedEvents: context.observedEvents, predicate: (event) => isProgressProofForPreview(event) || + (params.allowFinalReplacementAsCompletion === true && + isFinalReplacement(event, previewRootEventId)) || (params.allowTopLevelFinalWithProgress === true && isFinalReply(event)), roomId: context.roomId, since: preview.since, timeoutMs: context.timeoutMs, }) .catch((err: unknown) => throwProgressTimeout(err, previewRootEventId)); - if (isFinalReply(progressOrFinal.event)) { + if ( + params.allowFinalReplacementAsCompletion === true && + isFinalReplacement(progressOrFinal.event, previewRootEventId) + ) { + finalReplacementBeforeProgress = progressOrFinal; + progress = progressOrFinal; + } else if (isFinalReply(progressOrFinal.event)) { topLevelFinalBeforeProgress = progressOrFinal; progress = await client .waitForRoomEvent({ @@ -1099,6 +1116,7 @@ async function runMatrixToolProgressScenario( const finalized = topLevelFinalBeforeProgress ?? + finalReplacementBeforeProgress ?? (await client .waitForRoomEvent({ observedEvents: context.observedEvents, @@ -1202,6 +1220,7 @@ export async function runToolProgressCommandPreviewScenario(context: MatrixQaSce expectedPreviewKind: "notice", finalText: buildMatrixQaToken("MATRIX_QA_TOOL_PROGRESS_COMMAND"), label: "tool progress command preview", + allowFinalReplacementAsCompletion: true, progressPattern: /\bcompleted\b|\bexit\s+0\b/i, rejectProgressBodyPattern: /`(?![^`]*\bcompleted\b)[^`]*(?:matrix-command-progress-start|print text\s*→\s*run sleep 2)[^`]*`/i, diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts index a6c350394bd..78cdc5c1ce8 100644 --- a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts +++ b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts @@ -3392,6 +3392,52 @@ describe("matrix live qa scenarios", () => { expect(artifacts.reply?.tokenMatched).toBe(true); }); + it("accepts a final replacement as Matrix command completion", async () => { + const previewEventId = "$tool-progress-command-final-replacement-preview"; + mockMatrixQaRoomClient({ + driverEventId: "$tool-progress-command-final-replacement-trigger", + events: [ + { + event: matrixQaMessageEvent({ + kind: "notice", + eventId: previewEventId, + body: "Working\n`🛠️ print text → run sleep 2`", + }), + since: "driver-sync-preview", + }, + { + event: ({ sendTextMessage }) => + matrixQaMessageEvent({ + kind: "notice", + eventId: "$tool-progress-command-final-replacement", + body: readMatrixQaReplyDirective( + mockMessageBody(sendTextMessage, "sendTextMessage"), + "MATRIX_QA_TOOL_PROGRESS_COMMAND", + ), + relatesTo: { + relType: "m.replace", + eventId: previewEventId, + }, + }), + since: "driver-sync-final", + }, + ], + }); + + const scenario = requireMatrixQaScenario("matrix-room-tool-progress-command-preview"); + + const result = await runMatrixQaScenario(scenario, matrixQaScenarioContext()); + const artifacts = result.artifacts as { + previewBodyPreview?: unknown; + previewEventId?: unknown; + reply?: { eventId?: unknown; tokenMatched?: unknown }; + }; + expect(artifacts.previewBodyPreview).toMatch(/^MATRIX_QA_TOOL_PROGRESS_COMMAND_/); + expect(artifacts.previewEventId).toBe(previewEventId); + expect(artifacts.reply?.eventId).toBe("$tool-progress-command-final-replacement"); + expect(artifacts.reply?.tokenMatched).toBe(true); + }); + it("reports Matrix tool progress preview candidates when the progress wait times out", async () => { const previewEvent = matrixQaMessageEvent({ kind: "notice",