fix(qa): keep Matrix mention preview finals strict

vincentkoc · vincentkoc · commit b8cf83aeb3b5 · 2026-05-27T16:15:34.000+02:00
diff --git a/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts b/extensions/qa-matrix/src/runners/contract/scenario-runtime-room.ts
@@ -868,6 +868,7 @@ async function runMatrixToolProgressScenario(
     expectedPreviewKind: MatrixQaObservedEvent["kind"];
     finalText: string;
     allowFinalOnly?: boolean;
+    allowFinalBeforeProgress?: boolean;
     allowTopLevelFinalWithProgress?: boolean;
     label: string;
     allowGenericProgressLine?: boolean;
@@ -926,7 +927,9 @@ async function runMatrixToolProgressScenario(
       observedEvents: context.observedEvents,
       predicate: (event) =>
         isProgressEvent(event) ||
-        ((params.allowFinalOnly === true || params.allowTopLevelFinalWithProgress === true) &&
+        ((params.allowFinalOnly === true ||
+          params.allowFinalBeforeProgress === true ||
+          params.allowTopLevelFinalWithProgress === true) &&
           isFinalReply(event)),
       roomId: context.roomId,
       since: startSince,
@@ -946,7 +949,10 @@ async function runMatrixToolProgressScenario(
       );
     });
   if (isFinalReply(preview.event)) {
-    if (params.allowTopLevelFinalWithProgress === true && params.allowFinalOnly !== true) {
+    if (
+      (params.allowFinalBeforeProgress === true || params.allowTopLevelFinalWithProgress === true) &&
+      params.allowFinalOnly !== true
+    ) {
       const progressAfterFinal = await client
         .waitForRoomEvent({
           observedEvents: context.observedEvents,
@@ -1230,7 +1236,7 @@ export async function runToolProgressMentionSafetyScenario(context: MatrixQaScen
     expectedPreviewKind: "message",
     finalText: buildMatrixQaToken("MATRIX_QA_TOOL_PROGRESS_MENTION_SAFE"),
     label: "tool progress mention safety",
-    allowTopLevelFinalWithProgress: true,
+    allowFinalBeforeProgress: true,
     mentionSafety: true,
     progressPattern: /@room|@alice:matrix-qa\.test|!room:matrix-qa\.test/i,
     triggerBodyBuilder: buildMatrixToolProgressMentionSafetyPrompt,
diff --git a/extensions/qa-matrix/src/runners/contract/scenarios.test.ts b/extensions/qa-matrix/src/runners/contract/scenarios.test.ts
@@ -3982,6 +3982,52 @@ describe("matrix live qa scenarios", () => {
     );
   });
 
+  it("does not accept top-level finals after a Matrix mention-safety preview", async () => {
+    const context = matrixQaScenarioContext();
+    const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");
+    const sendTextMessage = vi.fn().mockResolvedValue("$tool-progress-mention-stale-trigger");
+    const previewEvent = matrixQaMessageEvent({
+      kind: "message",
+      eventId: "$tool-progress-mention-stale-preview",
+      body:
+        "Working...\n- `read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed`",
+      formattedBody:
+        "Working...<br><ul><li><code>read matrix-progress-@room-@alice:matrix-qa.test-!room:matrix-qa.test.txt failed</code></li></ul>",
+      mentions: {},
+    });
+    const waitForRoomEvent = vi
+      .fn()
+      .mockImplementationOnce(async () => {
+        context.observedEvents.push(previewEvent);
+        return { event: previewEvent, since: "driver-sync-preview" };
+      })
+      .mockImplementationOnce(async (params: { predicate: (event: MatrixQaObservedEvent) => boolean }) => {
+        const topLevelFinal = matrixQaMessageEvent({
+          kind: "message",
+          eventId: "$tool-progress-mention-stale-final",
+          body: readMatrixQaReplyDirective(
+            mockMessageBody(sendTextMessage, "sendTextMessage"),
+            "MATRIX_QA_TOOL_PROGRESS_MENTION_SAFE_FIXED",
+          ),
+        });
+        expect(params.predicate(topLevelFinal)).toBe(false);
+        context.observedEvents.push(topLevelFinal);
+        throw new Error("timed out after 8000ms waiting for Matrix room event");
+      });
+    createMatrixQaClient.mockReturnValue({
+      primeRoom,
+      sendTextMessage,
+      waitForRoomEvent,
+    });
+
+    const scenario = requireMatrixQaScenario("matrix-room-tool-progress-mention-safety");
+
+    await expect(runMatrixQaScenario(scenario, context)).rejects.toThrow(
+      /timed out after 8000ms waiting for Matrix room event/,
+    );
+    expect(waitForRoomEvent).toHaveBeenCalledTimes(2);
+  });
+
   it("preserves separate finalized block events when Matrix block streaming is enabled", async () => {
     const primeRoom = vi.fn().mockResolvedValue("driver-sync-start");
     const sendTextMessage = vi.fn().mockResolvedValue("$block-stream-trigger");
