Skip to content

Commit 963ad1d

Browse files
gumadeirasgumadeiras
committed
QA: extend Matrix live contract coverage
1 parent 3830e68 commit 963ad1d

15 files changed

Lines changed: 1432 additions & 286 deletions

‎extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,15 +40,19 @@ function createStubTransport(baseUrl = "http://127.0.0.1:43123") {
4040

4141
describe("startQaLiveLaneGateway", () => {
4242
const gatewayStop = vi.fn();
43+
const gatewayCall = vi.fn();
4344
const mockStop = vi.fn();
4445

4546
beforeEach(() => {
4647
gatewayStop.mockReset();
48+
gatewayCall.mockReset();
4749
mockStop.mockReset();
4850
startQaGatewayChild.mockReset();
4951
startQaMockOpenAiServer.mockReset();
5052

5153
startQaGatewayChild.mockResolvedValue({
54+
call: gatewayCall,
55+
cfg: {},
5256
stop: gatewayStop,
5357
});
5458
startQaMockOpenAiServer.mockResolvedValue({

‎extensions/qa-matrix/src/runners/contract/runtime.test.ts‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
22
import { afterEach, describe, expect, it, vi } from "vitest";
3+
import { renderQaMarkdownReport } from "../../report.js";
34
import { __testing as liveTesting } from "./runtime.js";
45

56
afterEach(() => {
@@ -228,6 +229,18 @@ describe("matrix live qa runtime", () => {
228229
scenarios: [],
229230
startedAt: "2026-04-10T10:00:00.000Z",
230231
sutAccountId: "sut",
232+
timings: {
233+
artifactWriteMs: 5,
234+
canaryMs: 40,
235+
harnessBootMs: 100,
236+
initialGatewayBootMs: 200,
237+
provisioningMs: 300,
238+
scenarioGatewayBootMs: 50,
239+
scenarioRestartGatewayMs: 60,
240+
scenarioTransportInterruptMs: 70,
241+
scenarios: [],
242+
totalMs: 825,
243+
},
231244
userIds: {
232245
driver: "@driver:matrix-qa.test",
233246
observer: "@observer:matrix-qa.test",
@@ -300,6 +313,27 @@ describe("matrix live qa runtime", () => {
300313
],
301314
startedAt: "2026-04-10T10:00:00.000Z",
302315
sutAccountId: "sut",
316+
timings: {
317+
artifactWriteMs: 5,
318+
canaryMs: 40,
319+
harnessBootMs: 100,
320+
initialGatewayBootMs: 200,
321+
provisioningMs: 300,
322+
scenarioGatewayBootMs: 50,
323+
scenarioRestartGatewayMs: 60,
324+
scenarioTransportInterruptMs: 70,
325+
scenarios: [
326+
{
327+
durationMs: 80,
328+
gatewayBootMs: 0,
329+
gatewayRestartMs: 0,
330+
id: "matrix-mention-gating",
331+
title: "Matrix room message without mention does not trigger",
332+
transportInterruptMs: 0,
333+
},
334+
],
335+
totalMs: 905,
336+
},
303337
userIds: {
304338
driver: "@driver:matrix-qa.test",
305339
observer: "@observer:matrix-qa.test",
@@ -322,9 +356,128 @@ describe("matrix live qa runtime", () => {
322356
},
323357
},
324358
],
359+
timings: {
360+
totalMs: 905,
361+
},
325362
});
326363
});
327364

365+
it("keeps failing Matrix scenario details and timings complete in summary + report output", () => {
366+
const summary = liveTesting.buildMatrixQaSummary({
367+
artifactPaths: {
368+
observedEvents: "/tmp/observed.json",
369+
report: "/tmp/report.md",
370+
summary: "/tmp/summary.json",
371+
},
372+
checks: [{ name: "Matrix harness ready", status: "pass" }],
373+
config: {
374+
default: liveTesting.buildMatrixQaConfigSnapshot({
375+
driverUserId: "@driver:matrix-qa.test",
376+
observerUserId: "@observer:matrix-qa.test",
377+
sutUserId: "@sut:matrix-qa.test",
378+
topology: {
379+
defaultRoomId: "!room:matrix-qa.test",
380+
defaultRoomKey: "main",
381+
rooms: [],
382+
},
383+
}),
384+
scenarios: [],
385+
},
386+
finishedAt: "2026-04-10T10:05:00.000Z",
387+
harness: {
388+
baseUrl: "http://127.0.0.1:28008/",
389+
composeFile: "/tmp/docker-compose.yml",
390+
dmRoomIds: [],
391+
image: "ghcr.io/matrix-construct/tuwunel:v1.5.1",
392+
roomId: "!room:matrix-qa.test",
393+
roomIds: ["!room:matrix-qa.test"],
394+
serverName: "matrix-qa.test",
395+
},
396+
observedEventCount: 6,
397+
scenarios: [
398+
{
399+
id: "matrix-reaction-not-a-reply",
400+
title: "Matrix reactions do not trigger a fresh bot reply",
401+
status: "fail",
402+
details: [
403+
"unexpected SUT reply after reaction from @driver:matrix-qa.test",
404+
"reaction event: $reaction",
405+
"unexpected reply event: $reply",
406+
].join("\n"),
407+
},
408+
],
409+
startedAt: "2026-04-10T10:00:00.000Z",
410+
sutAccountId: "sut",
411+
timings: {
412+
artifactWriteMs: 5,
413+
canaryMs: 40,
414+
harnessBootMs: 100,
415+
initialGatewayBootMs: 200,
416+
provisioningMs: 300,
417+
scenarioGatewayBootMs: 50,
418+
scenarioRestartGatewayMs: 60,
419+
scenarioTransportInterruptMs: 70,
420+
scenarios: [
421+
{
422+
durationMs: 8_000,
423+
gatewayBootMs: 0,
424+
gatewayRestartMs: 0,
425+
id: "matrix-reaction-not-a-reply",
426+
title: "Matrix reactions do not trigger a fresh bot reply",
427+
transportInterruptMs: 0,
428+
},
429+
],
430+
totalMs: 825,
431+
},
432+
userIds: {
433+
driver: "@driver:matrix-qa.test",
434+
observer: "@observer:matrix-qa.test",
435+
sut: "@sut:matrix-qa.test",
436+
},
437+
});
438+
439+
expect(summary).toMatchObject({
440+
counts: {
441+
total: 2,
442+
passed: 1,
443+
failed: 1,
444+
},
445+
scenarios: [
446+
{
447+
id: "matrix-reaction-not-a-reply",
448+
status: "fail",
449+
details: expect.stringContaining("reaction event: $reaction"),
450+
},
451+
],
452+
timings: {
453+
scenarios: [
454+
{
455+
id: "matrix-reaction-not-a-reply",
456+
durationMs: 8_000,
457+
},
458+
],
459+
},
460+
});
461+
462+
const report = renderQaMarkdownReport({
463+
title: "Matrix QA Report",
464+
startedAt: new Date(summary.startedAt),
465+
finishedAt: new Date(summary.finishedAt),
466+
checks: summary.checks,
467+
scenarios: summary.scenarios.map((scenario) => ({
468+
details: scenario.details,
469+
name: scenario.title,
470+
status: scenario.status,
471+
})),
472+
notes: [`observed events: ${summary.observedEventsPath}`],
473+
});
474+
475+
expect(report).toContain("### Matrix reactions do not trigger a fresh bot reply");
476+
expect(report).toContain("unexpected SUT reply after reaction from @driver:matrix-qa.test");
477+
expect(report).toContain("reaction event: $reaction");
478+
expect(report).toContain("observed events: /tmp/observed.json");
479+
});
480+
328481
it("batches Matrix scenarios by config key while preserving stable in-group order", () => {
329482
const scenarios = liveTesting.findMatrixQaScenarios([
330483
"matrix-top-level-reply-shape",

0 commit comments

Comments
 (0)