Skip to content

Commit 988447c

Browse files
gumadeirasgumadeiras
committed
QA Matrix: expand contract coverage
1 parent 0f7c40e commit 988447c

26 files changed

Lines changed: 4407 additions & 548 deletions

‎extensions/qa-matrix/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
"description": "OpenClaw Matrix QA runner plugin",
66
"type": "module",
77
"devDependencies": {
8+
"@openclaw/matrix": "workspace:*",
89
"@openclaw/plugin-sdk": "workspace:*",
910
"openclaw": "workspace:*"
1011
},

‎extensions/qa-matrix/src/runners/contract/runtime.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -609,6 +609,8 @@ export async function runMatrixQaLive(params: {
609609
baseUrl: harness.baseUrl,
610610
canary: canaryArtifact,
611611
driverAccessToken: provisioning.driver.accessToken,
612+
driverDeviceId: provisioning.driver.deviceId,
613+
driverPassword: provisioning.driver.password,
612614
driverUserId: provisioning.driver.userId,
613615
interruptTransport: async () => {
614616
writeMatrixQaProgress(`transport interrupt start ${scenario.id}`);
@@ -626,7 +628,10 @@ export async function runMatrixQaLive(params: {
626628
},
627629
observedEvents,
628630
observerAccessToken: provisioning.observer.accessToken,
631+
observerDeviceId: provisioning.observer.deviceId,
632+
observerPassword: provisioning.observer.password,
629633
observerUserId: provisioning.observer.userId,
634+
outputDir,
630635
restartGateway: async () => {
631636
if (!gatewayHarness) {
632637
throw new Error("Matrix restart scenario requires a live gateway");
@@ -644,6 +649,8 @@ export async function runMatrixQaLive(params: {
644649
},
645650
roomId: provisioning.roomId,
646651
sutAccessToken: provisioning.sut.accessToken,
652+
sutDeviceId: provisioning.sut.deviceId,
653+
sutPassword: provisioning.sut.password,
647654
syncState,
648655
syncStreams,
649656
sutUserId: provisioning.sut.userId,

‎extensions/qa-matrix/src/runners/contract/scenario-catalog.ts‎

Lines changed: 198 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@ export type MatrixQaScenarioId =
2323
| "matrix-room-block-streaming"
2424
| "matrix-room-image-understanding-attachment"
2525
| "matrix-room-generated-image-delivery"
26+
| "matrix-media-type-coverage"
27+
| "matrix-attachment-only-ignored"
28+
| "matrix-unsupported-media-safe"
2629
| "matrix-dm-reply-shape"
2730
| "matrix-dm-shared-session-notice"
2831
| "matrix-dm-thread-reply-override"
@@ -33,12 +36,31 @@ export type MatrixQaScenarioId =
3336
| "matrix-reaction-notification"
3437
| "matrix-reaction-threaded"
3538
| "matrix-reaction-not-a-reply"
39+
| "matrix-reaction-redaction-observed"
3640
| "matrix-restart-resume"
41+
| "matrix-post-restart-room-continue"
3742
| "matrix-room-membership-loss"
3843
| "matrix-homeserver-restart-resume"
3944
| "matrix-mention-gating"
45+
| "matrix-mention-metadata-spoof-block"
4046
| "matrix-observer-allowlist-override"
41-
| "matrix-allowlist-block";
47+
| "matrix-allowlist-block"
48+
| "matrix-multi-actor-ordering"
49+
| "matrix-inbound-edit-ignored"
50+
| "matrix-inbound-edit-no-duplicate-trigger"
51+
| "matrix-e2ee-basic-reply"
52+
| "matrix-e2ee-thread-follow-up"
53+
| "matrix-e2ee-bootstrap-success"
54+
| "matrix-e2ee-recovery-key-lifecycle"
55+
| "matrix-e2ee-device-sas-verification"
56+
| "matrix-e2ee-qr-verification"
57+
| "matrix-e2ee-stale-device-hygiene"
58+
| "matrix-e2ee-dm-sas-verification"
59+
| "matrix-e2ee-restart-resume"
60+
| "matrix-e2ee-verification-notice-no-trigger"
61+
| "matrix-e2ee-artifact-redaction"
62+
| "matrix-e2ee-media-image"
63+
| "matrix-e2ee-key-bootstrap-failure";
4264

4365
export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition<MatrixQaScenarioId> & {
4466
configOverrides?: MatrixQaConfigOverrides;
@@ -48,6 +70,8 @@ export type MatrixQaScenarioDefinition = LiveTransportScenarioDefinition<MatrixQ
4870
export const MATRIX_QA_BLOCK_ROOM_KEY = "block";
4971
export const MATRIX_QA_DRIVER_DM_ROOM_KEY = "driver-dm";
5072
export const MATRIX_QA_DRIVER_DM_SHARED_ROOM_KEY = "driver-dm-shared";
73+
export const MATRIX_QA_E2EE_ROOM_KEY = "e2ee";
74+
export const MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY = "e2ee-verification-dm";
5175
export const MATRIX_QA_HOMESERVER_ROOM_KEY = "homeserver";
5276
export const MATRIX_QA_MEDIA_ROOM_KEY = "media";
5377
export const MATRIX_QA_MEMBERSHIP_ROOM_KEY = "membership";
@@ -72,6 +96,7 @@ function buildMatrixQaDmTopology(
7296
}
7397

7498
function buildMatrixQaSingleGroupTopology(params: {
99+
encrypted?: boolean;
75100
key: string;
76101
name: string;
77102
requireMention: boolean;
@@ -80,6 +105,7 @@ function buildMatrixQaSingleGroupTopology(params: {
80105
defaultRoomKey: "main",
81106
rooms: [
82107
{
108+
encrypted: params.encrypted === true,
83109
key: params.key,
84110
kind: "group",
85111
members: ["driver", "observer", "sut"],
@@ -144,6 +170,31 @@ const MATRIX_QA_HOMESERVER_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({
144170
requireMention: true,
145171
});
146172

173+
const MATRIX_QA_E2EE_ROOM_TOPOLOGY = buildMatrixQaSingleGroupTopology({
174+
encrypted: true,
175+
key: MATRIX_QA_E2EE_ROOM_KEY,
176+
name: "Matrix QA E2EE Room",
177+
requireMention: true,
178+
});
179+
180+
const MATRIX_QA_E2EE_VERIFICATION_DM_TOPOLOGY: MatrixQaTopologySpec = {
181+
defaultRoomKey: "main",
182+
rooms: [
183+
{
184+
encrypted: true,
185+
key: MATRIX_QA_E2EE_VERIFICATION_DM_ROOM_KEY,
186+
kind: "dm",
187+
members: ["driver", "observer"],
188+
name: "Matrix QA E2EE Verification DM",
189+
},
190+
],
191+
};
192+
193+
const MATRIX_QA_E2EE_CONFIG = {
194+
encryption: true,
195+
startupVerification: "off",
196+
} satisfies MatrixQaConfigOverrides;
197+
147198
export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
148199
{
149200
id: "matrix-thread-follow-up",
@@ -214,13 +265,34 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
214265
{
215266
id: "matrix-room-image-understanding-attachment",
216267
timeoutMs: 60_000,
217-
title: "Matrix image attachments reach the model vision path",
268+
title: "Matrix captioned image attachments reach the model vision path",
218269
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
219270
},
220271
{
221272
id: "matrix-room-generated-image-delivery",
222273
timeoutMs: 60_000,
223-
title: "Matrix generated images deliver as real image attachments",
274+
title: "Matrix generated images deliver as real image attachments while streaming",
275+
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
276+
configOverrides: {
277+
streaming: "quiet",
278+
},
279+
},
280+
{
281+
id: "matrix-media-type-coverage",
282+
timeoutMs: 90_000,
283+
title: "Matrix media attachments cover image, audio, video, PDF, and EPUB transport",
284+
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
285+
},
286+
{
287+
id: "matrix-attachment-only-ignored",
288+
timeoutMs: 8_000,
289+
title: "Matrix attachment-only group media does not bypass mention gating",
290+
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
291+
},
292+
{
293+
id: "matrix-unsupported-media-safe",
294+
timeoutMs: 45_000,
295+
title: "Matrix unsupported media attachments do not block caption replies",
224296
topology: MATRIX_QA_MEDIA_ROOM_TOPOLOGY,
225297
},
226298
{
@@ -302,13 +374,24 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
302374
timeoutMs: 8_000,
303375
title: "Matrix reactions do not trigger a fresh bot reply",
304376
},
377+
{
378+
id: "matrix-reaction-redaction-observed",
379+
timeoutMs: 45_000,
380+
title: "Matrix reaction removals are observed as redactions",
381+
},
305382
{
306383
id: "matrix-restart-resume",
307384
standardId: "restart-resume",
308385
timeoutMs: 60_000,
309386
title: "Matrix lane resumes cleanly after gateway restart",
310387
topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY,
311388
},
389+
{
390+
id: "matrix-post-restart-room-continue",
391+
timeoutMs: 75_000,
392+
title: "Matrix restarted room continues after the first recovered reply",
393+
topology: MATRIX_QA_RESTART_ROOM_TOPOLOGY,
394+
},
312395
{
313396
id: "matrix-room-membership-loss",
314397
timeoutMs: 75_000,
@@ -327,6 +410,11 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
327410
timeoutMs: 8_000,
328411
title: "Matrix room message without mention does not trigger",
329412
},
413+
{
414+
id: "matrix-mention-metadata-spoof-block",
415+
timeoutMs: 8_000,
416+
title: "Matrix metadata-only mention spoof does not trigger",
417+
},
330418
{
331419
id: "matrix-observer-allowlist-override",
332420
timeoutMs: 45_000,
@@ -339,7 +427,113 @@ export const MATRIX_QA_SCENARIOS: MatrixQaScenarioDefinition[] = [
339427
id: "matrix-allowlist-block",
340428
standardId: "allowlist-block",
341429
timeoutMs: 8_000,
342-
title: "Matrix allowlist blocks non-driver replies",
430+
title: "Matrix sender allowlist blocks observer replies",
431+
},
432+
{
433+
id: "matrix-multi-actor-ordering",
434+
timeoutMs: 60_000,
435+
title: "Matrix blocked observer traffic does not poison later driver replies",
436+
},
437+
{
438+
id: "matrix-inbound-edit-ignored",
439+
timeoutMs: 8_000,
440+
title: "Matrix inbound edits cannot turn ignored messages into triggers",
441+
},
442+
{
443+
id: "matrix-inbound-edit-no-duplicate-trigger",
444+
timeoutMs: 45_000,
445+
title: "Matrix inbound edits do not duplicate already handled triggers",
446+
},
447+
{
448+
id: "matrix-e2ee-basic-reply",
449+
timeoutMs: 75_000,
450+
title: "Matrix E2EE encrypted room replies decrypt end-to-end",
451+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
452+
configOverrides: MATRIX_QA_E2EE_CONFIG,
453+
},
454+
{
455+
id: "matrix-e2ee-thread-follow-up",
456+
timeoutMs: 75_000,
457+
title: "Matrix E2EE encrypted threads preserve reply shape",
458+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
459+
configOverrides: MATRIX_QA_E2EE_CONFIG,
460+
},
461+
{
462+
id: "matrix-e2ee-bootstrap-success",
463+
timeoutMs: 90_000,
464+
title: "Matrix E2EE bootstrap verifies the owner device and backup",
465+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
466+
configOverrides: MATRIX_QA_E2EE_CONFIG,
467+
},
468+
{
469+
id: "matrix-e2ee-recovery-key-lifecycle",
470+
timeoutMs: 90_000,
471+
title: "Matrix E2EE recovery key restores and resets room-key backup",
472+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
473+
configOverrides: MATRIX_QA_E2EE_CONFIG,
474+
},
475+
{
476+
id: "matrix-e2ee-device-sas-verification",
477+
timeoutMs: 90_000,
478+
title: "Matrix E2EE device verification completes SAS emoji compare",
479+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
480+
configOverrides: MATRIX_QA_E2EE_CONFIG,
481+
},
482+
{
483+
id: "matrix-e2ee-qr-verification",
484+
timeoutMs: 90_000,
485+
title: "Matrix E2EE QR verification completes identity scan",
486+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
487+
configOverrides: MATRIX_QA_E2EE_CONFIG,
488+
},
489+
{
490+
id: "matrix-e2ee-stale-device-hygiene",
491+
timeoutMs: 90_000,
492+
title: "Matrix E2EE stale own devices can be removed without deleting the current device",
493+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
494+
configOverrides: MATRIX_QA_E2EE_CONFIG,
495+
},
496+
{
497+
id: "matrix-e2ee-dm-sas-verification",
498+
timeoutMs: 90_000,
499+
title: "Matrix E2EE DM verification notices stay scoped and complete SAS",
500+
topology: MATRIX_QA_E2EE_VERIFICATION_DM_TOPOLOGY,
501+
configOverrides: MATRIX_QA_E2EE_CONFIG,
502+
},
503+
{
504+
id: "matrix-e2ee-restart-resume",
505+
timeoutMs: 90_000,
506+
title: "Matrix E2EE encrypted rooms resume after gateway restart",
507+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
508+
configOverrides: MATRIX_QA_E2EE_CONFIG,
509+
},
510+
{
511+
id: "matrix-e2ee-verification-notice-no-trigger",
512+
timeoutMs: 30_000,
513+
title: "Matrix E2EE verification notices do not trigger replies",
514+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
515+
configOverrides: MATRIX_QA_E2EE_CONFIG,
516+
},
517+
{
518+
id: "matrix-e2ee-artifact-redaction",
519+
timeoutMs: 75_000,
520+
title: "Matrix E2EE decrypted payloads stay out of default event artifacts",
521+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
522+
configOverrides: MATRIX_QA_E2EE_CONFIG,
523+
},
524+
{
525+
id: "matrix-e2ee-media-image",
526+
timeoutMs: 90_000,
527+
title: "Matrix E2EE encrypted image attachments reach the model vision path",
528+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
529+
configOverrides: MATRIX_QA_E2EE_CONFIG,
530+
},
531+
{
532+
id: "matrix-e2ee-key-bootstrap-failure",
533+
timeoutMs: 90_000,
534+
title: "Matrix E2EE bootstrap reports room-key backup failures",
535+
topology: MATRIX_QA_E2EE_ROOM_TOPOLOGY,
536+
configOverrides: MATRIX_QA_E2EE_CONFIG,
343537
},
344538
];
345539

0 commit comments

Comments
 (0)