Skip to content

Commit f9aec04

Browse files
vincentkocvincentkoc
authored
fix(qa): stabilize live transport lanes
Wire QA fallback models into live gateway config, fix Slack allowlist-block coverage, and keep WhatsApp live artifacts useful while redacting raw credential metadata.\n\nVerification: focused QA Vitest; autoreview clean; AWS Crabbox pnpm check:changed run_0207de7d47aa; QA-Lab branch-defined transport run 26565521272 with Matrix transport 56/56 and Slack/Discord/Telegram/parity clear. WhatsApp remains blocked by stale shared Convex WhatsApp Web credentials returning Baileys 401 before scenarios.
1 parent b008989 commit f9aec04

7 files changed

Lines changed: 108 additions & 16 deletions

File tree

‎.github/workflows/qa-live-transports-convex.yml‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ env:
5252
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
5353
NODE_VERSION: "24.x"
5454
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }}
55+
OPENCLAW_CI_OPENAI_FALLBACK_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_FALLBACK_MODEL || 'openai/gpt-5.4' }}
5556
OPENCLAW_BUILD_PRIVATE_QA: "1"
5657
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
5758

@@ -288,7 +289,7 @@ jobs:
288289
--runtime-parity-tier live-only \
289290
--concurrency "${QA_PARITY_CONCURRENCY}" \
290291
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
291-
--alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \
292+
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
292293
--runtime-pair openclaw,codex \
293294
--fast \
294295
--allow-failures \
@@ -373,7 +374,7 @@ jobs:
373374
--output-dir "${output_dir}" \
374375
--provider-mode live-frontier \
375376
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
376-
--alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \
377+
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
377378
--profile "${INPUT_MATRIX_PROFILE}" \
378379
--fast
379380
)
@@ -457,7 +458,7 @@ jobs:
457458
--output-dir "${output_dir}" \
458459
--provider-mode live-frontier \
459460
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
460-
--alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \
461+
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
461462
--profile "${{ matrix.profile }}" \
462463
--fast
463464
)
@@ -555,7 +556,7 @@ jobs:
555556
--output-dir "${output_dir}" \
556557
--provider-mode live-frontier \
557558
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
558-
--alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \
559+
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
559560
--fast \
560561
--credential-source convex \
561562
--credential-role ci \
@@ -649,7 +650,7 @@ jobs:
649650
--output-dir "${output_dir}" \
650651
--provider-mode live-frontier \
651652
--model openai/gpt-5.5 \
652-
--alt-model openai/gpt-5.5 \
653+
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
653654
--fast \
654655
--credential-source convex \
655656
--credential-role ci \
@@ -746,7 +747,7 @@ jobs:
746747
--output-dir "${output_dir}" \
747748
--provider-mode live-frontier \
748749
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
749-
--alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \
750+
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
750751
--fast \
751752
--credential-source convex \
752753
--credential-role ci \
@@ -840,7 +841,7 @@ jobs:
840841
--output-dir "${output_dir}" \
841842
--provider-mode live-frontier \
842843
--model "${OPENCLAW_CI_OPENAI_MODEL}" \
843-
--alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \
844+
--alt-model "${OPENCLAW_CI_OPENAI_FALLBACK_MODEL}" \
844845
--fast \
845846
--credential-source convex \
846847
--credential-role ci \

‎extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,27 @@ describe("Slack live QA runtime helpers", () => {
106106
expect(account?.channels?.C123456789?.users).toEqual(["U999999999"]);
107107
});
108108

109+
it("overrides both owner and channel allowlists for block scenarios", () => {
110+
const cfg = testing.buildSlackQaConfig(
111+
{},
112+
{
113+
channelId: "C123456789",
114+
driverBotUserId: "U999999999",
115+
overrides: {
116+
allowFrom: ["U_NEVER_ALLOWED"],
117+
users: ["U_NEVER_ALLOWED"],
118+
},
119+
sutAccountId: "sut",
120+
sutAppToken: "xapp-sut",
121+
sutBotToken: "xoxb-sut",
122+
},
123+
);
124+
125+
const account = cfg.channels?.slack?.accounts?.sut;
126+
expect(account?.allowFrom).toEqual(["U_NEVER_ALLOWED"]);
127+
expect(account?.channels?.C123456789?.users).toEqual(["U_NEVER_ALLOWED"]);
128+
});
129+
109130
it("extracts Slack native approval button values from blocks", () => {
110131
expect(
111132
testing.collectSlackActionValues([

‎extensions/qa-lab/src/live-transports/slack/slack-live.runtime.ts‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ type SlackQaBeforeRunResult =
9797
};
9898

9999
type SlackQaConfigOverrides = {
100+
allowFrom?: string[];
100101
approvals?: {
101102
exec?: boolean;
102103
plugin?: boolean;
@@ -334,7 +335,10 @@ const SLACK_QA_SCENARIOS: SlackQaScenarioDefinition[] = [
334335
standardId: "allowlist-block",
335336
title: "Slack non-allowlisted sender does not trigger",
336337
timeoutMs: 8_000,
337-
configOverrides: { users: ["U_OPENCLAW_QA_NEVER_ALLOWED"] },
338+
configOverrides: {
339+
allowFrom: ["U_OPENCLAW_QA_NEVER_ALLOWED"],
340+
users: ["U_OPENCLAW_QA_NEVER_ALLOWED"],
341+
},
338342
buildRun: (sutUserId) => {
339343
const token = `SLACK_QA_BLOCK_${randomUUID().slice(0, 8).toUpperCase()}`;
340344
return {
@@ -656,7 +660,7 @@ function buildSlackQaConfig(
656660
mode: "socket",
657661
botToken: params.sutBotToken,
658662
appToken: params.sutAppToken,
659-
allowFrom: [params.driverBotUserId],
663+
allowFrom: params.overrides?.allowFrom ?? [params.driverBotUserId],
660664
groupPolicy: "allowlist",
661665
allowBots: true,
662666
replyToMode: params.overrides?.replyToMode ?? "off",

‎extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,33 @@ describe("WhatsApp QA live runtime", () => {
9595
]);
9696
});
9797

98+
it("derives a stable non-secret credential fingerprint", () => {
99+
expect(testing.fingerprintWhatsAppCredentialId("cred-stale-row")).toMatch(
100+
/^sha256:[0-9a-f]{16}$/,
101+
);
102+
expect(testing.fingerprintWhatsAppCredentialId("cred-stale-row")).toBe(
103+
testing.fingerprintWhatsAppCredentialId("cred-stale-row"),
104+
);
105+
expect(testing.fingerprintWhatsAppCredentialId(undefined)).toBeUndefined();
106+
});
107+
108+
it("keeps credential fingerprints visible in redacted reports", () => {
109+
const report = testing.renderWhatsAppQaMarkdown({
110+
cleanupIssues: [],
111+
credentialFingerprint: "sha256:1234567890abcdef",
112+
credentialSource: "convex",
113+
finishedAt: "2026-05-04T12:01:00.000Z",
114+
redactMetadata: true,
115+
scenarios: [],
116+
startedAt: "2026-05-04T12:00:00.000Z",
117+
sutPhoneE164: "+15550000002",
118+
});
119+
120+
expect(report).toContain("Credential fingerprint: `sha256:1234567890abcdef`");
121+
expect(report).toContain("SUT phone: `<redacted>`");
122+
expect(report).not.toContain("+15550000002");
123+
});
124+
98125
it("unpacks auth archives into a caller-provided temp directory", async () => {
99126
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-wa-qa-test-"));
100127
try {

‎extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { execFile } from "node:child_process";
2-
import { randomUUID } from "node:crypto";
2+
import { createHash, randomUUID } from "node:crypto";
33
import fs from "node:fs/promises";
44
import path from "node:path";
55
import { promisify } from "node:util";
@@ -132,6 +132,7 @@ type WhatsAppQaSummary = {
132132
total: number;
133133
};
134134
credentials: {
135+
credentialFingerprint?: string;
135136
credentialId?: string;
136137
kind: string;
137138
ownerId?: string;
@@ -692,6 +693,7 @@ function toObservedWhatsAppArtifacts(params: {
692693

693694
function renderWhatsAppQaMarkdown(params: {
694695
cleanupIssues: string[];
696+
credentialFingerprint?: string;
695697
credentialSource: "convex" | "env";
696698
finishedAt: string;
697699
gatewayDebugDirPath?: string;
@@ -704,6 +706,9 @@ function renderWhatsAppQaMarkdown(params: {
704706
"# WhatsApp QA Report",
705707
"",
706708
`- Credential source: \`${params.credentialSource}\``,
709+
...(params.credentialFingerprint
710+
? [`- Credential fingerprint: \`${params.credentialFingerprint}\``]
711+
: []),
707712
`- SUT phone: \`${params.redactMetadata ? "<redacted>" : (params.sutPhoneE164 ?? "<unavailable>")}\``,
708713
`- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``,
709714
`- Started: ${params.startedAt}`,
@@ -731,6 +736,14 @@ function renderWhatsAppQaMarkdown(params: {
731736
return lines.join("\n");
732737
}
733738

739+
function fingerprintWhatsAppCredentialId(credentialId: string | undefined) {
740+
if (!credentialId) {
741+
return undefined;
742+
}
743+
const digest = createHash("sha256").update(credentialId).digest("hex").slice(0, 16);
744+
return `sha256:${digest}`;
745+
}
746+
734747
function createMissingGroupJidScenarioResult(params: {
735748
explicitScenarioSelection: boolean;
736749
scenario: WhatsAppQaScenarioDefinition;
@@ -970,12 +983,14 @@ export async function runWhatsAppQaLive(params: {
970983
const passed = scenarioResults.filter((entry) => entry.status === "pass").length;
971984
const failed = scenarioResults.filter((entry) => entry.status === "fail").length;
972985
const skipped = scenarioResults.filter((entry) => entry.status === "skip").length;
986+
const credentialFingerprint = fingerprintWhatsAppCredentialId(credentialLease?.credentialId);
973987
const summary: WhatsAppQaSummary = {
974988
credentials: credentialLease
975989
? {
976990
source: credentialLease.source,
977991
kind: credentialLease.kind,
978992
role: credentialLease.role,
993+
credentialFingerprint,
979994
credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId,
980995
ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId,
981996
}
@@ -1016,6 +1031,7 @@ export async function runWhatsAppQaLive(params: {
10161031
reportPath,
10171032
`${renderWhatsAppQaMarkdown({
10181033
cleanupIssues,
1034+
credentialFingerprint,
10191035
credentialSource: credentialLease?.source ?? requestedCredentialSource,
10201036
finishedAt,
10211037
gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined,
@@ -1041,8 +1057,10 @@ export const testing = {
10411057
buildWhatsAppQaConfig,
10421058
createMissingGroupJidScenarioResult,
10431059
findScenarios,
1060+
fingerprintWhatsAppCredentialId,
10441061
isTransientWhatsAppQaDriverError,
10451062
parseWhatsAppQaCredentialPayload,
1063+
renderWhatsAppQaMarkdown,
10461064
resolveWhatsAppQaRuntimeEnv,
10471065
resolveWhatsAppMetadataRedaction,
10481066
toObservedWhatsAppArtifacts,

‎extensions/qa-lab/src/qa-gateway-config.test.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,16 @@ function getPrimaryModel(value: unknown): string | undefined {
4141
return undefined;
4242
}
4343

44+
function getModelFallbacks(value: unknown): string[] | undefined {
45+
if (value && typeof value === "object" && "fallbacks" in value) {
46+
const fallbacks = (value as { fallbacks?: unknown }).fallbacks;
47+
return Array.isArray(fallbacks)
48+
? fallbacks.filter((fallback): fallback is string => typeof fallback === "string")
49+
: undefined;
50+
}
51+
return undefined;
52+
}
53+
4454
describe("buildQaGatewayConfig", () => {
4555
it("keeps mock-openai as the default provider lane", () => {
4656
const cfg = buildQaGatewayConfig({
@@ -53,6 +63,8 @@ describe("buildQaGatewayConfig", () => {
5363
});
5464

5565
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("mock-openai/gpt-5.5");
66+
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual(["mock-openai/gpt-5.5-alt"]);
67+
expect(getModelFallbacks(cfg.agents?.list?.[0]?.model)).toEqual(["mock-openai/gpt-5.5-alt"]);
5668
expect(cfg.models?.providers?.["mock-openai"]?.baseUrl).toBe("http://127.0.0.1:44080/v1");
5769
expect(cfg.models?.providers?.["mock-openai"]?.request).toEqual({ allowPrivateNetwork: true });
5870
expect(cfg.models?.providers?.["mock-openai"]?.models).toEqual(
@@ -100,6 +112,12 @@ describe("buildQaGatewayConfig", () => {
100112
});
101113

102114
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.5");
115+
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toEqual([
116+
"anthropic/claude-opus-4-7",
117+
]);
118+
expect(getModelFallbacks(cfg.agents?.list?.[0]?.model)).toEqual([
119+
"anthropic/claude-opus-4-7",
120+
]);
103121
expect(cfg.models?.providers?.openai?.api).toBe("openai-responses");
104122
expect(cfg.models?.providers?.openai?.request).toEqual({ allowPrivateNetwork: true });
105123
expect(cfg.models?.providers?.openai?.models.map((model) => model.id)).toContain("gpt-5.5");
@@ -195,6 +213,8 @@ describe("buildQaGatewayConfig", () => {
195213

196214
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("openai/gpt-5.5");
197215
expect(getPrimaryModel(cfg.agents?.list?.[0]?.model)).toBe("openai/gpt-5.5");
216+
expect(getModelFallbacks(cfg.agents?.defaults?.model)).toBeUndefined();
217+
expect(getModelFallbacks(cfg.agents?.list?.[0]?.model)).toBeUndefined();
198218
expect(cfg.models).toBeUndefined();
199219
expect(cfg.plugins?.allow).toEqual(["acpx", "memory-core", "openai", "qa-channel"]);
200220
expect(cfg.plugins?.entries?.openai).toEqual({ enabled: true });

‎extensions/qa-lab/src/qa-gateway-config.ts‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ function normalizeQaGatewayModelRef(input: string | undefined, fallback: string)
3535
return model && model.length > 0 ? model : fallback;
3636
}
3737

38+
function buildQaModelSelection(primaryModel: string, alternateModel: string) {
39+
const fallbacks = alternateModel !== primaryModel ? [alternateModel] : undefined;
40+
return fallbacks ? { primary: primaryModel, fallbacks } : { primary: primaryModel };
41+
}
42+
3843
export function buildQaGatewayConfig(params: {
3944
bind: "loopback" | "lan";
4045
gatewayPort: number;
@@ -144,9 +149,7 @@ export function buildQaGatewayConfig(params: {
144149
agents: {
145150
defaults: {
146151
workspace: params.workspaceDir,
147-
model: {
148-
primary: primaryModel,
149-
},
152+
model: buildQaModelSelection(primaryModel, alternateModel),
150153
...(imageGenerationModelRef
151154
? {
152155
imageGenerationModel: {
@@ -180,9 +183,7 @@ export function buildQaGatewayConfig(params: {
180183
{
181184
id: "qa",
182185
default: true,
183-
model: {
184-
primary: primaryModel,
185-
},
186+
model: buildQaModelSelection(primaryModel, alternateModel),
186187
identity: {
187188
name: "C-3PO QA",
188189
theme: "Flustered Protocol Droid",

0 commit comments

Comments
 (0)