Skip to content

Commit 508024a

Browse files
steipetesteipete
committed
feat(qa): add live suite runner and harness
1 parent 4bb965e commit 508024a

12 files changed

Lines changed: 779 additions & 135 deletions

File tree

‎.agents/skills/openclaw-qa-testing/SKILL.md‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
---
2+
name: openclaw-qa-testing
3+
description: Run, watch, debug, and extend OpenClaw QA testing with qa-lab and qa-channel. Use when Codex needs to execute the repo-backed QA suite, inspect live QA artifacts, debug failing scenarios, add new QA scenarios, or explain the OpenClaw QA workflow. Prefer the live OpenAI lane with regular openai/gpt-5.4 in fast mode; do not use gpt-5.4-pro or gpt-5.4-mini unless the user explicitly overrides that policy.
4+
---
5+
6+
# OpenClaw QA Testing
7+
8+
Use this skill for `qa-lab` / `qa-channel` work. Repo-local QA only.
9+
10+
## Read first
11+
12+
- `docs/concepts/qa-e2e-automation.md`
13+
- `docs/help/testing.md`
14+
- `docs/channels/qa-channel.md`
15+
- `qa/QA_KICKOFF_TASK.md`
16+
- `qa/seed-scenarios.json`
17+
- `extensions/qa-lab/src/suite.ts`
18+
19+
## Model policy
20+
21+
- Live OpenAI lane: `openai/gpt-5.4`
22+
- Fast mode: on
23+
- Do not use:
24+
- `openai/gpt-5.4-pro`
25+
- `openai/gpt-5.4-mini`
26+
- Only change model policy if the user explicitly asks.
27+
28+
## Default workflow
29+
30+
1. Read the seed plan and current suite implementation.
31+
2. Decide lane:
32+
- mock/dev: `mock-openai`
33+
- real validation: `live-openai`
34+
3. For live OpenAI, use:
35+
36+
```bash
37+
OPENCLAW_LIVE_OPENAI_KEY="${OPENAI_API_KEY}" \
38+
pnpm openclaw qa suite \
39+
--provider-mode live-openai \
40+
--model openai/gpt-5.4 \
41+
--alt-model openai/gpt-5.4 \
42+
--fast \
43+
--output-dir .artifacts/qa-e2e/run-all-live-openai-<tag>
44+
```
45+
46+
4. Watch outputs:
47+
- summary: `.artifacts/qa-e2e/run-all-live-openai-<tag>/qa-suite-summary.json`
48+
- report: `.artifacts/qa-e2e/run-all-live-openai-<tag>/qa-suite-report.md`
49+
5. If the user wants to watch the live UI, find the current `openclaw-qa` listen port and report `http://127.0.0.1:<port>`.
50+
6. If a scenario fails, fix the product or harness root cause, then rerun the full lane.
51+
52+
## Repo facts
53+
54+
- Seed scenarios live in `qa/`.
55+
- Main live runner: `extensions/qa-lab/src/suite.ts`
56+
- QA lab server: `extensions/qa-lab/src/lab-server.ts`
57+
- Child gateway harness: `extensions/qa-lab/src/gateway-child.ts`
58+
- Synthetic channel: `extensions/qa-channel/`
59+
60+
## What “done” looks like
61+
62+
- Full suite green for the requested lane.
63+
- User gets:
64+
- watch URL if applicable
65+
- pass/fail counts
66+
- artifact paths
67+
- concise note on what was fixed
68+
69+
## Common failure patterns
70+
71+
- Live timeout too short:
72+
- widen live waits in `extensions/qa-lab/src/suite.ts`
73+
- Discovery cannot find repo files:
74+
- point prompts at `repo/...` inside seeded workspace
75+
- Subagent proof too brittle:
76+
- prefer stable final reply evidence over transient child-session listing
77+
- Harness “rebuild” delay:
78+
- dirty tree can trigger a pre-run build; expect that before ports appear
79+
80+
## When adding scenarios
81+
82+
- Add scenario metadata to `qa/seed-scenarios.json`
83+
- Keep kickoff expectations in `qa/QA_KICKOFF_TASK.md` aligned
84+
- Add executable coverage in `extensions/qa-lab/src/suite.ts`
85+
- Prefer end-to-end assertions over mock-only checks
86+
- Save outputs under `.artifacts/qa-e2e/`

‎.agents/skills/openclaw-qa-testing/agents/openai.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
interface:
2+
display_name: "QA Test OpenClaw"
3+
short_description: "Run and debug qa-lab and qa-channel scenarios"
4+
default_prompt: "Use $openclaw-qa-testing to run or extend the OpenClaw QA suite with qa-lab and qa-channel, using regular openai/gpt-5.4 in fast mode for live OpenAI runs."

‎extensions/qa-lab/src/cli.runtime.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,21 @@ export async function runQaLabSelfCheckCommand(opts: { output?: string }) {
1616
}
1717
}
1818

19-
export async function runQaSuiteCommand(opts: { outputDir?: string }) {
19+
export async function runQaSuiteCommand(opts: {
20+
outputDir?: string;
21+
providerMode?: "mock-openai" | "live-openai";
22+
primaryModel?: string;
23+
alternateModel?: string;
24+
fastMode?: boolean;
25+
}) {
2026
const result = await runQaSuite({
2127
outputDir: opts.outputDir ? path.resolve(opts.outputDir) : undefined,
28+
providerMode: opts.providerMode,
29+
primaryModel: opts.primaryModel,
30+
alternateModel: opts.alternateModel,
31+
fastMode: opts.fastMode,
2232
});
33+
process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
2334
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
2435
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
2536
}

‎extensions/qa-lab/src/cli.ts‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,13 @@ async function runQaSelfCheck(opts: { output?: string }) {
1414
await runtime.runQaLabSelfCheckCommand(opts);
1515
}
1616

17-
async function runQaSuite(opts: { outputDir?: string }) {
17+
async function runQaSuite(opts: {
18+
outputDir?: string;
19+
providerMode?: "mock-openai" | "live-openai";
20+
primaryModel?: string;
21+
alternateModel?: string;
22+
fastMode?: boolean;
23+
}) {
1824
const runtime = await loadQaLabCliRuntime();
1925
await runtime.runQaSuiteCommand(opts);
2026
}
@@ -71,9 +77,27 @@ export function registerQaLabCli(program: Command) {
7177
qa.command("suite")
7278
.description("Run all repo-backed QA scenarios against the real QA gateway lane")
7379
.option("--output-dir <path>", "Suite artifact directory")
74-
.action(async (opts: { outputDir?: string }) => {
75-
await runQaSuite(opts);
76-
});
80+
.option("--provider-mode <mode>", "Provider mode: mock-openai or live-openai", "mock-openai")
81+
.option("--model <ref>", "Primary provider/model ref")
82+
.option("--alt-model <ref>", "Alternate provider/model ref")
83+
.option("--fast", "Enable provider fast mode where supported", false)
84+
.action(
85+
async (opts: {
86+
outputDir?: string;
87+
providerMode?: "mock-openai" | "live-openai";
88+
model?: string;
89+
altModel?: string;
90+
fast?: boolean;
91+
}) => {
92+
await runQaSuite({
93+
outputDir: opts.outputDir,
94+
providerMode: opts.providerMode,
95+
primaryModel: opts.model,
96+
alternateModel: opts.altModel,
97+
fastMode: opts.fast,
98+
});
99+
},
100+
);
77101

78102
qa.command("ui")
79103
.description("Start the private QA debugger UI and local QA bus")

‎extensions/qa-lab/src/gateway-child.ts‎

Lines changed: 20 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ async function waitForGatewayReady(baseUrl: string, logs: () => string, timeoutM
2828
const startedAt = Date.now();
2929
while (Date.now() - startedAt < timeoutMs) {
3030
try {
31-
const response = await fetch(`${baseUrl}/readyz`);
31+
const response = await fetch(`${baseUrl}/healthz`);
3232
if (response.ok) {
3333
return;
3434
}
@@ -70,8 +70,13 @@ async function runCliJson(params: { cwd: string; env: NodeJS.ProcessEnv; args: s
7070

7171
export async function startQaGatewayChild(params: {
7272
repoRoot: string;
73-
providerBaseUrl: string;
73+
providerBaseUrl?: string;
7474
qaBusBaseUrl: string;
75+
providerMode?: "mock-openai" | "live-openai";
76+
primaryModel?: string;
77+
alternateModel?: string;
78+
fastMode?: boolean;
79+
controlUiEnabled?: boolean;
7580
}) {
7681
const tempRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qa-suite-"));
7782
const workspaceDir = path.join(tempRoot, "workspace");
@@ -101,6 +106,11 @@ export async function startQaGatewayChild(params: {
101106
providerBaseUrl: params.providerBaseUrl,
102107
qaBusBaseUrl: params.qaBusBaseUrl,
103108
workspaceDir,
109+
providerMode: params.providerMode,
110+
primaryModel: params.primaryModel,
111+
alternateModel: params.alternateModel,
112+
fastMode: params.fastMode,
113+
controlUiEnabled: params.controlUiEnabled,
104114
});
105115
await fs.writeFile(configPath, `${JSON.stringify(cfg, null, 2)}\n`, "utf8");
106116

@@ -149,6 +159,7 @@ export async function startQaGatewayChild(params: {
149159
const wsUrl = `ws://127.0.0.1:${gatewayPort}`;
150160
const logs = () =>
151161
`${Buffer.concat(stdout).toString("utf8")}\n${Buffer.concat(stderr).toString("utf8")}`.trim();
162+
const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1";
152163

153164
try {
154165
await waitForGatewayReady(baseUrl, logs);
@@ -190,9 +201,12 @@ export async function startQaGatewayChild(params: {
190201
"--params",
191202
JSON.stringify(rpcParams ?? {}),
192203
],
204+
}).catch((error) => {
205+
const details = error instanceof Error ? error.message : String(error);
206+
throw new Error(`${details}\nGateway logs:\n${logs()}`);
193207
});
194208
},
195-
async stop() {
209+
async stop(opts?: { keepTemp?: boolean }) {
196210
if (!child.killed) {
197211
child.kill("SIGTERM");
198212
await Promise.race([
@@ -204,7 +218,9 @@ export async function startQaGatewayChild(params: {
204218
}),
205219
]);
206220
}
207-
await fs.rm(tempRoot, { recursive: true, force: true });
221+
if (!(opts?.keepTemp ?? keepTemp)) {
222+
await fs.rm(tempRoot, { recursive: true, force: true });
223+
}
208224
},
209225
};
210226
}

‎extensions/qa-lab/src/lab-server.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,4 +222,71 @@ describe("qa-lab server", () => {
222222
};
223223
expect(snapshot.messages.filter((message) => message.direction === "outbound")).toHaveLength(0);
224224
});
225+
226+
it("exposes structured outcomes and can attach control-ui after startup", async () => {
227+
const lab = await startQaLabServer({
228+
host: "127.0.0.1",
229+
port: 0,
230+
embeddedGateway: "disabled",
231+
});
232+
cleanups.push(async () => {
233+
await lab.stop();
234+
});
235+
236+
const initialOutcomes = (await (await fetch(`${lab.baseUrl}/api/outcomes`)).json()) as {
237+
run: null | unknown;
238+
};
239+
expect(initialOutcomes.run).toBeNull();
240+
241+
lab.setScenarioRun({
242+
kind: "suite",
243+
status: "running",
244+
startedAt: "2026-04-06T09:00:00.000Z",
245+
scenarios: [
246+
{
247+
id: "channel-chat-baseline",
248+
name: "Channel baseline conversation",
249+
status: "pass",
250+
steps: [{ name: "reply check", status: "pass", details: "ok" }],
251+
finishedAt: "2026-04-06T09:00:01.000Z",
252+
},
253+
{
254+
id: "cron-one-minute-ping",
255+
name: "Cron one-minute ping",
256+
status: "running",
257+
startedAt: "2026-04-06T09:00:02.000Z",
258+
},
259+
],
260+
});
261+
lab.setControlUi({
262+
controlUiUrl: "http://127.0.0.1:18789/",
263+
controlUiToken: "late-token",
264+
});
265+
266+
const bootstrap = (await (await fetch(`${lab.baseUrl}/api/bootstrap`)).json()) as {
267+
controlUiEmbeddedUrl: string | null;
268+
};
269+
expect(bootstrap.controlUiEmbeddedUrl).toBe("http://127.0.0.1:18789/#token=late-token");
270+
271+
const outcomes = (await (await fetch(`${lab.baseUrl}/api/outcomes`)).json()) as {
272+
run: {
273+
status: string;
274+
counts: { total: number; passed: number; running: number };
275+
scenarios: Array<{ id: string; status: string }>;
276+
};
277+
};
278+
expect(outcomes.run.status).toBe("running");
279+
expect(outcomes.run.counts).toEqual({
280+
total: 2,
281+
pending: 0,
282+
running: 1,
283+
passed: 1,
284+
failed: 0,
285+
skipped: 0,
286+
});
287+
expect(outcomes.run.scenarios.map((scenario) => scenario.id)).toEqual([
288+
"channel-chat-baseline",
289+
"cron-one-minute-ping",
290+
]);
291+
});
225292
});

0 commit comments

Comments
 (0)