Skip to content

Commit bb7e982

Browse files
gumadeirasgumadeiras
committed
QA Lab: add provider registry
Move mock and live provider behavior behind provider-owned definitions so suite, manual, Matrix, and transport lanes share defaults, auth staging, model config, and standalone server startup. Add AIMock as a first-class local provider mode while keeping mock-openai as the scenario-aware deterministic lane.
1 parent 4acab55 commit bb7e982

60 files changed

Lines changed: 1964 additions & 998 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/concepts/qa-e2e-automation.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,22 @@ The baseline list should stay broad enough to cover:
151151
- repo-reading and docs-reading
152152
- one small build task such as Lobster Invaders
153153

154+
## Provider mock lanes
155+
156+
`qa suite` has two local provider mock lanes:
157+
158+
- `mock-openai` is the scenario-aware OpenClaw mock. It remains the default
159+
deterministic mock lane for repo-backed QA and parity gates.
160+
- `aimock` starts an AIMock-backed provider server for experimental protocol,
161+
fixture, record/replay, and chaos coverage. It is additive and does not
162+
replace the `mock-openai` scenario dispatcher.
163+
164+
Provider-lane implementation lives under `extensions/qa-lab/src/providers/`.
165+
Each provider owns its defaults, local server startup, gateway model config,
166+
auth-profile staging needs, and live/mock capability flags. Shared suite and
167+
gateway code should route through the provider registry instead of branching on
168+
provider names.
169+
154170
## Transport adapters
155171

156172
`qa-lab` owns a generic transport seam for markdown QA scenarios.

‎docs/help/testing.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,10 @@ These commands sit beside the main test suites when you need QA-lab realism:
5252
gateway workers, up to 64 workers or the selected scenario count. Use
5353
`--concurrency <count>` to tune the worker count, or `--concurrency 1` for
5454
the older serial lane.
55+
- Supports provider modes `live-frontier`, `mock-openai`, and `aimock`.
56+
`aimock` starts a local AIMock-backed provider server for experimental
57+
fixture and protocol-mock coverage without replacing the scenario-aware
58+
`mock-openai` lane.
5559
- `pnpm openclaw qa suite --runner multipass`
5660
- Runs the same QA suite inside a disposable Multipass Linux VM.
5761
- Keeps the same scenario-selection behavior as `qa suite` on the host.
@@ -65,6 +69,9 @@ These commands sit beside the main test suites when you need QA-lab realism:
6569
`.artifacts/qa-e2e/...`.
6670
- `pnpm qa:lab:up`
6771
- Starts the Docker-backed QA site for operator-style QA work.
72+
- `pnpm openclaw qa aimock`
73+
- Starts only the local AIMock provider server for direct protocol smoke
74+
testing.
6875
- `pnpm openclaw qa matrix`
6976
- Runs the Matrix live QA lane against a disposable Docker-backed Tuwunel homeserver.
7077
- This QA host is repo/dev-only today. Packaged OpenClaw installs do not ship

‎extensions/qa-lab/api.ts‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ export * from "./src/cli.js";
66
export * from "./src/harness-runtime.js";
77
export * from "./src/lab-server.js";
88
export * from "./src/docker-harness.js";
9-
export * from "./src/mock-openai-server.js";
109
export * from "./src/qa-agent-bootstrap.js";
1110
export * from "./src/qa-agent-workspace.js";
1211
export * from "./src/qa-gateway-config.js";

‎extensions/qa-lab/src/character-eval.ts‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -3,38 +3,28 @@ import path from "node:path";
33
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
44
import { runQaManualLane } from "./manual-lane.runtime.js";
55
import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js";
6+
import {
7+
QA_FRONTIER_CHARACTER_EVAL_MODELS,
8+
QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS,
9+
QA_FRONTIER_CHARACTER_JUDGE_MODELS,
10+
QA_FRONTIER_CHARACTER_THINKING_BY_MODEL,
11+
} from "./providers/live-frontier/character-eval.js";
612
import { type QaThinkingLevel } from "./qa-gateway-config.js";
713
import { extractQaVisibleReplyLeakText } from "./reply-failure.js";
814
import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
915
import type { QaSuiteResult } from "./suite.js";
1016

1117
const DEFAULT_CHARACTER_SCENARIO_ID = "character-vibes-gollum";
12-
const DEFAULT_CHARACTER_EVAL_MODELS = Object.freeze([
13-
"openai/gpt-5.4",
14-
"openai/gpt-5.2",
15-
"openai/gpt-5",
16-
"anthropic/claude-opus-4-6",
17-
"anthropic/claude-sonnet-4-6",
18-
"zai/glm-5.1",
19-
"moonshot/kimi-k2.5",
20-
"google/gemini-3.1-pro-preview",
21-
]);
18+
const DEFAULT_CHARACTER_EVAL_MODELS = QA_FRONTIER_CHARACTER_EVAL_MODELS;
2219
const DEFAULT_CHARACTER_THINKING: QaThinkingLevel = "high";
2320
const DEFAULT_CHARACTER_EVAL_CONCURRENCY = 16;
2421
const DEFAULT_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
25-
Object.freeze({
26-
"openai/gpt-5.4": "xhigh",
27-
"openai/gpt-5.2": "xhigh",
28-
"openai/gpt-5": "xhigh",
29-
});
30-
const DEFAULT_JUDGE_MODELS = Object.freeze(["openai/gpt-5.4", "anthropic/claude-opus-4-6"]);
22+
QA_FRONTIER_CHARACTER_THINKING_BY_MODEL;
23+
const DEFAULT_JUDGE_MODELS = QA_FRONTIER_CHARACTER_JUDGE_MODELS;
3124
const DEFAULT_JUDGE_THINKING: QaThinkingLevel = "xhigh";
3225
const DEFAULT_JUDGE_TIMEOUT_MS = 300_000;
3326
const DEFAULT_JUDGE_MODEL_OPTIONS: Readonly<Record<string, QaCharacterModelOptions>> =
34-
Object.freeze({
35-
"openai/gpt-5.4": { thinkingDefault: "xhigh" },
36-
"anthropic/claude-opus-4-6": { thinkingDefault: "high" },
37-
});
27+
QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS;
3828

3929
type QaCharacterRunStatus = "pass" | "fail";
4030

‎extensions/qa-lab/src/cli.runtime.test.ts‎

Lines changed: 25 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -24,14 +24,8 @@ const {
2424
writeQaDockerHarnessFiles: vi.fn(),
2525
buildQaDockerHarnessImage: vi.fn(),
2626
runQaDockerUp: vi.fn(),
27-
defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(
28-
(mode, options) =>
29-
mode === "live-frontier"
30-
? "openai/gpt-5.4"
31-
: options?.alternate
32-
? "mock-openai/gpt-5.4-alt"
33-
: "mock-openai/gpt-5.4",
34-
),
27+
defaultQaRuntimeModelForMode:
28+
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
3529
}));
3630

3731
vi.mock("./manual-lane.runtime.js", () => ({
@@ -83,6 +77,8 @@ import {
8377
runQaSuiteCommand,
8478
} from "./cli.runtime.js";
8579
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
80+
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
81+
import type { QaProviderModeInput } from "./run-config.js";
8682

8783
describe("qa cli runtime", () => {
8884
let stdoutWrite: ReturnType<typeof vi.spyOn>;
@@ -100,11 +96,7 @@ describe("qa cli runtime", () => {
10096
runQaDockerUp.mockReset();
10197
defaultQaRuntimeModelForMode.mockImplementation(
10298
(mode: string, options?: { alternate?: boolean }) =>
103-
mode === "live-frontier"
104-
? "openai/gpt-5.4"
105-
: options?.alternate
106-
? "mock-openai/gpt-5.4-alt"
107-
: "mock-openai/gpt-5.4",
99+
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
108100
);
109101
runQaSuiteFromRuntime.mockResolvedValue({
110102
watchUrl: "http://127.0.0.1:43124",
@@ -234,22 +226,6 @@ describe("qa cli runtime", () => {
234226
);
235227
});
236228

237-
it("normalizes legacy live-openai suite runs onto the frontier provider mode", async () => {
238-
await runQaSuiteCommand({
239-
repoRoot: "/tmp/openclaw-repo",
240-
providerMode: "live-openai",
241-
scenarioIds: ["approval-turn-tool-followthrough"],
242-
});
243-
244-
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith(
245-
expect.objectContaining({
246-
repoRoot: path.resolve("/tmp/openclaw-repo"),
247-
transportId: "qa-channel",
248-
providerMode: "live-frontier",
249-
}),
250-
);
251-
});
252-
253229
it("passes host suite concurrency through", async () => {
254230
await runQaSuiteCommand({
255231
repoRoot: "/tmp/openclaw-repo",
@@ -602,6 +578,25 @@ describe("qa cli runtime", () => {
602578
});
603579
});
604580

581+
it("defaults manual aimock runs onto the aimock model lane", async () => {
582+
await runQaManualLaneCommand({
583+
repoRoot: "/tmp/openclaw-repo",
584+
providerMode: "aimock",
585+
message: "read qa kickoff and reply short",
586+
});
587+
588+
expect(runQaManualLane).toHaveBeenCalledWith({
589+
repoRoot: path.resolve("/tmp/openclaw-repo"),
590+
transportId: "qa-channel",
591+
providerMode: "aimock",
592+
primaryModel: "aimock/gpt-5.4",
593+
alternateModel: "aimock/gpt-5.4-alt",
594+
fastMode: undefined,
595+
message: "read qa kickoff and reply short",
596+
timeoutMs: undefined,
597+
});
598+
});
599+
605600
it("defaults manual frontier runs onto the frontier model lane", async () => {
606601
await runQaManualLaneCommand({
607602
repoRoot: "/tmp/openclaw-repo",
@@ -640,31 +635,11 @@ describe("qa cli runtime", () => {
640635
});
641636
});
642637

643-
it("normalizes legacy live-openai manual runs onto the frontier provider mode", async () => {
644-
await runQaManualLaneCommand({
645-
repoRoot: "/tmp/openclaw-repo",
646-
providerMode: "live-openai",
647-
message: "read qa kickoff and reply short",
648-
});
649-
650-
expect(runQaManualLane).toHaveBeenCalledWith(
651-
expect.objectContaining({
652-
repoRoot: path.resolve("/tmp/openclaw-repo"),
653-
transportId: "qa-channel",
654-
providerMode: "live-frontier",
655-
primaryModel: "openai/gpt-5.4",
656-
alternateModel: "openai/gpt-5.4",
657-
}),
658-
);
659-
});
660-
661638
it("defaults manual frontier runs onto Codex OAuth when the runtime resolver prefers it", async () => {
662639
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
663640
mode === "live-frontier"
664641
? "openai-codex/gpt-5.4"
665-
: options?.alternate
666-
? "mock-openai/gpt-5.4-alt"
667-
: "mock-openai/gpt-5.4",
642+
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
668643
);
669644

670645
await runQaManualLaneCommand({

‎extensions/qa-lab/src/cli.runtime.ts‎

Lines changed: 25 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,13 @@ import { runQaDockerUp } from "./docker-up.runtime.js";
1414
import type { QaCliBackendAuthMode } from "./gateway-child.js";
1515
import { startQaLabServer } from "./lab-server.js";
1616
import { runQaManualLane } from "./manual-lane.runtime.js";
17-
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
1817
import { runQaMultipass } from "./multipass.runtime.js";
18+
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
19+
import {
20+
QA_FRONTIER_PARITY_BASELINE_LABEL,
21+
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
22+
} from "./providers/live-frontier/parity.js";
23+
import { startQaProviderServer } from "./providers/server-runtime.js";
1924
import {
2025
addQaCredentialSet,
2126
listQaCredentialSets,
@@ -419,8 +424,8 @@ export async function runQaParityReportCommand(opts: {
419424
) as QaParitySuiteSummary;
420425

421426
const comparison = buildQaAgenticParityComparison({
422-
candidateLabel: opts.candidateLabel?.trim() || "openai/gpt-5.4",
423-
baselineLabel: opts.baselineLabel?.trim() || "anthropic/claude-opus-4-6",
427+
candidateLabel: opts.candidateLabel?.trim() || QA_FRONTIER_PARITY_CANDIDATE_LABEL,
428+
baselineLabel: opts.baselineLabel?.trim() || QA_FRONTIER_PARITY_BASELINE_LABEL,
424429
candidateSummary,
425430
baselineSummary,
426431
});
@@ -488,7 +493,9 @@ export async function runQaManualLaneCommand(opts: {
488493
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
489494
const transportId = normalizeQaTransportId(opts.transportId);
490495
const providerMode: QaProviderMode =
491-
opts.providerMode === undefined ? "live-frontier" : normalizeQaProviderMode(opts.providerMode);
496+
opts.providerMode === undefined
497+
? DEFAULT_QA_LIVE_PROVIDER_MODE
498+
: normalizeQaProviderMode(opts.providerMode);
492499
const models = resolveQaManualLaneModels({
493500
providerMode,
494501
primaryModel: opts.primaryModel,
@@ -748,12 +755,23 @@ export async function runQaDockerUpCommand(opts: {
748755
process.stdout.write(`Stop: ${result.stopCommand}\n`);
749756
}
750757

751-
export async function runQaMockOpenAiCommand(opts: { host?: string; port?: number }) {
752-
const server = await startQaMockOpenAiServer({
758+
export async function runQaProviderServerCommand(
759+
providerMode: QaProviderMode,
760+
opts: { host?: string; port?: number },
761+
) {
762+
const provider = getQaProvider(providerMode);
763+
const standaloneCommand = provider.standaloneCommand;
764+
if (!standaloneCommand) {
765+
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
766+
}
767+
const server = await startQaProviderServer(providerMode, {
753768
host: opts.host,
754769
port: Number.isFinite(opts.port) ? opts.port : undefined,
755770
});
756-
await runInterruptibleServer("QA mock OpenAI", server);
771+
if (!server) {
772+
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
773+
}
774+
await runInterruptibleServer(standaloneCommand.serverLabel, server);
757775
}
758776

759777
export const __testing = {

‎extensions/qa-lab/src/cli.test.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,13 @@ const {
4444
runQaCredentialsAddCommand,
4545
runQaCredentialsListCommand,
4646
runQaCredentialsRemoveCommand,
47+
runQaProviderServerCommand,
4748
runQaTelegramCommand,
4849
} = vi.hoisted(() => ({
4950
runQaCredentialsAddCommand: vi.fn(),
5051
runQaCredentialsListCommand: vi.fn(),
5152
runQaCredentialsRemoveCommand: vi.fn(),
53+
runQaProviderServerCommand: vi.fn(),
5254
runQaTelegramCommand: vi.fn(),
5355
}));
5456

@@ -70,6 +72,7 @@ vi.mock("./cli.runtime.js", () => ({
7072
runQaCredentialsAddCommand,
7173
runQaCredentialsListCommand,
7274
runQaCredentialsRemoveCommand,
75+
runQaProviderServerCommand,
7376
}));
7477

7578
import { registerQaLabCli } from "./cli.js";
@@ -82,6 +85,7 @@ describe("qa cli registration", () => {
8285
runQaCredentialsAddCommand.mockReset();
8386
runQaCredentialsListCommand.mockReset();
8487
runQaCredentialsRemoveCommand.mockReset();
88+
runQaProviderServerCommand.mockReset();
8589
runQaTelegramCommand.mockReset();
8690
listQaRunnerCliContributions
8791
.mockReset()
@@ -116,6 +120,20 @@ describe("qa cli registration", () => {
116120
);
117121
});
118122

123+
it("registers standalone provider server commands from the provider registry", async () => {
124+
const qa = program.commands.find((command) => command.name() === "qa");
125+
expect(qa?.commands.map((command) => command.name())).toEqual(
126+
expect.arrayContaining(["mock-openai", "aimock"]),
127+
);
128+
129+
await program.parseAsync(["node", "openclaw", "qa", "aimock", "--port", "44080"]);
130+
131+
expect(runQaProviderServerCommand).toHaveBeenCalledWith("aimock", {
132+
host: "127.0.0.1",
133+
port: 44080,
134+
});
135+
});
136+
119137
it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => {
120138
listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]);
121139
const blockedProgram = new Command();

0 commit comments

Comments
 (0)