Skip to content

Commit 37dcf38

Browse files
vincentkocvincentkoc
committed
fix(qa): expose codex tools for runtime parity
1 parent 2c9f68f commit 37dcf38

15 files changed

Lines changed: 454 additions & 48 deletions

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ Docs: https://docs.openclaw.ai
123123
- Mac app: cache settings config schema/drafts and load channel config in parallel with channel probes, making repeated Channels and Config tab switches responsive over remote tunnels.
124124
- Control UI: negotiate the Gateway protocol from shared constants so rebuilt dashboards connect to current gateways instead of reporting a protocol mismatch.
125125
- Mac app: let menu gateway/session error text wrap across a few lines and stop rebuilding dynamic Context/Gateway menu rows while the menu is open, reducing flicker.
126+
- QA-Lab: expose Codex runtime tools during private parity runs and treat completed structural/tool-shape runtime drift as advisory, while preserving real runtime failures as lane blockers.
126127
- Mac app: make device pairing approval sheets friendlier, with concise Mac/device copy, shortened identifiers, friendly scope labels, and Approve as the primary action.
127128
- Providers/Qwen: honor session thinking level for `qwen-chat-template` payloads so `/think off` disables nested llama.cpp chat-template thinking controls. Fixes #82768. Thanks @bfox55.
128129
- Feishu/wiki: reject numeric wiki space IDs before creating Lark clients and keep numeric-looking IDs documented as quoted opaque strings, preventing JavaScript precision loss in knowledge base calls. Fixes #45301. (#82769) Thanks @hyspacex.

‎extensions/codex/src/app-server/dynamic-tool-profile.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { CodexPluginConfig } from "./config.js";
1+
import type { CodexDynamicToolsLoading, CodexPluginConfig } from "./config.js";
22

33
export const CODEX_APP_SERVER_OWNED_DYNAMIC_TOOL_EXCLUDES = [
44
"read",
@@ -19,18 +19,44 @@ const DYNAMIC_TOOL_NAME_ALIASES: Record<string, string> = {
1919
"apply-patch": "apply_patch",
2020
};
2121

22+
type CodexDynamicToolProfileEnv = {
23+
OPENCLAW_BUILD_PRIVATE_QA?: string;
24+
OPENCLAW_QA_FORCE_RUNTIME?: string;
25+
};
26+
2227
export function normalizeCodexDynamicToolName(name: string): string {
2328
const normalized = name.trim().toLowerCase();
2429
return DYNAMIC_TOOL_NAME_ALIASES[normalized] ?? normalized;
2530
}
2631

32+
export function isForcedPrivateQaCodexRuntime(
33+
env: CodexDynamicToolProfileEnv = process.env,
34+
): boolean {
35+
return (
36+
env.OPENCLAW_BUILD_PRIVATE_QA === "1" &&
37+
env.OPENCLAW_QA_FORCE_RUNTIME?.trim().toLowerCase() === "codex"
38+
);
39+
}
40+
41+
export function resolveCodexDynamicToolsLoading(
42+
config: Pick<CodexPluginConfig, "codexDynamicToolsLoading">,
43+
env: CodexDynamicToolProfileEnv = process.env,
44+
): CodexDynamicToolsLoading {
45+
return isForcedPrivateQaCodexRuntime(env)
46+
? "direct"
47+
: (config.codexDynamicToolsLoading ?? "searchable");
48+
}
49+
2750
export function filterCodexDynamicTools<T extends { name: string }>(
2851
tools: T[],
2952
config: Pick<CodexPluginConfig, "codexDynamicToolsExclude">,
53+
env: CodexDynamicToolProfileEnv = process.env,
3054
): T[] {
3155
const excludes = new Set<string>();
32-
for (const name of CODEX_APP_SERVER_OWNED_DYNAMIC_TOOL_EXCLUDES) {
33-
excludes.add(name);
56+
if (!isForcedPrivateQaCodexRuntime(env)) {
57+
for (const name of CODEX_APP_SERVER_OWNED_DYNAMIC_TOOL_EXCLUDES) {
58+
excludes.add(name);
59+
}
3460
}
3561
for (const name of config.codexDynamicToolsExclude ?? []) {
3662
const trimmed = normalizeCodexDynamicToolName(name);

‎extensions/codex/src/app-server/run-attempt.test.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -646,6 +646,21 @@ describe("runCodexAppServerAttempt", () => {
646646
).toEqual(["message"]);
647647
});
648648

649+
it("exposes app-server-owned tools directly for forced private QA Codex runtime", () => {
650+
const tools = ["read", "write", "image_generate", "message"].map((name) => ({ name }));
651+
const privateQaCodexEnv = {
652+
OPENCLAW_BUILD_PRIVATE_QA: "1",
653+
OPENCLAW_QA_FORCE_RUNTIME: "codex",
654+
};
655+
656+
expect(
657+
__testing
658+
.filterCodexDynamicTools(tools, {}, privateQaCodexEnv)
659+
.map((tool) => tool.name),
660+
).toEqual(["read", "write", "image_generate", "message"]);
661+
expect(__testing.resolveCodexDynamicToolsLoading({}, privateQaCodexEnv)).toBe("direct");
662+
});
663+
649664
it("starts Codex threads without duplicate OpenClaw workspace tools by default", async () => {
650665
const sessionFile = path.join(tempDir, "session.jsonl");
651666
const workspaceDir = path.join(tempDir, "workspace");
@@ -897,6 +912,38 @@ describe("runCodexAppServerAttempt", () => {
897912
expect((factoryOptions[0] as { modelApi?: unknown }).modelApi).toBe("openai-responses");
898913
});
899914

915+
it("enables gateway subagent binding for forced private QA Codex runs", async () => {
916+
vi.stubEnv("OPENCLAW_BUILD_PRIVATE_QA", "1");
917+
vi.stubEnv("OPENCLAW_QA_FORCE_RUNTIME", "codex");
918+
const sessionFile = path.join(tempDir, "session.jsonl");
919+
const workspaceDir = path.join(tempDir, "workspace");
920+
const params = createParams(sessionFile, workspaceDir);
921+
params.disableTools = false;
922+
params.runtimePlan = createCodexRuntimePlanFixture();
923+
const factoryOptions: unknown[] = [];
924+
__testing.setOpenClawCodingToolsFactoryForTests((options) => {
925+
factoryOptions.push(options);
926+
return [createRuntimeDynamicTool("sessions_spawn")];
927+
});
928+
929+
const tools = await __testing.buildDynamicTools({
930+
params,
931+
resolvedWorkspace: workspaceDir,
932+
effectiveWorkspace: workspaceDir,
933+
sandboxSessionKey: params.sessionKey!,
934+
sandbox: null as never,
935+
runAbortController: new AbortController(),
936+
sessionAgentId: "main",
937+
pluginConfig: {},
938+
onYieldDetected: () => undefined,
939+
});
940+
941+
expect(factoryOptions).toHaveLength(1);
942+
const factoryOption = factoryOptions[0] as { allowGatewaySubagentBinding?: unknown };
943+
expect(factoryOption.allowGatewaySubagentBinding).toBe(true);
944+
expect(tools.map((tool) => tool.name)).toEqual(["sessions_spawn"]);
945+
});
946+
900947
it("normalizes Codex dynamic toolsAllow entries before filtering", () => {
901948
const tools = ["exec", "apply_patch", "read", "message"].map((name) => ({ name }));
902949

‎extensions/codex/src/app-server/run-attempt.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,12 @@ import {
7878
resolveCodexContextEngineProjectionMaxChars,
7979
resolveCodexContextEngineProjectionReserveTokens,
8080
} from "./context-engine-projection.js";
81-
import { filterCodexDynamicTools, normalizeCodexDynamicToolName } from "./dynamic-tool-profile.js";
81+
import {
82+
filterCodexDynamicTools,
83+
isForcedPrivateQaCodexRuntime,
84+
normalizeCodexDynamicToolName,
85+
resolveCodexDynamicToolsLoading,
86+
} from "./dynamic-tool-profile.js";
8287
import { createCodexDynamicToolBridge, type CodexDynamicToolBridge } from "./dynamic-tools.js";
8388
import { handleCodexAppServerElicitationRequest } from "./elicitation-bridge.js";
8489
import { CodexAppServerEventProjector } from "./event-projector.js";
@@ -618,7 +623,7 @@ export async function runCodexAppServerAttempt(
618623
const toolBridge = createCodexDynamicToolBridge({
619624
tools,
620625
signal: runAbortController.signal,
621-
loading: pluginConfig.codexDynamicToolsLoading ?? "searchable",
626+
loading: resolveCodexDynamicToolsLoading(pluginConfig),
622627
directToolNames: shouldForceMessageTool(params) ? ["message"] : [],
623628
hookContext: {
624629
agentId: sessionAgentId,
@@ -2748,7 +2753,8 @@ async function buildDynamicTools(input: DynamicToolBuildParams) {
27482753
senderUsername: params.senderUsername,
27492754
senderE164: params.senderE164,
27502755
senderIsOwner: params.senderIsOwner,
2751-
allowGatewaySubagentBinding: params.allowGatewaySubagentBinding,
2756+
allowGatewaySubagentBinding:
2757+
params.allowGatewaySubagentBinding || isForcedPrivateQaCodexRuntime(),
27522758
...sessionKeys,
27532759
sessionId: params.sessionId,
27542760
runId: params.runId,
@@ -3933,6 +3939,7 @@ export const __testing = {
39333939
isInvalidCodexImagePayloadError,
39343940
remapCodexContextFilePath,
39353941
resolveDynamicToolCallTimeoutMs,
3942+
resolveCodexDynamicToolsLoading,
39363943
restrictCodexAppServerSandboxForOpenClawSandbox,
39373944
resolveCodexAppServerForOpenClawToolPolicy,
39383945
resolveOpenClawCodingToolsSessionKeys,

‎extensions/codex/src/app-server/side-question.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,10 @@ import { handleCodexAppServerApprovalRequest } from "./approval-bridge.js";
1616
import { refreshCodexAppServerAuthTokens } from "./auth-bridge.js";
1717
import { isCodexAppServerApprovalRequest, type CodexAppServerClient } from "./client.js";
1818
import { readCodexPluginConfig, resolveCodexAppServerRuntimeOptions } from "./config.js";
19-
import { filterCodexDynamicTools } from "./dynamic-tool-profile.js";
19+
import {
20+
filterCodexDynamicTools,
21+
resolveCodexDynamicToolsLoading,
22+
} from "./dynamic-tool-profile.js";
2023
import { createCodexDynamicToolBridge, type CodexDynamicToolBridge } from "./dynamic-tools.js";
2124
import { handleCodexAppServerElicitationRequest } from "./elicitation-bridge.js";
2225
import {
@@ -378,7 +381,7 @@ async function createCodexSideToolBridge(input: {
378381
return createCodexDynamicToolBridge({
379382
tools,
380383
signal: input.signal,
381-
loading: input.pluginConfig.codexDynamicToolsLoading ?? "searchable",
384+
loading: resolveCodexDynamicToolsLoading(input.pluginConfig),
382385
hookContext: {
383386
agentId: input.sessionAgentId,
384387
config: input.params.cfg,

‎extensions/qa-lab/src/agentic-parity-report.test.ts‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary {
6666
},
6767
{
6868
name: "Compaction retry after mutating tool",
69-
status: "fail",
69+
status: "pass",
7070
steps: [],
7171
runtimeParity: {
7272
scenarioId: "compaction-retry-after-mutating-tool",
@@ -97,8 +97,8 @@ function makeRuntimeParitySummary(): QaRuntimeParitySuiteSummary {
9797
],
9898
counts: {
9999
total: 2,
100-
passed: 1,
101-
failed: 1,
100+
passed: 2,
101+
failed: 0,
102102
},
103103
run: {
104104
providerMode: "mock-openai",
@@ -801,9 +801,28 @@ status=done`,
801801
});
802802

803803
expect(report.runtimePair).toEqual(["pi", "codex"]);
804-
expect(report.pass).toBe(false);
804+
expect(report.pass).toBe(true);
805805
expect(report.driftCounts.none).toBe(1);
806806
expect(report.driftCounts["tool-call-shape"]).toBe(1);
807+
expect(report.failures).toEqual([]);
808+
});
809+
810+
it("fails runtime parity reports when a runtime cell fails", () => {
811+
const summary = makeRuntimeParitySummary();
812+
const scenario = summary.scenarios[1];
813+
if (!scenario?.runtimeParity) {
814+
throw new Error("runtime parity fixture missing");
815+
}
816+
scenario.status = "fail";
817+
scenario.runtimeParity.cells.codex.runtimeErrorClass = "tool-error";
818+
819+
const report = buildQaRuntimeParityReport({
820+
summary,
821+
comparedAt: "2026-05-10T00:00:00.000Z",
822+
});
823+
824+
expect(report.pass).toBe(false);
825+
expect(report.failedScenarios).toBe(1);
807826
expect(report.failures).toContain(
808827
"Compaction retry after mutating tool drift=tool-call-shape (tool call 1 differs).",
809828
);

‎extensions/qa-lab/src/agentic-parity-report.ts‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ import {
44
} from "./agentic-parity.js";
55
import type {
66
RuntimeId,
7-
RuntimeParityCell,
87
RuntimeParityDrift,
98
RuntimeParityResult,
109
} from "./runtime-parity.js";
10+
import { isRuntimeParityResultPass, runtimeParityCellStatus } from "./runtime-parity.js";
1111

1212
type QaParityReportStep = {
1313
name: string;
@@ -260,13 +260,6 @@ function normalizeRuntimePair(
260260
return ["pi", "codex"];
261261
}
262262

263-
function runtimeCellStatus(cell: RuntimeParityCell | undefined): "pass" | "fail" | "missing" {
264-
if (!cell) {
265-
return "missing";
266-
}
267-
return cell.runtimeErrorClass || cell.transportErrorClass ? "fail" : "pass";
268-
}
269-
270263
function requiredCoverageStatus(
271264
scenario: QaParityReportScenario | undefined,
272265
): "pass" | "fail" | "skip" | "missing" {
@@ -637,9 +630,9 @@ export function buildQaRuntimeParityReport(params: {
637630
driftCounts[parity.drift] += 1;
638631
const piCell = parity.cells.pi;
639632
const codexCell = parity.cells.codex;
640-
const piStatus = runtimeCellStatus(piCell);
641-
const codexStatus = runtimeCellStatus(codexCell);
642-
const status = scenario.status === "pass" ? "pass" : "fail";
633+
const piStatus = runtimeParityCellStatus(piCell);
634+
const codexStatus = runtimeParityCellStatus(codexCell);
635+
const status = isRuntimeParityResultPass(parity) ? "pass" : "fail";
643636
if (status === "fail") {
644637
failures.push(
645638
`${scenario.name} drift=${parity.drift}${parity.driftDetails ? ` (${parity.driftDetails})` : ""}.`,
@@ -660,12 +653,8 @@ export function buildQaRuntimeParityReport(params: {
660653
});
661654

662655
const totalScenarios = params.summary.counts?.total ?? scenarios.length;
663-
const passedScenarios =
664-
params.summary.counts?.passed ??
665-
scenarios.filter((scenario) => scenario.status === "pass").length;
666-
const failedScenarios =
667-
params.summary.counts?.failed ??
668-
scenarios.filter((scenario) => scenario.status === "fail").length;
656+
const passedScenarios = scenarios.filter((scenario) => scenario.status === "pass").length;
657+
const failedScenarios = scenarios.filter((scenario) => scenario.status === "fail").length;
669658

670659
return {
671660
runtimePair,
@@ -680,7 +669,7 @@ export function buildQaRuntimeParityReport(params: {
680669
pass: failures.length === 0 && failedScenarios === 0,
681670
failures,
682671
notes: [
683-
"Runtime parity treats none and text-only drift as pass; all structural, tool-shape, and failure-mode drift classes fail the lane.",
672+
"Runtime parity fails runtime, transport, and failure-mode drift; structural and tool-shape drift is recorded as advisory when both runtimes complete.",
684673
"Token totals here are assistant-message usage captured from the normalized transcript, not provider transport payloads.",
685674
],
686675
};

‎extensions/qa-lab/src/cli.runtime.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -868,6 +868,7 @@ describe("qa cli runtime", () => {
868868
finalText: "done",
869869
usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
870870
wallClockMs: 10,
871+
runtimeErrorClass: "tool-error",
871872
bootStateLines: [],
872873
},
873874
},

0 commit comments

Comments
 (0)