fix(qa-lab): fail missing parity tool results

vincentkoc · vincentkoc · commit 03f1bf9a4df0 · 2026-06-07T06:25:28.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -44,6 +44,7 @@ Docs: https://docs.openclaw.ai
 - Release/CI/E2E: plugin lifecycle matrix resource sampling now fails phases that exceed RSS, wall-clock, or CPU ceilings instead of only logging the measurements.
 - Release/CI/E2E: Codex npm plugin live assertions now cap transcript discovery and diagnostic log reads so failure proof stays bounded.
 - Tests/state isolation: QA Lab valid-tool-call metrics now require runtime tool-call evidence when runtime parity data is available instead of counting tool-backed scenario pass status alone.
+- Tests/state isolation: QA Lab runtime parity now fails planned-only tool-call rows without matching tool results instead of treating matching mock plans as real tool evidence.
 - Tests/state isolation: provider, media, auth, cron, task, session, sandbox, Gateway, and Codex timeout fixtures now scope more home/state/env data per test, reducing cross-test leakage and making release validation failures less noisy. (#90027, #89974)
 
 ## 2026.6.2
diff --git a/extensions/qa-lab/src/runtime-parity.test.ts b/extensions/qa-lab/src/runtime-parity.test.ts
@@ -0,0 +1,92 @@
+// Qa Lab tests cover runtime parity classification behavior.
+import { describe, expect, it } from "vitest";
+import {
+  __testing,
+  runRuntimeParityScenario,
+  type RuntimeId,
+  type RuntimeParityCell,
+  type RuntimeParityToolCall,
+} from "./runtime-parity.js";
+
+function makeRuntimeParityCell(
+  runtime: RuntimeId,
+  toolCalls: RuntimeParityToolCall[],
+): RuntimeParityCell {
+  return {
+    runtime,
+    transcriptBytes: '{"message":{"role":"assistant","content":"done"}}\n',
+    toolCalls,
+    finalText: "done",
+    usage: {
+      inputTokens: 1,
+      outputTokens: 1,
+      totalTokens: 2,
+    },
+    wallClockMs: 10,
+    bootStateLines: [],
+  };
+}
+
+describe("runtime parity", () => {
+  it("marks planned mock tool calls without outputs as missing tool results", () => {
+    const toolCalls = __testing.resolveToolCallOrderFromMockRequests([
+      {
+        plannedToolName: "read_file",
+        plannedToolArgs: { path: "README.md" },
+      },
+    ]);
+
+    expect(toolCalls).toHaveLength(1);
+    expect(toolCalls[0]).toMatchObject({
+      tool: "read_file",
+      errorClass: "tool-result-missing",
+    });
+  });
+
+  it("keeps resolved mock tool calls eligible for no-drift parity", async () => {
+    const toolCalls = __testing.resolveToolCallOrderFromMockRequests([
+      {
+        plannedToolName: "read_file",
+        plannedToolArgs: { path: "README.md" },
+      },
+      {
+        toolOutput: JSON.stringify({ ok: true }),
+      },
+    ]);
+
+    expect(toolCalls).toHaveLength(1);
+    expect(toolCalls[0]?.errorClass).toBeUndefined();
+
+    const result = await runRuntimeParityScenario({
+      scenarioId: "resolved-tool",
+      runCell: async (runtime) => ({
+        scenarioStatus: "pass",
+        cell: makeRuntimeParityCell(runtime, toolCalls),
+      }),
+    });
+
+    expect(result.drift).toBe("none");
+  });
+
+  it("classifies planned-only matching tool calls as failure-mode", async () => {
+    const toolCalls = __testing.resolveToolCallOrderFromMockRequests([
+      {
+        plannedToolName: "read_file",
+        plannedToolArgs: { path: "README.md" },
+      },
+    ]);
+
+    const result = await runRuntimeParityScenario({
+      scenarioId: "planned-only-tool",
+      runCell: async (runtime) => ({
+        scenarioStatus: "pass",
+        cell: makeRuntimeParityCell(runtime, toolCalls),
+      }),
+    });
+
+    expect(result).toMatchObject({
+      drift: "failure-mode",
+      driftDetails: "at least one runtime planned a tool call without a tool result",
+    });
+  });
+});
diff --git a/extensions/qa-lab/src/runtime-parity.ts b/extensions/qa-lab/src/runtime-parity.ts
@@ -134,6 +134,7 @@ const HEARTBEAT_RESPONSE_TOOL_NAME = "heartbeat_respond";
 const HEARTBEAT_TRANSCRIPT_PROMPT = "[OpenClaw heartbeat poll]";
 const HEARTBEAT_TASK_PROMPT_PREFIX =
   "Run the following periodic tasks (only those due based on their intervals):";
+const TOOL_RESULT_MISSING_ERROR_CLASS = "tool-result-missing";
 const BOOT_STATE_LINE_RE =
   /\b(?:FailoverError|No API key found|Codex app-server|auth profile|runtime policy|restart mode:|plugin|doctor)\b/i;
 const TOOL_RESULT_ERROR_RE = /\b(?:error|failed|failure|timeout|denied|enoent|not found)\b/i;
@@ -393,6 +394,17 @@ function classifyToolResultError(params: {
   return undefined;
 }
 
+function finalizeToolCallOrder(ordered: RuntimeParityPendingToolCall[]): RuntimeParityToolCall[] {
+  return ordered.map(({ _resolved, ...toolCall }) =>
+    _resolved
+      ? toolCall
+      : {
+          ...toolCall,
+          errorClass: toolCall.errorClass ?? TOOL_RESULT_MISSING_ERROR_CLASS,
+        },
+  );
+}
+
 function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): RuntimeParityToolCall[] {
   const ordered: RuntimeParityPendingToolCall[] = [];
   const byId = new Map<string, number>();
@@ -481,7 +493,7 @@ function resolveToolCallOrder(records: RuntimeParityTranscriptRecord[]): Runtime
     }
   }
 
-  return ordered.map(({ _resolved: _ignored, ...toolCall }) => toolCall);
+  return finalizeToolCallOrder(ordered);
 }
 
 function resolveToolCallOrderFromMockRequests(
@@ -545,7 +557,7 @@ function resolveToolCallOrderFromMockRequests(
     enqueueUnresolved(ordered.length - 1);
   }
 
-  return ordered.map(({ _resolved: _ignored, ...toolCall }) => toolCall);
+  return finalizeToolCallOrder(ordered);
 }
 
 function classifyScenarioError(details: string | undefined): string | undefined {
@@ -750,6 +762,10 @@ function isHardFailureRuntimeError(errorClass: string | undefined) {
   );
 }
 
+function hasMissingToolResult(toolCalls: readonly RuntimeParityToolCall[]) {
+  return toolCalls.some((toolCall) => toolCall.errorClass === TOOL_RESULT_MISSING_ERROR_CLASS);
+}
+
 function summarizeSentinelErrorClass(findings: readonly GatewayLogSentinelFinding[]) {
   if (findings.length === 0) {
     return undefined;
@@ -781,6 +797,16 @@ function classifyRuntimeParityCells(params: {
     };
   }
 
+  if (
+    hasMissingToolResult(params.openclaw.toolCalls) ||
+    hasMissingToolResult(params.codex.toolCalls)
+  ) {
+    return {
+      drift: "failure-mode",
+      driftDetails: "at least one runtime planned a tool call without a tool result",
+    };
+  }
+
   const toolCallShapeDetails = compareToolCallShape(
     params.openclaw.toolCalls,
     params.codex.toolCalls,
@@ -1025,3 +1051,10 @@ export async function runRuntimeParityScenario(params: {
     ...(drift.driftDetails ? { driftDetails: drift.driftDetails } : {}),
   };
 }
+
+export const testing = {
+  classifyRuntimeParityCells,
+  resolveToolCallOrderFromMockRequests,
+};
+
+export { testing as __testing };
