Skip to content

Commit 5f89fbe

Browse files
keshavbotagentkeshavbotagent
authored
fix(codex): recover app-server completion stalls
Fix Codex app-server completion-stall recovery so replay-safe stdio completion-idle failures retry once, while progress/terminal turn-watch timeouts only surface timeout payloads. Also preserve post-tool completion guards for scoped native response deltas and stabilize the oversized CONNECT timeout regression test picked up from latest main. Co-authored-by: Kelaw - Keshav's Agent <keshavbotagent@gmail.com>
1 parent bc848b3 commit 5f89fbe

21 files changed

Lines changed: 753 additions & 108 deletions

‎docs/plugins/codex-harness-reference.md‎

Lines changed: 28 additions & 23 deletions
Large diffs are not rendered by default.

‎docs/plugins/codex-harness.md‎

Lines changed: 28 additions & 23 deletions
Large diffs are not rendered by default.

‎extensions/codex-supervisor/src/supervisor.test.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -808,12 +808,15 @@ describe("connectCodexAppServerEndpoint", () => {
808808
const sawProbeRequest = new Promise<void>((resolve) => {
809809
server.once("connection", (socket) => {
810810
socket.on("message", (data) => {
811-
const text = Array.isArray(data)
812-
? Buffer.concat(data).toString("utf8")
813-
: data instanceof ArrayBuffer
814-
? Buffer.from(new Uint8Array(data)).toString("utf8")
815-
: Buffer.from(data).toString("utf8");
816-
const request = JSON.parse(text) as Record<string, unknown>;
811+
const messageText =
812+
typeof data === "string"
813+
? data
814+
: Array.isArray(data)
815+
? Buffer.concat(data).toString()
816+
: data instanceof ArrayBuffer
817+
? Buffer.from(new Uint8Array(data)).toString()
818+
: Buffer.from(data).toString();
819+
const request = JSON.parse(messageText) as Record<string, unknown>;
817820
if (request.method === "initialize") {
818821
socket.send(JSON.stringify({ id: request.id, result: {} }));
819822
}

‎extensions/codex/openclaw.plugin.json‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,8 @@
176176
},
177177
"postToolRawAssistantCompletionIdleTimeoutMs": {
178178
"type": "number",
179-
"minimum": 1
179+
"minimum": 1,
180+
"default": 300000
180181
},
181182
"approvalPolicy": {
182183
"type": "string",
@@ -360,7 +361,7 @@
360361
},
361362
"appServer.postToolRawAssistantCompletionIdleTimeoutMs": {
362363
"label": "Post-Tool Raw Assistant Completion Idle Timeout",
363-
"help": "Completion-idle guard after a tool handoff when Codex emits raw assistant completion or progress without turn/completed. Defaults to the assistant completion idle timeout when unset.",
364+
"help": "Completion-idle guard after a tool handoff when Codex emits raw assistant completion or progress without turn/completed. Defaults to 300000 ms when unset.",
364365
"advanced": true
365366
},
366367
"appServer.approvalPolicy": {

‎extensions/codex/src/app-server/attempt-notification-state.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import {
44
isAssistantCompletionReleaseNotification,
55
isCodexTurnAbortMarkerNotification,
66
isNativeToolProgressNotification,
7+
isNativeResponseStreamDeltaNotification,
78
isPendingOpenClawDynamicToolCompletionNotification,
89
isRawAssistantCompletionNotification,
910
isRawReasoningCompletionNotification,
@@ -99,9 +100,10 @@ export function applyCodexTurnNotificationState(params: {
99100
params.turnId,
100101
);
101102
const isTurnCompletion = notification.method === "turn/completed" && isCurrentTurnNotification;
103+
const isNativeResponseStreamDelta = isNativeResponseStreamDeltaNotification(notification);
102104
let turnCrossedToolHandoff = params.turnCrossedToolHandoff;
103105

104-
if (isCurrentTurnNotification) {
106+
if (isCurrentTurnNotification && !isNativeResponseStreamDelta) {
105107
turnWatches.touchActivity(`notification:${notification.method}`, {
106108
details: describeNotificationActivity(notification),
107109
attemptProgress: true,
@@ -174,6 +176,9 @@ export function applyCodexTurnNotificationState(params: {
174176
} else if (isCurrentTurnNotification && assistantCompletionCanRelease) {
175177
turnWatches.armAssistantCompletionIdleWatch(describeNotificationActivity(notification));
176178
} else if (postToolRawAssistantCompletionNeedsTerminalGuard) {
179+
// A post-tool assistant status can be followed by native Codex streaming a
180+
// large custom tool input. Forwarded raw deltas refresh activity at enqueue
181+
// time; keep this guard conservative for versions that do not forward them.
177182
turnWatches.armCompletionIdleWatch({
178183
timeoutMs: params.postToolRawAssistantCompletionIdleTimeoutMs,
179184
});
@@ -203,6 +208,7 @@ export function applyCodexTurnNotificationState(params: {
203208
!turnWatches.isCompletionIdleWatchPinnedByTerminalError() &&
204209
notification.method !== "turn/completed" &&
205210
isCurrentTurnNotification &&
211+
!isNativeResponseStreamDelta &&
206212
!trackedDynamicToolCompletion &&
207213
!rawToolOutputCompletion &&
208214
!postToolRawAssistantCompletionNeedsTerminalGuard &&

‎extensions/codex/src/app-server/attempt-notifications.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,12 @@ export function isNativeToolProgressNotification(notification: CodexServerNotifi
179179
}
180180
}
181181

182+
export function isNativeResponseStreamDeltaNotification(
183+
notification: CodexServerNotification,
184+
): boolean {
185+
return notification.method.startsWith("response.") && notification.method.endsWith(".delta");
186+
}
187+
182188
export function isRawAssistantCompletionNotification(
183189
notification: CodexServerNotification,
184190
): boolean {

‎extensions/codex/src/app-server/attempt-results.test.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,28 @@ describe("Codex app-server attempt results", () => {
8989
replayInvalid: true,
9090
livenessState: "abandoned",
9191
});
92+
expect(
93+
buildCodexAppServerPromptTimeoutOutcome({
94+
result: createResult({
95+
assistantTexts: ["I am changing the data model now..."],
96+
}),
97+
turnCompletionIdleTimedOut: true,
98+
}),
99+
).toEqual({
100+
message:
101+
"Codex stopped before confirming the turn was complete. The response may be incomplete; retry if needed.",
102+
});
103+
expect(
104+
buildCodexAppServerPromptTimeoutOutcome({
105+
result: createResult({
106+
toolMetas: [{ toolName: "exec" }],
107+
}),
108+
turnCompletionIdleTimedOut: true,
109+
}),
110+
).toEqual({
111+
message:
112+
"Codex stopped before confirming the turn was complete. The response may be incomplete; retry if needed.",
113+
});
92114
});
93115

94116
it("classifies replay blocked reasons", () => {

‎extensions/codex/src/app-server/attempt-results.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,12 @@ export function buildCodexAppServerPromptTimeoutOutcome(params: {
2727
const completionIdleTimeoutHadPotentialSideEffects = hasCodexAppServerPotentialSideEffectEvidence(
2828
params.result,
2929
);
30+
const replayBlockedReason = resolveCodexAppServerReplayBlockedReason(params.result);
3031
if (
3132
!params.turnCompletionIdleTimedOut ||
3233
(params.result.itemLifecycle.completedCount === 0 &&
33-
!completionIdleTimeoutHadPotentialSideEffects)
34+
!completionIdleTimeoutHadPotentialSideEffects &&
35+
replayBlockedReason === undefined)
3436
) {
3537
return undefined;
3638
}

‎extensions/codex/src/app-server/attempt-timeouts.test.ts‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { afterEach, describe, expect, it, vi } from "vitest";
22
import {
33
CODEX_APP_SERVER_STARTUP_TIMEOUT_FLOOR_MS,
4+
CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS,
45
CODEX_TURN_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS,
56
CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS,
67
CODEX_TURN_TERMINAL_IDLE_TIMEOUT_MS,
@@ -36,6 +37,11 @@ describe("Codex app-server attempt timeouts", () => {
3637
});
3738

3839
it("normalizes turn idle timeout overrides", () => {
40+
expect(CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS).toBe(5 * 60_000);
41+
expect(CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS).toBeGreaterThan(
42+
CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS,
43+
);
44+
3945
expect(resolveCodexTurnCompletionIdleTimeoutMs(undefined)).toBe(
4046
CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS,
4147
);
@@ -54,9 +60,21 @@ describe("Codex app-server attempt timeouts", () => {
5460
expect(resolveCodexTurnAssistantCompletionIdleTimeoutMs(9.8)).toBe(9);
5561
expect(resolveCodexTurnAssistantCompletionIdleTimeoutMs(-10)).toBe(1);
5662

57-
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 123)).toBe(123);
58-
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(Number.NaN, 123)).toBe(123);
59-
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, Number.NaN)).toBe(1);
63+
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 123)).toBe(
64+
CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS,
65+
);
66+
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(Number.NaN, 123)).toBe(
67+
CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS,
68+
);
69+
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 120_000)).toBe(
70+
CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS,
71+
);
72+
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, 6 * 60_000)).toBe(
73+
6 * 60_000,
74+
);
75+
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(undefined, Number.NaN)).toBe(
76+
CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS,
77+
);
6078
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(7.9, 123)).toBe(7);
6179
expect(resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(0, 123)).toBe(1);
6280

‎extensions/codex/src/app-server/attempt-timeouts.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ import { parseFiniteNumber } from "openclaw/plugin-sdk/number-runtime";
33
export const CODEX_APP_SERVER_STARTUP_TIMEOUT_FLOOR_MS = 100;
44
export const CODEX_TURN_COMPLETION_IDLE_TIMEOUT_MS = 60_000;
55
export const CODEX_TURN_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS = 10_000;
6+
// Native Codex can stream a large custom tool input after a raw assistant
7+
// progress item. Forwarded deltas count as activity, but older native paths may
8+
// not surface them, so keep this terminal guard conservative.
9+
export const CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS = 5 * 60_000;
610
export const CODEX_POST_REASONING_SOURCE_REPLY_IDLE_TIMEOUT_MS = 5 * 60_000;
711
export const CODEX_TURN_TERMINAL_IDLE_TIMEOUT_MS = 30 * 60_000;
812

@@ -91,7 +95,11 @@ export function resolveCodexPostToolRawAssistantCompletionIdleTimeoutMs(
9195
value: number | undefined,
9296
fallbackMs: number,
9397
): number {
94-
return resolvePositiveIntegerTimeoutMs(value, fallbackMs);
98+
const defaultMs = Math.max(
99+
resolvePositiveIntegerTimeoutMs(undefined, fallbackMs),
100+
CODEX_POST_TOOL_RAW_ASSISTANT_COMPLETION_IDLE_TIMEOUT_MS,
101+
);
102+
return resolvePositiveIntegerTimeoutMs(value, defaultMs);
95103
}
96104

97105
export function resolveCodexTurnTerminalIdleTimeoutMs(value: number | undefined): number {

0 commit comments

Comments
 (0)