fix(agent): retry transient gateway handshake closes

vincentkoc · vincentkoc · commit ff79299d68e3 · 2026-05-22T12:24:50.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -28,6 +28,7 @@ Docs: https://docs.openclaw.ai
 
 ### Fixes
 
+- CLI/agents: retry transient normal-close Gateway handshakes before falling back to embedded `openclaw agent` execution.
 - Providers/Gemini: strip fractional seconds from web-search time range filters so Gemini accepts freshness-bound search requests. (#85071) Thanks @Noerr.
 - OpenAI Codex: preserve image input support for sparse `openai-codex/gpt-5.5` catalog rows. (#85095) Thanks @sercada.
 - Plugins/discovery: strip `-plugin` package suffixes when deriving plugin id hints so package names line up with manifest ids. (#85170) Thanks @JulyanXu.
diff --git a/src/commands/agent-via-gateway.test.ts b/src/commands/agent-via-gateway.test.ts
@@ -167,6 +167,21 @@ function createGatewayClosedError() {
   });
 }
 
+function createGatewayNormalCloseError() {
+  const err = new Error("gateway closed (1000 normal closure): no close reason");
+  err.name = "GatewayTransportError";
+  return Object.assign(err, {
+    kind: "closed",
+    code: 1000,
+    reason: "no close reason",
+    connectionDetails: {
+      url: "ws://127.0.0.1:18789",
+      urlSource: "local loopback",
+      message: "Gateway target: ws://127.0.0.1:18789",
+    },
+  });
+}
+
 vi.mock("../config/config.js", () => ({ getRuntimeConfig: loadConfig, loadConfig }));
 vi.mock("../gateway/call.js", () => ({
   callGateway,
@@ -1021,6 +1036,36 @@ describe("agentCliCommand", () => {
       expect(agentCommand).not.toHaveBeenCalled();
     });
   });
+
+  it("aborts while waiting for a transient gateway retry", async () => {
+    vi.useFakeTimers();
+    try {
+      await withTempStore(async () => {
+        const signals = createSignalProcess();
+        callGateway.mockRejectedValueOnce(createGatewayNormalCloseError());
+
+        const run = agentCliCommand({ message: "hi", to: "+1555" }, runtime, {
+          process: signals.processLike,
+        });
+        for (
+          let attempt = 0;
+          attempt < 10 && mockMessages(runtime.error).length === 0;
+          attempt += 1
+        ) {
+          await Promise.resolve();
+        }
+        signals.emit("SIGTERM");
+
+        await expect(run).resolves.toBeUndefined();
+        expect(callGateway).toHaveBeenCalledTimes(1);
+        expect(agentCommand).not.toHaveBeenCalled();
+        expect(runtime.exit).toHaveBeenCalledWith(143);
+      });
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   it("stays silent when the gateway returns an intentional empty reply", async () => {
     await withTempStore(async () => {
       callGateway.mockResolvedValue({
@@ -1173,6 +1218,45 @@ describe("agentCliCommand", () => {
     });
   });
 
+  it("retries transient normal gateway closes before embedded fallback", async () => {
+    vi.useFakeTimers();
+    try {
+      await withTempStore(async () => {
+        callGateway
+          .mockRejectedValueOnce(createGatewayNormalCloseError())
+          .mockRejectedValueOnce(createGatewayNormalCloseError())
+          .mockResolvedValue({
+            runId: "idem-1",
+            status: "ok",
+            result: {
+              payloads: [{ text: "remote" }],
+              meta: { stub: true },
+            },
+          });
+
+        const command = agentCliCommand({ message: "hi", to: "+1555" }, runtime);
+        await vi.advanceTimersByTimeAsync(1_000);
+        await vi.advanceTimersByTimeAsync(2_000);
+        await command;
+
+        expect(callGateway).toHaveBeenCalledTimes(3);
+        const idempotencyKeys = callGateway.mock.calls.map(
+          ([call]) => (call as { params?: { idempotencyKey?: unknown } }).params?.idempotencyKey,
+        );
+        expect(new Set(idempotencyKeys).size).toBe(1);
+        expect(agentCommand).not.toHaveBeenCalled();
+        expect(
+          mockMessages(runtime.error).filter((message) =>
+            message.includes("Gateway agent connection closed during handshake"),
+          ),
+        ).toHaveLength(2);
+        expect(runtime.log).toHaveBeenCalledWith("remote");
+      });
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+
   it("preserves explicit session keys for embedded fallback when the gateway closes", async () => {
     await withTempStore(async () => {
       callGateway.mockRejectedValue(createGatewayClosedError());
diff --git a/src/commands/agent-via-gateway.ts b/src/commands/agent-via-gateway.ts
@@ -52,6 +52,7 @@ const EMBEDDED_FALLBACK_META = {
   fallbackFrom: "gateway",
 } as const;
 const GATEWAY_TIMEOUT_FALLBACK_SESSION_PREFIX = "gateway-fallback-";
+const GATEWAY_TRANSIENT_CONNECT_RETRY_DELAYS_MS = [1_000, 2_000, 5_000, 10_000, 15_000] as const;
 
 type AgentCliOpts = {
   message: string;
@@ -147,6 +148,15 @@ function isGatewayAgentEmbeddedFallbackError(err: unknown): boolean {
   return isGatewayTransportError(err);
 }
 
+function isTransientGatewayAgentConnectClose(err: unknown): boolean {
+  if (!isGatewayTransportError(err) || err.kind !== "closed") {
+    return false;
+  }
+  const code = typeof err.code === "number" ? err.code : undefined;
+  const reason = normalizeOptionalString(err.reason);
+  return code === 1000 && (!reason || reason === "no close reason");
+}
+
 function validateExplicitSessionKeyForDispatch(
   opts: Pick<AgentCliOpts, "agent" | "sessionKey">,
 ): void {
@@ -268,8 +278,28 @@ function resolveAgentCliProcessLike(deps: AgentCliDeps | undefined): AgentCliPro
   return isAgentCliProcessLike(processLike) ? processLike : process;
 }
 
-function delayMs(ms: number): Promise<void> {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+function createAbortDelayError(): Error {
+  const err = new Error("gateway agent retry aborted");
+  err.name = "AbortError";
+  return err;
+}
+
+function delayMs(ms: number, signal?: AbortSignal): Promise<void> {
+  if (signal?.aborted) {
+    return Promise.reject(createAbortDelayError());
+  }
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      signal?.removeEventListener("abort", onAbort);
+      reject(createAbortDelayError());
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
 }
 
 function isConfirmedChatAbortResponseForRun(value: unknown, runId: string): boolean {
@@ -631,6 +661,34 @@ async function agentViaGatewayCommand(
   return response;
 }
 
+async function agentViaGatewayCommandWithTransientRetries(
+  opts: AgentCliOpts,
+  runtime: RuntimeEnv,
+  signalBridge: ReturnType<typeof createAgentCliSignalBridge>,
+) {
+  for (const [attempt, retryDelayMs] of [
+    ...GATEWAY_TRANSIENT_CONNECT_RETRY_DELAYS_MS,
+    0,
+  ].entries()) {
+    try {
+      return await agentViaGatewayCommand(opts, runtime, signalBridge);
+    } catch (err) {
+      if (isAbortError(err)) {
+        throw err;
+      }
+      const isFinalAttempt = attempt === GATEWAY_TRANSIENT_CONNECT_RETRY_DELAYS_MS.length;
+      if (isFinalAttempt || !isTransientGatewayAgentConnectClose(err)) {
+        throw err;
+      }
+      runtime.error?.(
+        `Gateway agent connection closed during handshake; retrying in ${retryDelayMs}ms before embedded fallback.`,
+      );
+      await delayMs(retryDelayMs, signalBridge.signal);
+    }
+  }
+  throw new Error("Gateway agent retry loop exhausted unexpectedly.");
+}
+
 export async function agentCliCommand(
   opts: AgentCliOpts,
   runtime: RuntimeEnv,
@@ -639,11 +697,14 @@ export async function agentCliCommand(
   protectJsonStdout(opts);
   const dispatchOpts = normalizeSessionKeyOptsForDispatch(opts);
   validateExplicitSessionKeyForDispatch(dispatchOpts);
+  const gatewayDispatchOpts = dispatchOpts.runId
+    ? dispatchOpts
+    : { ...dispatchOpts, runId: randomIdempotencyKey() };
   const signalBridge = createAgentCliSignalBridge(resolveAgentCliProcessLike(deps));
   const localOpts = {
-    ...dispatchOpts,
-    agentId: dispatchOpts.agent,
-    replyAccountId: dispatchOpts.replyAccount,
+    ...gatewayDispatchOpts,
+    agentId: gatewayDispatchOpts.agent,
+    replyAccountId: gatewayDispatchOpts.replyAccount,
     cleanupBundleMcpOnRunEnd: true,
     cleanupCliLiveSessionOnRunEnd: true,
     abortSignal: signalBridge.signal,
@@ -655,7 +716,11 @@ export async function agentCliCommand(
     }
 
     try {
-      const result = await agentViaGatewayCommand(dispatchOpts, runtime, signalBridge);
+      const result = await agentViaGatewayCommandWithTransientRetries(
+        gatewayDispatchOpts,
+        runtime,
+        signalBridge,
+      );
       return returnAfterSignalExit(result, signalBridge.getReceivedSignal(), runtime);
     } catch (err) {
       if (isAbortError(err)) {
