fix(e2e): stop credential retries after deadline

vincentkoc · vincentkoc · commit 440e737c67dc · 2026-05-30T04:21:01.000+02:00
diff --git a/scripts/e2e/npm-telegram-rtt-credentials.mjs b/scripts/e2e/npm-telegram-rtt-credentials.mjs
@@ -350,12 +350,20 @@ async function acquireWithRetry(config) {
   let attempt = 0;
   while (true) {
     attempt += 1;
+    const attemptElapsedMs = Date.now() - startedAt;
+    const attemptRemainingMs = config.acquireTimeoutMs - attemptElapsedMs;
+    if (attemptRemainingMs <= 0) {
+      throw taggedError(
+        `credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`,
+        "ETIMEDOUT",
+      );
+    }
     try {
       return await postBroker({
         authToken: config.authToken,
         bodyMaxBytes: config.httpBodyMaxBytes,
         label: "credential broker acquire",
-        timeoutMs: config.httpTimeoutMs,
+        timeoutMs: Math.min(config.httpTimeoutMs, attemptRemainingMs),
         url: config.acquireUrl,
         body: {
           kind: "telegram",
@@ -375,9 +383,14 @@ async function acquireWithRetry(config) {
       const fallbackDelay = RETRY_BACKOFF_MS[Math.min(attempt - 1, RETRY_BACKOFF_MS.length - 1)];
       const retryAfterMs = error instanceof BrokerError ? error.retryAfterMs : undefined;
       const delayMs = retryAfterMs ?? fallbackDelay;
-      await new Promise((resolve) =>
-        setTimeout(resolve, Math.min(delayMs, Math.max(config.acquireTimeoutMs - elapsedMs, 0))),
-      );
+      const remainingMs = config.acquireTimeoutMs - elapsedMs;
+      if (delayMs >= remainingMs) {
+        throw taggedError(
+          `credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`,
+          "ETIMEDOUT",
+        );
+      }
+      await new Promise((resolve) => setTimeout(resolve, delayMs));
     }
   }
 }
diff --git a/test/scripts/rtt-harness.test.ts b/test/scripts/rtt-harness.test.ts
@@ -257,6 +257,108 @@ describe("RTT harness", () => {
     }
   });
 
+  it("does not start another credential acquire after retry delay exhausts the deadline", async () => {
+    let requests = 0;
+    const server = createServer((_request, response) => {
+      requests += 1;
+      response.writeHead(503, { "content-type": "application/json" });
+      response.end(
+        JSON.stringify({
+          status: "error",
+          code: "POOL_EXHAUSTED",
+          message: "credential pool exhausted",
+          retryAfterMs: 1_000,
+        }),
+      );
+    });
+    const { port } = await listenOnLoopback(server);
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-retry-"));
+    tempDirs.push(tempDir);
+    const startedAt = Date.now();
+
+    try {
+      await execFileAsync(
+        process.execPath,
+        [
+          CREDENTIAL_SCRIPT_PATH,
+          "acquire",
+          "--lease-file",
+          path.join(tempDir, "lease.json"),
+          "--credential-env-file",
+          path.join(tempDir, "credentials.env"),
+        ],
+        {
+          env: {
+            ...credentialBrokerEnv(port),
+            OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "75",
+            OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "250",
+          },
+          maxBuffer: 128 * 1024,
+        },
+      );
+      throw new Error("Expected credential acquire to fail.");
+    } catch (error) {
+      const execError = error as Error & { stderr?: string };
+      expect(execError.stderr).toContain("credential broker acquire timed out after 75ms");
+      expect(Date.now() - startedAt).toBeLessThan(500);
+      expect(requests).toBe(1);
+    } finally {
+      await closeServer(server);
+    }
+  });
+
+  it("caps credential acquire HTTP retries to the remaining acquire deadline", async () => {
+    let requests = 0;
+    const server = createServer((_request, response) => {
+      requests += 1;
+      if (requests === 1) {
+        response.writeHead(503, { "content-type": "application/json" });
+        response.end(
+          JSON.stringify({
+            status: "error",
+            code: "POOL_EXHAUSTED",
+            message: "credential pool exhausted",
+            retryAfterMs: 1,
+          }),
+        );
+      }
+    });
+    const { port } = await listenOnLoopback(server);
+    const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-cap-"));
+    tempDirs.push(tempDir);
+    const startedAt = Date.now();
+
+    try {
+      await execFileAsync(
+        process.execPath,
+        [
+          CREDENTIAL_SCRIPT_PATH,
+          "acquire",
+          "--lease-file",
+          path.join(tempDir, "lease.json"),
+          "--credential-env-file",
+          path.join(tempDir, "credentials.env"),
+        ],
+        {
+          env: {
+            ...credentialBrokerEnv(port),
+            OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "100",
+            OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "900",
+          },
+          maxBuffer: 128 * 1024,
+        },
+      );
+      throw new Error("Expected credential acquire to fail.");
+    } catch (error) {
+      const execError = error as Error & { stderr?: string };
+      expect(execError.stderr).toContain("credential broker acquire timed out after");
+      expect(Date.now() - startedAt).toBeLessThan(500);
+      expect(requests).toBe(2);
+    } finally {
+      await closeServer(server);
+    }
+  });
+
   it("preserves empty broker responses for successful lease release", async () => {
     const server = createServer((_request, response) => {
       response.writeHead(204);
