fix(json): retry on transient File changed during read race condition (#85029)

clawsweeper[bot] · samson1357924 · Takhoffman · web-flow · commit 7f943b5d8f83 · 2026-05-21T16:56:46.000Z
Summary: - The PR wraps the async JSON file readers in `src/infra/json-files.ts` with bounded retries for fs-safe `File changed during read` races, adds regression tests, and adds a changelog entry. - Reproducibility: yes. Source inspection shows fs-safe throws `File changed during read`, current main re-exp ... R proof includes before/after gateway logs; I did not run a new live race harness in this read-only review. Automerge notes: - PR branch already contained follow-up commit before automerge: fix(json): preserve strict reader types (Promise<T> for readJson/read… - PR branch already contained follow-up commit before automerge: test(json): add retry-success and retry-exhaustion coverage - PR branch already contained follow-up commit before automerge: fix(json): resolve lint warnings (prefer-exponentiation-operator, cur… - PR branch already contained follow-up commit before automerge: fix(json): retry on transient File changed during read race condition Validation: - ClawSweeper review passed for head 00602a1. - Required merge gates passed before the squash merge. Prepared head SHA: 00602a1 Review: #85029 (comment) Co-authored-by: samson1357924 <98934496+samson1357924@users.noreply.github.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: takhoffman Co-authored-by: takhoffman <781889+takhoffman@users.noreply.github.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,7 @@ Docs: https://docs.openclaw.ai
 
 ### Fixes
 
+- Infra/json: retry transient `File changed during read` races while loading JSON state so config and state reads recover instead of failing the turn. (#84285)
 - Gateway/sessions: preserve compatible session auth profile overrides when switching models within the same provider, including provider-auth aliases. Fixes #81837. (#81886) Thanks @TurboTheTurtle.
 - Gateway/status: surface inbound delivery telemetry counters and transport-liveness warnings in `openclaw status --all`. Fixes #49577. (#72724)
 - Providers/Ollama: treat Docker/OrbStack host aliases as local Ollama endpoints so `ollama-local` marker auth works when OpenClaw runs inside a VM/container and Ollama runs on the host. Fixes #84875.
diff --git a/src/infra/json-files.test.ts b/src/infra/json-files.test.ts
@@ -1,12 +1,15 @@
-import fs from "node:fs/promises";
+import fs from "node:fs";
+import fsPromises from "node:fs/promises";
 import path from "node:path";
 import { afterEach, describe, expect, it, vi } from "vitest";
 import { withTempDir } from "../test-helpers/temp-dir.js";
 import {
   JsonFileReadError,
   createAsyncLock,
   readDurableJsonFile,
+  readJson,
   readJsonFile,
+  tryReadJson,
   writeJsonAtomic,
   writeTextAtomic,
 } from "./json-files.js";
@@ -26,7 +29,7 @@ describe("json file helpers", () => {
       name: "reads valid json",
       setup: async (base: string) => {
         const filePath = path.join(base, "valid.json");
-        await fs.writeFile(filePath, '{"ok":true}', "utf8");
+        await fsPromises.writeFile(filePath, '{"ok":true}', "utf8");
         return filePath;
       },
       expected: { ok: true },
@@ -35,7 +38,7 @@ describe("json file helpers", () => {
       name: "returns null for invalid files",
       setup: async (base: string) => {
         const filePath = path.join(base, "invalid.json");
-        await fs.writeFile(filePath, "{not-json}", "utf8");
+        await fsPromises.writeFile(filePath, "{not-json}", "utf8");
         return filePath;
       },
       expected: null,
@@ -56,8 +59,8 @@ describe("json file helpers", () => {
       const validPath = path.join(base, "valid.json");
       const invalidPath = path.join(base, "invalid.json");
       const missingPath = path.join(base, "missing.json");
-      await fs.writeFile(validPath, '{"ok":true}', "utf8");
-      await fs.writeFile(invalidPath, "{not-json}", "utf8");
+      await fsPromises.writeFile(validPath, '{"ok":true}', "utf8");
+      await fsPromises.writeFile(invalidPath, "{not-json}", "utf8");
 
       await expect(readDurableJsonFile(validPath)).resolves.toEqual({ ok: true });
       await expect(readDurableJsonFile(missingPath)).resolves.toBeNull();
@@ -82,7 +85,7 @@ describe("json file helpers", () => {
         { trailingNewline: true, dirMode: 0o755 },
       );
 
-      await expect(fs.readFile(filePath, "utf8")).resolves.toBe(
+      await expect(fsPromises.readFile(filePath, "utf8")).resolves.toBe(
         '{\n  "ok": true,\n  "nested": {\n    "value": 1\n  }\n}\n',
       );
     });
@@ -95,56 +98,56 @@ describe("json file helpers", () => {
     await withTempDir({ prefix: "openclaw-json-files-" }, async (base) => {
       const filePath = path.join(base, "nested", "note.txt");
       await writeTextAtomic(filePath, input, { trailingNewline: true });
-      await expect(fs.readFile(filePath, "utf8")).resolves.toBe(expected);
+      await expect(fsPromises.readFile(filePath, "utf8")).resolves.toBe(expected);
     });
   });
 
   it("can skip durable fsync work for hot state writes", async () => {
     await withTempDir({ prefix: "openclaw-json-files-" }, async (base) => {
       const filePath = path.join(base, "state.json");
-      const openSpy = vi.spyOn(fs, "open");
+      const openSpy = vi.spyOn(fsPromises, "open");
 
       await writeTextAtomic(filePath, "new", { durable: false });
 
       expect(openSpy).not.toHaveBeenCalled();
-      await expect(fs.readFile(filePath, "utf8")).resolves.toBe("new");
+      await expect(fsPromises.readFile(filePath, "utf8")).resolves.toBe("new");
     });
   });
 
   it("preserves text when Windows rename reports EPERM", async () => {
     await withTempDir({ prefix: "openclaw-json-files-" }, async (base) => {
       const filePath = path.join(base, "state.json");
-      await fs.writeFile(filePath, "old", "utf8");
+      await fsPromises.writeFile(filePath, "old", "utf8");
 
       Object.defineProperty(process, "platform", { value: "win32", configurable: true });
       const renameError = Object.assign(new Error("EPERM"), { code: "EPERM" });
-      const renameSpy = vi.spyOn(fs, "rename").mockRejectedValueOnce(renameError);
+      const renameSpy = vi.spyOn(fsPromises, "rename").mockRejectedValueOnce(renameError);
 
       await writeTextAtomic(filePath, "new");
 
       expect(renameSpy).toHaveBeenCalledOnce();
-      await expect(fs.readFile(filePath, "utf8")).resolves.toBe("new");
+      await expect(fsPromises.readFile(filePath, "utf8")).resolves.toBe("new");
     });
   });
 
   it("refuses Windows copy fallback through symlink destinations", async () => {
     await withTempDir({ prefix: "openclaw-json-files-" }, async (base) => {
       const filePath = path.join(base, "state.json");
       const outsidePath = path.join(base, "outside.json");
-      await fs.writeFile(outsidePath, "outside", "utf8");
-      await fs.symlink(outsidePath, filePath);
+      await fsPromises.writeFile(outsidePath, "outside", "utf8");
+      await fsPromises.symlink(outsidePath, filePath);
 
       Object.defineProperty(process, "platform", { value: "win32", configurable: true });
       const renameError = Object.assign(new Error("EPERM"), { code: "EPERM" });
-      vi.spyOn(fs, "rename").mockRejectedValueOnce(renameError);
+      vi.spyOn(fsPromises, "rename").mockRejectedValueOnce(renameError);
 
       await expect(writeTextAtomic(filePath, "new")).rejects.toThrow(
         "Refusing copy fallback through symlink destination",
       );
 
-      const fileStat = await fs.lstat(filePath);
+      const fileStat = await fsPromises.lstat(filePath);
       expect(fileStat.isSymbolicLink()).toBe(true);
-      await expect(fs.readFile(outsidePath, "utf8")).resolves.toBe("outside");
+      await expect(fsPromises.readFile(outsidePath, "utf8")).resolves.toBe("outside");
     });
   });
 
@@ -185,4 +188,77 @@ describe("json file helpers", () => {
     await expect(second).resolves.toBe("ok");
     expect(events).toEqual(expectedEvents);
   });
+
+  describe("retry behaviors on 'File changed during read'", () => {
+    /**
+     * Helper: spy on fsPromises.lstat for our target file path.
+     * Returns a real Stats object with a modified ino to trigger
+     * verifyStableReadTarget in @openclaw/fs-safe.
+     * Object.assign + Object.create preserves the Stats prototype.
+     */
+    function setupLstatSpy(targetPath: string, targetCallCount: number): () => number {
+      const origLstat = fsPromises.lstat.bind(fsPromises);
+      let callCount = 0;
+
+      vi.spyOn(fsPromises, "lstat").mockImplementation(async (p, ...args) => {
+        const stat = await origLstat(p, ...args);
+        const pathStr = typeof p === "string" ? p : String(p);
+        if (pathStr === targetPath) {
+          callCount++;
+          if (callCount <= targetCallCount) {
+            // Modify ino: for BigInt ino add 100n, for number ino add 100
+            const modifiedIno = typeof stat.ino === "bigint" ? stat.ino + 100n : stat.ino + 100;
+
+            // Clone stat preserving prototype, override ino
+            return Object.assign(Object.create(Object.getPrototypeOf(stat)), stat, {
+              ino: modifiedIno,
+            });
+          }
+        }
+        return stat;
+      });
+
+      return () => callCount;
+    }
+
+    it("retries on transient File changed during read and succeeds", async () => {
+      await withTempDir({ prefix: "openclaw-json-files-retry-" }, async (base) => {
+        const filePath = path.join(base, "config.json");
+        await fsPromises.writeFile(filePath, '{"ok":true}', "utf8");
+
+        // Only fail lstat once (first call) — retry should succeed on 2nd attempt
+        const getCalls = setupLstatSpy(filePath, 1);
+
+        const result = await readJson<{ ok: boolean }>(filePath);
+        expect(result).toEqual({ ok: true });
+        // Should have at least 2 lstat calls: one failed, one successful
+        expect(getCalls()).toBeGreaterThanOrEqual(2);
+      });
+    });
+
+    it("throws JsonFileReadError after exhausting retries on persistent race", async () => {
+      await withTempDir({ prefix: "openclaw-json-files-exhaust-" }, async (base) => {
+        const filePath = path.join(base, "config.json");
+        await fsPromises.writeFile(filePath, '{"ok":true}', "utf8");
+
+        // Always fail lstat — all 3 retries should exhaust
+        setupLstatSpy(filePath, Infinity);
+
+        await expect(readJson(filePath)).rejects.toThrow(JsonFileReadError);
+      });
+    });
+
+    it("tryReadJson returns null after exhausting retries", async () => {
+      await withTempDir({ prefix: "openclaw-json-files-try-" }, async (base) => {
+        const filePath = path.join(base, "config.json");
+        await fsPromises.writeFile(filePath, '{"ok":true}', "utf8");
+
+        // Always fail lstat — tryReadJson catches and returns null
+        setupLstatSpy(filePath, Infinity);
+
+        const result = await tryReadJson(filePath);
+        expect(result).toBeNull();
+      });
+    });
+  });
 });
diff --git a/src/infra/json-files.ts b/src/infra/json-files.ts
@@ -1,24 +1,115 @@
 import "./fs-safe-defaults.js";
+import {
+  JsonFileReadError,
+  readJson as readJsonImpl,
+  readJsonIfExists as readJsonIfExistsImpl,
+} from "@openclaw/fs-safe/json";
 import { replaceFileAtomic } from "./replace-file.js";
 
 export {
   JsonFileReadError,
-  readJson,
-  readJson as readJsonFileStrict,
-  readJsonIfExists,
-  readJsonIfExists as readDurableJsonFile,
   readJsonSync,
   readRootJsonObjectSync,
   readRootJsonSync,
   readRootStructuredFileSync,
-  tryReadJson,
-  tryReadJson as readJsonFile,
   tryReadJsonSync,
   tryReadJsonSync as readJsonFileSync,
   writeJson,
   writeJson as writeJsonAtomic,
   writeJsonSync,
 } from "@openclaw/fs-safe/json";
+
+const RETRY_MAX_ATTEMPTS = 3;
+const RETRY_BASE_DELAY_MS = 50;
+
+/**
+ * Recursively walks the error cause chain to detect
+ * "File changed during read" errors wrapped inside
+ * JsonFileReadError by @openclaw/fs-safe.
+ */
+function isFileChangedDuringRead(err: unknown): boolean {
+  let current: unknown = err;
+  while (current) {
+    if (current instanceof Error) {
+      if (
+        typeof current.message === "string" &&
+        current.message.includes("File changed during read")
+      ) {
+        return true;
+      }
+      current = (current as Error & { cause?: unknown }).cause;
+    } else {
+      break;
+    }
+  }
+  return false;
+}
+
+async function withRetryOnFileChanged<T>(fn: () => Promise<T>): Promise<T> {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      if (isFileChangedDuringRead(err) && attempt < RETRY_MAX_ATTEMPTS - 1) {
+        await new Promise((r) => setTimeout(r, RETRY_BASE_DELAY_MS * 2 ** attempt));
+        continue;
+      }
+      throw err;
+    }
+  }
+}
+
+export async function readJson<T>(filePath: string): Promise<T> {
+  try {
+    return await withRetryOnFileChanged(() => readJsonImpl<T>(filePath));
+  } catch (err) {
+    throw err instanceof JsonFileReadError ? err : new JsonFileReadError(filePath, "read", err);
+  }
+}
+
+export async function readJsonFileStrict<T>(filePath: string): Promise<T> {
+  return readJson<T>(filePath);
+}
+
+export async function readJsonIfExists<T>(filePath: string): Promise<T | null> {
+  try {
+    return await withRetryOnFileChanged(() => readJsonIfExistsImpl<T>(filePath));
+  } catch (err) {
+    if (err instanceof JsonFileReadError) {
+      throw err;
+    }
+    throw new JsonFileReadError(filePath, "read", err);
+  }
+}
+
+export async function readDurableJsonFile<T>(filePath: string): Promise<T | null> {
+  return readJsonIfExists<T>(filePath);
+}
+
+/**
+ * tryReadJson delegates to readJsonIfExists instead of the internal
+ * tryReadJsonImpl from @openclaw/fs-safe. The fs-safe implementation
+ * swallows all errors internally and returns null, which prevents
+ * the retry wrapper from detecting transient "File changed during read"
+ * race conditions.
+ *
+ * By routing through readJsonIfExists, fs-safe propagates errors on
+ * race conditions, our retry wrapper intercepts and retries them,
+ * and the outer try-catch still handles parse errors / file-not-found
+ * gracefully.
+ */
+export async function tryReadJson<T>(filePath: string): Promise<T | null> {
+  try {
+    return await readJsonIfExists<T>(filePath);
+  } catch {
+    return null;
+  }
+}
+
+export async function readJsonFile<T>(filePath: string): Promise<T | null> {
+  return tryReadJson<T>(filePath);
+}
+
 export { createAsyncLock } from "@openclaw/fs-safe/advanced";
 
 export type WriteTextAtomicOptions = {
