Skip to content

Commit 538d36e

Browse files
steipetesteipete
authored
refactor: move session metadata to SQLite (#91322)
* refactor: move session metadata to sqlite * test: seed session stores with sqlite fixtures * test: seed remaining session stores with sqlite fixtures * fix: stabilize sqlite session cache freshness * test: seed cli transcript metadata in sqlite
1 parent b2c1de7 commit 538d36e

149 files changed

Lines changed: 4333 additions & 3990 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/.generated/plugin-sdk-api-baseline.sha256‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
16202a4c1ba8816643ad4cc81536c6ff9bfea38b01826d090c2195230dc85ab3 plugin-sdk-api-baseline.json
2-
a674e0fc5998b343fd1235438794c9c342fcd6e538157650109d2d30c184b7bc plugin-sdk-api-baseline.jsonl
1+
8be695e0892078773d78dae5f4c5a20ee57f30c5df227be6a89cc316fc4b4e10 plugin-sdk-api-baseline.json
2+
45944cb6fa30a094c4f104d19eec7afc5822be05c88bbd2aa4a8b93a7cba9de8 plugin-sdk-api-baseline.jsonl

‎docs/plugins/sdk-subpaths.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -246,7 +246,7 @@ usage endpoint failed or returned no usable usage data.
246246
| `plugin-sdk/reply-history` | Shared short-window reply-history helpers. New message-turn code should use `createChannelHistoryWindow`; lower-level map helpers remain deprecated compatibility exports only |
247247
| `plugin-sdk/reply-reference` | `createReplyReferencePlanner` |
248248
| `plugin-sdk/reply-chunking` | Narrow text/markdown chunking helpers |
249-
| `plugin-sdk/session-store-runtime` | Session workflow helpers (`getSessionEntry`, `listSessionEntries`, `patchSessionEntry`, `upsertSessionEntry`), legacy session store path/session-key helpers, updated-at reads, and deprecated whole-store mutation helpers |
249+
| `plugin-sdk/session-store-runtime` | Session workflow helpers (`getSessionEntry`, `listSessionEntries`, `patchSessionEntry`, `upsertSessionEntry`), target discovery, legacy session store path/session-key helpers, updated-at reads, and deprecated whole-store mutation helpers |
250250
| `plugin-sdk/cron-store-runtime` | Cron store path/load/save helpers |
251251
| `plugin-sdk/state-paths` | State/OAuth dir path helpers |
252252
| `plugin-sdk/plugin-state-runtime` | Plugin sidecar SQLite keyed-state types |

‎extensions/codex/src/app-server/startup-binding.test.ts‎

Lines changed: 29 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
import fs from "node:fs/promises";
33
import os from "node:os";
44
import path from "node:path";
5+
import { saveSessionStore } from "openclaw/plugin-sdk/session-store-runtime";
56
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
67
import { readCodexAppServerBinding, writeCodexAppServerBinding } from "./session-binding.js";
78
import { rotateOversizedCodexAppServerStartupBinding } from "./startup-binding.js";
@@ -34,14 +35,17 @@ describe("Codex app-server startup binding", () => {
3435

3536
async function writeSessionRecord(sessionFile: string, record: Record<string, unknown>) {
3637
await fs.mkdir(path.dirname(sessionFile), { recursive: true });
37-
await fs.writeFile(
38+
await saveSessionStore(
3839
path.join(path.dirname(sessionFile), "sessions.json"),
39-
JSON.stringify({
40+
{
4041
"agent:main:session-1": {
42+
sessionId: "session-1",
4143
sessionFile,
44+
updatedAt: Date.now(),
4245
...record,
4346
},
44-
}),
47+
},
48+
{ skipMaintenance: true },
4549
);
4650
}
4751

@@ -78,29 +82,33 @@ describe("Codex app-server startup binding", () => {
7882
expect(savedBinding?.threadId).toBe("thread-existing");
7983
});
8084

81-
it("reuses the session record cache while sessions.json is unchanged", async () => {
85+
it("reads updated SQLite-backed session records between startup checks", async () => {
8286
const sessionFile = path.join(tempDir, "session.jsonl");
8387
const workspaceDir = path.join(tempDir, "workspace");
8488
const agentDir = path.join(tempDir, "agent");
8589
await writeExistingBinding(sessionFile, workspaceDir, { dynamicToolsFingerprint: "[]" });
8690
await writeSessionRecord(sessionFile, { totalTokens: 12_000 });
87-
const sessionsJson = path.join(path.dirname(sessionFile), "sessions.json");
88-
const readFileSpy = vi.spyOn(fs, "readFile");
89-
90-
for (let i = 0; i < 2; i += 1) {
91-
const binding = await rotateOversizedCodexAppServerStartupBinding({
92-
binding: await readCodexAppServerBinding(sessionFile),
93-
sessionFile,
94-
agentDir,
95-
config: undefined,
96-
});
97-
expect(binding?.threadId).toBe("thread-existing");
98-
}
99-
100-
const sessionStoreReads = readFileSpy.mock.calls.filter(
101-
([file]) => typeof file === "string" && file === sessionsJson,
102-
);
103-
expect(sessionStoreReads).toHaveLength(1);
91+
92+
const firstBinding = await rotateOversizedCodexAppServerStartupBinding({
93+
binding: await readCodexAppServerBinding(sessionFile),
94+
sessionFile,
95+
agentDir,
96+
config: undefined,
97+
});
98+
expect(firstBinding?.threadId).toBe("thread-existing");
99+
100+
await writeSessionRecord(sessionFile, { totalTokens: 400_000 });
101+
102+
const secondBinding = await rotateOversizedCodexAppServerStartupBinding({
103+
binding: await readCodexAppServerBinding(sessionFile),
104+
sessionFile,
105+
agentDir,
106+
config: undefined,
107+
});
108+
109+
expect(secondBinding).toBeUndefined();
110+
const savedBinding = await readCodexAppServerBinding(sessionFile);
111+
expect(savedBinding).toBeUndefined();
104112
});
105113

106114
it("checks native rollout token pressure under default compaction config", async () => {

‎extensions/codex/src/app-server/startup-binding.ts‎

Lines changed: 6 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
embeddedAgentLog,
1010
type EmbeddedRunAttemptParams,
1111
} from "openclaw/plugin-sdk/agent-harness-runtime";
12+
import { loadSessionStore } from "openclaw/plugin-sdk/session-store-runtime";
1213
import { resolveCodexAppServerHomeDir } from "./auth-bridge.js";
1314
import { isJsonObject, type JsonValue } from "./protocol.js";
1415
import { clearCodexAppServerBinding, type CodexAppServerThreadBinding } from "./session-binding.js";
@@ -34,15 +35,6 @@ const CODEX_APP_SERVER_BYTE_UNITS: Record<string, number> = {
3435
tb: 1024 * 1024 * 1024 * 1024,
3536
tib: 1024 * 1024 * 1024 * 1024,
3637
};
37-
type CodexSessionRecordCacheEntry = {
38-
sessionsFile: string;
39-
mtimeMs: number;
40-
size: number;
41-
record: (Record<string, unknown> & { sessionKey: string }) | undefined;
42-
};
43-
44-
const codexSessionRecordCache = new Map<string, CodexSessionRecordCacheEntry>();
45-
4638
function parseCodexAppServerByteLimit(value: unknown): number | undefined {
4739
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
4840
return Math.floor(value);
@@ -123,52 +115,24 @@ async function listCodexAppServerRolloutFilesForThread(
123115
async function readCodexSessionRecordForSessionFile(
124116
sessionFile: string,
125117
): Promise<(Record<string, unknown> & { sessionKey: string }) | undefined> {
126-
const sessionsFile = path.join(path.dirname(sessionFile), "sessions.json");
118+
const storePath = path.join(path.dirname(sessionFile), "sessions.json");
127119
const resolvedSessionFile = path.resolve(sessionFile);
128-
let stat: Awaited<ReturnType<typeof fs.stat>>;
120+
let store: Record<string, unknown>;
129121
try {
130-
stat = await fs.stat(sessionsFile);
122+
store = loadSessionStore(storePath, { skipCache: true }) as Record<string, unknown>;
131123
} catch {
132-
codexSessionRecordCache.delete(resolvedSessionFile);
133124
return undefined;
134125
}
135-
const cached = codexSessionRecordCache.get(resolvedSessionFile);
136-
if (
137-
cached?.sessionsFile === sessionsFile &&
138-
cached.mtimeMs === stat.mtimeMs &&
139-
cached.size === stat.size
140-
) {
141-
return cached.record;
142-
}
143-
let store: JsonValue | undefined;
144-
try {
145-
store = JSON.parse(await fs.readFile(sessionsFile, "utf8")) as JsonValue;
146-
} catch {
147-
codexSessionRecordCache.delete(resolvedSessionFile);
148-
return undefined;
149-
}
150-
if (!isJsonObject(store)) {
151-
codexSessionRecordCache.delete(resolvedSessionFile);
152-
return undefined;
153-
}
154-
let found: (Record<string, unknown> & { sessionKey: string }) | undefined;
155126
for (const [sessionKey, record] of Object.entries(store)) {
156127
if (!isJsonObject(record) || typeof record.sessionFile !== "string") {
157128
continue;
158129
}
159130
if (path.resolve(record.sessionFile) !== resolvedSessionFile) {
160131
continue;
161132
}
162-
found = { sessionKey, ...record };
163-
break;
133+
return { sessionKey, ...record };
164134
}
165-
codexSessionRecordCache.set(resolvedSessionFile, {
166-
sessionsFile,
167-
mtimeMs: stat.mtimeMs,
168-
size: stat.size,
169-
record: found,
170-
});
171-
return found;
135+
return undefined;
172136
}
173137

174138
type CodexAppServerRolloutTokenSnapshot = {

‎extensions/feishu/src/doctor.test.ts‎

Lines changed: 51 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,11 @@
22
import fs from "node:fs";
33
import os from "node:os";
44
import path from "node:path";
5-
import { loadSessionStore } from "openclaw/plugin-sdk/session-store-runtime";
5+
import {
6+
loadSessionStore,
7+
saveSessionStore,
8+
type SessionEntry,
9+
} from "openclaw/plugin-sdk/session-store-runtime";
610
import { afterEach, beforeEach, describe, expect, it } from "vitest";
711
import type { OpenClawConfig } from "../runtime-api.js";
812
import { isFeishuSessionStoreKey, runFeishuDoctorSequence } from "./doctor.js";
@@ -59,10 +63,13 @@ function storePath(agentId = "main"): string {
5963
return path.join(sessionsDir(agentId), "sessions.json");
6064
}
6165

62-
function writeStore(entries: Record<string, unknown>, agentId = "main"): string {
66+
async function writeStore(
67+
entries: Record<string, SessionEntry>,
68+
agentId = "main",
69+
): Promise<string> {
6370
const target = storePath(agentId);
6471
fs.mkdirSync(path.dirname(target), { recursive: true });
65-
fs.writeFileSync(target, JSON.stringify(entries, null, 2));
72+
await saveSessionStore(target, entries, { skipMaintenance: true });
6673
return target;
6774
}
6875

@@ -131,7 +138,7 @@ describe("Feishu doctor state repair", () => {
131138
fs.writeFileSync(path.join(feishuDedupDir, "default.json"), JSON.stringify({ msg1: 1 }));
132139

133140
writeTranscript("sess-ok", [sessionHeader("sess-ok"), userMessage("hello")]);
134-
writeStore({
141+
await writeStore({
135142
"agent:main:feishu:direct:ou_user": {
136143
sessionId: "sess-ok",
137144
sessionFile: "sess-ok.jsonl",
@@ -155,15 +162,16 @@ describe("Feishu doctor state repair", () => {
155162
]);
156163
const customStorePath = path.join(stateDir(), "custom-sessions", "sessions.json");
157164
fs.mkdirSync(path.dirname(customStorePath), { recursive: true });
158-
fs.writeFileSync(
165+
await saveSessionStore(
159166
customStorePath,
160-
JSON.stringify({
167+
{
161168
"agent:main:feishu:direct:ou_user": {
162169
sessionId: "sess-abs",
163170
sessionFile: transcriptPath,
164171
updatedAt: Date.now(),
165172
},
166-
}),
173+
},
174+
{ skipMaintenance: true },
167175
);
168176

169177
const result = await runFeishuDoctorSequence({
@@ -187,7 +195,7 @@ describe("Feishu doctor state repair", () => {
187195
userMessage("world"),
188196
userMessage(""),
189197
]);
190-
writeStore({
198+
await writeStore({
191199
"agent:main:feishu:direct:ou_user": {
192200
sessionId: "sess-separated-blanks",
193201
sessionFile: "sess-separated-blanks.jsonl",
@@ -230,7 +238,7 @@ describe("Feishu doctor state repair", () => {
230238
sessionHeader("sess-ok"),
231239
userMessage("hello"),
232240
]);
233-
const targetStorePath = writeStore({
241+
const targetStorePath = await writeStore({
234242
"agent:main:feishu:direct:ou_user": {
235243
sessionId: "sess-ok",
236244
sessionFile: "sess-ok.jsonl",
@@ -286,7 +294,7 @@ describe("Feishu doctor state repair", () => {
286294
userMessage(""),
287295
]);
288296

289-
const targetStorePath = writeStore({
297+
const targetStorePath = await writeStore({
290298
"agent:main:feishu:direct:ou_user": {
291299
sessionId: "sess-bad",
292300
sessionFile: "sess-bad.jsonl",
@@ -345,14 +353,46 @@ describe("Feishu doctor state repair", () => {
345353
).toBe(true);
346354
});
347355

356+
it("archives unhealthy Feishu sessions from SQLite-only retired agent stores", async () => {
357+
const retiredAgent = "retired";
358+
const transcriptPath = writeTranscript(
359+
"sess-retired-bad",
360+
[sessionHeader("sess-retired-bad"), userMessage(""), userMessage(""), userMessage("")],
361+
retiredAgent,
362+
);
363+
const targetStorePath = storePath(retiredAgent);
364+
const entries: Record<string, SessionEntry> = {
365+
"agent:retired:feishu:direct:ou_user": {
366+
sessionId: "sess-retired-bad",
367+
sessionFile: "sess-retired-bad.jsonl",
368+
updatedAt: Date.now(),
369+
},
370+
};
371+
await saveSessionStore(targetStorePath, entries, { skipMaintenance: true });
372+
expect(fs.existsSync(targetStorePath)).toBe(false);
373+
374+
const result = await runFeishuDoctorSequence({
375+
cfg: feishuConfig(),
376+
env: process.env,
377+
shouldRepair: true,
378+
});
379+
380+
expect(result.warningNotes).toEqual([]);
381+
expect(result.changeNotes.join("\n")).toContain("Removed 1 Feishu-scoped session entry");
382+
383+
const store = loadSessionStore(targetStorePath, { skipCache: true });
384+
expect(store["agent:retired:feishu:direct:ou_user"]).toBeUndefined();
385+
expect(fs.existsSync(transcriptPath)).toBe(false);
386+
});
387+
348388
it("archives unhealthy default-scope sessions when metadata identifies Feishu", async () => {
349389
const transcriptPath = writeTranscript("sess-default-feishu-bad", [
350390
sessionHeader("sess-default-feishu-bad"),
351391
userMessage(""),
352392
userMessage(""),
353393
userMessage(""),
354394
]);
355-
const targetStorePath = writeStore({
395+
const targetStorePath = await writeStore({
356396
"agent:main:main": {
357397
sessionId: "sess-default-feishu-bad",
358398
sessionFile: "sess-default-feishu-bad.jsonl",

0 commit comments

Comments
 (0)