Skip to content

Commit 2a1e1a1

Browse files
steipetesteipete
committed
refactor(memory-wiki): store import runs in sqlite
1 parent ea3a915 commit 2a1e1a1

8 files changed

Lines changed: 844 additions & 102 deletions

File tree

‎extensions/memory-wiki/doctor-contract-api.test.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,11 @@ import {
99
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
1010
import { afterEach, beforeEach, describe, expect, it } from "vitest";
1111
import { stateMigrations } from "./doctor-contract-api.js";
12+
import {
13+
createMemoryWikiImportRunStateStore,
14+
readMemoryWikiImportRunRecord,
15+
resolveMemoryWikiImportRunRecordPath,
16+
} from "./src/import-runs-state.js";
1217
import {
1318
createMemoryWikiSourceSyncStateStore,
1419
readMemoryWikiSourceSyncState,
@@ -112,6 +117,78 @@ describe("memory-wiki doctor source sync migration", () => {
112117
await expect(fs.stat(`${legacyPath}.migrated`)).resolves.toBeDefined();
113118
});
114119

120+
it("detects and migrates legacy import-run records into plugin state", async () => {
121+
const stateDir = await makeTempDir();
122+
const vaultRoot = path.join(stateDir, "vault");
123+
const legacyPath = resolveMemoryWikiImportRunRecordPath(vaultRoot, "chatgpt-alpha");
124+
const snapshotPath = path.join(
125+
vaultRoot,
126+
".openclaw-wiki",
127+
"import-runs",
128+
"chatgpt-alpha",
129+
"snapshots",
130+
"alpha.md",
131+
);
132+
await fs.mkdir(path.dirname(legacyPath), { recursive: true });
133+
await fs.mkdir(path.dirname(snapshotPath), { recursive: true });
134+
await fs.writeFile(snapshotPath, "previous page\n", "utf8");
135+
await fs.writeFile(
136+
legacyPath,
137+
`${JSON.stringify({
138+
version: 1,
139+
runId: "chatgpt-alpha",
140+
importType: "chatgpt",
141+
exportPath: "/tmp/chatgpt",
142+
sourcePath: "/tmp/chatgpt/conversations.json",
143+
appliedAt: "2026-04-10T10:00:00.000Z",
144+
conversationCount: 2,
145+
createdCount: 1,
146+
updatedCount: 1,
147+
skippedCount: 0,
148+
createdPaths: ["sources/new.md"],
149+
updatedPaths: [{ path: "sources/existing.md", snapshotPath: "snapshots/alpha.md" }],
150+
})}\n`,
151+
);
152+
const params = migrationParams({ stateDir, vaultRoot });
153+
const migration = stateMigrations.find(
154+
(entry) => entry.id === "memory-wiki-import-runs-json-to-plugin-state",
155+
);
156+
if (!migration) {
157+
throw new Error("Expected import-run migration");
158+
}
159+
160+
await expect(migration.detectLegacyState(params)).resolves.toEqual({
161+
preview: [expect.stringContaining("Memory Wiki import runs:")],
162+
});
163+
await expect(migration.migrateLegacyState(params)).resolves.toEqual({
164+
changes: [
165+
"Migrated Memory Wiki import runs -> plugin state (1 imported, 0 existing)",
166+
expect.stringContaining("Archived Memory Wiki import-run legacy record ->"),
167+
],
168+
warnings: [],
169+
});
170+
const store = createMemoryWikiImportRunStateStore(params.context.openPluginStateKeyedStore);
171+
await expect(readMemoryWikiImportRunRecord(vaultRoot, "chatgpt-alpha", store)).resolves.toEqual(
172+
{
173+
version: 1,
174+
runId: "chatgpt-alpha",
175+
importType: "chatgpt",
176+
exportPath: "/tmp/chatgpt",
177+
sourcePath: "/tmp/chatgpt/conversations.json",
178+
appliedAt: "2026-04-10T10:00:00.000Z",
179+
conversationCount: 2,
180+
createdCount: 1,
181+
updatedCount: 1,
182+
skippedCount: 0,
183+
createdPaths: ["sources/new.md"],
184+
updatedPaths: [{ path: "sources/existing.md", snapshotPath: "snapshots/alpha.md" }],
185+
},
186+
);
187+
await expect(fs.stat(legacyPath)).rejects.toMatchObject({ code: "ENOENT" });
188+
await expect(fs.stat(`${legacyPath}.migrated`)).resolves.toBeDefined();
189+
await expect(fs.readFile(snapshotPath, "utf8")).resolves.toBe("previous page\n");
190+
});
191+
115192
it("merges legacy entries with existing plugin state before archiving", async () => {
116193
const stateDir = await makeTempDir();
117194
const vaultRoot = path.join(stateDir, "vault");

‎extensions/memory-wiki/doctor-contract-api.ts‎

Lines changed: 113 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,20 @@
11
// Memory Wiki doctor contract migrates shipped source-sync state.
22
import fs from "node:fs/promises";
3+
import path from "node:path";
34
import type { OpenClawConfig } from "openclaw/plugin-sdk/plugin-entry";
45
import type { PluginDoctorStateMigration } from "openclaw/plugin-sdk/runtime-doctor";
56
import { resolveMemoryWikiConfig, type MemoryWikiPluginConfig } from "./src/config.js";
67
export { legacyConfigRules, normalizeCompatibilityConfig } from "./src/config-compat.js";
8+
import {
9+
countMemoryWikiImportRunStateRows,
10+
createMemoryWikiImportRunStateStore,
11+
listMemoryWikiImportRunRecords,
12+
MEMORY_WIKI_IMPORT_RUN_STATE_MAX_ENTRIES,
13+
MEMORY_WIKI_IMPORT_RUN_STATE_NAMESPACE,
14+
readLegacyMemoryWikiImportRunRecords,
15+
resolveMemoryWikiImportRunsDir,
16+
writeMemoryWikiImportRunRecord,
17+
} from "./src/import-runs-state.js";
718
import {
819
createMemoryWikiSourceSyncStateStore,
920
MEMORY_WIKI_SOURCE_SYNC_STATE_MAX_ENTRIES,
@@ -52,24 +63,61 @@ async function fileExists(filePath: string): Promise<boolean> {
5263

5364
async function archiveLegacySource(params: {
5465
filePath: string;
66+
label: string;
5567
changes: string[];
5668
warnings: string[];
5769
}): Promise<void> {
5870
const archivedPath = `${params.filePath}.migrated`;
5971
if (await fileExists(archivedPath)) {
6072
params.warnings.push(
61-
`Left migrated Memory Wiki source-sync source in place because ${archivedPath} already exists`,
73+
`Left migrated ${params.label} in place because ${archivedPath} already exists`,
6274
);
6375
return;
6476
}
6577
try {
6678
await fs.rename(params.filePath, archivedPath);
67-
params.changes.push(`Archived Memory Wiki source-sync legacy source -> ${archivedPath}`);
79+
params.changes.push(`Archived ${params.label} -> ${archivedPath}`);
6880
} catch (err) {
69-
params.warnings.push(`Failed archiving Memory Wiki source-sync legacy source: ${String(err)}`);
81+
params.warnings.push(`Failed archiving ${params.label}: ${String(err)}`);
7082
}
7183
}
7284

85+
async function archiveLegacyImportRunRecords(params: {
86+
vaultRoot: string;
87+
changes: string[];
88+
warnings: string[];
89+
}): Promise<void> {
90+
const importRunsDir = resolveMemoryWikiImportRunsDir(params.vaultRoot);
91+
const entries = await fs
92+
.readdir(importRunsDir, { withFileTypes: true })
93+
.catch((error: unknown) => {
94+
if (isRecord(error) && error.code === "ENOENT") {
95+
return [];
96+
}
97+
throw error;
98+
});
99+
for (const entry of entries) {
100+
if (!entry.isFile() || !entry.name.endsWith(".json")) {
101+
continue;
102+
}
103+
await archiveLegacySource({
104+
filePath: path.join(importRunsDir, entry.name),
105+
label: "Memory Wiki import-run legacy record",
106+
changes: params.changes,
107+
warnings: params.warnings,
108+
});
109+
}
110+
}
111+
112+
function countImportRunStateRows(
113+
records: Array<{ createdPaths: string[]; updatedPaths: unknown[] }>,
114+
): number {
115+
return records.reduce(
116+
(total, record) => total + 1 + record.createdPaths.length + record.updatedPaths.length,
117+
0,
118+
);
119+
}
120+
73121
export const stateMigrations: PluginDoctorStateMigration[] = [
74122
{
75123
id: "memory-wiki-source-sync-json-to-plugin-state",
@@ -131,7 +179,68 @@ export const stateMigrations: PluginDoctorStateMigration[] = [
131179
changes.push(
132180
`Migrated Memory Wiki source sync -> plugin state (${importedCount} imported, ${existingCount} existing)`,
133181
);
134-
await archiveLegacySource({ filePath, changes, warnings });
182+
await archiveLegacySource({
183+
filePath,
184+
label: "Memory Wiki source-sync legacy source",
185+
changes,
186+
warnings,
187+
});
188+
}
189+
return { changes, warnings };
190+
},
191+
},
192+
{
193+
id: "memory-wiki-import-runs-json-to-plugin-state",
194+
label: "Memory Wiki import run records",
195+
async detectLegacyState(params) {
196+
const previews: string[] = [];
197+
for (const vaultRoot of resolveConfiguredVaultRoots({
198+
config: params.config,
199+
env: params.env,
200+
})) {
201+
const records = await readLegacyMemoryWikiImportRunRecords(vaultRoot);
202+
if (records.length === 0) {
203+
continue;
204+
}
205+
previews.push(
206+
`- Memory Wiki import runs: ${resolveMemoryWikiImportRunsDir(vaultRoot)}/*.json -> plugin state (${MEMORY_WIKI_IMPORT_RUN_STATE_NAMESPACE}, ${records.length} records)`,
207+
);
208+
}
209+
return previews.length > 0 ? { preview: previews } : null;
210+
},
211+
async migrateLegacyState(params) {
212+
const changes: string[] = [];
213+
const warnings: string[] = [];
214+
const store = createMemoryWikiImportRunStateStore(params.context.openPluginStateKeyedStore);
215+
for (const vaultRoot of resolveConfiguredVaultRoots({
216+
config: params.config,
217+
env: params.env,
218+
})) {
219+
const records = await readLegacyMemoryWikiImportRunRecords(vaultRoot);
220+
if (records.length === 0) {
221+
continue;
222+
}
223+
const existingRecords = await listMemoryWikiImportRunRecords(vaultRoot, store);
224+
const existingRunIds = new Set(existingRecords.map((record) => record.runId));
225+
const importedRecords = records.filter((record) => !existingRunIds.has(record.runId));
226+
const nextRowCount =
227+
(await countMemoryWikiImportRunStateRows(store)) +
228+
countImportRunStateRows(importedRecords);
229+
if (nextRowCount > MEMORY_WIKI_IMPORT_RUN_STATE_MAX_ENTRIES) {
230+
warnings.push(
231+
`Skipped Memory Wiki import-run import for ${vaultRoot}: ${nextRowCount} state rows exceeds ${MEMORY_WIKI_IMPORT_RUN_STATE_MAX_ENTRIES}`,
232+
);
233+
continue;
234+
}
235+
let importedCount = 0;
236+
for (const record of importedRecords) {
237+
await writeMemoryWikiImportRunRecord(vaultRoot, record, store);
238+
importedCount += 1;
239+
}
240+
changes.push(
241+
`Migrated Memory Wiki import runs -> plugin state (${importedCount} imported, ${existingRunIds.size} existing)`,
242+
);
243+
await archiveLegacyImportRunRecords({ vaultRoot, changes, warnings });
135244
}
136245
return { changes, warnings };
137246
},

‎extensions/memory-wiki/index.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ import { registerWikiCli } from "./src/cli.js";
44
import { memoryWikiConfigSchema, resolveMemoryWikiConfig } from "./src/config.js";
55
import { createWikiCorpusSupplement } from "./src/corpus-supplement.js";
66
import { registerMemoryWikiGatewayMethods } from "./src/gateway.js";
7+
import {
8+
configureMemoryWikiImportRunStateStore,
9+
createMemoryWikiImportRunStateStore,
10+
} from "./src/import-runs-state.js";
711
import { createWikiPromptSectionBuilder } from "./src/prompt-section.js";
812
import {
913
configureMemoryWikiSourceSyncStateStore,
@@ -27,6 +31,9 @@ export default definePluginEntry({
2731
configureMemoryWikiSourceSyncStateStore(
2832
createMemoryWikiSourceSyncStateStore(api.runtime.state.openKeyedStore),
2933
);
34+
configureMemoryWikiImportRunStateStore(
35+
createMemoryWikiImportRunStateStore(api.runtime.state.openKeyedStore),
36+
);
3037

3138
api.registerMemoryPromptSupplement(createWikiPromptSectionBuilder(config));
3239
api.registerMemoryCorpusSupplement(

0 commit comments

Comments
 (0)