Skip to content

Commit dc3df62

Browse files
steipetesteipete
committed
refactor(memory-host): own package contract surface
1 parent 6fadc56 commit dc3df62

56 files changed

Lines changed: 1111 additions & 602 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎packages/memory-host-sdk/src/engine-embeddings.ts‎

Lines changed: 73 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,73 @@
1-
export * from "../../../src/memory-host-sdk/engine-embeddings.js";
1+
// Real workspace contract for memory embedding providers and batch helpers.
2+
3+
export {
4+
getMemoryEmbeddingProvider,
5+
listRegisteredMemoryEmbeddingProviders,
6+
listMemoryEmbeddingProviders,
7+
listRegisteredMemoryEmbeddingProviderAdapters,
8+
} from "../../../src/plugins/memory-embedding-provider-runtime.js";
9+
export type {
10+
MemoryEmbeddingBatchChunk,
11+
MemoryEmbeddingBatchOptions,
12+
MemoryEmbeddingProvider,
13+
MemoryEmbeddingProviderAdapter,
14+
MemoryEmbeddingProviderCreateOptions,
15+
MemoryEmbeddingProviderCreateResult,
16+
MemoryEmbeddingProviderRuntime,
17+
} from "../../../src/plugins/memory-embedding-providers.js";
18+
export { createLocalEmbeddingProvider, DEFAULT_LOCAL_MODEL } from "./host/embeddings.js";
19+
export { extractBatchErrorMessage, formatUnavailableBatchError } from "./host/batch-error-utils.js";
20+
export { postJsonWithRetry } from "./host/batch-http.js";
21+
export { applyEmbeddingBatchOutputLine } from "./host/batch-output.js";
22+
export {
23+
EMBEDDING_BATCH_ENDPOINT,
24+
type EmbeddingBatchStatus,
25+
type ProviderBatchOutputLine,
26+
} from "./host/batch-provider-common.js";
27+
export {
28+
buildEmbeddingBatchGroupOptions,
29+
runEmbeddingBatchGroups,
30+
type EmbeddingBatchExecutionParams,
31+
} from "./host/batch-runner.js";
32+
export {
33+
resolveBatchCompletionFromStatus,
34+
resolveCompletedBatchResult,
35+
throwIfBatchTerminalFailure,
36+
type BatchCompletionResult,
37+
} from "./host/batch-status.js";
38+
export { uploadBatchJsonlFile } from "./host/batch-upload.js";
39+
export {
40+
buildBatchHeaders,
41+
normalizeBatchBaseUrl,
42+
type BatchHttpClientConfig,
43+
} from "./host/batch-utils.js";
44+
export { enforceEmbeddingMaxInputTokens } from "./host/embedding-chunk-limits.js";
45+
export {
46+
isMissingEmbeddingApiKeyError,
47+
mapBatchEmbeddingsByIndex,
48+
sanitizeEmbeddingCacheHeaders,
49+
} from "./host/embedding-provider-adapter-utils.js";
50+
export { sanitizeAndNormalizeEmbedding } from "./host/embedding-vectors.js";
51+
export { debugEmbeddingsLog } from "./host/embeddings-debug.js";
52+
export { normalizeEmbeddingModelWithPrefixes } from "./host/embeddings-model-normalize.js";
53+
export {
54+
resolveRemoteEmbeddingBearerClient,
55+
type RemoteEmbeddingProviderId,
56+
} from "./host/embeddings-remote-client.js";
57+
export {
58+
createRemoteEmbeddingProvider,
59+
resolveRemoteEmbeddingClient,
60+
type RemoteEmbeddingClient,
61+
} from "./host/embeddings-remote-provider.js";
62+
export { fetchRemoteEmbeddingVectors } from "./host/embeddings-remote-fetch.js";
63+
export {
64+
estimateStructuredEmbeddingInputBytes,
65+
estimateUtf8Bytes,
66+
} from "./host/embedding-input-limits.js";
67+
export { hasNonTextEmbeddingParts, type EmbeddingInput } from "./host/embedding-inputs.js";
68+
export { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./host/remote-http.js";
69+
export {
70+
buildCaseInsensitiveExtensionGlob,
71+
classifyMemoryMultimodalPath,
72+
getMemoryMultimodalExtensions,
73+
} from "./host/multimodal.js";

‎packages/memory-host-sdk/src/engine-foundation.ts‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,48 @@
1-
export * from "../../../src/memory-host-sdk/engine-foundation.js";
1+
// Real workspace contract for memory engine foundation concerns.
2+
3+
export {
4+
resolveAgentContextLimits,
5+
resolveAgentDir,
6+
resolveAgentWorkspaceDir,
7+
resolveDefaultAgentId,
8+
resolveSessionAgentId,
9+
} from "../../../src/agents/agent-scope.js";
10+
export {
11+
resolveMemorySearchConfig,
12+
resolveMemorySearchSyncConfig,
13+
type ResolvedMemorySearchConfig,
14+
type ResolvedMemorySearchSyncConfig,
15+
} from "../../../src/agents/memory-search.js";
16+
export { parseDurationMs } from "../../../src/cli/parse-duration.js";
17+
export { loadConfig } from "../../../src/config/config.js";
18+
export { resolveStateDir } from "../../../src/config/paths.js";
19+
export { resolveSessionTranscriptsDirForAgent } from "../../../src/config/sessions/paths.js";
20+
export {
21+
hasConfiguredSecretInput,
22+
normalizeResolvedSecretInputString,
23+
} from "../../../src/config/types.secrets.js";
24+
export { writeFileWithinRoot } from "../../../src/infra/fs-safe.js";
25+
export { createSubsystemLogger } from "../../../src/logging/subsystem.js";
26+
export { detectMime } from "../../../src/media/mime.js";
27+
export { resolveGlobalSingleton } from "../../../src/shared/global-singleton.js";
28+
export { onSessionTranscriptUpdate } from "../../../src/sessions/transcript-events.js";
29+
export { splitShellArgs } from "../../../src/utils/shell-argv.js";
30+
export { runTasksWithConcurrency } from "../../../src/utils/run-with-concurrency.js";
31+
export {
32+
shortenHomeInString,
33+
shortenHomePath,
34+
resolveUserPath,
35+
truncateUtf16Safe,
36+
} from "../../../src/utils.js";
37+
export type { OpenClawConfig } from "../../../src/config/config.js";
38+
export type { SessionSendPolicyConfig } from "../../../src/config/types.base.js";
39+
export type { SecretInput } from "../../../src/config/types.secrets.js";
40+
export type {
41+
MemoryBackend,
42+
MemoryCitationsMode,
43+
MemoryQmdConfig,
44+
MemoryQmdIndexPath,
45+
MemoryQmdMcporterConfig,
46+
MemoryQmdSearchMode,
47+
} from "../../../src/config/types.memory.js";
48+
export type { MemorySearchConfig } from "../../../src/config/types.tools.js";

‎packages/memory-host-sdk/src/engine-qmd.ts‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,26 @@
1-
export * from "../../../src/memory-host-sdk/engine-qmd.js";
1+
// Real workspace contract for QMD/session/query helpers used by the memory engine.
2+
3+
export { extractKeywords, isQueryStopWordToken } from "./host/query-expansion.js";
4+
export {
5+
buildSessionEntry,
6+
listSessionFilesForAgent,
7+
loadDreamingNarrativeTranscriptPathSetForAgent,
8+
loadSessionTranscriptClassificationForAgent,
9+
normalizeSessionTranscriptPathForComparison,
10+
sessionPathForFile,
11+
type BuildSessionEntryOptions,
12+
type SessionFileEntry,
13+
type SessionTranscriptClassification,
14+
} from "./host/session-files.js";
15+
export { parseUsageCountedSessionIdFromFileName } from "../../../src/config/sessions/artifacts.js";
16+
export { parseQmdQueryJson, type QmdQueryResult } from "./host/qmd-query-parser.js";
17+
export {
18+
deriveQmdScopeChannel,
19+
deriveQmdScopeChatType,
20+
isQmdScopeAllowed,
21+
} from "./host/qmd-scope.js";
22+
export {
23+
checkQmdBinaryAvailability,
24+
resolveCliSpawnInvocation,
25+
runCliCommand,
26+
} from "./host/qmd-process.js";

‎packages/memory-host-sdk/src/engine-storage.ts‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,48 @@
1-
export * from "../../../src/memory-host-sdk/engine-storage.js";
1+
// Real workspace contract for memory engine storage/index helpers.
2+
3+
export {
4+
buildFileEntry,
5+
buildMultimodalChunkForIndexing,
6+
chunkMarkdown,
7+
cosineSimilarity,
8+
ensureDir,
9+
hashText,
10+
listMemoryFiles,
11+
normalizeExtraMemoryPaths,
12+
parseEmbedding,
13+
remapChunkLines,
14+
runWithConcurrency,
15+
type MemoryChunk,
16+
type MemoryFileEntry,
17+
} from "./host/internal.js";
18+
export { readMemoryFile } from "./host/read-file.js";
19+
export {
20+
buildMemoryReadResult,
21+
buildMemoryReadResultFromSlice,
22+
DEFAULT_MEMORY_READ_LINES,
23+
DEFAULT_MEMORY_READ_MAX_CHARS,
24+
type MemoryReadResult,
25+
} from "./host/read-file-shared.js";
26+
export { resolveMemoryBackendConfig } from "./host/backend-config.js";
27+
export type {
28+
ResolvedMemoryBackendConfig,
29+
ResolvedQmdConfig,
30+
ResolvedQmdMcporterConfig,
31+
} from "./host/backend-config.js";
32+
export type {
33+
MemoryEmbeddingProbeResult,
34+
MemoryProviderStatus,
35+
MemorySearchManager,
36+
MemorySearchRuntimeDebug,
37+
MemorySearchResult,
38+
MemorySource,
39+
MemorySyncProgressUpdate,
40+
} from "./host/types.js";
41+
export { ensureMemoryIndexSchema } from "./host/memory-schema.js";
42+
export { loadSqliteVecExtension } from "./host/sqlite-vec.js";
43+
export {
44+
closeMemorySqliteWalMaintenance,
45+
configureMemorySqliteWalMaintenance,
46+
requireNodeSqlite,
47+
} from "./host/sqlite.js";
48+
export { isFileMissingError, statRegularFile } from "./host/fs-utils.js";

‎packages/memory-host-sdk/src/engine.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,7 @@
1-
export * from "../../../src/memory-host-sdk/engine.js";
1+
// Aggregate workspace contract for the memory engine surface.
2+
// Keep focused subpaths preferred for new code.
3+
4+
export * from "./engine-foundation.js";
5+
export * from "./engine-storage.js";
6+
export * from "./engine-embeddings.js";
7+
export * from "./engine-qmd.js";

‎packages/memory-host-sdk/src/host/backend-config.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ import type {
1414
} from "../../../../src/config/types.memory.js";
1515
import { CANONICAL_ROOT_MEMORY_FILENAME } from "../../../../src/memory/root-memory-files.js";
1616
import { normalizeAgentId } from "../../../../src/routing/session-key.js";
17-
import { normalizeLowercaseStringOrEmpty } from "../../../../src/shared/string-coerce.js";
1817
import { resolveUserPath } from "../../../../src/utils.js";
1918
import { splitShellArgs } from "../../../../src/utils/shell-argv.js";
19+
import { normalizeLowercaseStringOrEmpty } from "./string-utils.js";
2020

2121
export type ResolvedMemoryBackendConfig = {
2222
backend: MemoryBackend;

‎packages/memory-host-sdk/src/host/batch-error-utils.ts‎

Lines changed: 33 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,33 @@
1-
export * from "../../../../src/memory-host-sdk/host/batch-error-utils.js";
1+
import { formatErrorMessage } from "../../../../src/infra/errors.js";
2+
3+
type BatchOutputErrorLike = {
4+
error?: { message?: string };
5+
response?: {
6+
body?:
7+
| string
8+
| {
9+
error?: { message?: string };
10+
};
11+
};
12+
};
13+
14+
function getResponseErrorMessage(line: BatchOutputErrorLike | undefined): string | undefined {
15+
const body = line?.response?.body;
16+
if (typeof body === "string") {
17+
return body || undefined;
18+
}
19+
if (!body || typeof body !== "object") {
20+
return undefined;
21+
}
22+
return typeof body.error?.message === "string" ? body.error.message : undefined;
23+
}
24+
25+
export function extractBatchErrorMessage(lines: BatchOutputErrorLike[]): string | undefined {
26+
const first = lines.find((line) => line.error?.message || getResponseErrorMessage(line));
27+
return first?.error?.message ?? getResponseErrorMessage(first);
28+
}
29+
30+
export function formatUnavailableBatchError(err: unknown): string | undefined {
31+
const message = formatErrorMessage(err);
32+
return message ? `error file unavailable: ${message}` : undefined;
33+
}

‎packages/memory-host-sdk/src/host/batch-http.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export async function postJsonWithRetry<T>(params: {
66
url: string;
77
headers: Record<string, string>;
88
ssrfPolicy?: SsrFPolicy;
9+
fetchImpl?: typeof fetch;
910
body: unknown;
1011
errorPrefix: string;
1112
}): Promise<T> {
@@ -15,6 +16,7 @@ export async function postJsonWithRetry<T>(params: {
1516
url: params.url,
1617
headers: params.headers,
1718
ssrfPolicy: params.ssrfPolicy,
19+
fetchImpl: params.fetchImpl,
1820
body: params.body,
1921
errorPrefix: params.errorPrefix,
2022
attachStatus: true,

‎packages/memory-host-sdk/src/host/embedding-defaults.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
export const DEFAULT_LOCAL_MODEL =
2+
"hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf";

‎packages/memory-host-sdk/src/host/embedding-provider-adapter-utils.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { normalizeLowercaseStringOrEmpty } from "./string-utils.js";
2+
3+
export function isMissingEmbeddingApiKeyError(err: unknown): boolean {
4+
return err instanceof Error && err.message.includes("No API key found for provider");
5+
}
6+
7+
export function sanitizeEmbeddingCacheHeaders(
8+
headers: Record<string, string>,
9+
excludedHeaderNames: string[],
10+
): Array<[string, string]> {
11+
const excluded = new Set(
12+
excludedHeaderNames.map((name) => normalizeLowercaseStringOrEmpty(name)),
13+
);
14+
return Object.entries(headers)
15+
.filter(([key]) => !excluded.has(normalizeLowercaseStringOrEmpty(key)))
16+
.toSorted(([a], [b]) => a.localeCompare(b))
17+
.map(([key, value]) => [key, value]);
18+
}
19+
20+
export function mapBatchEmbeddingsByIndex(
21+
byCustomId: Map<string, number[]>,
22+
count: number,
23+
): number[][] {
24+
const embeddings: number[][] = [];
25+
for (let index = 0; index < count; index += 1) {
26+
embeddings.push(byCustomId.get(String(index)) ?? []);
27+
}
28+
return embeddings;
29+
}

0 commit comments

Comments
 (0)