fix(memory-core): use CJK-aware tokenizer for dreaming dedupe (#80613)

clawsweeper[bot] · clawsweeper[bot] · commit ca9c02734c53 · 2026-05-25T21:35:18.000Z
diff --git a/extensions/memory-core/src/memory/mmr.test.ts b/extensions/memory-core/src/memory/mmr.test.ts
@@ -172,8 +172,7 @@ describe("textSimilarity", () => {
     expect(textSimilarity("Привет мир", "Привет мир")).toBe(1);
     // Empty vs empty — preserve identity (both literally equal).
     expect(textSimilarity("", "")).toBe(1);
-    // Whitespace-only normalization is case-folded only, not trimmed — keep
-    // distinct strings distinct.
+    // Normalized equality is case-insensitive for non-tokenized text.
     expect(textSimilarity("Привет МИР", "привет мир")).toBe(1);
   });
 });
