I tried what you say and you are right. I extended it to all UTF-16 characters (of course it also contains non-printable characters).

I insert some code snippets that I made for testing purposes.

Code snippets

Okydk's case

import { create, insert, search } from "@orama/orama";
import { stemmer } from '@orama/stemmers/danish'

const schema = { title: "string" };

const movieDB = await create({ 
  schema,
  components: {
    tokenizer: {
      stemming: true,
      stemmer
    },
  },
});

const charactersToTry = ["foo", "æ", "ø", "å", "😄"];

const results = [];
for (const term of charactersToTry) {
  await insert(movieDB, { title: term });
  const count = (await search(movieDB, { term })).count;
  results.push({ term, count });
}

console.table(results);

Result

┌─────────┬───────┬───────┐
│ (index) │ term  │ count │
├─────────┼───────┼───────┤
│    0    │ 'foo' │   1   │
│    1    │  'æ'  │   0   │
│    2    │  'ø'  │   0   │
│    3    │  'å'  │   0   │
│    4    │ '😄'  │   0   │
└─────────┴───────┴───────┘

Generic

import { create, insert, search } from "@orama/orama";
import { stemmer } from '@orama/stemmers/danish'

const totalCharactersCount = 1112064;

const schema = { title: "string" };
const movieDB = await create({ 
  schema,
  components: {
    tokenizer: {
      stemming: true,
      stemmer
    },
  },
});

const charactersToTry = Array.from({ length: totalCharactersCount }, (_, i) =>
  String.fromCharCode(i),
);

const results = [];
for (const term of charactersToTry) {
  await insert(movieDB, { title: term });
  const count = (await search(movieDB, { term })).count;
  results.push({ term, count });
}

const missingCharacters = results.filter((r) => r.count === 0);

console.log(`Missing characters of UTF-16: ${missingCharacters.length}`);
console.log(
  `Available characters of UTF-16: ${
    totalCharactersCount - missingCharacters.length
  }`,
);

// Not available chars
// console.table(results.filter((r) => r.count === 0));

// Available chars
// console.table(results.filter((r) => r.count > 0));

Result

Missing characters of UTF-16: 1110687
Available characters of UTF-16: 1377

(the same for English language)

Uh oh!

Indexing of special characters #510

Description

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions