feat(lyra): removes diacritics to keep index smaller

implements #75
oramasearch · Aug 4, 2022 · e8396c3 · e8396c3
1 parent 929a6cd
commit e8396c3
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 11 deletions.
diff --git a/packages/lyra/src/tokenizer/diacritics.ts b/packages/lyra/src/tokenizer/diacritics.ts
@@ -0,0 +1,23 @@
+const diacritics = [
+  { char: "A", base: /[\300-\306]/g },
+  { char: "a", base: /[\340-\346]/g },
+  { char: "E", base: /[\310-\313]/g },
+  { char: "e", base: /[\350-\353]/g },
+  { char: "I", base: /[\314-\317]/g },
+  { char: "i", base: /[\354-\357]/g },
+  { char: "O", base: /[\322-\330]/g },
+  { char: "o", base: /[\362-\370]/g },
+  { char: "U", base: /[\331-\334]/g },
+  { char: "u", base: /[\371-\374]/g },
+  { char: "N", base: /[\321]/g },
+  { char: "n", base: /[\361]/g },
+  { char: "C", base: /[\307]/g },
+  { char: "c", base: /[\347]/g },
+];
+
+export function replaceDiacritics(str: string): string {
+  for (const { char, base } of diacritics) {
+    str = str.replace(base, char);
+  }
+  return str;
+}
diff --git a/packages/lyra/src/tokenizer/index.ts b/packages/lyra/src/tokenizer/index.ts
@@ -1,4 +1,5 @@
 import { Language } from "./languages";
+import { replaceDiacritics } from "./diacritics";
 
 const splitRegex: Record<Language, RegExp> = {
   dutch: /[^a-z0-9_'-]+/gim,
@@ -16,7 +17,7 @@ export function tokenize(input: string, language: Language = "english") {
   if (typeof input !== "string") return [input];
 
   const splitRule = splitRegex[language];
-  const tokens = input.toLowerCase().split(splitRule);
+  const tokens = input.toLowerCase().split(splitRule).map(replaceDiacritics);
   return Array.from(new Set(trim(tokens)));
 }
 

diff --git a/packages/lyra/tap-snapshots/tests/tokenizer.test.ts.test.cjs b/packages/lyra/tap-snapshots/tests/tokenizer.test.ts.test.cjs
@@ -62,7 +62,7 @@ Array [
   "ai",
   "fait",
   "des",
-  "gâteaux",
+  "gateaux",
 ]
 `
 
@@ -100,13 +100,13 @@ Array [
 
 exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctly in norwegian > Should tokenize and stem correctly in norwegian-O2 1`] = `
 Array [
-  "å",
+  "a",
   "sove",
   "er",
   "en",
   "vanskelig",
   "ting",
-  "når",
+  "nar",
   "testene",
   "mislykkes",
 ]
@@ -124,10 +124,10 @@ Array [
 exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctly in portuguese > Should tokenize and stem correctly in portuguese-O2 1`] = `
 Array [
   "dormir",
-  "é",
+  "e",
   "uma",
   "coisa",
-  "difícil",
+  "dificil",
   "quando",
   "os",
   "testes",
@@ -156,7 +156,7 @@ Array [
 
 exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctly in spanish > Should tokenize and stem correctly in spanish-O1 1`] = `
 Array [
-  "cociné",
+  "cocine",
   "unos",
   "pasteles",
 ]
@@ -179,7 +179,7 @@ exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctl
 Array [
   "jag",
   "lagade",
-  "några",
+  "nagra",
   "kakor",
 ]
 `
@@ -188,11 +188,11 @@ exports[`tests/tokenizer.test.ts TAP Tokenizer Should tokenize and stem correctl
 Array [
   "att",
   "sova",
-  "är",
+  "ar",
   "en",
-  "svår",
+  "svar",
   "sak",
-  "när",
+  "nar",
   "testerna",
   "misslyckas",
 ]