Better normalization cache

They key seems to be too specific. Specially by using the prop, which
basically makes it redudant to cache tokens that are found in different
props. The goal of that cache seems to be to trade memory for time, but
right now seems to be storing equal computations in different keys which
basically is inefficient. The only thing that the prop is needed for is
the `stemmerSkipProperties`.

Author: masylum

packages/orama/src/components/index.ts

@@ -156,11 +156,9 @@ export async function insertTokenScoreParameters(
 export async function removeDocumentScoreParameters(
   index: Index,
   prop: string,
-  id: DocumentID,
+  internalId: DocumentID,
   docsCount: number
 ): Promise<void> {
-  const internalId = getInternalDocumentId(index.sharedInternalDocumentStore, id)
-
   index.avgFieldLength[prop] =
     (index.avgFieldLength[prop] * docsCount - index.fieldLengths[prop][internalId]!) / (docsCount - 1)
   index.fieldLengths[prop][internalId] = undefined

packages/orama/src/components/tokenizer/index.ts

@@ -16,15 +16,13 @@ export interface DefaultTokenizer extends Tokenizer {
 }
 
 export function normalizeToken(this: DefaultTokenizer, prop: string, token: string): string {
-  const key = `${this.language}:${prop}:${token}`
-
-  if (this.normalizationCache.has(key)) {
-    return this.normalizationCache.get(key)!
+  if (this.normalizationCache.has(token)) {
+    return this.normalizationCache.get(token)!
   }
 
   // Remove stopwords if enabled
-  if (this.stopWords?.includes(token)) {
-    this.normalizationCache.set(key, '')
+  if (this.stopWords?.has(token)) {
+    this.normalizationCache.set(token, '')
     return ''
   }
 
@@ -34,7 +32,7 @@ export function normalizeToken(this: DefaultTokenizer, prop: string, token: stri
   }
 
   token = replaceDiacritics(token)
-  this.normalizationCache.set(key, token)
+  this.normalizationCache.set(token, token)
   return token
 }
 

packages/orama/tests/search.test.ts

@@ -725,7 +725,7 @@ t.test('search method', (t) => {
   t.test('with custom tokenizer', async (t) => {
     t.plan(4)
 
-    const normalizationCache = new Map([['english:foo:dogs', 'Dogs']])
+    const normalizationCache = new Map([['dogs', 'Dogs']])
 
     const db = await create({
       schema: {