Merge pull request 'Glove embeddings' (#210) from glove-embeddings into develop

Author: strangetom

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
     rev: v5.0.0
     hooks:
       - id: check-added-large-files
+        args: ['--maxkb=5000']
         stages: [pre-commit]
       - id: check-ast
         stages: [pre-commit]
@@ -15,7 +16,7 @@ repos:
       - id: debug-statements
         stages: [pre-commit]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.9.6
+    rev: v0.11.6
     hooks:
       - id: ruff
         args: ["--fix"]

README.md

@@ -48,13 +48,13 @@ The model has the following accuracy on a test data set of 20% of the total data
 
 ```
 Sentence-level results:
-	Accuracy: 94.50%
+	Accuracy: 94.56%
 
 Word-level results:
-	Accuracy 97.78%
-	Precision (micro) 97.76%
-	Recall (micro) 97.78%
-	F1 score (micro) 97.76%
+	Accuracy 97.77%
+	Precision (micro) 97.75%
+	Recall (micro) 97.77%
+	F1 score (micro) 97.75%
 ```
 
 ## Development

ingredient_parser/en/ModelCard.en.md

@@ -124,7 +124,7 @@ The model has the following performance metrics:
 
 | Word level accuracy | Sentence level accuracy |
 | ------------------- | ----------------------- |
-| 97.78 ± 0.18%       | 94.50 ± 0.42%           |
+| 97.77 ± 0.18%       | 94.56 ± 0.44%           |
 
 These metrics were determined by executing 20 training/evaluation cycles and calculating the mean and standard deviation for the two metrics across all cycles. The uncertainty values provided represent the 99.7% confidence bounds (i.e. 3x standard deviation). The uncertainty is due to the randomisation of the selection of training and evaluation data whenever the model is trained.
 

ingredient_parser/en/_embeddings.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python3
+
+import gzip
+from importlib.resources import as_file, files
+from typing import Any
+
+import numpy as np
+
+
+class GloVeModel:
+    def __init__(self, vec_file: str):
+        self.vec_file = vec_file
+        self._load_vectors_from_file(vec_file)
+
+    def __repr__(self) -> str:
+        return f"GloVeModel(vec_file={self.vec_file})"
+
+    def __str__(self) -> str:
+        return f"GloVeModel(vocab_size={self.vocab_size}, dimensions={self.dimension})"
+
+    def __len__(self) -> int:
+        return self.vocab_size
+
+    def __contains__(self, token: str) -> bool:
+        return token in self.vectors
+
+    def __getitem__(self, token: str) -> np.ndarray:
+        return self.vectors[token]
+
+    def get(self, token: str, default: Any) -> Any:
+        """If token in vector keys, return vector, otherwise return default.
+
+        Parameters
+        ----------
+        token : str
+            Token to return vector for.
+        default : Any
+            Default value if token not in vector keys.
+
+        Returns
+        -------
+        Any
+            Vector, or default value.
+        """
+        if token in self.vectors:
+            return self.vectors[token]
+        else:
+            return default
+
+    def _load_vectors_from_file(self, vec_file: str) -> None:
+        """Load vectors from gzipped txt file in word2vec format.
+
+        The first line of the file contains the header which is the vocabulary size
+        (i.e. number of vectors) and the dimenisions of the vectors.
+
+        All remaining rows contain the token followed by the numeric elements of the
+        vector, separated by a space
+
+        Parameters
+        ----------
+        vec_file : str
+            File to load vectors from.
+        """
+        vectors = {}
+        with as_file(files(__package__) / vec_file) as p:
+            with gzip.open(p, "rt") as f:
+                # Read first line as header
+                header = f.readline().rstrip()
+                self.vocab_size, self.dimension = map(int, header.split())
+
+                # Read remaining lines and load vectors
+                for line in f:
+                    parts = line.rstrip().split()
+                    token = parts[0]
+                    vector = np.array([float(v) for v in parts[1:]], dtype=np.float32)
+                    vectors[token] = vector
+
+        self.vectors = vectors

ingredient_parser/en/_foundationfoods.py

@@ -10,6 +10,7 @@
 import numpy as np
 
 from ..dataclasses import FoundationFood
+from ._embeddings import GloVeModel
 from ._loaders import load_embeddings_model
 from ._utils import prepare_embeddings_tokens, tokenize
 
@@ -47,8 +48,8 @@
 # Increasing value indicates decreasing preference.
 PREFERRED_DATATYPES = {
     "foundation_food": 0,  #  Most preferred
-    "sr_legacy_food": 1,
-    "survey_fndds_food": 2,
+    "survey_fndds_food": 1,
+    "sr_legacy_food": 2,
 }
 
 
@@ -118,8 +119,8 @@ class uSIF:
     ----------
     a : float
         'a' parameter.
-    embeddings : floret.floret._floret
-        Floret embeddings model.
+    embeddings : GloVeModel
+        GloVe embeddings model.
     embeddings_dimension : int
         Dimension of embeddings model.
     fdc_ingredients : dict[str, list[FDCIngredient]]
@@ -132,9 +133,9 @@ class uSIF:
         Dictionary of token probabilities.
     """
 
-    def __init__(self, embeddings, fdc_ingredients: list[FDCIngredient]):
+    def __init__(self, embeddings: GloVeModel, fdc_ingredients: list[FDCIngredient]):
         self.embeddings = embeddings
-        self.embeddings_dimension: int = embeddings.get_dimension()
+        self.embeddings_dimension: int = embeddings.dimension
 
         self.fdc_ingredients: list[FDCIngredient] = fdc_ingredients
         self.token_prob: dict[str, float] = self._estimate_token_probability(
@@ -284,7 +285,7 @@ def _cosine_similarity(self, vec1: np.ndarray, vec2: np.ndarray) -> float:
         )
 
     def find_candidate_matches(
-        self, tokens: list[str], cutoff: float = 0.3
+        self, tokens: list[str], n: int
     ) -> list[FDCIngredientMatch]:
         """Find best candidate matches between input token and FDC ingredients with a
         cosine similarity of no more than cutoff.
@@ -293,29 +294,29 @@ def find_candidate_matches(
         ----------
         tokens : list[str]
             List of tokens.
-        cutoff : float
-            Maximum allowable score of returned matches.
+        n : int
+            Number of matches to return, sorted by score.
 
         Returns
         -------
         list[FDCIngredientMatch]
-            List of candidate matching FDC ingredient.
+            List of candidate matching FDC ingredients.
         """
         prepared_tokens = prepare_embeddings_tokens(tuple(tokens))
         input_token_vector = self._embed(prepared_tokens)
 
         candidates = []
         for idx, vec in enumerate(self.fdc_vectors):
             score = self._cosine_similarity(input_token_vector, vec)
-            if score <= cutoff:
-                candidates.append(
-                    FDCIngredientMatch(
-                        fdc=self.fdc_ingredients[idx],
-                        score=score,
-                    )
+            candidates.append(
+                FDCIngredientMatch(
+                    fdc=self.fdc_ingredients[idx],
+                    score=score,
                 )
+            )
 
-        return candidates
+        sorted_candidates = sorted(candidates, key=lambda x: x.score)
+        return sorted_candidates[:n]
 
 
 class FuzzyEmbeddingMatcher:
@@ -332,11 +333,11 @@ class FuzzyEmbeddingMatcher:
 
     Attributes
     ----------
-    embeddings : floret.floret._floret
+    embeddings : GloVeModel
         Floret embeddings model.
     """
 
-    def __init__(self, embeddings):
+    def __init__(self, embeddings: GloVeModel):
         self.embeddings = embeddings
 
     @lru_cache
@@ -536,7 +537,6 @@ def find_best_match(
 
         sorted_matches = sorted(scored, key=lambda x: x.score)
         return self._select_best_match(sorted_matches)
-        # return sorted_matches[0]
 
 
 @lru_cache
@@ -591,12 +591,14 @@ def match_foundation_foods(tokens: list[str]) -> FoundationFood | None:
         Matching foundation food, or None if no match can be found.
     """
     prepared_tokens = prepare_embeddings_tokens(tuple(tokens))
+    if not prepared_tokens:
+        return None
 
     if tuple(prepared_tokens) in FOUNDATION_FOOD_OVERRIDES:
         return FOUNDATION_FOOD_OVERRIDES[tuple(prepared_tokens)]
 
     u = get_usif_matcher()
-    candidate_matches = u.find_candidate_matches(prepared_tokens)
+    candidate_matches = u.find_candidate_matches(prepared_tokens, n=50)
     if not candidate_matches:
         return None
 

ingredient_parser/en/_loaders.py

@@ -3,9 +3,10 @@
 from functools import lru_cache
 from importlib.resources import as_file, files
 
-import floret
 import pycrfsuite
 
+from ._embeddings import GloVeModel
+
 
 @lru_cache
 def load_parser_model() -> pycrfsuite.Tagger:  # type: ignore
@@ -26,16 +27,15 @@ def load_parser_model() -> pycrfsuite.Tagger:  # type: ignore
 
 
 @lru_cache
-def load_embeddings_model() -> floret.floret._floret:  # type: ignore
+def load_embeddings_model() -> GloVeModel:  # type: ignore
     """Load embeddings model.
 
     This function is cached so that when the model has been loaded once, it does not
     need to be loaded again, the cached model is returned.
 
     Returns
     -------
-    floret.floret._floret
-        Embeddigns model.
+    GloVeModel
+        Embeddings model.
     """
-    with as_file(files(__package__) / "ingredient_embeddings.25d.floret.bin") as p:
-        return floret.load_model(str(p))
+    return GloVeModel("ingredient_embeddings.25d.glove.txt.gz")

ingredient_parser/en/_utils.py

@@ -9,6 +9,8 @@
 import nltk.stem.porter as nsp
 import pint
 
+from ingredient_parser.en._loaders import load_embeddings_model
+
 from .._common import UREG, consume, download_nltk_resources, is_float, is_range
 from ..dataclasses import IngredientAmount
 from ._constants import (
@@ -553,10 +555,13 @@ def prepare_embeddings_tokens(tokens: tuple[str, ...]) -> list[str]:
     list[str]
         Prepared tokens.
     """
+    embeddings = load_embeddings_model()
+
     return [
         stem(token.lower())
         for token in tokens
-        if not token.isnumeric()
+        if stem(token.lower()) in embeddings
+        and not token.isnumeric()
         and not token.isdigit()
         and not token.isdecimal()
         and not token.isspace()