Bugfix: De-duplicate ingredient names.

Author: strangetom

ingredient_parser/en/postprocess.py

@@ -288,7 +288,40 @@ def _postprocess_names(self) -> tuple[list[IngredientText], list[FoundationFood]
                     if ff:
                         foundation_foods.add(ff)
 
-        return names, list(foundation_foods)
+        return self._deduplicate_names(names), list(foundation_foods)
+
+    def _deduplicate_names(self, names: list[IngredientText]) -> list[IngredientText]:
+        """Deduplicate list of names.
+
+        Where the same name text appears in multiple IngredientText objects, the
+        confidence values are averaged, and the minimum starting_index is kept for the
+        dedeuplicated names.
+
+        Parameters
+        ----------
+        names : list[IngredientText]
+            List of names.
+
+        Returns
+        -------
+        list[IngredientText]
+            Deduplicaed list of names.
+        """
+        name_dict = defaultdict(list)
+        for name in names:
+            name_dict[name.text].append(name)
+
+        deduped_names = []
+        for text, name_objs in name_dict.items():
+            deduped_names.append(
+                IngredientText(
+                    text=text,
+                    confidence=mean([n.confidence for n in name_objs]),
+                    starting_index=min([n.starting_index for n in name_objs]),
+                )
+            )
+
+        return deduped_names
 
     def _group_name_labels(self, name_labels: list[str]) -> list[list[tuple[int, str]]]:
         """Group name labels according to name label type.

tests/postprocess/test_process_names.py

@@ -112,3 +112,21 @@ def test_multiple_modified_variant_names(self):
         p = PostProcessor(sentence, tokens, pos_tags, labels, scores)
         names, _ = p._postprocess_names()
         assert names == expected
+
+    def test_deuplicate_ingredient_names(self):
+        """
+        Test that a list containing one IngredientText objects is returned
+        """
+        sentence = "1/2 cup sugar plus 1 1/2 tablespoons sugar"
+        tokens = ["#1$2", "cup", "sugar", "plus", "1#1$2", "tablespoon", "sugar"]
+        pos_tags = ["CD", "NN", "NN", "CC", "CD", "NN", "NN"]
+        labels = ["QTY", "UNIT", "B_NAME_TOK", "COMMENT", "QTY", "UNIT", "B_NAME_TOK"]
+        scores = [0.0] * len(tokens)
+
+        expected = [
+            IngredientText(text="sugar", confidence=0, starting_index=2),
+        ]
+
+        p = PostProcessor(sentence, tokens, pos_tags, labels, scores)
+        names, _ = p._postprocess_names()
+        assert names == expected