additional semi join (#947)

Author: bpblanken

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples.py

@@ -74,6 +74,12 @@ def update_table(self, ht: hl.Table) -> hl.Table:
             # and either present or not present in the existing annotations table.
             callset_variants_ht = ht.semi_join(callset_ht)
             ht = ht.anti_join(callset_ht)
+            lookup_ht = hl.read_table(
+                lookup_table_path(
+                    self.reference_genome,
+                    self.dataset_type,
+                ),
+            )
             callset_variants_ht = callset_variants_ht.annotate(
                 **get_fields(
                     callset_variants_ht,
@@ -89,6 +95,11 @@ def update_table(self, ht: hl.Table) -> hl.Table:
             )
             ht = ht.union(callset_variants_ht, unify=True)
 
+            # Variants may have fallen out of the callset and
+            # have been removed from the lookup table during modification.
+            # Ensure we don't proceed with those variants.
+            ht = ht.semi_join(lookup_ht)
+
         # Fix up the globals and mark the table as updated with these callset/project pairs.
         ht = self.annotate_globals(ht)
         return ht.annotate_globals(