Fix optional fields join

Author: bpblanken

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples.py

@@ -177,7 +177,7 @@ def update_table(self, ht: hl.Table) -> hl.Table:
                 remapped_and_subsetted_callset_path(
                     self.reference_genome,
                     self.dataset_type,
-                    callset_path,
+                    callset_path,ca
                     project_guid,
                 ),
             ).rows()
@@ -188,6 +188,13 @@ def update_table(self, ht: hl.Table) -> hl.Table:
                 self.project_pedigree_paths,
             )
         ]
+
+        # Drop any fields potentially unshared/unused by the annotations.
+        for i, callset_ht in enumerate(callset_hts):
+            for row_field in self.dataset_type.optional_row_fields:
+                if hasattr(callset_ht, row_field):
+                    callset_hts[i] = callset_ht.drop(row_field)
+
         callset_ht = functools.reduce(
             (lambda ht1, ht2: ht1.union(ht2, unify=True)),
             callset_hts,