i guess this is necessary

Author: bpblanken

v03_pipeline/lib/misc/family_entries.py

@@ -98,13 +98,18 @@ def remove_family_guids(
     family_guids: hl.SetExpression,
 ) -> hl.Table:
     # Remove families from the existing project table structure (both the entries arrays and the globals are mutated)
-    family_indexes_to_keep = hl.array(
+    family_indexes_to_keep = hl.eval(hl.array(
         hl.enumerate(ht.globals.family_guids)
         .filter(lambda item: ~family_guids.contains(item[1]))
         .map(lambda item: item[0]),
-    )
+    ))
     ht = ht.annotate(
-        family_entries=family_indexes_to_keep.map(lambda i: ht.family_entries[i]),
+        # NB: this "should" work without the extra if statement (and does in the tests)
+        # however, experiments on dataproc showed this statement hanging with an empty 
+        # indexes array.
+        family_entries=family_indexes_to_keep.map(lambda i: ht.family_entries[i])
+        if len(family_indexes_to_keep) > 0
+        else hl.empty_array(ht.family_entries.dtype.element_type)
     )
     return ht.annotate_globals(
         family_guids=ht.family_guids.filter(

v03_pipeline/lib/misc/lookup.py

@@ -135,9 +135,10 @@ def remove_project(
     )
     ht = ht.annotate(
         project_stats=(
-            project_indexes_to_keep.map(
-                lambda i: ht.project_stats[i],
-            )
+            # See "remove_family_guids" func for why this was necessary
+            project_stats=project_indexes_to_keep.map(lambda i: ht.project_stats[i])
+            if len(project_indexes_to_keep) > 0
+            else hl.empty_array(ht.project_stats.dtype.element_type)
         ),
     )
     ht = ht.filter(hl.any(ht.project_stats.map(hl.is_defined)))