Merge branch 'dev' of github.com:broadinstitute/seqr-loading-pipelines

Author: bpblanken

v03_pipeline/lib/methods/relatedness.py

@@ -30,7 +30,7 @@ def call_relatedness(
     # - brew install lz4
     # - CXXFLAGS='-I/opt/homebrew/include/' HAIL_COMPILE_NATIVES=1 make -C hail install
     # Hail issue here: https://discuss.hail.is/t/noclassdeffounderror-could-not-initialize-class-is-hail-methods-ibsffi/2453
-    kin_ht = hl.identity_by_descent(mt, maf=mt.info.AF[1], min=0.10, max=1.0)
+    kin_ht = hl.identity_by_descent(mt, maf=mt.info.AF[0], min=0.10, max=1.0)
     kin_ht = kin_ht.key_by('i', 'j')
     return kin_ht.select(
         ibd0=kin_ht.ibd.Z0,

v03_pipeline/lib/reference_data/compare_globals.py

@@ -67,13 +67,7 @@ def from_ht(
         paths = dict(rdc_globals_struct.paths)
         versions = dict(rdc_globals_struct.versions)
         # enums are nested structs
-        enums = {k: dict(v) for k, v in rdc_globals_struct.enums.items()}
-
-        for global_dict in [paths, versions, enums]:
-            for dataset in list(global_dict.keys()):
-                if dataset not in datasets:
-                    global_dict.pop(dataset)
-
+        enums = {k: dict(v) for k, v in rdc_globals_struct.enums.items() if k in paths}
         selects = {}
         for dataset in datasets:
             if dataset in ht.row:

v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset.py

@@ -44,22 +44,25 @@ def complete(self) -> bool:
                 )
             return False
 
-        for rdc in self.reference_dataset_collections:
-            datasets = rdc.datasets(self.dataset_type)
-            annotations_ht_globals = Globals.from_ht(
-                hl.read_table(self.output().path),
-                datasets,
-            )
-            rdc_ht_globals = Globals.from_ht(
-                self.rdc_annotation_dependencies[f'{rdc.value}_ht'],
-                datasets,
-            )
-            self._datasets_to_update.extend(
-                get_datasets_to_update(
-                    annotations_ht_globals,
-                    rdc_ht_globals,
-                ),
-            )
+        datasets_to_check = [
+            dataset
+            for rdc in self.reference_dataset_collections
+            for dataset in rdc.datasets(self.dataset_type)
+        ]
+        annotations_ht_globals = Globals.from_ht(
+            hl.read_table(self.output().path),
+            datasets_to_check,
+        )
+        rdc_ht_globals = Globals.from_dataset_configs(
+            self.reference_genome,
+            datasets_to_check,
+        )
+        self._datasets_to_update.extend(
+            get_datasets_to_update(
+                annotations_ht_globals,
+                rdc_ht_globals,
+            ),
+        )
         logger.info(f'Datasets to update: {self._datasets_to_update}')
         return not self._datasets_to_update