use dataset configs in UpdateVATWithUpdatedRDC complete() take 2 (#723)

* use datasets in Globals constructors

* simpler mock

* cleaner set operations

* start to build config in test

* use dataset configs in UpdateVATWithUpdatedRDC complete() take 2

* fixes

* oops

* Delete v03_pipeline/lib/tasks/reference_data/write_cached_reference_dataset_query.py

* Delete v03_pipeline/lib/tasks/reference_data/write_cached_reference_dataset_query_test.py

* oops

* fix mocks

* Filter to paths

Author: jklugherz

v03_pipeline/lib/reference_data/compare_globals.py

@@ -67,13 +67,7 @@ def from_ht(
         paths = dict(rdc_globals_struct.paths)
         versions = dict(rdc_globals_struct.versions)
         # enums are nested structs
-        enums = {k: dict(v) for k, v in rdc_globals_struct.enums.items()}
-
-        for global_dict in [paths, versions, enums]:
-            for dataset in list(global_dict.keys()):
-                if dataset not in datasets:
-                    global_dict.pop(dataset)
-
+        enums = {k: dict(v) for k, v in rdc_globals_struct.enums.items() if k in paths}
         selects = {}
         for dataset in datasets:
             if dataset in ht.row:

v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset.py

@@ -44,22 +44,25 @@ def complete(self) -> bool:
                 )
             return False
 
-        for rdc in self.reference_dataset_collections:
-            datasets = rdc.datasets(self.dataset_type)
-            annotations_ht_globals = Globals.from_ht(
-                hl.read_table(self.output().path),
-                datasets,
-            )
-            rdc_ht_globals = Globals.from_ht(
-                self.rdc_annotation_dependencies[f'{rdc.value}_ht'],
-                datasets,
-            )
-            self._datasets_to_update.extend(
-                get_datasets_to_update(
-                    annotations_ht_globals,
-                    rdc_ht_globals,
-                ),
-            )
+        datasets_to_check = [
+            dataset
+            for rdc in self.reference_dataset_collections
+            for dataset in rdc.datasets(self.dataset_type)
+        ]
+        annotations_ht_globals = Globals.from_ht(
+            hl.read_table(self.output().path),
+            datasets_to_check,
+        )
+        rdc_ht_globals = Globals.from_dataset_configs(
+            self.reference_genome,
+            datasets_to_check,
+        )
+        self._datasets_to_update.extend(
+            get_datasets_to_update(
+                annotations_ht_globals,
+                rdc_ht_globals,
+            ),
+        )
         logger.info(f'Datasets to update: {self._datasets_to_update}')
         return not self._datasets_to_update