make updating crdqs a dependency of updating vat (#912)

jklugherz · web-flow · commit a25980488db9 · 2024-09-30T16:34:39.000-04:00
* make updating crdqs a dependency of updating vat

* remove from init

* fix import
diff --git a/v03_pipeline/bin/pipeline_worker.py b/v03_pipeline/bin/pipeline_worker.py
@@ -13,7 +13,6 @@
     project_remap_path,
 )
 from v03_pipeline.lib.tasks import (
-    UpdateCachedReferenceDatasetQueries,
     UpdateVariantAnnotationsTableWithNewSamplesTask,
     WriteProjectFamilyTablesTask,
 )
@@ -50,10 +49,6 @@ def main():
                 k: v for k, v in lpr.model_dump().items() if k != 'projects_to_run'
             }
             tasks = [
-                UpdateCachedReferenceDatasetQueries(
-                    reference_genome=lpr.reference_genome,
-                    dataset_type=lpr.dataset_type,
-                ),
                 UpdateVariantAnnotationsTableWithNewSamplesTask(
                     project_guids=lpr.projects_to_run,
                     project_remap_paths=project_remap_paths,
diff --git a/v03_pipeline/lib/tasks/__init__.py b/v03_pipeline/lib/tasks/__init__.py
@@ -8,9 +8,6 @@
 from v03_pipeline.lib.tasks.migrate_all_variant_annotations_tables import (
     MigrateAllVariantAnnotationsTablesTask,
 )
-from v03_pipeline.lib.tasks.reference_data.update_cached_reference_dataset_queries import (
-    UpdateCachedReferenceDatasetQueries,
-)
 from v03_pipeline.lib.tasks.update_lookup_table import (
     UpdateLookupTableTask,
 )
@@ -53,7 +50,6 @@
     'UpdateVariantAnnotationsTableWithNewSamplesTask',
     'UpdateVariantAnnotationsTableWithDeletedProjectTask',
     'UpdateVariantAnnotationsTableWithDeletedFamiliesTask',
-    'UpdateCachedReferenceDatasetQueries',
     'WriteMetadataForRunTask',
     'WriteProjectFamilyTablesTask',
 ]
diff --git a/v03_pipeline/lib/tasks/base/base_update_variant_annotations_table.py b/v03_pipeline/lib/tasks/base/base_update_variant_annotations_table.py
@@ -13,6 +13,9 @@
 )
 from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask
 from v03_pipeline.lib.tasks.files import GCSorLocalTarget
+from v03_pipeline.lib.tasks.reference_data.update_cached_reference_dataset_queries import (
+    UpdateCachedReferenceDatasetQueries,
+)
 from v03_pipeline.lib.tasks.reference_data.updated_reference_dataset_collection import (
     UpdatedReferenceDatasetCollectionTask,
 )
@@ -32,19 +35,24 @@ def output(self) -> luigi.Target:
         )
 
     def requires(self) -> list[luigi.Task]:
-        return [
-            (
-                UpdatedReferenceDatasetCollectionTask(
-                    self.reference_genome,
-                    self.dataset_type,
-                    rdc,
-                )
+        requirements = [
+            UpdateCachedReferenceDatasetQueries(
+                reference_genome=self.reference_genome,
+                dataset_type=self.dataset_type,
+            ),
+        ]
+        requirements.extend(
+            UpdatedReferenceDatasetCollectionTask(
+                self.reference_genome,
+                self.dataset_type,
+                rdc,
             )
             for rdc in ReferenceDatasetCollection.for_reference_genome_dataset_type(
                 self.reference_genome,
                 self.dataset_type,
             )
-        ]
+        )
+        return requirements
 
     def initialize_table(self) -> hl.Table:
         key_type = self.dataset_type.table_key_type(self.reference_genome)
diff --git a/v03_pipeline/lib/tasks/base/base_update_variant_annotations_table_test.py b/v03_pipeline/lib/tasks/base/base_update_variant_annotations_table_test.py
@@ -53,8 +53,16 @@ def setUp(self) -> None:
     @patch(
         'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdatedReferenceDatasetCollectionTask',
     )
-    def test_should_create_initialized_table(self, mock_update_rdc_task) -> None:
+    @patch(
+        'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdateCachedReferenceDatasetQueries',
+    )
+    def test_should_create_initialized_table(
+        self,
+        mock_update_crdqs_task,
+        mock_update_rdc_task,
+    ) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         vat_task = BaseUpdateVariantAnnotationsTableTask(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
diff --git a/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py b/v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py
@@ -649,6 +649,9 @@
 @mock.patch(
     'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdatedReferenceDatasetCollectionTask',
 )
+@mock.patch(
+    'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdateCachedReferenceDatasetQueries',
+)
 @mock.patch(
     'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.BaseUpdateVariantAnnotationsTableTask.initialize_table',
 )
@@ -719,9 +722,11 @@ def setUp(self) -> None:
     def test_update_vat_with_updated_rdc_snv_indel_38(
         self,
         mock_initialize_table,
+        mock_update_crdqs_task,
         mock_update_rdc_task,
     ):
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_initialize_table.return_value = hl.Table.parallelize(
             [
                 hl.Struct(
@@ -927,9 +932,11 @@ def test_update_vat_with_updated_rdc_snv_indel_38(
     def test_update_vat_with_updated_rdc_mito_38(
         self,
         mock_initialize_table,
+        mock_update_crdqs_task,
         mock_update_rdc_task,
     ):
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_initialize_table.return_value = hl.Table.parallelize(
             [
                 hl.Struct(
@@ -1075,9 +1082,11 @@ def test_update_vat_with_updated_rdc_mito_38(
     def test_update_vat_with_updated_rdc_snv_indel_37(
         self,
         mock_initialize_table,
+        mock_update_crdqs_task,
         mock_update_rdc_task,
     ):
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_initialize_table.return_value = hl.Table.parallelize(
             [
                 hl.Struct(
diff --git a/v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py b/v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py
@@ -88,6 +88,9 @@
 @patch(
     'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdatedReferenceDatasetCollectionTask',
 )
+@patch(
+    'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdateCachedReferenceDatasetQueries',
+)
 class UpdateVariantAnnotationsTableWithNewSamplesTaskTest(MockedDatarootTestCase):
     def setUp(self) -> None:
         super().setUp()
@@ -154,9 +157,11 @@ def setUp(self) -> None:
     def test_missing_pedigree(
         self,
         mock_update_vat_with_rdc_task,
+        mock_update_crdqs_task,
         mock_update_rdc_task,
     ) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_update_vat_with_rdc_task.return_value = MockCompleteTask()
         uvatwns_task = UpdateVariantAnnotationsTableWithNewSamplesTask(
             reference_genome=ReferenceGenome.GRCh38,
@@ -180,9 +185,11 @@ def test_missing_pedigree(
     def test_missing_interval_reference(
         self,
         mock_update_vat_with_rdc_task,
+        mock_update_crdqs_task,
         mock_update_rdc_task,
     ) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_update_vat_with_rdc_task.return_value = MockCompleteTask()
         shutil.rmtree(
             valid_reference_dataset_collection_path(
@@ -233,10 +240,12 @@ def test_multiple_update_vat(
         mock_updated_cached_reference_dataset_query,
         mock_env: Mock,
         mock_register_alleles: Mock,
+        mock_update_crdqs_task,
         mock_update_rdc_task: Mock,
     ) -> None:
         mock_updated_cached_reference_dataset_query.return_value = MockCompleteTask()
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_update_vat_with_rdc_task.return_value = (
             BaseUpdateVariantAnnotationsTableTask(
                 reference_genome=ReferenceGenome.GRCh38,
@@ -679,9 +688,11 @@ def test_update_vat_grch37(
         mock_vep: Mock,
         mock_update_vat_with_rdc_task: Mock,
         mock_register_alleles: Mock,
+        mock_update_crdqs_task,
         mock_update_rdc_task: Mock,
     ) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_update_vat_with_rdc_task.return_value = (
             BaseUpdateVariantAnnotationsTableTask(
                 reference_genome=ReferenceGenome.GRCh37,
@@ -845,12 +856,14 @@ def test_update_vat_without_accessing_private_datasets(
         mock_rdc_env: Mock,
         mock_update_vat_with_rdc_task: Mock,
         mock_register_alleles: Mock,
+        mock_update_crdqs_task,
         mock_update_rdc_task: Mock,
     ) -> None:
         mock_load_gencode_ensembl_to_refseq_id.return_value = hl.dict(
             {'ENST00000327044': 'NM_015658.4'},
         )
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_update_vat_with_rdc_task.return_value = (
             BaseUpdateVariantAnnotationsTableTask(
                 reference_genome=ReferenceGenome.GRCh38,
@@ -913,9 +926,11 @@ def test_mito_update_vat(
         self,
         mock_update_vat_with_rdc_task: Mock,
         mock_register_alleles: Mock,
+        mock_update_crdqs_task,
         mock_update_rdc_task: Mock,
     ) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_update_vat_with_rdc_task.return_value = (
             BaseUpdateVariantAnnotationsTableTask(
                 reference_genome=ReferenceGenome.GRCh38,
@@ -1198,9 +1213,11 @@ def test_mito_update_vat(
     def test_sv_update_vat(
         self,
         mock_load_gencode: Mock,
+        mock_update_crdqs_task,
         mock_update_rdc_task: Mock,
     ) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         mock_load_gencode.return_value = GENE_ID_MAPPING
         worker = luigi.worker.Worker()
         update_variant_annotations_task = (
@@ -1778,9 +1795,11 @@ def test_sv_update_vat(
 
     def test_gcnv_update_vat(
         self,
+        mock_update_crdqs_task,
         mock_update_rdc_task,
     ) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         worker = luigi.worker.Worker()
         update_variant_annotations_task = (
             UpdateVariantAnnotationsTableWithNewSamplesTask(
diff --git a/v03_pipeline/lib/tasks/write_relatedness_check_table.py b/v03_pipeline/lib/tasks/write_relatedness_check_table.py
@@ -3,7 +3,7 @@
 import luigi.util
 
 from v03_pipeline.lib.methods.relatedness import call_relatedness
-from v03_pipeline.lib.model import CachedReferenceDatasetQuery, Env
+from v03_pipeline.lib.model import CachedReferenceDatasetQuery
 from v03_pipeline.lib.paths import (
     relatedness_check_table_path,
 )
@@ -27,21 +27,14 @@ def output(self) -> luigi.Target:
             ),
         )
 
-    def requires(self) -> luigi.Task:
-        requirements = [
+    def requires(self):
+        return [
             self.clone(ValidateCallsetTask),
+            self.clone(
+                UpdatedCachedReferenceDatasetQuery,
+                crdq=CachedReferenceDatasetQuery.GNOMAD_QC,
+            ),
         ]
-        if Env.ACCESS_PRIVATE_REFERENCE_DATASETS:
-            requirements = [
-                *requirements,
-                (
-                    self.clone(
-                        UpdatedCachedReferenceDatasetQuery,
-                        crdq=CachedReferenceDatasetQuery.GNOMAD_QC,
-                    )
-                ),
-            ]
-        return requirements
 
     def create_table(self) -> hl.Table:
         callset_mt = hl.read_matrix_table(self.input()[0].path)
diff --git a/v03_pipeline/lib/tasks/write_variant_annotations_vcf_test.py b/v03_pipeline/lib/tasks/write_variant_annotations_vcf_test.py
@@ -10,6 +10,7 @@
 from v03_pipeline.lib.tasks.write_variant_annotations_vcf import (
     WriteVariantAnnotationsVCF,
 )
+from v03_pipeline.lib.test.mock_complete_task import MockCompleteTask
 from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
 
 TEST_SV_VCF = 'v03_pipeline/var/test/callsets/sv_1.vcf'
@@ -39,11 +40,16 @@ class WriteVariantAnnotationsVCFTest(MockedDatarootTestCase):
     @patch(
         'v03_pipeline.lib.tasks.write_new_variants_table.load_gencode_gene_symbol_to_gene_id',
     )
+    @patch(
+        'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdateCachedReferenceDatasetQueries',
+    )
     def test_sv_export_vcf(
         self,
+        mock_update_crdqs_task: Mock,
         mock_load_gencode: Mock,
     ) -> None:
         mock_load_gencode.return_value = GENE_ID_MAPPING
+        mock_update_crdqs_task.return_value = MockCompleteTask()
         worker = luigi.worker.Worker()
         update_variant_annotations_task = (
             UpdateVariantAnnotationsTableWithNewSamplesTask(

Original file line number	Diff line number	Diff line change
`@@ -13,7 +13,6 @@`
`13`	`13`	`project_remap_path,`
`14`	`14`	`)`
`15`	`15`	`from v03_pipeline.lib.tasks import (`
`16`		`- UpdateCachedReferenceDatasetQueries,`
`17`	`16`	`UpdateVariantAnnotationsTableWithNewSamplesTask,`
`18`	`17`	`WriteProjectFamilyTablesTask,`
`19`	`18`	`)`
`@@ -50,10 +49,6 @@ def main():`
`50`	`49`	`k: v for k, v in lpr.model_dump().items() if k != 'projects_to_run'`
`51`	`50`	`}`
`52`	`51`	`tasks = [`
`53`		`- UpdateCachedReferenceDatasetQueries(`
`54`		`- reference_genome=lpr.reference_genome,`
`55`		`- dataset_type=lpr.dataset_type,`
`56`		`- ),`
`57`	`52`	`UpdateVariantAnnotationsTableWithNewSamplesTask(`
`58`	`53`	`project_guids=lpr.projects_to_run,`
`59`	`54`	`project_remap_paths=project_remap_paths,`
Original file line number	Diff line number	Diff line change
`@@ -8,9 +8,6 @@`
`8`	`8`	`from v03_pipeline.lib.tasks.migrate_all_variant_annotations_tables import (`
`9`	`9`	`MigrateAllVariantAnnotationsTablesTask,`
`10`	`10`	`)`
`11`		`-from v03_pipeline.lib.tasks.reference_data.update_cached_reference_dataset_queries import (`
`12`		`- UpdateCachedReferenceDatasetQueries,`
`13`		`-)`
`14`	`11`	`from v03_pipeline.lib.tasks.update_lookup_table import (`
`15`	`12`	`UpdateLookupTableTask,`
`16`	`13`	`)`
`@@ -53,7 +50,6 @@`
`53`	`50`	`'UpdateVariantAnnotationsTableWithNewSamplesTask',`
`54`	`51`	`'UpdateVariantAnnotationsTableWithDeletedProjectTask',`
`55`	`52`	`'UpdateVariantAnnotationsTableWithDeletedFamiliesTask',`
`56`		`- 'UpdateCachedReferenceDatasetQueries',`
`57`	`53`	`'WriteMetadataForRunTask',`
`58`	`54`	`'WriteProjectFamilyTablesTask',`
`59`	`55`	`]`