add env vars too!

bpblanken · bpblanken · commit 3cd656b5733e · 2024-06-20T21:26:11.000-04:00
diff --git a/v03_pipeline/lib/model/environment.py b/v03_pipeline/lib/model/environment.py
@@ -28,6 +28,8 @@
 ACCESS_PRIVATE_REFERENCE_DATASETS = (
     os.environ.get('ACCESS_PRIVATE_REFERENCE_DATASETS') == '1'
 )
+CHECK_SEX_AND_RELATEDNESS = os.environ.get('CHECK_SEX_AND_RELATEDNESS') == '1'
+EXPECT_WES_FILTERS = os.environ.get('EXPECT_WES_FILTERS') == '1'
 REFERENCE_DATA_AUTO_UPDATE = os.environ.get('REFERENCE_DATA_AUTO_UPDATE') == '1'
 SHOULD_REGISTER_ALLELES = os.environ.get('SHOULD_REGISTER_ALLELES') == '1'
 
@@ -36,6 +38,8 @@
 class Env:
     ACCESS_PRIVATE_REFERENCE_DATASETS: bool = ACCESS_PRIVATE_REFERENCE_DATASETS
     ALLELE_REGISTRY_SECRET_NAME: str | None = ALLELE_REGISTRY_SECRET_NAME
+    CHECK_SEX_AND_RELATEDNESS: bool = CHECK_SEX_AND_RELATEDNESS
+    EXPECT_WES_FILTERS: bool = EXPECT_WES_FILTERS
     HAIL_TMPDIR: str = HAIL_TMPDIR
     HAIL_SEARCH_DATA: str = HAIL_SEARCH_DATA
     LIFTOVER_REF_PATH: str = LIFTOVER_REF_PATH
diff --git a/v03_pipeline/lib/paths.py b/v03_pipeline/lib/paths.py
@@ -222,6 +222,7 @@ def valid_filters_path(
     callset_path: str,
 ) -> str | None:
     if (
+        not Env.EXPECT_WES_FILTERS or
         not dataset_type.expect_filters(sample_type)
         or 'part_one_outputs' not in callset_path
     ):
@@ -240,7 +241,7 @@ def valid_reference_dataset_collection_path(
 ) -> str | None:
     if (
         not Env.ACCESS_PRIVATE_REFERENCE_DATASETS
-        and reference_dataset_collection.access_control == AccessControl.PRIVATE
+        or reference_dataset_collection.access_control == AccessControl.PUBLIC
     ):
         return None
     return os.path.join(
diff --git a/v03_pipeline/lib/paths_test.py b/v03_pipeline/lib/paths_test.py
@@ -66,14 +66,16 @@ def test_valid_filters_path(self) -> None:
             ),
             None,
         )
-        self.assertEqual(
-            valid_filters_path(
-                DatasetType.SNV_INDEL,
-                SampleType.WES,
-                'gs://bucket/RDG_Broad_WES_Internal_Oct2023/part_one_outputs/chr*/*.vcf.gz',
-            ),
-            'gs://bucket/RDG_Broad_WES_Internal_Oct2023/part_two_outputs/*.filtered.*.vcf.gz',
-        )
+        with patch('v03_pipeline.lib.paths.Env') as mock_env:
+            mock_env.EXPECT_WES_FILTERS = True
+            self.assertEqual(
+                valid_filters_path(
+                    DatasetType.SNV_INDEL,
+                    SampleType.WES,
+                    'gs://bucket/RDG_Broad_WES_Internal_Oct2023/part_one_outputs/chr*/*.vcf.gz',
+                ),
+                'gs://bucket/RDG_Broad_WES_Internal_Oct2023/part_two_outputs/*.filtered.*.vcf.gz',
+            )
 
     def test_project_table_path(self) -> None:
         self.assertEqual(
diff --git a/v03_pipeline/lib/tasks/update_lookup_table_test.py b/v03_pipeline/lib/tasks/update_lookup_table_test.py
@@ -26,7 +26,6 @@ def test_skip_update_lookup_table_task(self) -> None:
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
         )
         worker.add(uslt_task)
         worker.run()
@@ -58,7 +57,6 @@ def test_update_lookup_table_task(self) -> None:
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
         )
         worker.add(uslt_task)
         worker.run()
diff --git a/v03_pipeline/lib/tasks/update_project_table_test.py b/v03_pipeline/lib/tasks/update_project_table_test.py
@@ -22,7 +22,6 @@ def test_update_project_table_task(self) -> None:
             project_remap_path=TEST_REMAP,
             project_pedigree_path=TEST_PEDIGREE_3,
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
         )
         worker.add(upt_task)
         worker.run()
diff --git a/v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py b/v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py
@@ -163,7 +163,6 @@ def test_missing_pedigree(
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=['bad_pedigree'],
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
         )
         worker = luigi.worker.Worker()
@@ -197,7 +196,6 @@ def test_missing_interval_reference(
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
         )
         worker = luigi.worker.Worker()
@@ -366,7 +364,6 @@ def test_multiple_update_vat(
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=False,
-            skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
         )
         worker.add(uvatwns_task_3)
@@ -418,7 +415,6 @@ def test_multiple_update_vat(
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_4],
             skip_validation=False,
-            skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
         )
         worker.add(uvatwns_task_4)
@@ -689,7 +685,6 @@ def test_update_vat_grch37(
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
         )
         worker.add(uvatwns_task)
@@ -769,7 +764,6 @@ def test_update_vat_without_accessing_private_datasets(
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
         )
         worker.add(uvatwns_task)
@@ -827,7 +821,6 @@ def test_mito_update_vat(
                 project_remap_paths=['not_a_real_file'],
                 project_pedigree_paths=[TEST_PEDIGREE_5],
                 skip_validation=True,
-                skip_check_sex_and_relatedness=True,
                 run_id=TEST_RUN_ID,
             )
         )
@@ -1092,7 +1085,6 @@ def test_sv_update_vat(
                 project_remap_paths=['not_a_real_file'],
                 project_pedigree_paths=[TEST_PEDIGREE_5],
                 skip_validation=True,
-                skip_check_sex_and_relatedness=True,
                 run_id=TEST_RUN_ID,
             )
         )
@@ -1654,7 +1646,6 @@ def test_gcnv_update_vat(
                 project_remap_paths=['not_a_real_file'],
                 project_pedigree_paths=[TEST_PEDIGREE_5],
                 skip_validation=True,
-                skip_check_sex_and_relatedness=True,
                 run_id=TEST_RUN_ID,
             )
         )
diff --git a/v03_pipeline/lib/tasks/write_family_table_test.py b/v03_pipeline/lib/tasks/write_family_table_test.py
@@ -26,7 +26,6 @@ def test_snv_write_family_table_task(self) -> None:
             project_pedigree_path=TEST_PEDIGREE_3,
             family_guid='abc_1',
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
         )
         worker.add(wft_task)
         worker.run()
@@ -164,7 +163,6 @@ def test_sv_write_family_table_task(self) -> None:
             project_pedigree_path=TEST_PEDIGREE_5,
             family_guid='family_2_1',
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
         )
         worker.add(write_family_table_task)
         worker.run()
@@ -417,7 +415,6 @@ def test_gcnv_write_family_table_task(self) -> None:
             project_pedigree_path=TEST_PEDIGREE_5,
             family_guid='family_2_1',
             skip_validation=True,
-            skip_check_sex_and_relatedness=True,
         )
         worker.add(write_family_table_task)
         worker.run()
diff --git a/v03_pipeline/lib/tasks/write_imported_callset.py b/v03_pipeline/lib/tasks/write_imported_callset.py
@@ -55,8 +55,12 @@ def output(self) -> luigi.Target:
 
     def requires(self) -> list[luigi.Task]:
         requirements = []
-        if not self.skip_expect_filters and self.dataset_type.expect_filters(
-            self.sample_type,
+        if (
+            Env.EXPECT_WES_FILTERS
+            and not self.skip_expect_filters
+            and self.dataset_type.expect_filters(
+                self.sample_type,
+            )
         ):
             requirements = [
                 *requirements,
@@ -87,7 +91,8 @@ def requires(self) -> list[luigi.Task]:
                 ),
             ]
         if (
-            not self.skip_check_sex_and_relatedness
+            Env.CHECK_SEX_AND_RELATEDNESS
+            and not self.skip_check_sex_and_relatedness
             and self.dataset_type.check_sex_and_relatedness
         ):
             requirements = [
@@ -124,8 +129,12 @@ def create_table(self) -> hl.MatrixTable:
             self.dataset_type,
         )
         filters_path = None
-        if not self.skip_expect_filters and self.dataset_type.expect_filters(
-            self.sample_type,
+        if (
+            Env.EXPECT_WES_FILTERS
+            and not self.skip_expect_filters
+            and self.dataset_type.expect_filters(
+                self.sample_type,
+            )
         ):
             filters_path = valid_filters_path(
                 self.dataset_type,
@@ -178,7 +187,8 @@ def create_table(self) -> hl.MatrixTable:
                 self.sample_type,
             )
         if (
-            not self.skip_check_sex_and_relatedness
+            Env.CHECK_SEX_AND_RELATEDNESS
+            and not self.skip_check_sex_and_relatedness
             and self.dataset_type.check_sex_and_relatedness
         ):
             sex_check_ht = hl.read_table(
diff --git a/v03_pipeline/lib/tasks/write_metadata_for_run_test.py b/v03_pipeline/lib/tasks/write_metadata_for_run_test.py
@@ -23,7 +23,6 @@ def test_write_metadata_for_run_task(self) -> None:
             project_guids=['R0113_test_project', 'R0114_project4'],
             project_remap_paths=[TEST_REMAP_2, TEST_REMAP_2],
             project_pedigree_paths=[TEST_PEDIGREE_3, TEST_PEDIGREE_4],
-            skip_check_sex_and_relatedness=True,
             skip_validation=True,
             run_id='run_123456',
         )
diff --git a/v03_pipeline/lib/tasks/write_remapped_and_subsetted_callset.py b/v03_pipeline/lib/tasks/write_remapped_and_subsetted_callset.py
@@ -15,6 +15,7 @@
 )
 from v03_pipeline.lib.misc.pedigree import parse_pedigree_ht_to_families
 from v03_pipeline.lib.misc.sample_ids import remap_sample_ids, subset_samples
+from v03_pipeline.lib.model.environment import Env
 from v03_pipeline.lib.paths import remapped_and_subsetted_callset_path
 from v03_pipeline.lib.tasks.base.base_loading_run_params import BaseLoadingRunParams
 from v03_pipeline.lib.tasks.base.base_write import BaseWriteTask
@@ -88,7 +89,8 @@ def create_table(self) -> hl.MatrixTable:
         families_failed_relatedness_check = {}
         families_failed_sex_check = {}
         if (
-            not self.skip_check_sex_and_relatedness
+            Env.CHECK_SEX_AND_RELATEDNESS
+            and not self.skip_check_sex_and_relatedness
             and self.dataset_type.check_sex_and_relatedness
         ):
             relatedness_check_ht = hl.read_table(self.input()[2].path)

Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,6 @@ def test_update_project_table_task(self) -> None:`
`22`	`22`	`project_remap_path=TEST_REMAP,`
`23`	`23`	`project_pedigree_path=TEST_PEDIGREE_3,`
`24`	`24`	`skip_validation=True,`
`25`		`- skip_check_sex_and_relatedness=True,`
`26`	`25`	`)`
`27`	`26`	`worker.add(upt_task)`
`28`	`27`	`worker.run()`
Original file line number	Diff line number	Diff line change
`@@ -163,7 +163,6 @@ def test_missing_pedigree(`
`163`	`163`	`project_remap_paths=[TEST_REMAP],`
`164`	`164`	`project_pedigree_paths=['bad_pedigree'],`
`165`	`165`	`skip_validation=True,`
`166`		`- skip_check_sex_and_relatedness=True,`
`167`	`166`	`run_id=TEST_RUN_ID,`
`168`	`167`	`)`
`169`	`168`	`worker = luigi.worker.Worker()`
`@@ -197,7 +196,6 @@ def test_missing_interval_reference(`
`197`	`196`	`project_remap_paths=[TEST_REMAP],`
`198`	`197`	`project_pedigree_paths=[TEST_PEDIGREE_3],`
`199`	`198`	`skip_validation=True,`
`200`		`- skip_check_sex_and_relatedness=True,`
`201`	`199`	`run_id=TEST_RUN_ID,`
`202`	`200`	`)`
`203`	`201`	`worker = luigi.worker.Worker()`
`@@ -366,7 +364,6 @@ def test_multiple_update_vat(`
`366`	`364`	`project_remap_paths=[TEST_REMAP],`
`367`	`365`	`project_pedigree_paths=[TEST_PEDIGREE_3],`
`368`	`366`	`skip_validation=False,`
`369`		`- skip_check_sex_and_relatedness=True,`
`370`	`367`	`run_id=TEST_RUN_ID,`
`371`	`368`	`)`
`372`	`369`	`worker.add(uvatwns_task_3)`
`@@ -418,7 +415,6 @@ def test_multiple_update_vat(`
`418`	`415`	`project_remap_paths=[TEST_REMAP],`
`419`	`416`	`project_pedigree_paths=[TEST_PEDIGREE_4],`
`420`	`417`	`skip_validation=False,`
`421`		`- skip_check_sex_and_relatedness=True,`
`422`	`418`	`run_id=TEST_RUN_ID,`
`423`	`419`	`)`
`424`	`420`	`worker.add(uvatwns_task_4)`
`@@ -689,7 +685,6 @@ def test_update_vat_grch37(`
`689`	`685`	`project_remap_paths=[TEST_REMAP],`
`690`	`686`	`project_pedigree_paths=[TEST_PEDIGREE_3],`
`691`	`687`	`skip_validation=True,`
`692`		`- skip_check_sex_and_relatedness=True,`
`693`	`688`	`run_id=TEST_RUN_ID,`
`694`	`689`	`)`
`695`	`690`	`worker.add(uvatwns_task)`
`@@ -769,7 +764,6 @@ def test_update_vat_without_accessing_private_datasets(`
`769`	`764`	`project_remap_paths=[TEST_REMAP],`
`770`	`765`	`project_pedigree_paths=[TEST_PEDIGREE_3],`
`771`	`766`	`skip_validation=True,`
`772`		`- skip_check_sex_and_relatedness=True,`
`773`	`767`	`run_id=TEST_RUN_ID,`
`774`	`768`	`)`
`775`	`769`	`worker.add(uvatwns_task)`
`@@ -827,7 +821,6 @@ def test_mito_update_vat(`
`827`	`821`	`project_remap_paths=['not_a_real_file'],`
`828`	`822`	`project_pedigree_paths=[TEST_PEDIGREE_5],`
`829`	`823`	`skip_validation=True,`
`830`		`- skip_check_sex_and_relatedness=True,`
`831`	`824`	`run_id=TEST_RUN_ID,`
`832`	`825`	`)`
`833`	`826`	`)`
`@@ -1092,7 +1085,6 @@ def test_sv_update_vat(`
`1092`	`1085`	`project_remap_paths=['not_a_real_file'],`
`1093`	`1086`	`project_pedigree_paths=[TEST_PEDIGREE_5],`
`1094`	`1087`	`skip_validation=True,`
`1095`		`- skip_check_sex_and_relatedness=True,`
`1096`	`1088`	`run_id=TEST_RUN_ID,`
`1097`	`1089`	`)`
`1098`	`1090`	`)`
`@@ -1654,7 +1646,6 @@ def test_gcnv_update_vat(`
`1654`	`1646`	`project_remap_paths=['not_a_real_file'],`
`1655`	`1647`	`project_pedigree_paths=[TEST_PEDIGREE_5],`
`1656`	`1648`	`skip_validation=True,`
`1657`		`- skip_check_sex_and_relatedness=True,`
`1658`	`1649`	`run_id=TEST_RUN_ID,`
`1659`	`1650`	`)`
`1660`	`1651`	`)`
Original file line number	Diff line number	Diff line change
`@@ -23,7 +23,6 @@ def test_write_metadata_for_run_task(self) -> None:`
`23`	`23`	`project_guids=['R0113_test_project', 'R0114_project4'],`
`24`	`24`	`project_remap_paths=[TEST_REMAP_2, TEST_REMAP_2],`
`25`	`25`	`project_pedigree_paths=[TEST_PEDIGREE_3, TEST_PEDIGREE_4],`
`26`		`- skip_check_sex_and_relatedness=True,`
`27`	`26`	`skip_validation=True,`
`28`	`27`	`run_id='run_123456',`
`29`	`28`	`)`