Remove reference data env var (#874)

bpblanken · web-flow · commit 6c7c5782783c · 2024-08-19T13:50:57.000-04:00
* Remove reference data env var

* lint

* Remove env var from unit tests
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
@@ -36,7 +36,6 @@ jobs:
         run: ruff . --output-format github
       - name: Unit Tests
         run: |
-          export REFERENCE_DATA_AUTO_UPDATE=1
           export ACCESS_PRIVATE_REFERENCE_DATASETS=1
           export PYSPARK_SUBMIT_ARGS='--driver-memory 8G pyspark-shell'
           nosetests --with-coverage --cover-package v03_pipeline/lib v03_pipeline/lib
diff --git a/v03_pipeline/lib/model/environment.py b/v03_pipeline/lib/model/environment.py
@@ -34,7 +34,6 @@
 )
 CHECK_SEX_AND_RELATEDNESS = os.environ.get('CHECK_SEX_AND_RELATEDNESS') == '1'
 EXPECT_WES_FILTERS = os.environ.get('EXPECT_WES_FILTERS') == '1'
-REFERENCE_DATA_AUTO_UPDATE = os.environ.get('REFERENCE_DATA_AUTO_UPDATE') == '1'
 SHOULD_REGISTER_ALLELES = os.environ.get('SHOULD_REGISTER_ALLELES') == '1'
 
 
@@ -51,7 +50,6 @@ class Env:
     LOADING_DATASETS: str = LOADING_DATASETS
     PRIVATE_REFERENCE_DATASETS: str = PRIVATE_REFERENCE_DATASETS
     PROJECT_ID: str | None = PROJECT_ID
-    REFERENCE_DATA_AUTO_UPDATE: bool = REFERENCE_DATA_AUTO_UPDATE
     REFERENCE_DATASETS: str = REFERENCE_DATASETS
     SHOULD_REGISTER_ALLELES: bool = SHOULD_REGISTER_ALLELES
     VEP_CONFIG_PATH: str | None = VEP_CONFIG_PATH
diff --git a/v03_pipeline/lib/tasks/base/base_update_variant_annotations_table.py b/v03_pipeline/lib/tasks/base/base_update_variant_annotations_table.py
@@ -6,15 +6,13 @@
     get_rdc_annotation_dependencies,
 )
 from v03_pipeline.lib.model import (
-    Env,
     ReferenceDatasetCollection,
 )
 from v03_pipeline.lib.paths import (
-    valid_reference_dataset_collection_path,
     variant_annotations_table_path,
 )
 from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask
-from v03_pipeline.lib.tasks.files import GCSorLocalTarget, HailTableTask
+from v03_pipeline.lib.tasks.files import GCSorLocalTarget
 from v03_pipeline.lib.tasks.reference_data.updated_reference_dataset_collection import (
     UpdatedReferenceDatasetCollectionTask,
 )
@@ -41,14 +39,6 @@ def requires(self) -> list[luigi.Task]:
                     self.dataset_type,
                     rdc,
                 )
-                if Env.REFERENCE_DATA_AUTO_UPDATE
-                else HailTableTask(
-                    valid_reference_dataset_collection_path(
-                        self.reference_genome,
-                        self.dataset_type,
-                        rdc,
-                    ),
-                )
             )
             for rdc in ReferenceDatasetCollection.for_reference_genome_dataset_type(
                 self.reference_genome,
diff --git a/v03_pipeline/lib/tasks/reference_data/updated_cached_reference_dataset_query.py b/v03_pipeline/lib/tasks/reference_data/updated_cached_reference_dataset_query.py
@@ -4,7 +4,6 @@
 from v03_pipeline.lib.logger import get_logger
 from v03_pipeline.lib.model import (
     CachedReferenceDatasetQuery,
-    Env,
     ReferenceDatasetCollection,
 )
 from v03_pipeline.lib.paths import (
@@ -64,12 +63,6 @@ def output(self) -> luigi.Target:
         )
 
     def requires(self) -> luigi.Task:
-        if Env.REFERENCE_DATA_AUTO_UPDATE and not self.crdq.query_raw_dataset:
-            return UpdatedReferenceDatasetCollectionTask(
-                self.reference_genome,
-                self.dataset_type,
-                ReferenceDatasetCollection.COMBINED,
-            )
         if self.crdq.query_raw_dataset:
             return HailTableTask(
                 get_ht_path(
@@ -78,12 +71,10 @@ def requires(self) -> luigi.Task:
                     ],
                 ),
             )
-        return HailTableTask(
-            valid_reference_dataset_collection_path(
-                self.reference_genome,
-                self.dataset_type,
-                ReferenceDatasetCollection.COMBINED,
-            ),
+        return UpdatedReferenceDatasetCollectionTask(
+            self.reference_genome,
+            self.dataset_type,
+            ReferenceDatasetCollection.COMBINED,
         )
 
     def create_table(self) -> hl.Table:
diff --git a/v03_pipeline/lib/tasks/validate_callset.py b/v03_pipeline/lib/tasks/validate_callset.py
@@ -20,7 +20,7 @@
 )
 from v03_pipeline.lib.tasks.base.base_loading_run_params import BaseLoadingRunParams
 from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask
-from v03_pipeline.lib.tasks.files import CallsetTask, GCSorLocalTarget, HailTableTask
+from v03_pipeline.lib.tasks.files import CallsetTask, GCSorLocalTarget
 from v03_pipeline.lib.tasks.reference_data.updated_cached_reference_dataset_query import (
     UpdatedCachedReferenceDatasetQuery,
 )
@@ -59,14 +59,6 @@ def requires(self) -> list[luigi.Task]:
                         UpdatedCachedReferenceDatasetQuery,
                         crdq=CachedReferenceDatasetQuery.GNOMAD_CODING_AND_NONCODING_VARIANTS,
                     )
-                    if Env.REFERENCE_DATA_AUTO_UPDATE
-                    else HailTableTask(
-                        cached_reference_dataset_query_path(
-                            self.reference_genome,
-                            self.dataset_type,
-                            CachedReferenceDatasetQuery.GNOMAD_CODING_AND_NONCODING_VARIANTS,
-                        ),
-                    ),
                 ),
             ]
         if (
diff --git a/v03_pipeline/lib/tasks/write_new_variants_table.py b/v03_pipeline/lib/tasks/write_new_variants_table.py
@@ -22,9 +22,6 @@
     load_gencode_gene_symbol_to_gene_id,
 )
 from v03_pipeline.lib.tasks.base.base_loading_run_params import BaseLoadingRunParams
-from v03_pipeline.lib.tasks.base.base_update_variant_annotations_table import (
-    BaseUpdateVariantAnnotationsTableTask,
-)
 from v03_pipeline.lib.tasks.base.base_write import BaseWriteTask
 from v03_pipeline.lib.tasks.files import GCSorLocalTarget
 from v03_pipeline.lib.tasks.reference_data.update_variant_annotations_table_with_updated_reference_dataset import (
@@ -81,20 +78,12 @@ def output(self) -> luigi.Target:
         )
 
     def requires(self) -> list[luigi.Task]:
-        if Env.REFERENCE_DATA_AUTO_UPDATE:
-            requirements = [
-                UpdateVariantAnnotationsTableWithUpdatedReferenceDataset(
-                    self.reference_genome,
-                    self.dataset_type,
-                ),
-            ]
-        else:
-            requirements = [
-                BaseUpdateVariantAnnotationsTableTask(
-                    self.reference_genome,
-                    self.dataset_type,
-                ),
-            ]
+        requirements = [
+            UpdateVariantAnnotationsTableWithUpdatedReferenceDataset(
+                self.reference_genome,
+                self.dataset_type,
+            ),
+        ]
         if self.dataset_type.has_lookup_table:
             # NB: the lookup table task has remapped and subsetted callset tasks as dependencies.
             # Also note that force is passed here,
diff --git a/v03_pipeline/lib/tasks/write_relatedness_check_table.py b/v03_pipeline/lib/tasks/write_relatedness_check_table.py
@@ -5,12 +5,11 @@
 from v03_pipeline.lib.methods.relatedness import call_relatedness
 from v03_pipeline.lib.model import CachedReferenceDatasetQuery, Env
 from v03_pipeline.lib.paths import (
-    cached_reference_dataset_query_path,
     relatedness_check_table_path,
 )
 from v03_pipeline.lib.tasks.base.base_loading_run_params import BaseLoadingRunParams
 from v03_pipeline.lib.tasks.base.base_write import BaseWriteTask
-from v03_pipeline.lib.tasks.files import GCSorLocalTarget, HailTableTask
+from v03_pipeline.lib.tasks.files import GCSorLocalTarget
 from v03_pipeline.lib.tasks.reference_data.updated_cached_reference_dataset_query import (
     UpdatedCachedReferenceDatasetQuery,
 )
@@ -40,14 +39,6 @@ def requires(self) -> luigi.Task:
                         UpdatedCachedReferenceDatasetQuery,
                         crdq=CachedReferenceDatasetQuery.GNOMAD_QC,
                     )
-                    if Env.REFERENCE_DATA_AUTO_UPDATE
-                    else HailTableTask(
-                        cached_reference_dataset_query_path(
-                            self.reference_genome,
-                            self.dataset_type,
-                            CachedReferenceDatasetQuery.GNOMAD_QC,
-                        ),
-                    )
                 ),
             ]
         return requirements

Original file line number	Diff line number	Diff line change
`@@ -6,15 +6,13 @@`
`6`	`6`	`get_rdc_annotation_dependencies,`
`7`	`7`	`)`
`8`	`8`	`from v03_pipeline.lib.model import (`
`9`		`- Env,`
`10`	`9`	`ReferenceDatasetCollection,`
`11`	`10`	`)`
`12`	`11`	`from v03_pipeline.lib.paths import (`
`13`		`- valid_reference_dataset_collection_path,`
`14`	`12`	`variant_annotations_table_path,`
`15`	`13`	`)`
`16`	`14`	`from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask`
`17`		`-from v03_pipeline.lib.tasks.files import GCSorLocalTarget, HailTableTask`
	`15`	`+from v03_pipeline.lib.tasks.files import GCSorLocalTarget`
`18`	`16`	`from v03_pipeline.lib.tasks.reference_data.updated_reference_dataset_collection import (`
`19`	`17`	`UpdatedReferenceDatasetCollectionTask,`
`20`	`18`	`)`
`@@ -41,14 +39,6 @@ def requires(self) -> list[luigi.Task]:`
`41`	`39`	`self.dataset_type,`
`42`	`40`	`rdc,`
`43`	`41`	`)`
`44`		`- if Env.REFERENCE_DATA_AUTO_UPDATE`
`45`		`- else HailTableTask(`
`46`		`- valid_reference_dataset_collection_path(`
`47`		`- self.reference_genome,`
`48`		`- self.dataset_type,`
`49`		`- rdc,`
`50`		`- ),`
`51`		`- )`
`52`	`42`	`)`
`53`	`43`	`for rdc in ReferenceDatasetCollection.for_reference_genome_dataset_type(`
`54`	`44`	`self.reference_genome,`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@`
`20`	`20`	`)`
`21`	`21`	`from v03_pipeline.lib.tasks.base.base_loading_run_params import BaseLoadingRunParams`
`22`	`22`	`from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask`
`23`		`-from v03_pipeline.lib.tasks.files import CallsetTask, GCSorLocalTarget, HailTableTask`
	`23`	`+from v03_pipeline.lib.tasks.files import CallsetTask, GCSorLocalTarget`
`24`	`24`	`from v03_pipeline.lib.tasks.reference_data.updated_cached_reference_dataset_query import (`
`25`	`25`	`UpdatedCachedReferenceDatasetQuery,`
`26`	`26`	`)`
`@@ -59,14 +59,6 @@ def requires(self) -> list[luigi.Task]:`
`59`	`59`	`UpdatedCachedReferenceDatasetQuery,`
`60`	`60`	`crdq=CachedReferenceDatasetQuery.GNOMAD_CODING_AND_NONCODING_VARIANTS,`
`61`	`61`	`)`
`62`		`- if Env.REFERENCE_DATA_AUTO_UPDATE`
`63`		`- else HailTableTask(`
`64`		`- cached_reference_dataset_query_path(`
`65`		`- self.reference_genome,`
`66`		`- self.dataset_type,`
`67`		`- CachedReferenceDatasetQuery.GNOMAD_CODING_AND_NONCODING_VARIANTS,`
`68`		`- ),`
`69`		`- ),`
`70`	`62`	`),`
`71`	`63`	`]`
`72`	`64`	`if (`