Merge pull request #711 from broadinstitute/benb/prefer_env_vars_for_vep_path

prefer env vars for vep path

Author: bpblanken

v03_pipeline/lib/annotations/fields_test.py

@@ -41,7 +41,6 @@ def test_get_formatting_fields(self, mock_vep: Mock, mock_validate: Mock) -> Non
             ht,
             DatasetType.SNV_INDEL,
             ReferenceGenome.GRCh38,
-            None,
         )
         ht = ht.annotate(rsid='abcd')
         for reference_genome, expected_fields in [

v03_pipeline/lib/annotations/shared_test.py

@@ -40,7 +40,6 @@ def test_sorted_transcript_consequences(
             ht,
             DatasetType.SNV_INDEL,
             ReferenceGenome.GRCh38,
-            None,
         )
         ht = ht.select(
             sorted_transcript_consequences=sorted_transcript_consequences(ht),

v03_pipeline/lib/model/environment.py

@@ -18,6 +18,8 @@
     'REFERENCE_DATASETS',
     '/seqr-reference-data',
 )
+VEP_CONFIG_PATH = os.environ.get('VEP_CONFIG_PATH', None)
+VEP_CONFIG_URI = os.environ.get('VEP_CONFIG_URI', None)
 
 
 @dataclass
@@ -30,3 +32,5 @@ class Env:
     LOADING_DATASETS: str = LOADING_DATASETS
     PRIVATE_REFERENCE_DATASETS: str = PRIVATE_REFERENCE_DATASETS
     REFERENCE_DATASETS: str = REFERENCE_DATASETS
+    VEP_CONFIG_PATH: str | None = VEP_CONFIG_PATH
+    VEP_CONFIG_URI: str | None = VEP_CONFIG_URI

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples.py

@@ -52,10 +52,6 @@ class UpdateVariantAnnotationsTableWithNewSamplesTask(BaseVariantAnnotationsTabl
         default='gs://hail-common/references/grch38_to_grch37.over.chain.gz',
         description='Path to GRCh38 to GRCh37 coordinates file',
     )
-    vep_config_json_path = luigi.OptionalParameter(
-        default=None,
-        description='Path of hail vep config .json file',
-    )
 
     @property
     def other_annotation_dependencies(self) -> dict[str, hl.Table]:
@@ -216,7 +212,6 @@ def update_table(self, ht: hl.Table) -> hl.Table:
             new_variants_ht,
             self.dataset_type,
             self.reference_genome,
-            self.vep_config_json_path,
         )
 
         # 2) Select down to the formatting annotations fields and

v03_pipeline/lib/vep.py

@@ -1,10 +1,10 @@
 import hail as hl
 
-from v03_pipeline.lib.model import DatasetType, ReferenceGenome
+from v03_pipeline.lib.model import DatasetType, Env, ReferenceGenome
 
 
-def validate_vep_config_reference_genome(reference_genome, config: str) -> None:
-    with open(config) as f:
+def validate_vep_config_reference_genome(reference_genome) -> None:
+    with open(Env.VEP_CONFIG_PATH) as f:
         if reference_genome.value not in f.read():
             msg = f'Vep config does not match supplied reference genome {reference_genome.value}'
             raise ValueError(msg)
@@ -14,19 +14,13 @@ def run_vep(
     ht: hl.Table,
     dataset_type: DatasetType,
     reference_genome: ReferenceGenome,
-    vep_config_json_path: str | None,
 ) -> hl.Table:
     if not dataset_type.veppable:
         return ht
-    config = (
-        vep_config_json_path
-        if vep_config_json_path is not None
-        else f'file:///vep_data/vep-{reference_genome.value}-gcloud.json'
-    )
-    validate_vep_config_reference_genome(reference_genome, config)
+    validate_vep_config_reference_genome(reference_genome)
     return hl.vep(
         ht,
-        config=config,
+        config=Env.VEP_CONFIG_URI,
         name='vep',
         block_size=1000,
         tolerate_parse_error=True,