merge

Author: bpblanken

requirements-dev.txt

@@ -38,7 +38,7 @@ idna==3.6
     #   requests
 imagesize==1.4.1
     # via sphinx
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   -c requirements.txt
     #   sphinx

requirements.in

@@ -1,5 +1,6 @@
 elasticsearch==7.9.1
 google-api-python-client>=1.8.0
-hail==0.2.122
+hail==0.2.128
 luigi>=3.4.0
 gnomad==0.6.4
+google-cloud-storage>=2.14.0

requirements.txt

@@ -16,8 +16,6 @@ asttokens==2.4.1
     # via stack-data
 async-timeout==4.0.3
     # via aiohttp
-asyncinit==0.2.4
-    # via hail
 attrs==23.1.0
     # via aiohttp
 avro==1.11.3
@@ -68,7 +66,7 @@ commonmark==0.9.1
     # via rich
 contourpy==1.2.0
     # via bokeh
-cryptography==42.0.0
+cryptography==42.0.4
     # via
     #   azure-identity
     #   azure-storage-blob
@@ -119,19 +117,19 @@ google-auth-httplib2==0.1.1
     # via google-api-python-client
 google-auth-oauthlib==0.8.0
     # via hail
-google-cloud-core==2.3.3
+google-cloud-core==2.4.1
     # via google-cloud-storage
-google-cloud-storage==2.13.0
-    # via hail
+google-cloud-storage==2.14.0
+    # via -r requirements.in
 google-crc32c==1.5.0
     # via
     #   google-cloud-storage
     #   google-resumable-media
-google-resumable-media==2.6.0
+google-resumable-media==2.7.0
     # via google-cloud-storage
 googleapis-common-protos==1.61.0
     # via google-api-core
-hail==0.2.122
+hail==0.2.128
     # via -r requirements.in
 hdbscan==0.8.33
     # via gnomad
@@ -204,7 +202,7 @@ numpy==1.26.2
     #   scipy
 oauthlib==3.2.2
     # via requests-oauthlib
-orjson==3.9.10
+orjson==3.9.11
     # via hail
 packaging==23.2
     # via
@@ -254,9 +252,7 @@ pygments==2.17.2
     #   ipython
     #   rich
 pyjwt[crypto]==2.8.0
-    # via
-    #   msal
-    #   pyjwt
+    # via msal
 pyparsing==3.1.1
     # via httplib2
 pyspark==3.3.3

v03_pipeline/lib/annotations/enums.py

@@ -204,7 +204,7 @@
     'Likely_pathogenic/Likely_risk_allele',
     'Established_risk_allele',
     'Likely_risk_allele',
-    'Conflicting_interpretations_of_pathogenicity',
+    'Conflicting_classifications_of_pathogenicity',
     'Uncertain_risk_allele',
     'Uncertain_significance/Uncertain_risk_allele',
     'Uncertain_significance',

v03_pipeline/lib/annotations/fields_test.py

@@ -31,14 +31,16 @@ def setUp(self) -> None:
             ),
         )
 
+    @patch('v03_pipeline.lib.vep.validate_vep_config_reference_genome')
     @patch('v03_pipeline.lib.vep.hl.vep')
-    def test_get_formatting_fields(self, mock_vep: Mock) -> None:
+    def test_get_formatting_fields(self, mock_vep: Mock, mock_validate: Mock) -> None:
         ht = hl.read_table(TEST_COMBINED_1)
         mock_vep.return_value = ht.annotate(vep=MOCK_VEP_DATA)
+        mock_validate.return_value = None
         ht = run_vep(
             ht,
             DatasetType.SNV_INDEL,
-            None,
+            ReferenceGenome.GRCh38,
         )
         ht = ht.annotate(rsid='abcd')
         for reference_genome, expected_fields in [

v03_pipeline/lib/annotations/shared_test.py

@@ -10,8 +10,13 @@
 
 
 class SharedAnnotationsTest(unittest.TestCase):
+    @patch('v03_pipeline.lib.vep.validate_vep_config_reference_genome')
     @patch('v03_pipeline.lib.vep.hl.vep')
-    def test_sorted_transcript_consequences(self, mock_vep: Mock) -> None:
+    def test_sorted_transcript_consequences(
+        self,
+        mock_vep: Mock,
+        mock_validate: Mock,
+    ) -> None:
         ht = hl.Table.parallelize(
             [
                 {
@@ -30,10 +35,11 @@ def test_sorted_transcript_consequences(self, mock_vep: Mock) -> None:
             key=['locus', 'alleles'],
         )
         mock_vep.return_value = ht.annotate(vep=MOCK_VEP_DATA)
+        mock_validate.return_value = None
         ht = run_vep(
             ht,
             DatasetType.SNV_INDEL,
-            None,
+            ReferenceGenome.GRCh38,
         )
         ht = ht.select(
             sorted_transcript_consequences=sorted_transcript_consequences(ht),

v03_pipeline/lib/logger.py

@@ -2,7 +2,7 @@
 
 LOG_CONFIG = {
     'version': 1,
-    'disable_existing_loggers': True,
+    'disable_existing_loggers': False,
     'propagate': True,
     'formatters': {
         'default': {
@@ -22,6 +22,12 @@
             'handlers': ['default'],
             'propagate': True,
         },
+        'py4j': {
+            'level': 'CRITICAL',
+        },
+        'urllib3': {
+            'level': 'CRITICAL',
+        },
     },
 }
 

v03_pipeline/lib/model/definitions.py

@@ -60,8 +60,10 @@ def contig_recoding(self, include_mt: bool = False) -> dict[str, str]:
             },
         }[self]
 
-        if include_mt:
+        if include_mt and self == ReferenceGenome.GRCh38:
             recode.update({'MT': 'chrM'})
+        if include_mt and self == ReferenceGenome.GRCh37:
+            recode.update({'chrM': 'MT'})
 
         return recode
 

v03_pipeline/lib/model/definitions_test.py

@@ -104,7 +104,7 @@ def test_contig_recoding(self):
             'chr22': '22',
             'chrX': 'X',
             'chrY': 'Y',
-            'MT': 'chrM',
+            'chrM': 'MT',
         }
         self.assertDictEqual(
             ReferenceGenome.GRCh37.contig_recoding(include_mt=True),
@@ -136,8 +136,9 @@ def test_contig_recoding(self):
             '22': 'chr22',
             'X': 'chrX',
             'Y': 'chrY',
+            'MT': 'chrM',
         }
         self.assertDictEqual(
-            ReferenceGenome.GRCh38.contig_recoding(),
+            ReferenceGenome.GRCh38.contig_recoding(include_mt=True),
             expected_38_recode,
         )

v03_pipeline/lib/model/environment.py

@@ -18,6 +18,8 @@
     'REFERENCE_DATASETS',
     '/seqr-reference-data',
 )
+VEP_CONFIG_PATH = os.environ.get('VEP_CONFIG_PATH', None)
+VEP_CONFIG_URI = os.environ.get('VEP_CONFIG_URI', None)
 
 
 @dataclass
@@ -30,3 +32,5 @@ class Env:
     LOADING_DATASETS: str = LOADING_DATASETS
     PRIVATE_REFERENCE_DATASETS: str = PRIVATE_REFERENCE_DATASETS
     REFERENCE_DATASETS: str = REFERENCE_DATASETS
+    VEP_CONFIG_PATH: str | None = VEP_CONFIG_PATH
+    VEP_CONFIG_URI: str | None = VEP_CONFIG_URI