Merge pull request #710 from broadinstitute/dev

main <-  dev

Author: bpblanken

.github/workflows/unit-tests.yml

@@ -30,6 +30,8 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
+      - name: Check Ruff Format
+        run: ruff format --check v03_pipeline --diff
       - name: Check Ruff
         run: ruff . --output-format github
       - name: Unit Tests

requirements-dev.txt

@@ -38,7 +38,7 @@ idna==3.6
     #   requests
 imagesize==1.4.1
     # via sphinx
-jinja2==3.1.2
+jinja2==3.1.3
     # via
     #   -c requirements.txt
     #   sphinx

requirements.in

@@ -1,5 +1,6 @@
 elasticsearch==7.9.1
 google-api-python-client>=1.8.0
-hail==0.2.122
+hail==0.2.128
 luigi>=3.4.0
 gnomad==0.6.4
+google-cloud-storage>=2.14.0

requirements.txt

@@ -16,8 +16,6 @@ asttokens==2.4.1
     # via stack-data
 async-timeout==4.0.3
     # via aiohttp
-asyncinit==0.2.4
-    # via hail
 attrs==23.1.0
     # via aiohttp
 avro==1.11.3
@@ -119,19 +117,19 @@ google-auth-httplib2==0.1.1
     # via google-api-python-client
 google-auth-oauthlib==0.8.0
     # via hail
-google-cloud-core==2.3.3
+google-cloud-core==2.4.1
     # via google-cloud-storage
-google-cloud-storage==2.13.0
-    # via hail
+google-cloud-storage==2.14.0
+    # via -r requirements.in
 google-crc32c==1.5.0
     # via
     #   google-cloud-storage
     #   google-resumable-media
-google-resumable-media==2.6.0
+google-resumable-media==2.7.0
     # via google-cloud-storage
 googleapis-common-protos==1.61.0
     # via google-api-core
-hail==0.2.122
+hail==0.2.128
     # via -r requirements.in
 hdbscan==0.8.33
     # via gnomad
@@ -204,7 +202,7 @@ numpy==1.26.2
     #   scipy
 oauthlib==3.2.2
     # via requests-oauthlib
-orjson==3.9.15
+orjson==3.9.11
     # via hail
 packaging==23.2
     # via
@@ -254,9 +252,7 @@ pygments==2.17.2
     #   ipython
     #   rich
 pyjwt[crypto]==2.8.0
-    # via
-    #   msal
-    #   pyjwt
+    # via msal
 pyparsing==3.1.1
     # via httplib2
 pyspark==3.3.3

v03_pipeline/lib/annotations/enums.py

@@ -204,7 +204,7 @@
     'Likely_pathogenic/Likely_risk_allele',
     'Established_risk_allele',
     'Likely_risk_allele',
-    'Conflicting_interpretations_of_pathogenicity',
+    'Conflicting_classifications_of_pathogenicity',
     'Uncertain_risk_allele',
     'Uncertain_significance/Uncertain_risk_allele',
     'Uncertain_significance',

v03_pipeline/lib/annotations/fields_test.py

@@ -31,13 +31,16 @@ def setUp(self) -> None:
             ),
         )
 
+    @patch('v03_pipeline.lib.vep.validate_vep_config_reference_genome')
     @patch('v03_pipeline.lib.vep.hl.vep')
-    def test_get_formatting_fields(self, mock_vep: Mock) -> None:
+    def test_get_formatting_fields(self, mock_vep: Mock, mock_validate: Mock) -> None:
         ht = hl.read_table(TEST_COMBINED_1)
         mock_vep.return_value = ht.annotate(vep=MOCK_VEP_DATA)
+        mock_validate.return_value = None
         ht = run_vep(
             ht,
             DatasetType.SNV_INDEL,
+            ReferenceGenome.GRCh38,
             None,
         )
         ht = ht.annotate(rsid='abcd')

v03_pipeline/lib/annotations/mito.py

@@ -44,7 +44,7 @@ def HL(mt: hl.MatrixTable, **_: Any) -> hl.Expression:  # noqa: N802
     return hl.if_else(is_called, mt.HL, 0)
 
 
-def high_constraint_region(
+def high_constraint_region_mito(
     ht: hl.Table,
     interval_ht: hl.Table,
     **_: Any,

v03_pipeline/lib/annotations/shared_test.py

@@ -10,8 +10,13 @@
 
 
 class SharedAnnotationsTest(unittest.TestCase):
+    @patch('v03_pipeline.lib.vep.validate_vep_config_reference_genome')
     @patch('v03_pipeline.lib.vep.hl.vep')
-    def test_sorted_transcript_consequences(self, mock_vep: Mock) -> None:
+    def test_sorted_transcript_consequences(
+        self,
+        mock_vep: Mock,
+        mock_validate: Mock,
+    ) -> None:
         ht = hl.Table.parallelize(
             [
                 {
@@ -30,9 +35,11 @@ def test_sorted_transcript_consequences(self, mock_vep: Mock) -> None:
             key=['locus', 'alleles'],
         )
         mock_vep.return_value = ht.annotate(vep=MOCK_VEP_DATA)
+        mock_validate.return_value = None
         ht = run_vep(
             ht,
             DatasetType.SNV_INDEL,
+            ReferenceGenome.GRCh38,
             None,
         )
         ht = ht.select(

v03_pipeline/lib/logger.py

@@ -0,0 +1,42 @@
+import logging.config
+
+LOG_CONFIG = {
+    'version': 1,
+    'disable_existing_loggers': False,
+    'propagate': True,
+    'formatters': {
+        'default': {
+            'format': '%(asctime)s - %(module)s - %(levelname)s - %(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+        },
+    },
+    'handlers': {
+        'default': {
+            'formatter': 'default',
+            'class': 'logging.StreamHandler',
+        },
+    },
+    'loggers': {
+        '': {
+            'level': 'INFO',
+            'handlers': ['default'],
+            'propagate': True,
+        },
+        'py4j': {
+            'level': 'CRITICAL',
+        },
+        'urllib3': {
+            'level': 'CRITICAL',
+        },
+    },
+}
+
+_CONFIGURED = False
+
+
+def get_logger(name: str):
+    global _CONFIGURED  # noqa: PLW0603
+    if not _CONFIGURED:
+        logging.config.dictConfig(LOG_CONFIG)
+        _CONFIGURED = True
+    return logging.getLogger(name)

v03_pipeline/lib/model/dataset_type.py

@@ -175,7 +175,7 @@ def formatting_annotation_fns(
             DatasetType.MITO: [
                 mito.common_low_heteroplasmy,
                 mito.haplogroup,
-                mito.high_constraint_region,
+                mito.high_constraint_region_mito,
                 mito.mitotip,
                 mito.rsid,
                 shared.sorted_transcript_consequences,