merge

Author: bpblanken

.github/workflows/unit-tests.yml

@@ -30,15 +30,11 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
-      - name: Check Import Sort
-        run: isort . --check --diff
-      - name: Check Black
-        run: black . --check --diff
       - name: Check Ruff
         run: ruff . --output-format github
       - name: Unit Tests
         run: |
-          export ACCESS_PRIVATE_DATASETS=1
+          export ACCESS_PRIVATE_REFERENCE_DATASETS=1
           export PYSPARK_SUBMIT_ARGS='--driver-memory 8G pyspark-shell'
           nosetests --with-coverage --cover-package v03_pipeline/lib v03_pipeline/lib
           coverage report --omit '*test*' --fail-under=75

.gitignore

@@ -31,7 +31,6 @@ downloads/
 eggs/
 .eggs/
 lib64/
-parts/
 sdist/
 wheels/
 share/python-wheels/

download_and_create_reference_datasets/v02/hail_scripts/write_clinvar_ht.py

@@ -1,18 +1,20 @@
 import tempfile
 
 import hail as hl
+
+from v03_pipeline.lib.model import ReferenceGenome
 from v03_pipeline.lib.reference_data.clinvar import (
     download_and_import_latest_clinvar_vcf,
     CLINVAR_GOLD_STARS_LOOKUP,
 )
 from hail_scripts.utils.hail_utils import write_ht
 
-CLINVAR_PATH = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh{genome_version}/clinvar.vcf.gz'
-CLINVAR_HT_PATH = 'gs://seqr-reference-data/GRCh{genome_version}/clinvar/clinvar.GRCh{genome_version}.ht'
+CLINVAR_PATH = 'ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_{reference_genome}/clinvar.vcf.gz'
+CLINVAR_HT_PATH = 'gs://seqr-reference-data/{reference_genome}/clinvar/clinvar.{reference_genome}.ht'
 
-for genome_version in ["37", "38"]:
-    clinvar_url = CLINVAR_PATH.format(genome_version=genome_version)
-    ht = download_and_import_latest_clinvar_vcf(clinvar_url, genome_version)
+for reference_genome in ReferenceGenome:
+    clinvar_url = CLINVAR_PATH.format(reference_genome=reference_genome.value)
+    ht = download_and_import_latest_clinvar_vcf(clinvar_url, reference_genome)
     timestamp = hl.eval(ht.version)
     ht = ht.annotate(
         gold_stars=CLINVAR_GOLD_STARS_LOOKUP.get(hl.delimit(ht.info.CLNREVSTAT))
@@ -21,7 +23,7 @@
     ht = ht.repartition(100)
     write_ht(
         ht,
-        CLINVAR_HT_PATH.format(genome_version=genome_version).replace(".ht", ".")
+        CLINVAR_HT_PATH.format(reference_genome=reference_genome.value).replace(".ht", ".")
         + timestamp
         + ".ht",
     )

download_and_create_reference_datasets/v02/hail_scripts/write_combined_interval_ref_data.py

@@ -3,7 +3,7 @@
 
 import hail as hl
 
-from v03_pipeline.lib.reference_data.combine import join_hts
+from v03_pipeline.lib.reference_data.dataset_table_operations import join_hts
 
 VERSION = '2.0.5'
 OUTPUT_PATH = "gs://seqr-reference-data/GRCh38/combined_interval_reference_data/combined_interval_reference_data.ht"

download_and_create_reference_datasets/v02/hail_scripts/write_combined_reference_data_ht.py

@@ -3,7 +3,7 @@
 
 import hail as hl
 
-from v03_pipeline.lib.reference_data.combine import join_hts
+from v03_pipeline.lib.reference_data.dataset_table_operations import join_hts
 from v03_pipeline.lib.reference_data.config import CONFIG
 
 VERSION = '2.0.4'

download_and_create_reference_datasets/v02/hail_scripts/write_dbnsfp_ht.py

@@ -5,12 +5,12 @@
     '2.9.3': {
         'reference_genome': '37',
         'source_path': 'gs://seqr-reference-data/GRCh37/dbNSFP/v2.9.3/dbNSFP2.9.3_variant.chr*.gz',
-        'output_path': 'gs://seqr-reference-data/GRCh37/dbNSFP/v2.9.3/dbNSFP2.9.3_variant.ht',
+        'output_path': 'gs://seqr-reference-data/GRCh37/dbNSFP/v2.9.3/dbNSFP2.9.3_variant.with_new_scores.ht',
     },
     '4.2': {
         'reference_genome': '38',
         'source_path': 'gs://seqr-reference-data/GRCh38/dbNSFP/v4.2/dbNSFP4.2a_variant.chr*.gz',
-        'output_path': 'gs://seqr-reference-data/GRCh38/dbNSFP/v4.2/dbNSFP4.2a_variant.ht',
+        'output_path': 'gs://seqr-reference-data/GRCh38/dbNSFP/v4.2/dbNSFP4.2a_variant.with_new_scores.ht',
     },
 }
 
@@ -21,9 +21,9 @@
         'pos(1-coor)': tint,
         'ref': tstr,
         'alt': tstr,
-        'SIFT_pred': tstr,
+        'SIFT_score': tstr,
         'Polyphen2_HDIV_pred': tstr,
-        'Polyphen2_HVAR_pred': tstr,
+        'Polyphen2_HVAR_score': tstr,
         'LRT_pred': tstr,
         'MutationTaster_pred': tstr,
         'MutationAssessor_pred': tstr,
@@ -68,16 +68,16 @@
         'pos(1-based)': tint,
         'ref': tstr,
         'alt': tstr,
-        'SIFT_pred': tstr,
-        'Polyphen2_HVAR_pred': tstr,
+        'SIFT_score': tstr,
+        'Polyphen2_HVAR_score': tstr,
         'MutationTaster_pred': tstr,
         'FATHMM_pred': tstr,
         'VEST4_score': tstr,
         'MetaSVM_pred': tstr,
         'REVEL_score': tstr,
         'GERP++_RS': tstr,
         'phastCons100way_vertebrate': tstr,
-        'fathmm-MKL_coding_pred': tstr,
+        'fathmm-MKL_coding_score': tfloat,
         'MutPred_score': tstr,
     }
 }

download_and_create_reference_datasets/v02/mito/write_combined_mito_reference_data_hts.py

@@ -4,7 +4,7 @@
 
 import hail as hl
 
-from v03_pipeline.lib.reference_data.combine import join_hts
+from v03_pipeline.lib.reference_data.dataset_table_operations import join_hts
 
 VERSION = '2.0.4'
 OUTPUT_PATH = 'gs://seqr-reference-data/GRCh38/mitochondrial/all_mito_reference_data/combined_reference_data_chrM.ht'

luigi_pipeline/tests/test_hail_tasks.py

@@ -40,12 +40,10 @@ def _hail_matrix_table_task(self):
 
     def _set_validation_configs(self):
         global_config = GlobalConfig()
-        global_config.param_kwargs[
-            'validation_37_coding_ht'
-        ] = global_config.validation_37_coding_ht = 'tests/data/validation_37_coding.ht'
-        global_config.param_kwargs[
-            'validation_37_noncoding_ht'
-        ] = (
+        global_config.param_kwargs['validation_37_coding_ht'] = (
+            global_config.validation_37_coding_ht
+        ) = 'tests/data/validation_37_coding.ht'
+        global_config.param_kwargs['validation_37_noncoding_ht'] = (
             global_config.validation_37_noncoding_ht
         ) = 'tests/data/validation_37_noncoding.ht'
 

pyproject.toml

@@ -14,40 +14,6 @@ include = ["v03_pipeline*"]
 exclude = ["v03_pipeline.bin", "v03_pipeline*test*"]
 namespaces = false
 
-[tool.black]
-line-length = 88
-skip-string-normalization = true
-extend-exclude = '''
-(
-    download_and_create_reference_datasets/v02
-    | gcloud_dataproc
-    | hail_scripts/computed_fields
-    | hail_scripts/elasticsearch
-    | hail_scripts/update_models
-    | hail_scripts/utils
-    | kubernetes
-    | luigi_pipeline/lib
-    | luigi_pipeline/seqr.*\.py$
-    | luigi_pipeline/tests/data
-)
-'''
-
-[tool.isort]
-include_trailing_comma = 'True'
-known_external_package = ['hail_scripts']
-line_length = 88
-profile = 'black'
-sections = ['FUTURE', 'STDLIB', 'THIRDPARTY', 'EXTERNAL_PACKAGE', 'FIRSTPARTY', 'LOCALFOLDER']
-skip_glob = [
-    'download_and_create_reference_datasets/v02/*',
-    'gcloud_dataproc/*',
-    'hail_scripts/computed_fields/*',
-    'hail_scripts/elasticsearch/*',
-    'hail_scripts/update_models/*',
-    'hail_scripts/utils/*',
-    'kubernetes/*'
-]
-
 [tool.mypy]
 packages = 'v03_pipeline'
 python_version = "3.10"
@@ -71,7 +37,6 @@ ignore = [
     "G004",  # logging-f-string, these are fine for now
 
     # Rule Groupings
-    "I",     # isort is enabled so this is unnecessary (for now, maybe we want to disable isort)
     "D",     # pydocstyle is for docs... we have none
     "FBT",   # flake-boolean-trap... disallows boolean args to functions... fixing this code will require refactors.
     "ANN",   # flake8-annotations is for typed code
@@ -102,3 +67,6 @@ inline-quotes = "single"
 
 [tool.ruff.pylint]
 max-args = 6
+
+[tool.ruff.format]
+quote-style = "single"

requirements-dev.in

@@ -1,9 +1,7 @@
 -c requirements.txt
-black>=23.7.0
 coverage>=7.2.2
-isort>=5.12.0
 mypy>=1.2.0
 nose-py3>=1.6.3 # bug in regular nose here: https://github.com/nose-devs/nose/issues/1099#issuecomment-577647469
 pip-tools>=6.12.3
 responses>=0.23.1
-ruff>=0.0.259
+ruff>=0.1.8