Run ruff format during CI. (#697)

* Run ruff format

* commas

* one more format

* Delete v03_pipeline/lib/tasks/reference_data/updated_cached_reference_dataset_query_test.py

---------

Co-authored-by: Julia Klugherz <juliaklugherz@gmail.com>

Author: bpblanken

.github/workflows/unit-tests.yml

@@ -30,6 +30,8 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
           pip install -r requirements-dev.txt
+      - name: Check Ruff Format
+        run: ruff format --check v03_pipeline --diff
       - name: Check Ruff
         run: ruff . --output-format github
       - name: Unit Tests

v03_pipeline/lib/logger.py

@@ -27,8 +27,9 @@
 
 _CONFIGURED = False
 
+
 def get_logger(name: str):
-    global _CONFIGURED # noqa: PLW0603
+    global _CONFIGURED  # noqa: PLW0603
     if not _CONFIGURED:
         logging.config.dictConfig(LOG_CONFIG)
         _CONFIGURED = True

v03_pipeline/lib/reference_data/clinvar.py

@@ -38,6 +38,7 @@
 
 logger = get_logger(__name__)
 
+
 def safely_move_to_gcs(tmp_file_name, gcs_tmp_file_name):
     try:
         subprocess.run(
@@ -106,7 +107,10 @@ def download_and_import_latest_clinvar_vcf(
 
     with tempfile.NamedTemporaryFile(suffix='.vcf.gz', delete=False) as tmp_file:
         urllib.request.urlretrieve(clinvar_url, tmp_file.name)  # noqa: S310
-        gcs_tmp_file_name = os.path.join(Env.HAIL_TMPDIR, os.path.basename(tmp_file.name))
+        gcs_tmp_file_name = os.path.join(
+            Env.HAIL_TMPDIR,
+            os.path.basename(tmp_file.name),
+        )
         safely_move_to_gcs(tmp_file.name, gcs_tmp_file_name)
         mt = hl.import_vcf(
             gcs_tmp_file_name,

v03_pipeline/lib/reference_data/compare_globals.py

@@ -51,8 +51,12 @@ def from_dataset_configs(
                 ),
             )
             enums[dataset] = dataset_config.get('enum_select', {})
-            dataset_ht = dataset_ht.select(**get_all_select_fields(dataset_ht, dataset_config))
-            dataset_ht = dataset_ht.transmute(**get_enum_select_fields(dataset_ht, dataset_config))
+            dataset_ht = dataset_ht.select(
+                **get_all_select_fields(dataset_ht, dataset_config),
+            )
+            dataset_ht = dataset_ht.transmute(
+                **get_enum_select_fields(dataset_ht, dataset_config),
+            )
             selects[dataset] = set(dataset_ht.row) - set(dataset_ht.key)
         return cls(paths, versions, enums, selects)
 
@@ -67,13 +71,17 @@ def from_ht(
         paths = dict(rdc_globals_struct.paths)
         versions = dict(rdc_globals_struct.versions)
         # enums are nested structs
-        enums = {k: dict(v) for k,v in rdc_globals_struct.enums.items()}
+        enums = {k: dict(v) for k, v in rdc_globals_struct.enums.items()}
 
         selects = {}
         for dataset in rdc.datasets(dataset_type):
             if dataset in ht.row:
                 # NB: handle an edge case (mito high constraint) where we annotate a bool from the reference dataset collection
-                selects[dataset] = set(ht[dataset]) if isinstance(ht[dataset], hl.StructExpression) else set()
+                selects[dataset] = (
+                    set(ht[dataset])
+                    if isinstance(ht[dataset], hl.StructExpression)
+                    else set()
+                )
         return cls(paths, versions, enums, selects)
 
 

v03_pipeline/lib/reference_data/compare_globals_test.py

@@ -22,15 +22,21 @@ class CompareGlobalsTest(unittest.TestCase):
                 '38': {
                     'custom_import': None,
                     'source_path': 'a_path',  # 'a' has a custom import
-                    'select': {'test_select': 'info.test_select', 'test_enum': 'test_enum'},
+                    'select': {
+                        'test_select': 'info.test_select',
+                        'test_enum': 'test_enum',
+                    },
                     'version': 'a_version',
                     'enum_select': {'test_enum': ['A', 'B']},
                 },
             },
             'b': {  # b is missing version
                 '38': {
                     'path': 'b_path',
-                    'select': {'test_select': 'info.test_select', 'test_enum': 'test_enum'},
+                    'select': {
+                        'test_select': 'info.test_select',
+                        'test_enum': 'test_enum',
+                    },
                     'enum_select': {'test_enum': ['C', 'D']},
                     'custom_select': lambda ht: {'field2': ht.info.test_select_2},
                 },
@@ -93,7 +99,10 @@ def test_create_globals_from_dataset_ht_configs(
         )
         self.assertTrue(
             dataset_config_globals.selects
-            == {'a': {'test_select', 'test_enum_id'}, 'b': {'test_select', 'field2', 'test_enum_id'}},
+            == {
+                'a': {'test_select', 'test_enum_id'},
+                'b': {'test_select', 'field2', 'test_enum_id'},
+            },
         )
 
     def test_from_rdc_or_annotations_ht(self):

v03_pipeline/lib/reference_data/dataset_table_operations_test.py

@@ -583,4 +583,3 @@ def test_update_or_create_joined_ht_drop_a_dataset(
                 ),
             ],
         )
-

v03_pipeline/lib/tasks/base/base_hail_table_task.py

@@ -7,6 +7,7 @@
 
 logger = get_logger(__name__)
 
+
 class BaseHailTableTask(luigi.Task):
     reference_genome = luigi.EnumParameter(enum=ReferenceGenome)
     dataset_type = luigi.EnumParameter(enum=DatasetType)
@@ -25,24 +26,29 @@ def init_hail(self):
         # Interval ref data join causes shuffle death, this prevents it
         hl._set_flags(use_new_shuffle='1', no_whole_stage_codegen='1')  # noqa: SLF001
 
+
 # NB: these are defined over luigi.Task instead of the BaseHailTableTask so that
 # they work on file dependencies.
 @luigi.Task.event_handler(luigi.Event.DEPENDENCY_MISSING)
 def dependency_missing(task):
     logger.info(f'{task} dependency_missing')
 
+
 @luigi.Task.event_handler(luigi.Event.DEPENDENCY_PRESENT)
 def dependency_present(task):
     logger.info(f'{task} dependency_present')
 
+
 @luigi.Task.event_handler(luigi.Event.START)
 def start(task):
     logger.info(f'{task} start')
 
+
 @luigi.Task.event_handler(luigi.Event.FAILURE)
 def failure(task, _):
     logger.exception(f'{task} failure')
 
+
 @luigi.Task.event_handler(luigi.Event.SUCCESS)
 def success(task):
     logger.info(f'{task} success')

v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset.py

@@ -14,7 +14,6 @@
 class UpdateVariantAnnotationsTableWithUpdatedReferenceDataset(
     BaseVariantAnnotationsTableTask,
 ):
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._datasets_to_update = []
@@ -66,7 +65,13 @@ def update_table(self, ht: hl.Table) -> hl.Table:
             if dataset in ht.row:
                 ht = ht.drop(dataset)
             if rdc.requires_annotation:
-                formatting_fn = next(x for x in self.dataset_type.formatting_annotation_fns(self.reference_genome) if x.__name__ == dataset)
+                formatting_fn = next(
+                    x
+                    for x in self.dataset_type.formatting_annotation_fns(
+                        self.reference_genome,
+                    )
+                    if x.__name__ == dataset
+                )
                 ht = ht.annotate(
                     **get_fields(
                         ht,

v03_pipeline/lib/tasks/reference_data/updated_reference_dataset_collection.py

@@ -54,7 +54,9 @@ def complete(self) -> bool:
                 self.dataset_type,
             ),
         )
-        logger.info(f'Datasets to update: {self._datasets_to_update} for {self.reference_dataset_collection}')
+        logger.info(
+            f'Datasets to update: {self._datasets_to_update} for {self.reference_dataset_collection}',
+        )
         return not self._datasets_to_update
 
     def output(self) -> luigi.Target: