Merge pull request #694 from broadinstitute/dev

Dev

Author: bpblanken

.github/workflows/unit-tests.yml

@@ -34,6 +34,7 @@ jobs:
         run: ruff . --output-format github
       - name: Unit Tests
         run: |
+          export REFERENCE_DATA_AUTO_UPDATE=1
           export ACCESS_PRIVATE_REFERENCE_DATASETS=1
           export PYSPARK_SUBMIT_ARGS='--driver-memory 8G pyspark-shell'
           nosetests --with-coverage --cover-package v03_pipeline/lib v03_pipeline/lib

v03_pipeline/bin/write_combined_interval_reference_ht.py

@@ -79,9 +79,13 @@ def gt_stats(
         ref_samples_length = row.ref_samples[project_guid].length()
         heteroplasmic_samples_length = row.heteroplasmic_samples[project_guid].length()
         homoplasmic_samples_length = row.homoplasmic_samples[project_guid].length()
-        AC_het += heteroplasmic_samples_length # noqa: N806
-        AC_hom += homoplasmic_samples_length # noqa: N806
-        AN += (ref_samples_length + heteroplasmic_samples_length + homoplasmic_samples_length) # noqa: N806
+        AC_het += heteroplasmic_samples_length  # noqa: N806
+        AC_hom += homoplasmic_samples_length  # noqa: N806
+        AN += (  # noqa: N806
+            ref_samples_length
+            + heteroplasmic_samples_length
+            + homoplasmic_samples_length
+        )
     return hl.Struct(
         AC_het=AC_het,
         AF_het=hl.float32(AC_het / AN),

v03_pipeline/bin/write_combined_reference_ht.py

@@ -25,9 +25,21 @@ def test_allele_count_annotations(self) -> None:
             [
                 {
                     'id': 0,
-                    'ref_samples': hl.Struct(project_1={'a', 'c'}, project_2=set(), R0607_gregor_training_project_=set()),
-                    'het_samples': hl.Struct(project_1={'b', 'd'}, project_2=set(), R0607_gregor_training_project_=set()),
-                    'hom_samples': hl.Struct(project_1={'e', 'f'}, project_2=set(), R0607_gregor_training_project_={'l', 'm'}),
+                    'ref_samples': hl.Struct(
+                        project_1={'a', 'c'},
+                        project_2=set(),
+                        R0607_gregor_training_project_=set(),
+                    ),
+                    'het_samples': hl.Struct(
+                        project_1={'b', 'd'},
+                        project_2=set(),
+                        R0607_gregor_training_project_=set(),
+                    ),
+                    'hom_samples': hl.Struct(
+                        project_1={'e', 'f'},
+                        project_2=set(),
+                        R0607_gregor_training_project_={'l', 'm'},
+                    ),
                 },
                 {
                     'id': 1,
@@ -36,8 +48,16 @@ def test_allele_count_annotations(self) -> None:
                         project_2=set(),
                         R0607_gregor_training_project_={'l', 'm'},
                     ),
-                    'het_samples': hl.Struct(project_1=set(), project_2=set(), R0607_gregor_training_project_=set()),
-                    'hom_samples': hl.Struct(project_1=set(), project_2=set(), R0607_gregor_training_project_=set()),
+                    'het_samples': hl.Struct(
+                        project_1=set(),
+                        project_2=set(),
+                        R0607_gregor_training_project_=set(),
+                    ),
+                    'hom_samples': hl.Struct(
+                        project_1=set(),
+                        project_2=set(),
+                        R0607_gregor_training_project_=set(),
+                    ),
                 },
             ],
             hl.tstruct(