Bugfix to ensure the validation/gnomad_qc table is updated before being used, + a unit test! (#783)

* add unit test for crdq updating

* closer

* fix the mock

* lint

* mock the task

* Lint

* rename test

Author: bpblanken

luigi_pipeline/tests/test_seqr_loading_tasks.py

@@ -14,7 +14,7 @@ def setUp(self):
         # Create a temporary directory
         self.test_mt = hl.import_vcf(TEST_DATA_MT_1KG)
 
-    def _sample_type_stats_return_value(  # noqa: PLR0913
+    def _sample_type_stats_return_value(
         self,
         nc_match_count,
         nc_total_count,

pyproject.toml

@@ -63,6 +63,7 @@ inline-quotes = "single"
 '*test*' = [
     'ARG002', # allow unused method arguments
     'SLF001', # allow private access
+    'PLR0913', # allow high arity functions
 ]
 
 [tool.ruff.pylint]

v03_pipeline/lib/model/cached_reference_dataset_query.py

@@ -22,8 +22,6 @@ class CachedReferenceDatasetQuery(Enum):
 
     @property
     def access_control(self) -> AccessControl:
-        if self == CachedReferenceDatasetQuery.GNOMAD_QC:
-            return AccessControl.PRIVATE
         return AccessControl.PUBLIC
 
     def dataset(self, dataset_type: DatasetType) -> str | None:

v03_pipeline/lib/reference_data/config.py

@@ -386,8 +386,8 @@ def custom_mpc_select(ht):
             'source_path': 'gs://seqr-reference-data/gnomad_qc/GRCh37/gnomad.joint.high_callrate_common_biallelic_snps.pruned.mt',
         },
         '38': {
-            'version': 'v4.0',
-            'source_path': 'gs://gcp-public-data--gnomad/release/4.0/pca/gnomad.v4.0.pca_loadings.ht',
+            'version': '4.0',
+            'path': 'gs://gcp-public-data--gnomad/release/4.0/pca/gnomad.v4.0.pca_loadings.ht',
         },
     },
     'exac': {

v03_pipeline/lib/tasks/__init__.py

@@ -1,5 +1,5 @@
-from v03_pipeline.lib.tasks.reference_data.write_cached_reference_dataset_query import (
-    WriteCachedReferenceDatasetQuery,
+from v03_pipeline.lib.tasks.reference_data.update_cached_reference_dataset_queries import (
+    UpdateCachedReferenceDatasetQueries,
 )
 from v03_pipeline.lib.tasks.update_lookup_table import (
     UpdateLookupTableTask,
@@ -33,7 +33,7 @@
     'UpdateVariantAnnotationsTableWithNewSamplesTask',
     'UpdateVariantAnnotationsTableWithDeletedProjectTask',
     'UpdateVariantAnnotationsTableWithDeletedFamiliesTask',
-    'WriteCachedReferenceDatasetQuery',
+    'UpdateCachedReferenceDatasetQueries',
     'WriteMetadataForRunTask',
     'WriteProjectFamilyTablesTask',
 ]

v03_pipeline/lib/tasks/reference_data/write_cached_reference_dataset_query.py renamed to v03_pipeline/lib/tasks/reference_data/update_cached_reference_dataset_queries.py

@@ -11,7 +11,7 @@
 )
 
 
-class WriteCachedReferenceDatasetQuery(luigi.Task):
+class UpdateCachedReferenceDatasetQueries(luigi.Task):
     reference_genome = luigi.EnumParameter(enum=ReferenceGenome)
     dataset_type = luigi.EnumParameter(enum=DatasetType)
     sample_type = luigi.EnumParameter(enum=SampleType)

v03_pipeline/lib/tasks/reference_data/write_cached_reference_dataset_query_test.py renamed to v03_pipeline/lib/tasks/reference_data/update_cached_reference_dataset_queries_test.py

@@ -9,20 +9,20 @@
     ReferenceGenome,
     SampleType,
 )
-from v03_pipeline.lib.tasks.reference_data.write_cached_reference_dataset_query import (
-    WriteCachedReferenceDatasetQuery,
+from v03_pipeline.lib.tasks.reference_data.update_cached_reference_dataset_queries import (
+    UpdateCachedReferenceDatasetQueries,
 )
 from v03_pipeline.lib.test.mock_complete_task import MockCompleteTask
 
 
 @mock.patch(
-    'v03_pipeline.lib.tasks.reference_data.write_cached_reference_dataset_query.UpdatedCachedReferenceDatasetQuery',
+    'v03_pipeline.lib.tasks.reference_data.update_cached_reference_dataset_queries.UpdatedCachedReferenceDatasetQuery',
 )
-class WriteCachedReferenceDatasetQueryTest(unittest.TestCase):
+class UpdateCachedReferenceDatasetQueriesTest(unittest.TestCase):
     def test_37_snv_indel(self, mock_crdq_task):
         mock_crdq_task.return_value = MockCompleteTask()
         worker = luigi.worker.Worker()
-        task = WriteCachedReferenceDatasetQuery(
+        task = UpdateCachedReferenceDatasetQueries(
             reference_genome=ReferenceGenome.GRCh37,
             dataset_type=DatasetType.SNV_INDEL,
             sample_type=SampleType.WGS,
@@ -62,7 +62,7 @@ def test_37_snv_indel(self, mock_crdq_task):
     def test_38_snv_indel(self, mock_crdq_task):
         mock_crdq_task.return_value = MockCompleteTask()
         worker = luigi.worker.Worker()
-        task = WriteCachedReferenceDatasetQuery(
+        task = UpdateCachedReferenceDatasetQueries(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
             sample_type=SampleType.WGS,
@@ -102,7 +102,7 @@ def test_38_snv_indel(self, mock_crdq_task):
     def test_38_mito(self, mock_crdq_task):
         mock_crdq_task.return_value = MockCompleteTask()
         worker = luigi.worker.Worker()
-        task = WriteCachedReferenceDatasetQuery(
+        task = UpdateCachedReferenceDatasetQueries(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.MITO,
             sample_type=SampleType.WGS,
@@ -124,7 +124,7 @@ def test_38_mito(self, mock_crdq_task):
     def test_38_sv(self, mock_crdq_task):
         mock_crdq_task.return_value = MockCompleteTask()
         worker = luigi.worker.Worker()
-        task = WriteCachedReferenceDatasetQuery(
+        task = UpdateCachedReferenceDatasetQueries(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SV,
             sample_type=SampleType.WGS,

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py

@@ -202,6 +202,9 @@ def test_missing_interval_reference(
         worker.run()
         self.assertFalse(uvatwns_task.complete())
 
+    @patch(
+        'v03_pipeline.lib.tasks.write_imported_callset.UpdatedCachedReferenceDatasetQuery',
+    )
     @patch(
         'v03_pipeline.lib.tasks.write_new_variants_table.UpdateVariantAnnotationsTableWithUpdatedReferenceDataset',
     )
@@ -219,7 +222,9 @@ def test_multiple_update_vat(
         mock_standard_contigs: Mock,
         mock_update_vat_with_rdc_task: Mock,
         mock_update_rdc_task: Mock,
+        mock_updated_cached_reference_dataset_query,
     ) -> None:
+        mock_updated_cached_reference_dataset_query.return_value = MockCompleteTask()
         mock_update_rdc_task.return_value = MockCompleteTask()
         mock_update_vat_with_rdc_task.return_value = (
             BaseUpdateVariantAnnotationsTableTask(
@@ -233,6 +238,7 @@ def test_multiple_update_vat(
         mock_standard_contigs.return_value = {'chr1'}
         # This creates a mock validation table with 1 coding and 1 non-coding variant
         # explicitly chosen from the VCF.
+        # NB: This is the one and only place validation is enabled in the task tests!
         coding_and_noncoding_variants_ht = hl.Table.parallelize(
             [
                 {
@@ -260,6 +266,17 @@ def test_multiple_update_vat(
                 noncoding=hl.tbool,
             ),
             key='locus',
+            globals=hl.Struct(
+                paths=hl.Struct(
+                    gnomad_genomes='gs://gcp-public-data--gnomad/release/4.1/ht/genomes/gnomad.genomes.v4.1.sites.ht',
+                ),
+                versions=hl.Struct(
+                    gnomad_genomes='4.1',
+                ),
+                enums=hl.Struct(
+                    gnomad_genomes=hl.Struct(),
+                ),
+            ),
         )
         coding_and_noncoding_variants_ht.write(
             valid_cached_reference_dataset_query_path(

v03_pipeline/lib/tasks/write_imported_callset.py

@@ -14,13 +14,17 @@
 )
 from v03_pipeline.lib.misc.vets import annotate_vets
 from v03_pipeline.lib.model import CachedReferenceDatasetQuery
+from v03_pipeline.lib.model.environment import Env
 from v03_pipeline.lib.paths import (
     imported_callset_path,
     sex_check_table_path,
     valid_cached_reference_dataset_query_path,
 )
 from v03_pipeline.lib.tasks.base.base_write import BaseWriteTask
 from v03_pipeline.lib.tasks.files import CallsetTask, GCSorLocalTarget, HailTableTask
+from v03_pipeline.lib.tasks.reference_data.updated_cached_reference_dataset_query import (
+    UpdatedCachedReferenceDatasetQuery,
+)
 from v03_pipeline.lib.tasks.write_sex_check_table import WriteSexCheckTableTask
 
 
@@ -71,11 +75,20 @@ def requires(self) -> list[luigi.Task]:
         if self.validate and self.dataset_type.can_run_validation:
             requirements = [
                 *requirements,
-                HailTableTask(
-                    valid_cached_reference_dataset_query_path(
-                        self.reference_genome,
-                        self.dataset_type,
-                        CachedReferenceDatasetQuery.GNOMAD_CODING_AND_NONCODING_VARIANTS,
+                (
+                    UpdatedCachedReferenceDatasetQuery(
+                        reference_genome=self.reference_genome,
+                        dataset_type=self.dataset_type,
+                        sample_type=self.sample_type,
+                        crdq=CachedReferenceDatasetQuery.GNOMAD_CODING_AND_NONCODING_VARIANTS,
+                    )
+                    if Env.REFERENCE_DATA_AUTO_UPDATE
+                    else HailTableTask(
+                        valid_cached_reference_dataset_query_path(
+                            self.reference_genome,
+                            self.dataset_type,
+                            CachedReferenceDatasetQuery.GNOMAD_CODING_AND_NONCODING_VARIANTS,
+                        ),
                     ),
                 ),
             ]

v03_pipeline/lib/tasks/write_relatedness_check_table.py

@@ -9,6 +9,9 @@
 )
 from v03_pipeline.lib.tasks.base.base_write import BaseWriteTask
 from v03_pipeline.lib.tasks.files import GCSorLocalTarget, HailTableTask
+from v03_pipeline.lib.tasks.reference_data.updated_cached_reference_dataset_query import (
+    UpdatedCachedReferenceDatasetQuery,
+)
 from v03_pipeline.lib.tasks.write_imported_callset import WriteImportedCallsetTask
 
 
@@ -36,12 +39,21 @@ def requires(self) -> luigi.Task:
         if Env.ACCESS_PRIVATE_REFERENCE_DATASETS:
             requirements = [
                 *requirements,
-                HailTableTask(
-                    valid_cached_reference_dataset_query_path(
-                        self.reference_genome,
-                        self.dataset_type,
-                        CachedReferenceDatasetQuery.GNOMAD_QC,
-                    ),
+                (
+                    UpdatedCachedReferenceDatasetQuery(
+                        reference_genome=self.reference_genome,
+                        dataset_type=self.dataset_type,
+                        sample_type=self.sample_type,
+                        crdq=CachedReferenceDatasetQuery.GNOMAD_QC,
+                    )
+                    if Env.REFERENCE_DATA_AUTO_UPDATE
+                    else HailTableTask(
+                        valid_cached_reference_dataset_query_path(
+                            self.reference_genome,
+                            self.dataset_type,
+                            CachedReferenceDatasetQuery.GNOMAD_QC,
+                        ),
+                    )
                 ),
             ]
         return requirements

v03_pipeline/lib/tasks/write_relatedness_check_table_test.py

@@ -0,0 +1,112 @@
+import shutil
+from unittest import mock
+
+import hail as hl
+import luigi.worker
+
+from v03_pipeline.lib.misc.io import import_vcf
+from v03_pipeline.lib.model import (
+    CachedReferenceDatasetQuery,
+    DatasetType,
+    ReferenceGenome,
+    SampleType,
+)
+from v03_pipeline.lib.paths import (
+    imported_callset_path,
+    relatedness_check_table_path,
+    valid_cached_reference_dataset_query_path,
+)
+from v03_pipeline.lib.tasks.write_relatedness_check_table import (
+    WriteRelatednessCheckTableTask,
+)
+from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
+
+TEST_GNOMAD_QC_HT = 'v03_pipeline/var/test/reference_data/gnomad_qc_crdq.ht'
+TEST_VCF = 'v03_pipeline/var/test/callsets/1kg_30variants.vcf'
+
+MOCK_CONFIG = {
+    'gnomad_qc': {
+        '38': {
+            'version': '4.0',
+            'source_path': TEST_GNOMAD_QC_HT,
+            'custom_import': lambda *_: hl.Table.parallelize(
+                [],
+                hl.tstruct(
+                    locus=hl.tlocus('GRCh38'),
+                    alleles=hl.tarray(hl.tstr),
+                ),
+                key=['locus', 'alleles'],
+            ),
+        },
+    },
+}
+
+
+class WriteRelatednessCheckTableTaskTest(MockedDatarootTestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        self.gnomad_qc_path = valid_cached_reference_dataset_query_path(
+            ReferenceGenome.GRCh38,
+            DatasetType.SNV_INDEL,
+            CachedReferenceDatasetQuery.GNOMAD_QC,
+        )
+        shutil.copytree(
+            TEST_GNOMAD_QC_HT,
+            self.gnomad_qc_path,
+        )
+
+        # Force imported callset to be complete
+        ht = import_vcf(TEST_VCF, ReferenceGenome.GRCh38)
+        ht = ht.annotate_globals(sample_type=SampleType.WGS.value)
+        ht.write(
+            imported_callset_path(
+                ReferenceGenome.GRCh38,
+                DatasetType.SNV_INDEL,
+                TEST_VCF,
+            ),
+        )
+
+    @mock.patch.dict(
+        'v03_pipeline.lib.reference_data.compare_globals.CONFIG',
+        MOCK_CONFIG,
+    )
+    @mock.patch.dict(
+        'v03_pipeline.lib.tasks.reference_data.updated_cached_reference_dataset_query.CONFIG',
+        MOCK_CONFIG,
+    )
+    def test_relatedness_check_table_task_gnomad_qc_updated(
+        self,
+    ) -> None:
+        ht = hl.read_table(
+            self.gnomad_qc_path,
+        )
+        self.assertEqual(
+            hl.eval(ht.versions.gnomad_qc),
+            'v3.1',
+        )
+        worker = luigi.worker.Worker()
+        task = WriteRelatednessCheckTableTask(
+            reference_genome=ReferenceGenome.GRCh38,
+            dataset_type=DatasetType.SNV_INDEL,
+            sample_type=SampleType.WGS,
+            callset_path=TEST_VCF,
+        )
+        worker.add(task)
+        worker.run()
+        self.assertTrue(task.complete())
+        ht = hl.read_table(self.gnomad_qc_path)
+        self.assertEqual(
+            hl.eval(ht.versions.gnomad_qc),
+            '4.0',
+        )
+        ht = hl.read_table(
+            relatedness_check_table_path(
+                ReferenceGenome.GRCh38,
+                DatasetType.SNV_INDEL,
+                TEST_VCF,
+            ),
+        )
+        self.assertEqual(
+            ht.collect(),
+            [],
+        )

v03_pipeline/var/test/reference_data/gnomad_qc_crdq.ht/.README.txt.crc

@@ -0,0 +1,3 @@
+This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
+  Written with version 0.2.128-eead8100a1c1
+  Created at 2024/05/10 15:28:58