Resolve bugs and increment versions. (#994)

* hmtvar float32

* Also fix topmed

* Bump versions

* Fix exac

* Eigen

* Splice ai

* put this back

* No change to `mitimpact`

* Remove duplicates based on transcript

* update mock tables

* Remove from here

* Back to main

* Back to main

* Fix unit tests

* Update splice_ai.py

Author: bpblanken

v03_pipeline/lib/reference_datasets/reference_dataset.py

@@ -263,15 +263,15 @@ def get_ht(
         },
         ReferenceGenome.GRCh37: {
             DATASET_TYPES: frozenset([DatasetType.SNV_INDEL]),
-            VERSION: '1.0',
+            VERSION: '1.1',
             PATH: [
                 'gs://seqr-reference-data/GRCh37/spliceai/spliceai_scores.masked.snv.hg19.vcf.gz',
                 'gs://seqr-reference-data/GRCh37/spliceai/spliceai_scores.masked.indel.hg19.vcf.gz',
             ],
         },
         ReferenceGenome.GRCh38: {
             DATASET_TYPES: frozenset([DatasetType.SNV_INDEL]),
-            VERSION: '1.0',
+            VERSION: '1.1',
             # NB: SpliceAI data is only available to download for authenticated Illumina users, so we will host the data
             PATH: [
                 'gs://seqr-reference-data/GRCh38/spliceai/spliceai_scores.masked.snv.hg38.vcf.gz',

v03_pipeline/lib/reference_datasets/splice_ai.py

@@ -9,6 +9,26 @@
 from v03_pipeline.lib.reference_datasets.misc import vcf_to_ht
 
 
+def remove_duplicate_scores(ht: hl.Table):
+    #
+    # SpliceAI has duplicate rows of the ilk:
+    #
+    #  1:861264      | ["C","A"]  | NA   | -1.00e+01 | NA       | ["A|AL645608.1|0.00|0.00|0.00|0.00|2|27|12|1"] |
+    #  1:861264      | ["C","A"]  | NA   | -1.00e+01 | NA       | ["A|SAMD11|0.02|0.01|0.00|0.00|14|38|14|38"]
+    #
+    count_ht = ht.group_by(*ht.key).aggregate(n=hl.agg.count())
+    duplicate_variants_ht = count_ht.filter(count_ht.n > 1)
+    duplicates_ht = ht.semi_join(duplicate_variants_ht)
+    non_duplicates_ht = ht.anti_join(duplicates_ht)
+    return non_duplicates_ht.union(
+        # Remove rows that 1) are part of duplicate variant groupings
+        # and 2) contain dots.
+        duplicates_ht.filter(
+            ~duplicates_ht.info.SpliceAI[0].split(delim='\\|')[1].contains('.'),
+        ),
+    )
+
+
 def get_ht(
     paths: list[str],
     reference_genome: ReferenceGenome,
@@ -26,6 +46,7 @@ def get_ht(
         # of partititons.
     )
     ht, _ = checkpoint(ht)
+    ht = remove_duplicate_scores(ht)
 
     # SpliceAI INFO field description from the VCF header: SpliceAIv1.3 variant annotation. These include
     # delta scores (DS) and delta positions (DP) for acceptor gain (AG), acceptor loss (AL), donor gain (DG), and
@@ -39,7 +60,7 @@ def get_ht(
         .map(hl.float32),
     )
     ht = ht.annotate(delta_score=hl.max(ht.delta_scores))
-    ht = ht.annotate(
+    return ht.annotate(
         splice_consequence_id=hl.if_else(
             ht.delta_score > 0,
             # Splice Consequence enum ID is the index of the max score
@@ -48,6 +69,3 @@ def get_ht(
             num_delta_scores,
         ),
     ).drop('delta_scores')
-    return ht.group_by(*ht.key).aggregate(
-        splice_consequence_id=hl.agg.min(ht.splice_consequence_id),
-    )

v03_pipeline/lib/tasks/reference_data/update_variant_annotations_table_with_updated_reference_dataset_test.py

@@ -146,7 +146,7 @@ def test_update_vat_snv_indel_38(
                             eigen='1.1',
                             clinvar='2024-11-11',
                             exac='1.1',
-                            splice_ai='1.0',
+                            splice_ai='1.1',
                             topmed='1.1',
                             hgmd='1.0',
                             gnomad_exomes='1.0',
@@ -425,7 +425,7 @@ def test_update_vat_snv_indel_37(
                             eigen='1.1',
                             clinvar='2024-11-11',
                             exac='1.1',
-                            splice_ai='1.0',
+                            splice_ai='1.1',
                             topmed='1.1',
                             hgmd='1.0',
                             gnomad_exomes='1.0',

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py

@@ -501,7 +501,7 @@ def test_multiple_update_vat(
                             exac='1.1',
                             gnomad_exomes='1.0',
                             gnomad_genomes='1.0',
-                            splice_ai='1.0',
+                            splice_ai='1.1',
                             topmed='1.1',
                             gnomad_non_coding_constraint='1.0',
                             screen='1.0',
@@ -762,7 +762,7 @@ def test_update_vat_without_accessing_private_datasets(
                         exac='1.1',
                         gnomad_exomes='1.0',
                         gnomad_genomes='1.0',
-                        splice_ai='1.0',
+                        splice_ai='1.1',
                         topmed='1.1',
                         gnomad_non_coding_constraint='1.0',
                         screen='1.0',