add new clinvar pathogenicity and contig filter workaround (#713)

* add new clinvar pathogenicity and contig filter workaround

* remove filter

* tables

* change filtering

Author: jklugherz

v03_pipeline/lib/annotations/enums.py

@@ -198,6 +198,7 @@
 CLINVAR_PATHOGENICITIES = [
     'Pathogenic',
     'Pathogenic/Likely_pathogenic',
+    'Pathogenic/Likely_pathogenic/Established_risk_allele',
     'Pathogenic/Likely_pathogenic/Likely_risk_allele',
     'Pathogenic/Likely_risk_allele',
     'Likely_pathogenic',

v03_pipeline/lib/reference_data/clinvar.py

@@ -64,6 +64,10 @@ def parsed_clnsig(ht: hl.Table):
             'Likely_pathogenic,_low_penetrance',
             'Likely_pathogenic|low_penetrance',
         )
+        .replace(
+            '/Pathogenic,_low_penetrance/Established_risk_allele',
+            '/Established_risk_allele|low_penetrance',
+        )
         .replace(
             '/Pathogenic,_low_penetrance',
             '|low_penetrance',

v03_pipeline/lib/reference_data/clinvar_test.py

@@ -31,6 +31,14 @@ def test_parsed_clnsig(self):
                 },
                 {'info': hl.Struct(CLNSIG=['Likely_pathogenic', '_low_penetrance'])},
                 {'info': hl.Struct(CLNSIG=['association|protective'])},
+                {
+                    'info': hl.Struct(
+                        CLNSIG=[
+                            'Pathogenic/Likely_pathogenic/Pathogenic',
+                            '_low_penetrance/Established_risk_allele',
+                        ],
+                    ),
+                },
             ],
             hl.tstruct(info=hl.tstruct(CLNSIG=hl.tarray(hl.tstr))),
         )
@@ -42,6 +50,10 @@ def test_parsed_clnsig(self):
                 ['Likely_pathogenic', 'low_penetrance', 'association', 'protective'],
                 ['Likely_pathogenic', 'low_penetrance'],
                 ['association', 'protective'],
+                [
+                    'Pathogenic/Likely_pathogenic/Established_risk_allele',
+                    'low_penetrance',
+                ],
             ],
         )
 
@@ -66,8 +78,8 @@ def test_parsed_and_mapped_clnsigconf(self):
                 None,
                 [
                     hl.Struct(count=9, pathogenicity_id=0),
-                    hl.Struct(count=2, pathogenicity_id=4),
-                    hl.Struct(count=1, pathogenicity_id=11),
+                    hl.Struct(count=2, pathogenicity_id=5),
+                    hl.Struct(count=1, pathogenicity_id=12),
                 ],
             ],
         )

v03_pipeline/lib/reference_data/config.py

@@ -192,7 +192,7 @@ def custom_mpc_select(ht):
                 'pathogenicity': CLINVAR_PATHOGENICITIES,
                 'assertion': CLINVAR_ASSERTIONS,
             },
-            'filter': lambda ht: ht.locus.contig != 'MT',
+            'filter': lambda ht: ~(ht.locus.contig == 'MT'),
         },
         '38': {
             'custom_import': download_and_import_latest_clinvar_vcf,
@@ -203,7 +203,7 @@ def custom_mpc_select(ht):
                 'pathogenicity': CLINVAR_PATHOGENICITIES,
                 'assertion': CLINVAR_ASSERTIONS,
             },
-            'filter': lambda ht: ht.locus.contig != 'chrM',
+            'filter': lambda ht: ~(ht.locus.contig == 'chrM'),
         },
     },
     'dbnsfp': {

v03_pipeline/var/test/reference_data/test_combined_1.ht/.README.txt.crc

@@ -1,3 +1,3 @@
 This folder comprises a Hail (www.hail.is) native Table or MatrixTable.
   Written with version 0.2.128-eead8100a1c1
-  Created at 2024/02/28 14:36:31
+  Created at 2024/03/01 15:52:48