main <- dev (#860)

* Move vep files (#844)

* Add mito local constraint (#845)

* Add mito local constraint

* Fix tests

* lint

* Benb/migration task (#834)

* split import and validate

* lint and share function

* ruff

* change dep

* tweak update

* lint

* wrong method

* correct method

* mocks

* change sample type annotation on test

* hack on migration

* sort return list

* move the migration

* still hacking

* better!

* getting there

* Cleaner

* ruff

* Finish it off

* migration

* rename var

* add migrations to annotations table

* fix test import

* actually fix the test

* add migrations

* not used here

* use globals

* missed one

* a hilarious typo

* Update migrate_variant_annotations_table.py

* correct sign

* add lookup migration

* Add lookup table migration

* adjust migration

* ruff

* Add to tasks

* ensure a migration cannot run before a previous migration!

* ruff

* fix bug

* lint

* add referencegenomedatasetype

* Annoying but fixed

* Add new SV annotations for VCF export. (#857)

* Add SV annotations

* ruff

* push

* ruff

* Update update_variant_annotations_table_with_new_samples_test.py

* Add a task to export the SV annotations table to VCF. (#858)

* Export VCF task

* Fix test

* lint

* Resolve the assumption in the pipeline that remap/pedigree files are immutable. (#856)

* add remap_pedigree hash

* add func

* all the imports

* ruff

* Fix it

* support missing remap

* ruff

* ruff

* ruff

* tweak the type

* tweak the type

* Fix test

* ruff

* add remap pedigree hash

* Explicit int32

* lint

* Update io.py

* ruff

* lint

* hash

* Flappy test

* wrong pedigree

* bad colon

* finish tests

* add a test

* add pedigree

* Fix test

Author: bpblanken

pyproject.toml

@@ -64,6 +64,9 @@ inline-quotes = "single"
     'SLF001', # allow private access
     'PLR0913', # allow high arity functions
 ]
+'*migration*' = [
+    'N999', # allow invalid module names
+]
 
 [tool.ruff.pylint]
 max-args = 6

v03_pipeline/lib/annotations/sv.py

@@ -81,6 +81,41 @@ def _sv_types(ht: hl.Table) -> hl.ArrayExpression:
     return ht.alleles[1].replace('[<>]', '').split(':', 2)
 
 
+def alleles(ht: hl.Table, **_: Any) -> hl.ArrayExpression:
+    return hl.array(
+        [
+            'N',
+            hl.if_else(
+                (
+                    hl.is_defined(ht.sv_type_detail_id)
+                    & (hl.array(SV_TYPES)[ht.sv_type_id] != 'CPX')
+                ),
+                hl.format(
+                    '<%s:%s>',
+                    hl.array(SV_TYPES)[ht.sv_type_id],
+                    hl.array(SV_TYPE_DETAILS)[ht.sv_type_detail_id],
+                ),
+                hl.format('<%s>', hl.array(SV_TYPES)[ht.sv_type_id]),
+            ),
+        ],
+    )
+
+
+def info(ht: hl.Table, **_: Any) -> hl.StructExpression:
+    return hl.Struct(
+        ALGORITHMS=ht.algorithms,
+        END=ht.start_locus.position,
+        CHR2=ht.end_locus.contig,
+        END2=ht.end_locus.position,
+        SVTYPE=hl.array(SV_TYPES)[ht.sv_type_id],
+        SVLEN=ht.sv_len,
+    )
+
+
+def locus(ht: hl.Table, **_: Any) -> hl.LocusExpression:
+    return ht.start_locus
+
+
 def algorithms(ht: hl.Table, **_: Any) -> hl.Expression:
     return hl.str(',').join(ht['info.ALGORITHMS'])
 
@@ -205,6 +240,10 @@ def strvctvre(ht: hl.Table, **_: Any) -> hl.Expression:
     return hl.struct(score=hl.parse_float32(ht['info.StrVCTVRE']))
 
 
+def sv_len(ht: hl.Table, **_: Any) -> hl.Expression:
+    return ht['info.SVLEN']
+
+
 def sv_type_id(ht: hl.Table, **_: Any) -> hl.Expression:
     return SV_TYPES_LOOKUP[_sv_types(ht)[0]]
 

v03_pipeline/lib/annotations/sv_test.py

@@ -0,0 +1,171 @@
+import unittest
+
+import hail as hl
+
+from v03_pipeline.lib.annotations.fields import get_fields
+from v03_pipeline.lib.model import DatasetType
+
+
+class SVTest(unittest.TestCase):
+    def test_sv_export_annotations(self) -> None:
+        ht = hl.Table.parallelize(
+            [
+                hl.Struct(
+                    id=0,
+                    algorithms='manta',
+                    end_locus=hl.Locus(
+                        contig='chr5',
+                        position=20404,
+                        reference_genome='GRCh38',
+                    ),
+                    start_locus=hl.Locus(
+                        contig='chr1',
+                        position=180928,
+                        reference_genome='GRCh38',
+                    ),
+                    sv_len=123,
+                    sv_type_id=2,
+                    sv_type_detail_id=None,
+                ),
+                hl.Struct(
+                    id=1,
+                    algorithms='manta',
+                    end_locus=hl.Locus(
+                        contig='chr1',
+                        position=789481,
+                        reference_genome='GRCh38',
+                    ),
+                    start_locus=hl.Locus(
+                        contig='chr1',
+                        position=789481,
+                        reference_genome='GRCh38',
+                    ),
+                    sv_len=245,
+                    sv_type_id=2,
+                    sv_type_detail_id=None,
+                ),
+                hl.Struct(
+                    id=2,
+                    algorithms='manta',
+                    end_locus=hl.Locus(
+                        contig='chr1',
+                        position=6559723,
+                        reference_genome='GRCh38',
+                    ),
+                    start_locus=hl.Locus(
+                        contig='chr1',
+                        position=6558902,
+                        reference_genome='GRCh38',
+                    ),
+                    sv_len=245,
+                    sv_type_id=3,
+                    sv_type_detail_id=2,
+                ),
+                hl.Struct(
+                    id=3,
+                    algorithms='manta',
+                    end_locus=hl.Locus(
+                        contig='chr1',
+                        position=6559723,
+                        reference_genome='GRCh38',
+                    ),
+                    start_locus=hl.Locus(
+                        contig='chr1',
+                        position=6558902,
+                        reference_genome='GRCh38',
+                    ),
+                    sv_len=245,
+                    sv_type_id=7,
+                    sv_type_detail_id=6,
+                ),
+            ],
+            hl.tstruct(
+                id=hl.tint32,
+                algorithms=hl.tstr,
+                end_locus=hl.tlocus('GRCh38'),
+                start_locus=hl.tlocus('GRCh38'),
+                sv_len=hl.tint32,
+                sv_type_id=hl.tint32,
+                sv_type_detail_id=hl.tint32,
+            ),
+            key='id',
+        )
+        ht = ht.select(
+            **get_fields(
+                ht,
+                DatasetType.SV.export_vcf_annotation_fns,
+            ),
+        )
+        self.assertEqual(
+            ht.collect(),
+            [
+                hl.Struct(
+                    id=0,
+                    locus=hl.Locus(
+                        contig='chr1',
+                        position=180928,
+                        reference_genome='GRCh38',
+                    ),
+                    alleles=['N', '<BND>'],
+                    info=hl.Struct(
+                        ALGORITHMS='manta',
+                        END=180928,
+                        CHR2='chr5',
+                        END2=20404,
+                        SVTYPE='BND',
+                        SVLEN=123,
+                    ),
+                ),
+                hl.Struct(
+                    id=1,
+                    locus=hl.Locus(
+                        contig='chr1',
+                        position=789481,
+                        reference_genome='GRCh38',
+                    ),
+                    alleles=['N', '<BND>'],
+                    info=hl.Struct(
+                        ALGORITHMS='manta',
+                        END=789481,
+                        CHR2='chr1',
+                        END2=789481,
+                        SVTYPE='BND',
+                        SVLEN=245,
+                    ),
+                ),
+                hl.Struct(
+                    id=2,
+                    locus=hl.Locus(
+                        contig='chr1',
+                        position=6558902,
+                        reference_genome='GRCh38',
+                    ),
+                    alleles=['N', '<CPX>'],
+                    info=hl.Struct(
+                        ALGORITHMS='manta',
+                        END=6558902,
+                        CHR2='chr1',
+                        END2=6559723,
+                        SVTYPE='CPX',
+                        SVLEN=245,
+                    ),
+                ),
+                hl.Struct(
+                    id=3,
+                    locus=hl.Locus(
+                        contig='chr1',
+                        position=6558902,
+                        reference_genome='GRCh38',
+                    ),
+                    alleles=['N', '<INS:ME:SVA>'],
+                    info=hl.Struct(
+                        ALGORITHMS='manta',
+                        END=6558902,
+                        CHR2='chr1',
+                        END2=6559723,
+                        SVTYPE='INS',
+                        SVLEN=245,
+                    ),
+                ),
+            ],
+        )

v03_pipeline/lib/migration/__init__.py

@@ -0,0 +1,16 @@
+from abc import ABC, abstractmethod
+
+import hail as hl
+
+from v03_pipeline.lib.model import DatasetType, ReferenceGenome
+
+
+class BaseMigration(ABC):
+    reference_genome_dataset_types: frozenset[
+        tuple[ReferenceGenome, DatasetType]
+    ] = None
+
+    @staticmethod
+    @abstractmethod
+    def migrate(ht: hl.Table) -> hl.Table:
+        pass

v03_pipeline/lib/migration/base_migration.py

@@ -0,0 +1,33 @@
+import inspect
+import pkgutil
+import re
+
+from v03_pipeline.lib.migration.base_migration import BaseMigration
+
+MIGRATION_NAME_PATTERN = r'(\d\d\d\d_.*)'
+
+
+def list_migrations(
+    path: str,
+) -> list[tuple[str, BaseMigration]]:
+    migrations = []
+    for loader, name, _ in pkgutil.iter_modules([path]):
+        match = re.search(MIGRATION_NAME_PATTERN, name)
+        if match:
+            module = loader.find_module(name).load_module(name)
+            implemented_migration = next(
+                (
+                    m
+                    for m in module.__dict__.values()
+                    # Return objects that are
+                    # classes, subclasses of the BaseMigration
+                    # and also NOT the BaseMigration class.
+                    if inspect.isclass(m)
+                    and issubclass(m, BaseMigration)
+                    and m != BaseMigration
+                ),
+                None,
+            )
+            if implemented_migration:
+                migrations.append((match.group(1), implemented_migration))
+    return sorted(migrations, key=lambda x: x[0])

v03_pipeline/lib/migration/misc.py

@@ -0,0 +1,47 @@
+import os
+import shutil
+import tempfile
+import unittest
+
+from v03_pipeline.lib.migration.base_migration import BaseMigration
+from v03_pipeline.lib.migration.misc import list_migrations
+
+
+class TestListMigrations(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.TemporaryDirectory()
+        for migration in [
+            '__init__.py',
+            '1111_a_migration.py',
+            '0000_migration.py',
+            '001_test.py',
+            'abcd_test.py',
+            '0000_migration.txt',
+        ]:
+            with open(os.path.join(self.tmpdir.name, migration), 'w') as f:
+                f.write(
+                    """
+from v03_pipeline.lib.migration.base_migration import BaseMigration
+class ImplementedMigration(BaseMigration):
+    pass
+                    """,
+                )
+
+    def tearDown(self):
+        if os.path.isdir(self.tmpdir.name):
+            shutil.rmtree(self.tmpdir.name)
+
+    def test_list_migrations(self):
+        self.assertEqual(
+            [
+                (x, issubclass(y, BaseMigration))
+                for (x, y) in list_migrations(self.tmpdir.name)
+            ],
+            [
+                ('0000_migration', True),
+                ('1111_a_migration', True),
+            ],
+        )
+        self.assertTrue(
+            all(hasattr(x[1], 'migrate') for x in list_migrations(self.tmpdir.name)),
+        )

v03_pipeline/lib/migration/misc_test.py

@@ -1,8 +1,10 @@
+import hashlib
 import math
 import os
 import uuid
 
 import hail as hl
+import hailtop.fs as hfs
 
 from v03_pipeline.lib.misc.gcnv import parse_gcnv_genes
 from v03_pipeline.lib.misc.nested_field import parse_nested_field
@@ -200,6 +202,17 @@ def import_pedigree(pedigree_path: str) -> hl.Table:
     )
 
 
+def remap_pedigree_hash(remap_path: str, pedigree_path: str) -> hl.Int32Expression:
+    sha256 = hashlib.sha256()
+    if hfs.exists(remap_path):
+        with hfs.open(remap_path) as f1:
+            sha256.update(f1.read().encode('utf8'))
+    with hfs.open(pedigree_path) as f2:
+        sha256.update(f2.read().encode('utf8'))
+    # maximum 4 byte int
+    return hl.int32(int(sha256.hexdigest()[:8], 16))
+
+
 def checkpoint(t: hl.Table | hl.MatrixTable) -> tuple[hl.Table | hl.MatrixTable, str]:
     suffix = 'mt' if isinstance(t, hl.MatrixTable) else 'ht'
     read_fn = hl.read_matrix_table if isinstance(t, hl.MatrixTable) else hl.read_table

v03_pipeline/lib/misc/io.py

@@ -6,13 +6,16 @@
     compute_hail_n_partitions,
     file_size_bytes,
     import_imputed_sex,
+    remap_pedigree_hash,
 )
 
 TEST_IMPUTED_SEX = 'v03_pipeline/var/test/sex_check/test_imputed_sex.tsv'
 TEST_IMPUTED_SEX_UNEXPECTED_VALUE = (
     'v03_pipeline/var/test/sex_check/test_imputed_sex_unexpected_value.tsv'
 )
+TEST_PEDIGREE_3 = 'v03_pipeline/var/test/pedigrees/test_pedigree_3.tsv'
 TEST_MITO_MT = 'v03_pipeline/var/test/callsets/mito_1.mt'
+TEST_REMAP = 'v03_pipeline/var/test/remaps/test_remap_1.tsv'
 TEST_SV_VCF = 'v03_pipeline/var/test/callsets/sv_1.vcf'
 
 
@@ -46,3 +49,14 @@ def test_import_imputed_sex_unexpected_value(self) -> None:
             'Found unexpected value Unknown in imputed sex file',
             ht.collect,
         )
+
+    def test_remap_pedigree_hash(self) -> None:
+        self.assertEqual(
+            hl.eval(
+                remap_pedigree_hash(
+                    TEST_REMAP,
+                    TEST_PEDIGREE_3,
+                ),
+            ),
+            -560434714,
+        )