Merge pull request #1075 from broadinstitute/remap-from-ped-file

Remap from ped file

Author: hanars

v03_pipeline/bin/pipeline_worker.py

@@ -11,7 +11,6 @@
 from v03_pipeline.lib.paths import (
     loading_pipeline_queue_path,
     project_pedigree_path,
-    project_remap_path,
 )
 from v03_pipeline.lib.tasks.trigger_hail_backend_reload import TriggerHailBackendReload
 from v03_pipeline.lib.tasks.write_success_file import WriteSuccessFileTask
@@ -26,15 +25,6 @@ def main():
                 continue
             with open(loading_pipeline_queue_path()) as f:
                 lpr = LoadingPipelineRequest.model_validate_json(f.read())
-            project_remap_paths = [
-                project_remap_path(
-                    lpr.reference_genome,
-                    lpr.dataset_type,
-                    lpr.sample_type,
-                    project_guid,
-                )
-                for project_guid in lpr.projects_to_run
-            ]
             project_pedigree_paths = [
                 project_pedigree_path(
                     lpr.reference_genome,
@@ -49,7 +39,6 @@ def main():
             )
             loading_run_task_params = {
                 'project_guids': lpr.projects_to_run,
-                'project_remap_paths': project_remap_paths,
                 'project_pedigree_paths': project_pedigree_paths,
                 'run_id': run_id,
                 **{k: v for k, v in lpr.model_dump().items() if k != 'projects_to_run'},

v03_pipeline/lib/misc/io.py

@@ -283,31 +283,21 @@ def import_tdr_qc_metrics(file_path: str) -> hl.Table:
     return ht.key_by(ht.s)
 
 
-def import_remap(remap_path: str) -> hl.Table:
-    ht = hl.import_table(remap_path)
-    ht = ht.select(
-        s=ht.s,
-        seqr_id=ht.seqr_id,
-    )
-    return ht.key_by(ht.s)
-
-
 def import_pedigree(pedigree_path: str) -> hl.Table:
     ht = hl.import_table(pedigree_path, missing='')
+    optional_selects = {'remap_id': ht.VCF_ID} if 'VCF_ID' in ht.row else {}
     return ht.select(
         sex=ht.Sex,
         family_guid=ht.Family_GUID,
         s=ht.Individual_ID,
         maternal_s=ht.Maternal_ID,
         paternal_s=ht.Paternal_ID,
+        **optional_selects,
     )
 
 
-def remap_pedigree_hash(remap_path: str, pedigree_path: str) -> hl.Int32Expression:
+def remap_pedigree_hash(pedigree_path: str) -> hl.Int32Expression:
     sha256 = hashlib.sha256()
-    if hfs.exists(remap_path):
-        with hfs.open(remap_path) as f1:
-            sha256.update(f1.read().encode('utf8'))
     with hfs.open(pedigree_path) as f2:
         sha256.update(f2.read().encode('utf8'))
     # maximum 4 byte int

v03_pipeline/lib/misc/io_test.py

@@ -20,9 +20,8 @@
     'v03_pipeline/var/test/sex_check/test_imputed_sex_unexpected_value.tsv'
 )
 TEST_INVALID_VCF = 'v03_pipeline/var/test/callsets/improperly_formatted.vcf'
-TEST_PEDIGREE_3 = 'v03_pipeline/var/test/pedigrees/test_pedigree_3.tsv'
+TEST_PEDIGREE_3_REMAP = 'v03_pipeline/var/test/pedigrees/test_pedigree_3_remap.tsv'
 TEST_MITO_MT = 'v03_pipeline/var/test/callsets/mito_1.mt'
-TEST_REMAP = 'v03_pipeline/var/test/remaps/test_remap_1.tsv'
 
 
 class IOTest(unittest.TestCase):
@@ -61,11 +60,10 @@ def test_remap_pedigree_hash(self) -> None:
         self.assertEqual(
             hl.eval(
                 remap_pedigree_hash(
-                    TEST_REMAP,
-                    TEST_PEDIGREE_3,
+                    TEST_PEDIGREE_3_REMAP,
                 ),
             ),
-            -560434714,
+            573002191,
         )
 
     def test_import_vcf(self) -> None:
@@ -97,7 +95,7 @@ def test_import_vcf(self) -> None:
             SeqrValidationError,
             'VCF failed file format validation: Your input file has a malformed header: We never saw the required CHROM header line \\(starting with one #\\) for the input VCF file',
             import_vcf,
-            TEST_PEDIGREE_3,
+            TEST_PEDIGREE_3_REMAP,
             ReferenceGenome.GRCh38,
         )
         self.assertRaisesRegex(

v03_pipeline/lib/misc/pedigree.py

@@ -170,3 +170,10 @@ def parse_pedigree_ht_to_families(
     ):
         families.add(Family.parse(family_guid, list(rows)))
     return families
+
+
+def parse_pedigree_ht_to_remap_ht(pedigree_ht: hl.Table) -> hl.Table:
+    ht = pedigree_ht.filter(hl.is_defined(pedigree_ht.remap_id))
+    ht = ht.annotate(seqr_id=ht.s)
+    ht = ht.key_by(s=ht.remap_id)
+    return ht.select('seqr_id')

v03_pipeline/lib/paths.py

@@ -399,24 +399,6 @@ def clinvar_dataset_path(reference_genome: ReferenceGenome, etag: str) -> str:
     )
 
 
-def project_remap_path(
-    reference_genome: ReferenceGenome,
-    dataset_type: DatasetType,
-    sample_type: SampleType,
-    project_guid: str,
-) -> str:
-    return os.path.join(
-        pipeline_prefix(
-            Env.LOADING_DATASETS_DIR,
-            reference_genome,
-            dataset_type,
-        ),
-        'remaps',
-        sample_type.value,
-        f'{project_guid}_remap.tsv',
-    )
-
-
 def project_pedigree_path(
     reference_genome: ReferenceGenome,
     dataset_type: DatasetType,

v03_pipeline/lib/paths_test.py

@@ -15,7 +15,6 @@
     metadata_for_run_path,
     new_variants_table_path,
     project_pedigree_path,
-    project_remap_path,
     project_table_path,
     relatedness_check_table_path,
     remapped_and_subsetted_callset_path,
@@ -225,17 +224,6 @@ def test_new_variants_table_path(self) -> None:
             '/var/seqr/seqr-hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2023-06-26T18:30:09.349671+00:00/new_variants.ht',
         )
 
-    def test_project_remap_path(self) -> None:
-        self.assertEqual(
-            project_remap_path(
-                ReferenceGenome.GRCh38,
-                DatasetType.SNV_INDEL,
-                SampleType.WGS,
-                'R0652_pipeline_test',
-            ),
-            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/remaps/WGS/R0652_pipeline_test_remap.tsv',
-        )
-
     def test_project_pedigree_path(self) -> None:
         self.assertEqual(
             project_pedigree_path(

v03_pipeline/lib/tasks/base/base_loading_run_params.py

@@ -20,7 +20,6 @@ class BaseLoadingRunParams(luigi.Task):
     sample_type = luigi.EnumParameter(enum=SampleType)
     callset_path = luigi.Parameter()
     project_guids = luigi.ListParameter(default=[])
-    project_remap_paths = luigi.ListParameter(default=[])
     project_pedigree_paths = luigi.ListParameter(default=[])
     skip_check_sex_and_relatedness = luigi.BoolParameter(
         default=False,

v03_pipeline/lib/tasks/dataproc/misc_test.py

@@ -19,7 +19,6 @@ def test_to_kebab_str_args(self, _: Mock):
             sample_type=SampleType.WGS,
             callset_path='test_callset',
             project_guids=['R0113_test_project'],
-            project_remap_paths=['test_remap'],
             project_pedigree_paths=['test_pedigree'],
             run_id='a_misc_run',
         )
@@ -38,8 +37,6 @@ def test_to_kebab_str_args(self, _: Mock):
                 'test_callset',
                 '--project-guids',
                 '["R0113_test_project"]',
-                '--project-remap-paths',
-                '["test_remap"]',
                 '--project-pedigree-paths',
                 '["test_pedigree"]',
                 '--skip-check-sex-and-relatedness',

v03_pipeline/lib/tasks/dataproc/rsync_to_seqr_app_dirs_test.py

@@ -34,7 +34,6 @@ def test_rsync_to_seqr_app_dirs_no_sync(
             sample_type=SampleType.WGS,
             callset_path='test_callset',
             project_guids=['R0113_test_project'],
-            project_remap_paths=['test_remap'],
             project_pedigree_paths=['test_pedigree'],
             run_id='manual__2024-04-01',
         )
@@ -77,7 +76,6 @@ def test_rsync_to_seqr_app_dirs_sync(
             sample_type=SampleType.WGS,
             callset_path='test_callset',
             project_guids=['R0113_test_project'],
-            project_remap_paths=['test_remap'],
             project_pedigree_paths=['test_pedigree'],
             run_id='manual__2024-04-02',
         )

v03_pipeline/lib/tasks/dataproc/run_pipeline_on_dataproc_test.py

@@ -45,7 +45,6 @@ def test_job_already_exists_failed(
             sample_type=SampleType.WGS,
             callset_path='test_callset',
             project_guids=['R0113_test_project'],
-            project_remap_paths=['test_remap'],
             project_pedigree_paths=['test_pedigree'],
             run_id='manual__2024-04-03',
         )
@@ -79,7 +78,6 @@ def test_job_already_exists_success(
             sample_type=SampleType.WGS,
             callset_path='test_callset',
             project_guids=['R0113_test_project'],
-            project_remap_paths=['test_remap'],
             project_pedigree_paths=['test_pedigree'],
             run_id='manual__2024-04-04',
         )
@@ -111,7 +109,6 @@ def test_job_failed(
             sample_type=SampleType.WGS,
             callset_path='test_callset',
             project_guids=['R0113_test_project'],
-            project_remap_paths=['test_remap'],
             project_pedigree_paths=['test_pedigree'],
             run_id='manual__2024-04-05',
         )
@@ -152,7 +149,6 @@ def test_job_success(
             sample_type=SampleType.WGS,
             callset_path='test_callset',
             project_guids=['R0113_test_project'],
-            project_remap_paths=['test_remap'],
             project_pedigree_paths=['test_pedigree'],
             run_id='manual__2024-04-06',
         )