fix liftover

Author: bpblanken

v03_pipeline/lib/model/environment.py

@@ -4,6 +4,9 @@
 # NB: using os.environ.get inside the dataclass defaults gives a lint error.
 HAIL_TMPDIR = os.environ.get('HAIL_TMPDIR', '/tmp')  # noqa: S108
 HAIL_SEARCH_DATA = os.environ.get('HAIL_SEARCH_DATA', '/hail-search-data')
+LIFTOVER_REF_PATH = os.environ.get(
+    'LIFTOVER_REF_PATH', 'gs://hail-common/references/grch38_to_grch37.over.chain.gz'
+)
 LOADING_DATASETS = os.environ.get('LOADING_DATASETS', '/seqr-loading-temp')
 PRIVATE_REFERENCE_DATASETS = os.environ.get(
     'PRIVATE_REFERENCE_DATASETS',
@@ -34,6 +37,7 @@ class Env:
     ALLELE_REGISTRY_SECRET_NAME: str | None = ALLELE_REGISTRY_SECRET_NAME
     HAIL_TMPDIR: str = HAIL_TMPDIR
     HAIL_SEARCH_DATA: str = HAIL_SEARCH_DATA
+    LIFTOVER_REF_PATH: str = LIFTOVER_REF_PATH
     LOADING_DATASETS: str = LOADING_DATASETS
     PRIVATE_REFERENCE_DATASETS: str = PRIVATE_REFERENCE_DATASETS
     PROJECT_ID: str | None = PROJECT_ID

v03_pipeline/lib/tasks/base/base_loading_run_params.py

@@ -34,7 +34,3 @@ class BaseLoadingRunParams(luigi.Task):
         default=False,
         description='Is this a fully joint-called callset.',
     )
-    liftover_ref_path = luigi.OptionalParameter(
-        default='gs://hail-common/references/grch38_to_grch37.over.chain.gz',
-        description='Path to GRCh38 to GRCh37 coordinates file',
-    )

v03_pipeline/lib/tasks/update_lookup_table_test.py

@@ -7,7 +7,6 @@
 )
 from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
 
-TEST_LIFTOVER = 'v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz'
 TEST_VCF = 'v03_pipeline/var/test/callsets/1kg_30variants.vcf'
 TEST_REMAP = 'v03_pipeline/var/test/remaps/test_remap_1.tsv'
 TEST_PEDIGREE_3 = 'v03_pipeline/var/test/pedigrees/test_pedigree_3.tsv'
@@ -28,7 +27,6 @@ def test_skip_update_lookup_table_task(self) -> None:
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
-            liftover_ref_path=TEST_LIFTOVER,
         )
         worker.add(uslt_task)
         worker.run()
@@ -61,7 +59,6 @@ def test_update_lookup_table_task(self) -> None:
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
-            liftover_ref_path=TEST_LIFTOVER,
         )
         worker.add(uslt_task)
         worker.run()

v03_pipeline/lib/tasks/update_project_table_test.py

@@ -5,7 +5,6 @@
 from v03_pipeline.lib.tasks.update_project_table import UpdateProjectTableTask
 from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
 
-TEST_LIFTOVER = 'v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz'
 TEST_VCF = 'v03_pipeline/var/test/callsets/1kg_30variants.vcf'
 TEST_REMAP = 'v03_pipeline/var/test/remaps/test_remap_1.tsv'
 TEST_PEDIGREE_3 = 'v03_pipeline/var/test/pedigrees/test_pedigree_3.tsv'
@@ -24,7 +23,6 @@ def test_update_project_table_task(self) -> None:
             project_pedigree_path=TEST_PEDIGREE_3,
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
-            liftover_ref_path=TEST_LIFTOVER,
         )
         worker.add(upt_task)
         worker.run()

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py

@@ -43,7 +43,6 @@
 from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
 from v03_pipeline.var.test.vep.mock_vep_data import MOCK_37_VEP_DATA, MOCK_38_VEP_DATA
 
-TEST_LIFTOVER = 'v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz'
 TEST_MITO_MT = 'v03_pipeline/var/test/callsets/mito_1.mt'
 TEST_SNV_INDEL_VCF = 'v03_pipeline/var/test/callsets/1kg_30variants.vcf'
 TEST_SV_VCF = 'v03_pipeline/var/test/callsets/sv_1.vcf'
@@ -163,7 +162,6 @@ def test_missing_pedigree(
             project_guids=['R0113_test_project'],
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=['bad_pedigree'],
-            liftover_ref_path=TEST_LIFTOVER,
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
@@ -198,7 +196,6 @@ def test_missing_interval_reference(
             project_guids=['R0113_test_project'],
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
-            liftover_ref_path=TEST_LIFTOVER,
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
@@ -368,7 +365,6 @@ def test_multiple_update_vat(
             project_guids=['R0113_test_project'],
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
-            liftover_ref_path=TEST_LIFTOVER,
             skip_validation=False,
             skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
@@ -421,7 +417,6 @@ def test_multiple_update_vat(
             project_guids=['R0114_project4'],
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_4],
-            liftover_ref_path=TEST_LIFTOVER,
             skip_validation=False,
             skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
@@ -693,7 +688,6 @@ def test_update_vat_grch37(
             project_guids=['R0113_test_project'],
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
-            liftover_ref_path=TEST_LIFTOVER,
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
@@ -774,7 +768,6 @@ def test_update_vat_without_accessing_private_datasets(
             project_guids=['R0113_test_project'],
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
-            liftover_ref_path=TEST_LIFTOVER,
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
             run_id=TEST_RUN_ID,
@@ -833,7 +826,6 @@ def test_mito_update_vat(
                 project_guids=['R0115_test_project2'],
                 project_remap_paths=['not_a_real_file'],
                 project_pedigree_paths=[TEST_PEDIGREE_5],
-                liftover_ref_path=TEST_LIFTOVER,
                 skip_validation=True,
                 skip_check_sex_and_relatedness=True,
                 run_id=TEST_RUN_ID,
@@ -1099,7 +1091,6 @@ def test_sv_update_vat(
                 project_guids=['R0115_test_project2'],
                 project_remap_paths=['not_a_real_file'],
                 project_pedigree_paths=[TEST_PEDIGREE_5],
-                liftover_ref_path=TEST_LIFTOVER,
                 skip_validation=True,
                 skip_check_sex_and_relatedness=True,
                 run_id=TEST_RUN_ID,
@@ -1662,7 +1653,6 @@ def test_gcnv_update_vat(
                 project_guids=['R0115_test_project2'],
                 project_remap_paths=['not_a_real_file'],
                 project_pedigree_paths=[TEST_PEDIGREE_5],
-                liftover_ref_path=TEST_LIFTOVER,
                 skip_validation=True,
                 skip_check_sex_and_relatedness=True,
                 run_id=TEST_RUN_ID,

v03_pipeline/lib/tasks/write_family_table_test.py

@@ -5,7 +5,6 @@
 from v03_pipeline.lib.tasks.write_family_table import WriteFamilyTableTask
 from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
 
-TEST_LIFTOVER = 'v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz'
 TEST_GCNV_BED_FILE = 'v03_pipeline/var/test/callsets/gcnv_1.tsv'
 TEST_SNV_INDEL_VCF = 'v03_pipeline/var/test/callsets/1kg_30variants.vcf'
 TEST_SV_VCF = 'v03_pipeline/var/test/callsets/sv_1.vcf'
@@ -28,7 +27,6 @@ def test_snv_write_family_table_task(self) -> None:
             family_guid='abc_1',
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
-            liftover_ref_path=TEST_LIFTOVER,
         )
         worker.add(wft_task)
         worker.run()
@@ -167,7 +165,6 @@ def test_sv_write_family_table_task(self) -> None:
             family_guid='family_2_1',
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
-            liftover_ref_path=TEST_LIFTOVER,
         )
         worker.add(write_family_table_task)
         worker.run()
@@ -421,7 +418,6 @@ def test_gcnv_write_family_table_task(self) -> None:
             family_guid='family_2_1',
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
-            liftover_ref_path=TEST_LIFTOVER,
         )
         worker.add(write_family_table_task)
         worker.run()

v03_pipeline/lib/tasks/write_new_variants_table.py

@@ -62,6 +62,7 @@ def annotation_dependencies(self) -> dict[str, hl.Table]:
             deps['gencode_gene_symbol_to_gene_id_mapping'] = hl.literal(
                 load_gencode_gene_symbol_to_gene_id(GENCODE_RELEASE, ''),
             )
+        deps['liftover_ref_path'] = Env.LIFTOVER_REF_PATH
         return deps
 
     def output(self) -> luigi.Target:

v03_pipeline/lib/tasks/write_project_family_tables_test.py

@@ -7,7 +7,6 @@
 )
 from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
 
-TEST_LIFTOVER = 'v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz'
 TEST_SNV_INDEL_VCF = 'v03_pipeline/var/test/callsets/1kg_30variants.vcf'
 TEST_REMAP = 'v03_pipeline/var/test/remaps/test_remap_1.tsv'
 TEST_PEDIGREE_4 = 'v03_pipeline/var/test/pedigrees/test_pedigree_4.tsv'
@@ -26,7 +25,6 @@ def test_snv_write_project_family_tables_task(self) -> None:
             project_pedigree_path=TEST_PEDIGREE_4,
             skip_validation=True,
             skip_check_sex_and_relatedness=True,
-            liftover_ref_path=TEST_LIFTOVER,
         )
         worker.add(write_project_family_tables)
         worker.run()