Make sure WriteMetadataForRunTask is always in the dependency tree (#877)

* ruff

* tasks

* Make sure WriteMetadataForRunTask is always in the dependency tree

* fix test

* env vars

* remove liftover from hail

* ruff

* spelling

* move remove_liftover

* set value on mock

* ruff

* move from before tasks to after tasks

* move it back

Author: bpblanken

v03_pipeline/lib/tasks/update_lookup_table.py

@@ -9,12 +9,13 @@
     remove_family_guids,
 )
 from v03_pipeline.lib.model.constants import PROJECTS_EXCLUDED_FROM_LOOKUP
+from v03_pipeline.lib.paths import remapped_and_subsetted_callset_path
 from v03_pipeline.lib.tasks.base.base_loading_run_params import BaseLoadingRunParams
 from v03_pipeline.lib.tasks.base.base_update_lookup_table import (
     BaseUpdateLookupTableTask,
 )
-from v03_pipeline.lib.tasks.write_remapped_and_subsetted_callset import (
-    WriteRemappedAndSubsettedCallsetTask,
+from v03_pipeline.lib.tasks.write_metadata_for_run import (
+    WriteMetadataForRunTask,
 )
 
 
@@ -23,6 +24,7 @@ class UpdateLookupTableTask(BaseUpdateLookupTableTask):
     project_guids = luigi.ListParameter()
     project_remap_paths = luigi.ListParameter()
     project_pedigree_paths = luigi.ListParameter()
+    run_id = luigi.Parameter()
 
     def complete(self) -> bool:
         return (
@@ -53,22 +55,9 @@ def complete(self) -> bool:
     def requires(self) -> list[luigi.Task]:
         return [
             self.clone(
-                WriteRemappedAndSubsettedCallsetTask,
-                project_guid=project_guid,
-                project_remap_path=project_remap_path,
-                project_pedigree_path=project_pedigree_path,
+                WriteMetadataForRunTask,
                 force=False,
-            )
-            for (
-                project_guid,
-                project_remap_path,
-                project_pedigree_path,
-            ) in zip(
-                self.project_guids,
-                self.project_remap_paths,
-                self.project_pedigree_paths,
-                strict=True,
-            )
+            ),
         ]
 
     def update_table(self, ht: hl.Table) -> hl.Table:
@@ -89,7 +78,14 @@ def update_table(self, ht: hl.Table) -> hl.Table:
                     ),
                 )
                 continue
-            callset_mt = hl.read_matrix_table(self.input()[i].path)
+            callset_mt = hl.read_matrix_table(
+                remapped_and_subsetted_callset_path(
+                    self.reference_genome,
+                    self.dataset_type,
+                    self.callset_path,
+                    project_guid,
+                ),
+            )
             ht = remove_family_guids(
                 ht,
                 project_guid,

v03_pipeline/lib/tasks/update_lookup_table_test.py

@@ -12,6 +12,8 @@
 TEST_REMAP = 'v03_pipeline/var/test/remaps/test_remap_1.tsv'
 TEST_PEDIGREE_3 = 'v03_pipeline/var/test/pedigrees/test_pedigree_3.tsv'
 
+TEST_RUN_ID = 'manual__2024-04-03'
+
 
 class UpdateLookupTableTest(MockedDatarootTestCase):
     def test_skip_update_lookup_table_task(self) -> None:
@@ -27,6 +29,7 @@ def test_skip_update_lookup_table_task(self) -> None:
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
+            run_id=TEST_RUN_ID,
         )
         worker.add(uslt_task)
         worker.run()
@@ -64,6 +67,7 @@ def test_update_lookup_table_task(self) -> None:
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_3],
             skip_validation=True,
+            run_id=TEST_RUN_ID,
         )
         worker.add(uslt_task)
         worker.run()

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples_test.py

@@ -423,7 +423,7 @@ def test_multiple_update_vat(
             project_remap_paths=[TEST_REMAP],
             project_pedigree_paths=[TEST_PEDIGREE_4],
             skip_validation=False,
-            run_id=TEST_RUN_ID,
+            run_id=TEST_RUN_ID + '-another-run',
         )
         worker.add(uvatwns_task_4)
         worker.run()

v03_pipeline/lib/tasks/validate_callset.py

@@ -49,7 +49,7 @@ def output(self) -> luigi.Target:
 
     def requires(self) -> list[luigi.Task]:
         requirements = [
-            self.clone(WriteImportedCallsetTask),
+            self.clone(WriteImportedCallsetTask, force=False),
         ]
         if not self.skip_validation and self.dataset_type.can_run_validation:
             requirements = [

v03_pipeline/lib/tasks/write_new_variants_table.py

@@ -33,8 +33,8 @@
 from v03_pipeline.lib.tasks.update_lookup_table import (
     UpdateLookupTableTask,
 )
-from v03_pipeline.lib.tasks.write_remapped_and_subsetted_callset import (
-    WriteRemappedAndSubsettedCallsetTask,
+from v03_pipeline.lib.tasks.write_metadata_for_run import (
+    WriteMetadataForRunTask,
 )
 from v03_pipeline.lib.vep import run_vep
 
@@ -95,26 +95,10 @@ def requires(self) -> list[luigi.Task]:
                 self.clone(UpdateLookupTableTask),
             ]
         else:
-            requirements.extend(
-                [
-                    self.clone(
-                        WriteRemappedAndSubsettedCallsetTask,
-                        project_guid=project_guid,
-                        project_remap_path=project_remap_path,
-                        project_pedigree_path=project_pedigree_path,
-                    )
-                    for (
-                        project_guid,
-                        project_remap_path,
-                        project_pedigree_path,
-                    ) in zip(
-                        self.project_guids,
-                        self.project_remap_paths,
-                        self.project_pedigree_paths,
-                        strict=True,
-                    )
-                ],
-            )
+            requirements = [
+                *requirements,
+                self.clone(WriteMetadataForRunTask, force=False),
+            ]
         return requirements
 
     def complete(self) -> bool: