make run id a shared parameter (necessary for validation work) (#901)

* make run id shared

* A fix

* another test

* A few more

* Few more

* Unnecessary

* ruf

* run id

* string

* unused

* missed one

* Fix it correctly

* missed one

* Last one!

Author: bpblanken

.github/workflows/unit-tests.yml

@@ -36,7 +36,6 @@ jobs:
         run: ruff . --output-format github
       - name: Unit Tests
         run: |
-          export HAIL_TMP_DIR=/tmp
           export GRCH37_TO_GRCH38_LIFTOVER_REF_PATH=v03_pipeline/var/test/liftover/grch37_to_grch38.over.chain.gz
           export GRCH38_TO_GRCH37_LIFTOVER_REF_PATH=v03_pipeline/var/test/liftover/grch38_to_grch37.over.chain.gz
           export ACCESS_PRIVATE_REFERENCE_DATASETS=1

v03_pipeline/lib/tasks/base/base_loading_run_params.py

@@ -16,6 +16,7 @@ class BaseLoadingRunParams(luigi.Task):
     # - The "Loading Pipeline" params are shared with
     # tasks that may remove data from or change the
     # structure of the persisted Hail Tables.
+    run_id = luigi.Parameter()
     sample_type = luigi.EnumParameter(enum=SampleType)
     callset_path = luigi.Parameter()
     ignore_missing_samples_when_remapping = luigi.BoolParameter(

v03_pipeline/lib/tasks/update_lookup_table.py

@@ -24,7 +24,6 @@ class UpdateLookupTableTask(BaseUpdateLookupTableTask):
     project_guids = luigi.ListParameter()
     project_remap_paths = luigi.ListParameter()
     project_pedigree_paths = luigi.ListParameter()
-    run_id = luigi.Parameter()
 
     def complete(self) -> bool:
         return super().complete() and hl.eval(

v03_pipeline/lib/tasks/update_project_table_test.py

@@ -13,13 +13,16 @@
     'v03_pipeline/var/test/pedigrees/test_pedigree_3_different_families.tsv'
 )
 
+TEST_RUN_ID = 'manual__2024-04-03'
+
 
 class UpdateProjectTableTaskTest(MockedDatarootTestCase):
     def test_update_project_table_task(self) -> None:
         worker = luigi.worker.Worker()
         upt_task = UpdateProjectTableTask(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
+            run_id=TEST_RUN_ID,
             sample_type=SampleType.WGS,
             callset_path=TEST_VCF,
             project_guid='R0113_test_project',
@@ -128,6 +131,7 @@ def test_update_project_table_task_different_pedigree(self) -> None:
         upt_task = UpdateProjectTableTask(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
+            run_id=TEST_RUN_ID,
             sample_type=SampleType.WGS,
             callset_path=TEST_VCF,
             project_guid='R0113_test_project',
@@ -140,6 +144,7 @@ def test_update_project_table_task_different_pedigree(self) -> None:
         upt_task = UpdateProjectTableTask(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
+            run_id=TEST_RUN_ID,
             sample_type=SampleType.WGS,
             callset_path=TEST_VCF,
             project_guid='R0113_test_project',

v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples.py

@@ -23,7 +23,6 @@ class UpdateVariantAnnotationsTableWithNewSamplesTask(
     project_guids = luigi.ListParameter()
     project_remap_paths = luigi.ListParameter()
     project_pedigree_paths = luigi.ListParameter()
-    run_id = luigi.Parameter()
 
     def requires(self) -> list[luigi.Task]:
         return [

v03_pipeline/lib/tasks/write_metadata_for_run.py

@@ -17,7 +17,6 @@ class WriteMetadataForRunTask(luigi.Task):
     project_guids = luigi.ListParameter()
     project_remap_paths = luigi.ListParameter()
     project_pedigree_paths = luigi.ListParameter()
-    run_id = luigi.Parameter()
 
     def output(self) -> luigi.Target:
         return GCSorLocalTarget(

v03_pipeline/lib/tasks/write_new_variants_table.py

@@ -48,7 +48,6 @@ class WriteNewVariantsTableTask(BaseWriteTask):
     project_guids = luigi.ListParameter()
     project_remap_paths = luigi.ListParameter()
     project_pedigree_paths = luigi.ListParameter()
-    run_id = luigi.Parameter()
 
     @property
     def annotation_dependencies(self) -> dict[str, hl.Table]:

v03_pipeline/lib/tasks/write_project_family_tables_test.py

@@ -13,13 +13,16 @@
 TEST_PEDIGREE_4 = 'v03_pipeline/var/test/pedigrees/test_pedigree_4.tsv'
 TEST_PEDIGREE_4_SUBSET = 'v03_pipeline/var/test/pedigrees/test_pedigree_4_subset.tsv'
 
+TEST_RUN_ID = 'manual__2024-04-03'
+
 
 class WriteProjectFamilyTablesTest(MockedDatarootTestCase):
     def test_snv_write_project_family_tables_task(self) -> None:
         worker = luigi.worker.Worker()
         write_project_family_tables = WriteProjectFamilyTablesTask(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
+            run_id=TEST_RUN_ID,
             sample_type=SampleType.WGS,
             callset_path=TEST_SNV_INDEL_VCF,
             project_guid='R0113_test_project',
@@ -57,6 +60,7 @@ def test_snv_write_project_family_tables_task(self) -> None:
         write_project_family_tables_subset = WriteProjectFamilyTablesTask(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
+            run_id=TEST_RUN_ID,
             sample_type=SampleType.WGS,
             callset_path=TEST_SNV_INDEL_VCF,
             project_guid='R0113_test_project',

v03_pipeline/lib/tasks/write_variant_annotations_vcf_test.py

@@ -49,13 +49,13 @@ def test_sv_export_vcf(
             UpdateVariantAnnotationsTableWithNewSamplesTask(
                 reference_genome=ReferenceGenome.GRCh38,
                 dataset_type=DatasetType.SV,
+                run_id='run_id1',
                 sample_type=SampleType.WGS,
                 callset_path=TEST_SV_VCF,
                 project_guids=['R0115_test_project2'],
                 project_remap_paths=['not_a_real_file'],
                 project_pedigree_paths=[TEST_PEDIGREE_5],
                 skip_validation=True,
-                run_id='run_id1',
             )
         )
         worker.add(update_variant_annotations_task)
@@ -64,6 +64,7 @@ def test_sv_export_vcf(
         write_variant_annotations_vcf_task = WriteVariantAnnotationsVCF(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SV,
+            run_id='run_id1',
             sample_type=SampleType.WGS,
             callset_path=TEST_SV_VCF,
         )