diff --git a/v03_pipeline/lib/model/definitions.py b/v03_pipeline/lib/model/definitions.py index bd6adf90e..b460ef458 100644 --- a/v03_pipeline/lib/model/definitions.py +++ b/v03_pipeline/lib/model/definitions.py @@ -17,6 +17,7 @@ class Sex(Enum): class PipelineVersion(Enum): V02 = 'v02' V03 = 'v03' + V3_1 = 'v3.1' class ReferenceGenome(Enum): diff --git a/v03_pipeline/lib/paths.py b/v03_pipeline/lib/paths.py index 3dc0ae179..92d877746 100644 --- a/v03_pipeline/lib/paths.py +++ b/v03_pipeline/lib/paths.py @@ -14,14 +14,14 @@ ) -def _v03_pipeline_prefix( +def _pipeline_prefix( root: str, reference_genome: ReferenceGenome, dataset_type: DatasetType, ) -> str: return os.path.join( root, - PipelineVersion.V03.value, + PipelineVersion.V3_1.value, reference_genome.value, dataset_type.value, ) @@ -62,15 +62,17 @@ def cached_reference_dataset_query_path( def family_table_path( reference_genome: ReferenceGenome, dataset_type: DatasetType, + sample_type: SampleType, family_guid: str, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.HAIL_SEARCH_DATA, reference_genome, dataset_type, ), 'families', + sample_type.value, f'{family_guid}.ht', ) @@ -81,7 +83,7 @@ def imputed_sex_path( callset_path: str, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.LOADING_DATASETS, reference_genome, dataset_type, @@ -97,7 +99,7 @@ def imported_callset_path( callset_path: str, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.LOADING_DATASETS, reference_genome, dataset_type, @@ -125,15 +127,17 @@ def metadata_for_run_path( def project_table_path( reference_genome: ReferenceGenome, dataset_type: DatasetType, + sample_type: SampleType, project_guid: str, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.HAIL_SEARCH_DATA, reference_genome, dataset_type, ), 'projects', + sample_type.value, f'{project_guid}.ht', ) @@ -144,7 +148,7 @@ def relatedness_check_table_path( callset_path: str, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.LOADING_DATASETS, reference_genome, dataset_type, @@ -161,7 +165,7 @@ def remapped_and_subsetted_callset_path( project_guid: str, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.LOADING_DATASETS, reference_genome, dataset_type, @@ -177,7 +181,7 @@ def lookup_table_path( dataset_type: DatasetType, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.HAIL_SEARCH_DATA, reference_genome, dataset_type, @@ -191,7 +195,7 @@ def runs_path( dataset_type: DatasetType, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.HAIL_SEARCH_DATA, reference_genome, dataset_type, @@ -206,7 +210,7 @@ def sex_check_table_path( callset_path: str, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.LOADING_DATASETS, reference_genome, dataset_type, @@ -260,7 +264,7 @@ def variant_annotations_table_path( dataset_type: DatasetType, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.HAIL_SEARCH_DATA, reference_genome, dataset_type, @@ -274,7 +278,7 @@ def variant_annotations_vcf_path( dataset_type: DatasetType, ) -> str: return os.path.join( - _v03_pipeline_prefix( + _pipeline_prefix( Env.HAIL_SEARCH_DATA, reference_genome, dataset_type, diff --git a/v03_pipeline/lib/paths_test.py b/v03_pipeline/lib/paths_test.py index ff437cf45..4ba4a270c 100644 --- a/v03_pipeline/lib/paths_test.py +++ b/v03_pipeline/lib/paths_test.py @@ -42,9 +42,10 @@ def test_family_table_path(self) -> None: family_table_path( ReferenceGenome.GRCh37, DatasetType.SNV_INDEL, + SampleType.WES, 'franklin', ), - '/hail-search-data/v03/GRCh37/SNV_INDEL/families/franklin.ht', + '/hail-search-data/v3.1/GRCh37/SNV_INDEL/families/WES/franklin.ht', ) with patch('v03_pipeline.lib.paths.Env') as mock_env: mock_env.HAIL_SEARCH_DATA = 'gs://seqr-datasets/' @@ -52,9 +53,10 @@ def test_family_table_path(self) -> None: family_table_path( ReferenceGenome.GRCh37, DatasetType.SNV_INDEL, + SampleType.WES, 'franklin', ), - 'gs://seqr-datasets/v03/GRCh37/SNV_INDEL/families/franklin.ht', + 'gs://seqr-datasets/v3.1/GRCh37/SNV_INDEL/families/WES/franklin.ht', ) def test_valid_filters_path(self) -> None: @@ -82,9 +84,10 @@ def test_project_table_path(self) -> None: project_table_path( ReferenceGenome.GRCh38, DatasetType.MITO, + SampleType.WES, 'R0652_pipeline_test', ), - '/hail-search-data/v03/GRCh38/MITO/projects/R0652_pipeline_test.ht', + '/hail-search-data/v3.1/GRCh38/MITO/projects/WES/R0652_pipeline_test.ht', ) def test_valid_reference_dataset_collection_path(self) -> None: @@ -113,7 +116,7 @@ def test_lookup_table_path(self) -> None: ReferenceGenome.GRCh37, DatasetType.SV, ), - '/hail-search-data/v03/GRCh37/SV/lookup.ht', + '/hail-search-data/v3.1/GRCh37/SV/lookup.ht', ) def test_sex_check_table_path(self) -> None: @@ -123,7 +126,7 @@ def test_sex_check_table_path(self) -> None: DatasetType.SNV_INDEL, 'gs://abc.efg/callset.vcf.gz', ), - '/seqr-loading-temp/v03/GRCh38/SNV_INDEL/sex_check/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.ht', + '/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/sex_check/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.ht', ) def test_relatedness_check_table_path(self) -> None: @@ -133,7 +136,7 @@ def test_relatedness_check_table_path(self) -> None: DatasetType.SNV_INDEL, 'gs://abc.efg/callset.vcf.gz', ), - '/seqr-loading-temp/v03/GRCh38/SNV_INDEL/relatedness_check/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.ht', + '/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/relatedness_check/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.ht', ) def test_metadata_for_run_path(self) -> None: @@ -143,7 +146,7 @@ def test_metadata_for_run_path(self) -> None: DatasetType.SNV_INDEL, 'manual__2023-06-26T18:30:09.349671+00:00', ), - '/hail-search-data/v03/GRCh38/SNV_INDEL/runs/manual__2023-06-26T18:30:09.349671+00:00/metadata.json', + '/hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2023-06-26T18:30:09.349671+00:00/metadata.json', ) def test_variant_annotations_table_path(self) -> None: @@ -152,7 +155,7 @@ def test_variant_annotations_table_path(self) -> None: ReferenceGenome.GRCh38, DatasetType.GCNV, ), - '/hail-search-data/v03/GRCh38/GCNV/annotations.ht', + '/hail-search-data/v3.1/GRCh38/GCNV/annotations.ht', ) def test_remapped_and_subsetted_callset_path(self) -> None: @@ -163,7 +166,7 @@ def test_remapped_and_subsetted_callset_path(self) -> None: 'gs://abc.efg/callset.vcf.gz', 'R0111_tgg_bblanken_wes', ), - '/seqr-loading-temp/v03/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.mt', + '/seqr-loading-temp/v3.1/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.mt', ) self.assertEqual( remapped_and_subsetted_callset_path( @@ -172,7 +175,7 @@ def test_remapped_and_subsetted_callset_path(self) -> None: 'gs://abc.efg/callset/*.vcf.gz', 'R0111_tgg_bblanken_wes', ), - '/seqr-loading-temp/v03/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/bce53ccdb49a5ed2513044e1d0c6224e3ffcc323f770dc807d9175fd3c70a050.mt', + '/seqr-loading-temp/v3.1/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/bce53ccdb49a5ed2513044e1d0c6224e3ffcc323f770dc807d9175fd3c70a050.mt', ) def test_imported_callset_path(self) -> None: @@ -182,7 +185,7 @@ def test_imported_callset_path(self) -> None: DatasetType.SNV_INDEL, 'gs://abc.efg/callset.vcf.gz', ), - '/seqr-loading-temp/v03/GRCh38/SNV_INDEL/imported_callsets/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.mt', + '/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/imported_callsets/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.mt', ) def test_imputed_sex_path(self) -> None: @@ -192,7 +195,7 @@ def test_imputed_sex_path(self) -> None: DatasetType.SNV_INDEL, 'gs://abc.efg/callset.vcf.gz', ), - '/seqr-loading-temp/v03/GRCh38/SNV_INDEL/imputed_sex/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.tsv', + '/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/imputed_sex/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.tsv', ) def test_new_variants_table_path(self) -> None: @@ -202,5 +205,5 @@ def test_new_variants_table_path(self) -> None: DatasetType.SNV_INDEL, 'manual__2023-06-26T18:30:09.349671+00:00', ), - '/hail-search-data/v03/GRCh38/SNV_INDEL/runs/manual__2023-06-26T18:30:09.349671+00:00/new_variants.ht', + '/hail-search-data/v3.1/GRCh38/SNV_INDEL/runs/manual__2023-06-26T18:30:09.349671+00:00/new_variants.ht', ) diff --git a/v03_pipeline/lib/tasks/base/base_update_project_table.py b/v03_pipeline/lib/tasks/base/base_update_project_table.py index ad7ee0d1e..473a31bc2 100644 --- a/v03_pipeline/lib/tasks/base/base_update_project_table.py +++ b/v03_pipeline/lib/tasks/base/base_update_project_table.py @@ -1,12 +1,14 @@ import hail as hl import luigi +from v03_pipeline.lib.model import SampleType from v03_pipeline.lib.paths import project_table_path from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask from v03_pipeline.lib.tasks.files import GCSorLocalTarget class BaseUpdateProjectTableTask(BaseUpdateTask): + sample_type = luigi.EnumParameter(enum=SampleType) project_guid = luigi.Parameter() def output(self) -> luigi.Target: @@ -14,6 +16,7 @@ def output(self) -> luigi.Target: project_table_path( self.reference_genome, self.dataset_type, + self.sample_type, self.project_guid, ), ) diff --git a/v03_pipeline/lib/tasks/delete_family_table.py b/v03_pipeline/lib/tasks/delete_family_table.py index cbc929fd2..c7946035d 100644 --- a/v03_pipeline/lib/tasks/delete_family_table.py +++ b/v03_pipeline/lib/tasks/delete_family_table.py @@ -1,11 +1,13 @@ import luigi +from v03_pipeline.lib.model import SampleType from v03_pipeline.lib.paths import family_table_path from v03_pipeline.lib.tasks.base.base_delete_table import BaseDeleteTableTask from v03_pipeline.lib.tasks.files import GCSorLocalTarget class DeleteFamilyTableTask(BaseDeleteTableTask): + sample_type = luigi.EnumParameter(enum=SampleType) family_guid = luigi.Parameter() def output(self) -> luigi.Target: @@ -13,6 +15,7 @@ def output(self) -> luigi.Target: family_table_path( self.reference_genome, self.dataset_type, + self.sample_type, self.family_guid, ), ) diff --git a/v03_pipeline/lib/tasks/delete_family_table_test.py b/v03_pipeline/lib/tasks/delete_family_table_test.py index 43e92bb6b..e4d364abb 100644 --- a/v03_pipeline/lib/tasks/delete_family_table_test.py +++ b/v03_pipeline/lib/tasks/delete_family_table_test.py @@ -3,7 +3,7 @@ import hail as hl import luigi.worker -from v03_pipeline.lib.model import DatasetType, ReferenceGenome +from v03_pipeline.lib.model import DatasetType, ReferenceGenome, SampleType from v03_pipeline.lib.paths import family_table_path from v03_pipeline.lib.tasks.delete_family_table import DeleteFamilyTableTask from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase @@ -41,6 +41,7 @@ def setUp(self) -> None: family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WES, 'abc_1', ), ) @@ -50,6 +51,7 @@ def test_delete_family_table_task(self) -> None: task = DeleteFamilyTableTask( reference_genome=ReferenceGenome.GRCh38, dataset_type=DatasetType.SNV_INDEL, + sample_type=SampleType.WES, family_guid='abc_1', ) worker.add(task) @@ -60,6 +62,7 @@ def test_delete_family_table_task(self) -> None: family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WES, 'abc_1', ), ).exists(), diff --git a/v03_pipeline/lib/tasks/delete_family_tables.py b/v03_pipeline/lib/tasks/delete_family_tables.py index a68f4dc28..c09e9fca2 100644 --- a/v03_pipeline/lib/tasks/delete_family_tables.py +++ b/v03_pipeline/lib/tasks/delete_family_tables.py @@ -1,5 +1,6 @@ import luigi +from v03_pipeline.lib.model import SampleType from v03_pipeline.lib.tasks.base.base_hail_table import BaseHailTableTask from v03_pipeline.lib.tasks.delete_family_table import DeleteFamilyTableTask @@ -18,12 +19,14 @@ def complete(self) -> bool: ) def run(self): - for family_guid in self.family_guids: - self.dynamic_delete_family_table_tasks.add( - DeleteFamilyTableTask( - reference_genome=self.reference_genome, - dataset_type=self.dataset_type, - family_guid=family_guid, - ), - ) + for sample_type in SampleType: + for family_guid in self.family_guids: + self.dynamic_delete_family_table_tasks.add( + DeleteFamilyTableTask( + reference_genome=self.reference_genome, + dataset_type=self.dataset_type, + sample_type=sample_type, + family_guid=family_guid, + ), + ) yield self.dynamic_delete_family_table_tasks diff --git a/v03_pipeline/lib/tasks/delete_family_tables_test.py b/v03_pipeline/lib/tasks/delete_family_tables_test.py index 535299602..21d0e73cd 100644 --- a/v03_pipeline/lib/tasks/delete_family_tables_test.py +++ b/v03_pipeline/lib/tasks/delete_family_tables_test.py @@ -3,7 +3,7 @@ import hail as hl import luigi.worker -from v03_pipeline.lib.model import DatasetType, ReferenceGenome +from v03_pipeline.lib.model import DatasetType, ReferenceGenome, SampleType from v03_pipeline.lib.paths import family_table_path from v03_pipeline.lib.tasks.delete_family_tables import ( DeleteFamilyTablesTask, @@ -20,6 +20,7 @@ def setUp(self) -> None: family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, family_guid, ), ) @@ -30,6 +31,17 @@ def test_delete_project_family_tables_task(self) -> None: family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, + 'family_a', + ), + ).exists(), + ) + self.assertFalse( + pathlib.Path( + family_table_path( + ReferenceGenome.GRCh38, + DatasetType.SNV_INDEL, + SampleType.WES, 'family_a', ), ).exists(), @@ -49,6 +61,7 @@ def test_delete_project_family_tables_task(self) -> None: family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, family_guid, ), ).exists(), diff --git a/v03_pipeline/lib/tasks/delete_project_family_tables.py b/v03_pipeline/lib/tasks/delete_project_family_tables.py index befca9a45..5ebca0f03 100644 --- a/v03_pipeline/lib/tasks/delete_project_family_tables.py +++ b/v03_pipeline/lib/tasks/delete_project_family_tables.py @@ -1,6 +1,8 @@ import hail as hl +import hailtop.fs as hfs import luigi +from v03_pipeline.lib.model import SampleType from v03_pipeline.lib.paths import project_table_path from v03_pipeline.lib.tasks.base.base_hail_table import BaseHailTableTask from v03_pipeline.lib.tasks.delete_family_table import DeleteFamilyTableTask @@ -21,21 +23,32 @@ def complete(self) -> bool: ) def run(self): - project_table_task: luigi.Target = yield HailTableTask( - project_table_path( + project_tables = set() + for sample_type in SampleType: + project_ht_path = project_table_path( self.reference_genome, self.dataset_type, + sample_type, self.project_guid, - ), - ) - project_ht = hl.read_table(project_table_task.path) - family_guids = hl.eval(project_ht.globals.family_guids) - for family_guid in family_guids: - self.dynamic_delete_family_table_tasks.add( - DeleteFamilyTableTask( - reference_genome=self.reference_genome, - dataset_type=self.dataset_type, - family_guid=family_guid, - ), ) + if hfs.exists(project_ht_path): + project_table_task: luigi.Target = yield HailTableTask(project_ht_path) + project_ht = hl.read_table(project_table_task.path) + project_tables.add((project_ht, sample_type)) + + if len(project_tables) == 0: + msg = f'No project tables found for {self.project_guid}' + raise RuntimeError(msg) + + for project_ht, sample_type in project_tables: + family_guids = hl.eval(project_ht.globals.family_guids) + for family_guid in family_guids: + self.dynamic_delete_family_table_tasks.add( + DeleteFamilyTableTask( + reference_genome=self.reference_genome, + dataset_type=self.dataset_type, + sample_type=sample_type, + family_guid=family_guid, + ), + ) yield self.dynamic_delete_family_table_tasks diff --git a/v03_pipeline/lib/tasks/delete_project_family_tables_test.py b/v03_pipeline/lib/tasks/delete_project_family_tables_test.py index 75f11987a..e6f10edb2 100644 --- a/v03_pipeline/lib/tasks/delete_project_family_tables_test.py +++ b/v03_pipeline/lib/tasks/delete_project_family_tables_test.py @@ -3,7 +3,7 @@ import hail as hl import luigi.worker -from v03_pipeline.lib.model import DatasetType, ReferenceGenome +from v03_pipeline.lib.model import DatasetType, ReferenceGenome, SampleType from v03_pipeline.lib.paths import family_table_path, project_table_path from v03_pipeline.lib.tasks.delete_project_family_tables import ( DeleteProjectFamilyTablesTask, @@ -14,7 +14,7 @@ class DeleteTableTaskTest(MockedDatarootTestCase): def setUp(self) -> None: super().setUp() - ht = hl.Table.parallelize( + project_ht = hl.Table.parallelize( [ { 'locus': hl.Locus( @@ -123,33 +123,77 @@ def setUp(self) -> None: }, ), ) - ht.write( + project_ht.write( project_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, 'project_a', ), ) - for family_guid in hl.eval(ht.globals.family_guids): + for family_guid in hl.eval(project_ht.globals.family_guids): family_ht = hl.utils.range_table(100) family_ht.write( family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, family_guid, ), ) + def test_no_project_tables_for_project(self) -> None: + worker = luigi.worker.Worker() + task = DeleteProjectFamilyTablesTask( + reference_genome=ReferenceGenome.GRCh38, + dataset_type=DatasetType.SNV_INDEL, + project_guid='project_b', + ) + worker.add(task) + worker.run() + self.assertFalse(task.complete()) + def test_delete_project_family_tables_task(self) -> None: self.assertTrue( pathlib.Path( family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, 'family_a', ), ).exists(), ) + self.assertFalse( + pathlib.Path( + family_table_path( + ReferenceGenome.GRCh38, + DatasetType.SNV_INDEL, + SampleType.WES, + 'family_a', + ), + ).exists(), + ) + self.assertTrue( + pathlib.Path( + project_table_path( + ReferenceGenome.GRCh38, + DatasetType.SNV_INDEL, + SampleType.WGS, + 'project_a', + ), + ).exists(), + ) + self.assertFalse( + pathlib.Path( + project_table_path( + ReferenceGenome.GRCh38, + DatasetType.SNV_INDEL, + SampleType.WES, + 'project_a', + ), + ).exists(), + ) worker = luigi.worker.Worker() task = DeleteProjectFamilyTablesTask( reference_genome=ReferenceGenome.GRCh38, @@ -165,6 +209,7 @@ def test_delete_project_family_tables_task(self) -> None: family_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, family_guid, ), ).exists(), diff --git a/v03_pipeline/lib/tasks/delete_project_table.py b/v03_pipeline/lib/tasks/delete_project_table.py index 0a403ea3b..53ae98a97 100644 --- a/v03_pipeline/lib/tasks/delete_project_table.py +++ b/v03_pipeline/lib/tasks/delete_project_table.py @@ -1,5 +1,6 @@ import luigi +from v03_pipeline.lib.model import SampleType from v03_pipeline.lib.paths import project_table_path from v03_pipeline.lib.tasks.base.base_delete_table import BaseDeleteTableTask from v03_pipeline.lib.tasks.delete_project_family_tables import ( @@ -9,12 +10,14 @@ class DeleteProjectTableTask(BaseDeleteTableTask): + sample_type = luigi.EnumParameter(enum=SampleType) project_guid = luigi.Parameter() def requires(self) -> luigi.Task: return DeleteProjectFamilyTablesTask( self.reference_genome, self.dataset_type, + self.sample_type, self.project_guid, ) @@ -23,6 +26,7 @@ def output(self) -> luigi.Target: project_table_path( self.reference_genome, self.dataset_type, + self.sample_type, self.project_guid, ), ) diff --git a/v03_pipeline/lib/tasks/migrate_lookup_table.py b/v03_pipeline/lib/tasks/migrate_lookup_table.py index 1cf42b901..4fe9d7ac4 100644 --- a/v03_pipeline/lib/tasks/migrate_lookup_table.py +++ b/v03_pipeline/lib/tasks/migrate_lookup_table.py @@ -11,7 +11,7 @@ class MigrateLookupTableTask(BaseMigrateTask): @property - def migrations_path(self) -> list[str]: + def migrations_path(self): return v03_pipeline.migrations.lookup.__path__[0] def output(self) -> luigi.Target: diff --git a/v03_pipeline/lib/tasks/update_project_table_with_deleted_families_test.py b/v03_pipeline/lib/tasks/update_project_table_with_deleted_families_test.py index b1f5cc5af..72c475d3f 100644 --- a/v03_pipeline/lib/tasks/update_project_table_with_deleted_families_test.py +++ b/v03_pipeline/lib/tasks/update_project_table_with_deleted_families_test.py @@ -1,7 +1,7 @@ import hail as hl import luigi -from v03_pipeline.lib.model import DatasetType, ReferenceGenome +from v03_pipeline.lib.model import DatasetType, ReferenceGenome, SampleType from v03_pipeline.lib.paths import project_table_path from v03_pipeline.lib.tasks.update_project_table_with_deleted_families import ( UpdateProjectTableWithDeletedFamiliesTask, @@ -120,6 +120,7 @@ def setUp(self) -> None: project_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, 'project_a', ), ) @@ -129,6 +130,7 @@ def test_update_project_with_deleted_families(self): task = UpdateProjectTableWithDeletedFamiliesTask( dataset_type=DatasetType.SNV_INDEL, reference_genome=ReferenceGenome.GRCh38, + sample_type=SampleType.WGS, project_guid='project_a', family_guids=['family_b'], ) diff --git a/v03_pipeline/lib/tasks/write_family_table.py b/v03_pipeline/lib/tasks/write_family_table.py index ce4c8679c..4d6683664 100644 --- a/v03_pipeline/lib/tasks/write_family_table.py +++ b/v03_pipeline/lib/tasks/write_family_table.py @@ -23,6 +23,7 @@ def output(self) -> luigi.Target: family_table_path( self.reference_genome, self.dataset_type, + self.sample_type, self.family_guid, ), ) diff --git a/v03_pipeline/lib/tasks/write_project_family_tables_test.py b/v03_pipeline/lib/tasks/write_project_family_tables_test.py index 09ac0b030..27680d8a1 100644 --- a/v03_pipeline/lib/tasks/write_project_family_tables_test.py +++ b/v03_pipeline/lib/tasks/write_project_family_tables_test.py @@ -85,6 +85,7 @@ def test_snv_write_project_family_tables_task(self) -> None: project_table_path( ReferenceGenome.GRCh38, DatasetType.SNV_INDEL, + SampleType.WGS, 'R0113_test_project', ), ).family_guids.collect()[0],