Delete Project & Family Table Tasks. (#767)

* Delete project tasks

* cleanup

* ruff format

* well

* rename

* hacking away

* almost there!

* ruff

* Fix missing updates change

* ruff

* Remove debug code

* remove bad merge

* more precision in test

* project table

* allow for missing project

* remove some unnecessary checks

* test already deleted family

* Lots of renames

* More updates

* Sketch

* Flesh out test

* fix paths

* Rename base hail table

* a bunch more renames

* delete project table

* Add delete project families

* add comment

* test it!

* Fix

* add dep

Author: bpblanken

v03_pipeline/lib/tasks/base/base_delete_table.py

@@ -0,0 +1,19 @@
+import hailtop.fs as hfs
+
+from v03_pipeline.lib.logger import get_logger
+from v03_pipeline.lib.tasks.base.base_hail_table import BaseHailTableTask
+from v03_pipeline.lib.tasks.files import GCSorLocalFolderTarget, GCSorLocalTarget
+
+logger = get_logger(__name__)
+
+
+class BaseDeleteTableTask(BaseHailTableTask):
+    def complete(self) -> bool:
+        logger.info(f'DeleteTableTask: checking if {self.output().path} exists')
+        return (
+            not GCSorLocalTarget(self.output().path).exists()
+            and not GCSorLocalFolderTarget(self.output().path).exists()
+        )
+
+    def run(self) -> None:
+        hfs.rmtree(self.output().path)

v03_pipeline/lib/tasks/base/base_hail_table_task.py renamed to v03_pipeline/lib/tasks/base/base_hail_table.py

@@ -1,7 +1,7 @@
 import hail as hl
 
 from v03_pipeline.lib.misc.io import write
-from v03_pipeline.lib.tasks.base.base_hail_table_task import BaseHailTableTask
+from v03_pipeline.lib.tasks.base.base_hail_table import BaseHailTableTask
 
 
 class BaseUpdateTask(BaseHailTableTask):

v03_pipeline/lib/tasks/base/base_update_task.py renamed to v03_pipeline/lib/tasks/base/base_update.py

@@ -2,11 +2,11 @@
 import luigi
 
 from v03_pipeline.lib.paths import lookup_table_path
-from v03_pipeline.lib.tasks.base.base_update_task import BaseUpdateTask
+from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask
 from v03_pipeline.lib.tasks.files import GCSorLocalTarget
 
 
-class BaseLookupTableTask(BaseUpdateTask):
+class BaseUpdateLookupTableTask(BaseUpdateTask):
     def output(self) -> luigi.Target:
         return GCSorLocalTarget(
             lookup_table_path(

v03_pipeline/lib/tasks/base/base_lookup_table_task.py renamed to v03_pipeline/lib/tasks/base/base_update_lookup_table.py

@@ -2,11 +2,11 @@
 import luigi
 
 from v03_pipeline.lib.paths import project_table_path
-from v03_pipeline.lib.tasks.base.base_update_task import BaseUpdateTask
+from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask
 from v03_pipeline.lib.tasks.files import GCSorLocalTarget
 
 
-class BaseProjectTableTask(BaseUpdateTask):
+class BaseUpdateProjectTableTask(BaseUpdateTask):
     project_guid = luigi.Parameter()
 
     def output(self) -> luigi.Target:

v03_pipeline/lib/tasks/base/base_project_table_task.py renamed to v03_pipeline/lib/tasks/base/base_update_project_table.py

@@ -13,14 +13,14 @@
     valid_reference_dataset_collection_path,
     variant_annotations_table_path,
 )
-from v03_pipeline.lib.tasks.base.base_update_task import BaseUpdateTask
+from v03_pipeline.lib.tasks.base.base_update import BaseUpdateTask
 from v03_pipeline.lib.tasks.files import GCSorLocalTarget, HailTableTask
 from v03_pipeline.lib.tasks.reference_data.updated_reference_dataset_collection import (
     UpdatedReferenceDatasetCollectionTask,
 )
 
 
-class BaseVariantAnnotationsTableTask(BaseUpdateTask):
+class BaseUpdateVariantAnnotationsTableTask(BaseUpdateTask):
     @property
     def rdc_annotation_dependencies(self) -> dict[str, hl.Table]:
         return get_rdc_annotation_dependencies(self.dataset_type, self.reference_genome)

v03_pipeline/lib/tasks/base/base_variant_annotations_table.py renamed to v03_pipeline/lib/tasks/base/base_update_variant_annotations_table.py

@@ -11,8 +11,8 @@
     SampleType,
 )
 from v03_pipeline.lib.paths import valid_reference_dataset_collection_path
-from v03_pipeline.lib.tasks.base.base_variant_annotations_table import (
-    BaseVariantAnnotationsTableTask,
+from v03_pipeline.lib.tasks.base.base_update_variant_annotations_table import (
+    BaseUpdateVariantAnnotationsTableTask,
 )
 from v03_pipeline.lib.tasks.files import GCSorLocalFolderTarget
 from v03_pipeline.lib.test.mock_complete_task import MockCompleteTask
@@ -52,11 +52,11 @@ def setUp(self) -> None:
         )
 
     @patch(
-        'v03_pipeline.lib.tasks.base.base_variant_annotations_table.UpdatedReferenceDatasetCollectionTask',
+        'v03_pipeline.lib.tasks.base.base_update_variant_annotations_table.UpdatedReferenceDatasetCollectionTask',
     )
     def test_should_create_initialized_table(self, mock_update_rdc_task) -> None:
         mock_update_rdc_task.return_value = MockCompleteTask()
-        vat_task = BaseVariantAnnotationsTableTask(
+        vat_task = BaseUpdateVariantAnnotationsTableTask(
             reference_genome=ReferenceGenome.GRCh38,
             dataset_type=DatasetType.SNV_INDEL,
             sample_type=SampleType.WGS,

v03_pipeline/lib/tasks/base/base_variant_annotations_table_test.py renamed to v03_pipeline/lib/tasks/base/base_update_variant_annotations_table_test.py

@@ -1,7 +1,7 @@
 import hail as hl
 
 from v03_pipeline.lib.misc.io import write
-from v03_pipeline.lib.tasks.base.base_hail_table_task import BaseHailTableTask
+from v03_pipeline.lib.tasks.base.base_hail_table import BaseHailTableTask
 
 
 class BaseWriteTask(BaseHailTableTask):

v03_pipeline/lib/tasks/base/base_write_task.py renamed to v03_pipeline/lib/tasks/base/base_write.py

@@ -0,0 +1,18 @@
+import luigi
+
+from v03_pipeline.lib.paths import family_table_path
+from v03_pipeline.lib.tasks.base.base_delete_table import BaseDeleteTableTask
+from v03_pipeline.lib.tasks.files import GCSorLocalTarget
+
+
+class DeleteFamilyTableTask(BaseDeleteTableTask):
+    family_guid = luigi.Parameter()
+
+    def output(self) -> luigi.Target:
+        return GCSorLocalTarget(
+            family_table_path(
+                self.reference_genome,
+                self.dataset_type,
+                self.family_guid,
+            ),
+        )

v03_pipeline/lib/tasks/delete_family_table.py

@@ -0,0 +1,67 @@
+import pathlib
+
+import hail as hl
+import luigi.worker
+
+from v03_pipeline.lib.model import DatasetType, ReferenceGenome, SampleType
+from v03_pipeline.lib.paths import family_table_path
+from v03_pipeline.lib.tasks.delete_family_table import DeleteFamilyTableTask
+from v03_pipeline.lib.test.mocked_dataroot_testcase import MockedDatarootTestCase
+
+
+class DeleteTableTaskTest(MockedDatarootTestCase):
+    def setUp(self) -> None:
+        super().setUp()
+        ht = hl.Table.parallelize(
+            [
+                {
+                    'locus': hl.Locus(
+                        contig='chr1',
+                        position=876499,
+                        reference_genome='GRCh38',
+                    ),
+                    'alleles': ['A', 'G'],
+                },
+                {
+                    'locus': hl.Locus(
+                        contig='chr1',
+                        position=878314,
+                        reference_genome='GRCh38',
+                    ),
+                    'alleles': ['G', 'C'],
+                },
+            ],
+            hl.tstruct(
+                locus=hl.tlocus('GRCh38'),
+                alleles=hl.tarray(hl.tstr),
+            ),
+            key=['locus', 'alleles'],
+        )
+        ht.write(
+            family_table_path(
+                ReferenceGenome.GRCh38,
+                DatasetType.SNV_INDEL,
+                'abc_1',
+            ),
+        )
+
+    def test_delete_family_table_task(self) -> None:
+        worker = luigi.worker.Worker()
+        task = DeleteFamilyTableTask(
+            reference_genome=ReferenceGenome.GRCh38,
+            dataset_type=DatasetType.SNV_INDEL,
+            sample_type=SampleType.WGS,
+            family_guid='abc_1',
+        )
+        worker.add(task)
+        worker.run()
+        self.assertTrue(task.complete())
+        self.assertFalse(
+            pathlib.Path(
+                family_table_path(
+                    ReferenceGenome.GRCh38,
+                    DatasetType.SNV_INDEL,
+                    'abc_1',
+                ),
+            ).exists(),
+        )