Skip to content

Commit 7a9a693

Browse files
bpblankenjklugherz
bpblanken
and
jklugherz
authored
Benb/force task option (#757)
* add allele registry step in update vat with samples task * shh * existing tests pass * fix test deps * test * annotation_dependencies * ruff * take out the zero check * fix requirements new task name * move vep into new variants task * only annotate lookup from callset_ht * clean up mocks * r * force task * lint * whoops * remove force option from new variants * bad syntax * ruff format --------- Co-authored-by: Julia Klugherz <juliaklugherz@gmail.com>
1 parent a183132 commit 7a9a693

12 files changed

+127
-227
lines changed

‎v03_pipeline/lib/tasks/__init__.py

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
from v03_pipeline.lib.tasks.delete_old_runs import DeleteOldRunsTask
21
from v03_pipeline.lib.tasks.reference_data.write_cached_reference_dataset_query import (
32
WriteCachedReferenceDatasetQuery,
43
)
@@ -15,7 +14,6 @@
1514
)
1615

1716
__all__ = [
18-
'DeleteOldRunsTask',
1917
'UpdateProjectTableTask',
2018
'UpdateLookupTableTask',
2119
'UpdateVariantAnnotationsTableWithNewSamplesTask',

‎v03_pipeline/lib/tasks/delete_old_runs.py

Lines changed: 0 additions & 53 deletions
This file was deleted.

‎v03_pipeline/lib/tasks/delete_old_runs_test.py

Lines changed: 0 additions & 113 deletions
This file was deleted.

‎v03_pipeline/lib/tasks/update_lookup_table.py

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ class UpdateLookupTableTask(BaseUpdateTask):
3333
default=True,
3434
parsing=luigi.BoolParameter.EXPLICIT_PARSING,
3535
)
36+
force = luigi.BoolParameter(
37+
default=False,
38+
parsing=luigi.BoolParameter.EXPLICIT_PARSING,
39+
)
3640

3741
def output(self) -> luigi.Target:
3842
return GCSorLocalTarget(
@@ -43,31 +47,35 @@ def output(self) -> luigi.Target:
4347
)
4448

4549
def complete(self) -> bool:
46-
return super().complete() and hl.eval(
47-
hl.bind(
48-
lambda updates: hl.all(
49-
[
50-
updates.contains(
51-
hl.Struct(
52-
callset=callset_path,
53-
project_guid=project_guid,
54-
),
55-
)
56-
for (
57-
callset_path,
58-
project_guid,
59-
_,
60-
_,
61-
) in callset_project_pairs(
62-
self.callset_paths,
63-
self.project_guids,
64-
self.project_remap_paths,
65-
self.project_pedigree_paths,
66-
)
67-
],
50+
return (
51+
not self.force
52+
and super().complete()
53+
and hl.eval(
54+
hl.bind(
55+
lambda updates: hl.all(
56+
[
57+
updates.contains(
58+
hl.Struct(
59+
callset=callset_path,
60+
project_guid=project_guid,
61+
),
62+
)
63+
for (
64+
callset_path,
65+
project_guid,
66+
_,
67+
_,
68+
) in callset_project_pairs(
69+
self.callset_paths,
70+
self.project_guids,
71+
self.project_remap_paths,
72+
self.project_pedigree_paths,
73+
)
74+
],
75+
),
76+
hl.read_table(self.output().path).updates,
6877
),
69-
hl.read_table(self.output().path).updates,
70-
),
78+
)
7179
)
7280

7381
def requires(self) -> list[luigi.Task]:
@@ -83,6 +91,7 @@ def requires(self) -> list[luigi.Task]:
8391
self.ignore_missing_samples_when_subsetting,
8492
self.ignore_missing_samples_when_remapping,
8593
self.validate,
94+
self.force,
8695
)
8796
for (
8897
callset_path,

‎v03_pipeline/lib/tasks/update_project_table.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@ class UpdateProjectTableTask(BaseUpdateTask):
3232
default=True,
3333
parsing=luigi.BoolParameter.EXPLICIT_PARSING,
3434
)
35+
force = luigi.BoolParameter(
36+
default=False,
37+
parsing=luigi.BoolParameter.EXPLICIT_PARSING,
38+
)
3539
is_new_gcnv_joint_call = luigi.BoolParameter(
3640
default=False,
3741
description='Is this a fully joint-called callset.',
@@ -47,10 +51,14 @@ def output(self) -> luigi.Target:
4751
)
4852

4953
def complete(self) -> bool:
50-
return super().complete() and hl.eval(
51-
hl.read_table(self.output().path).updates.contains(
52-
self.callset_path,
53-
),
54+
return (
55+
not self.force
56+
and super().complete()
57+
and hl.eval(
58+
hl.read_table(self.output().path).updates.contains(
59+
self.callset_path,
60+
),
61+
)
5462
)
5563

5664
def requires(self) -> luigi.Task:
@@ -65,6 +73,7 @@ def requires(self) -> luigi.Task:
6573
self.ignore_missing_samples_when_subsetting,
6674
self.ignore_missing_samples_when_remapping,
6775
self.validate,
76+
self.force,
6877
)
6978

7079
def initialize_table(self) -> hl.Table:

‎v03_pipeline/lib/tasks/update_variant_annotations_table_with_new_samples.py

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,10 @@ class UpdateVariantAnnotationsTableWithNewSamplesTask(BaseVariantAnnotationsTabl
3030
default=True,
3131
parsing=luigi.BoolParameter.EXPLICIT_PARSING,
3232
)
33+
force = luigi.BoolParameter(
34+
default=False,
35+
parsing=luigi.BoolParameter.EXPLICIT_PARSING,
36+
)
3337
liftover_ref_path = luigi.OptionalParameter(
3438
default='gs://hail-common/references/grch38_to_grch37.over.chain.gz',
3539
description='Path to GRCh38 to GRCh37 coordinates file',
@@ -50,37 +54,42 @@ def requires(self) -> list[luigi.Task]:
5054
self.ignore_missing_samples_when_subsetting,
5155
self.ignore_missing_samples_when_remapping,
5256
self.validate,
57+
self.force,
5358
self.liftover_ref_path,
5459
self.run_id,
5560
),
5661
]
5762

5863
def complete(self) -> bool:
59-
return super().complete() and hl.eval(
60-
hl.bind(
61-
lambda updates: hl.all(
62-
[
63-
updates.contains(
64-
hl.Struct(
65-
callset=callset_path,
66-
project_guid=project_guid,
67-
),
68-
)
69-
for (
70-
callset_path,
71-
project_guid,
72-
_,
73-
_,
74-
) in callset_project_pairs(
75-
self.callset_paths,
76-
self.project_guids,
77-
self.project_remap_paths,
78-
self.project_pedigree_paths,
79-
)
80-
],
64+
return (
65+
not self.force
66+
and super().complete()
67+
and hl.eval(
68+
hl.bind(
69+
lambda updates: hl.all(
70+
[
71+
updates.contains(
72+
hl.Struct(
73+
callset=callset_path,
74+
project_guid=project_guid,
75+
),
76+
)
77+
for (
78+
callset_path,
79+
project_guid,
80+
_,
81+
_,
82+
) in callset_project_pairs(
83+
self.callset_paths,
84+
self.project_guids,
85+
self.project_remap_paths,
86+
self.project_pedigree_paths,
87+
)
88+
],
89+
),
90+
hl.read_table(self.output().path).updates,
8191
),
82-
hl.read_table(self.output().path).updates,
83-
),
92+
)
8493
)
8594

8695
def update_table(self, ht: hl.Table) -> hl.Table:

0 commit comments

Comments
 (0)