Checkpoint no longer necessary with improved caching (#840)

* Checkpoint no longer necessary with improved caching

* move the reset to clinvar

Author: bpblanken

v03_pipeline/lib/reference_data/clinvar.py

@@ -125,8 +125,10 @@ def get_clinvar_ht(
         ht = hl.read_table(clinvar_ht_path)
     else:
         logger.info('Cached clinvar ht not found, downloading latest clinvar vcf')
+        hl._set_flags(use_new_shuffle=None, no_whole_stage_codegen='1')  # noqa: SLF001
         ht = download_and_import_latest_clinvar_vcf(clinvar_url, reference_genome)
         write(ht, clinvar_ht_path, repartition=False)
+        hl._set_flags(use_new_shuffle='1', no_whole_stage_codegen='1')  # noqa: SLF001
     return ht
 
 

v03_pipeline/lib/reference_data/dataset_table_operations.py

@@ -4,7 +4,6 @@
 import hail as hl
 import pytz
 
-from v03_pipeline.lib.misc.io import checkpoint
 from v03_pipeline.lib.misc.nested_field import parse_nested_field
 from v03_pipeline.lib.model import (
     DatasetType,
@@ -36,10 +35,7 @@ def update_or_create_joined_ht(
             continue
 
         # Join the new one!
-        hl._set_flags(use_new_shuffle=None, no_whole_stage_codegen='1')  # noqa: SLF001
         dataset_ht = get_dataset_ht(dataset, reference_genome)
-        dataset_ht, _ = checkpoint(dataset_ht)
-        hl._set_flags(use_new_shuffle='1', no_whole_stage_codegen='1')  # noqa: SLF001
         joined_ht = joined_ht.join(dataset_ht, 'outer')
         joined_ht = annotate_dataset_globals(joined_ht, dataset, dataset_ht)
 
@@ -216,10 +212,7 @@ def join_hts(
         ),
     )
     for dataset in reference_dataset_collection.datasets(dataset_type):
-        hl._set_flags(use_new_shuffle=None, no_whole_stage_codegen='1')  # noqa: SLF001
         dataset_ht = get_dataset_ht(dataset, reference_genome)
-        dataset_ht, _ = checkpoint(dataset_ht)
-        hl._set_flags(use_new_shuffle='1', no_whole_stage_codegen='1')  # noqa: SLF001
         joined_ht = joined_ht.join(dataset_ht, 'outer')
         joined_ht = annotate_dataset_globals(joined_ht, dataset, dataset_ht)
     return joined_ht