lower partitions

Author: bpblanken

v03_pipeline/lib/reference_datasets/splice_ai.py

@@ -20,7 +20,7 @@ def get_ht(
     ht, checkpoint_path = checkpoint(ht)
     # The default partitions are too big, leading to OOMs.
     ht = ht.repartition(
-        int(compute_hail_n_partitions(file_size_bytes(checkpoint_path)) * 10),
+        int(compute_hail_n_partitions(file_size_bytes(checkpoint_path)) * 2),
         # Note that shuffle=True here, since this is one of the few
         # cases in the pipeline where we want to increase the number
         # of partititons.