Handle duplicates in splice_ai and improve memory consumption when printing error message

Author: bpblanken

v03_pipeline/lib/misc/validation.py

@@ -53,7 +53,7 @@ def validate_no_duplicate_variants(
     ht = ht.select()
     if ht.count() > 0:
         variant_format = dataset_type.table_key_format_fn(reference_genome)
-        msg = f'Variants are present multiple times in the callset: {[variant_format(v) for v in ht.collect()][:10]}'
+        msg = f'Variants are present multiple times in the callset: {[variant_format(v) for v in ht.take(10)]}'
         raise SeqrValidationError(msg)
 
 

v03_pipeline/lib/reference_datasets/splice_ai.py

@@ -39,7 +39,7 @@ def get_ht(
         .map(hl.float32),
     )
     ht = ht.annotate(delta_score=hl.max(ht.delta_scores))
-    return ht.annotate(
+    ht = ht.annotate(
         splice_consequence_id=hl.if_else(
             ht.delta_score > 0,
             # Splice Consequence enum ID is the index of the max score
@@ -48,3 +48,4 @@ def get_ht(
             num_delta_scores,
         ),
     ).drop('delta_scores')
+    return ht.group_by(*ht.key).aggregate(splice_consequence_id=hl.agg.min(ht.splice_consequence_id))