Add modified time to callset path (#1036)

* Add modified time to callset path

* Fix tests

* ruff

* add test

* test not worth it

* actual test

Author: bpblanken

v03_pipeline/lib/paths.py

@@ -2,6 +2,8 @@
 import os
 import re
 
+import hailtop.fs as hfs
+
 from v03_pipeline.lib.model import (
     AccessControl,
     DatasetType,
@@ -59,6 +61,25 @@ def _v03_reference_dataset_prefix(
     )
 
 
+def _callset_path_hash(callset_path: str) -> str:
+    # Include the most recent modified time of any
+    # of the callset shards if they exist.
+    try:
+        # hfs.ls throws FileNotFoundError if a non-wildcard is passed
+        # but not found, but does not throw if a wildcard is passed and
+        # there are no results.
+        shards = hfs.ls(callset_path)
+        if not shards:
+            key = callset_path
+        else:
+            key = callset_path + str(max(f.modification_time for f in shards))
+    except FileNotFoundError:
+        key = callset_path
+    return hashlib.sha256(
+        key.encode('utf8'),
+    ).hexdigest()
+
+
 def family_table_path(
     reference_genome: ReferenceGenome,
     dataset_type: DatasetType,
@@ -114,7 +135,7 @@ def imported_callset_path(
             dataset_type,
         ),
         'imported_callsets',
-        f'{hashlib.sha256(callset_path.encode("utf8")).hexdigest()}.mt',
+        f'{_callset_path_hash(callset_path)}.mt',
     )
 
 
@@ -178,7 +199,7 @@ def relatedness_check_table_path(
             dataset_type,
         ),
         'relatedness_check',
-        f'{hashlib.sha256(callset_path.encode("utf8")).hexdigest()}.ht',
+        f'{_callset_path_hash(callset_path)}.ht',
     )
 
 
@@ -194,7 +215,7 @@ def relatedness_check_tsv_path(
             dataset_type,
         ),
         'relatedness_check',
-        f'{hashlib.sha256(callset_path.encode("utf8")).hexdigest()}.tsv',
+        f'{_callset_path_hash(callset_path)}.tsv',
     )
 
 
@@ -212,7 +233,7 @@ def remapped_and_subsetted_callset_path(
         ),
         'remapped_and_subsetted_callsets',
         project_guid,
-        f'{hashlib.sha256(callset_path.encode("utf8")).hexdigest()}.mt',
+        f'{_callset_path_hash(callset_path)}.mt',
     )
 
 
@@ -256,7 +277,7 @@ def sex_check_table_path(
             dataset_type,
         ),
         'sex_check',
-        f'{hashlib.sha256(callset_path.encode("utf8")).hexdigest()}.ht',
+        f'{_callset_path_hash(callset_path)}.ht',
     )
 
 

v03_pipeline/lib/paths_test.py

@@ -1,6 +1,8 @@
 import unittest
 from unittest.mock import patch
 
+import hailtop.fs as hfs
+
 from v03_pipeline.lib.model import (
     DatasetType,
     ReferenceGenome,
@@ -24,6 +26,8 @@
     variant_annotations_table_path,
 )
 
+TEST_VCF = 'v03_pipeline/var/test/callsets/1kg_30varia*.vcf'
+
 
 class TestPaths(unittest.TestCase):
     def test_family_table_path(self) -> None:
@@ -39,15 +43,15 @@ def test_family_table_path(self) -> None:
         with patch('v03_pipeline.lib.paths.Env') as mock_env, patch(
             'v03_pipeline.lib.paths.FeatureFlag',
         ) as mock_ff:
-            mock_env.HAIL_SEARCH_DATA_DIR = 'gs://seqr-datasets/'
+            mock_env.HAIL_SEARCH_DATA_DIR = '/var/bucket/'
             self.assertEqual(
                 family_table_path(
                     ReferenceGenome.GRCh37,
                     DatasetType.SNV_INDEL,
                     SampleType.WES,
                     'franklin',
                 ),
-                'gs://seqr-datasets/v3.1/GRCh37/SNV_INDEL/families/WES/franklin.ht',
+                '/var/bucket/v3.1/GRCh37/SNV_INDEL/families/WES/franklin.ht',
             )
             mock_ff.INCLUDE_PIPELINE_VERSION_IN_PREFIX = False
             self.assertEqual(
@@ -57,15 +61,15 @@ def test_family_table_path(self) -> None:
                     SampleType.WES,
                     'franklin',
                 ),
-                'gs://seqr-datasets/GRCh37/SNV_INDEL/families/WES/franklin.ht',
+                '/var/bucket/GRCh37/SNV_INDEL/families/WES/franklin.ht',
             )
 
     def test_valid_filters_path(self) -> None:
         self.assertEqual(
             valid_filters_path(
                 DatasetType.MITO,
                 SampleType.WES,
-                'gs://bucket/RDG_Broad_WES_Internal_Oct2023/part_one_outputs/chr*/*.vcf.gz',
+                '/var/bucket/RDG_Broad_WES_Internal_Oct2023/part_one_outputs/chr*/*.vcf.gz',
             ),
             None,
         )
@@ -75,9 +79,9 @@ def test_valid_filters_path(self) -> None:
                 valid_filters_path(
                     DatasetType.SNV_INDEL,
                     SampleType.WES,
-                    'gs://bucket/RDG_Broad_WES_Internal_Oct2023/part_one_outputs/chr*/*.vcf.gz',
+                    '/var/bucket/RDG_Broad_WES_Internal_Oct2023/part_one_outputs/chr*/*.vcf.gz',
                 ),
-                'gs://bucket/RDG_Broad_WES_Internal_Oct2023/part_two_outputs/*.filtered.*.vcf.gz',
+                '/var/bucket/RDG_Broad_WES_Internal_Oct2023/part_two_outputs/*.filtered.*.vcf.gz',
             )
 
     def test_project_table_path(self) -> None:
@@ -105,19 +109,19 @@ def test_sex_check_table_path(self) -> None:
             sex_check_table_path(
                 ReferenceGenome.GRCh38,
                 DatasetType.SNV_INDEL,
-                'gs://abc.efg/callset.vcf.gz',
+                '/var/abc.efg/callset.vcf.gz',
             ),
-            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/sex_check/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.ht',
+            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/sex_check/f92b8ab6b5b8c41fa20d7d49a5626b96dcd2ba79fa6f61eab7ffb80d550d951c.ht',
         )
 
     def test_relatedness_check_table_path(self) -> None:
         self.assertEqual(
             relatedness_check_table_path(
                 ReferenceGenome.GRCh38,
                 DatasetType.SNV_INDEL,
-                'gs://abc.efg/callset.vcf.gz',
+                '/var/abc.efg/callset.vcf.gz',
             ),
-            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/relatedness_check/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.ht',
+            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/relatedness_check/f92b8ab6b5b8c41fa20d7d49a5626b96dcd2ba79fa6f61eab7ffb80d550d951c.ht',
         )
 
     def test_validation_errors_for_run_path(self) -> None:
@@ -154,31 +158,50 @@ def test_remapped_and_subsetted_callset_path(self) -> None:
             remapped_and_subsetted_callset_path(
                 ReferenceGenome.GRCh38,
                 DatasetType.GCNV,
-                'gs://abc.efg/callset.vcf.gz',
+                '/var/abc.efg/callset.vcf.gz',
                 'R0111_tgg_bblanken_wes',
             ),
-            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.mt',
+            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/f92b8ab6b5b8c41fa20d7d49a5626b96dcd2ba79fa6f61eab7ffb80d550d951c.mt',
         )
         self.assertEqual(
             remapped_and_subsetted_callset_path(
                 ReferenceGenome.GRCh38,
                 DatasetType.GCNV,
-                'gs://abc.efg/callset/*.vcf.gz',
+                '/var/abc.efg/callset/*.vcf.gz',
                 'R0111_tgg_bblanken_wes',
             ),
-            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/bce53ccdb49a5ed2513044e1d0c6224e3ffcc323f770dc807d9175fd3c70a050.mt',
+            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/GCNV/remapped_and_subsetted_callsets/R0111_tgg_bblanken_wes/26f481b386721f9889250c6549905660728ec9f77be4b8f7eeb6c4facc76282e.mt',
         )
 
     def test_imported_callset_path(self) -> None:
         self.assertEqual(
             imported_callset_path(
                 ReferenceGenome.GRCh38,
                 DatasetType.SNV_INDEL,
-                'gs://abc.efg/callset.vcf.gz',
+                '/var/abc.efg/callset.vcf.gz',
             ),
-            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/imported_callsets/ead56bb177a5de24178e1e622ce1d8beb3f8892bdae1c925d22ca0af4013d6dd.mt',
+            '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/imported_callsets/f92b8ab6b5b8c41fa20d7d49a5626b96dcd2ba79fa6f61eab7ffb80d550d951c.mt',
         )
 
+        with patch('v03_pipeline.lib.paths.hfs.ls') as mock_ls:
+            mock_ls.return_value = [
+                hfs.stat_result.FileListEntry(
+                    path='v03_pipeline/var/test/callsets/1kg_30variants.vcf',
+                    owner=None,
+                    size=104481,
+                    typ=hfs.stat_result.FileType(2),
+                    modification_time=1732033623.804012,
+                ),
+            ]
+            self.assertEqual(
+                imported_callset_path(
+                    ReferenceGenome.GRCh38,
+                    DatasetType.SNV_INDEL,
+                    TEST_VCF,
+                ),
+                '/var/seqr/seqr-loading-temp/v3.1/GRCh38/SNV_INDEL/imported_callsets/42f2c9e2025c4b61106b3fecfd30443f882a1849b73c6f6903a7e421c20117e0.mt',
+            )
+
     def test_tdr_metrics_path(self) -> None:
         self.assertEqual(
             tdr_metrics_path(