bugfix: make sure file copy runs after the file handle is closed and the buffer is flushed (#1008)

Author: bpblanken

v03_pipeline/lib/reference_datasets/clinvar.py

@@ -168,7 +168,7 @@ def get_ht(
         delete=False,
     ) as tmp_file, requests.get(clinvar_url, stream=True, timeout=10) as r:
         shutil.copyfileobj(r.raw, tmp_file)
-        cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
+    cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
     ht = vcf_to_ht(cloud_tmp_file, reference_genome)
     # Filter deletions present as single alleles
     ht = ht.filter(hl.len(ht.alleles) == BIALLELIC)

v03_pipeline/lib/reference_datasets/misc.py

@@ -180,7 +180,4 @@ def copy_to_cloud_storage(file_name: str) -> str:
     else:
         cloud_storage_path = os.path.join(Env.HAIL_TMP_DIR, os.path.basename(file_name))
     hfs.copy(file_name, cloud_storage_path)
-    if os.path.getsize(file_name) != hfs.stat(cloud_storage_path).size:
-        msg = 'Local and Cloud storage file sizes differ'
-        raise AssertionError(msg)
     return cloud_storage_path