Merge branch 'main' of github.com:broadinstitute/seqr-loading-pipelines into dev

Author: bpblanken

v03_pipeline/lib/reference_datasets/clinvar.py

@@ -113,7 +113,7 @@ def get_submission_summary_ht() -> hl.Table:
         timeout=10,
     ) as r:
         shutil.copyfileobj(r.raw, tmp_file)
-        cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
+    cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
     ht = hl.import_table(
         cloud_tmp_file,
         force=True,
@@ -168,7 +168,7 @@ def get_ht(
         delete=False,
     ) as tmp_file, requests.get(clinvar_url, stream=True, timeout=10) as r:
         shutil.copyfileobj(r.raw, tmp_file)
-        cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
+    cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
     ht = vcf_to_ht(cloud_tmp_file, reference_genome)
     # Filter deletions present as single alleles
     ht = ht.filter(hl.len(ht.alleles) == BIALLELIC)

v03_pipeline/lib/reference_datasets/helix_mito.py

@@ -23,7 +23,7 @@ def get_ht(
         delete=False,
     ) as tmp_file, requests.get(url, stream=True, timeout=10) as r:
         shutil.copyfileobj(r.raw, tmp_file)
-        cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
+    cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
     ht = hl.import_table(
         cloud_tmp_file,
         types={

v03_pipeline/lib/reference_datasets/misc.py

@@ -176,8 +176,8 @@ def copy_to_cloud_storage(file_name: str) -> str:
     if not Env.HAIL_TMP_DIR.startswith('gs://'):
         return file_name
     if os.path.isdir(file_name):
-        path = os.path.join(Env.HAIL_TMP_DIR, file_name.lstrip('/'))
+        cloud_storage_path = os.path.join(Env.HAIL_TMP_DIR, file_name.lstrip('/'))
     else:
-        path = os.path.join(Env.HAIL_TMP_DIR, os.path.basename(file_name))
-    hfs.copy(file_name, path)
-    return path
+        cloud_storage_path = os.path.join(Env.HAIL_TMP_DIR, os.path.basename(file_name))
+    hfs.copy(file_name, cloud_storage_path)
+    return cloud_storage_path

v03_pipeline/lib/reference_datasets/screen.py

@@ -21,7 +21,7 @@ def get_ht(path: str, reference_genome: ReferenceGenome) -> hl.Table:
         timeout=10,
     ) as r:
         shutil.copyfileobj(r.raw, tmp_file)
-        cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
+    cloud_tmp_file = copy_to_cloud_storage(tmp_file.name)
     ht = hl.import_table(
         cloud_tmp_file,
         no_header=True,

v03_pipeline/lib/tasks/delete_project_table.py

@@ -14,12 +14,7 @@ class DeleteProjectTableTask(BaseDeleteTableTask):
     project_guid = luigi.Parameter()
 
     def requires(self) -> luigi.Task:
-        return DeleteProjectFamilyTablesTask(
-            self.reference_genome,
-            self.dataset_type,
-            self.sample_type,
-            self.project_guid,
-        )
+        return self.clone(DeleteProjectFamilyTablesTask)
 
     def output(self) -> luigi.Target:
         return GCSorLocalTarget(