Various bugfixes

Author: aofarrel

matutils_and_friends.wdl

@@ -819,7 +819,6 @@ task cluster_CDPH_method {
 		echo "Contents of workdir:"
 		tree
 		# A_big.nwk									big tree, nwk format
-		# all_neighbors.tsv
 		# LONELY-subtree-n.nwk (n as variable)		subtrees (usually multiple) of unclustered samples
 		# lonely-subtree-assignments.tsv			which subtree each unclustered sample ended up in
 		# cluster_annotation_workdirIDs.tsv			can be used to annotate by nonpersistent cluster (but isn't, at least not yet)
@@ -864,7 +863,7 @@ task cluster_CDPH_method {
 		File? change_report_json = "change_report" + today + ".json"
 
 		# brand new samples list, not fully finished processing but here ya go
-		File new_samples = "new_samples" + today + ".tsv"
+		File? new_samples = "new_samples" + today + ".tsv"
 
 		# trees, all in nwk format for now
 		# A = not internally masked
@@ -883,19 +882,19 @@ task cluster_CDPH_method {
 		Array[File]? bcluster_matrices = glob("b*_dmtrx.tsv")  # !UnnecessaryQuantifier
 
 		# cluster information
-		File unclustered_neighbors = "lonely_closest_relatives.txt"
+		File? unclustered_neighbors = "unclustered_neighbors.txt"
 		#File rosetta_stone_20 = "rosetta_stone_20.tsv"
 		#File rosetta_stone_10 = "rosetta_stone_10.tsv"
 		#File rosetta_stone_5 = "rosetta_stone_5.tsv"
-		File nearest_and_furtherst_info = "all_neighbors.tsv"
+		#File nearest_and_furtherst_info = "all_neighbors.tsv"
 		Int n_big_clusters = read_int("n_big_clusters")
 		Int n_samples_in_clusters = read_int("n_samples_in_clusters")
 		Int n_samples_processed = read_int("n_samples_processed")
 		Int n_unclustered = read_int("n_unclustered")
 
 		# old, maybe restore later?
 		#Array[File] abig_subtrees = glob("abig-subtree-*.nwk")
-		File samp_cluster = "samp_persiscluster" + today + ".tsv" # for nextstrain conversion
+		File? samp_cluster = "samp_persiscluster" + today + ".tsv" # for nextstrain conversion
 		#File? persistent_cluster_translator = "mapped_persistent_cluster_ids_to_new_cluster_ids.tsv"
 		#Array[File] cluster_trees_json = glob("*.json")
 		#Array[File] metadata_tsvs = glob("*.tsv")  # for auspice.us, which supports nwk

process_clusters.py

@@ -118,8 +118,10 @@ def main():
     # Are there any samples present in all_persistent_samples not present in all_latest_samples?
     # If no: Literally who cares, the perl script will handle it
     # If yes: Iterate the *persistent* clusters rowwise to make sure they aren't decimated... or just give up
-    all_latest_samples_set = set(all_latest_samples["latest_cluster_id"].to_list())
-    all_persistent_samples_set = set(all_persistent_samples["latest_cluster_id"].to_list())
+    all_latest_samples_set = set(all_latest_samples["sample_id"].to_list())
+    all_persistent_samples_set = set(all_persistent_samples["sample_id"].to_list())
+    print(all_latest_samples_set)
+    print(all_persistent_samples_set)
     if all_persistent_samples_set.issubset(all_latest_samples_set):
         logging.info("All persistent samples is a subset of all latest samples")
     else:
@@ -797,6 +799,7 @@ def main():
     change_report = []
 
     for row in all_cluster_information.iter_rows(named=True):
+        logging.debug("Checking %s", row['cluster_id'])
         try:
             what_is = set(row["sample_id"])
         except TypeError:
@@ -805,15 +808,18 @@ def main():
             what_was = set(row["sample_id_previously"])
         except TypeError:
             what_was = set()
+        logging.debug("what is: %s", what_is)
+        logging.debug("what was: %s", what_was)
         if len(what_is.intersection(what_was)) == len(what_was):
-            change_report.append({"cluster": f"{row['cluster_id']}@{row['cluster_distance']}", "gained": None, "lost": None, "maintained": list(what_is.intersection(what_was))})
+            change_report.append({"cluster": f"{row['cluster_id']}@{row['cluster_distance']}", "gained": [], "lost": [], "maintained": list(what_is.intersection(what_was))})
         else:
             change_report.append({"cluster": f"{row['cluster_id']}@{row['cluster_distance']}", "gained": list(what_is - what_was), "lost": list(what_was - what_is), "maintained": list(what_is.intersection(what_was))})
     change_report_df = pl.DataFrame(change_report).with_columns([
         pl.when(pl.col('gained').list.len() == 0).then(None).otherwise(pl.col('gained')).alias("gained"),
         pl.when(pl.col('lost').list.len() == 0).then(None).otherwise(pl.col('lost')).alias("lost"),
         pl.when(pl.col('maintained').list.len() == 0).then(None).otherwise(pl.col('maintained')).alias("maintained"),
         ])
+    logging.info("Finished. Here's how clusters have changed:")
     print(change_report_df)
     change_report_df.write_ndjson(f'change_report{today.isoformat()}.json')
 
@@ -876,7 +882,7 @@ def get_nwk_and_matrix_plus_local_mask(big_ol_dataframe, combineddiff):
                         logging.debug("[%s] matUtils mask returned 0 (atree.pb --> masked btree.pb)", this_cluster_id)
                         subprocess.run(f"matUtils extract -i {btreepb} -t {btree}", shell=True, check=True)
                         logging.debug("[%s] matUtils extract returned 0 (masked btree.pb --> masked btree.nwk)", this_cluster_id)
-                        subprocess.run(f"python3 /scripts/find_clusters.py {btreepb} {btree} --type BM --collection-name {this_cluster_id} -jmatsu", shell=True, check=True)
+                        subprocess.run(f"python3 /scripts/find_clusters.py {btreepb} --type BM --collection-name {this_cluster_id} -jmatsu", shell=True, check=True)
                         logging.debug("[%s] ran find_clusters.py, looks like it returned 0", this_cluster_id)
                         bmatrix = f"b{this_cluster_id}_dmtrx.tsv" if os.path.exists(f"b{this_cluster_id}_dmtrx.tsv") else None
                     except subprocess.CalledProcessError as e:

tree_nine.wdl

@@ -271,7 +271,7 @@ workflow Tree_Nine {
 		# other cluster information
 		File? new_persistent_ids = cluster.new_persistent_ids
 		File? new_persistent_meta = cluster.new_persistent_meta
-		File? nearest_and_furtherst_info = cluster.nearest_and_furtherst_info
+		File? unclustered_neighbors = cluster.unclustered_neighbors
 		File? final_cluster_information_json = cluster.final_cluster_information_json
 		Int?  nb_n_clusters = cluster.n_big_clusters
 		Int?  nb_n_samps_unclustered = cluster.n_unclustered