Update cryovesnet evaluation

Author: constantinpape

scripts/baselines/cryo_ves_net/common.py

@@ -53,7 +53,18 @@ def _prepare_input(path, output_folder, input_key, resolution, rel_folder=None):
     return out_path, sub_folder
 
 
-def _process_output(tmp, tmp_file, output_folder, output_key, rel_folder=None, mask_file=None, mask_key=None):
+def _get_output_path(fname, output_folder, rel_folder=None):
+    if rel_folder is None:
+        this_output_folder = output_folder
+    else:
+        this_output_folder = os.path.join(output_folder, rel_folder)
+        os.makedirs(this_output_folder, exist_ok=True)
+
+    out_path = os.path.join(this_output_folder, f"{fname}.h5")
+    return out_path
+
+
+def _process_output(tmp, tmp_file, out_path, output_key, mask_file=None, mask_key=None):
     fname = Path(tmp_file).stem
     seg_path = os.path.join(tmp, "cryovesnet", f"{fname}_convex_labels.mrc")
     with mrcfile.open(seg_path, "r") as f:
@@ -66,13 +77,6 @@ def _process_output(tmp, tmp_file, output_folder, output_key, rel_folder=None, m
         seg = np.asarray(seg).copy()
         seg[~mask] = 0
 
-    if rel_folder is None:
-        this_output_folder = output_folder
-    else:
-        this_output_folder = os.path.join(output_folder, rel_folder)
-        os.makedirs(this_output_folder, exist_ok=True)
-
-    out_path = os.path.join(this_output_folder, f"{fname}.h5")
     with h5py.File(out_path, "a") as f:
         f.create_dataset(output_key, data=seg, compression="gzip")
 
@@ -101,16 +105,22 @@ def apply_cryo_vesnet(
     with tempfile.TemporaryDirectory() as tmp:
 
         for i, file in enumerate(files):
+            fname = Path(file).stem
 
             # Get the resolution info for this file.
             if resolution is None:
                 res = None
             else:
-                fname = Path(file).stem
                 res = resolution[fname] if isinstance(resolution, dict) else resolution
 
-            # Prepare the input files by copying them over or resaving them (if h5).
+            # Get the current output path, skip processing if it already exists
             rel_folder = os.path.split(os.path.relpath(file, input_folder))[0] if nested else None
+            out_path = _get_output_path(fname, output_folder, rel_folder=rel_folder)
+            if os.path.exists(out_path):
+                print("Skipping processing of the", file, "because the output at", out_path, "already exists")
+                continue
+
+            # Prepare the input files by copying them over or resaving them (if h5).
             tmp_file, sub_folder = _prepare_input(file, tmp, input_key, res, rel_folder=rel_folder)
 
             # Segment the vesicles in the file.
@@ -119,5 +129,5 @@ def apply_cryo_vesnet(
             # Write the output file.
             mask_file = None if mask_files is None else mask_files[i]
             _process_output(
-                sub_folder, tmp_file, output_folder, output_key, rel_folder, mask_file=mask_file, mask_key=mask_key
+                sub_folder, tmp_file, out_path, output_key, mask_file=mask_file, mask_key=mask_key
             )

scripts/baselines/cryo_ves_net/evaluate_cooper.py

@@ -13,6 +13,7 @@
     "01_hoi_maus_2020_incomplete",
     "02_hcc_nanogold",
     "03_hog_cs1sy7",
+    "04",
     "05_stem750_sv_training",
     "07_hoi_s1sy7_tem250_ihgp",
     "10_tem_single_release",
@@ -30,8 +31,8 @@ def evaluate_dataset(ds_name):
         return results
 
     print("Evaluating ds", ds_name)
-    input_files = sorted(glob(os.path.join(INPUT_ROOT, ds_name, "*.h5")))
-    pred_files = sorted(glob(os.path.join(OUTPUT_ROOT, ds_name, "*.h5")))
+    input_files = sorted(glob(os.path.join(INPUT_ROOT, ds_name, "**/*.h5"), recursive=True))
+    pred_files = sorted(glob(os.path.join(OUTPUT_ROOT, ds_name, "**/*.h5"), recursive=True))
 
     results = {
         "dataset": [],
@@ -76,7 +77,7 @@ def main():
                           "07_hoi_s1sy7_tem250_ihgp",
                           "10_tem_single_release",
                           "11_tem_multiple_release"],
-        "STEM": ["05_stem750_sv_training"],
+        "STEM": ["04", "05_stem750_sv_training"],
         "Overall": DATASETS,
     }
 

scripts/baselines/cryo_ves_net/evaluate_cryo.py

@@ -0,0 +1,61 @@
+import os
+from glob import glob
+
+import h5py
+import pandas as pd
+from elf.evaluation.matching import matching
+
+
+INPUT_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/fernandez-busnadiego/vesicle_gt/v3"  # noqa
+OUTPUT_FOLDER = "./predictions/cryo"
+
+
+def evaluate_dataset(ds_name="cryo"):
+    result_folder = "./results/cryo"
+    os.makedirs(result_folder, exist_ok=True)
+    result_path = os.path.join(result_folder, f"{ds_name}.csv")
+    if os.path.exists(result_path):
+        results = pd.read_csv(result_path)
+        return results
+
+    print("Evaluating ds", ds_name)
+    input_files = sorted(glob(os.path.join(INPUT_FOLDER, "*.h5")))
+    pred_files = sorted(glob(os.path.join(OUTPUT_FOLDER, "*.h5")))
+
+    results = {
+        "dataset": [],
+        "file": [],
+        "precision": [],
+        "recall": [],
+        "f1-score": [],
+    }
+    for inf, predf in zip(input_files, pred_files):
+        fname = os.path.basename(inf)
+
+        with h5py.File(inf, "r") as f:
+            gt = f["/labels/vesicles"][:]
+        with h5py.File(predf, "r") as f:
+            seg = f["/prediction/vesicles/cryovesnet"][:]
+        assert gt.shape == seg.shape
+
+        scores = matching(seg, gt)
+
+        results["dataset"].append(ds_name)
+        results["file"].append(fname)
+        results["precision"].append(scores["precision"])
+        results["recall"].append(scores["recall"])
+        results["f1-score"].append(scores["f1"])
+
+    results = pd.DataFrame(results)
+    results.to_csv(result_path, index=False)
+    return results
+
+
+def main():
+    result = evaluate_dataset()
+    print(result)
+    print(result["f1-score"].mean())
+
+
+if __name__ == "__main__":
+    main()

scripts/baselines/cryo_ves_net/evaluate_endbulb.py

@@ -0,0 +1,63 @@
+import os
+from glob import glob
+
+import h5py
+import pandas as pd
+from elf.evaluation.matching import matching
+
+
+INPUT_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/wichmann/extracted/endbulb_of_held/Automatische_Segmentierung_Dataset_Validierung"  # noqa
+OUTPUT_FOLDER = "./predictions/endbulb"
+
+
+def evaluate_dataset(ds_name="endbulb"):
+    result_folder = "./results/endbulb"
+    os.makedirs(result_folder, exist_ok=True)
+    result_path = os.path.join(result_folder, f"{ds_name}.csv")
+    if os.path.exists(result_path):
+        results = pd.read_csv(result_path)
+        return results
+
+    print("Evaluating ds", ds_name)
+    input_files = sorted(glob(os.path.join(INPUT_FOLDER, "*.h5")))
+    pred_files = sorted(glob(os.path.join(OUTPUT_FOLDER, "*.h5")))
+
+    results = {
+        "dataset": [],
+        "file": [],
+        "precision": [],
+        "recall": [],
+        "f1-score": [],
+    }
+    for inf, predf in zip(input_files, pred_files):
+        fname = os.path.basename(inf)
+
+        with h5py.File(inf, "r") as f:
+            gt = f["/labels/vesicles"][:]
+        with h5py.File(predf, "r") as f:
+            seg = f["/prediction/vesicles/cryovesnet"][:]
+        assert gt.shape == seg.shape
+
+        scores = matching(seg, gt)
+
+        results["dataset"].append(ds_name)
+        results["file"].append(fname)
+        results["precision"].append(scores["precision"])
+        results["recall"].append(scores["recall"])
+        results["f1-score"].append(scores["f1"])
+
+    results = pd.DataFrame(results)
+    results.to_csv(result_path, index=False)
+    return results
+
+
+def main():
+    result = evaluate_dataset()
+    print(result)
+    print()
+    print(result["f1-score"].mean())
+    print(result["f1-score"].std())
+
+
+if __name__ == "__main__":
+    main()

scripts/baselines/cryo_ves_net/evaluate_inner_ear.py

@@ -0,0 +1,63 @@
+import os
+from glob import glob
+
+import h5py
+import pandas as pd
+from elf.evaluation.matching import matching
+
+
+INPUT_FOLDER = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/moser/vesicle_gt"  # noqa
+OUTPUT_FOLDER = "./predictions/inner_ear"
+
+
+def evaluate_dataset(ds_name="inner_ear"):
+    result_folder = "./results/inner_ear"
+    os.makedirs(result_folder, exist_ok=True)
+    result_path = os.path.join(result_folder, f"{ds_name}.csv")
+    if os.path.exists(result_path):
+        results = pd.read_csv(result_path)
+        return results
+
+    print("Evaluating ds", ds_name)
+    input_files = sorted(glob(os.path.join(INPUT_FOLDER, "*.h5")))
+    pred_files = sorted(glob(os.path.join(OUTPUT_FOLDER, "*.h5")))
+
+    results = {
+        "dataset": [],
+        "file": [],
+        "precision": [],
+        "recall": [],
+        "f1-score": [],
+    }
+    for inf, predf in zip(input_files, pred_files):
+        fname = os.path.basename(inf)
+
+        with h5py.File(inf, "r") as f:
+            gt = f["/labels/vesicles"][:]
+        with h5py.File(predf, "r") as f:
+            seg = f["/prediction/vesicles/cryovesnet"][:]
+        assert gt.shape == seg.shape
+
+        scores = matching(seg, gt)
+
+        results["dataset"].append(ds_name)
+        results["file"].append(fname)
+        results["precision"].append(scores["precision"])
+        results["recall"].append(scores["recall"])
+        results["f1-score"].append(scores["f1"])
+
+    results = pd.DataFrame(results)
+    results.to_csv(result_path, index=False)
+    return results
+
+
+def main():
+    result = evaluate_dataset()
+    print(result)
+    print()
+    print(result["f1-score"].mean())
+    print(result["f1-score"].std())
+
+
+if __name__ == "__main__":
+    main()

scripts/baselines/cryo_ves_net/segment_cooper.py

@@ -33,7 +33,26 @@ def segment_dataset(ds_name, resolution):
     )
 
 
+def segment_04():
+    ds_name = "04"
+    input_folder = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/cooper/ground_truth/04Dataset_for_vesicle_eval"  # noqa
+    output_folder = os.path.join(OUTPUT_ROOT, ds_name)
+
+    n_inputs = len(glob(os.path.join(input_folder, "**/*.h5"), recursive=True))
+    n_outputs = len(glob(os.path.join(output_folder, "**/*.h5"), recursive=True))
+    if n_inputs == n_outputs:
+        print(ds_name, "is already processed")
+        return
+
+    resolution = (8.681, 8.681, 8.681)
+    apply_cryo_vesnet(
+        input_folder, output_folder, pattern="*.h5", input_key="raw", resolution=resolution, nested=True,
+        mask_folder=input_folder, mask_key="labels/compartment"
+    )
+
+
 def main():
+    segment_04()
     for ds_name, resolution in RESOLUTIONS.items():
         resolution = tuple(res * 10 for res in resolution)
         segment_dataset(ds_name, resolution)

scripts/baselines/cryo_ves_net/segment_cryo.py

@@ -2,8 +2,8 @@
 
 
 def main():
-    input_folder = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/fernandez-busnadiego/vesicle_gt/v2"  # noqa
-    output_folder = "./cryo-vesnet-test"
+    input_folder = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/fernandez-busnadiego/vesicle_gt/v3"  # noqa
+    output_folder = "./predictions/cryo"
 
     # Resolution in Angstrom in XYZ
     # The two tomograms have a different resolution.

scripts/baselines/cryo_ves_net/segment_endbulb.py

@@ -0,0 +1,20 @@
+from common import apply_cryo_vesnet
+
+
+def main():
+    input_folder = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/wichmann/extracted/endbulb_of_held/Automatische_Segmentierung_Dataset_Validierung"  # noqa
+    output_folder = "./predictions/endbulb"
+
+    # Resolution in Angstrom in XYZ
+    # The two tomograms have a different resolution.
+    resolution = (17.48,) * 3
+    apply_cryo_vesnet(
+        input_folder, output_folder,
+        pattern="*.h5", input_key="raw",
+        mask_folder=input_folder, mask_key="labels/endbulb",
+        resolution=resolution
+    )
+
+
+if __name__ == "__main__":
+    main()

scripts/baselines/cryo_ves_net/segment_inner_ear.py

@@ -0,0 +1,19 @@
+from common import apply_cryo_vesnet
+
+
+def main():
+    input_folder = "/mnt/lustre-emmy-hdd/projects/nim00007/data/synaptic-reconstruction/moser/vesicle_gt"  # noqa
+    output_folder = "./predictions/inner_ear"
+
+    # Resolution in Angstrom in XYZ
+    # The two tomograms have a different resolution.
+    resolution = (11.8, 11.8, 11.88)
+    apply_cryo_vesnet(
+        input_folder, output_folder,
+        pattern="*.h5", input_key="raw",
+        resolution=resolution
+    )
+
+
+if __name__ == "__main__":
+    main()