Update all measurements for the inner ear analysis

Author: constantinpape

scripts/inner_ear/analysis/analyze_distances.py

@@ -32,7 +32,6 @@ def _plot_all(distances):
     plt.show()
 
 
-# TODO rename the method names.
 # We only care about the following distances:
 # - MP-V -> PD, AZ (Boundary)
 # - Docked-V -> PD, AZ
@@ -98,7 +97,7 @@ def _plot(pool_name, distance_col, structure_name, ax):
 
 
 def for_tomos_with_annotation(plot_all=True):
-    manual_assignments, semi_automatic_assignments, automatic_assignments = get_measurements_with_annotation()
+    manual_assignments, semi_automatic_assignments, proofread_assignments = get_measurements_with_annotation()
 
     manual_distances = manual_assignments[
         ["tomogram", "pool", "ribbon_distance [nm]", "pd_distance [nm]", "boundary_distance [nm]"]
@@ -110,38 +109,38 @@ def for_tomos_with_annotation(plot_all=True):
     ]
     semi_automatic_distances["approach"] = ["semi_automatic"] * len(semi_automatic_distances)
 
-    automatic_distances = automatic_assignments[
+    proofread_distances = proofread_assignments[
         ["tomogram", "pool", "ribbon_distance [nm]", "pd_distance [nm]", "boundary_distance [nm]"]
     ]
-    automatic_distances["approach"] = ["automatic"] * len(automatic_distances)
+    proofread_distances["approach"] = ["proofread"] * len(proofread_distances)
 
-    distances = pd.concat([manual_distances, semi_automatic_distances, automatic_distances])
+    distances = pd.concat([manual_distances, semi_automatic_distances, proofread_distances])
     if plot_all:
-        distances.to_excel("./results/distances_with_manual_annotations.xlsx", index=False)
+        distances.to_excel("./results/distances_tomos_with_manual_annotations.xlsx", index=False)
         _plot_all(distances)
     else:
-        _plot_selected(distances, save_path="./results/selected_distances_manual_annotations.xlsx")
+        _plot_selected(distances, save_path="./results/selected_distances_tomos_with_manual_annotations.xlsx")
 
 
 def for_all_tomos(plot_all=True):
-    semi_automatic_assignments, automatic_assignments = get_all_measurements()
+    semi_automatic_assignments, proofread_assignments = get_all_measurements()
 
     semi_automatic_distances = semi_automatic_assignments[
         ["tomogram", "pool", "ribbon_distance [nm]", "pd_distance [nm]", "boundary_distance [nm]"]
     ]
     semi_automatic_distances["approach"] = ["semi_automatic"] * len(semi_automatic_distances)
 
-    automatic_distances = automatic_assignments[
+    proofread_distances = proofread_assignments[
         ["tomogram", "pool", "ribbon_distance [nm]", "pd_distance [nm]", "boundary_distance [nm]"]
     ]
-    automatic_distances["approach"] = ["automatic"] * len(automatic_distances)
+    proofread_distances["approach"] = ["proofread"] * len(proofread_distances)
 
-    distances = pd.concat([semi_automatic_distances, automatic_distances])
+    distances = pd.concat([semi_automatic_distances, proofread_distances])
     if plot_all:
-        distances.to_excel("./results/distances_all_tomograms.xlsx", index=False)
+        distances.to_excel("./results/distances_all_tomos.xlsx", index=False)
         _plot_all(distances)
     else:
-        _plot_selected(distances, save_path="./results/selected_distances_all_tomograms.xlsx")
+        _plot_selected(distances, save_path="./results/selected_distances_all_tomos.xlsx")
 
 
 def main():

scripts/inner_ear/analysis/analyze_vesicle_diameters.py

@@ -10,10 +10,12 @@
 from synaptic_reconstruction.imod.export import load_points_from_imodinfo
 from synaptic_reconstruction.file_utils import get_data_path
 
+from common import get_finished_tomos
+
 sys.path.append("../processing")
 
 
-def aggregate_radii(data_root, table, save_path, get_tab):
+def aggregate_radii(data_root, table, save_path, get_tab, include_names):
     if os.path.exists(save_path):
         return
 
@@ -24,6 +26,14 @@ def aggregate_radii(data_root, table, save_path, get_tab):
             continue
 
         tomo_name = os.path.relpath(folder, os.path.join(data_root, "Electron-Microscopy-Susi/Analyse"))
+        if (
+            tomo_name in ("WT strong stim/Mouse 1/modiolar/1", "WT strong stim/Mouse 1/modiolar/2") and
+            (row["EM alt vs. Neu"] == "neu")
+        ):
+            continue
+        if tomo_name not in include_names:
+            continue
+
         tab_path = get_tab(folder)
         if tab_path is None:
             continue
@@ -38,7 +48,7 @@ def aggregate_radii(data_root, table, save_path, get_tab):
     radius_table.to_excel(save_path, index=False)
 
 
-def aggregate_radii_imod(data_root, table, save_path):
+def aggregate_radii_imod(data_root, table, save_path, include_names):
     if os.path.exists(save_path):
         return
 
@@ -49,6 +59,15 @@ def aggregate_radii_imod(data_root, table, save_path):
             continue
 
         tomo_name = os.path.relpath(folder, os.path.join(data_root, "Electron-Microscopy-Susi/Analyse"))
+        tomo_name = os.path.relpath(folder, os.path.join(data_root, "Electron-Microscopy-Susi/Analyse"))
+        if (
+            tomo_name in ("WT strong stim/Mouse 1/modiolar/1", "WT strong stim/Mouse 1/modiolar/2") and
+            (row["EM alt vs. Neu"] == "neu")
+        ):
+            continue
+        if tomo_name not in include_names:
+            continue
+
         annotation_folder = os.path.join(folder, "manuell")
         if not os.path.exists(annotation_folder):
             annotation_folder = os.path.join(folder, "Manuell")
@@ -84,7 +103,7 @@ def aggregate_radii_imod(data_root, table, save_path):
     radius_table.to_excel(save_path, index=False)
 
 
-def get_tab_automatic(folder):
+def get_tab_semi_automatic(folder):
     tab_name = "measurements_uncorrected_assignments.xlsx"
     res_path = os.path.join(folder, "korrektur", tab_name)
     if not os.path.exists(res_path):
@@ -94,7 +113,7 @@ def get_tab_automatic(folder):
     return res_path
 
 
-def get_tab_semi_automatic(folder):
+def get_tab_proofread(folder):
     tab_name = "measurements.xlsx"
     res_path = os.path.join(folder, "korrektur", tab_name)
     if not os.path.exists(res_path):
@@ -121,11 +140,22 @@ def main():
     table_path = os.path.join(data_root, "Electron-Microscopy-Susi", "Übersicht.xlsx")
     table = parse_table(table_path, data_root)
 
-    # TODO get the radii from imod
-    aggregate_radii(data_root, table, save_path="./results/vesicle_radii_automatic.xlsx", get_tab=get_tab_automatic)
-    aggregate_radii(data_root, table, save_path="./results/vesicle_radii_semi_automatic.xlsx", get_tab=get_tab_semi_automatic)  # noqa
-    aggregate_radii(data_root, table, save_path="./results/vesicle_radii_manual.xlsx", get_tab=get_tab_manual)
-    aggregate_radii_imod(data_root, table, save_path="./results/vesicle_radii_imod.xlsx")
+    all_tomos = get_finished_tomos()
+    aggregate_radii(
+        data_root, table, save_path="./results/vesicle_radii_semi_automatic.xlsx", get_tab=get_tab_semi_automatic,
+        include_names=all_tomos
+    )
+
+    aggregate_radii(
+        data_root, table, save_path="./results/vesicle_radii_proofread.xlsx", get_tab=get_tab_proofread,
+        include_names=all_tomos
+    )
+
+    # aggregate_radii(data_root, table, save_path="./results/vesicle_radii_manual.xlsx", get_tab=get_tab_manual)
+    aggregate_radii_imod(
+        data_root, table, save_path="./results/vesicle_radii_manual.xlsx",
+        include_names=all_tomos
+    )
 
 
 if __name__ == "__main__":

scripts/inner_ear/analysis/analyze_vesicle_pools.py

@@ -35,68 +35,67 @@ def plot_pools(data, errors):
 
 
 def for_tomos_with_annotation():
-    manual_assignments, semi_automatic_assignments, automatic_assignments = get_measurements_with_annotation()
+    manual_assignments, semi_automatic_assignments, proofread_assignments = get_measurements_with_annotation()
 
     manual_counts = manual_assignments.groupby(["tomogram", "pool"]).size().unstack(fill_value=0)
     semi_automatic_counts = semi_automatic_assignments.groupby(["tomogram", "pool"]).size().unstack(fill_value=0)
-    automatic_counts = automatic_assignments.groupby(["tomogram", "pool"]).size().unstack(fill_value=0)
+    proofread_counts = proofread_assignments.groupby(["tomogram", "pool"]).size().unstack(fill_value=0)
 
     manual_stats = manual_counts.agg(["mean", "std"]).transpose().reset_index()
     semi_automatic_stats = semi_automatic_counts.agg(["mean", "std"]).transpose().reset_index()
-    automatic_stats = automatic_counts.agg(["mean", "std"]).transpose().reset_index()
+    proofread_stats = proofread_counts.agg(["mean", "std"]).transpose().reset_index()
 
     data = pd.DataFrame({
         "Pool": manual_stats["pool"],
         "Semi-automatic": semi_automatic_stats["mean"],
-        "Automatic": automatic_stats["mean"],
+        "Proofread": proofread_stats["mean"],
         "Manual": manual_stats["mean"],
     })
     errors = pd.DataFrame({
         "Pool": manual_stats["pool"],
         "Semi-automatic": semi_automatic_stats["std"],
-        "Automatic": automatic_stats["std"],
+        "Proofread": proofread_stats["std"],
         "Manual": manual_stats["std"],
     })
 
     plot_pools(data, errors)
 
-    output_path = "./results/vesicle_pools_with_manual_annotations.xlsx"
+    output_path = "./results/vesicle_pools_tomos_with_manual_annotations.xlsx"
     data.to_excel(output_path, index=False, sheet_name="Average")
     with pd.ExcelWriter(output_path, engine="openpyxl", mode="a") as writer:
         errors.to_excel(writer, sheet_name="StandardDeviation", index=False)
 
 
 def for_all_tomos():
-    semi_automatic_assignments, automatic_assignments = get_all_measurements()
+    semi_automatic_assignments, proofread_assignments = get_all_measurements()
 
-    automatic_counts = automatic_assignments.groupby(["tomogram", "pool"]).size().unstack(fill_value=0)
-    automatic_stats = automatic_counts.agg(["mean", "std"]).transpose().reset_index()
+    proofread_counts = proofread_assignments.groupby(["tomogram", "pool"]).size().unstack(fill_value=0)
+    proofread_stats = proofread_counts.agg(["mean", "std"]).transpose().reset_index()
 
     semi_automatic_counts = semi_automatic_assignments.groupby(["tomogram", "pool"]).size().unstack(fill_value=0)
     semi_automatic_stats = semi_automatic_counts.agg(["mean", "std"]).transpose().reset_index()
 
     data = pd.DataFrame({
-        "Pool": automatic_stats["pool"],
+        "Pool": proofread_stats["pool"],
         "Semi-automatic": semi_automatic_stats["mean"],
-        "Automatic": automatic_stats["mean"],
+        "Proofread": proofread_stats["mean"],
     })
     errors = pd.DataFrame({
-        "Pool": automatic_stats["pool"],
+        "Pool": proofread_stats["pool"],
         "Semi-automatic": semi_automatic_stats["std"],
-        "Automatic": automatic_stats["std"],
+        "Proofread": proofread_stats["std"],
     })
 
     plot_pools(data, errors)
 
-    output_path = "./results/vesicle_pools_all_tomograms.xlsx"
+    output_path = "./results/vesicle_pools_all_tomos.xlsx"
     data.to_excel(output_path, index=False, sheet_name="Average")
     with pd.ExcelWriter(output_path, engine="openpyxl", mode="a") as writer:
         errors.to_excel(writer, sheet_name="StandardDeviation", index=False)
 
 
-# TODO: export the ribbon and pd stats (first need to discuss this with Fid)
 def main():
-    # for_tomos_with_annotation()
+    for_tomos_with_annotation()
     for_all_tomos()
 
 

scripts/inner_ear/analysis/common.py

@@ -1,27 +1,51 @@
 # import os
 import sys
 
+import numpy as np
 import pandas as pd
 
 sys.path.append("../processing")
 
 from parse_table import get_data_root  # noqa
 
 
+def get_finished_tomos():
+    # data_root = get_data_root()
+    # val_table = os.path.join(data_root, "Electron-Microscopy-Susi", "Validierungs-Tabelle-v3.xlsx")
+
+    val_table = "/home/pape/Desktop/sfb1286/mboc_synapse/misc/Validierungs-Tabelle-v3-passt.xlsx"
+    val_table = pd.read_excel(val_table)
+
+    val_table = val_table[val_table["Kommentar 22.11.24"] == "passt"]
+    n_tomos = len(val_table)
+    assert n_tomos > 0
+
+    tomo_names = []
+    for _, row in val_table.iterrows():
+        name = "/".join([
+            row.Bedingung, f"Mouse {int(row.Maus)}",
+            row["Ribbon-Orientierung"].lower().rstrip("?"),
+            str(int(row["OwnCloud-Unterordner"]))]
+        )
+        tomo_names.append(name)
+
+    return tomo_names
+
+
 def get_manual_assignments():
-    result_path = "../results/20240917_1/fully_manual_analysis_results.xlsx"
+    result_path = "../results/20241124_1/fully_manual_analysis_results.xlsx"
     results = pd.read_excel(result_path)
     return results
 
 
-def get_semi_automatic_assignments(tomograms):
-    result_path = "../results/20240917_1/automatic_analysis_results.xlsx"
+def get_proofread_assignments(tomograms):
+    result_path = "../results/20241124_1/automatic_analysis_results.xlsx"
     results = pd.read_excel(result_path)
     results = results[results["tomogram"].isin(tomograms)]
     return results
 
 
-def get_automatic_assignments(tomograms):
+def get_semi_automatic_assignments(tomograms):
     result_path = "../results/fully_automatic_analysis_results.xlsx"
     results = pd.read_excel(result_path)
     results = results[results["tomogram"].isin(tomograms)]
@@ -30,48 +54,29 @@ def get_automatic_assignments(tomograms):
 
 def get_measurements_with_annotation():
     manual_assignments = get_manual_assignments()
-    manual_tomograms = pd.unique(manual_assignments["tomogram"])
-    semi_automatic_assignments = get_semi_automatic_assignments(manual_tomograms)
 
-    tomograms = pd.unique(semi_automatic_assignments["tomogram"])
-    manual_assignments = manual_assignments[manual_assignments["tomogram"].isin(tomograms)]
-    assert len(pd.unique(manual_assignments["tomogram"])) == len(pd.unique(semi_automatic_assignments["tomogram"]))
+    # Get the tomos with manual annotations and the ones which are fully done in proofreading.
+    manual_tomos = pd.unique(manual_assignments["tomogram"])
+    finished_tomos = get_finished_tomos()
+    # Intersect them to get the tomos we are using.
+    tomos = np.intersect1d(manual_tomos, finished_tomos)
 
-    automatic_assignments = get_automatic_assignments(tomograms)
-    filtered_tomograms = pd.unique(manual_assignments["tomogram"])
-    assert len(filtered_tomograms) == len(pd.unique(automatic_assignments["tomogram"]))
+    manual_assignments = manual_assignments[manual_assignments["tomogram"].isin(tomos)]
+    semi_automatic_assignments = get_semi_automatic_assignments(tomos)
+    proofread_assignments = get_proofread_assignments(tomos)
 
-    print("Tomograms with manual annotations:", len(filtered_tomograms))
-    return manual_assignments, semi_automatic_assignments, automatic_assignments
+    print("Tomograms with manual annotations:", len(tomos))
+    return manual_assignments, semi_automatic_assignments, proofread_assignments
 
 
 def get_all_measurements():
-    # data_root = get_data_root()
-    # val_table = os.path.join(data_root, "Electron-Microscopy-Susi", "Validierungs-Tabelle-v3.xlsx")
-
-    val_table = "/home/pape/Desktop/sfb1286/mboc_synapse/misc/Validierungs-Tabelle-v3-passt.xlsx"
-    val_table = pd.read_excel(val_table)
-
-    val_table = val_table[val_table["Kommentar 22.11.24"] == "passt"]
-    n_tomos = len(val_table)
-    print("All tomograms:", n_tomos)
-    assert n_tomos > 0
-    tomo_names = []
-    for _, row in val_table.iterrows():
-        name = "/".join([
-            row.Bedingung, f"Mouse {int(row.Maus)}",
-            row["Ribbon-Orientierung"].lower().rstrip("?"),
-            str(int(row["OwnCloud-Unterordner"]))]
-        )
-        tomo_names.append(name)
-
-    semi_automatic_assignments = get_semi_automatic_assignments(tomo_names)
-    filtered_tomo_names = pd.unique(semi_automatic_assignments["tomogram"]).tolist()
+    tomos = get_finished_tomos()
+    print("All tomograms:", len(tomos))
 
-    automatic_assignments = get_automatic_assignments(tomo_names)
-    assert len(filtered_tomo_names) == len(pd.unique(automatic_assignments["tomogram"]))
+    semi_automatic_assignments = get_semi_automatic_assignments(tomos)
+    proofread_assignments = get_proofread_assignments(tomos)
 
-    return semi_automatic_assignments, automatic_assignments
+    return semi_automatic_assignments, proofread_assignments
 
 
 def main():