Update rules (#16)

* update plot pca and roc

* update plot scripts

* fix typo + add logs

* update common

* new report files

* Update report

* update rule graph

* update readme

Author: gibsramen

README.md

@@ -57,7 +57,7 @@ Qadabra generates many results files including many intermediate files that can
 Each tool's output is stored in a separate subdirectory.
 For the R tools, an RDS object with the tool's R data is saved.
 The raw outputs are processed and concatenated into a file called `concatenated_differentials.tsv`.
-A Qurro visualization of all the tool ranks is generated at `qurro/index.html`.
+A Qurro visualization of all the tool ranks is generated at `results/qurro/index.html`.
 An interactive table with all the tool outputs is at `results/differentials_table.html`.
 
 For each tool, the ranked features are used for machine learning models.
@@ -71,8 +71,7 @@ The `ml` subdirectory of each tool contains the features used, sample log-ratios
 The differential rank plots of each tool are plotted as `<tool_name>_differentials.svg`.
 A heatmap of the pairwise Spearman correlation among all pairs of tools is available as well.
 We also generated interactive plots to help compare the ranks of different microbes from the tools.
-`figures/pca.html` generates a PCA plot of all the microbes, showing the concordance and discordance of results as well as the contribution of the tools.
+`figures/pca.svg` generates a PCA plot of all the microbes, showing the concordance and discordance of results as well as the contribution of the tools.
 You can use the `figures/rank_comparisons.html` webpage to dynamically explore the relationship between pairs of tools.
 The `upset` subdirectory contains [UpSet](https://doi.org/10.1109%2FTVCG.2014.2346248) plots comparing the features from each tool.
-
 Finally, the `roc` subdirectory contains ROC plots of all tools at each percentile of features.

config/config.yaml

@@ -1,5 +1,6 @@
 table: "data/table.biom"
 metadata: "data/metadata.tsv"
+stylesheet: "config/qadabra.mplstyle"
 model:
     covariate: anemia
     target: anemic

imgs/rule_graph.png

@@ -0,0 +1 @@
+Table of differentials from all tools.

workflow/report/create_table.rst

@@ -0,0 +1 @@
+Plot of differentials from all tools. Differentials are mean-centered and scaled before PCA.

workflow/report/plot_pca.rst

@@ -0,0 +1 @@
+Interactive plot of rank comparisons. Select the two tools to compare and the plot will update.

workflow/report/plot_rank_comparison.rst

@@ -0,0 +1 @@
+Qurro visualization of differentials from all tools. For more information on Qurro, see `Fedarko 2020 <https://academic.oup.com/nargab/article/2/2/lqaa023/5826153>`_.

workflow/report/qurro.rst

@@ -30,7 +30,8 @@ all_viz_files.extend(
         pctile=config["log_ratio_feat_pcts"],
     )
 )
-all_viz_files.extend(expand("figures/{fig}.html", fig=["pca", "rank_comparisons"]))
+all_viz_files.append("figures/rank_comparisons.html")
+all_viz_files.append("figures/pca.svg")
 
 all_input = all_viz_files.copy()
 all_input.extend(["results/qurro", "results/differentials_table.html"])

workflow/rules/common.smk

@@ -159,41 +159,3 @@ rule combine_differentials:
         "../envs/qadabra-default.yaml"
     script:
         "../scripts/concatenate_differentials.py"
-
-
-rule qurro:
-    input:
-        ranks="results/concatenated_differentials.tsv",
-        table=config["table"],
-        metadata=config["metadata"],
-    output:
-        report(
-            directory("results/qurro"),
-            htmlindex="index.html",
-            category="Visualization",
-        ),
-    log:
-        "log/qurro.log",
-    conda:
-        "../envs/qadabra-songbird.yaml"
-    shell:
-        """
-        qurro \
-            --ranks {input.ranks} \
-            --table {input.table} \
-            --sample-metadata {input.metadata} \
-            --output-dir {output} > {log} 2>&1
-        """
-
-
-rule create_table:
-    input:
-        "results/concatenated_differentials.tsv",
-    output:
-        report("results/differentials_table.html", category="Visualization"),
-    log:
-        "log/create_table.log",
-    conda:
-        "../envs/qadabra-default.yaml"
-    script:
-        "../scripts/create_table.py"

workflow/rules/diffab.smk

@@ -1,7 +1,7 @@
 import re
 
 
-stylesheet = "config/qadabra.mplstyle"
+stylesheet = config["stylesheet"]
 
 
 rule plot_differentials:
@@ -11,14 +11,12 @@ rule plot_differentials:
         report(
             "figures/{tool}_differentials.svg",
             caption="../report/plot_differentials.rst",
-            category="Visualization",
-            subcategory="Differentials",
+            category="Differentials",
+            subcategory="Rank Plots",
             labels={"tool": "{tool}"},
         ),
     log:
         "log/plot_differentials.{tool}.log",
-    params:
-        stylesheet,
     conda:
         "../envs/qadabra-default.yaml"
     script:
@@ -32,19 +30,18 @@ rule plot_rank_correlation:
         report(
             "figures/spearman_heatmap.svg",
             caption="../report/plot_rank_correlation.rst",
-            category="Visualization",
+            category="Differentials",
+            subcategory="Comparison"
         ),
     log:
         "log/plot_rank_correlation.log",
-    params:
-        stylesheet,
     conda:
         "../envs/qadabra-default.yaml"
     script:
         "../scripts/plot_rank_correlations.py"
 
 
-rule upset:
+rule plot_upset:
     input:
         expand(
             "results/ml/{tool}/pctile_feats/pctile_{{pctile}}.tsv",
@@ -54,23 +51,19 @@ rule upset:
         numerator=report(
             "figures/upset/upset.pctile_{pctile}.numerator.svg",
             caption="../report/plot_upset.rst",
-            category="Visualization",
-            subcategory="UpSet",
+            category="UpSet",
             labels={"percentile": "{pctile}", "type": "numerator"},
         ),
         denominator=report(
             "figures/upset/upset.pctile_{pctile}.denominator.svg",
             caption="../report/plot_upset.rst",
-            category="Visualization",
-            subcategory="UpSet",
+            category="UpSet",
             labels={"percentile": "{pctile}", "type": "denominator"},
         ),
     wildcard_constraints:
         tool="(?!pca_pc1)\w*",
     log:
         "log/upset.pctile_{pctile}.log",
-    params:
-        stylesheet,
     conda:
         "../envs/qadabra-default.yaml"
     script:
@@ -87,14 +80,11 @@ rule plot_roc:
         report(
             "figures/roc/roc.pctile_{pctile}.svg",
             caption="../report/plot_roc.rst",
-            category="Visualization",
-            subcategory="ROC",
+            category="ROC",
             labels={"percentile": "{pctile}"},
         ),
     log:
         "log/plot_roc.{pctile}.log",
-    params:
-        stylesheet,
     conda:
         "../envs/qadabra-default.yaml"
     script:
@@ -108,9 +98,10 @@ rule plot_pca:
         prop_exp="results/pca/proportion_explained.tsv",
     output:
         report(
-            "figures/pca.html",
-            category="Visualization",
-            subcategory="Differentials",
+            "figures/pca.svg",
+            caption="../report/plot_pca.rst",
+            category="Differentials",
+            subcategory="Comparison"
         ),
     log:
         "log/plot_pca.log",
@@ -126,12 +117,58 @@ rule plot_rank_comparison:
     output:
         report(
             "figures/rank_comparisons.html",
-            category="Visualization",
-            subcategory="Differentials",
+            caption="../report/plot_rank_comparison.rst",
+            category="Differentials",
+            subcategory="Comparison"
         ),
     log:
         "log/plot_rank_comparison.log",
     conda:
         "../envs/qadabra-default.yaml"
     script:
         "../scripts/plot_rank_comparison.py"
+
+
+rule qurro:
+    input:
+        ranks="results/concatenated_differentials.tsv",
+        table=config["table"],
+        metadata=config["metadata"],
+    output:
+        report(
+            directory("results/qurro"),
+            caption="../report/qurro.rst",
+            htmlindex="index.html",
+            category="Differentials",
+            subcategory="Comparison"
+        ),
+    log:
+        "log/qurro.log",
+    conda:
+        "../envs/qadabra-songbird.yaml"
+    shell:
+        """
+        qurro \
+            --ranks {input.ranks} \
+            --table {input.table} \
+            --sample-metadata {input.metadata} \
+            --output-dir {output} > {log} 2>&1
+        """
+
+
+rule create_table:
+    input:
+        "results/concatenated_differentials.tsv",
+    output:
+        report(
+            "results/differentials_table.html",
+            caption="../report/create_table.rst",
+            category="Differentials",
+            subcategory="Comparison"
+        ),
+    log:
+        "log/create_table.log",
+    conda:
+        "../envs/qadabra-default.yaml"
+    script:
+        "../scripts/create_table.py"

workflow/rules/visualization.smk

@@ -15,7 +15,10 @@
 fh.setFormatter(formatter)
 logger.addHandler(fh)
 
-plt.style.use(snakemake.params[0])
+logging.captureWarnings(True)
+logging.getLogger("py.warnings").addHandler(fh)
+
+plt.style.use(snakemake.config["stylesheet"])
 
 logger.info("Loading differentials...")
 diffs = pd.read_table(snakemake.input[0], sep="\t", index_col=0).squeeze()