Auto merge of #526 - Zeegomo:html-test-exp, r=pietroalbini

Tests for HTML reports

This adds support for other report types in minicrater. `IndexMap` is used instead of `HashMap` to ensure consistent serialization.
Needs to be merged after #520

Author: bors

Cargo.lock

@@ -69,6 +69,7 @@ remove_dir_all = "0.5.2"
 ctrlc = "3.1.3"
 prometheus = "0.7.0"
 cargo_metadata = "0.9.1"
+indexmap = "1.4.0"
 
 [dev-dependencies]
 assert_cmd = "0.10.1"

Cargo.toml

@@ -243,6 +243,8 @@ pub enum Crater {
         dest: Dest,
         #[structopt(name = "force", long = "force")]
         force: bool,
+        #[structopt(name = "output-templates", long = "output-templates")]
+        output_templates: bool,
     },
 
     #[structopt(name = "publish-report", about = "publish the experiment report to S3")]
@@ -258,6 +260,8 @@ pub enum Crater {
         s3_prefix: report::S3Prefix,
         #[structopt(name = "force", long = "force")]
         force: bool,
+        #[structopt(name = "output-templates", long = "output-templates")]
+        output_templates: bool,
     },
 
     #[structopt(name = "server")]
@@ -503,6 +507,7 @@ impl Crater {
                 ref ex,
                 ref dest,
                 force,
+                output_templates,
             } => {
                 let config = Config::load()?;
                 let db = Database::open()?;
@@ -527,6 +532,7 @@ impl Crater {
                         &experiment.get_crates(&db)?,
                         &report::FileWriter::create(dest.0.clone())?,
                         &config,
+                        output_templates,
                     );
 
                     if let Err(err) = res {
@@ -543,6 +549,7 @@ impl Crater {
                 ref ex,
                 ref s3_prefix,
                 force,
+                output_templates,
             } => {
                 let config = Config::load()?;
                 let db = Database::open()?;
@@ -569,6 +576,7 @@ impl Crater {
                         &experiment.get_crates(&db)?,
                         &report::S3Writer::create(client, s3_prefix.clone())?,
                         &config,
+                        output_templates,
                     );
 
                     if let Err(err) = res {

src/cli.rs

@@ -4,8 +4,8 @@ use crate::results::{
     FailureReason,
     TestResult::{self, BuildFail},
 };
+use indexmap::IndexMap;
 use std::collections::BTreeSet;
-use std::collections::HashMap;
 
 pub enum ToolchainSelect {
     Start,
@@ -22,20 +22,20 @@ pub enum ReportConfig {
 pub enum ReportCrates {
     Plain(Vec<CrateResult>),
     Complete {
-        tree: HashMap<Crate, Vec<CrateResult>>,
-        results: HashMap<TestResult, Vec<CrateResult>>,
+        tree: IndexMap<Crate, Vec<CrateResult>>,
+        results: IndexMap<TestResult, Vec<CrateResult>>,
     },
 }
 
 #[cfg_attr(test, derive(Debug, PartialEq))]
 pub struct TestResults {
-    pub categories: HashMap<Comparison, ReportCrates>,
-    pub info: HashMap<Comparison, u32>,
+    pub categories: IndexMap<Comparison, ReportCrates>,
+    pub info: IndexMap<Comparison, u32>,
 }
 
 fn analyze_detailed(toolchain: usize, crates: Vec<CrateResult>) -> ReportCrates {
-    let mut tree = HashMap::new();
-    let mut results = HashMap::new();
+    let mut tree = IndexMap::new();
+    let mut results = IndexMap::new();
 
     let mut root = Vec::new();
     for krate in crates {
@@ -75,7 +75,7 @@ fn analyze_detailed(toolchain: usize, crates: Vec<CrateResult>) -> ReportCrates
 }
 
 pub fn analyze_report(test: RawTestResults) -> TestResults {
-    let mut comparison = HashMap::new();
+    let mut comparison = IndexMap::new();
     for krate in test.crates {
         comparison
             .entry(krate.res)
@@ -86,9 +86,9 @@ pub fn analyze_report(test: RawTestResults) -> TestResults {
     let info = comparison
         .iter()
         .map(|(&key, vec)| (key, vec.len() as u32))
-        .collect::<HashMap<_, _>>();
+        .collect::<IndexMap<_, _>>();
 
-    let mut categories = HashMap::new();
+    let mut categories = IndexMap::new();
     for (cat, crates) in comparison {
         if let ReportConfig::Complete(toolchain) = cat.report_config() {
             // variants in an enum are numbered following an
@@ -191,18 +191,18 @@ mod tests {
                     panic!("invalid crate type")
                 }
             })
-            .collect::<HashMap<_, _>>();
+            .collect::<IndexMap<_, _>>();
         let analyzed = analyze_report(raw);
 
-        let mut info = HashMap::new();
+        let mut info = IndexMap::new();
         info.insert(Comparison::Regressed, 5);
         info.insert(Comparison::Fixed, 2);
         info.insert(Comparison::SameTestPass, 1);
 
         macro_rules! create_results {
             ($src:expr, $($key:expr => ($($krate:expr),*)),*) => {
                 {
-                    let mut map = HashMap::new();
+                    let mut map = IndexMap::new();
                     $(
                         let mut crates = Vec::new();
                         $(
@@ -248,7 +248,7 @@ mod tests {
 
         let test_pass = ReportCrates::Plain(vec![crates.remove("test-pass").unwrap()]);
 
-        let mut categories = HashMap::new();
+        let mut categories = IndexMap::new();
         categories.insert(Comparison::Regressed, regressed);
         categories.insert(Comparison::Fixed, fixed);
         categories.insert(Comparison::SameTestPass, test_pass);

src/report/analyzer.rs

@@ -5,7 +5,7 @@ use crate::prelude::*;
 use crate::report::{compare, ReportWriter};
 use crate::results::{EncodedLog, EncodingType, ReadResults};
 use flate2::{write::GzEncoder, Compression};
-use std::collections::HashMap;
+use indexmap::IndexMap;
 use tar::{Builder as TarBuilder, Header as TarHeader};
 
 #[derive(Serialize)]
@@ -23,7 +23,7 @@ pub fn write_logs_archives<DB: ReadResults, W: ReportWriter>(
 ) -> Fallible<Vec<Archive>> {
     let mut archives = Vec::new();
     let mut all = TarBuilder::new(GzEncoder::new(Vec::new(), Compression::default()));
-    let mut by_comparison = HashMap::new();
+    let mut by_comparison = IndexMap::new();
 
     for krate in crates {
         if config.should_skip(krate) {
@@ -86,7 +86,7 @@ pub fn write_logs_archives<DB: ReadResults, W: ReportWriter>(
         path: "logs-archives/all.tar.gz".to_string(),
     });
 
-    for (comparison, archive) in by_comparison.drain() {
+    for (comparison, archive) in by_comparison.drain(..) {
         let data = archive.into_inner()?.finish()?;
         dest.write_bytes(
             &format!("logs-archives/{}.tar.gz", comparison),

src/report/archives.rs

@@ -6,7 +6,7 @@ use crate::report::{
     ResultColor, ResultName, TestResults,
 };
 use crate::results::{EncodingType, FailureReason, TestResult};
-use std::collections::HashMap;
+use indexmap::IndexMap;
 
 #[derive(Serialize)]
 struct NavbarItem {
@@ -27,11 +27,11 @@ enum ReportCratesHTML {
     Plain(Vec<CrateResultHTML>),
     Tree {
         count: u32,
-        tree: HashMap<String, Vec<CrateResultHTML>>,
+        tree: IndexMap<String, Vec<CrateResultHTML>>,
     },
     RootResults {
         count: u32,
-        results: HashMap<String, Vec<CrateResultHTML>>,
+        results: IndexMap<String, Vec<CrateResultHTML>>,
     },
 }
 
@@ -62,11 +62,10 @@ struct ResultsContext<'a> {
     ex: &'a Experiment,
     nav: Vec<NavbarItem>,
     categories: Vec<(Comparison, ReportCratesHTML)>,
-    info: HashMap<Comparison, u32>,
+    info: IndexMap<Comparison, u32>,
     full: bool,
     crates_count: usize,
-
-    comparison_colors: HashMap<Comparison, Color>,
+    comparison_colors: IndexMap<Comparison, Color>,
     result_colors: Vec<Color>,
     result_names: Vec<String>,
 }
@@ -102,9 +101,10 @@ fn write_report<W: ReportWriter>(
     full: bool,
     to: &str,
     dest: &W,
+    output_templates: bool,
 ) -> Fallible<()> {
-    let mut comparison_colors = HashMap::new();
-    let mut test_results_to_int = HashMap::new();
+    let mut comparison_colors = IndexMap::new();
+    let mut test_results_to_int = IndexMap::new();
     let mut result_colors = Vec::new();
     let mut result_names = Vec::new();
 
@@ -165,7 +165,7 @@ fn write_report<W: ReportWriter>(
                                     .collect::<Vec<_>>(),
                             )
                         })
-                        .collect::<HashMap<_, _>>();
+                        .collect::<IndexMap<_, _>>();
                     let results = results
                         .into_iter()
                         .map(|(res, krates)| {
@@ -182,7 +182,7 @@ fn write_report<W: ReportWriter>(
                                     .collect::<Vec<_>>(),
                             )
                         })
-                        .collect::<HashMap<_, _>>();
+                        .collect::<IndexMap<_, _>>();
 
                     vec![
                         (
@@ -227,6 +227,14 @@ fn write_report<W: ReportWriter>(
     let html = minifier::html::minify(&assets::render_template("report/results.html", &context)?);
     dest.write_string(to, html.into(), &mime::TEXT_HTML)?;
 
+    if output_templates {
+        dest.write_string(
+            [to, ".context.json"].concat(),
+            serde_json::to_string(&context)?.into(),
+            &mime::APPLICATION_JSON,
+        )?;
+    }
+
     Ok(())
 }
 
@@ -235,6 +243,7 @@ fn write_downloads<W: ReportWriter>(
     crates_count: usize,
     available_archives: Vec<Archive>,
     dest: &W,
+    output_templates: bool,
 ) -> Fallible<()> {
     let context = DownloadsContext {
         ex,
@@ -247,6 +256,14 @@ fn write_downloads<W: ReportWriter>(
     let html = minifier::html::minify(&assets::render_template("report/downloads.html", &context)?);
     dest.write_string("downloads.html", html.into(), &mime::TEXT_HTML)?;
 
+    if output_templates {
+        dest.write_string(
+            "downloads.html.context.json",
+            serde_json::to_string(&context)?.into(),
+            &mime::APPLICATION_JSON,
+        )?;
+    }
+
     Ok(())
 }
 
@@ -256,12 +273,29 @@ pub fn write_html_report<W: ReportWriter>(
     res: &TestResults,
     available_archives: Vec<Archive>,
     dest: &W,
+    output_templates: bool,
 ) -> Fallible<()> {
     let js_in = assets::load("report.js")?;
     let css_in = assets::load("report.css")?;
-    write_report(ex, crates_count, res, false, "index.html", dest)?;
-    write_report(ex, crates_count, res, true, "full.html", dest)?;
-    write_downloads(ex, crates_count, available_archives, dest)?;
+    write_report(
+        ex,
+        crates_count,
+        res,
+        false,
+        "index.html",
+        dest,
+        output_templates,
+    )?;
+    write_report(
+        ex,
+        crates_count,
+        res,
+        true,
+        "full.html",
+        dest,
+        output_templates,
+    )?;
+    write_downloads(ex, crates_count, available_archives, dest, output_templates)?;
 
     info!("copying static assets");
     dest.write_bytes(