Auto merge of #389 - pietroalbini:minicrater-docs, r=pietroalbini

Add minicrater docs

This PR adds some docs about minicrater, making easier to contributors to change it.

Author: bors

README.md

@@ -44,4 +44,5 @@ guide](CONTRIBUTING.md).
 
 **Technical documentation:**
 
-* [Agent HTTP Api specification](docs/agent-http-api.md)
+* [minicrater docs](tests/minicrater/README.md)
+* [Agent HTTP API specification](docs/agent-http-api.md)

tests/minicrater/README.md

@@ -0,0 +1,59 @@
+# minicrater
+
+minicrater is the component of Crater's test suite that tests if the runs are
+actually executed and produce the correct result. It's inspired by rustc's
+compiletest.
+
+## Executing minicrater
+
+minicrater executions can take a few minutes to run, so it's ignored by default
+while running `cargo test`. You can run minicrater with:
+
+```
+$ cargo test minicrater -- --ignored --test-threads 1
+```
+
+The runs' output is hidden by default, but you can show it by setting the
+`MINICRATER_SHOW_OUTPUT` environment variable:
+
+```
+$ MINICRATER_SHOW_OUTPUT=1 cargo test minicrater -- --ignored --test-threads 1
+```
+
+## Adding tests to minicrater
+
+There are two ways to add a test to minicrater:
+
+* If your test requires different experiment configuration (like different
+  flags or `config.toml` file entries) and no other minicrater run has the
+  configuration you want you need to create a new minicrater run
+* Otherwise you can create a new local crate and have it tested by an existing
+  minicrater run (usually the full ones)
+
+minicrater runs are defined in `tests/minicrater/mod.rs`, and each run has
+additional configuration in the `tests/minicrater/<run>` directory: a
+`config.toml` with the configuration file used for the run, and
+`results.expected.json`, which contains the JSON output expected in the
+generated report.
+
+### Adding new local crates
+
+minicrater doesn't test public crates available on crates.io, but "local
+crates", which are dummy crates located in the `local-crates` directory at the
+top of the project. To add a new local crate you need to actually create the
+crate in that directory and then run:
+
+```
+$ cargo run create-lists
+```
+
+The `create-lists` command is only needed when you add or remove local crates,
+not when you edit one of them.
+
+### Adding new minicrater runs
+
+To add a new minicrater run, you need to add a new entry to
+`tests/minicrater/mod.rs` and a configuration file in
+`tests/minicrater/<run>/config.toml`. Then you run minicrater - expecting the
+experiment to be failing - and execute the command shown in the output to
+generate the expected JSON report.

tests/minicrater/driver.rs

@@ -0,0 +1,141 @@
+use crate::common::CommandCraterExt;
+use assert_cmd::prelude::*;
+use difference::Changeset;
+use rand::{self, distributions::Alphanumeric, Rng};
+use serde_json::{self, Value};
+use std::env;
+use std::path::PathBuf;
+use std::process::Command;
+
+trait CommandMinicraterExt {
+    fn minicrater_exec(&mut self);
+}
+
+impl CommandMinicraterExt for Command {
+    fn minicrater_exec(&mut self) {
+        if env::var_os("MINICRATER_SHOW_OUTPUT").is_some() {
+            assert!(self.status().unwrap().success());
+        } else {
+            self.assert().success();
+        }
+    }
+}
+
+pub(super) struct MinicraterRun {
+    pub(super) ex: &'static str,
+    pub(super) crate_select: &'static str,
+    pub(super) multithread: bool,
+    pub(super) ignore_blacklist: bool,
+}
+
+impl MinicraterRun {
+    pub(super) fn execute(&self) {
+        let ex_dir = PathBuf::from("tests").join("minicrater").join(self.ex);
+        let config_file = ex_dir.join("config.toml");
+        let expected_file = ex_dir.join("results.expected.json");
+        let actual_file = ex_dir.join("results.actual.json");
+
+        let threads_count = if self.multithread { num_cpus::get() } else { 1 };
+
+        let report_dir = tempfile::tempdir().expect("failed to create report dir");
+        let ex_arg = format!(
+            "--ex=minicrater-{}-{}",
+            self.ex,
+            rand::thread_rng()
+                .sample_iter(&Alphanumeric)
+                .take(10)
+                .collect::<String>()
+        );
+
+        // Create local list in the temp work dir
+        Command::crater()
+            .args(&["create-lists", "local"])
+            .env("CRATER_CONFIG", &config_file)
+            .minicrater_exec();
+
+        // Define the experiment
+        let crate_select = format!("--crate-select={}", self.crate_select);
+        let mut define_args = vec!["define-ex", &ex_arg, "stable", "beta", &crate_select];
+        if self.ignore_blacklist {
+            define_args.push("--ignore-blacklist");
+        }
+        Command::crater()
+            .args(&define_args)
+            .env("CRATER_CONFIG", &config_file)
+            .minicrater_exec();
+
+        // Execute the experiment
+        Command::crater()
+            .args(&[
+                "run-graph",
+                &ex_arg,
+                "--threads",
+                &threads_count.to_string(),
+            ])
+            .env("CRATER_CONFIG", &config_file)
+            .minicrater_exec();
+
+        // Generate the report
+        Command::crater()
+            .args(&["gen-report", &ex_arg])
+            .env("CRATER_CONFIG", &config_file)
+            .arg(report_dir.path())
+            .minicrater_exec();
+
+        // Read the JSON report
+        let json_report = ::std::fs::read(report_dir.path().join("results.json"))
+            .expect("failed to read json report");
+
+        // Delete the experiment
+        Command::crater()
+            .args(&["delete-ex", &ex_arg])
+            .env("CRATER_CONFIG", &config_file)
+            .minicrater_exec();
+
+        // Load the generated JSON report
+        let parsed_report: Value =
+            serde_json::from_slice(&json_report).expect("invalid json report");
+        let mut actual_report = serde_json::to_vec_pretty(&parsed_report).unwrap();
+        actual_report.push(b'\n');
+
+        // Load the expected JSON report
+        let expected_report = ::std::fs::read(&expected_file).unwrap_or(Vec::new());
+
+        // Write the actual JSON report
+        ::std::fs::write(&actual_file, &actual_report)
+            .expect("failed to write copy of the json report");
+
+        let changeset = Changeset::new(
+            &String::from_utf8(expected_report).expect("invalid utf-8 in the expected report"),
+            &String::from_utf8(actual_report.clone()).expect("invalid utf-8 in the actual report"),
+            "\n",
+        );
+        if changeset.distance != 0 {
+            eprintln!(
+                "Difference between expected and actual reports:\n{}",
+                changeset
+            );
+            eprintln!("To expect the new report in the future run:");
+            eprintln!(
+                "$ cp {} {}\n",
+                actual_file.to_string_lossy(),
+                expected_file.to_string_lossy()
+            );
+            panic!("invalid report generated by Crater");
+        }
+    }
+}
+
+#[macro_export]
+macro_rules! minicrater {
+    ($($name:ident $opts:tt,)*) => {
+        $(
+            #[test]
+            #[ignore]
+            fn $name() {
+                use $crate::minicrater::driver::MinicraterRun;
+                MinicraterRun $opts.execute();
+            }
+        )*
+    }
+}