Merge pull request #7 from oli-obk/runtest

dogfood ui_test

Author: oli-obk

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "ui_test"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 description = "A test framework for testing rustc diagnostics output"
@@ -22,3 +22,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 color-eyre = { version = "0.6.1", default-features = false, features = ["capture-spantrace"] }
 cargo_metadata = "0.15"
+
+[[test]]
+name = "integration"
+harness = false

Cargo.toml

@@ -8,6 +8,7 @@
 use std::collections::VecDeque;
 use std::ffi::OsString;
 use std::fmt::Write;
+use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::process::{Command, ExitStatus};
 use std::sync::atomic::{AtomicUsize, Ordering};
@@ -32,6 +33,8 @@ mod tests;
 #[derive(Debug)]
 pub struct Config {
     /// Arguments passed to the binary that is executed.
+    /// Take care to only append unless you actually meant to overwrite the defaults.
+    /// Overwriting the defaults may make `//~ ERROR` style comments stop working.
     pub args: Vec<OsString>,
     /// `None` to run on the host, otherwise a target triple
     pub target: Option<String>,
@@ -53,23 +56,28 @@ pub struct Config {
     pub dependency_builder: Option<DependencyBuilder>,
     /// Print one character per test instead of one line
     pub quiet: bool,
+    /// How many threads to use for running tests. Defaults to number of cores
+    pub num_test_threads: NonZeroUsize,
 }
 
 impl Default for Config {
     fn default() -> Self {
         Self {
-            args: vec![],
+            args: vec!["--error-format=json".into()],
             target: None,
             stderr_filters: vec![],
             stdout_filters: vec![],
             root_dir: PathBuf::new(),
-            mode: Mode::Fail,
+            mode: Mode::Fail {
+                require_patterns: true,
+            },
             program: PathBuf::from("rustc"),
             output_conflict_handling: OutputConflictHandling::Error,
             path_filter: vec![],
             dependencies_crate_manifest_path: None,
             dependency_builder: None,
             quiet: true,
+            num_test_threads: std::thread::available_parallelism().unwrap(),
         }
     }
 }
@@ -104,9 +112,6 @@ pub type Filter = Vec<(Regex, &'static str)>;
 pub fn run_tests(mut config: Config) -> Result<()> {
     eprintln!("   Compiler flags: {:?}", config.args);
 
-    // Get the triple with which to run the tests
-    let target = config.target.clone().unwrap_or_else(|| config.get_host());
-
     let dependencies = build_dependencies(&config)?;
     for (name, dependency) in dependencies.dependencies {
         config.args.push("--extern".into());
@@ -119,7 +124,14 @@ pub fn run_tests(mut config: Config) -> Result<()> {
         config.args.push("-L".into());
         config.args.push(import_path.into());
     }
-    let config = config;
+    run_tests_generic(config, |path| {
+        path.extension().map(|ext| ext == "rs").unwrap_or(false)
+    })
+}
+
+pub fn run_tests_generic(config: Config, file_filter: impl Fn(&Path) -> bool + Sync) -> Result<()> {
+    // Get the triple with which to run the tests
+    let target = config.target.clone().unwrap_or_else(|| config.get_host());
 
     // A channel for files to process
     let (submit, receive) = crossbeam::channel::unbounded();
@@ -148,7 +160,7 @@ pub fn run_tests(mut config: Config) -> Result<()> {
                     for entry in entries {
                         todo.push_back(entry.path());
                     }
-                } else if path.extension().map(|ext| ext == "rs").unwrap_or(false) {
+                } else if file_filter(&path) {
                     // Forward .rs files to the test workers.
                     submit.send(path).unwrap();
                 }
@@ -198,7 +210,7 @@ pub fn run_tests(mut config: Config) -> Result<()> {
         let mut threads = vec![];
 
         // Create N worker threads that receive files to test.
-        for _ in 0..std::thread::available_parallelism().unwrap().get() {
+        for _ in 0..config.num_test_threads.get() {
             let finished_files_sender = finished_files_sender.clone();
             threads.push(s.spawn(|_| -> Result<()> {
                 let finished_files_sender = finished_files_sender;
@@ -412,7 +424,6 @@ fn run_test(
     if !revision.is_empty() {
         miri.arg(format!("--cfg={revision}"));
     }
-    miri.arg("--error-format=json");
     for arg in &comments.compile_flags {
         miri.arg(arg);
     }
@@ -586,7 +597,12 @@ fn check_annotations(
         comments.error_pattern.is_some() || !comments.error_matches.is_empty(),
     ) {
         (Mode::Pass, true) | (Mode::Panic, true) => errors.push(Error::PatternFoundInPassTest),
-        (Mode::Fail, false) => errors.push(Error::NoPatternsFound),
+        (
+            Mode::Fail {
+                require_patterns: true,
+            },
+            false,
+        ) => errors.push(Error::NoPatternsFound),
         _ => {}
     }
 }
@@ -695,18 +711,23 @@ impl Config {
 
 #[derive(Copy, Clone, Debug)]
 pub enum Mode {
-    // The test passes a full execution of the rustc driver
+    /// The test passes a full execution of the rustc driver
     Pass,
-    // The rustc driver panicked
+    /// The rustc driver panicked
     Panic,
-    // The rustc driver emitted an error
-    Fail,
+    /// The rustc driver emitted an error
+    Fail {
+        /// Whether failing tests must have error patterns. Set to false if you just care about .stderr output.
+        require_patterns: bool,
+    },
 }
 
 impl Mode {
     fn ok(self, status: ExitStatus) -> Errors {
         match (status.code(), self) {
-            (Some(1), Mode::Fail) | (Some(101), Mode::Panic) | (Some(0), Mode::Pass) => vec![],
+            (Some(1), Mode::Fail { .. }) | (Some(101), Mode::Panic) | (Some(0), Mode::Pass) => {
+                vec![]
+            }
             _ => vec![Error::ExitStatus(self, status)],
         }
     }

src/lib.rs

@@ -7,18 +7,10 @@ use super::*;
 
 fn config() -> Config {
     Config {
-        args: vec![],
-        target: None,
-        stderr_filters: vec![],
-        stdout_filters: vec![],
         root_dir: PathBuf::from("$RUSTROOT"),
-        mode: Mode::Fail,
-        path_filter: vec![],
         program: PathBuf::from("cake"),
-        output_conflict_handling: OutputConflictHandling::Error,
-        dependencies_crate_manifest_path: None,
-        dependency_builder: None,
         quiet: false,
+        ..Config::default()
     }
 }
 

src/tests.rs

@@ -1,19 +1,73 @@
-#[test]
-fn integrations() {
-    run("basic");
-}
+use std::path::Path;
 
-fn run(name: &str) {
-    let path = std::path::Path::new(file!()).parent().unwrap();
+use ui_test::color_eyre::Result;
+use ui_test::*;
 
-    let mut cmd = std::process::Command::new(std::env::var_os("CARGO").unwrap());
-    cmd.arg("test");
+fn main() -> Result<()> {
+    run("integrations", Mode::Pass)?;
+    run(
+        "integrations",
+        Mode::Fail {
+            require_patterns: false,
+        },
+    )
+}
 
-    cmd.arg("--target-dir");
-    cmd.arg(path.parent().unwrap().join("target"));
+fn run(name: &str, mode: Mode) -> Result<()> {
+    let path = Path::new(file!()).parent().unwrap();
+    let root_dir = path.join(name);
+    let mut config = Config {
+        root_dir: root_dir.clone(),
+        args: vec![
+            "test".into(),
+            "--color".into(),
+            "never".into(),
+            "--target-dir".into(),
+            path.parent().unwrap().join("target").into(),
+            "--manifest-path".into(),
+        ],
+        program: "cargo".into(),
+        output_conflict_handling: if std::env::var_os("BLESS").is_some() {
+            OutputConflictHandling::Bless
+        } else {
+            OutputConflictHandling::Error
+        },
+        mode,
+        ..Config::default()
+    };
 
-    cmd.arg("--manifest-path");
-    cmd.arg(format!("{}/integrations/{name}/Cargo.toml", path.display()));
+    config.stderr_filter("in [0-9\\.]+s", "");
+    config.stderr_filter("( +Running [^(]+).*", "$1");
+    config.stderr_filter(" *(Compiling|Downloaded|Downloading) .*\n", "");
+    // The order of the `/deps` directory flag is flaky
+    config.stderr_filter("/deps", "");
+    config.stderr_filter(
+        &std::path::Path::new(path)
+            .canonicalize()
+            .unwrap()
+            .parent()
+            .unwrap()
+            .display()
+            .to_string(),
+        "$$DIR",
+    );
+    config.stderr_filter("[0-9a-f]+\\.rmeta", "$$HASH.rmeta");
 
-    assert!(cmd.status().unwrap().success());
+    run_tests_generic(config, |path| {
+        let fail = path
+            .parent()
+            .unwrap()
+            .file_name()
+            .unwrap()
+            .to_str()
+            .unwrap()
+            .ends_with("-fail");
+        path.ends_with("Cargo.toml")
+            && path.parent().unwrap().parent().unwrap() == root_dir
+            && match mode {
+                Mode::Pass => !fail,
+                Mode::Panic => unreachable!(),
+                Mode::Fail { .. } => fail,
+            }
+    })
 }