Add multiple tests

Author: oli-obk

src/lib.rs

@@ -8,6 +8,7 @@
 use std::collections::VecDeque;
 use std::ffi::OsString;
 use std::fmt::Write;
+use std::num::NonZeroUsize;
 use std::path::{Path, PathBuf};
 use std::process::{Command, ExitStatus};
 use std::sync::atomic::{AtomicUsize, Ordering};
@@ -55,6 +56,8 @@ pub struct Config {
     pub dependency_builder: Option<DependencyBuilder>,
     /// Print one character per test instead of one line
     pub quiet: bool,
+    /// How many threads to use for running tests. Defaults to number of cores
+    pub num_test_threads: NonZeroUsize,
 }
 
 impl Default for Config {
@@ -74,6 +77,7 @@ impl Default for Config {
             dependencies_crate_manifest_path: None,
             dependency_builder: None,
             quiet: true,
+            num_test_threads: std::thread::available_parallelism().unwrap(),
         }
     }
 }
@@ -206,7 +210,7 @@ pub fn run_tests_generic(config: Config, file_filter: impl Fn(&Path) -> bool + S
         let mut threads = vec![];
 
         // Create N worker threads that receive files to test.
-        for _ in 0..std::thread::available_parallelism().unwrap().get() {
+        for _ in 0..config.num_test_threads.get() {
             let finished_files_sender = finished_files_sender.clone();
             threads.push(s.spawn(|_| -> Result<()> {
                 let finished_files_sender = finished_files_sender;

tests/integrations/basic-fail/Cargo.stderr

@@ -3,10 +3,41 @@
      Running tests/ui_tests.rs 
    Compiler flags: ["--error-format=json", "--edition=2021"]
    Building test dependencies...
+tests/actual_tests/bad_pattern.rs ... FAILED
 tests/actual_tests/foomp.rs ... FAILED
 
+tests/actual_tests/bad_pattern.rs FAILED:
+command: "rustc" "--error-format=json" "--edition=2021" "--extern" "basic_fail=/home/ubuntu/crates/ui_test/$DIR/../../../target/debug/deps/libbasic_fail-8603f4ccb2fe730c.rmeta" "-L" "/home/ubuntu/crates/ui_test/$DIR/../../../target/debug" "-L" "/home/ubuntu/crates/ui_test/$DIR/../../../target/debug/deps" "tests/actual_tests/bad_pattern.rs"
+
+substring `miesmätsched types` not found in stderr output
+expected because of pattern here: tests/actual_tests/bad_pattern.rs:5
+
+There were 1 unmatched diagnostics at tests/actual_tests/bad_pattern.rs:4
+    Error: mismatched types
+
+full stderr:
+error[E0308]: mismatched types
+ --> tests/actual_tests/bad_pattern.rs:4:9
+  |
+4 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> /home/ubuntu/crates/ui_test/$DIR/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.
+
+
+
 tests/actual_tests/foomp.rs FAILED:
-command: "rustc" "--error-format=json" "--edition=2021" "--extern" "basic_fail=/home/ubuntu/crates/ui_test/$DIR/../../../target/debug/deps/libbasic_fail-8603f4ccb2fe730c.rmeta" "-L" "/home/ubuntu/crates/ui_test/$DIR/../../../target/debug/deps" "-L" "/home/ubuntu/crates/ui_test/$DIR/../../../target/debug" "tests/actual_tests/foomp.rs"
+command: "rustc" "--error-format=json" "--edition=2021" "--extern" "basic_fail=/home/ubuntu/crates/ui_test/$DIR/../../../target/debug/deps/libbasic_fail-8603f4ccb2fe730c.rmeta" "-L" "/home/ubuntu/crates/ui_test/$DIR/../../../target/debug" "-L" "/home/ubuntu/crates/ui_test/$DIR/../../../target/debug/deps" "tests/actual_tests/foomp.rs"
 
 actual output differed from expected tests/actual_tests/foomp.stderr
 Diff < left / right > :
@@ -55,7 +86,8 @@ For more information about this error, try `rustc --explain E0308`.
 
 
 FAILURES:
+    tests/actual_tests/bad_pattern.rs
     tests/actual_tests/foomp.rs
 
-test result: FAIL. 1 tests failed, 0 tests passed, 0 ignored, 0 filtered out
+test result: FAIL. 2 tests failed, 0 tests passed, 0 ignored, 0 filtered out
 error: test failed, to rerun pass '--test ui_tests'

tests/integrations/basic-fail/tests/actual_tests/bad_pattern.rs

@@ -0,0 +1,6 @@
+use basic_fail::add;
+
+fn main() {
+    add("42", 3);
+    //~^ ERROR: miesmätsched types
+}

tests/integrations/basic-fail/tests/actual_tests/bad_pattern.stderr

@@ -0,0 +1,17 @@
+error[E0308]: mismatched types
+ --> $DIR/bad_pattern.rs:4:9
+  |
+4 |     add("42", 3);
+  |     --- ^^^^ expected `usize`, found `&str`
+  |     |
+  |     arguments to this function are incorrect
+  |
+note: function defined here
+ --> $DIR/tests/integrations/basic-fail/src/lib.rs:1:8
+  |
+1 | pub fn add(left: usize, right: usize) -> usize {
+  |        ^^^
+
+error: aborting due to previous error
+
+For more information about this error, try `rustc --explain E0308`.

tests/integrations/basic-fail/tests/ui_tests.rs

@@ -1,3 +1,4 @@
+use std::num::NonZeroUsize;
 use ui_test::*;
 
 fn main() -> ui_test::color_eyre::Result<()> {
@@ -13,6 +14,8 @@ fn main() -> ui_test::color_eyre::Result<()> {
         }),
         // Never bless integrations-fail tests, we want to see stderr mismatches
         output_conflict_handling: OutputConflictHandling::Error,
+        // Make sure our tests are ordered for reliable output.
+        num_test_threads: NonZeroUsize::new(1).unwrap(),
         ..Config::default()
     };
     config.args.push("--edition=2021".into());