[cargo-nextest] add a "bench" command

See #2125 -- it can be helpful to run benchmarks with setup scripts and such
configured.

Author: sunshowers

cargo-nextest/src/dispatch.rs

@@ -7,7 +7,10 @@ use indent_write::indentable::Indented;
 use itertools::Itertools;
 use nextest_filtering::errors::FiltersetParseErrors;
 use nextest_metadata::NextestExitCode;
-use nextest_runner::{errors::*, helpers::plural, redact::Redactor};
+use nextest_runner::{
+    config::ConfigExperimental, errors::*, helpers::plural, redact::Redactor,
+    run_mode::NextestRunMode,
+};
 use owo_colors::OwoColorize;
 use semver::Version;
 use std::{error::Error, io, path::PathBuf, process::ExitStatus, string::FromUtf8Error};
@@ -212,6 +215,7 @@ pub enum ExpectedError {
     TestRunFailed,
     #[error("no tests to run")]
     NoTestsRun {
+        mode: NextestRunMode,
         /// The no-tests-run error was chosen because it was the default (we show a hint in this
         /// case)
         is_default: bool,
@@ -249,6 +253,11 @@ pub enum ExpectedError {
         name: &'static str,
         var_name: &'static str,
     },
+    #[error("config experimental features not enabled")]
+    ConfigExperimentalFeaturesNotEnabled {
+        config_file: Utf8PathBuf,
+        missing: Vec<ConfigExperimental>,
+    },
     #[error("filterset parse error")]
     FiltersetParseError {
         all_errors: Vec<FiltersetParseErrors>,
@@ -467,9 +476,7 @@ impl ExpectedError {
             | Self::DebugExtractWriteError { .. } => NextestExitCode::WRITE_OUTPUT_ERROR,
             #[cfg(feature = "self-update")]
             Self::UpdateError { .. } => NextestExitCode::UPDATE_ERROR,
-            Self::ExperimentalFeatureNotEnabled { .. } => {
-                NextestExitCode::EXPERIMENTAL_FEATURE_NOT_ENABLED
-            }
+            Self::ExperimentalFeatureNotEnabled { .. } | Self::ConfigExperimentalFeaturesNotEnabled { .. } => NextestExitCode::EXPERIMENTAL_FEATURE_NOT_ENABLED,
             Self::FiltersetParseError { .. } => NextestExitCode::INVALID_FILTERSET,
         }
     }
@@ -870,13 +877,16 @@ impl ExpectedError {
                 error!("test run failed");
                 None
             }
-            Self::NoTestsRun { is_default } => {
+            Self::NoTestsRun { mode, is_default } => {
                 let hint_str = if *is_default {
                     "\n(hint: use `--no-tests` to customize)"
                 } else {
                     ""
                 };
-                error!("no tests to run{hint_str}");
+                error!(
+                    "no {} to run{hint_str}",
+                    plural::tests_plural_if(*mode, true),
+                );
                 None
             }
             Self::ShowTestGroupsError { err } => {
@@ -936,6 +946,34 @@ impl ExpectedError {
                 );
                 None
             }
+            Self::ConfigExperimentalFeaturesNotEnabled {
+                config_file,
+                missing,
+            } => {
+                if missing.len() == 1 {
+                    let env_hint = if let Some(env_var) = missing[0].env_var() {
+                        format!(" or set {env_var}=1")
+                    } else {
+                        String::new()
+                    };
+                    error!(
+                        "experimental feature not enabled: {}\n(hint: add to the {} list in {}{})",
+                        missing[0].style(styles.bold),
+                        "experimental".style(styles.bold),
+                        config_file.style(styles.bold),
+                        env_hint,
+                    );
+                } else {
+                    error!(
+                        "experimental features not enabled: {}\n(hint: add to the {} list in {})",
+                        missing.iter().map(|f| f.style(styles.bold)).join(", "),
+                        "experimental".style(styles.bold),
+                        config_file.style(styles.bold),
+                    );
+                    // TODO: env_hint?
+                }
+                None
+            }
             Self::FiltersetParseError { all_errors } => {
                 for errors in all_errors {
                     for single_error in &errors.errors {

cargo-nextest/src/errors.rs

@@ -175,4 +175,5 @@ pub enum TestCaseFixtureProperty {
     MatchesCdylib = 4,
     MatchesTestMultiplyTwo = 8,
     NotInDefaultSetUnix = 16,
+    IsBenchmark = 32,
 }

fixture-data/src/models.rs

@@ -131,7 +131,10 @@ pub static EXPECTED_TEST_SUITES: LazyLock<IdOrdMap<TestSuiteFixture>> = LazyLock
             "my-bench",
             BuildPlatform::Target,
             id_ord_map! {
-                TestCaseFixture::new("bench_add_two", TestCaseFixtureStatus::Pass),
+                TestCaseFixture::new("bench_add_two", TestCaseFixtureStatus::Pass)
+                    .with_property(TestCaseFixtureProperty::IsBenchmark),
+                TestCaseFixture::new("bench_ignored", TestCaseFixtureStatus::IgnoredPass)
+                    .with_property(TestCaseFixtureProperty::IsBenchmark),
                 TestCaseFixture::new("tests::test_execute_bin", TestCaseFixtureStatus::Pass),
             },
         ),

fixture-data/src/nextest_tests.rs

@@ -28,3 +28,8 @@ __NEXTEST_TESTING_EXTRA_CONFIG_OVERRIDE_FORCE_IN_MAIN = { value = "test-FAILED-v
 __NEXTEST_TESTING_EXTRA_CONFIG_OVERRIDE_FORCE_IN_BOTH = { value = "test-FAILED-value-set-by-main-config", force = true }
 __NEXTEST_TESTING_EXTRA_CONFIG_OVERRIDE_FORCE_NONE = { value = "test-FAILED-value-set-by-main-config" }
 __NEXTEST_TESTING_EXTRA_CONFIG_OVERRIDE_FORCE_FALSE = { value = "test-FAILED-value-set-by-main-config", force = true }
+
+[term]
+# Disable OSC 9;4 terminal integration for output reproducibility in case it's enabled globally.
+progress.when = "auto"
+progress.term-integration = false

fixtures/nextest-tests/.cargo/config

@@ -5,7 +5,7 @@
 nextest-version = { required = "0.9.54", recommended = "0.9.56" }
 
 # This must be on one line for test_setup_scripts_not_enabled to work.
-experimental = ["setup-scripts", "wrapper-scripts"]
+experimental = ["benchmarks", "setup-scripts", "wrapper-scripts"]
 
 [profile.default]
 # disable fail-fast to ensure a deterministic test run

fixtures/nextest-tests/.config/nextest.toml

@@ -5,12 +5,17 @@
 
 extern crate test;
 
-// TODO: add benchmarks
 #[bench]
 fn bench_add_two(b: &mut test::Bencher) {
     b.iter(|| 2 + 2);
 }
 
+#[bench]
+#[ignore]
+fn bench_ignored(b: &mut test::Bencher) {
+    b.iter(|| 2 + 2);
+}
+
 #[cfg(test)]
 mod tests {
     /// Test that a binary can be successfully executed.

fixtures/nextest-tests/benches/my-bench.rs

@@ -174,6 +174,7 @@ pub enum RunProperty {
     WithSkipCdylibFilter = 4,
     // --exact test_multiply_two tests::test_multiply_two_cdylib
     WithMultiplyTwoExactFilter = 8,
+    Benchmarks = 16,
 }
 
 fn debug_run_properties(properties: u64) -> String {
@@ -190,6 +191,9 @@ fn debug_run_properties(properties: u64) -> String {
     if properties & RunProperty::WithMultiplyTwoExactFilter as u64 != 0 {
         ret.push_str("with-exact-filter ");
     }
+    if properties & RunProperty::Benchmarks as u64 != 0 {
+        ret.push_str("benchmarks ");
+    }
     ret
 }
 
@@ -229,6 +233,13 @@ pub fn check_run_output(stderr: &[u8], properties: u64) {
         for test in &fixture.test_cases {
             let name = format!("{} {}", binary_id, test.name);
 
+            if properties & RunProperty::Benchmarks as u64 != 0 {
+                // We don't consider skipped tests while running benchmarks.
+                if !test.has_property(TestCaseFixtureProperty::IsBenchmark) {
+                    continue;
+                }
+            }
+
             if test.has_property(TestCaseFixtureProperty::NotInDefaultSet)
                 && properties & RunProperty::WithDefaultFilter as u64 != 0
             {
@@ -343,7 +354,12 @@ pub fn check_run_output(stderr: &[u8], properties: u64) {
         }
     }
 
-    let tests_str = if run_count == 1 { "test" } else { "tests" };
+    let tests_str = match (run_count, properties & RunProperty::Benchmarks as u64 != 0) {
+        (1, false) => "test",
+        (1, true) => "benchmark",
+        (_, false) => "tests",
+        (_, true) => "benchmarks",
+    };
     let leak_fail_regex_str = if leak_fail_count > 0 {
         format!(r" \({leak_fail_count} due to being leaky\)")
     } else {

integration-tests/tests/integration/fixtures.rs

@@ -967,6 +967,30 @@ fn run_archive(archive_file: &Utf8Path) -> (TempProject, Utf8PathBuf) {
     (p2, extract_to.join("target"))
 }
 
+#[test]
+fn test_bench() {
+    set_env_vars();
+    let p = TempProject::new().unwrap();
+    let output = CargoNextestCli::for_test()
+        .args([
+            "--manifest-path",
+            p.manifest_path().as_str(),
+            "bench",
+            // Set the dev profile here to avoid a rebuild.
+            "--cargo-profile",
+            "dev",
+            "--no-capture",
+        ])
+        .unchecked(true)
+        .output();
+    assert_eq!(
+        output.exit_status.code(),
+        Some(0),
+        "correct exit code for command\n{output}",
+    );
+    check_run_output(&output.stderr, RunProperty::Benchmarks as u64);
+}
+
 #[test]
 fn test_show_config_test_groups() {
     set_env_vars();

integration-tests/tests/integration/main.rs

@@ -3,7 +3,7 @@ source: integration-tests/tests/integration/main.rs
 expression: output.stderr_as_str()
 snapshot_kind: text
 ---
-info: experimental features enabled: setup-scripts, wrapper-scripts
+info: experimental features enabled: setup-scripts, wrapper-scripts, benchmarks
   error: operator didn't match any binary names
    ╭────
  1 │ binary(unknown) & binary_id(unknown) & binary(=unknown) | binary_id(=unknown)