diff --git a/cargo-nextest/src/dispatch.rs b/cargo-nextest/src/dispatch.rs index 17a38c77268..56473a9d15a 100644 --- a/cargo-nextest/src/dispatch.rs +++ b/cargo-nextest/src/dispatch.rs @@ -37,6 +37,7 @@ use nextest_runner::{ redact::Redactor, reporter::{ FinalStatusLevel, ReporterBuilder, StatusLevel, TestOutputDisplay, TestOutputErrorSlice, + displayer::TestOutputDisplayStreams, events::{FinalRunStats, RunStatsFailureKind}, highlight_end, structured, }, @@ -57,6 +58,7 @@ use std::{ env::VarError, fmt, io::{Cursor, Write}, + str::FromStr, sync::{Arc, OnceLock}, time::Duration, }; @@ -1027,13 +1029,19 @@ fn non_zero_duration(input: &str) -> Result { #[derive(Debug, Default, Args)] #[command(next_help_heading = "Reporter options")] struct ReporterOpts { - /// Output stdout and stderr on failure + /// Output stdout and/or stderr on failure + /// + /// Takes the form of: '{value}' or 'stdout={value}' or 'stdout={value},stderr={value}' + /// where {value} is one of: 'immediate', 'immediate-final', 'final', 'never' #[arg(long, value_enum, value_name = "WHEN", env = "NEXTEST_FAILURE_OUTPUT")] - failure_output: Option, + failure_output: Option, - /// Output stdout and stderr on success + /// Output stdout and/or stderr on success + /// + /// Takes the form of: '{value}' or 'stdout={value}' or 'stdout={value},stderr={value}' + /// where {value} is one of: 'immediate', 'immediate-final', 'final', 'never' #[arg(long, value_enum, value_name = "WHEN", env = "NEXTEST_SUCCESS_OUTPUT")] - success_output: Option, + success_output: Option, // status_level does not conflict with --no-capture because pass vs skip still makes sense. /// Test statuses to output @@ -1152,6 +1160,68 @@ impl ReporterOpts { } } +#[derive(Debug, Clone, Copy)] +struct TestOutputDisplayStreamsOpt { + stdout: Option, + stderr: Option, +} + +impl FromStr for TestOutputDisplayStreamsOpt { + type Err = String; + + fn from_str(s: &str) -> Result { + // expected input has three forms + // - "{value}": where value is one of [immediate, immediate-final, final, never] + // - "{stream}={value}": where {stream} is one of [stdout, stderr] + // - "{stream}={value},{stream=value}": where the two {stream} keys cannot be the same + let (stdout, stderr) = if let Some((left, right)) = s.split_once(',') { + // the "{stream}={value},{stream=value}" case + let left = left + .split_once('=') + .map(|l| (l.0, TestOutputDisplayOpt::from_str(l.1, false))); + let right = right + .split_once('=') + .map(|r| (r.0, TestOutputDisplayOpt::from_str(r.1, false))); + match (left, right) { + (Some(("stderr", Ok(stderr))), Some(("stdout", Ok(stdout)))) => (Some(stdout), Some(stderr)), + (Some(("stdout", Ok(stdout))), Some(("stderr", Ok(stderr)))) => (Some(stdout), Some(stderr)), + (Some((stream @ "stdout" | stream @ "stderr", Err(_))), _) => return Err(format!("\n unrecognized setting for {stream}: [possible values: immediate, immediate-final, final, never]")), + (_, Some((stream @ "stdout" | stream @ "stderr", Err(_)))) => return Err(format!("\n unrecognized setting for {stream}: [possible values: immediate, immediate-final, final, never]")), + (Some(("stdout", _)), Some(("stdout", _))) => return Err("\n stdout specified twice".to_string()), + (Some(("stderr", _)), Some(("stderr", _))) => return Err("\n stderr specified twice".to_string()), + (Some((stream, _)), Some(("stdout" | "stderr", _))) => return Err(format!("\n unrecognized output stream '{stream}': [possible values: stdout, stderr]")), + (Some(("stdout" | "stderr", _)), Some((stream, _))) => return Err(format!("\n unrecognized output stream '{stream}': [possible values: stdout, stderr]")), + (_, _) => return Err("\n [possible values: immediate, immediate-final, final, never], or specify one or both output streams: stdout={}, stderr={}, stdout={},stderr={}".to_string()), + } + } else if let Some((stream, right)) = s.split_once('=') { + // the "{stream}={value}" case + let value = TestOutputDisplayOpt::from_str(right, false); + match (stream, value) { + ("stderr", Ok(stderr)) => (None, Some(stderr)), + ("stdout", Ok(stdout)) => (Some(stdout), None), + ("stdout" | "stderr", Err(_)) => return Err(format!("\n unrecognized setting for {stream}: [possible values: immediate, immediate-final, final, never]")), + (_, _) => return Err("\n unrecognized output stream, possible values: [stdout={}, stderr={}, stdout={},stderr={}]".to_string()) + } + } else if let Ok(value) = TestOutputDisplayOpt::from_str(s, false) { + // the "{value}" case + (Some(value), Some(value)) + } else { + // did not recognize one of the three cases + return Err("\n [possible values: immediate, immediate-final, final, never], or specify one or both output streams: stdout={}, stderr={}, stdout={},stderr={}".to_string()); + }; + Ok(Self { stdout, stderr }) + } +} + +impl From for TestOutputDisplayStreams { + fn from(value: TestOutputDisplayStreamsOpt) -> Self { + Self { + stdout: value.stdout.map(TestOutputDisplay::from), + stderr: value.stderr.map(TestOutputDisplay::from), + } + } +} + #[derive(Clone, Copy, Debug, ValueEnum)] enum TestOutputDisplayOpt { Immediate, diff --git a/nextest-runner/src/config/core/imp.rs b/nextest-runner/src/config/core/imp.rs index 98b92000c67..ccc19ebef9d 100644 --- a/nextest-runner/src/config/core/imp.rs +++ b/nextest-runner/src/config/core/imp.rs @@ -28,7 +28,7 @@ use crate::{ helpers::plural, list::TestList, platform::BuildPlatforms, - reporter::{FinalStatusLevel, StatusLevel, TestOutputDisplay}, + reporter::{FinalStatusLevel, StatusLevel, TestOutputDisplayStreams}, }; use camino::{Utf8Path, Utf8PathBuf}; use config::{ @@ -1049,14 +1049,14 @@ impl<'cfg> EvaluatableProfile<'cfg> { } /// Returns the failure output config for this profile. - pub fn failure_output(&self) -> TestOutputDisplay { + pub fn failure_output(&self) -> TestOutputDisplayStreams { self.custom_profile .and_then(|profile| profile.failure_output) .unwrap_or(self.default_profile.failure_output) } /// Returns the failure output config for this profile. - pub fn success_output(&self) -> TestOutputDisplay { + pub fn success_output(&self) -> TestOutputDisplayStreams { self.custom_profile .and_then(|profile| profile.success_output) .unwrap_or(self.default_profile.success_output) @@ -1225,8 +1225,8 @@ pub(in crate::config) struct DefaultProfileImpl { retries: RetryPolicy, status_level: StatusLevel, final_status_level: FinalStatusLevel, - failure_output: TestOutputDisplay, - success_output: TestOutputDisplay, + failure_output: TestOutputDisplayStreams, + success_output: TestOutputDisplayStreams, max_fail: MaxFail, slow_timeout: SlowTimeout, global_timeout: GlobalTimeout, @@ -1314,9 +1314,9 @@ pub(in crate::config) struct CustomProfileImpl { #[serde(default)] final_status_level: Option, #[serde(default)] - failure_output: Option, + failure_output: Option, #[serde(default)] - success_output: Option, + success_output: Option, #[serde( default, rename = "fail-fast", diff --git a/nextest-runner/src/config/overrides/imp.rs b/nextest-runner/src/config/overrides/imp.rs index e91ede36337..92ea08c677d 100644 --- a/nextest-runner/src/config/overrides/imp.rs +++ b/nextest-runner/src/config/overrides/imp.rs @@ -17,7 +17,7 @@ use crate::{ ConfigCompileError, ConfigCompileErrorKind, ConfigCompileSection, ConfigParseErrorKind, }, platform::BuildPlatforms, - reporter::TestOutputDisplay, + reporter::TestOutputDisplayStreams, }; use guppy::graph::cargo::BuildPlatform; use nextest_filtering::{ @@ -109,8 +109,8 @@ pub struct TestSettings<'p, Source = ()> { slow_timeout: (SlowTimeout, Source), leak_timeout: (LeakTimeout, Source), test_group: (TestGroup, Source), - success_output: (TestOutputDisplay, Source), - failure_output: (TestOutputDisplay, Source), + success_output: (TestOutputDisplayStreams, Source), + failure_output: (TestOutputDisplayStreams, Source), junit_store_success_output: (bool, Source), junit_store_failure_output: (bool, Source), } @@ -217,12 +217,12 @@ impl<'p> TestSettings<'p> { } /// Returns the success output setting for this test. - pub fn success_output(&self) -> TestOutputDisplay { + pub fn success_output(&self) -> TestOutputDisplayStreams { self.success_output.0 } /// Returns the failure output setting for this test. - pub fn failure_output(&self) -> TestOutputDisplay { + pub fn failure_output(&self) -> TestOutputDisplayStreams { self.failure_output.0 } @@ -704,8 +704,8 @@ pub(in crate::config) struct ProfileOverrideData { slow_timeout: Option, leak_timeout: Option, pub(in crate::config) test_group: Option, - success_output: Option, - failure_output: Option, + success_output: Option, + failure_output: Option, junit: DeserializedJunitOutput, } @@ -947,9 +947,9 @@ pub(in crate::config) struct DeserializedOverride { #[serde(default)] test_group: Option, #[serde(default)] - success_output: Option, + success_output: Option, #[serde(default)] - failure_output: Option, + failure_output: Option, #[serde(default)] junit: DeserializedJunitOutput, } @@ -1122,8 +1122,14 @@ mod tests { } ); assert_eq!(overrides.test_group(), &test_group("my-group")); - assert_eq!(overrides.success_output(), TestOutputDisplay::Never); - assert_eq!(overrides.failure_output(), TestOutputDisplay::Final); + assert_eq!( + overrides.success_output(), + TestOutputDisplayStreams::create_never() + ); + assert_eq!( + overrides.failure_output(), + TestOutputDisplayStreams::create_final() + ); // For clarity. #[expect(clippy::bool_assert_comparison)] { @@ -1173,9 +1179,12 @@ mod tests { assert_eq!(overrides.test_group(), &test_group("my-group")); assert_eq!( overrides.success_output(), - TestOutputDisplay::ImmediateFinal + TestOutputDisplayStreams::create_immediate_final() + ); + assert_eq!( + overrides.failure_output(), + TestOutputDisplayStreams::create_final() ); - assert_eq!(overrides.failure_output(), TestOutputDisplay::Final); // For clarity. #[expect(clippy::bool_assert_comparison)] { diff --git a/nextest-runner/src/reporter/displayer/imp.rs b/nextest-runner/src/reporter/displayer/imp.rs index c5807bc6c74..3fa9b261a54 100644 --- a/nextest-runner/src/reporter/displayer/imp.rs +++ b/nextest-runner/src/reporter/displayer/imp.rs @@ -22,7 +22,10 @@ use crate::{ helpers::{DisplayScriptInstance, DisplayTestInstance, plural}, list::{TestInstance, TestInstanceId}, reporter::{ - displayer::{formatters::DisplayHhMmSs, progress::TerminalProgress}, + displayer::{ + DisplayOutput, TestOutputDisplayStreams, formatters::DisplayHhMmSs, + progress::TerminalProgress, + }, events::*, helpers::Styles, imp::ReporterStderr, @@ -44,8 +47,8 @@ pub(crate) struct DisplayReporterBuilder { pub(crate) default_filter: CompiledDefaultFilter, pub(crate) status_levels: StatusLevels, pub(crate) test_count: usize, - pub(crate) success_output: Option, - pub(crate) failure_output: Option, + pub(crate) success_output: TestOutputDisplayStreams, + pub(crate) failure_output: TestOutputDisplayStreams, pub(crate) should_colorize: bool, pub(crate) no_capture: bool, pub(crate) hide_progress_bar: bool, @@ -120,14 +123,14 @@ impl DisplayReporterBuilder { // failure_output and success_output are meaningless if the runner isn't capturing any // output. - let force_success_output = match self.no_capture { - true => Some(TestOutputDisplay::Never), - false => self.success_output, - }; - let force_failure_output = match self.no_capture { - true => Some(TestOutputDisplay::Never), - false => self.failure_output, - }; + let mut force_success_output = self.success_output; + let mut force_failure_output = self.failure_output; + if self.no_capture { + force_success_output.stdout = Some(TestOutputDisplay::Never); + force_success_output.stderr = Some(TestOutputDisplay::Never); + force_failure_output.stdout = Some(TestOutputDisplay::Never); + force_failure_output.stderr = Some(TestOutputDisplay::Never); + } DisplayReporter { inner: DisplayReporterImpl { @@ -241,7 +244,7 @@ enum FinalOutput { Skipped(#[expect(dead_code)] MismatchReason), Executed { run_statuses: ExecutionStatuses, - display_output: bool, + display_output: Option, }, } @@ -472,7 +475,7 @@ impl<'a> DisplayReporterImpl<'a> { // Always display failing setup script output if it exists. We // may change this in the future. if !run_status.result.is_success() { - self.write_setup_script_execute_status(run_status, writer)?; + self.write_setup_script_execute_status(run_status, display_output, writer)?; } } TestEventKind::TestStarted { @@ -569,12 +572,12 @@ impl<'a> DisplayReporterImpl<'a> { !run_status.result.is_success(), "only failing tests are retried" ); - if self + if let Some(display_output) = self .unit_output .failure_output(*failure_output) - .is_immediate() + .display_output_immediate() { - self.write_test_execute_status(run_status, true, writer)?; + self.write_test_execute_status(run_status, true, display_output, writer)?; } // The final output doesn't show retries, so don't store this result in @@ -648,8 +651,8 @@ impl<'a> DisplayReporterImpl<'a> { if output_on_test_finished.write_status_line { self.write_status_line(*stress_index, *test_instance, describe, writer)?; } - if output_on_test_finished.show_immediate { - self.write_test_execute_status(last_status, false, writer)?; + if let Some(display_output) = output_on_test_finished.show_immediate { + self.write_test_execute_status(last_status, false, display_output, writer)?; } if let OutputStoreFinal::Yes { display_output } = output_on_test_finished.store_final @@ -1036,8 +1039,13 @@ impl<'a> DisplayReporterImpl<'a> { run_statuses.describe(), writer, )?; - if *display_output { - self.write_test_execute_status(last_status, false, writer)?; + if let Some(display_output) = *display_output { + self.write_test_execute_status( + last_status, + false, + display_output, + writer, + )?; } } } @@ -1358,6 +1366,7 @@ impl<'a> DisplayReporterImpl<'a> { &self.styles, &self.output_spec_for_info(UnitKind::Script), output, + display_output, &mut writer, )?; } @@ -1405,6 +1414,7 @@ impl<'a> DisplayReporterImpl<'a> { &self.styles, &self.output_spec_for_info(UnitKind::Test), output, + display_output, &mut writer, )?; } @@ -1715,6 +1725,7 @@ impl<'a> DisplayReporterImpl<'a> { fn write_setup_script_execute_status( &self, run_status: &SetupScriptExecuteStatus, + display_output: DisplayOutput, writer: &mut dyn Write, ) -> io::Result<()> { let spec = self.output_spec_for_finished(run_status.result, false); @@ -1722,6 +1733,7 @@ impl<'a> DisplayReporterImpl<'a> { &self.styles, &spec, &run_status.output, + display_output, writer, )?; @@ -1742,6 +1754,7 @@ impl<'a> DisplayReporterImpl<'a> { &self, run_status: &ExecuteStatus, is_retry: bool, + display_output: DisplayOutput, writer: &mut dyn Write, ) -> io::Result<()> { let spec = self.output_spec_for_finished(run_status.result, is_retry); @@ -1749,6 +1762,7 @@ impl<'a> DisplayReporterImpl<'a> { &self.styles, &spec, &run_status.output, + display_output, writer, )?; @@ -2109,8 +2123,8 @@ mod tests { final_status_level: FinalStatusLevel::Fail, }, test_count: 0, - success_output: Some(TestOutputDisplay::Immediate), - failure_output: Some(TestOutputDisplay::Immediate), + success_output: TestOutputDisplayStreams::create_immediate(), + failure_output: TestOutputDisplayStreams::create_immediate(), should_colorize: false, no_capture: true, hide_progress_bar: false, @@ -2682,14 +2696,24 @@ mod tests { |reporter| { assert!(reporter.inner.no_capture, "no_capture is true"); assert_eq!( - reporter.inner.unit_output.force_failure_output(), - Some(TestOutputDisplay::Never), - "failure output is never, overriding other settings" + reporter.inner.unit_output.force_failure_output().stdout, + TestOutputDisplayStreams::create_never().stdout, + "failure output for stdout is never, overriding other settings" + ); + assert_eq!( + reporter.inner.unit_output.force_failure_output().stderr, + TestOutputDisplayStreams::create_never().stderr, + "failure output for stderr is never, overriding other settings" + ); + assert_eq!( + reporter.inner.unit_output.force_success_output().stdout, + TestOutputDisplayStreams::create_never().stdout, + "success output for stdout is never, overriding other settings" ); assert_eq!( - reporter.inner.unit_output.force_success_output(), - Some(TestOutputDisplay::Never), - "success output is never, overriding other settings" + reporter.inner.unit_output.force_success_output().stderr, + TestOutputDisplayStreams::create_never().stderr, + "success output for stderr is never, overriding other settings" ); assert_eq!( reporter.inner.status_levels.status_level, diff --git a/nextest-runner/src/reporter/displayer/status_level.rs b/nextest-runner/src/reporter/displayer/status_level.rs index da205fa2a52..9aee2edb29a 100644 --- a/nextest-runner/src/reporter/displayer/status_level.rs +++ b/nextest-runner/src/reporter/displayer/status_level.rs @@ -5,8 +5,10 @@ //! //! Status levels play a role that's similar to log levels in typical loggers. -use super::TestOutputDisplay; -use crate::reporter::events::CancelReason; +use crate::reporter::{ + displayer::{DisplayOutput, TestOutputDisplayStreams}, + events::CancelReason, +}; use serde::Deserialize; /// Status level to show in the reporter output. @@ -89,17 +91,18 @@ pub(crate) struct StatusLevels { impl StatusLevels { pub(super) fn compute_output_on_test_finished( &self, - display: TestOutputDisplay, + display: TestOutputDisplayStreams, cancel_status: Option, test_status_level: StatusLevel, test_final_status_level: FinalStatusLevel, ) -> OutputOnTestFinished { let write_status_line = self.status_level >= test_status_level; - let is_immediate = display.is_immediate(); + let is_immediate = display.display_output_immediate(); // We store entries in the final output map if either the final status level is high enough or // if `display` says we show the output at the end. - let is_final = display.is_final() || self.final_status_level >= test_final_status_level; + let is_final = display.display_output_final().is_some() + || self.final_status_level >= test_final_status_level; // This table is tested below. The basic invariant is that we generally follow what // is_immediate and is_final suggests, except: @@ -124,19 +127,24 @@ impl StatusLevels { // // [2] If there's a signal, we shouldn't display output twice at the end since it's // redundant -- instead, just show the output as part of the immediate display. - let show_immediate = is_immediate && cancel_status <= Some(CancelReason::Signal); + let show_immediate = if cancel_status <= Some(CancelReason::Signal) { + is_immediate + } else { + None + }; let store_final = if is_final && cancel_status < Some(CancelReason::Signal) - || !is_immediate && is_final && cancel_status == Some(CancelReason::Signal) + || is_immediate.is_none() && is_final && cancel_status == Some(CancelReason::Signal) { OutputStoreFinal::Yes { - display_output: display.is_final(), + display_output: display.display_output_final(), } - } else if is_immediate && is_final && cancel_status == Some(CancelReason::Signal) { + } else if is_immediate.is_some() && is_final && cancel_status == Some(CancelReason::Signal) + { // In this special case, we already display the output once as the test is being // cancelled, so don't display it again at the end since that's redundant. OutputStoreFinal::Yes { - display_output: false, + display_output: None, } } else { OutputStoreFinal::No @@ -153,7 +161,7 @@ impl StatusLevels { #[derive(Debug, PartialEq, Eq)] pub(super) struct OutputOnTestFinished { pub(super) write_status_line: bool, - pub(super) show_immediate: bool, + pub(super) show_immediate: Option, pub(super) store_final: OutputStoreFinal, } @@ -164,7 +172,9 @@ pub(super) enum OutputStoreFinal { /// Store the output. display_output controls whether stdout and stderr should actually be /// displayed at the end. - Yes { display_output: bool }, + Yes { + display_output: Option, + }, } #[cfg(test)] @@ -179,7 +189,7 @@ mod tests { #[proptest(cases = 64)] fn on_test_finished_dont_write_status_line( - display: TestOutputDisplay, + display: TestOutputDisplayStreams, cancel_status: Option, #[filter(StatusLevel::Pass < #test_status_level)] test_status_level: StatusLevel, test_final_status_level: FinalStatusLevel, @@ -201,7 +211,7 @@ mod tests { #[proptest(cases = 64)] fn on_test_finished_write_status_line( - display: TestOutputDisplay, + display: TestOutputDisplayStreams, cancel_status: Option, #[filter(StatusLevel::Pass >= #test_status_level)] test_status_level: StatusLevel, test_final_status_level: FinalStatusLevel, @@ -223,7 +233,7 @@ mod tests { #[proptest(cases = 64)] fn on_test_finished_with_interrupt( // We always hide output on interrupt. - display: TestOutputDisplay, + display: TestOutputDisplayStreams, // cancel_status is fixed to Interrupt. // In this case, the status levels are not relevant for is_immediate and is_final. @@ -241,13 +251,13 @@ mod tests { test_status_level, test_final_status_level, ); - assert!(!actual.show_immediate); + assert!(actual.show_immediate.is_none()); assert_eq!(actual.store_final, OutputStoreFinal::No); } #[proptest(cases = 64)] fn on_test_finished_dont_show_immediate( - #[filter(!#display.is_immediate())] display: TestOutputDisplay, + #[filter(#display.display_output_immediate().is_none())] display: TestOutputDisplayStreams, cancel_status: Option, // The status levels are not relevant for show_immediate. test_status_level: StatusLevel, @@ -264,12 +274,12 @@ mod tests { test_status_level, test_final_status_level, ); - assert!(!actual.show_immediate); + assert!(actual.show_immediate.is_none()); } #[proptest(cases = 64)] fn on_test_finished_show_immediate( - #[filter(#display.is_immediate())] display: TestOutputDisplay, + #[filter(#display.display_output_immediate().is_none())] display: TestOutputDisplayStreams, #[filter(#cancel_status <= Some(CancelReason::Signal))] cancel_status: Option, // The status levels are not relevant for show_immediate. test_status_level: StatusLevel, @@ -286,14 +296,14 @@ mod tests { test_status_level, test_final_status_level, ); - assert!(actual.show_immediate); + assert!(actual.show_immediate.is_some()); } // Where we don't store final output: if display.is_final() is false, and if the test final // status level is too high. #[proptest(cases = 64)] fn on_test_finished_dont_store_final( - #[filter(!#display.is_final())] display: TestOutputDisplay, + #[filter(#display.display_output_final().is_none())] display: TestOutputDisplayStreams, cancel_status: Option, // The status level is not relevant for store_final. test_status_level: StatusLevel, @@ -330,7 +340,7 @@ mod tests { }; let actual = status_levels.compute_output_on_test_finished( - TestOutputDisplay::Final, + TestOutputDisplayStreams::create_final(), cancel_status, test_status_level, test_final_status_level, @@ -338,7 +348,10 @@ mod tests { assert_eq!( actual.store_final, OutputStoreFinal::Yes { - display_output: true + display_output: Some(DisplayOutput { + stdout: true, + stderr: true, + }) } ); } @@ -357,7 +370,7 @@ mod tests { }; let actual = status_levels.compute_output_on_test_finished( - TestOutputDisplay::ImmediateFinal, + TestOutputDisplayStreams::create_immediate_final(), cancel_status, test_status_level, test_final_status_level, @@ -365,7 +378,7 @@ mod tests { assert_eq!( actual.store_final, OutputStoreFinal::Yes { - display_output: true + display_output: None, } ); } @@ -383,7 +396,7 @@ mod tests { }; let actual = status_levels.compute_output_on_test_finished( - TestOutputDisplay::ImmediateFinal, + TestOutputDisplayStreams::create_immediate_final(), Some(CancelReason::Signal), test_status_level, test_final_status_level, @@ -391,7 +404,7 @@ mod tests { assert_eq!( actual.store_final, OutputStoreFinal::Yes { - display_output: false, + display_output: None, } ); } @@ -399,7 +412,7 @@ mod tests { // Case 4: if display.is_final() is *false* but the test_final_status_level is low enough. #[proptest(cases = 64)] fn on_test_finished_store_final_4( - #[filter(!#display.is_final())] display: TestOutputDisplay, + #[filter(#display.display_output_final().is_none())] display: TestOutputDisplayStreams, #[filter(#cancel_status <= Some(CancelReason::Signal))] cancel_status: Option, // The status level is not relevant for store_final. test_status_level: StatusLevel, @@ -421,7 +434,7 @@ mod tests { assert_eq!( actual.store_final, OutputStoreFinal::Yes { - display_output: false, + display_output: None, } ); } diff --git a/nextest-runner/src/reporter/displayer/unit_output.rs b/nextest-runner/src/reporter/displayer/unit_output.rs index 3666d58019b..621758ca76c 100644 --- a/nextest-runner/src/reporter/displayer/unit_output.rs +++ b/nextest-runner/src/reporter/displayer/unit_output.rs @@ -17,10 +17,170 @@ use indent_write::io::IndentWriter; use owo_colors::Style; use serde::Deserialize; use std::{ - fmt, + fmt::{self, Formatter}, io::{self, Write}, + str::FromStr, }; +/// When to display test output in the reporter. +#[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] +#[cfg_attr(test, derive(test_strategy::Arbitrary))] +pub struct TestOutputDisplayStreams { + /// When to display the stdout output of a test in the reporter + pub stdout: Option, + /// When to display the stderr output of a test in the reporter + pub stderr: Option, +} + +impl TestOutputDisplayStreams { + /// Create a `TestOutputDisplayStreams` with both stdout and stderr set to `Immediate` + pub fn create_immediate() -> Self { + Self { + stdout: Some(TestOutputDisplay::Immediate), + stderr: Some(TestOutputDisplay::Immediate), + } + } + /// Create a `TestOutputDisplayStreams` with both stdout and stderr set to `Final` + pub fn create_final() -> Self { + Self { + stdout: Some(TestOutputDisplay::Final), + stderr: Some(TestOutputDisplay::Final), + } + } + /// Create a `TestOutputDisplayStreams` with both stdout and stderr set to `ImmediateFinal` + pub fn create_immediate_final() -> Self { + Self { + stdout: Some(TestOutputDisplay::ImmediateFinal), + stderr: Some(TestOutputDisplay::ImmediateFinal), + } + } + /// Create a `TestOutputDisplayStreams` with both stdout and stderr set to `Never` + pub fn create_never() -> Self { + Self { + stdout: Some(TestOutputDisplay::Never), + stderr: Some(TestOutputDisplay::Never), + } + } + + /// Which output streams should be output immediately + /// + /// # Returns + /// Returns `None` when no output streams should be output + pub fn display_output_immediate(&self) -> Option { + let stdout = self.stdout.map_or(false, |t| t.is_immediate()); + let stderr = self.stderr.map_or(false, |t| t.is_immediate()); + if stdout || stderr { + Some(DisplayOutput { stdout, stderr }) + } else { + None + } + } + + /// Which output streams should be output at the end + /// + /// # Returns + /// Returns `None` when no output streams should be output + pub fn display_output_final(&self) -> Option { + let stdout = self.stdout.map_or(false, |t| t.is_final()); + let stderr = self.stderr.map_or(false, |t| t.is_final()); + if stdout || stderr { + Some(DisplayOutput { stdout, stderr }) + } else { + None + } + } +} + +impl<'de> Deserialize<'de> for TestOutputDisplayStreams { + fn deserialize(deserializer: D) -> Result + where + D: serde::de::Deserializer<'de>, + { + deserializer.deserialize_any(TestOutputDisplayStreamsVisitor) + } +} + +struct TestOutputDisplayStreamsVisitor; +impl<'de> serde::de::Visitor<'de> for TestOutputDisplayStreamsVisitor { + type Value = TestOutputDisplayStreams; + + fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { + formatter.write_str( + "a string with 'never', 'immediate', 'immediate-final' or 'final',\ +or a map with 'stdout' and/or 'stderr' as keys and the preceding values", + ) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + // the input is a string so we expect a TestOutputDisplay value + match v { + "never" => Ok(TestOutputDisplayStreams::create_never()), + "immediate" => Ok(TestOutputDisplayStreams::create_immediate()), + "immediate-final" => Ok(TestOutputDisplayStreams::create_immediate_final()), + "final" => Ok(TestOutputDisplayStreams::create_final()), + _ => Err(E::invalid_value( + serde::de::Unexpected::Str(v), + &"unrecognized value, expected 'never', 'immediate', 'immediate-final' or 'final'\ +or a map with 'stdout' and/or 'stderr' as keys and the preceding values", + )), + } + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + use serde::de::Error; + + // the input is a map, so we expect stdout and/or stderr as keys and TestOutputDisplay values + // as the value + let mut stdout = None; + let mut stderr = None; + while let Some((key, value)) = map.next_entry::<&str, &str>()? { + match key { + "stdout" => { + if stdout.is_some() { + return Err(A::Error::duplicate_field("stdout")); + } + stdout = Some(TestOutputDisplay::from_str(value).map_err(|_| { + A::Error::invalid_value( + serde::de::Unexpected::Str(value), + &"'never', 'immediate', 'immediate-final', or 'final'", + ) + })?); + } + "stderr" => { + if stderr.is_some() { + return Err(A::Error::duplicate_field("stderr")); + } + stderr = Some(TestOutputDisplay::from_str(value).map_err(|_| { + A::Error::invalid_value( + serde::de::Unexpected::Str(value), + &"'never', 'immediate', 'immediate-final', or 'final'", + ) + })?); + } + _ => return Err(A::Error::unknown_field(key, &["stdout", "stderr"])), + } + } + Ok(TestOutputDisplayStreams { stdout, stderr }) + } +} + +/// Simplified [`TestOutputDisplayStreams`] to tell which output streams should be output +/// +/// Used after the Never/Immediate/Final distinction has been made. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DisplayOutput { + /// Display the stdout output of the test + pub stdout: bool, + /// Display the stderr output of the test + pub stderr: bool, +} + /// When to display test output in the reporter. #[derive(Copy, Clone, Debug, Eq, PartialEq, Deserialize)] #[cfg_attr(test, derive(test_strategy::Arbitrary))] @@ -59,6 +219,22 @@ impl TestOutputDisplay { } } +impl FromStr for TestOutputDisplay { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + match s { + "immediate" => Ok(TestOutputDisplay::Immediate), + "immediate-final" => Ok(TestOutputDisplay::ImmediateFinal), + "final" => Ok(TestOutputDisplay::Final), + "never" => Ok(TestOutputDisplay::Never), + _ => Err( + "unrecognized value, expected 'never', 'immediate', 'immediate-final', or 'final'", + ), + } + } +} + /// Formatting options for writing out child process output. /// /// TODO: should these be lazily generated? Can't imagine this ever being @@ -74,15 +250,15 @@ pub(super) struct ChildOutputSpec { } pub(super) struct UnitOutputReporter { - force_success_output: Option, - force_failure_output: Option, + force_success_output: TestOutputDisplayStreams, + force_failure_output: TestOutputDisplayStreams, display_empty_outputs: bool, } impl UnitOutputReporter { pub(super) fn new( - force_success_output: Option, - force_failure_output: Option, + force_success_output: TestOutputDisplayStreams, + force_failure_output: TestOutputDisplayStreams, ) -> Self { // Ordinarily, empty stdout and stderr are not displayed. This // environment variable is set in integration tests to ensure that they @@ -97,23 +273,35 @@ impl UnitOutputReporter { } } - pub(super) fn success_output(&self, test_setting: TestOutputDisplay) -> TestOutputDisplay { - self.force_success_output.unwrap_or(test_setting) + pub(super) fn success_output( + &self, + test_setting: TestOutputDisplayStreams, + ) -> TestOutputDisplayStreams { + TestOutputDisplayStreams { + stdout: self.force_success_output.stdout.or(test_setting.stdout), + stderr: self.force_success_output.stderr.or(test_setting.stderr), + } } - pub(super) fn failure_output(&self, test_setting: TestOutputDisplay) -> TestOutputDisplay { - self.force_failure_output.unwrap_or(test_setting) + pub(super) fn failure_output( + &self, + test_setting: TestOutputDisplayStreams, + ) -> TestOutputDisplayStreams { + TestOutputDisplayStreams { + stdout: self.force_failure_output.stdout.or(test_setting.stdout), + stderr: self.force_failure_output.stderr.or(test_setting.stderr), + } } // These are currently only used by tests, but there's no principled // objection to using these functions elsewhere in the displayer. #[cfg(test)] - pub(super) fn force_success_output(&self) -> Option { + pub(super) fn force_success_output(&self) -> TestOutputDisplayStreams { self.force_success_output } #[cfg(test)] - pub(super) fn force_failure_output(&self) -> Option { + pub(super) fn force_failure_output(&self) -> TestOutputDisplayStreams { self.force_failure_output } @@ -122,6 +310,7 @@ impl UnitOutputReporter { styles: &Styles, spec: &ChildOutputSpec, exec_output: &ChildExecutionOutput, + display_output: DisplayOutput, mut writer: &mut dyn Write, ) -> io::Result<()> { match exec_output { @@ -151,7 +340,14 @@ impl UnitOutputReporter { } else { None }; - self.write_child_output(styles, spec, output, highlight_slice, writer)?; + self.write_child_output( + styles, + spec, + output, + highlight_slice, + display_output, + writer, + )?; } ChildExecutionOutput::StartError(error) => { @@ -175,12 +371,13 @@ impl UnitOutputReporter { spec: &ChildOutputSpec, output: &ChildOutput, highlight_slice: Option>, + display_output: DisplayOutput, mut writer: &mut dyn Write, ) -> io::Result<()> { match output { ChildOutput::Split(split) => { if let Some(stdout) = &split.stdout { - if self.display_empty_outputs || !stdout.is_empty() { + if self.display_empty_outputs || (!stdout.is_empty() && display_output.stdout) { writeln!(writer, "{}", spec.stdout_header)?; // If there's no output indent, this is a no-op, though @@ -200,7 +397,7 @@ impl UnitOutputReporter { } if let Some(stderr) = &split.stderr { - if self.display_empty_outputs || !stderr.is_empty() { + if self.display_empty_outputs || (!stderr.is_empty() && display_output.stderr) { writeln!(writer, "{}", spec.stderr_header)?; let mut indent_writer = IndentWriter::new(spec.output_indent, writer); @@ -215,7 +412,7 @@ impl UnitOutputReporter { } } ChildOutput::Combined { output } => { - if self.display_empty_outputs || !output.is_empty() { + if self.display_empty_outputs || (!output.is_empty() && display_output.stdout) { writeln!(writer, "{}", spec.combined_header)?; let mut indent_writer = IndentWriter::new(spec.output_indent, writer); diff --git a/nextest-runner/src/reporter/events.rs b/nextest-runner/src/reporter/events.rs index 2e2f9940e99..bdfb1f4d078 100644 --- a/nextest-runner/src/reporter/events.rs +++ b/nextest-runner/src/reporter/events.rs @@ -6,10 +6,11 @@ //! These types form the interface between the test runner and the test //! reporter. The root structure for all events is [`TestEvent`]. -use super::{FinalStatusLevel, StatusLevel, TestOutputDisplay}; +use super::{FinalStatusLevel, StatusLevel}; use crate::{ config::{elements::LeakTimeoutResult, scripts::ScriptId}, list::{TestInstance, TestInstanceId, TestList}, + reporter::displayer::TestOutputDisplayStreams, runner::{StressCondition, StressCount}, test_output::ChildExecutionOutput, }; @@ -199,7 +200,7 @@ pub enum TestEventKind<'a> { delay_before_next_attempt: Duration, /// Whether failure outputs are printed out. - failure_output: TestOutputDisplay, + failure_output: TestOutputDisplayStreams, }, /// A retry has started. @@ -223,10 +224,10 @@ pub enum TestEventKind<'a> { test_instance: TestInstance<'a>, /// Test setting for success output. - success_output: TestOutputDisplay, + success_output: TestOutputDisplayStreams, /// Test setting for failure output. - failure_output: TestOutputDisplay, + failure_output: TestOutputDisplayStreams, /// Whether the JUnit report should store success output for this test. junit_store_success_output: bool, diff --git a/nextest-runner/src/reporter/imp.rs b/nextest-runner/src/reporter/imp.rs index 8cf596f590d..7f1028689cd 100644 --- a/nextest-runner/src/reporter/imp.rs +++ b/nextest-runner/src/reporter/imp.rs @@ -6,7 +6,7 @@ //! The main structure in this module is [`TestReporter`]. use super::{ - FinalStatusLevel, StatusLevel, TestOutputDisplay, + FinalStatusLevel, StatusLevel, displayer::{DisplayReporter, DisplayReporterBuilder, StatusLevels}, }; use crate::{ @@ -14,7 +14,10 @@ use crate::{ config::core::EvaluatableProfile, errors::WriteEventError, list::TestList, - reporter::{aggregator::EventAggregator, events::*, structured::StructuredReporter}, + reporter::{ + aggregator::EventAggregator, displayer::TestOutputDisplayStreams, events::*, + structured::StructuredReporter, + }, }; /// Standard error destination for the reporter. @@ -35,8 +38,8 @@ pub enum ReporterStderr<'a> { pub struct ReporterBuilder { no_capture: bool, should_colorize: bool, - failure_output: Option, - success_output: Option, + failure_output: TestOutputDisplayStreams, + success_output: TestOutputDisplayStreams, status_level: Option, final_status_level: Option, @@ -62,14 +65,14 @@ impl ReporterBuilder { } /// Sets the conditions under which test failures are output. - pub fn set_failure_output(&mut self, failure_output: TestOutputDisplay) -> &mut Self { - self.failure_output = Some(failure_output); + pub fn set_failure_output(&mut self, failure_output: TestOutputDisplayStreams) -> &mut Self { + self.failure_output = failure_output; self } /// Sets the conditions under which test successes are output. - pub fn set_success_output(&mut self, success_output: TestOutputDisplay) -> &mut Self { - self.success_output = Some(success_output); + pub fn set_success_output(&mut self, success_output: TestOutputDisplayStreams) -> &mut Self { + self.success_output = success_output; self } diff --git a/nextest-runner/src/reporter/mod.rs b/nextest-runner/src/reporter/mod.rs index 106e5f7d551..3d000e5a432 100644 --- a/nextest-runner/src/reporter/mod.rs +++ b/nextest-runner/src/reporter/mod.rs @@ -13,7 +13,7 @@ mod helpers; mod imp; pub mod structured; -pub use displayer::{FinalStatusLevel, StatusLevel, TestOutputDisplay}; +pub use displayer::{FinalStatusLevel, StatusLevel, TestOutputDisplay, TestOutputDisplayStreams}; pub use error_description::*; pub use helpers::highlight_end; pub use imp::*; diff --git a/nextest-runner/src/runner/internal_events.rs b/nextest-runner/src/runner/internal_events.rs index 18c18739894..548a0893e80 100644 --- a/nextest-runner/src/runner/internal_events.rs +++ b/nextest-runner/src/runner/internal_events.rs @@ -12,7 +12,7 @@ use crate::{ config::scripts::{ScriptId, SetupScriptConfig}, list::TestInstance, reporter::{ - TestOutputDisplay, + TestOutputDisplayStreams, events::{ ExecuteStatus, ExecutionResult, InfoResponse, RetryData, SetupScriptEnvMap, SetupScriptExecuteStatus, StressIndex, UnitState, @@ -91,7 +91,7 @@ pub(super) enum ExecutorEvent<'a> { AttemptFailedWillRetry { stress_index: Option, test_instance: TestInstance<'a>, - failure_output: TestOutputDisplay, + failure_output: TestOutputDisplayStreams, run_status: ExecuteStatus, delay_before_next_attempt: Duration, }, @@ -105,8 +105,8 @@ pub(super) enum ExecutorEvent<'a> { Finished { stress_index: Option, test_instance: TestInstance<'a>, - success_output: TestOutputDisplay, - failure_output: TestOutputDisplay, + success_output: TestOutputDisplayStreams, + failure_output: TestOutputDisplayStreams, junit_store_success_output: bool, junit_store_failure_output: bool, last_run_status: ExecuteStatus,