Revamp the end-of-test summary (#4089)

* Start collecting metrics on summary output

Co-authored-by: oleiade <theo@crevon.me>

* Include groups & scenarios metrics on summary

Co-authored-by: oleiade <theo@crevon.me>

* Display nested groups in the summary

Co-authored-by: oleiade <theo@crevon.me>

* Include nested checks to the end-of-test summary

Co-authored-by: oleiade <theo@crevon.me>

* Store Threholds to Summary output and its report

* Print Threholds as part of the summary output

* Add JSDoc documentation to summary.js

* Apply JS linter recommendations to summary.js

* Factor decoration in a ANSIFormatter class

* Reorganize the summary.js file for easier maintenance

* Fulfill JSDoc documentation of summary.js

* Clean up the old summary data (except for the user-defined handler)

* Par the end-of-test summary with the design

* Align metrics sections within the same block

* Move full-summary example to internal/cmd/testdata

* Pass render options missing to ANSIFormatter

* Accommodate tests to the new summary format

* Refine flaky test with undetermined amount of iterations

* Apply suggestions from code review

Co-authored-by: Oleg Bespalov <oleg.bespalov@grafana.com>

* Skip some HTTP metrics when on 'compact' mode

* Move 'summary' output to 'internal'

* Stop holding *metrics.Metric on summary.Output

* Move lib.Summary and related types and logic into internal/lib/summary pkg

* Fix duplicated thresholds on submetrics

---------

Co-authored-by: oleiade <theo@crevon.me>
Co-authored-by: Oleg Bespalov <oleg.bespalov@grafana.com>

Author: 3 people

.github/workflows/xk6-tests/xk6-test.js

@@ -11,7 +11,7 @@ export let options = {
 
 export function handleSummary(data) {
     return {
-        'summary-results.txt': data.metrics.foos.values.count.toString(),
+        'summary-results.txt': data.metrics.custom.foos.values.count.toString(),
     };
 }
 

internal/cmd/builtin_output_gen.go

@@ -37,6 +37,7 @@ const (
 	builtinOutputKafka
 	builtinOutputStatsd
 	builtinOutputExperimentalOpentelemetry
+	builtinOutputSummary
 )
 
 // TODO: move this to an output sub-module after we get rid of the old collectors?

internal/cmd/outputs.go

@@ -11,6 +11,7 @@ func TestBuiltinOutputString(t *testing.T) {
 	exp := []string{
 		"cloud", "csv", "datadog", "experimental-prometheus-rw",
 		"influxdb", "json", "kafka", "statsd", "experimental-opentelemetry",
+		"summary",
 	}
 	assert.Equal(t, exp, builtinOutputStrings())
 }

internal/cmd/outputs_test.go

@@ -23,8 +23,10 @@ import (
 	"go.k6.io/k6/internal/event"
 	"go.k6.io/k6/internal/execution"
 	"go.k6.io/k6/internal/execution/local"
+	"go.k6.io/k6/internal/lib/summary"
 	"go.k6.io/k6/internal/lib/trace"
 	"go.k6.io/k6/internal/metrics/engine"
+	summaryoutput "go.k6.io/k6/internal/output/summary"
 	"go.k6.io/k6/internal/ui/pb"
 	"go.k6.io/k6/js/common"
 	"go.k6.io/k6/lib"
@@ -189,26 +191,76 @@ func (c *cmdRun) run(cmd *cobra.Command, args []string) (err error) {
 	}
 
 	executionState := execScheduler.GetState()
-	if !testRunState.RuntimeOptions.NoSummary.Bool {
-		defer func() {
-			logger.Debug("Generating the end-of-test summary...")
-			summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, &lib.Summary{
-				Metrics:         metricsEngine.ObservedMetrics,
-				RootGroup:       testRunState.GroupSummary.Group(),
-				TestRunDuration: executionState.GetCurrentTestRunDuration(),
-				NoColor:         c.gs.Flags.NoColor,
-				UIState: lib.UIState{
-					IsStdOutTTY: c.gs.Stdout.IsTTY,
-					IsStdErrTTY: c.gs.Stderr.IsTTY,
-				},
+	if !testRunState.RuntimeOptions.NoSummary.Bool { //nolint:nestif
+		sm, err := summary.ValidateMode(testRunState.RuntimeOptions.SummaryMode.String)
+		if err != nil {
+			logger.WithError(err).Warnf(
+				"invalid summary mode %q, falling back to \"compact\" (default)",
+				testRunState.RuntimeOptions.SummaryMode.String,
+			)
+		}
+
+		switch sm {
+		// TODO: Remove this code block once we stop supporting the legacy summary, and just leave the default.
+		case summary.ModeLegacy:
+			// At the end of the test run
+			defer func() {
+				logger.Debug("Generating the end-of-test summary...")
+
+				legacySummary := &lib.LegacySummary{
+					Metrics:         metricsEngine.ObservedMetrics,
+					RootGroup:       testRunState.GroupSummary.Group(),
+					TestRunDuration: executionState.GetCurrentTestRunDuration(),
+					NoColor:         c.gs.Flags.NoColor,
+					UIState: lib.UIState{
+						IsStdOutTTY: c.gs.Stdout.IsTTY,
+						IsStdErrTTY: c.gs.Stderr.IsTTY,
+					},
+				}
+
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, legacySummary, nil)
+				if hsErr == nil {
+					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
+				}
+				if hsErr != nil {
+					logger.WithError(hsErr).Error("failed to handle the end-of-test summary")
+				}
+			}()
+		default:
+			// Instantiates the summary output
+			summaryOutput, err := summaryoutput.New(output.Params{
+				RuntimeOptions: testRunState.RuntimeOptions,
+				Logger:         c.gs.Logger,
 			})
-			if hsErr == nil {
-				hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
+			if err != nil {
+				logger.WithError(err).Error("failed to initialize the end-of-test summary output")
 			}
-			if hsErr != nil {
-				logger.WithError(hsErr).Error("failed to handle the end-of-test summary")
-			}
-		}()
+			outputs = append(outputs, summaryOutput)
+
+			// At the end of the test run
+			defer func() {
+				logger.Debug("Generating the end-of-test summary...")
+
+				summary := summaryOutput.Summary(
+					executionState.GetCurrentTestRunDuration(),
+					metricsEngine.ObservedMetrics,
+					test.initRunner.GetOptions(),
+				)
+
+				// TODO: We should probably try to move these out of the summary,
+				// likely as an additional argument like options.
+				summary.NoColor = c.gs.Flags.NoColor
+				summary.EnableColors = !summary.NoColor && c.gs.Stdout.IsTTY
+
+				summaryResult, hsErr := test.initRunner.HandleSummary(globalCtx, nil, summary)
+				if hsErr == nil {
+					hsErr = handleSummaryResult(c.gs.FS, c.gs.Stdout, c.gs.Stderr, summaryResult)
+				}
+				if hsErr != nil {
+					logger.WithError(hsErr).Error("failed to handle the end-of-test summary")
+				}
+			}()
+		}
 	}
 
 	waitInitDone := emitEvent(&event.Event{Type: event.Init})

internal/cmd/run.go

@@ -326,7 +326,7 @@ func TestThresholdsRuntimeBehavior(t *testing.T) {
 			name:              "#2518: submetrics without values should be rendered under their parent metric #2518",
 			testFilename:      "thresholds/thresholds_on_submetric_without_samples.js",
 			expExitCode:       0,
-			expStdoutContains: "     one..................: 0   0/s\n       { tag:xyz }........: 0   0/s\n",
+			expStdoutContains: "    one....................................: 0   0/s\n      { tag:xyz }..........................: 0   0/s\n",
 		},
 		{
 			name:         "#2512: parsing threshold names containing parsable tokens should be valid",
@@ -337,7 +337,7 @@ func TestThresholdsRuntimeBehavior(t *testing.T) {
 			name:                 "#2520: thresholds over metrics without values should avoid division by zero and displaying NaN values",
 			testFilename:         "thresholds/empty_sink_no_nan.js",
 			expExitCode:          0,
-			expStdoutContains:    "rate.................: 0.00%",
+			expStdoutContains:    "rate...................................: 0.00%",
 			expStdoutNotContains: "NaN",
 		},
 	}

internal/cmd/run_test.go

@@ -9,6 +9,7 @@ import (
 	"gopkg.in/guregu/null.v3"
 
 	"go.k6.io/k6/cmd/state"
+	"go.k6.io/k6/internal/lib/summary"
 	"go.k6.io/k6/lib"
 )
 
@@ -31,6 +32,8 @@ extended: base + sets "global" as alias for "globalThis"
 	flags.StringArrayP("env", "e", nil, "add/override environment variable with `VAR=value`")
 	flags.Bool("no-thresholds", false, "don't run thresholds")
 	flags.Bool("no-summary", false, "don't show the summary at the end of the test")
+	flags.String("summary-mode", summary.ModeCompact.String(), "determine the summary mode,"+
+		" \"compact\", \"full\" or \"legacy\"")
 	flags.String(
 		"summary-export",
 		"",
@@ -76,6 +79,7 @@ func runtimeOptionsFromFlags(flags *pflag.FlagSet) lib.RuntimeOptions {
 		CompatibilityMode:    getNullString(flags, "compatibility-mode"),
 		NoThresholds:         getNullBool(flags, "no-thresholds"),
 		NoSummary:            getNullBool(flags, "no-summary"),
+		SummaryMode:          getNullString(flags, "summary-mode"),
 		SummaryExport:        getNullString(flags, "summary-export"),
 		TracesOutput:         getNullString(flags, "traces-output"),
 		Env:                  make(map[string]string),
@@ -94,6 +98,11 @@ func populateRuntimeOptionsFromEnv(opts lib.RuntimeOptions, environment map[stri
 		opts.CompatibilityMode = null.StringFrom(envVar)
 	}
 
+	if _, err := lib.ValidateCompatibilityMode(opts.CompatibilityMode.String); err != nil {
+		// some early validation
+		return opts, err
+	}
+
 	if err := saveBoolFromEnv(environment, "K6_INCLUDE_SYSTEM_ENV_VARS", &opts.IncludeSystemEnvVars); err != nil {
 		return opts, err
 	}
@@ -106,7 +115,11 @@ func populateRuntimeOptionsFromEnv(opts lib.RuntimeOptions, environment map[stri
 		return opts, err
 	}
 
-	if _, err := lib.ValidateCompatibilityMode(opts.CompatibilityMode.String); err != nil {
+	if envVar, ok := environment["K6_SUMMARY_MODE"]; !opts.SummaryMode.Valid && ok {
+		opts.SummaryMode = null.StringFrom(envVar)
+	}
+
+	if _, err := summary.ValidateMode(opts.SummaryMode.String); err != nil {
 		// some early validation
 		return opts, err
 	}