test: default to not honor the performance baseline checks

But add an option to check them

Signed-off-by: Pablo Barbáchano <pablob@amazon.com>

Author: pb8o

tests/conftest.py

@@ -125,6 +125,15 @@ def test_with_any_microvm(test_microvm_any):
 METRICS = get_metrics_logger()
 
 
+def pytest_addoption(parser):
+    """Pytest hook. Add command line options."""
+    parser.addoption(
+        "--perf-fail",
+        action="store_true",
+        help="fail the test if the baseline does not match",
+    )
+
+
 @pytest.fixture(scope="function", autouse=True)
 def record_props(request, record_property):
     """Decorate test results with additional properties.

tests/framework/stats/consumer.py

@@ -100,7 +100,7 @@ def _reset(self):
         """Reset the results of this consumer, used in a previous exercise."""
         self._results = defaultdict()
 
-    def process(self, fail_fast=False) -> (dict, dict):
+    def process(self, check=True, fail_fast=False) -> (dict, dict):
         """Generate statistics as a dictionary."""
         self._validate()
         for ms_name in self._results:
@@ -127,7 +127,7 @@ def process(self, fail_fast=False) -> (dict, dict):
                     }
 
                 pass_criteria = st_def.pass_criteria
-                if pass_criteria:
+                if check and pass_criteria:
                     # if the statistic definition contains a criteria but the
                     # corresponding baseline is not defined, the test should fail.
                     if pass_criteria.baseline == {}:

tests/framework/stats/core.py

@@ -53,6 +53,7 @@ def __init__(self, name="<PLACEHOLDER>", iterations=1, custom={}):
         self._failure_aggregator = CoreException()
         self.metrics_test = None
         self.metrics = None
+        self.check_baseline = True
 
     def add_pipe(self, producer: Producer, consumer: Consumer, tag=None):
         """Add a new producer-consumer pipe."""
@@ -83,7 +84,7 @@ def run_exercise(self, fail_fast=False) -> Result:
                         self.metrics.flush()
 
             try:
-                stats, custom = pipe.consumer.process(fail_fast)
+                stats, custom = pipe.consumer.process(check=self.check_baseline)
             except (ProcessingException, AssertionError) as err:
                 self._failure_aggregator.add_row(f"Failed on '{tag}':")
                 self._failure_aggregator.add_row(err)

tests/integration_tests/performance/conftest.py

@@ -85,6 +85,7 @@ def send_metrics(metrics, stats: core.Core):
 def st_core(metrics, results_file_dumper, guest_kernel, rootfs, request):
     """Helper fixture to dump results and publish metrics"""
     stats = core.Core()
+    stats.check_baseline = request.config.getoption("--perf-fail")
     stats.iterations = 1
     stats.custom = {
         "instance": global_props.instance,