[CI][snap_perf]: re-gather baselines

Gather baselines for all combinations of setup
that we currently support: 4.14 Sync, 5.10 Sync and Async.

Signed-off-by: Diana Popa <dpopa@amazon.com>

Author: dianpopa

tests/integration_tests/performance/test_snapshot_perf.py

@@ -6,6 +6,8 @@
 import logging
 import os
 import platform
+import pytest
+
 from conftest import _test_images_s3_bucket
 from framework.artifacts import ArtifactCollection, ArtifactSet
 from framework.defs import DEFAULT_TEST_IMAGES_S3_BUCKET
@@ -28,6 +30,10 @@
 SAMPLE_COUNT = 3
 USEC_IN_MSEC = 1000
 PLATFORM = platform.machine()
+ENGINES = ["Sync"]
+
+if is_io_uring_supported():
+    ENGINES.append("Async")
 
 # Latencies in milliseconds.
 # The latency for snapshot creation has high variance due to scheduler noise.
@@ -60,12 +66,40 @@
 # TODO: Update the table after fix. Target is < 5ms.
 LOAD_LATENCY_BASELINES = {
     "x86_64": {
-        "2vcpu_256mb.json": {"target": 9},
-        "2vcpu_512mb.json": {"target": 9},
+        "4.14": {
+            "sync": {
+                "2vcpu_256mb.json": {"target": 9},
+                "2vcpu_512mb.json": {"target": 9},
+            }
+        },
+        "5.10": {
+            "sync": {
+                "2vcpu_256mb.json": {"target": 60},
+                "2vcpu_512mb.json": {"target": 60},
+            },
+            "async": {
+                "2vcpu_256mb.json": {"target": 190},
+                "2vcpu_512mb.json": {"target": 190},
+            },
+        },
     },
     "aarch64": {
-        "2vcpu_256mb.json": {"target": 3},
-        "2vcpu_512mb.json": {"target": 3},
+        "4.14": {
+            "sync": {
+                "2vcpu_256mb.json": {"target": 2},
+                "2vcpu_512mb.json": {"target": 2},
+            }
+        },
+        "5.10": {
+            "sync": {
+                "2vcpu_256mb.json": {"target": 2},
+                "2vcpu_512mb.json": {"target": 2},
+            },
+            "async": {
+                "2vcpu_256mb.json": {"target": 125},
+                "2vcpu_512mb.json": {"target": 130},
+            },
+        },
     },
 }
 
@@ -88,9 +122,11 @@ def snapshot_create_measurements(vm_type, snapshot_type):
     return [latency]
 
 
-def snapshot_resume_measurements(vm_type):
+def snapshot_resume_measurements(vm_type, io_engine):
     """Define measurements for snapshot resume tests."""
-    load_latency = LOAD_LATENCY_BASELINES[platform.machine()][vm_type]
+    load_latency = LOAD_LATENCY_BASELINES[platform.machine()][
+        get_kernel_version(level=1)
+    ][io_engine][vm_type]
 
     latency = types.MeasurementDef.create_measurement(
         "latency",
@@ -278,6 +314,7 @@ def _test_snapshot_resume_latency(context):
     vm_builder = context.custom["builder"]
     snapshot_type = context.custom["snapshot_type"]
     file_dumper = context.custom["results_file_dumper"]
+    io_engine = context.custom["io_engine"]
     diff_snapshots = snapshot_type == SnapshotType.DIFF
 
     logger.info(
@@ -302,6 +339,7 @@ def _test_snapshot_resume_latency(context):
         config=context.microvm,
         diff_snapshots=diff_snapshots,
         use_ramdisk=True,
+        io_engine=io_engine,
     )
     basevm = vm_instance.vm
     basevm.start()
@@ -341,7 +379,8 @@ def _test_snapshot_resume_latency(context):
         func_kwargs={},
     )
     eager_map(
-        cons.set_measurement_def, snapshot_resume_measurements(context.microvm.name())
+        cons.set_measurement_def,
+        snapshot_resume_measurements(context.microvm.name(), io_engine.lower()),
     )
 
     st_core.add_pipe(producer=prod, consumer=cons, tag=context.microvm.name())
@@ -360,6 +399,7 @@ def _test_older_snapshot_resume_latency(context):
     logger = context.custom["logger"]
     snapshot_type = context.custom["snapshot_type"]
     file_dumper = context.custom["results_file_dumper"]
+    io_engine = context.custom["io_engine"]
 
     firecracker = context.firecracker
     jailer = firecracker.jailer()
@@ -410,7 +450,8 @@ def _test_older_snapshot_resume_latency(context):
         func_kwargs={},
     )
     eager_map(
-        cons.set_measurement_def, snapshot_resume_measurements(context.microvm.name())
+        cons.set_measurement_def,
+        snapshot_resume_measurements(context.microvm.name(), io_engine.lower()),
     )
 
     st_core.add_pipe(producer=prod, consumer=cons, tag=context.microvm.name())
@@ -439,8 +480,8 @@ def test_snapshot_create_full_latency(
     # - Rootfs: Ubuntu 18.04
     # - Microvm: 2vCPU with 256/512 MB RAM
     # TODO: Multiple microvm sizes must be tested in the async pipeline.
-    microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_512mb"))
-    microvm_artifacts.insert(artifacts.microvms(keyword="2vcpu_256mb"))
+    microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_256mb"))
+    microvm_artifacts.insert(artifacts.microvms(keyword="2vcpu_512mb"))
     kernel_artifacts = ArtifactSet(artifacts.kernels())
     disk_artifacts = ArtifactSet(artifacts.disks(keyword="ubuntu"))
 
@@ -479,8 +520,8 @@ def test_snapshot_create_diff_latency(
     # - Rootfs: Ubuntu 18.04
     # - Microvm: 2vCPU with 256/512 MB RAM
     # TODO: Multiple microvm sizes must be tested in the async pipeline.
-    microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_512mb"))
-    microvm_artifacts.insert(artifacts.microvms(keyword="2vcpu_256mb"))
+    microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_256mb"))
+    microvm_artifacts.insert(artifacts.microvms(keyword="2vcpu_512mb"))
     kernel_artifacts = ArtifactSet(artifacts.kernels())
     disk_artifacts = ArtifactSet(artifacts.disks(keyword="ubuntu"))
 
@@ -504,7 +545,10 @@ def test_snapshot_create_diff_latency(
     test_matrix.run_test(_test_snapshot_create_latency)
 
 
-def test_snapshot_resume_latency(network_config, bin_cloner_path, results_file_dumper):
+@pytest.mark.parametrize("io_engine", ENGINES)
+def test_snapshot_resume_latency(
+    network_config, bin_cloner_path, results_file_dumper, io_engine
+):
     """
     Test scenario: Snapshot load performance measurement.
 
@@ -518,8 +562,8 @@ def test_snapshot_resume_latency(network_config, bin_cloner_path, results_file_d
     # - Rootfs: Ubuntu 18.04
     # - Microvm: 2vCPU with 256/512 MB RAM
     # TODO: Multiple microvm sizes must be tested in the async pipeline.
-    microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_512mb"))
-    microvm_artifacts.insert(artifacts.microvms(keyword="2vcpu_256mb"))
+    microvm_artifacts = ArtifactSet(artifacts.microvms(keyword="2vcpu_256mb"))
+    microvm_artifacts.insert(artifacts.microvms(keyword="2vcpu_512mb"))
 
     kernel_artifacts = ArtifactSet(artifacts.kernels())
     disk_artifacts = ArtifactSet(artifacts.disks(keyword="ubuntu"))
@@ -533,6 +577,7 @@ def test_snapshot_resume_latency(network_config, bin_cloner_path, results_file_d
         "snapshot_type": SnapshotType.FULL,
         "name": "resume_latency",
         "results_file_dumper": results_file_dumper,
+        "io_engine": io_engine,
     }
 
     # Create the test matrix.
@@ -544,7 +589,8 @@ def test_snapshot_resume_latency(network_config, bin_cloner_path, results_file_d
     test_matrix.run_test(_test_snapshot_resume_latency)
 
 
-def test_older_snapshot_resume_latency(bin_cloner_path, results_file_dumper):
+@pytest.mark.parametrize("io_engine", ENGINES)
+def test_older_snapshot_resume_latency(bin_cloner_path, results_file_dumper, io_engine):
     """
     Test scenario: Older snapshot load performance measurement.
 
@@ -570,6 +616,7 @@ def test_older_snapshot_resume_latency(bin_cloner_path, results_file_dumper):
         "snapshot_type": SnapshotType.FULL,
         "name": "old_snapshot_resume_latency",
         "results_file_dumper": results_file_dumper,
+        "io_engine": io_engine,
     }
 
     # Create the test matrix.