test: run throughput perf tests with secret freedom enabled

Aadditionally parametrize some of our throughput performance tests
(network, block and vsock) by memory config, so that they run with
secret freedom (and hence bounce buffering) enabled. Also add it to the
boottime test, because bouncing can impact the time taken to read the
rootfs.

Skip them on m6g.metal because secret freedom does not work here for
architectural reasons (and our patches do not take this into account, so
trying to use secret freedom here would result in host kernel panics).

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>

Author: roypat

tests/conftest.py

@@ -386,6 +386,20 @@ def io_engine(request):
     return request.param
 
 
+secret_free_test_cases = [False]
+if (
+    global_props.host_linux_version_metrics == "next"
+    and global_props.instance != "m6g.metal"
+):
+    secret_free_test_cases.append(True)
+
+
+@pytest.fixture(params=secret_free_test_cases)
+def secret_free(request):
+    """Supported secret hiding configuration, based on hardware"""
+    return request.param
+
+
 @pytest.fixture
 def results_dir(request):
     """

tests/framework/microvm.py

@@ -249,6 +249,7 @@ def __init__(
         self.disks_vhost_user = {}
         self.vcpus_count = None
         self.mem_size_bytes = None
+        self.secret_free = False
         self.cpu_template_name = "None"
         # The given custom CPU template will be set in basic_config() but could
         # be overwritten via set_cpu_template().
@@ -463,6 +464,7 @@ def dimensions(self):
             "rootfs": self.rootfs_file.name,
             "vcpus": str(self.vcpus_count),
             "guest_memory": f"{self.mem_size_bytes / (1024 * 1024)}MB",
+            "secret_free": str(self.secret_free or False),
         }
 
     @property
@@ -730,6 +732,7 @@ def basic_config(
         rootfs_io_engine=None,
         cpu_template: Optional[str] = None,
         enable_entropy_device=False,
+        secret_free=None,
     ):
         """Shortcut for quickly configuring a microVM.
 
@@ -748,15 +751,23 @@ def basic_config(
 
         Reference: file:../../src/vmm/src/vmm_config/boot_source.rs::DEFAULT_KERNEL_CMDLINE
         """
+        # Have to do it this way as otherwise A/B-tests fail if the 'A' revision
+        # of Firecracker doesn't know about the secret_free parameter.
+        kwargs = {}
+        if secret_free:
+            kwargs["secret_free"] = True
+
         self.api.machine_config.put(
             vcpu_count=vcpu_count,
             smt=smt,
             mem_size_mib=mem_size_mib,
             track_dirty_pages=track_dirty_pages,
             huge_pages=huge_pages,
+            **kwargs,
         )
         self.vcpus_count = vcpu_count
         self.mem_size_bytes = mem_size_mib * 2**20
+        self.secret_free = secret_free or False
 
         if self.custom_cpu_template is not None:
             self.set_cpu_template(self.custom_cpu_template)

tests/integration_tests/performance/test_block_ab.py

@@ -163,14 +163,20 @@ def test_block_performance(
     fio_block_size,
     fio_engine,
     io_engine,
+    secret_free,
     metrics,
 ):
     """
     Execute block device emulation benchmarking scenarios.
     """
+    if secret_free and io_engine == "Async":
+        pytest.skip("userspace bounce buffers not supported with async block engine")
+
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB, secret_free=secret_free
+    )
     vm.add_net_iface()
     # Add a secondary block device for benchmark tests.
     fs = drive_tools.FilesystemFile(

tests/integration_tests/performance/test_boottime.py

@@ -8,8 +8,6 @@
 
 import pytest
 
-from framework.properties import global_props
-
 # Regex for obtaining boot time from some string.
 
 DEFAULT_BOOT_ARGS = (
@@ -18,14 +16,6 @@
 )
 
 
-DIMENSIONS = {
-    "instance": global_props.instance,
-    "cpu_model": global_props.cpu_model,
-    "host_os": global_props.host_os,
-    "host_kernel": "linux-" + global_props.host_linux_version_metrics,
-}
-
-
 def get_boottime_device_info(vm):
     """Auxiliary function for asserting the expected boot time."""
     boot_time_us = None
@@ -110,20 +100,16 @@ def to_ms(v, unit):
 )
 @pytest.mark.nonci
 def test_boottime(
-    microvm_factory, guest_kernel_acpi, rootfs_rw, vcpu_count, mem_size_mib, metrics
+    microvm_factory,
+    guest_kernel_acpi,
+    rootfs_rw,
+    vcpu_count,
+    mem_size_mib,
+    secret_free,
+    metrics,
 ):
     """Test boot time with different guest configurations"""
 
-    metrics.set_dimensions(
-        {
-            **DIMENSIONS,
-            "performance_test": "test_boottime",
-            "guest_kernel": guest_kernel_acpi.name,
-            "vcpus": str(vcpu_count),
-            "mem_size_mib": str(mem_size_mib),
-        }
-    )
-
     for _ in range(10):
         vm = microvm_factory.build(guest_kernel_acpi, rootfs_rw)
         vm.jailer.extra_args.update({"boot-timer": None})
@@ -133,11 +119,14 @@ def test_boottime(
             mem_size_mib=mem_size_mib,
             boot_args=DEFAULT_BOOT_ARGS + " init=/usr/local/bin/init",
             enable_entropy_device=True,
+            secret_free=secret_free,
         )
         vm.add_net_iface()
         vm.start()
         vm.pin_threads(0)
 
+        metrics.set_dimensions({"performance_test": "test_boottime", **vm.dimensions})
+
         boot_time_us, cpu_boot_time_us = get_boottime_device_info(vm)
         metrics.put_metric(
             "guest_boot_time",

tests/integration_tests/performance/test_network_ab.py

@@ -36,7 +36,7 @@ def consume_ping_output(ping_putput, request_per_round):
 
 
 @pytest.fixture
-def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs):
+def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs, secret_free):
     """Creates a microvm with the networking setup used by the performance tests in this file.
     This fixture receives its vcpu count via indirect parameterization"""
 
@@ -45,7 +45,9 @@ def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs):
 
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib)
+    vm.basic_config(
+        vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib, secret_free=secret_free
+    )
     vm.add_net_iface()
     vm.start()
     vm.pin_threads(0)

tests/integration_tests/performance/test_vsock_ab.py

@@ -73,7 +73,14 @@ def guest_command(self, port_offset):
 @pytest.mark.parametrize("payload_length", ["64K", "1024K"], ids=["p64K", "p1024K"])
 @pytest.mark.parametrize("mode", ["g2h", "h2g", "bd"])
 def test_vsock_throughput(
-    microvm_factory, guest_kernel_acpi, rootfs, vcpus, payload_length, mode, metrics
+    microvm_factory,
+    guest_kernel_acpi,
+    rootfs,
+    vcpus,
+    payload_length,
+    mode,
+    metrics,
+    secret_free,
 ):
     """
     Test vsock throughput for multiple vm configurations.
@@ -87,7 +94,9 @@ def test_vsock_throughput(
     mem_size_mib = 1024
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=mem_size_mib)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=mem_size_mib, secret_free=secret_free
+    )
     vm.add_net_iface()
     # Create a vsock device
     vm.api.vsock.put(vsock_id="vsock0", guest_cid=3, uds_path="/" + VSOCK_UDS_PATH)