test: run throughput perf tests with swiotlb enabled

On aarch64, additionally parametrize our throughput performance tests
(network, block and vsock) by swiotlb size, to get a feeling for the
performance impact bounce buffering will have. Try out different sizes
of swiotlb region to see whether we can identify a cut off. Also add it
to the boottime test, because swiotlb can impact the time taken to read
the rootfs.

Signed-off-by: Patrick Roy <roypat@amazon.co.uk>

Author: roypat

tests/conftest.py

@@ -376,6 +376,19 @@ def io_engine(request):
     return request.param
 
 
+# 4MiB is the smallest swiotlb size that seems to generally work for all
+# perf tests. 2MiB makes the vsock throughput test consistently fail, and 1MiB makes
+# the network throughput test occasionally fail (both due to connection issues).
+# The block test passes even with the minimum of 1MiB. We pick 8 to have enough
+# buffer to the failing cases.
+@pytest.fixture(params=[None, 8] if platform.machine() == "aarch64" else [None])
+def memory_config(request):
+    """Differently configured swiotlb regions. Only supported on aarch64"""
+    if request.param is None:
+        return None
+    return {"initial_swiotlb_size": request.param}
+
+
 @pytest.fixture
 def results_dir(request):
     """

tests/framework/microvm.py

@@ -248,6 +248,7 @@ def __init__(
         self.disks_vhost_user = {}
         self.vcpus_count = None
         self.mem_size_bytes = None
+        self.memory_config = None
         self.cpu_template_name = None
         # The given custom CPU template will be set in basic_config() but could
         # be overwritten via set_cpu_template().
@@ -462,6 +463,11 @@ def dimensions(self):
             "rootfs": self.rootfs_file.name,
             "vcpus": str(self.vcpus_count),
             "guest_memory": f"{self.mem_size_bytes / (1024 * 1024)}MB",
+            **(
+                {k: str(v) for k, v in self.memory_config.items()}
+                if self.memory_config
+                else {}
+            ),
         }
 
     @property
@@ -729,6 +735,7 @@ def basic_config(
         rootfs_io_engine=None,
         cpu_template: Optional[str] = None,
         enable_entropy_device=False,
+        memory_config=None,
     ):
         """Shortcut for quickly configuring a microVM.
 
@@ -747,15 +754,23 @@ def basic_config(
 
         Reference: file:../../src/vmm/src/vmm_config/boot_source.rs::DEFAULT_KERNEL_CMDLINE
         """
+        # Have to do it this way as otherwise A/B-tests fail if the 'A' revision
+        # of Firecracker doesn't know about the mem_config parameter.
+        kwargs = {}
+        if memory_config is not None:
+            kwargs["mem_config"] = memory_config
+
         self.api.machine_config.put(
             vcpu_count=vcpu_count,
             smt=smt,
             mem_size_mib=mem_size_mib,
             track_dirty_pages=track_dirty_pages,
             huge_pages=huge_pages,
+            **kwargs,
         )
         self.vcpus_count = vcpu_count
         self.mem_size_bytes = mem_size_mib * 2**20
+        self.memory_config = memory_config
 
         if self.custom_cpu_template is not None:
             self.set_cpu_template(self.custom_cpu_template)

tests/integration_tests/performance/test_block_ab.py

@@ -148,14 +148,20 @@ def test_block_performance(
     fio_mode,
     fio_block_size,
     io_engine,
+    memory_config,
     metrics,
 ):
     """
     Execute block device emulation benchmarking scenarios.
     """
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=GUEST_MEM_MIB, memory_config=memory_config
+    )
     vm.add_net_iface()
     # Add a secondary block device for benchmark tests.
     fs = drive_tools.FilesystemFile(

tests/integration_tests/performance/test_boottime.py

@@ -8,8 +8,6 @@
 
 import pytest
 
-from framework.properties import global_props
-
 # Regex for obtaining boot time from some string.
 TIMESTAMP_LOG_REGEX = r"Guest-boot-time\s+\=\s+(\d+)\s+us"
 
@@ -19,14 +17,6 @@
 )
 
 
-DIMENSIONS = {
-    "instance": global_props.instance,
-    "cpu_model": global_props.cpu_model,
-    "host_os": global_props.host_os,
-    "host_kernel": "linux-" + global_props.host_linux_version_metrics,
-}
-
-
 def _get_microvm_boottime(vm):
     """Auxiliary function for asserting the expected boot time."""
     boot_time_us = None
@@ -75,19 +65,18 @@ def find_events(log_data):
 )
 @pytest.mark.nonci
 def test_boottime(
-    microvm_factory, guest_kernel_acpi, rootfs_rw, vcpu_count, mem_size_mib, metrics
+    microvm_factory,
+    guest_kernel_acpi,
+    rootfs_rw,
+    vcpu_count,
+    mem_size_mib,
+    memory_config,
+    metrics,
 ):
     """Test boot time with different guest configurations"""
 
-    metrics.set_dimensions(
-        {
-            **DIMENSIONS,
-            "performance_test": "test_boottime",
-            "guest_kernel": guest_kernel_acpi.name,
-            "vcpus": str(vcpu_count),
-            "mem_size_mib": str(mem_size_mib),
-        }
-    )
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
 
     for _ in range(10):
         vm = microvm_factory.build(guest_kernel_acpi, rootfs_rw)
@@ -98,10 +87,14 @@ def test_boottime(
             mem_size_mib=mem_size_mib,
             boot_args=DEFAULT_BOOT_ARGS + " init=/usr/local/bin/init",
             enable_entropy_device=True,
+            memory_config=memory_config,
         )
         vm.add_net_iface()
         vm.start()
         vm.pin_threads(0)
+
+        metrics.set_dimensions({"performance_test": "test_boottime", **vm.dimensions})
+
         boottime_us = _get_microvm_boottime(vm)
         metrics.put_metric("boot_time", boottime_us, unit="Microseconds")
         timestamps = find_events(vm.log_data)

tests/integration_tests/performance/test_network_ab.py

@@ -36,16 +36,21 @@ def consume_ping_output(ping_putput, request_per_round):
 
 
 @pytest.fixture
-def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs):
+def network_microvm(request, microvm_factory, guest_kernel_acpi, rootfs, memory_config):
     """Creates a microvm with the networking setup used by the performance tests in this file.
     This fixture receives its vcpu count via indirect parameterization"""
 
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     guest_mem_mib = 1024
     guest_vcpus = request.param
 
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib)
+    vm.basic_config(
+        vcpu_count=guest_vcpus, mem_size_mib=guest_mem_mib, memory_config=memory_config
+    )
     vm.add_net_iface()
     vm.start()
     vm.pin_threads(0)

tests/integration_tests/performance/test_vsock_ab.py

@@ -73,7 +73,14 @@ def guest_command(self, port_offset):
 @pytest.mark.parametrize("payload_length", ["64K", "1024K"], ids=["p64K", "p1024K"])
 @pytest.mark.parametrize("mode", ["g2h", "h2g", "bd"])
 def test_vsock_throughput(
-    microvm_factory, guest_kernel_acpi, rootfs, vcpus, payload_length, mode, metrics
+    microvm_factory,
+    guest_kernel_acpi,
+    rootfs,
+    vcpus,
+    payload_length,
+    mode,
+    metrics,
+    memory_config,
 ):
     """
     Test vsock throughput for multiple vm configurations.
@@ -84,10 +91,15 @@ def test_vsock_throughput(
     if mode == "bd" and vcpus < 2:
         pytest.skip("bidrectional test only done with at least 2 vcpus")
 
+    if memory_config is not None and "6.1" not in guest_kernel_acpi.name:
+        pytest.skip("swiotlb only supported on aarch64/6.1")
+
     mem_size_mib = 1024
     vm = microvm_factory.build(guest_kernel_acpi, rootfs, monitor_memory=False)
     vm.spawn(log_level="Info", emit_metrics=True)
-    vm.basic_config(vcpu_count=vcpus, mem_size_mib=mem_size_mib)
+    vm.basic_config(
+        vcpu_count=vcpus, mem_size_mib=mem_size_mib, memory_config=memory_config
+    )
     vm.add_net_iface()
     # Create a vsock device
     vm.api.vsock.put(vsock_id="vsock0", guest_cid=3, uds_path="/" + VSOCK_UDS_PATH)