feat: save block perf tests data

Save logs and json files for block perf tests.

For log parsing, specify the root_dir for `glob.glob`
otherwise it wll struggle with directory names like:
`test_block_performance[vmlinux-5.10.233-Sync-bs4096-randread-1vcpu]`

Signed-off-by: Egor Lazarchuk <yegorlz@amazon.co.uk>

Author: ShadowCurse

tests/integration_tests/performance/test_block_ab.py

@@ -5,7 +5,6 @@
 import concurrent
 import glob
 import os
-import shutil
 from pathlib import Path
 
 import pytest
@@ -45,7 +44,7 @@ def prepare_microvm_for_test(microvm):
     check_output("echo 3 > /proc/sys/vm/drop_caches")
 
 
-def run_fio(microvm, mode, block_size, fio_engine="libaio"):
+def run_fio(microvm, mode, block_size, test_output_dir, fio_engine="libaio"):
     """Run a fio test in the specified mode with block size bs."""
     cmd = (
         CmdBuilder("fio")
@@ -71,16 +70,10 @@ def run_fio(microvm, mode, block_size, fio_engine="libaio"):
         .with_arg(f"--write_bw_log={mode}")
         .with_arg(f"--write_lat_log={mode}")
         .with_arg("--log_avg_msec=1000")
+        .with_arg("--output-format=json+")
         .build()
     )
 
-    logs_path = Path(microvm.jailer.chroot_base_with_id()) / "fio_output"
-
-    if logs_path.is_dir():
-        shutil.rmtree(logs_path)
-
-    logs_path.mkdir()
-
     prepare_microvm_for_test(microvm)
 
     # Start the CPU load monitor.
@@ -93,21 +86,31 @@ def run_fio(microvm, mode, block_size, fio_engine="libaio"):
         )
 
         # Print the fio command in the log and run it
-        rc, _, stderr = microvm.ssh.run(f"cd /tmp; {cmd}")
+        rc, stdout, stderr = microvm.ssh.run(f"cd /tmp; {cmd}")
         assert rc == 0, stderr
         assert stderr == ""
 
-        microvm.ssh.scp_get("/tmp/*.log", logs_path)
+        fio_json_path = Path(test_output_dir / "fio.json")
+        with open(fio_json_path, "w", encoding="utf-8") as f:
+            f.write(stdout)
+
+        microvm.ssh.scp_get("/tmp/*.log", test_output_dir)
         microvm.ssh.check_output("rm /tmp/*.log")
 
-        return logs_path, cpu_load_future.result()
+        return cpu_load_future.result()
 
 
-def process_fio_log_files(logs_glob):
-    """Parses all fio log files matching the given glob and yields tuples of same-timestamp read and write metrics"""
+def process_fio_log_files(root_dir, logs_glob):
+    """
+    Parses all fio log files in the root_dir matching the given glob and
+    yields tuples of same-timestamp read and write metrics
+    """
+    # We specify `root_dir` for `glob.glob` because otherwise it will
+    # struggle with directory with names like:
+    # test_block_performance[vmlinux-5.10.233-Sync-bs4096-randread-1vcpu]
     data = [
-        Path(pathname).read_text("UTF-8").splitlines()
-        for pathname in glob.glob(logs_glob)
+        Path(root_dir / pathname).read_text("UTF-8").splitlines()
+        for pathname in glob.glob(logs_glob, root_dir=root_dir)
     ]
 
     assert data, "no log files found!"
@@ -134,13 +137,13 @@ def process_fio_log_files(logs_glob):
 
 def emit_fio_metrics(logs_dir, metrics):
     """Parses the fio logs in `{logs_dir}/*_[clat|bw].*.log and emits their contents as CloudWatch metrics"""
-    for bw_read, bw_write in process_fio_log_files(f"{logs_dir}/*_bw.*.log"):
+    for bw_read, bw_write in process_fio_log_files(logs_dir, "*_bw.*.log"):
         if bw_read:
             metrics.put_metric("bw_read", sum(bw_read), "Kilobytes/Second")
         if bw_write:
             metrics.put_metric("bw_write", sum(bw_write), "Kilobytes/Second")
 
-    for lat_read, lat_write in process_fio_log_files(f"{logs_dir}/*_clat.*.log"):
+    for lat_read, lat_write in process_fio_log_files(logs_dir, "*_clat.*.log"):
         # latency values in fio logs are in nanoseconds, but cloudwatch only supports
         # microseconds as the more granular unit, so need to divide by 1000.
         for value in lat_read:
@@ -164,6 +167,7 @@ def test_block_performance(
     fio_engine,
     io_engine,
     metrics,
+    results_dir,
 ):
     """
     Execute block device emulation benchmarking scenarios.
@@ -192,9 +196,9 @@ def test_block_performance(
 
     vm.pin_threads(0)
 
-    logs_dir, cpu_util = run_fio(vm, fio_mode, fio_block_size, fio_engine)
+    cpu_util = run_fio(vm, fio_mode, fio_block_size, results_dir, fio_engine)
 
-    emit_fio_metrics(logs_dir, metrics)
+    emit_fio_metrics(results_dir, metrics)
 
     for thread_name, values in cpu_util.items():
         for value in values:
@@ -213,6 +217,7 @@ def test_block_vhost_user_performance(
     fio_mode,
     fio_block_size,
     metrics,
+    results_dir,
 ):
     """
     Execute block device emulation benchmarking scenarios.
@@ -242,9 +247,9 @@ def test_block_vhost_user_performance(
     next_cpu = vm.pin_threads(0)
     vm.disks_vhost_user["scratch"].pin(next_cpu)
 
-    logs_dir, cpu_util = run_fio(vm, fio_mode, fio_block_size)
+    cpu_util = run_fio(vm, fio_mode, fio_block_size, results_dir)
 
-    emit_fio_metrics(logs_dir, metrics)
+    emit_fio_metrics(results_dir, metrics)
 
     for thread_name, values in cpu_util.items():
         for value in values: