From 6766df23b5778770901dc0964f7a3245cb2f578f Mon Sep 17 00:00:00 2001 From: Davide Trentin Date: Thu, 27 Feb 2025 16:25:39 -0500 Subject: [PATCH] test(oracle): add OCI example tests for OFED userspace tools and iperf3 Add test cases to the Oracle cluster example, to validate the presence of Nvidia firmware CLI tools and to confirm that the throughput measured by iperf3 is acceptable. --- VERSION | 2 +- .../oracle/oracle-example-cluster-test.py | 233 ++++++++++++++++-- 2 files changed, 216 insertions(+), 19 deletions(-) diff --git a/VERSION b/VERSION index 63c70000..d43e29f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1!10.10.0 +1!10.10.1 diff --git a/examples/oracle/oracle-example-cluster-test.py b/examples/oracle/oracle-example-cluster-test.py index 53105b84..605bb73b 100644 --- a/examples/oracle/oracle-example-cluster-test.py +++ b/examples/oracle/oracle-example-cluster-test.py @@ -3,6 +3,7 @@ """Basic examples of various lifecycle with an OCI instance.""" import logging +import re import threading import time from datetime import datetime @@ -51,7 +52,7 @@ def cluster() -> Generator[List[OciInstance], None, None]: class TestOracleClusterBasic: """Test basic functionalities of Oracle Cluster.""" - def test_basic_ping_on_private_ips(self, cluster: List[OciInstance]): # pylint: disable=W0621 + def test_basic_ping_on_private_ips(self, cluster: List[OciInstance]): """ Test that cluster instances can ping each other on private IPs. @@ -71,7 +72,7 @@ def test_basic_ping_on_private_ips(self, cluster: List[OciInstance]): # pylint: logger.info("Successfully pinged %s from %s", private_ip, instance.private_ip) -def setup_mofed_iptables_rules(instance: OciInstance): +def setup_mofed_iptables_rules(instance: OciInstance) -> OciInstance: """ Set up IPTABLES rules for RDMA usage. @@ -126,7 +127,73 @@ def ensure_image_is_rdma_ready(instance: OciInstance): r = instance.execute("ibstatus") if not r.stdout or not r.ok: logger.info("Infiniband status: %s", r.stdout + "\n" + r.stderr) - pytest.skip("The image beiing used is not RDMA ready") + pytest.skip("The image being used is not RDMA ready") + + +def ensure_second_vnics_ready(test_cluster: List[OciInstance]): + """ + Check if all cluster instances have a secondary VNIC and attach and configure one if not. + + If the instance already has a secondary VNIC, it will skip the attachment process. + + Otherwise, it will do the following to set up the secondary VNIC: + - Attach a secondary VNIC to the instance + - Configure the secondary VNIC using information from the IMDS + - Set up the iptables rules on the appropriate NIC for RDMA usage + + Args: + test_cluster (List[OciInstance]): The cluster (list of instances) to check and configure. + """ + for instance in test_cluster: + if instance.secondary_vnic_private_ip: + logger.info( + "Instance %s already has a secondary VNIC, not attaching one.", instance.name + ) + continue + logger.info("Creating a secondary VNIC on instance %s", instance.name) + # create a secondary VNIC on the 2nd vnic on the private subnet for RDMA usage + instance.add_network_interface( + nic_index=1, + subnet_name="private subnet-mofed-vcn", # use the private subnet for mofed testing + ) + instance.configure_secondary_vnic() + setup_mofed_iptables_rules(instance) + + +def get_private_nic_pci_address(instance: OciInstance) -> str: + """ + Get the PCI address of the second NIC on the instance (mlx5_1) which is used for RDMA. + + The `mst status -v` command returns an output in the following format, where the + PCI address can be extracted from the column at index 2: + + ``` + $ sudo mst status -v | grep mlx5_1 + ConnectX6DX(rev:0) NA 4b:00.1 mlx5_1 net-ens300f1np1 + ``` + + Args: + instance (OciInstance): The instance to get the PCI address from. + + Returns: + str: The PCI address of the second NIC. + + Raises: + ValueError: If a valid PCI address cannot be parsed from the mst output. + """ + r = instance.execute("sudo mst status -v | grep mlx5_1") + if not r.ok or not r.stdout: + raise ValueError("Failed to retrieve PCI address: mst status command failed") + + try: + pciaddr = r.stdout.split()[2] + except IndexError: + raise ValueError("Failed to retrieve the second column of the mst status output") + + if not re.match(r".*[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$", pciaddr): + raise ValueError(f"Invalid PCI address format: {pciaddr}") + + return pciaddr class TestOracleClusterRdma: @@ -135,7 +202,7 @@ class TestOracleClusterRdma: @pytest.fixture(scope="class") def mofed_cluster( self, - cluster: List[OciInstance], # pylint: disable=W0621 + cluster: List[OciInstance], ) -> Generator[List[OciInstance], None, None]: """ Configure cluster for RDMA testing. @@ -144,20 +211,7 @@ def mofed_cluster( List[OciInstance]: RDMA-ready cluster instances. """ ensure_image_is_rdma_ready(cluster[0]) - for instance in cluster: - if instance.secondary_vnic_private_ip: - logger.info( - "Instance %s already has a secondary VNIC, not attaching one.", instance.name - ) - continue - logger.info("Creating a secondary VNIC on instance %s", instance.name) - # create a secondary VNIC on the 2nd vnic on the private subnet for RDMA usage - instance.add_network_interface( - nic_index=1, - subnet_name="private subnet-mofed-vcn", # use the private subnet for mofed testing - ) - instance.configure_secondary_vnic() - setup_mofed_iptables_rules(instance) + ensure_second_vnics_ready(cluster) yield cluster @@ -295,3 +349,146 @@ def start_server(): ) logger.info("ucx_perftest output: %s", r.stdout) assert r.ok, "Failed to run ucx_perftest" + + +class TestOracleClusterOfedTools: + """ + Test Nvidia tools included in OFED userspace package. + + Validate that CLI tools included in OFED are installed and executable. + Only verify query commands to avoid affecting the physical NIC firmware. + """ + + def test_mst_status(self, cluster: List[OciInstance]): + """ + Run mst status to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + + r = dut_instance.execute("sudo mst status") + logger.info("mst status output: %s", r.stdout) + assert r.ok, "Failed to run mst status" + assert "MST modules" in r.stdout + assert "PCI Devices" in r.stdout + + def test_mlxconfig(self, cluster: List[OciInstance]): + """ + Run mlxconfig to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo mlxconfig -d {pci_addr} q") + logger.info("mlxconfig query output: %s", r.stdout) + assert r.ok, "Failed to run mlxconfig query" + assert "ConnectX" in r.stdout + + def test_mlxfwmanager(self, cluster: List[OciInstance]): + """ + Run mlxfwmanager to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo mlxfwmanager -d {pci_addr} --query") + logger.info("mlxfwmanager query output: %s", r.stdout) + assert r.ok, "Failed to run mlxfwmanager query" + assert "ConnectX" in r.stdout + assert "Device Type:" in r.stdout + assert "Part Number:" in r.stdout + + def test_flint(self, cluster: List[OciInstance]): + """ + Run flint to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo flint -d {pci_addr} q") + logger.info("flint query output: %s", r.stdout) + assert r.ok, "Failed to run flint query" + assert "Image type:" in r.stdout + assert "FW Version:" in r.stdout + assert "Product Version:" in r.stdout + + def test_mlxfwreset(self, cluster: List[OciInstance]): + """ + Run mlxfwreset to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo mlxfwreset -d {pci_addr} q") + logger.info("mlxfwreset query output: %s", r.stdout) + assert r.ok, "Failed to run mlxfwreset query" + assert "3: Driver restart and PCI reset" in r.stdout + assert "0: Tool is the owner" in r.stdout + + +class TestOracleClusterPerformance: + """Test traffic performance between Oracle Cluster instances.""" + + @pytest.fixture(scope="class") + def private_vnic_cluster( + self, + cluster: List[OciInstance], + ) -> Generator[List[OciInstance], None, None]: + """ + Cluster with private VNIC pair. + + Yields: + List[OciInstance]: Instances of cluster with private VNIC. + """ + ensure_second_vnics_ready(cluster) + + yield cluster + + def test_iperf3(self, private_vnic_cluster: List[OciInstance]): + """ + Test iperf3 between two instances. + + This tests the following: + - iperf3 successfully runs between two instances + - iperf3 throughput is greater than the minimum threshold (45.0) + + Args: + private_vnic_cluster (List[OciInstance]): Cluster using private VNICs + """ + min_throughput = 28.0 + server_instance = private_vnic_cluster[0] + client_instance = private_vnic_cluster[1] + + def start_server(): + """Start the iperf3 server on the "server_instance".""" + server_instance.execute("iperf3 -s -1") + + server_thread = threading.Thread(target=start_server) + server_thread.start() + + # Wait for iperf3 server to start before starting the client + time.sleep(5) + r = client_instance.execute( + f"iperf3 -c {server_instance.secondary_vnic_private_ip} -P 40 -Z | grep SUM" + ) + iperf3_output = r.stdout + logger.info("iperf3 output: %s", iperf3_output) + assert r.ok, "Failed to run iperf3" + + throughput = iperf3_output.splitlines()[-1].split()[5] + print("iperf3 measured throughput: %s" % throughput) + assert float(throughput) > min_throughput