diff --git a/VERSION b/VERSION index 63c70000..d43e29f5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1!10.10.0 +1!10.10.1 diff --git a/examples/oracle/oracle-example-cluster-test.py b/examples/oracle/oracle-example-cluster-test.py index 53105b84..605bb73b 100644 --- a/examples/oracle/oracle-example-cluster-test.py +++ b/examples/oracle/oracle-example-cluster-test.py @@ -3,6 +3,7 @@ """Basic examples of various lifecycle with an OCI instance.""" import logging +import re import threading import time from datetime import datetime @@ -51,7 +52,7 @@ def cluster() -> Generator[List[OciInstance], None, None]: class TestOracleClusterBasic: """Test basic functionalities of Oracle Cluster.""" - def test_basic_ping_on_private_ips(self, cluster: List[OciInstance]): # pylint: disable=W0621 + def test_basic_ping_on_private_ips(self, cluster: List[OciInstance]): """ Test that cluster instances can ping each other on private IPs. @@ -71,7 +72,7 @@ def test_basic_ping_on_private_ips(self, cluster: List[OciInstance]): # pylint: logger.info("Successfully pinged %s from %s", private_ip, instance.private_ip) -def setup_mofed_iptables_rules(instance: OciInstance): +def setup_mofed_iptables_rules(instance: OciInstance) -> OciInstance: """ Set up IPTABLES rules for RDMA usage. @@ -126,7 +127,73 @@ def ensure_image_is_rdma_ready(instance: OciInstance): r = instance.execute("ibstatus") if not r.stdout or not r.ok: logger.info("Infiniband status: %s", r.stdout + "\n" + r.stderr) - pytest.skip("The image beiing used is not RDMA ready") + pytest.skip("The image being used is not RDMA ready") + + +def ensure_second_vnics_ready(test_cluster: List[OciInstance]): + """ + Check if all cluster instances have a secondary VNIC and attach and configure one if not. + + If the instance already has a secondary VNIC, it will skip the attachment process. + + Otherwise, it will do the following to set up the secondary VNIC: + - Attach a secondary VNIC to the instance + - Configure the secondary VNIC using information from the IMDS + - Set up the iptables rules on the appropriate NIC for RDMA usage + + Args: + test_cluster (List[OciInstance]): The cluster (list of instances) to check and configure. + """ + for instance in test_cluster: + if instance.secondary_vnic_private_ip: + logger.info( + "Instance %s already has a secondary VNIC, not attaching one.", instance.name + ) + continue + logger.info("Creating a secondary VNIC on instance %s", instance.name) + # create a secondary VNIC on the 2nd vnic on the private subnet for RDMA usage + instance.add_network_interface( + nic_index=1, + subnet_name="private subnet-mofed-vcn", # use the private subnet for mofed testing + ) + instance.configure_secondary_vnic() + setup_mofed_iptables_rules(instance) + + +def get_private_nic_pci_address(instance: OciInstance) -> str: + """ + Get the PCI address of the second NIC on the instance (mlx5_1) which is used for RDMA. + + The `mst status -v` command returns an output in the following format, where the + PCI address can be extracted from the column at index 2: + + ``` + $ sudo mst status -v | grep mlx5_1 + ConnectX6DX(rev:0) NA 4b:00.1 mlx5_1 net-ens300f1np1 + ``` + + Args: + instance (OciInstance): The instance to get the PCI address from. + + Returns: + str: The PCI address of the second NIC. + + Raises: + ValueError: If a valid PCI address cannot be parsed from the mst output. + """ + r = instance.execute("sudo mst status -v | grep mlx5_1") + if not r.ok or not r.stdout: + raise ValueError("Failed to retrieve PCI address: mst status command failed") + + try: + pciaddr = r.stdout.split()[2] + except IndexError: + raise ValueError("Failed to retrieve the second column of the mst status output") + + if not re.match(r".*[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$", pciaddr): + raise ValueError(f"Invalid PCI address format: {pciaddr}") + + return pciaddr class TestOracleClusterRdma: @@ -135,7 +202,7 @@ class TestOracleClusterRdma: @pytest.fixture(scope="class") def mofed_cluster( self, - cluster: List[OciInstance], # pylint: disable=W0621 + cluster: List[OciInstance], ) -> Generator[List[OciInstance], None, None]: """ Configure cluster for RDMA testing. @@ -144,20 +211,7 @@ def mofed_cluster( List[OciInstance]: RDMA-ready cluster instances. """ ensure_image_is_rdma_ready(cluster[0]) - for instance in cluster: - if instance.secondary_vnic_private_ip: - logger.info( - "Instance %s already has a secondary VNIC, not attaching one.", instance.name - ) - continue - logger.info("Creating a secondary VNIC on instance %s", instance.name) - # create a secondary VNIC on the 2nd vnic on the private subnet for RDMA usage - instance.add_network_interface( - nic_index=1, - subnet_name="private subnet-mofed-vcn", # use the private subnet for mofed testing - ) - instance.configure_secondary_vnic() - setup_mofed_iptables_rules(instance) + ensure_second_vnics_ready(cluster) yield cluster @@ -295,3 +349,146 @@ def start_server(): ) logger.info("ucx_perftest output: %s", r.stdout) assert r.ok, "Failed to run ucx_perftest" + + +class TestOracleClusterOfedTools: + """ + Test Nvidia tools included in OFED userspace package. + + Validate that CLI tools included in OFED are installed and executable. + Only verify query commands to avoid affecting the physical NIC firmware. + """ + + def test_mst_status(self, cluster: List[OciInstance]): + """ + Run mst status to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + + r = dut_instance.execute("sudo mst status") + logger.info("mst status output: %s", r.stdout) + assert r.ok, "Failed to run mst status" + assert "MST modules" in r.stdout + assert "PCI Devices" in r.stdout + + def test_mlxconfig(self, cluster: List[OciInstance]): + """ + Run mlxconfig to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo mlxconfig -d {pci_addr} q") + logger.info("mlxconfig query output: %s", r.stdout) + assert r.ok, "Failed to run mlxconfig query" + assert "ConnectX" in r.stdout + + def test_mlxfwmanager(self, cluster: List[OciInstance]): + """ + Run mlxfwmanager to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo mlxfwmanager -d {pci_addr} --query") + logger.info("mlxfwmanager query output: %s", r.stdout) + assert r.ok, "Failed to run mlxfwmanager query" + assert "ConnectX" in r.stdout + assert "Device Type:" in r.stdout + assert "Part Number:" in r.stdout + + def test_flint(self, cluster: List[OciInstance]): + """ + Run flint to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo flint -d {pci_addr} q") + logger.info("flint query output: %s", r.stdout) + assert r.ok, "Failed to run flint query" + assert "Image type:" in r.stdout + assert "FW Version:" in r.stdout + assert "Product Version:" in r.stdout + + def test_mlxfwreset(self, cluster: List[OciInstance]): + """ + Run mlxfwreset to confirm it is installed. + + Args: + cluster (List[OciInstance]): cluster instances + """ + dut_instance = cluster[0] + pci_addr = get_private_nic_pci_address(dut_instance) + + r = dut_instance.execute(f"sudo mlxfwreset -d {pci_addr} q") + logger.info("mlxfwreset query output: %s", r.stdout) + assert r.ok, "Failed to run mlxfwreset query" + assert "3: Driver restart and PCI reset" in r.stdout + assert "0: Tool is the owner" in r.stdout + + +class TestOracleClusterPerformance: + """Test traffic performance between Oracle Cluster instances.""" + + @pytest.fixture(scope="class") + def private_vnic_cluster( + self, + cluster: List[OciInstance], + ) -> Generator[List[OciInstance], None, None]: + """ + Cluster with private VNIC pair. + + Yields: + List[OciInstance]: Instances of cluster with private VNIC. + """ + ensure_second_vnics_ready(cluster) + + yield cluster + + def test_iperf3(self, private_vnic_cluster: List[OciInstance]): + """ + Test iperf3 between two instances. + + This tests the following: + - iperf3 successfully runs between two instances + - iperf3 throughput is greater than the minimum threshold (45.0) + + Args: + private_vnic_cluster (List[OciInstance]): Cluster using private VNICs + """ + min_throughput = 28.0 + server_instance = private_vnic_cluster[0] + client_instance = private_vnic_cluster[1] + + def start_server(): + """Start the iperf3 server on the "server_instance".""" + server_instance.execute("iperf3 -s -1") + + server_thread = threading.Thread(target=start_server) + server_thread.start() + + # Wait for iperf3 server to start before starting the client + time.sleep(5) + r = client_instance.execute( + f"iperf3 -c {server_instance.secondary_vnic_private_ip} -P 40 -Z | grep SUM" + ) + iperf3_output = r.stdout + logger.info("iperf3 output: %s", iperf3_output) + assert r.ok, "Failed to run iperf3" + + throughput = iperf3_output.splitlines()[-1].split()[5] + print("iperf3 measured throughput: %s" % throughput) + assert float(throughput) > min_throughput