Group test_starccm and openfoam, and improve logging message output

- Move duplicated codes to common file
- Modify two tests' config files to make sure they are the same and can be used by both tests
- Add dependencies.install.sh in openfoam testdir as well to avoid failures
- Create a new shared_performance_test_cluster fixture, to group two tests in the same stack
- Now, if openfoam and starccm are not installed, it will not log errors
- Now, when percentage_difference is 0, will log matching baseline message

Author: hehe7318

tests/integration-tests/tests/performance_tests/common.py

@@ -27,6 +27,7 @@
 PYTEST_PARAMETERIZE_VALUES = [(NUM_COMPUTE_NODES, 1)]
 TEST_RUNNER_SCRIPT = "/shared/assets/workloads/scale-test/run-scale-test.sh"
 ROUND_UP_FACTOR = 100_000_000
+PERF_TEST_DIFFERENCE_TOLERANCE = 3
 
 METRICS = [
     dict(name="jobRunTime", unit="ms"),
@@ -222,3 +223,29 @@ def write_results_to_output_dir(
         paths["baseline"]["statistics.json"],
         paths[candidate_configuration]["statistics.json"],
     )
+
+
+def perf_test_difference(observed_value, baseline_value):
+    percentage_difference = 100 * (observed_value - baseline_value) / baseline_value
+    return percentage_difference
+
+
+def _log_output_performance_difference(node, performance_degradation, observed_value, baseline_value):
+    percentage_difference = perf_test_difference(observed_value, baseline_value)
+    if percentage_difference < 0:
+        outcome = "improvement"
+    elif percentage_difference == 0:
+        outcome = "matching baseline"
+    elif percentage_difference <= PERF_TEST_DIFFERENCE_TOLERANCE:
+        outcome = "degradation (within tolerance)"
+    else:
+        outcome = "degradation (above tolerance)"
+        performance_degradation[node] = {
+            "baseline": baseline_value,
+            "observed": observed_value,
+            "percentage_difference": percentage_difference,
+        }
+    logging.info(
+        f"Nodes: {node}, Baseline: {baseline_value} seconds, Observed: {observed_value} seconds, "
+        f"Percentage difference: {percentage_difference}%, Outcome: {outcome}"
+    )

tests/integration-tests/tests/performance_tests/conftest.py

@@ -0,0 +1,41 @@
+# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "LICENSE.txt" file accompanying this file.
+# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, express or implied.
+# See the License for the specific language governing permissions and limitations under the License.
+
+import logging
+
+import boto3
+import pytest
+
+OSS_REQUIRING_EXTRA_DEPS = ["alinux2023", "rhel8", "rocky8"]
+NUMBER_OF_NODES = [8, 16, 32]
+
+
+@pytest.fixture(scope="session")
+def shared_performance_test_cluster(
+    vpc_stack, pcluster_config_reader, clusters_factory, test_datadir, s3_bucket_factory
+):
+
+    def _shared_performance_test_cluster(instance, os, region, scheduler):
+        bucket_name = s3_bucket_factory()
+        s3 = boto3.client("s3")
+        s3.upload_file(str(test_datadir / "dependencies.install.sh"), bucket_name, "scripts/dependencies.install.sh")
+
+        cluster_config = pcluster_config_reader(
+            bucket_name=bucket_name,
+            install_extra_deps=os in OSS_REQUIRING_EXTRA_DEPS,
+            number_of_nodes=max(NUMBER_OF_NODES),
+        )
+        cluster = clusters_factory(cluster_config)
+        logging.info("Cluster Created")
+        return cluster
+
+    return _shared_performance_test_cluster

tests/integration-tests/tests/performance_tests/test_openfoam.py

@@ -4,6 +4,8 @@
 import pytest
 from remote_command_executor import RemoteCommandExecutionError, RemoteCommandExecutor
 
+from tests.performance_tests.common import _log_output_performance_difference
+
 # timeout in seconds
 OPENFOAM_INSTALLATION_TIMEOUT = 300
 OPENFOAM_JOB_TIMEOUT = 5400  # Takes long time because during the first time, it's not only execute the job but also
@@ -17,20 +19,15 @@
     "rhel8": {8: 742, 16: 376, 32: 185},  # v3.6.0 just a placeholder, RHEL8 not supported
     "rocky8": {8: 742, 16: 376, 32: 185},  # v3.8.0 just a placeholder, Rocky8 not supported
 }
-PERF_TEST_DIFFERENCE_TOLERANCE = 3
-
-
-def perf_test_difference(observed_value, baseline_value):
-    percentage_difference = 100 * (observed_value - baseline_value) / baseline_value
-    return percentage_difference
 
 
 def openfoam_installed(headnode):
     cmd = '[ -d "/shared/SubspaceBenchmarks" ]'
     try:
-        headnode.run_remote_command(cmd)
+        headnode.run_remote_command(cmd, log_error=False)
         return True
     except RemoteCommandExecutionError:
+        logging.info("OpenFOAM is not installed on the head node.")
         return False
 
 
@@ -61,13 +58,11 @@ def test_openfoam(
     region,
     scheduler,
     pcluster_config_reader,
-    clusters_factory,
+    shared_performance_test_cluster,
     number_of_nodes,
     test_datadir,
 ):
-    cluster_config = pcluster_config_reader(number_of_nodes=max(number_of_nodes))
-    cluster = clusters_factory(cluster_config)
-    logging.info("Cluster Created")
+    cluster = shared_performance_test_cluster(instance, os, region, scheduler)
     remote_command_executor = RemoteCommandExecutor(cluster)
     if not openfoam_installed(remote_command_executor):
         logging.info("Installing OpenFOAM")
@@ -92,22 +87,7 @@ def test_openfoam(
     # Check results and log performance degradation
     for node, observed_value in zip(number_of_nodes, [observed_value_8, observed_value_16, observed_value_32]):
         baseline_value = BASELINE_CLUSTER_SIZE_ELAPSED_SECONDS[os][node]
-        percentage_difference = perf_test_difference(observed_value, baseline_value)
-        if percentage_difference < 0:
-            outcome = "improvement"
-        elif percentage_difference <= PERF_TEST_DIFFERENCE_TOLERANCE:
-            outcome = "degradation (within tolerance)"
-        else:
-            outcome = "degradation (above tolerance)"
-            performance_degradation[node] = {
-                "baseline": baseline_value,
-                "observed": observed_value,
-                "percentage_difference": percentage_difference,
-            }
-        logging.info(
-            f"Nodes: {node}, Baseline: {baseline_value} seconds, Observed: {observed_value} seconds, "
-            f"Percentage difference: {percentage_difference}%, Outcome: {outcome}"
-        )
+        _log_output_performance_difference(node, performance_degradation, observed_value, baseline_value)
 
     if performance_degradation:
         pytest.fail(f"Performance degradation detected: {performance_degradation}")

tests/integration-tests/tests/performance_tests/test_openfoam/test_openfoam/dependencies.install.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# This script installs the necessary software stack for StarCCM+.
+# Note: The same cluster is shared by both test_openfoam and test_starccm.
+# The cluster will be created by whichever test (test_openfoam or test_starccm) is executed first.
+# If test_openfoam is executed first, it will also need to install the required dependencies.
+set -ex
+
+sudo yum install -y libnsl

tests/integration-tests/tests/performance_tests/test_openfoam/test_openfoam/pcluster.config.yaml

@@ -16,12 +16,23 @@ HeadNode:
       - BucketName: performance-tests-resources-for-parallelcluster
         KeyName: openfoam/*
         EnableWriteAccess: false
+      - BucketName: performance-tests-resources-for-parallelcluster
+        KeyName: starccm/*
+        EnableWriteAccess: false
+{% if install_extra_deps %}
+      - BucketName: {{ bucket_name }}
+        KeyName: scripts/dependencies.install.sh
+        EnableWriteAccess: false
+  CustomActions:
+    OnNodeConfigured:
+      Script: s3://{{ bucket_name }}/scripts/dependencies.install.sh
+{% endif %}
 Scheduling:
   Scheduler: slurm
   SlurmQueues:
     - Name: q1
       ComputeResources:
-        - Name: c5n18xl-efa
+        - Name: c5n-18xl-efa
           InstanceType: {{ instance }}
           MinCount: {{ number_of_nodes }}
           MaxCount: {{ number_of_nodes }}
@@ -37,6 +48,15 @@ Scheduling:
       Iam:
         AdditionalIamPolicies:
           - Policy: arn:{{partition}}:iam::aws:policy/AmazonSSMManagedInstanceCore # Required to report patching status
+{% if install_extra_deps %}
+        S3Access:
+          - BucketName: {{ bucket_name }}
+            KeyName: scripts/dependencies.install.sh
+            EnableWriteAccess: false
+      CustomActions:
+        OnNodeConfigured:
+          Script: s3://{{ bucket_name }}/scripts/dependencies.install.sh
+{% endif %}
 SharedStorage:
   - MountDir: /shared
     Name: shared-fsx

tests/integration-tests/tests/performance_tests/test_starccm.py

@@ -7,6 +7,7 @@
 from remote_command_executor import RemoteCommandExecutionError, RemoteCommandExecutor
 
 from tests.common.utils import assert_no_file_handler_leak, get_compute_ip_to_num_files
+from tests.performance_tests.common import _log_output_performance_difference
 
 # timeout in seconds
 STARCCM_INSTALLATION_TIMEOUT = 1800
@@ -22,7 +23,6 @@
     "rhel8": {8: 66.494, 16: 36.154, 32: 20.347},  # v3.6.0
     "rocky8": {8: 66.859, 16: 36.184, 32: 21.090},  # v3.8.0
 }
-PERF_TEST_DIFFERENCE_TOLERANCE = 3
 
 OSS_REQUIRING_EXTRA_DEPS = ["alinux2023", "rhel8", "rocky8"]
 
@@ -34,17 +34,13 @@ def get_starccm_secrets(region_name):
     return secrets["podkey"], secrets["licpath"]
 
 
-def perf_test_difference(observed_value, baseline_value):
-    percentage_difference = 100 * (observed_value - baseline_value) / baseline_value
-    return percentage_difference
-
-
 def starccm_installed(headnode):
     cmd = "/shared/STAR-CCM+/18.02.008/STAR-CCM+18.02.008/star/bin/starccm+ --version"
     try:
-        headnode.run_remote_command(cmd)
+        headnode.run_remote_command(cmd, log_error=False)
         return True
     except RemoteCommandExecutionError:
+        logging.info("STAR-CCM+ is not installed on the head node.")
         return False
 
 
@@ -76,24 +72,13 @@ def test_starccm(
     region,
     scheduler,
     pcluster_config_reader,
-    clusters_factory,
+    shared_performance_test_cluster,
     number_of_nodes,
     test_datadir,
     scheduler_commands_factory,
     s3_bucket_factory,
 ):
-    # Create S3 bucket for custom actions scripts
-    bucket_name = s3_bucket_factory()
-    s3 = boto3.client("s3")
-    s3.upload_file(str(test_datadir / "dependencies.install.sh"), bucket_name, "scripts/dependencies.install.sh")
-
-    cluster_config = pcluster_config_reader(
-        bucket_name=bucket_name,
-        install_extra_deps=os in OSS_REQUIRING_EXTRA_DEPS,
-        number_of_nodes=max(number_of_nodes),
-    )
-    cluster = clusters_factory(cluster_config)
-    logging.info("Cluster Created")
+    cluster = shared_performance_test_cluster(instance, os, region, scheduler)
     remote_command_executor = RemoteCommandExecutor(cluster)
     scheduler_commands = scheduler_commands_factory(remote_command_executor)
     init_num_files = get_compute_ip_to_num_files(remote_command_executor, scheduler_commands)
@@ -126,22 +111,7 @@ def test_starccm(
     # Check results and log performance degradation
     for node, observed_value in zip(number_of_nodes, [observed_value_8, observed_value_16, observed_value_32]):
         baseline_value = BASELINE_CLUSTER_SIZE_ELAPSED_SECONDS[os][node]
-        percentage_difference = perf_test_difference(observed_value, baseline_value)
-        if percentage_difference < 0:
-            outcome = "improvement"
-        elif percentage_difference <= PERF_TEST_DIFFERENCE_TOLERANCE:
-            outcome = "degradation (within tolerance)"
-        else:
-            outcome = "degradation (above tolerance)"
-            performance_degradation[node] = {
-                "baseline": baseline_value,
-                "observed": observed_value,
-                "percentage_difference": percentage_difference,
-            }
-        logging.info(
-            f"Nodes: {node}, Baseline: {baseline_value} seconds, Observed: {observed_value} seconds, "
-            f"Percentage difference: {percentage_difference}%, Outcome: {outcome}"
-        )
+        _log_output_performance_difference(node, performance_degradation, observed_value, baseline_value)
 
     assert_no_file_handler_leak(init_num_files, remote_command_executor, scheduler_commands)
 

tests/integration-tests/tests/performance_tests/test_starccm/test_starccm/pcluster.config.yaml

@@ -1,6 +1,8 @@
 Region: {{ region }}
 Image:
   Os: {{ os }}
+Imds:
+  ImdsSupport: v2.0
 HeadNode:
   InstanceType: {{ instance }}
   Networking:
@@ -11,6 +13,9 @@ HeadNode:
     AdditionalIamPolicies:
       - Policy: arn:{{partition}}:iam::aws:policy/AmazonSSMManagedInstanceCore #Required to report patching status
     S3Access:
+      - BucketName: performance-tests-resources-for-parallelcluster
+        KeyName: openfoam/*
+        EnableWriteAccess: false
       - BucketName: performance-tests-resources-for-parallelcluster
         KeyName: starccm/*
         EnableWriteAccess: false
@@ -59,5 +64,7 @@ SharedStorage:
     FsxLustreSettings:
       StorageCapacity: 2400
       DeploymentType: PERSISTENT_1
+      AutomaticBackupRetentionDays: 30
+      DailyAutomaticBackupStartTime: 00:00
       PerUnitStorageThroughput: 100
       StorageType: SSD