Merge branch 'develop' into wip/xuanqi-develop-314

hehe7318 · web-flow · commit beebd0abf185 · 2025-06-23T14:37:18.000-04:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -19,6 +19,12 @@ CHANGELOG
 3.13.1
 ------
 
+**BUG FIXES**
+- Fix build image failures occurring on non-latest versions of Rocky Linux 9.
+
+3.13.1
+------
+
 **CHANGES**
 - Upgrade Slurm to version 24.05.8.
 - Upgrade EFA installer to 1.41.0 (from 1.38.1).
diff --git a/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml b/cli/src/pcluster/resources/imagebuilder/parallelcluster.yaml
@@ -182,7 +182,7 @@ phases:
                 fi
               fi
 
-      - name: PinKernelVersion
+      - name: PinVersion
         action: ExecuteBash
         inputs:
           commands:
@@ -191,7 +191,38 @@ phases:
               OS='{{ build.OperatingSystemName.outputs.stdout }}'
               PLATFORM='{{ build.PlatformName.outputs.stdout }}'
               KERNEL_VERSION=$(uname -a)
+              RELEASE_VERSION='{{ build.OperatingSystemVersion.outputs.stdout }}'
               if [[ ${!PLATFORM} == RHEL ]]; then
+                if [[ ${!OS} == rhel9 ]] || [[ ${!OS} == rocky9 ]]; then
+                  if [[ ! -f /etc/yum/vars/releasever ]]; then
+                    echo "yes" > /opt/parallelcluster/pin_releasesever
+                    echo ${!RELEASE_VERSION} > /etc/yum/vars/releasever
+                    yum clean all
+                  fi
+                fi
+                PACKAGE_LIST="kernel-headers-$(uname -r) kernel-devel-$(uname -r)"
+                if [[ ${!OS} != "rocky8" ]] && [[ ${!OS} != "rhel8" ]]; then
+                  PACKAGE_LIST+=" kernel-devel-matched-$(uname -r)"
+                fi
+
+                if [[ ${!OS} == "rocky8" ]] || [[ ${!OS} == "rocky9" ]] ; then
+                  for PACKAGE in ${!PACKAGE_LIST}
+                  do
+                    yum install -y ${!PACKAGE}
+                    if [ $? -ne 0 ]; then
+                      # Enable vault repository
+                      sed -i 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=http://dl.rockylinux.org/vault/rocky|g' /etc/yum.repos.d/*.repo
+                      sed -i 's|^#baseurl=https://dl.rockylinux.org/$contentdir|baseurl=https://dl.rockylinux.org/vault/rocky|g' /etc/yum.repos.d/*.repo
+                      yum install -y ${!PACKAGE}
+                    fi
+                  done
+                else
+                  for PACKAGE in ${!PACKAGE_LIST}
+                  do
+                    yum -y install ${!PACKAGE}
+                  done
+                fi
+
                 yum install -y yum-plugin-versionlock
                 # listing all the packages because wildcard does not work as expected
                 yum versionlock kernel kernel-core kernel-modules
@@ -202,9 +233,24 @@ phases:
                   yum versionlock redhat-release
                 fi
               else
+                apt-get -y install linux-headers-$(uname -r)
                 apt-mark hold linux-aws* linux-base* linux-headers* linux-image* 
               fi
-              echo "Kernel version is ${!KERNEL_VERSION}"     
+              echo "Kernel version is ${!KERNEL_VERSION}"
+
+      - name: DisableNouveau
+        action: ExecuteBash
+        inputs:
+          commands:
+            - |
+              set -v
+              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
+              /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub
+              if [[ ${!PLATFORM} == RHEL ]]; then
+                grub2-mkconfig -o /boot/grub2/grub.cfg
+              elif [[ ${!PLATFORM} == DEBIAN ]]; then
+                update-grub
+              fi
 
       # Install prerequisite OS packages
       - name: InstallPrerequisite
@@ -215,16 +261,8 @@ phases:
               set -v
               OS='{{ build.OperatingSystemName.outputs.stdout }}'
               PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              VERSION='{{ build.OperatingSystemVersion.outputs.stdout }}'
 
               if [[ ${!PLATFORM} == RHEL ]]; then
-                if [[ ${!OS} == rhel9 ]] || [[ ${!OS} == rocky9 ]]; then
-                  if [[ ! -f /etc/yum/vars/releasever ]]; then
-                    echo "yes" > /opt/parallelcluster/pin_releasesever
-                    echo ${!VERSION} > /etc/yum/vars/releasever
-                    yum clean all
-                  fi
-                fi
                 yum -y update krb5-libs
                 yum -y groupinstall development && sudo yum -y install wget jq
                 if [[ ${!OS} != alinux2023 ]]; then
@@ -244,6 +282,13 @@ phases:
                 apt-get -y install build-essential curl wget jq
               fi
 
+      - name: RebootStep
+        action: Reboot
+        onFailure: Abort
+        maxAttempts: 2
+        inputs:
+          delaySeconds: 10
+
       # Install Cinc
       - name: InstallCinc
         action: ExecuteBash
diff --git a/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml b/cli/src/pcluster/resources/imagebuilder/update_and_reboot.yaml
@@ -97,20 +97,6 @@ phases:
                 fi
               fi
 
-      - name: DisableNouveau
-        action: ExecuteBash
-        inputs:
-          commands:
-            - |
-              set -v
-              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              /bin/sed -r -i -e 's/GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 rd.driver.blacklist=nouveau nouveau.modeset=0"/' /etc/default/grub
-              if [[ ${!PLATFORM} == RHEL ]]; then
-                grub2-mkconfig -o /boot/grub2/grub.cfg
-              elif [[ ${!PLATFORM} == DEBIAN ]]; then
-                update-grub
-              fi
-
       - name: DisableUnattendedUpgrades
         action: ExecuteBash
         inputs:
@@ -129,20 +115,6 @@ phases:
                 # update package index
                 DEBIAN_FRONTEND=noninteractive apt-get -y update
               fi
-
-      - name: InstallEfiBootManager
-        action: ExecuteBash
-        inputs:
-          commands:
-            - |
-              set -v
-              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              ARCH=$(uname -m)
-              if [[ `echo ${!ARCH}` == 'aarch64' ]] && [[ ${!PLATFORM} == DEBIAN ]]; then
-                # temporary workaround to solve https://bugs.launchpad.net/ubuntu/+source/grub2-signed/+bug/1936857
-                apt-get -y install efibootmgr
-              fi
-
       - name: InstallPrerequisites
         action: ExecuteBash
         inputs:
@@ -244,42 +216,6 @@ phases:
         maxAttempts: 2
         inputs:
           delaySeconds: 10
-      - name: InstallAdditionalKernelPackages
-        action: ExecuteBash
-        inputs:
-          commands:
-            - |
-              set -v
-              OS='{{ build.OperatingSystemName.outputs.stdout }}'
-              PLATFORM='{{ build.PlatformName.outputs.stdout }}'
-              DISABLE_KERNEL_UPDATE='{{ build.DisableKernelUpdate.outputs.stdout }}'
-
-              if [[ ${!PLATFORM} == RHEL ]]; then
-                # Install kernel-devel during OS update, so that headers are aligned with new kernel.
-                # The same is done for Debian through `apt-get -y install linux-aws`
-                if [[ ${!OS} == "rocky8" ]] ; then
-                  PACKAGE="kernel-devel-$(uname -r)"
-                  RELEASE_VERSION=$(source /etc/os-release && echo ${!VERSION_ID})
-
-                  # try to install kernel source for a specific release version
-                  yum install -y ${!PACKAGE} --releasever ${!RELEASE_VERSION}
-                  if [ $? -ne 0 ]; then
-                    yum install -y wget
-                    # Previous releases are moved into a vault area once a new minor release version is available for at least a week.
-                    # https://wiki.rockylinux.org/rocky/repo/#notes-on-devel
-                    wget https://dl.rockylinux.org/vault/rocky/${!RELEASE_VERSION}/BaseOS/$(uname -m)/os/Packages/k/${!PACKAGE}.rpm
-                    yum install -y ./${!PACKAGE}.rpm
-                  fi
-                else
-                  yum -y install kernel-headers-$(uname -r)
-                  yum -y install kernel-devel-$(uname -r)
-                fi
-
-              elif [[ ${!PLATFORM} == DEBIAN ]]; then
-                if [[ ${!DISABLE_KERNEL_UPDATE} != true ]]; then
-                   apt-get -y install linux-aws linux-headers-aws linux-image-aws
-                fi
-              fi
 
       - name: RemoveKernelPin
         action: ExecuteBash
diff --git a/tests/integration-tests/README.md b/tests/integration-tests/README.md
@@ -38,21 +38,22 @@ that lists all the available options:
 python -m test_runner --help
 usage: test_runner.py [-h] --key-name KEY_NAME --key-path KEY_PATH [-n PARALLELISM] [--sequential] [--credential CREDENTIAL] [--use-default-iam-credentials] [--retry-on-failures] [--tests-root-dir TESTS_ROOT_DIR] [--global-build-number GLOBAL_BUILD_NUMBER] [-c TESTS_CONFIG] [-i [INSTANCES ...]]
                       [-o [OSS ...]] [-s [SCHEDULERS ...]] [-r [REGIONS ...]] [-f FEATURES [FEATURES ...]] [--show-output] [--reports {html,junitxml,json,cw} [{html,junitxml,json,cw} ...]] [--cw-region CW_REGION] [--cw-namespace CW_NAMESPACE] [--cw-timestamp-day-start] [--output-dir OUTPUT_DIR]
-                      [--custom-node-url CUSTOM_NODE_URL] [--custom-cookbook-url CUSTOM_COOKBOOK_URL] [--createami-custom-cookbook-url CREATEAMI_CUSTOM_COOKBOOK_URL] [--createami-custom-node-url CREATEAMI_CUSTOM_NODE_URL] [--custom-awsbatchcli-url CUSTOM_AWSBATCHCLI_URL] [--pre-install PRE_INSTALL]
-                      [--post-install POST_INSTALL] [--instance-types-data INSTANCE_TYPES_DATA] [--custom-ami CUSTOM_AMI] [--pcluster-git-ref PCLUSTER_GIT_REF] [--cookbook-git-ref COOKBOOK_GIT_REF] [--node-git-ref NODE_GIT_REF] [--ami-owner AMI_OWNER] [--available-amis-oss-x86 [AVAILABLE_AMIS_OSS_X86 ...]]
-                      [--available-amis-oss-arm [AVAILABLE_AMIS_OSS_ARM ...]] [--benchmarks] [--benchmarks-target-capacity BENCHMARKS_TARGET_CAPACITY] [--benchmarks-max-time BENCHMARKS_MAX_TIME] [--scaling-test-config SCALING_TEST_CONFIG]
-                      [--cluster-custom-resource-service-token CLUSTER_CUSTOM_RESOURCE_SERVICE_TOKEN] [--resource-bucket RESOURCE_BUCKET] [--lambda-layer-source LAMBDA_LAYER_SOURCE] [--api-definition-s3-uri API_DEFINITION_S3_URI] [--api-infrastructure-s3-uri API_INFRASTRUCTURE_S3_URI] [--api-uri API_URI]
-                      [--policies-uri POLICIES_URI] [--vpc-stack VPC_STACK] [--cluster CLUSTER] [--no-delete] [--delete-logs-on-success] [--stackname-suffix STACKNAME_SUFFIX] [--dry-run] [--iam-user-role-stack-name IAM_USER_ROLE_STACK_NAME] [--directory-stack-name DIRECTORY_STACK_NAME]
-                      [--slurm-database-stack-name SLURM_DATABASE_STACK_NAME] [--slurm-dbd-stack-name SLURM_DBD_STACK_NAME] [--munge-key-secret-arn MUNGE_KEY_SECRET_ARN] [--external-shared-storage-stack-name EXTERNAL_SHARED_STORAGE_STACK_NAME] [--bucket-name BUCKET_NAME]
-                      [--custom-security-groups-stack-name CUSTOM_SECURITY_GROUPS_STACK_NAME] [--force-run-instances] [--force-elastic-ip] [--retain-ad-stack] [--proxy-stack PROXY_STACK] [--build-image-roles-stack BUILD_IMAGE_ROLES_STACK] [--api-stack API_STACK]
+                      [--custom-node-url CUSTOM_NODE_URL] [--custom-cookbook-url CUSTOM_COOKBOOK_URL] [--createami-custom-cookbook-url CREATEAMI_CUSTOM_COOKBOOK_URL] [--createami-custom-node-url CREATEAMI_CUSTOM_NODE_URL] [--custom-awsbatchcli-url CUSTOM_AWSBATCHCLI_URL]
+                      [--pcluster-installer-path PCLUSTER_INSTALLER_PATH] [--pre-install PRE_INSTALL] [--post-install POST_INSTALL] [--instance-types-data INSTANCE_TYPES_DATA] [--custom-ami CUSTOM_AMI] [--pcluster-git-ref PCLUSTER_GIT_REF] [--cookbook-git-ref COOKBOOK_GIT_REF]
+                      [--node-git-ref NODE_GIT_REF] [--ami-owner AMI_OWNER] [--available-amis-oss-x86 [AVAILABLE_AMIS_OSS_X86 ...]] [--available-amis-oss-arm [AVAILABLE_AMIS_OSS_ARM ...]] [--benchmarks] [--benchmarks-target-capacity BENCHMARKS_TARGET_CAPACITY] [--benchmarks-max-time BENCHMARKS_MAX_TIME]
+                      [--scaling-test-config SCALING_TEST_CONFIG] [--cluster-custom-resource-service-token CLUSTER_CUSTOM_RESOURCE_SERVICE_TOKEN] [--resource-bucket RESOURCE_BUCKET] [--lambda-layer-source LAMBDA_LAYER_SOURCE] [--api-definition-s3-uri API_DEFINITION_S3_URI]
+                      [--api-infrastructure-s3-uri API_INFRASTRUCTURE_S3_URI] [--api-uri API_URI] [--policies-uri POLICIES_URI] [--vpc-stack VPC_STACK] [--cluster CLUSTER] [--no-delete] [--delete-logs-on-success] [--stackname-suffix STACKNAME_SUFFIX] [--dry-run]
+                      [--iam-user-role-stack-name IAM_USER_ROLE_STACK_NAME] [--directory-stack-name DIRECTORY_STACK_NAME] [--slurm-database-stack-name SLURM_DATABASE_STACK_NAME] [--slurm-dbd-stack-name SLURM_DBD_STACK_NAME] [--munge-key-secret-arn MUNGE_KEY_SECRET_ARN]
+                      [--external-shared-storage-stack-name EXTERNAL_SHARED_STORAGE_STACK_NAME] [--bucket-name BUCKET_NAME] [--custom-security-groups-stack-name CUSTOM_SECURITY_GROUPS_STACK_NAME] [--force-run-instances] [--force-elastic-ip] [--retain-ad-stack] [--proxy-stack PROXY_STACK]
+                      [--build-image-roles-stack BUILD_IMAGE_ROLES_STACK] [--api-stack API_STACK]
 
 Run integration tests suite.
 
 options:
   -h, --help            show this help message and exit
   --key-name KEY_NAME   Key to use for EC2 instances (default: None)
   --key-path KEY_PATH   Path to the key to use for SSH connections (default: None)
-  -n PARALLELISM, --parallelism PARALLELISM
+  -n, --parallelism PARALLELISM
                         Tests parallelism for every region. (default: None)
   --sequential          Run tests in a single process. When not specified tests will spawn a process for each region under test. (default: False)
   --credential CREDENTIAL
@@ -67,18 +68,16 @@ options:
                         The build number passed from the testing pipelines (default: 0)
 
 Test dimensions:
-  -c TESTS_CONFIG, --tests-config TESTS_CONFIG
-                        Config file that specifies the tests to run and the dimensions to enable for each test. Note that when a config file is used the following flags are ignored: instances, regions, oss, schedulers. Refer to the docs for further details on the config format: https://github.com/aws/aws-
-                        parallelcluster/blob/develop/tests/integration-tests/README.md (default: None)
-  -i [INSTANCES ...], --instances [INSTANCES ...]
+  -c, --tests-config TESTS_CONFIG
+                        Config file that specifies the tests to run and the dimensions to enable for each test. Note that when a config file is used the following flags are ignored: instances, regions, oss, schedulers. Refer to the docs for further details on the config format: https://github.com/aws/aws-parallelcluster/blob/develop/tests/integration-tests/README.md (default: None)
+  -i, --instances [INSTANCES ...]
                         AWS instances under test. Ignored when tests-config is used. (default: [])
-  -o [OSS ...], --oss [OSS ...]
-                        OSs under test. Ignored when tests-config is used. (default: [])
-  -s [SCHEDULERS ...], --schedulers [SCHEDULERS ...]
+  -o, --oss [OSS ...]   OSs under test. Ignored when tests-config is used. (default: [])
+  -s, --schedulers [SCHEDULERS ...]
                         Schedulers under test. Ignored when tests-config is used. (default: [])
-  -r [REGIONS ...], --regions [REGIONS ...]
+  -r, --regions [REGIONS ...]
                         AWS regions where tests are executed. Ignored when tests-config is used. (default: [])
-  -f FEATURES [FEATURES ...], --features FEATURES [FEATURES ...]
+  -f, --features FEATURES [FEATURES ...]
                         Run only tests for the listed features. Prepending the not keyword to the feature name causes the feature to be excluded. (default: )
 
 Test reports:
@@ -105,6 +104,8 @@ Custom packages and templates:
                         URL to a custom node package for the createami command. (default: None)
   --custom-awsbatchcli-url CUSTOM_AWSBATCHCLI_URL
                         URL to a custom awsbatch cli package. (default: None)
+  --pcluster-installer-path PCLUSTER_INSTALLER_PATH
+                        Path to ParallelCluster installer. (default: None)
   --pre-install PRE_INSTALL
                         URL to a pre install script (default: None)
   --post-install POST_INSTALL
@@ -312,6 +313,54 @@ the test will be executed in
 * `eu-west-1` using the AZ with ZoneId `euw1-az1` (ZoneId is consistent across accounts)
 * `eu-central-1` using a random AZ available in the region
 
+#### OS Rotation
+The framework includes automatic OS rotation to ensure that all supported operating systems are tested regularly:
+
+- The framework calculates a rotation index daily, and assign values to the jinja variables upon test running
+- Tests can be constrained to specific available AMIs using the `--available-amis-oss-x86` and `--available-amis-oss-arm` parameters
+- It can be customized to filter OS choices based on feature compatibility (e.g., DCV, Lustre, Batch scheduler support)
+
+Use Jinja variables in test configs with the pattern:
+```
+      dimensions:
+        - oss: [ {{ OS_X86_3 }} ]
+```
+
+#### Instance Type Rotation
+
+Similar to OS rotation, the framework also rotates instance types for dynamic testing:
+
+- Instance types are automatically discovered and rotated for major regions (us-east-1, us-west-2, eu-west-1)
+- Excludes legacy instance types (m1, m2, t1, etc.) for better test reliability
+
+Use Jinja variables in test configs with the pattern:
+```
+      dimensions:
+        - instances: [{{ US_EAST_1_INSTANCE_TYPE_0 }}.xlarge]
+```
+
+#### Capacity Reservation Management
+
+The framework includes automatic capacity reservation management for tests that require specific instance types with guaranteed capacity. This feature:
+
+- Automatically detects capacity reservation requirements in test configuration files using Jinja variables
+- Creates or modifies existing EC2 capacity reservations as needed
+- Supports placement groups and time-limited reservations
+- Falls back to default availability zones if reservations cannot be created
+
+Use Jinja variables in test configs with the pattern:
+```
+      dimensions:
+        - regions: [{{ INSTANCE_TYPE_CAPACITY_RESERVATION_COUNT_INSTANCES_HOURS_HOURS_[YESPG|NOPG]_[OS] }}]
+```
+
+Example:
+```
+      # Reserve 2 c5.xlarge instances for 2 hours with placement group
+      dimensions:
+        - regions: []{{ c5_xlarge_CAPACITY_RESERVATION_2_INSTANCES_2_HOURS_YESPG_alinux2023 }}]
+```
+
 #### Using CLI options
 
 The following options can be used to control the parametrization of test cases:
@@ -390,6 +439,13 @@ metrics. You can use the options `--cw-region` (default `us-east-1`) and `--cw-n
 (default `ParallelCluster/IntegrationTests`) to specify what region and what metric namespace
 you want to use for the published metrics.
 
+### Test Metadata Management
+
+The framework automatically manages test metadata using DynamoDB tables for tracking test execution and results across regions. This system:
+
+- Creates metadata tables automatically in the appropriate reporting region
+- Tracks test execution metadata for analysis and reporting
+
 ### Parallelize Tests Execution
 The following options can be used to control tests parallelism:
 * `--sequential`: by default the tests orchestrator executes a separate parallel process for each region under test.
diff --git a/tests/integration-tests/framework/fixture_utils.py b/tests/integration-tests/framework/fixture_utils.py
@@ -126,7 +126,7 @@ def release(self):
                 return
 
         timeout = time.time() + 4 * 60 * 60  # 4 hours from now
-        while data.counter > 1:
+        while min(data.counter, len(data.currently_using_processes)) > 1:
             logging.info(
                 "Waiting for all processes to release shared fixture %s, currently in use by %d processes (%s)",
                 self.name,
