Fix cluster deletion failure when placement groups are enabled (#6373)

* Revert "Remove wait for shutting-down instances"

This reverts commit 4e80c57.

* Prevents Lambda function from timing out if instance termination exceeds 15 minutes.

Author: hehe7318

CHANGELOG.md

@@ -5,7 +5,6 @@ CHANGELOG
 ------
 
 **ENHANCEMENTS**
-
 - Add support for custom actions on login nodes.
 - Allow DCV connection on login nodes.
 - Add support for ap-southeast-3 region.
@@ -15,8 +14,10 @@ CHANGELOG
 **BUG FIXES**
 - Fix validator `EfaPlacementGroupValidator` so that it does not suggest to configure a Placement Group when Capacity Blocks are used.
 - Fix sporadic cluster creation failures with managed FSx for Lustre.
+- Fix cluster deletion failure when placement group is enabled.
 - Fix issue with login nodes being marked unhealthy when restricting SSH access.
 - Fix `retrieve_supported_regions` so that it can get the correct S3 url.
+
 3.10.1
 ------
 

cli/src/pcluster/resources/custom_resources/custom_resources_code/cleanup_resources.py

@@ -100,8 +100,17 @@ def _delete_s3_artifacts(event):
 
 
 def _terminate_cluster_nodes(event):
+    """
+    Terminate all EC2 instances associated with the given cluster.
+
+    This function iterates over all EC2 instances associated with the specified cluster and attempts to terminate them.
+    It handles retries for instances that fail to terminate initially and ensures that the function does not exceed
+    the Lambda execution timeout.
+    """
     try:
-        logger.info("Compute fleet clean-up: STARTED")
+        start_time = time.time()
+        max_exeution_time = 14 * 60  # Maximum allowed time for Lambda function execution (14 minutes) to avoid timeout
+        logger.info("Compute fleet nodes terminate: STARTED")
         stack_name = event["ResourceProperties"]["StackName"]
         ec2 = boto3.client("ec2", config=boto3_config)
 
@@ -117,12 +126,44 @@ def _terminate_cluster_nodes(event):
                         logger.error("Failed when terminating instances with error %s", e)
                         completed_successfully = False
                         continue
-        logger.info("Compute fleet clean-up: COMPLETED. Instances are either in shutting-down or terminated state")
+            logger.info("Sleeping for 10 seconds to allow all instances to initiate shut-down")
+            time.sleep(10)
+
+        while _has_shuttingdown_instances(stack_name):
+            # This logic prevents Lambda function from timing out if instance termination exceeds 15 minutes
+            # TODO: This approach may cause potential cluster deletion failure when PlacementGroups are enabled
+            # and instance termination time exceeds 15 minutes simultaneously. Resolve the above potential failure.
+            if time.time() - start_time > max_exeution_time:
+                logger.warning(
+                    "Lambda execution time has exceeded 14 minutes, approaching timeout. "
+                    "Returning from Lambda after a 30-second delay; instances may still be in a shutting-down state. "
+                    "Note: Instances in shutting-down state are not recoverable and are not billed during this period."
+                )
+                time.sleep(30)
+                return
+            logger.info("Waiting for all nodes terminated...")
+            time.sleep(10)
+
+        # Sleep for 30 more seconds to give PlacementGroups the time to update
+        time.sleep(30)
+
+        logger.info("Compute fleet nodes terminate: COMPLETED")
     except Exception as e:
         logger.error("Failed when terminating instances with error %s", e)
         raise
 
 
+def _has_shuttingdown_instances(stack_name):
+    ec2 = boto3.client("ec2", config=boto3_config)
+    filters = [
+        {"Name": "tag:parallelcluster:cluster-name", "Values": [stack_name]},
+        {"Name": "instance-state-name", "Values": ["shutting-down"]},
+    ]
+
+    result = ec2.describe_instances(Filters=filters)
+    return len(result.get("Reservations", [])) > 0
+
+
 def _describe_instance_ids_iterator(stack_name, instance_state=("pending", "running", "stopping", "stopped")):
     ec2 = boto3.client("ec2", config=boto3_config)
     filters = [