[DBInstance] Introduce ApplyImmediately flag to address unintended reboots and interruptions

aws-rds-dbinstance/aws-rds-dbinstance.json

@@ -473,6 +473,10 @@
         "type": "string"
       },
       "description": "A list of the VPC security group IDs to assign to the DB instance. The list can include both the physical IDs of existing VPC security groups and references to AWS::EC2::SecurityGroup resources created in the template."
+    },
+    "ApplyImmediately": {
+      "type": "boolean",
+      "description": "Specifies whether the modifications in this request and any pending modifications are asynchronously applied as soon as possible, regardless of the PreferredMaintenanceWindow setting for the DB instance. By default, this parameter is enabled."
     }
   },
   "additionalProperties": false,
@@ -549,7 +553,8 @@
     "/properties/SourceRegion",
     "/properties/TdeCredentialPassword",
     "/properties/UseDefaultProcessorFeatures",
-    "/properties/UseLatestRestorableTime"
+    "/properties/UseLatestRestorableTime",
+    "/properties/ApplyImmediately"
   ],
   "readOnlyProperties": [
     "/properties/Endpoint",

aws-rds-dbinstance/src/main/java/software/amazon/rds/dbinstance/BaseHandlerStd.java

@@ -25,6 +25,7 @@
 import software.amazon.rds.common.request.ValidatedRequest;
 import software.amazon.rds.common.request.Validations;
 import software.amazon.rds.dbinstance.client.*;
+import software.amazon.rds.dbinstance.util.ResourceModelHelper;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -596,7 +597,10 @@ protected boolean isDBParameterGroupStabilized(
     ) {
         final DBInstance dbInstance = fetchDBInstance(rdsProxyClient, model);
 
-        return DBInstancePredicates.isDBParameterGroupInSync(dbInstance);
+        if(ResourceModelHelper.shouldApplyImmediately(model)) {
+            return DBInstancePredicates.isDBParameterGroupInSync(dbInstance);
+        }
+        return DBInstancePredicates.isDBParameterGroupNotApplying(dbInstance);
     }
 
     protected boolean isDBClusterParameterGroupStabilized(
@@ -605,7 +609,10 @@ protected boolean isDBClusterParameterGroupStabilized(
     ) {
         final DBCluster dbCluster = fetchDBCluster(rdsProxyClient, model);
 
-        return DBInstancePredicates.isDBClusterParameterGroupInSync(model, dbCluster);
+        if(ResourceModelHelper.shouldApplyImmediately(model)) {
+            return DBInstancePredicates.isDBClusterParameterGroupInSync(model, dbCluster);
+        }
+        return DBInstancePredicates.isDBClusterParameterGroupNotApplying(model, dbCluster);
     }
 
     protected boolean isDBInstanceRoleStabilized(

aws-rds-dbinstance/src/main/java/software/amazon/rds/dbinstance/DBInstancePredicates.java

@@ -21,6 +21,7 @@
 import software.amazon.rds.dbinstance.status.OptionGroupStatus;
 import software.amazon.rds.dbinstance.status.ReadReplicaStatus;
 import software.amazon.rds.dbinstance.status.VPCSecurityGroupStatus;
+import software.amazon.rds.dbinstance.util.ResourceModelHelper;
 
 import java.util.Collections;
 import java.util.List;
@@ -158,6 +159,13 @@ public static boolean isDBClusterParameterGroupInSync(final ResourceModel model,
                 .anyMatch(member -> DBParameterGroupStatus.InSync.equalsString(member.dbClusterParameterGroupStatus()));
     }
 
+    public static boolean isDBClusterParameterGroupNotApplying(final ResourceModel model, final DBCluster dbCluster) {
+        return Optional.ofNullable(dbCluster.dbClusterMembers()).orElse(Collections.emptyList())
+            .stream()
+            .filter(member -> model.getDBInstanceIdentifier().equalsIgnoreCase(member.dbInstanceIdentifier()))
+            .noneMatch(member -> DBParameterGroupStatus.Applying.equalsString(member.dbClusterParameterGroupStatus()));
+    }
+
     public static boolean isDBClusterMember(final ResourceModel model) {
         return StringUtils.isNotBlank(model.getDBClusterIdentifier());
     }
@@ -193,6 +201,23 @@ public static boolean isDBInstanceStabilizedAfterMutate(
     ) {
         assertNoTerminalStatus(dbInstance);
 
+        if(ResourceModelHelper.shouldApplyImmediately(model)){
+            return isStabilizedWithChangesAppliedImmediately(dbInstance, model, requestLogger);
+        }
+
+        return isStabilizedWithoutChangesAppliedImmediately(dbInstance, requestLogger);
+    }
+
+    /***
+     * Stabilization logic that ensures all the changes are applied.
+     */
+    private static boolean isStabilizedWithChangesAppliedImmediately(
+        final DBInstance dbInstance,
+        final ResourceModel model,
+        final RequestLogger requestLogger
+    ) {
+        assertNoTerminalStatus(dbInstance);
+
         final boolean isDBInstanceStabilizedAfterMutateResult = isDBInstanceAvailable(dbInstance) &&
                 isReplicationComplete(dbInstance) &&
                 isDBParameterGroupNotApplying(dbInstance) &&
@@ -202,18 +227,48 @@ public static boolean isDBInstanceStabilizedAfterMutate(
                 isDomainMembershipsJoined(dbInstance) &&
                 isMasterUserSecretStabilized(dbInstance);
 
-        requestLogger.log(String.format("isDBInstanceStabilizedAfterMutate: %b", isDBInstanceStabilizedAfterMutateResult),
-                ImmutableMap.of("isDBInstanceAvailable", isDBInstanceAvailable(dbInstance),
-                        "isReplicationComplete", isReplicationComplete(dbInstance),
-                        "isDBParameterGroupNotApplying", isDBParameterGroupNotApplying(dbInstance),
-                        "isNoPendingChanges", isNoPendingChanges(dbInstance),
-                        "isCaCertificateChangesApplied", isCaCertificateChangesApplied(dbInstance, model),
-                        "isVpcSecurityGroupsActive", isVpcSecurityGroupsActive(dbInstance),
-                        "isDomainMembershipsJoined", isDomainMembershipsJoined(dbInstance),
-                        "isMasterUserSecretStabilized", isMasterUserSecretStabilized(dbInstance)),
-                ImmutableMap.of("Description", "isDBInstanceStabilizedAfterMutate method will be repeatedly" +
-                        " called with a backoff mechanism after the modify call until it returns true. This" +
-                        " process will continue until all included flags are true."));
+        requestLogger.log(String.format("isStabilizedWithChangesAppliedImmediately: %b", isDBInstanceStabilizedAfterMutateResult),
+            ImmutableMap.of("isDBInstanceAvailable", isDBInstanceAvailable(dbInstance),
+                "isReplicationComplete", isReplicationComplete(dbInstance),
+                "isDBParameterGroupNotApplying", isDBParameterGroupNotApplying(dbInstance),
+                "isNoPendingChanges", isNoPendingChanges(dbInstance),
+                "isCaCertificateChangesApplied", isCaCertificateChangesApplied(dbInstance, model),
+                "isVpcSecurityGroupsActive", isVpcSecurityGroupsActive(dbInstance),
+                "isDomainMembershipsJoined", isDomainMembershipsJoined(dbInstance),
+                "isMasterUserSecretStabilized", isMasterUserSecretStabilized(dbInstance)),
+                ImmutableMap.of("Description", "isStabilizedWithChangesAppliedImmediately method will be repeatedly" +
+                " called with a backoff mechanism after the modify call until it returns true. This" +
+                " process will continue until all included flags are true."));
+
+        return isDBInstanceStabilizedAfterMutateResult;
+    }
+
+    /***
+     * Stabilization logic that excludes the settings that are not applied immediately. This happens when
+     * ApplyImmediately is set to false. The excluded settings is based on the following doc
+     * https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/USER_ModifyInstance.Settings.html
+     */
+    private static boolean isStabilizedWithoutChangesAppliedImmediately(
+        final DBInstance dbInstance,
+        final RequestLogger requestLogger
+    ) {
+        assertNoTerminalStatus(dbInstance);
+
+        final boolean isDBInstanceStabilizedAfterMutateResult = isDBInstanceAvailable(dbInstance) &&
+            isReplicationComplete(dbInstance) &&
+            isDBParameterGroupNotApplying(dbInstance) &&
+            isVpcSecurityGroupsActive(dbInstance) &&
+            isMasterUserSecretStabilized(dbInstance);
+
+        requestLogger.log(String.format("isStabilizedWithoutChangesAppliedImmediately: %b", isDBInstanceStabilizedAfterMutateResult),
+            ImmutableMap.of("isDBInstanceAvailable", isDBInstanceAvailable(dbInstance),
+                "isReplicationComplete", isReplicationComplete(dbInstance),
+                "isDBParameterGroupNotApplying", isDBParameterGroupNotApplying(dbInstance),
+                "isVpcSecurityGroupsActive", isVpcSecurityGroupsActive(dbInstance),
+                "isMasterUserSecretStabilized", isMasterUserSecretStabilized(dbInstance)),
+                ImmutableMap.of("Description", "isStabilizedWithoutChangesAppliedImmediately method will be repeatedly" +
+                " called with a backoff mechanism after the modify call until it returns true. This" +
+                " process will continue until all included flags are true."));
 
         return isDBInstanceStabilizedAfterMutateResult;
     }

aws-rds-dbinstance/src/main/java/software/amazon/rds/dbinstance/Translator.java

@@ -414,7 +414,7 @@ public static ModifyDbInstanceRequest modifyDbInstanceRequestV12(
     ) {
         final ModifyDbInstanceRequest.Builder builder = ModifyDbInstanceRequest.builder()
                 .allowMajorVersionUpgrade(desiredModel.getAllowMajorVersionUpgrade())
-                .applyImmediately(Boolean.TRUE)
+                .applyImmediately(ResourceModelHelper.shouldApplyImmediately(desiredModel))
                 .autoMinorVersionUpgrade(diff(previousModel.getAutoMinorVersionUpgrade(), desiredModel.getAutoMinorVersionUpgrade()))
                 .backupRetentionPeriod(diff(previousModel.getBackupRetentionPeriod(), desiredModel.getBackupRetentionPeriod()))
                 .dbInstanceClass(diff(previousModel.getDBInstanceClass(), desiredModel.getDBInstanceClass()))
@@ -451,7 +451,7 @@ public static ModifyDbInstanceRequest modifyDbInstanceRequest(
     ) {
         final ModifyDbInstanceRequest.Builder builder = ModifyDbInstanceRequest.builder()
                 .allowMajorVersionUpgrade(desiredModel.getAllowMajorVersionUpgrade())
-                .applyImmediately(Boolean.TRUE)
+                .applyImmediately(ResourceModelHelper.shouldApplyImmediately(desiredModel))
                 .autoMinorVersionUpgrade(diff(previousModel.getAutoMinorVersionUpgrade(), desiredModel.getAutoMinorVersionUpgrade()))
                 .backupRetentionPeriod(diff(previousModel.getBackupRetentionPeriod(), desiredModel.getBackupRetentionPeriod()))
                 .copyTagsToSnapshot(diff(previousModel.getCopyTagsToSnapshot(), desiredModel.getCopyTagsToSnapshot()))
@@ -674,7 +674,7 @@ public static ModifyDbInstanceRequest updateAllocatedStorageRequest(final Resour
         return ModifyDbInstanceRequest.builder()
                 .dbInstanceIdentifier(desiredModel.getDBInstanceIdentifier())
                 .allocatedStorage(getAllocatedStorage(desiredModel))
-                .applyImmediately(Boolean.TRUE)
+                .applyImmediately(ResourceModelHelper.shouldApplyImmediately(desiredModel))
                 .build();
     }
 

aws-rds-dbinstance/src/main/java/software/amazon/rds/dbinstance/UpdateHandler.java

@@ -252,8 +252,9 @@ private boolean shouldReboot(
     ) {
         try {
             final DBInstance dbInstance = fetchDBInstance(proxyClient, progress.getResourceModel());
+            final boolean applyImmediately = ResourceModelHelper.shouldApplyImmediately(progress.getResourceModel());
             if (!CollectionUtils.isNullOrEmpty(dbInstance.dbParameterGroups())) {
-                return DBParameterGroupStatus.PendingReboot.equalsString(dbInstance.dbParameterGroups().get(0).parameterApplyStatus());
+                return applyImmediately && DBParameterGroupStatus.PendingReboot.equalsString(dbInstance.dbParameterGroups().get(0).parameterApplyStatus());
             }
         } catch (DbInstanceNotFoundException e) {
             return false;
@@ -267,10 +268,11 @@ private boolean shouldRebootCluster(
     ) {
         final String dbInstanceIdentifier = progress.getResourceModel().getDBInstanceIdentifier();
         final DBCluster dbCluster = fetchDBCluster(proxyClient, progress.getResourceModel());
+        final boolean applyImmediately =  ResourceModelHelper.shouldApplyImmediately(progress.getResourceModel());
         if (!CollectionUtils.isNullOrEmpty(dbCluster.dbClusterMembers())) {
             for (final DBClusterMember member : dbCluster.dbClusterMembers()) {
                 if (dbInstanceIdentifier.equalsIgnoreCase(member.dbInstanceIdentifier())) {
-                    return DBParameterGroupStatus.PendingReboot.equalsString(member.dbClusterParameterGroupStatus());
+                    return applyImmediately && DBParameterGroupStatus.PendingReboot.equalsString(member.dbClusterParameterGroupStatus());
                 }
             }
         }

aws-rds-dbinstance/src/main/java/software/amazon/rds/dbinstance/util/ResourceModelHelper.java

@@ -127,6 +127,12 @@ public static boolean shouldReboot(final ResourceModel model) {
         return StringUtils.hasValue(model.getDBParameterGroupName());
     }
 
+    public static boolean shouldApplyImmediately(final ResourceModel model) {
+        Boolean applyImmediately = model.getApplyImmediately();
+        // default to true
+        return applyImmediately == null || applyImmediately;
+    }
+
     public static Boolean getDefaultMultiAzForEngine(final String engine) {
         if (SQLSERVER_ENGINES_WITH_MIRRORING.contains(engine)) {
             return null;