fix (DBInstance): Handle throttling error for DBInstance resource

aws-rds-cfn-common/src/main/java/software/amazon/rds/common/error/ErrorStatus.java

@@ -20,7 +20,11 @@ static ErrorStatus ignore(final OperationStatus status) {
     }
 
     static ErrorStatus retry(final int callbackDelay) {
-        return new RetryErrorStatus(OperationStatus.IN_PROGRESS, callbackDelay);
+        return retry(callbackDelay, null);
+    }
+
+    static ErrorStatus retry(final int callbackDelay, final HandlerErrorCode handlerErrorCode) {
+        return new RetryErrorStatus(OperationStatus.IN_PROGRESS, callbackDelay, handlerErrorCode);
     }
 
     static ErrorStatus conditional(Function<Exception, ErrorStatus> condition) {

aws-rds-cfn-common/src/main/java/software/amazon/rds/common/error/RetryErrorStatus.java

@@ -1,6 +1,7 @@
 package software.amazon.rds.common.error;
 
 import lombok.Getter;
+import software.amazon.cloudformation.proxy.HandlerErrorCode;
 import software.amazon.cloudformation.proxy.OperationStatus;
 
 public class RetryErrorStatus implements ErrorStatus {
@@ -10,8 +11,12 @@ public class RetryErrorStatus implements ErrorStatus {
     @Getter
     private final int callbackDelay;
 
-    public RetryErrorStatus(final OperationStatus status, int callbackDelay) {
+    @Getter
+    HandlerErrorCode handlerErrorCode;
+
+    public RetryErrorStatus(final OperationStatus status, int callbackDelay, final HandlerErrorCode handlerErrorCode) {
         this.status = status;
         this.callbackDelay = callbackDelay;
+        this.handlerErrorCode = handlerErrorCode;
     }
 }

aws-rds-cfn-common/src/main/java/software/amazon/rds/common/handler/Commons.java

@@ -12,6 +12,7 @@
 import software.amazon.awssdk.core.exception.SdkServiceException;
 import software.amazon.awssdk.services.rds.model.KmsKeyNotAccessibleException;
 import software.amazon.cloudformation.proxy.HandlerErrorCode;
+import software.amazon.cloudformation.proxy.OperationStatus;
 import software.amazon.cloudformation.proxy.ProgressEvent;
 import software.amazon.cloudformation.resource.ResourceTypeSchema;
 import software.amazon.rds.common.error.ErrorRuleSet;
@@ -93,7 +94,17 @@ public static <M, C> ProgressEvent<M, C> handleException(
             }
         } else if (errorStatus instanceof RetryErrorStatus) {
             RetryErrorStatus retryErrorStatus = (RetryErrorStatus) errorStatus;
-            return ProgressEvent.defaultInProgressHandler(context, retryErrorStatus.getCallbackDelay(), model);
+            if (retryErrorStatus.getHandlerErrorCode() == null) {
+                return ProgressEvent.defaultInProgressHandler(context, retryErrorStatus.getCallbackDelay(), model);
+            } else {
+                return ProgressEvent.<M, C>builder()
+                    .callbackContext(context)
+                    .resourceModel(model)
+                    .errorCode(retryErrorStatus.getHandlerErrorCode())
+                    .callbackDelaySeconds(retryErrorStatus.getCallbackDelay())
+                    .status(OperationStatus.IN_PROGRESS)
+                    .build();
+            }
         } else if (errorStatus instanceof HandlerErrorStatus) {
             final HandlerErrorStatus handlerErrorStatus = (HandlerErrorStatus) errorStatus;
             // We need to set model and context to null in case of AlreadyExists errors

aws-rds-cfn-test-common/src/main/java/software/amazon/rds/test/common/core/AbstractTestBase.java

@@ -33,13 +33,17 @@ protected String newStackId() {
     }
 
     protected Consumer<ProgressEvent<ModelT, ContextT>> expectInProgress(int pause) {
+        return expectInProgress(pause, null);
+    }
+
+    protected Consumer<ProgressEvent<ModelT, ContextT>> expectInProgress(int pause, final HandlerErrorCode errorCode) {
         return (response) -> {
             Assertions.assertThat(response).isNotNull();
             Assertions.assertThat(response.getStatus()).isEqualTo(OperationStatus.IN_PROGRESS);
             Assertions.assertThat(response.getCallbackDelaySeconds()).isEqualTo(pause);
             Assertions.assertThat(response.getResourceModels()).isNull();
             Assertions.assertThat(response.getMessage()).isNull();
-            Assertions.assertThat(response.getErrorCode()).isNull();
+            Assertions.assertThat(response.getErrorCode()).isEqualTo(errorCode);
         };
     }
 
@@ -170,4 +174,39 @@ protected <RequestT extends AwsRequest, ResponseT extends AwsResponse> void test
         );
         expectation.verify();
     }
+
+    @ExcludeFromJacocoGeneratedReport
+    protected <RequestT extends AwsRequest, ResponseT extends AwsResponse> void test_handleRequest_throttle(
+        final MethodCallExpectation<RequestT, ResponseT> expectation,
+        final ContextT context,
+        final Supplier<ModelT> desiredStateSupplier,
+        final Object requestException,
+        final int callbackDelay
+    ) {
+        test_handleRequest_throttle(expectation, context, null, desiredStateSupplier, requestException, callbackDelay);
+    }
+
+    @ExcludeFromJacocoGeneratedReport
+    protected <RequestT extends AwsRequest, ResponseT extends AwsResponse> void test_handleRequest_throttle(
+        final MethodCallExpectation<RequestT, ResponseT> expectation,
+        final ContextT context,
+        final Supplier<ModelT> previousStateSupplier,
+        final Supplier<ModelT> desiredStateSupplier,
+        final Object requestException,
+        final int callbackDelay
+    ) {
+        final Exception exception = requestException instanceof Exception ? (Exception) requestException : newAwsServiceException(requestException);
+
+        expectation.setup()
+            .thenThrow(exception);
+
+        test_handleRequest_base(
+            context,
+            null,
+            previousStateSupplier,
+            desiredStateSupplier,
+            expectInProgress(callbackDelay, HandlerErrorCode.Throttling)
+        );
+        expectation.verify();
+    }
 }

aws-rds-dbinstance/src/main/java/software/amazon/rds/dbinstance/BaseHandlerStd.java

@@ -75,6 +75,8 @@ public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
 
     protected static final String DB_INSTANCE_STABILIZATION_TIME = "dbinstance-stabilization-time";
 
+    protected static final int CALLBACK_DELAY = 6;
+
     protected final HandlerConfig config;
 
     protected RequestLogger requestLogger;
@@ -115,6 +117,9 @@ public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
                     ErrorCode.InsufficientDBInstanceCapacity,
                     ErrorCode.SnapshotQuotaExceeded,
                     ErrorCode.StorageQuotaExceeded)
+            .withErrorCodes(ErrorStatus.retry(CALLBACK_DELAY, HandlerErrorCode.Throttling),
+                    ErrorCode.ThrottlingException,
+                    ErrorCode.Throttling)
             .withErrorCodes(ErrorStatus.failWith(HandlerErrorCode.InvalidRequest),
                     ErrorCode.DBSubnetGroupNotAllowedFault,
                     ErrorCode.InvalidParameterCombination,
@@ -167,8 +172,16 @@ public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
                     InvalidSubnetException.class)
             .build();
 
-    protected static final ErrorRuleSet DESCRIBE_AUTOMATED_BACKUPS_SOFTFAIL_ERROR_RULE_SET = ErrorRuleSet
+
+    protected static final ErrorRuleSet READ_HANDLER_ERROR_RULE_SET = ErrorRuleSet
         .extend(DEFAULT_DB_INSTANCE_ERROR_RULE_SET)
+        .withErrorCodes(ErrorStatus.failWith(HandlerErrorCode.Throttling),
+            ErrorCode.ThrottlingException,
+            ErrorCode.Throttling)
+        .build();
+
+    protected static final ErrorRuleSet DESCRIBE_AUTOMATED_BACKUPS_SOFTFAIL_ERROR_RULE_SET = ErrorRuleSet
+        .extend(READ_HANDLER_ERROR_RULE_SET)
         .withErrorClasses(ErrorStatus.ignore(OperationStatus.IN_PROGRESS),
             DbInstanceAutomatedBackupNotFoundException.class)
         .withErrorCodes(ErrorStatus.ignore(OperationStatus.IN_PROGRESS),

aws-rds-dbinstance/src/main/java/software/amazon/rds/dbinstance/ReadHandler.java

@@ -72,7 +72,7 @@ protected ProgressEvent<ResourceModel, CallbackContext> describeDbInstance(
             .handleError((describeRequest, exception, client, model, context) -> Commons.handleException(
                 ProgressEvent.progress(model, context),
                 exception,
-                DEFAULT_DB_INSTANCE_ERROR_RULE_SET,
+                READ_HANDLER_ERROR_RULE_SET,
                 requestLogger
             ))
             .done((describeRequest, describeResponse, automatedBackupProxyInvocation, model, context) -> {

aws-rds-dbinstance/src/test/java/software/amazon/rds/dbinstance/AbstractHandlerTest.java

@@ -9,8 +9,12 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.function.Supplier;
+import java.util.stream.Stream;
 
 import org.json.JSONObject;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.ArgumentsProvider;
 import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 import org.mockito.stubbing.OngoingStubbing;
@@ -42,6 +46,7 @@
 import software.amazon.cloudformation.proxy.ProxyClient;
 import software.amazon.cloudformation.proxy.ResourceHandlerRequest;
 import software.amazon.cloudformation.proxy.delay.Constant;
+import software.amazon.rds.common.error.ErrorCode;
 import software.amazon.rds.common.handler.Tagging;
 import software.amazon.rds.common.logging.RequestLogger;
 import software.amazon.rds.common.printer.FilteredJsonPrinter;
@@ -200,6 +205,7 @@ public abstract class AbstractHandlerTest extends AbstractTestBase<DBInstance, R
     protected static final Integer AUTOMATIC_BACKUP_REPLICATION_RETENTION_PERIOD = 1;
     protected static final String CURRENT_REGION = "eu-west-1";
 
+    protected static final int CALLBACK_DELAY = 6;
 
     protected static final ResourceModel RESOURCE_MODEL_NO_IDENTIFIER;
     protected static final ResourceModel RESOURCE_MODEL_ALTER;
@@ -889,4 +895,14 @@ public ArgumentCaptor<DeleteDbInstanceRequest> verify() {
     protected static String getAutomaticBackupArn(final String region) {
         return String.format("arn:aws:rds:%s:1234567890:auto-backup:ab-test", region);
     }
+
+    static class ThrottleExceptionArgumentsProvider implements ArgumentsProvider {
+        @Override
+        public Stream<? extends Arguments> provideArguments(ExtensionContext extensionContext) throws Exception {
+            return Stream.of(
+                Arguments.of(ErrorCode.ThrottlingException),
+                Arguments.of(ErrorCode.Throttling)
+            );
+        }
+    }
 }

aws-rds-dbinstance/src/test/java/software/amazon/rds/dbinstance/CreateHandlerTest.java

@@ -1627,7 +1627,6 @@ public Stream<? extends Arguments> provideArguments(ExtensionContext extensionCo
                     Arguments.of(ErrorCode.DBSubnetGroupNotAllowedFault, HandlerErrorCode.InvalidRequest),
                     Arguments.of(ErrorCode.InvalidParameterCombination, HandlerErrorCode.InvalidRequest),
                     Arguments.of(ErrorCode.StorageTypeNotSupportedFault, HandlerErrorCode.InvalidRequest),
-                    Arguments.of(ErrorCode.ThrottlingException, HandlerErrorCode.Throttling),
                     // Put exception classes below
                     Arguments.of(AuthorizationNotFoundException.builder().message(MSG_GENERIC_ERR).build(), HandlerErrorCode.InvalidRequest),
                     Arguments.of(CertificateNotFoundException.builder().message(MSG_GENERIC_ERR).build(), HandlerErrorCode.NotFound),
@@ -2355,4 +2354,53 @@ public void fetchEngineForUnknownScenario() {
                 expectFailed(HandlerErrorCode.InvalidRequest)
         );
     }
+
+    @ParameterizedTest
+    @ArgumentsSource(ThrottleExceptionArgumentsProvider.class)
+    public void handleRequest_CreateDBInstance_HandleThrottleException(
+        final Object requestException
+    ) {
+        test_handleRequest_throttle(
+            expectCreateDBInstanceCall(),
+            new CallbackContext(),
+            () -> RESOURCE_MODEL_BLDR().build(),
+            requestException,
+            CALLBACK_DELAY
+        );
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(ThrottleExceptionArgumentsProvider.class)
+    public void handleRequest_CreateDBInstanceReadReplica_HandleThrottleException(
+        final Object requestException
+    ) {
+        test_handleRequest_throttle(
+            expectCreateDBInstanceReadReplicaCall(),
+            new CallbackContext(),
+            () -> RESOURCE_MODEL_READ_REPLICA,
+            requestException,
+            CALLBACK_DELAY
+        );
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(ThrottleExceptionArgumentsProvider.class)
+    public void handleRequest_RestoreDBInstanceFromSnapshot_HandleThrottleException(
+        final Object requestException
+    ) {
+        when(rdsProxy.client().describeDBSnapshots(any(DescribeDbSnapshotsRequest.class)))
+            .thenReturn(DescribeDbSnapshotsResponse.builder()
+                .dbSnapshots(DBSnapshot.builder()
+                    .engine(ENGINE_MYSQL)
+                    .build())
+                .build());
+
+        test_handleRequest_throttle(
+            expectRestoreDBInstanceFromDBSnapshotCall(),
+            new CallbackContext(),
+            () -> RESOURCE_MODEL_RESTORING_FROM_SNAPSHOT,
+            requestException,
+            CALLBACK_DELAY
+        );
+    }
 }

aws-rds-dbinstance/src/test/java/software/amazon/rds/dbinstance/DeleteHandlerTest.java

@@ -445,4 +445,36 @@ public void handleRequest_DeleteDBInstance_HandleExceptionInDescribe(
                 expectResponseCode
         );
     }
+
+    @ParameterizedTest
+    @ArgumentsSource(ThrottleExceptionArgumentsProvider.class)
+    public void handleRequest_DeleteDBInstance_HandleThrottleExceptionInDelete(
+        final Object requestException
+    ) {
+        when(rdsProxy.client().describeDBInstances(any(DescribeDbInstancesRequest.class)))
+            .thenReturn(DescribeDbInstancesResponse.builder().dbInstances(DB_INSTANCE_ACTIVE).build());
+
+        test_handleRequest_throttle(
+            expectDeleteDBInstanceCall(),
+            new CallbackContext(),
+            () -> RESOURCE_MODEL_BLDR().build(),
+            requestException,
+            CALLBACK_DELAY
+        );
+    }
+
+    @ParameterizedTest
+    @ArgumentsSource(ThrottleExceptionArgumentsProvider.class)
+    public void handleRequest_DeleteDBInstance_HandleThrottleExceptionInDescribe(
+        final Object requestException
+    ) {
+        expectServiceInvocation = false;
+        test_handleRequest_throttle(
+            expectDescribeDBInstancesCall(),
+            new CallbackContext(),
+            () -> RESOURCE_MODEL_BLDR().build(),
+            requestException,
+            CALLBACK_DELAY
+        );
+    }
 }

aws-rds-dbinstance/src/test/java/software/amazon/rds/dbinstance/ReadHandlerTest.java

@@ -18,6 +18,8 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ArgumentsSource;
 import org.mockito.ArgumentMatchers;
 import org.mockito.Mock;
 import org.mockito.Mockito;
@@ -36,6 +38,7 @@
 import software.amazon.awssdk.services.rds.model.DescribeDbInstanceAutomatedBackupsResponse;
 import software.amazon.awssdk.services.rds.model.DescribeDbInstancesRequest;
 import software.amazon.cloudformation.proxy.AmazonWebServicesClientProxy;
+import software.amazon.cloudformation.proxy.HandlerErrorCode;
 import software.amazon.cloudformation.proxy.ProxyClient;
 import software.amazon.rds.test.common.core.HandlerName;
 
@@ -193,4 +196,18 @@ public void handleRequest_AutomaticBackupReplicationAccessDenied() {
         verify(crossRegionRdsProxy.client(), times(1)).describeDBInstanceAutomatedBackups(any(DescribeDbInstanceAutomatedBackupsRequest.class));
         Assertions.assertThat(context.isAutomaticBackupReplicationStarted()).isFalse();
     }
+
+    @ParameterizedTest
+    @ArgumentsSource(ThrottleExceptionArgumentsProvider.class)
+    public void handleRequest_DescribeDBInstance_HandleThrottleException(
+        final Object requestException
+    ) {
+        test_handleRequest_error(
+            expectDescribeDBInstancesCall(),
+            new CallbackContext(),
+            () -> RESOURCE_MODEL_BLDR().build(),
+            requestException,
+            HandlerErrorCode.Throttling
+        );
+    }
 }

aws-rds-dbinstance/src/test/java/software/amazon/rds/dbinstance/UpdateHandlerTest.java

@@ -1979,4 +1979,26 @@ public void handleRequest_EngineLifecycleSupportShouldFail() {
                 expectFailed(HandlerErrorCode.InvalidRequest)
         );
     }
+
+    @ParameterizedTest
+    @ArgumentsSource(ThrottleExceptionArgumentsProvider.class)
+    public void handleRequest_ModifyDBInstance_HandleThrottleException(
+        final Object requestException
+    ) {
+        final DBInstance fakeDBInstance = DBInstance.builder().build();
+        when(rdsProxy.client().describeDBInstances(any(DescribeDbInstancesRequest.class)))
+            .thenReturn(DescribeDbInstancesResponse.builder().dbInstances(fakeDBInstance).build());
+
+        final CallbackContext context = new CallbackContext();
+        context.setStorageAllocated(true);
+
+        test_handleRequest_throttle(
+            expectModifyDBInstanceCall(),
+            context,
+            () -> RESOURCE_MODEL_BLDR().build(),
+            () -> RESOURCE_MODEL_ALTER,
+            requestException,
+            CALLBACK_DELAY
+        );
+    }
 }