Skip to content

Commit dd20afb

Browse files
brianlaoawsbrianlaoaws
committed
Add support to download hook target data for stack-level hooks
Stack level hooks will not be provided with invocation payload information, unlike resource level hooks. Instead, Hooks Service will pass in an S3 presigned URL that points to a file that contains the stack-level invocation payload. The base hook handler (before it reaches the customer's handler code), will use that URL to download the data and set it on the target model that is passed to the customer's handler.
1 parent 17ab056 commit dd20afb

File tree

6 files changed

+147
-38
lines changed

6 files changed

+147
-38
lines changed

‎src/main/java/software/amazon/cloudformation/HookAbstractWrapper.java

Lines changed: 66 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,18 @@
1717
import com.amazonaws.AmazonServiceException;
1818
import com.amazonaws.retry.RetryUtils;
1919
import com.fasterxml.jackson.core.type.TypeReference;
20+
import java.io.ByteArrayOutputStream;
2021
import java.io.IOException;
2122
import java.io.InputStream;
2223
import java.io.OutputStream;
24+
import java.net.URL;
2325
import java.nio.charset.StandardCharsets;
2426
import java.time.Instant;
27+
import java.util.Collections;
2528
import java.util.Date;
29+
import java.util.Map;
30+
31+
import com.google.common.annotations.VisibleForTesting;
2632
import org.apache.commons.io.FileUtils;
2733
import org.apache.commons.io.IOUtils;
2834
import org.apache.commons.lang3.exception.ExceptionUtils;
@@ -31,10 +37,15 @@
3137
import org.slf4j.Logger;
3238
import org.slf4j.LoggerFactory;
3339
import software.amazon.awssdk.awscore.exception.AwsServiceException;
40+
import software.amazon.awssdk.http.HttpExecuteRequest;
41+
import software.amazon.awssdk.http.HttpExecuteResponse;
3442
import software.amazon.awssdk.http.HttpStatusCode;
3543
import software.amazon.awssdk.http.HttpStatusFamily;
3644
import software.amazon.awssdk.http.SdkHttpClient;
45+
import software.amazon.awssdk.http.SdkHttpMethod;
46+
import software.amazon.awssdk.http.SdkHttpRequest;
3747
import software.amazon.awssdk.http.apache.ApacheHttpClient;
48+
import software.amazon.awssdk.utils.IoUtils;
3849
import software.amazon.cloudformation.encryption.Cipher;
3950
import software.amazon.cloudformation.encryption.KMSCipher;
4051
import software.amazon.cloudformation.exceptions.BaseHandlerException;
@@ -63,6 +74,7 @@
6374
import software.amazon.cloudformation.proxy.hook.HookInvocationRequest;
6475
import software.amazon.cloudformation.proxy.hook.HookProgressEvent;
6576
import software.amazon.cloudformation.proxy.hook.HookRequestContext;
77+
import software.amazon.cloudformation.proxy.hook.HookRequestData;
6678
import software.amazon.cloudformation.proxy.hook.HookStatus;
6779
import software.amazon.cloudformation.resource.SchemaValidator;
6880
import software.amazon.cloudformation.resource.Serializer;
@@ -89,6 +101,9 @@ public abstract class HookAbstractWrapper<TargetT, CallbackT, ConfigurationT> {
89101
final SchemaValidator validator;
90102
final TypeReference<HookInvocationRequest<ConfigurationT, CallbackT>> typeReference;
91103

104+
final TypeReference<Map<String, Object>> hookStackPayloadS3TypeReference = new TypeReference<>() {
105+
};
106+
92107
private MetricsPublisher providerMetricsPublisher;
93108

94109
private CloudWatchLogHelper cloudWatchLogHelper;
@@ -222,18 +237,20 @@ private ProgressEvent<TargetT, CallbackT> processInvocation(final JSONObject raw
222237

223238
assert request != null : "Invalid request object received. Request object is null";
224239

225-
if (request.getRequestData() == null || request.getRequestData().getTargetModel() == null) {
226-
throw new TerminalException("Invalid request object received. Target Model can not be null.");
227-
}
228-
229-
// TODO: Include hook schema validation here after schema is finalized
240+
boolean isPayloadRemote = isHookInvocationPayloadRemote(request.getRequestData());
230241

231242
try {
232243
// initialise dependencies with platform credentials
233244
initialiseRuntime(request.getHookTypeName(), request.getRequestData().getProviderCredentials(),
234245
request.getRequestData().getProviderLogGroupName(), request.getAwsAccountId(),
235246
request.getRequestData().getHookEncryptionKeyArn(), request.getRequestData().getHookEncryptionKeyRole());
236247

248+
if (isPayloadRemote) {
249+
Map<String, Object> targetModelData = retrieveHookInvocationPayloadFromS3(request.getRequestData().getPayload());
250+
251+
request.getRequestData().setTargetModel(targetModelData);
252+
}
253+
237254
// transform the request object to pass to caller
238255
HookHandlerRequest hookHandlerRequest = transform(request);
239256
ConfigurationT typeConfiguration = request.getHookModel();
@@ -366,6 +383,50 @@ private void writeResponse(final OutputStream outputStream, final HookProgressEv
366383
outputStream.flush();
367384
}
368385

386+
public Map<String, Object> retrieveHookInvocationPayloadFromS3(final String s3PresignedUrl) {
387+
if (s3PresignedUrl != null) {
388+
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
389+
390+
try {
391+
URL presignedUrl = new URL(s3PresignedUrl);
392+
SdkHttpRequest httpRequest = SdkHttpRequest.builder().method(SdkHttpMethod.GET).uri(presignedUrl.toURI())
393+
.build();
394+
395+
HttpExecuteRequest executeRequest = HttpExecuteRequest.builder().request(httpRequest).build();
396+
397+
HttpExecuteResponse response = HTTP_CLIENT.prepareRequest(executeRequest).call();
398+
399+
response.responseBody().ifPresentOrElse(abortableInputStream -> {
400+
try {
401+
IoUtils.copy(abortableInputStream, byteArrayOutputStream);
402+
} catch (IOException e) {
403+
throw new RuntimeException(e);
404+
}
405+
}, () -> loggerProxy.log("Hook invocation payload is empty."));
406+
407+
String str = byteArrayOutputStream.toString(StandardCharsets.UTF_8);
408+
409+
return this.serializer.deserialize(str, hookStackPayloadS3TypeReference);
410+
} catch (Exception exp) {
411+
loggerProxy.log("Failed to retrieve hook invocation payload" + exp.toString());
412+
}
413+
}
414+
return Collections.emptyMap();
415+
}
416+
417+
@VisibleForTesting
418+
protected boolean isHookInvocationPayloadRemote(HookRequestData hookRequestData) {
419+
if (hookRequestData == null || hookRequestData.getTargetModel() == null) {
420+
throw new TerminalException("Invalid request object received. Target Model can not be null.");
421+
}
422+
423+
if (hookRequestData.getTargetModel().isEmpty() && hookRequestData.getPayload() == null) {
424+
throw new TerminalException("No payload data set.");
425+
}
426+
427+
return hookRequestData.getTargetModel().isEmpty();
428+
}
429+
369430
/**
370431
* Transforms the incoming request to the subset of typed models which the
371432
* handler implementor needs

‎src/main/java/software/amazon/cloudformation/proxy/hook/HookRequestData.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ public class HookRequestData {
2929
private String targetType;
3030
private String targetLogicalId;
3131
private Map<String, Object> targetModel;
32+
private String payload;
3233
private String callerCredentials;
3334
private String providerCredentials;
3435
private String providerLogGroupName;

‎src/test/java/software/amazon/cloudformation/HookLambdaWrapperOverride.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import java.nio.charset.StandardCharsets;
2020
import java.util.LinkedList;
2121
import java.util.List;
22+
import java.util.Map;
2223
import java.util.Queue;
2324
import lombok.Data;
2425
import lombok.EqualsAndHashCode;
@@ -46,6 +47,8 @@
4647
@EqualsAndHashCode(callSuper = true)
4748
public class HookLambdaWrapperOverride extends HookLambdaWrapper<TestModel, TestContext, TestConfigurationModel> {
4849

50+
private Map<String, Object> hookInvocationPayloadFromS3;
51+
4952
/**
5053
* This .ctor provided for testing
5154
*/
@@ -128,6 +131,15 @@ protected HookHandlerRequest transform(final HookInvocationRequest<TestConfigura
128131

129132
public HookHandlerRequest transformResponse;
130133

134+
@Override
135+
public Map<String, Object> retrieveHookInvocationPayloadFromS3(final String s3PresignedUrl) {
136+
return hookInvocationPayloadFromS3;
137+
}
138+
139+
public void setHookInvocationPayloadFromS3(Map<String, Object> input) {
140+
hookInvocationPayloadFromS3 = input;
141+
}
142+
131143
@Override
132144
protected TypeReference<HookInvocationRequest<TestConfigurationModel, TestContext>> getTypeReference() {
133145
return new TypeReference<HookInvocationRequest<TestConfigurationModel, TestContext>>() {

‎src/test/java/software/amazon/cloudformation/HookLambdaWrapperTest.java

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,23 @@
3030
import java.io.IOException;
3131
import java.io.InputStream;
3232
import java.io.OutputStream;
33+
import java.util.Collections;
34+
import java.util.List;
35+
import java.util.Map;
36+
37+
import com.google.common.collect.ImmutableList;
38+
import com.google.common.collect.ImmutableMap;
39+
import org.junit.jupiter.api.Assertions;
3340
import org.junit.jupiter.api.BeforeEach;
41+
import org.junit.jupiter.api.Test;
3442
import org.junit.jupiter.api.extension.ExtendWith;
3543
import org.junit.jupiter.params.ParameterizedTest;
3644
import org.junit.jupiter.params.provider.CsvSource;
3745
import org.mockito.Mock;
3846
import org.mockito.junit.jupiter.MockitoExtension;
3947
import software.amazon.awssdk.http.SdkHttpClient;
4048
import software.amazon.cloudformation.encryption.KMSCipher;
49+
import software.amazon.cloudformation.exceptions.TerminalException;
4150
import software.amazon.cloudformation.injection.CredentialsProvider;
4251
import software.amazon.cloudformation.loggers.CloudWatchLogPublisher;
4352
import software.amazon.cloudformation.loggers.LogPublisher;
@@ -48,6 +57,7 @@
4857
import software.amazon.cloudformation.proxy.ProgressEvent;
4958
import software.amazon.cloudformation.proxy.hook.HookHandlerRequest;
5059
import software.amazon.cloudformation.proxy.hook.HookProgressEvent;
60+
import software.amazon.cloudformation.proxy.hook.HookRequestData;
5161
import software.amazon.cloudformation.proxy.hook.HookStatus;
5262
import software.amazon.cloudformation.proxy.hook.targetmodel.ChangedResource;
5363
import software.amazon.cloudformation.proxy.hook.targetmodel.StackHookTargetModel;
@@ -333,6 +343,22 @@ public void invokeHandler_WithStackLevelHook_returnsSuccess(final String request
333343

334344
lenient().when(cipher.decryptCredentials(any())).thenReturn(new Credentials("123", "123", "123"));
335345

346+
wrapper.setHookInvocationPayloadFromS3(Map.of(
347+
"Template", "template string here",
348+
"PreviousTemplate", "previous template string here",
349+
"ResolvedTemplate", "resolved template string here",
350+
"ChangedResources", List.of(
351+
Map.of(
352+
"LogicalResourceId", "SomeLogicalResourceId",
353+
"ResourceType", "AWS::S3::Bucket",
354+
"Action", "CREATE",
355+
"LineNumber", 3,
356+
"ResourceProperties", "<Resource Properties as json string>",
357+
"PreviousResourceProperties", "<Resource Properties as json string>"
358+
)
359+
)
360+
));
361+
336362
try (final InputStream in = loadRequestStream(requestDataPath); final OutputStream out = new ByteArrayOutputStream()) {
337363
final Context context = getLambdaContext();
338364

@@ -369,6 +395,30 @@ public void invokeHandler_WithStackLevelHook_returnsSuccess(final String request
369395
}
370396
}
371397

398+
@Test
399+
public void testIsHookInvocationPayloadRemote() {
400+
List<HookRequestData> invalidHookRequestDataObjects = ImmutableList.of(
401+
HookRequestData.builder().targetModel(null).build(),
402+
HookRequestData.builder().targetModel(null).payload(null).build(),
403+
HookRequestData.builder().targetModel(Collections.emptyMap()).payload(null).build(),
404+
HookRequestData.builder().targetModel(Collections.emptyMap()).payload(null).build()
405+
);
406+
407+
invalidHookRequestDataObjects.forEach(requestData -> {
408+
Assertions.assertThrows(TerminalException.class, () -> wrapper.isHookInvocationPayloadRemote(requestData));
409+
});
410+
411+
Assertions.assertThrows(TerminalException.class, () -> wrapper.isHookInvocationPayloadRemote(null));
412+
413+
HookRequestData bothFieldsPopulated = HookRequestData.builder().targetModel(ImmutableMap.of("foo", "bar")).payload("http://s3PresignedUrl").build();
414+
HookRequestData onlyTargetModelPopulated = HookRequestData.builder().targetModel(ImmutableMap.of("foo", "bar")).payload(null).build();
415+
HookRequestData onlyPayloadPopulated = HookRequestData.builder().targetModel(Collections.emptyMap()).payload("http://s3PresignedUrl").build();
416+
417+
Assertions.assertFalse(wrapper.isHookInvocationPayloadRemote(bothFieldsPopulated));
418+
Assertions.assertFalse(wrapper.isHookInvocationPayloadRemote(onlyTargetModelPopulated));
419+
Assertions.assertTrue(wrapper.isHookInvocationPayloadRemote(onlyPayloadPopulated));
420+
}
421+
372422
private final String expectedStringWhenStrictDeserializingWithExtraneousFields = "Unrecognized field \"targetName\" (class software.amazon.cloudformation.proxy.hook.HookInvocationRequest), not marked as ignorable (10 known properties: \"requestContext\", \"stackId\", \"clientRequestToken\", \"hookModel\", \"hookTypeName\", \"requestData\", \"actionInvocationPoint\", \"awsAccountId\", \"changeSetId\", \"hookTypeVersion\"])\n"
373423
+ " at [Source: (String)\"{\n" + " \"clientRequestToken\": \"123456\",\n" + " \"awsAccountId\": \"123456789012\",\n"
374424
+ " \"stackId\": \"arn:aws:cloudformation:us-east-1:123456789012:stack/SampleStack/e722ae60-fe62-11e8-9a0e-0ae8cc519968\",\n"

‎src/test/java/software/amazon/cloudformation/data/hook/preCreate.request.with-stack-level-hook.json

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,21 +14,8 @@
1414
"targetName": "STACK",
1515
"targetType": "STACK",
1616
"targetLogicalId": "myStack",
17-
"targetModel": {
18-
"template": "template string here",
19-
"previousTemplate": "previous template string here",
20-
"resolvedTemplate": "resolved template string here",
21-
"changedResources": [
22-
{
23-
"LogicalResourceId": "SomeLogicalResourceId",
24-
"ResourceType": "AWS::S3::Bucket",
25-
"LineNumber": 3,
26-
"Action": "CREATE",
27-
"ResourceProperties": "<Resource Properties as json string>",
28-
"PreviousResourceProperties": "<Resource Properties as json string>"
29-
}
30-
]
31-
},
17+
"targetModel": {},
18+
"payload": "http://someS3PresignedUrl",
3219
"callerCredentials": "callerCredentials",
3320
"providerCredentials": "providerCredentials",
3421
"providerLogGroupName": "providerLoggingGroupName",

‎src/test/java/software/amazon/cloudformation/proxy/hook/targetmodel/HookTargetModelTest.java

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -222,11 +222,11 @@ public void testStackHookTargetModel() throws Exception {
222222
Assertions.assertEquals(changedResources, targetModel.getChangedResources());
223223
Assertions.assertNull(targetModel.getHookTargetTypeReference());
224224
Assertions.assertEquals(
225-
"{\"Template\":\"{\\\"key1\\\":\\\"value1\\\"}\",\"PreviousTemplate\":\"{\\\"previousKey1\\\":\\\"" +
226-
"previousValue1\\\"}\",\"ResolvedTemplate\":\"{\\\"resolvedKey1\\\":\\\"resolvedValue1\\\"}\",\"ChangedResources\"" +
227-
":[{\"LogicalResourceId\":\"SomeLogicalResourceId\",\"ResourceType\":\"AWS::S3::Bucket\",\"LineNumber\":" +
228-
"11,\"Action\":\"CREATE\",\"ResourceProperties\":\"{\\\"BucketName\\\": \\\"some-bucket-name\\\"\"," +
229-
"\"PreviousResourceProperties\":\"{\\\"BucketName\\\": \\\"some-prev-bucket-name\\\"\"}]}",
225+
"{\"Template\":\"{\\\"key1\\\":\\\"value1\\\"}\",\"PreviousTemplate\":\"{\\\"previousKey1\\\":\\\""
226+
+ "previousValue1\\\"}\",\"ResolvedTemplate\":\"{\\\"resolvedKey1\\\":\\\"resolvedValue1\\\"}\",\"ChangedResources\""
227+
+ ":[{\"LogicalResourceId\":\"SomeLogicalResourceId\",\"ResourceType\":\"AWS::S3::Bucket\",\"LineNumber\":"
228+
+ "11,\"Action\":\"CREATE\",\"ResourceProperties\":\"{\\\"BucketName\\\": \\\"some-bucket-name\\\"\","
229+
+ "\"PreviousResourceProperties\":\"{\\\"BucketName\\\": \\\"some-prev-bucket-name\\\"\"}]}",
230230
OBJECT_MAPPER.writeValueAsString(targetModel));
231231
}
232232

@@ -252,14 +252,12 @@ public void testStackHookTargetModelWithAdditionalPropertiesInInput() throws Exc
252252
Assertions.assertEquals(changedResources, targetModel.getChangedResources());
253253
Assertions.assertNull(targetModel.getHookTargetTypeReference());
254254

255-
Assertions.assertEquals(
256-
"{\"Template\":\"{\\\"key1\\\":\\\"value1\\\"}\",\"PreviousTemplate\":\"{\\\"previousKey1\\\":" +
257-
"\\\"previousValue1\\\"}\",\"ResolvedTemplate\":\"{\\\"resolvedKey1\\\":\\\"resolvedValue1\\\"}\"," +
258-
"\"ChangedResources\":[{\"LogicalResourceId\":\"SomeLogicalResourceId\",\"ResourceType\":" +
259-
"\"AWS::S3::Bucket\",\"LineNumber\":11,\"Action\":\"CREATE\",\"ResourceProperties\":\"{" +
260-
"\\\"BucketName\\\": \\\"some-bucket-name\\\"\",\"PreviousResourceProperties\":\"{\\\"BucketName\\\":" +
261-
" \\\"some-prev-bucket-name\\\"\"}]}",
262-
OBJECT_MAPPER.writeValueAsString(targetModel));
255+
Assertions.assertEquals("{\"Template\":\"{\\\"key1\\\":\\\"value1\\\"}\",\"PreviousTemplate\":\"{\\\"previousKey1\\\":"
256+
+ "\\\"previousValue1\\\"}\",\"ResolvedTemplate\":\"{\\\"resolvedKey1\\\":\\\"resolvedValue1\\\"}\","
257+
+ "\"ChangedResources\":[{\"LogicalResourceId\":\"SomeLogicalResourceId\",\"ResourceType\":"
258+
+ "\"AWS::S3::Bucket\",\"LineNumber\":11,\"Action\":\"CREATE\",\"ResourceProperties\":\"{"
259+
+ "\\\"BucketName\\\": \\\"some-bucket-name\\\"\",\"PreviousResourceProperties\":\"{\\\"BucketName\\\":"
260+
+ " \\\"some-prev-bucket-name\\\"\"}]}", OBJECT_MAPPER.writeValueAsString(targetModel));
263261
}
264262

265263
@Test
@@ -282,11 +280,11 @@ public void testStackHookTargetModelWithMissingPropertiesInInput() throws Except
282280
Assertions.assertEquals(changedResources, targetModel.getChangedResources());
283281
Assertions.assertNull(targetModel.getHookTargetTypeReference());
284282
Assertions.assertEquals(
285-
"{\"Template\":\"{\\\"key1\\\":\\\"value1\\\"}\",\"PreviousTemplate\":null,\"ResolvedTemplate\":" +
286-
"\"{\\\"resolvedKey1\\\":\\\"resolvedValue1\\\"}\",\"ChangedResources\":[{\"LogicalResourceId\":" +
287-
"\"SomeLogicalResourceId\",\"ResourceType\":\"AWS::S3::Bucket\",\"LineNumber\":null,\"Action\":" +
288-
"\"CREATE\",\"ResourceProperties\":\"{\\\"BucketName\\\": \\\"some-bucket-name\\\"\"," +
289-
"\"PreviousResourceProperties\":\"{\\\"BucketName\\\": \\\"some-prev-bucket-name\\\"\"}]}",
283+
"{\"Template\":\"{\\\"key1\\\":\\\"value1\\\"}\",\"PreviousTemplate\":null,\"ResolvedTemplate\":"
284+
+ "\"{\\\"resolvedKey1\\\":\\\"resolvedValue1\\\"}\",\"ChangedResources\":[{\"LogicalResourceId\":"
285+
+ "\"SomeLogicalResourceId\",\"ResourceType\":\"AWS::S3::Bucket\",\"LineNumber\":null,\"Action\":"
286+
+ "\"CREATE\",\"ResourceProperties\":\"{\\\"BucketName\\\": \\\"some-bucket-name\\\"\","
287+
+ "\"PreviousResourceProperties\":\"{\\\"BucketName\\\": \\\"some-prev-bucket-name\\\"\"}]}",
290288
OBJECT_MAPPER.writeValueAsString(targetModel));
291289
}
292290

0 commit comments

Comments
 (0)