Skip to content

Commit b791808

Browse files
tkhill-AWStkhill-AWS
authored
Java v2: Update S3 examples of uploading from a stream of unknown size to remove use of BlockingInputStreamAsyncRequestBody (#7513)
* Update examples for uploading from an inputstream of unknown size
1 parent 5762d3c commit b791808

File tree

5 files changed

+126
-32
lines changed

5 files changed

+126
-32
lines changed

‎.doc_gen/metadata/s3_metadata.yaml

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3294,13 +3294,15 @@ s3_Scenario_UploadStream:
32943294
- description: Use the <ulink url="sdk-for-java/latest/developer-guide/crt-based-s3-client.html" type="documentation">&AWS;
32953295
CRT-based S3 Client</ulink>.
32963296
snippet_tags:
3297-
- s3.java2.async_stream.import
3298-
- s3.java2.async_stream.main
3297+
- s3.java2.async_stream.complete
3298+
- description: Use the standard <ulink url="sdk-for-java/latest/developer-guide/s3-async-client-multipart.html#s3-async-client-mp-on" type="documentation">
3299+
asynchronous S3 client with multipart upload enabled</ulink>.
3300+
snippet_tags:
3301+
- s3.java2.async_stream_mp.complete
32993302
- description: Use the <ulink url="sdk-for-java/latest/developer-guide/transfer-manager.html" type="documentation">&S3;
33003303
Transfer Manager</ulink>.
33013304
snippet_tags:
3302-
- s3.tm.java2.upload_stream.import
3303-
- s3.tm.java2.upload_stream.main
3305+
- s3.tm.java2.upload_stream.complete
33043306
Swift:
33053307
versions:
33063308
- sdk_version: 1

‎javav2/example_code/s3/src/main/java/com/example/s3/async/PutObjectFromStreamAsync.java

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,21 @@
44

55
// snippet-start:[s3.java2.async_stream.complete]
66
// snippet-start:[s3.java2.async_stream.import]
7+
78
import com.example.s3.util.AsyncExampleUtils;
89
import org.slf4j.Logger;
910
import org.slf4j.LoggerFactory;
1011
import software.amazon.awssdk.core.async.AsyncRequestBody;
11-
import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody;
1212
import software.amazon.awssdk.core.exception.SdkException;
1313
import software.amazon.awssdk.services.s3.S3AsyncClient;
1414
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
1515

1616
import java.io.ByteArrayInputStream;
17+
import java.io.InputStream;
1718
import java.util.UUID;
1819
import java.util.concurrent.CompletableFuture;
20+
import java.util.concurrent.ExecutorService;
21+
import java.util.concurrent.Executors;
1922
// snippet-end:[s3.java2.async_stream.import]
2023

2124
public class PutObjectFromStreamAsync {
@@ -28,7 +31,8 @@ public static void main(String[] args) {
2831
AsyncExampleUtils.createBucket(bucketName);
2932
try {
3033
PutObjectFromStreamAsync example = new PutObjectFromStreamAsync();
31-
PutObjectResponse putObjectResponse = example.putObjectFromStream(AsyncExampleUtils.client, bucketName, key);
34+
S3AsyncClient s3AsyncClientCrt = S3AsyncClient.crtCreate();
35+
PutObjectResponse putObjectResponse = example.putObjectFromStreamCrt(s3AsyncClientCrt, bucketName, key);
3236
logger.info("Object {} etag: {}", key, putObjectResponse.eTag());
3337
logger.info("Object {} uploaded to bucket {}.", key, bucketName);
3438
} catch (SdkException e) {
@@ -41,29 +45,29 @@ public static void main(String[] args) {
4145

4246
// snippet-start:[s3.java2.async_stream.main]
4347
/**
44-
* @param s33CrtAsyncClient - To upload content from a stream of unknown size, use the AWS CRT-based S3 client. For more information, see
45-
* https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/crt-based-s3-client.html.
48+
* @param s33CrtAsyncClient - To upload content from a stream of unknown size, use can the AWS CRT-based S3 client.
4649
* @param bucketName - The name of the bucket.
4750
* @param key - The name of the object.
4851
* @return software.amazon.awssdk.services.s3.model.PutObjectResponse - Returns metadata pertaining to the put object operation.
4952
*/
50-
public PutObjectResponse putObjectFromStream(S3AsyncClient s33CrtAsyncClient, String bucketName, String key) {
51-
52-
BlockingInputStreamAsyncRequestBody body =
53-
AsyncRequestBody.forBlockingInputStream(null); // 'null' indicates a stream will be provided later.
54-
55-
CompletableFuture<PutObjectResponse> responseFuture =
56-
s33CrtAsyncClient.putObject(r -> r.bucket(bucketName).key(key), body);
53+
public PutObjectResponse putObjectFromStreamCrt(S3AsyncClient s33CrtAsyncClient, String bucketName, String key) {
5754

5855
// AsyncExampleUtils.randomString() returns a random string up to 100 characters.
5956
String randomString = AsyncExampleUtils.randomString();
6057
logger.info("random string to upload: {}: length={}", randomString, randomString.length());
58+
InputStream inputStream = new ByteArrayInputStream(randomString.getBytes());
59+
60+
// Executor required to handle reading from the InputStream on a separate thread so the main upload is not blocked.
61+
ExecutorService executor = Executors.newSingleThreadExecutor();
62+
// Specify `null` for the content length when you don't know the content length.
63+
AsyncRequestBody body = AsyncRequestBody.fromInputStream(inputStream, null, executor);
6164

62-
// Provide the stream of data to be uploaded.
63-
body.writeInputStream(new ByteArrayInputStream(randomString.getBytes()));
65+
CompletableFuture<PutObjectResponse> responseFuture =
66+
s33CrtAsyncClient.putObject(r -> r.bucket(bucketName).key(key), body);
6467

6568
PutObjectResponse response = responseFuture.join(); // Wait for the response.
6669
logger.info("Object {} uploaded to bucket {}.", key, bucketName);
70+
executor.shutdown();
6771
return response;
6872
}
6973
}

‎javav2/example_code/s3/src/main/java/com/example/s3/async/PutObjectFromStreamAsyncMp.java

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
package com.example.s3.async;
4+
5+
// snippet-start:[s3.java2.async_stream_mp.complete]
6+
// snippet-start:[s3.java2.async_stream_mp.import]
7+
8+
import com.example.s3.util.AsyncExampleUtils;
9+
import org.slf4j.Logger;
10+
import org.slf4j.LoggerFactory;
11+
import software.amazon.awssdk.core.async.AsyncRequestBody;
12+
import software.amazon.awssdk.core.exception.SdkException;
13+
import software.amazon.awssdk.services.s3.S3AsyncClient;
14+
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
15+
16+
import java.io.ByteArrayInputStream;
17+
import java.io.InputStream;
18+
import java.util.UUID;
19+
import java.util.concurrent.CompletableFuture;
20+
import java.util.concurrent.ExecutorService;
21+
import java.util.concurrent.Executors;
22+
// snippet-end:[s3.java2.async_stream_mp.import]
23+
24+
public class PutObjectFromStreamAsyncMp {
25+
private static final Logger logger = LoggerFactory.getLogger(PutObjectFromStreamAsyncMp.class);
26+
27+
public static void main(String[] args) {
28+
String bucketName = "amzn-s3-demo-bucket-" + UUID.randomUUID(); // Change bucket name.
29+
String key = UUID.randomUUID().toString();
30+
31+
AsyncExampleUtils.createBucket(bucketName);
32+
try {
33+
PutObjectFromStreamAsyncMp example = new PutObjectFromStreamAsyncMp();
34+
S3AsyncClient s3AsyncClientMp = S3AsyncClient.builder().multipartEnabled(true).build();
35+
PutObjectResponse putObjectResponse = example.putObjectFromStreamMp(s3AsyncClientMp, bucketName, key);
36+
logger.info("Object {} etag: {}", key, putObjectResponse.eTag());
37+
logger.info("Object {} uploaded to bucket {}.", key, bucketName);
38+
} catch (SdkException e) {
39+
logger.error(e.getMessage(), e);
40+
} finally {
41+
AsyncExampleUtils.deleteObject(bucketName, key);
42+
AsyncExampleUtils.deleteBucket(bucketName);
43+
}
44+
}
45+
46+
// snippet-start:[s3.java2.async_stream_mp.main]
47+
/**
48+
* @param s3AsyncClientMp - To upload content from a stream of unknown size, use can the S3 asynchronous client with multipart enabled.
49+
* @param bucketName - The name of the bucket.
50+
* @param key - The name of the object.
51+
* @return software.amazon.awssdk.services.s3.model.PutObjectResponse - Returns metadata pertaining to the put object operation.
52+
*/
53+
public PutObjectResponse putObjectFromStreamMp(S3AsyncClient s3AsyncClientMp, String bucketName, String key) {
54+
55+
// AsyncExampleUtils.randomString() returns a random string up to 100 characters.
56+
String randomString = AsyncExampleUtils.randomString();
57+
logger.info("random string to upload: {}: length={}", randomString, randomString.length());
58+
InputStream inputStream = new ByteArrayInputStream(randomString.getBytes());
59+
60+
// Executor required to handle reading from the InputStream on a separate thread so the main upload is not blocked.
61+
ExecutorService executor = Executors.newSingleThreadExecutor();
62+
// Specify `null` for the content length when you don't know the content length.
63+
AsyncRequestBody body = AsyncRequestBody.fromInputStream(inputStream, null, executor);
64+
65+
CompletableFuture<PutObjectResponse> responseFuture =
66+
s3AsyncClientMp.putObject(r -> r.bucket(bucketName).key(key), body);
67+
68+
PutObjectResponse response = responseFuture.join(); // Wait for the response.
69+
logger.info("Object {} uploaded to bucket {}.", key, bucketName);
70+
executor.shutdown();
71+
return response;
72+
}
73+
}
74+
// snippet-end:[s3.java2.async_stream_mp.main]
75+
// snippet-end:[s3.java2.async_stream_mp.complete]

‎javav2/example_code/s3/src/main/java/com/example/s3/transfermanager/UploadStream.java

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,21 @@
44

55
// snippet-start:[s3.tm.java2.upload_stream.complete]
66
// snippet-start:[s3.tm.java2.upload_stream.import]
7+
78
import com.example.s3.util.AsyncExampleUtils;
89
import org.slf4j.Logger;
910
import org.slf4j.LoggerFactory;
1011
import software.amazon.awssdk.core.async.AsyncRequestBody;
11-
import software.amazon.awssdk.core.async.BlockingInputStreamAsyncRequestBody;
1212
import software.amazon.awssdk.core.exception.SdkException;
1313
import software.amazon.awssdk.transfer.s3.S3TransferManager;
1414
import software.amazon.awssdk.transfer.s3.model.CompletedUpload;
1515
import software.amazon.awssdk.transfer.s3.model.Upload;
1616

1717
import java.io.ByteArrayInputStream;
18+
import java.io.InputStream;
1819
import java.util.UUID;
20+
import java.util.concurrent.ExecutorService;
21+
import java.util.concurrent.Executors;
1922
// snippet-end:[s3.tm.java2.upload_stream.import]
2023

2124
public class UploadStream {
@@ -41,30 +44,31 @@ public static void main(String[] args) {
4144

4245
// snippet-start:[s3.tm.java2.upload_stream.main]
4346
/**
44-
* @param transferManager - To upload content from a stream of unknown size, use the S3TransferManager based on the AWS CRT-based S3 client.
45-
* For more information, see https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/transfer-manager.html.
47+
* @param transferManager - To upload content from a stream of unknown size, you can use the S3TransferManager based on the AWS CRT-based S3 client.
4648
* @param bucketName - The name of the bucket.
4749
* @param key - The name of the object.
4850
* @return - software.amazon.awssdk.transfer.s3.model.CompletedUpload - The result of the completed upload.
4951
*/
5052
public CompletedUpload uploadStream(S3TransferManager transferManager, String bucketName, String key) {
5153

52-
BlockingInputStreamAsyncRequestBody body =
53-
AsyncRequestBody.forBlockingInputStream(null); // 'null' indicates a stream will be provided later.
54+
// AsyncExampleUtils.randomString() returns a random string up to 100 characters.
55+
String randomString = AsyncExampleUtils.randomString();
56+
logger.info("random string to upload: {}: length={}", randomString, randomString.length());
57+
InputStream inputStream = new ByteArrayInputStream(randomString.getBytes());
58+
59+
// Executor required to handle reading from the InputStream on a separate thread so the main upload is not blocked.
60+
ExecutorService executor = Executors.newSingleThreadExecutor();
61+
// Specify `null` for the content length when you don't know the content length.
62+
AsyncRequestBody body = AsyncRequestBody.fromInputStream(inputStream, null, executor);
5463

5564
Upload upload = transferManager.upload(builder -> builder
5665
.requestBody(body)
5766
.putObjectRequest(req -> req.bucket(bucketName).key(key))
5867
.build());
5968

60-
// AsyncExampleUtils.randomString() returns a random string up to 100 characters.
61-
String randomString = AsyncExampleUtils.randomString();
62-
logger.info("random string to upload: {}: length={}", randomString, randomString.length());
63-
64-
// Provide the stream of data to be uploaded.
65-
body.writeInputStream(new ByteArrayInputStream(randomString.getBytes()));
66-
67-
return upload.completionFuture().join();
69+
CompletedUpload completedUpload = upload.completionFuture().join();
70+
executor.shutdown();
71+
return completedUpload;
6872
}
6973
}
7074
// snippet-end:[s3.tm.java2.upload_stream.main]

‎javav2/example_code/s3/src/test/java/com/example/s3/async/AsyncTests.java

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import org.junit.jupiter.api.Test;
1111
import org.slf4j.Logger;
1212
import org.slf4j.LoggerFactory;
13+
import software.amazon.awssdk.services.s3.S3AsyncClient;
1314
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
1415

1516
import java.util.UUID;
@@ -34,9 +35,17 @@ void tearDown() {
3435

3536
@Test
3637
@Tag("IntegrationTest")
37-
void putObjectFromStream() {
38+
void putObjectFromStreamTest() {
3839
PutObjectFromStreamAsync example = new PutObjectFromStreamAsync();
39-
PutObjectResponse putObjectResponse = example.putObjectFromStream(AsyncExampleUtils.client, bucketName, key);
40+
PutObjectResponse putObjectResponse = example.putObjectFromStreamCrt(AsyncExampleUtils.client, bucketName, key);
41+
Assertions.assertNotNull(putObjectResponse.eTag());
42+
}
43+
44+
@Test
45+
@Tag("IntegrationTest")
46+
void putObjectFromStreamMpTest() {
47+
PutObjectFromStreamAsyncMp example = new PutObjectFromStreamAsyncMp();
48+
PutObjectResponse putObjectResponse = example.putObjectFromStreamMp(S3AsyncClient.builder().multipartEnabled(true).build(), bucketName, key);
4049
Assertions.assertNotNull(putObjectResponse.eTag());
4150
}
4251
}

0 commit comments

Comments
 (0)