Skip to content

Commit cf64793

Browse files
phantumcodephantumcode
authored
feat(storage): add delimiter support (#2871)
1 parent 45a2451 commit cf64793

File tree

18 files changed

+684
-29
lines changed

18 files changed

+684
-29
lines changed

‎aws-storage-s3/api/aws-storage-s3.api

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,10 +297,12 @@ public final class com/amplifyframework/storage/s3/request/AWSS3StorageGetPresig
297297
public final class com/amplifyframework/storage/s3/request/AWSS3StorageListRequest {
298298
public fun <init> (Ljava/lang/String;Lcom/amplifyframework/storage/StorageAccessLevel;Ljava/lang/String;)V
299299
public fun <init> (Ljava/lang/String;Lcom/amplifyframework/storage/StorageAccessLevel;Ljava/lang/String;ILjava/lang/String;)V
300+
public fun <init> (Ljava/lang/String;Lcom/amplifyframework/storage/StorageAccessLevel;Ljava/lang/String;ILjava/lang/String;Lcom/amplifyframework/storage/options/SubpathStrategy;)V
300301
public fun getAccessLevel ()Lcom/amplifyframework/storage/StorageAccessLevel;
301302
public fun getNextToken ()Ljava/lang/String;
302303
public fun getPageSize ()I
303304
public fun getPath ()Ljava/lang/String;
305+
public fun getSubpathStrategy ()Lcom/amplifyframework/storage/options/SubpathStrategy;
304306
public fun getTargetIdentityId ()Ljava/lang/String;
305307
}
306308

@@ -331,6 +333,8 @@ public abstract interface class com/amplifyframework/storage/s3/service/StorageS
331333
public abstract fun getTransfer (Ljava/lang/String;)Lcom/amplifyframework/storage/s3/transfer/TransferRecord;
332334
public abstract fun listFiles (Ljava/lang/String;Ljava/lang/String;)Ljava/util/List;
333335
public abstract fun listFiles (Ljava/lang/String;Ljava/lang/String;ILjava/lang/String;)Lcom/amplifyframework/storage/result/StorageListResult;
336+
public abstract fun listFiles (Ljava/lang/String;Ljava/lang/String;ILjava/lang/String;Lcom/amplifyframework/storage/options/SubpathStrategy;)Lcom/amplifyframework/storage/result/StorageListResult;
337+
public abstract fun listFiles (Ljava/lang/String;Ljava/lang/String;Lcom/amplifyframework/storage/options/SubpathStrategy;)Lcom/amplifyframework/storage/result/StorageListResult;
334338
public abstract fun pauseTransfer (Lcom/amplifyframework/storage/s3/transfer/TransferObserver;)V
335339
public abstract fun resumeTransfer (Lcom/amplifyframework/storage/s3/transfer/TransferObserver;)V
336340
public abstract fun uploadFile (Ljava/lang/String;Ljava/lang/String;Ljava/io/File;Lcom/amplifyframework/storage/ObjectMetadata;Z)Lcom/amplifyframework/storage/s3/transfer/TransferObserver;

‎aws-storage-s3/build.gradle.kts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ dependencies {
6060
androidTestImplementation(libs.test.androidx.runner)
6161
androidTestImplementation(libs.test.androidx.junit)
6262
androidTestImplementation(libs.test.androidx.workmanager)
63+
androidTestImplementation(libs.test.kotest.assertions)
6364
androidTestImplementation(project(":aws-storage-s3"))
6465

6566
androidTestUtil(libs.test.androidx.orchestrator)

‎aws-storage-s3/src/androidTest/java/com/amplifyframework/storage/s3/AWSS3StorageSubPathStrategyListTest.kt

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,201 @@
1+
/*
2+
* Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package com.amplifyframework.storage.s3
17+
18+
import android.content.Context
19+
import androidx.test.core.app.ApplicationProvider
20+
import com.amplifyframework.auth.cognito.AWSCognitoAuthPlugin
21+
import com.amplifyframework.storage.StorageCategory
22+
import com.amplifyframework.storage.StoragePath
23+
import com.amplifyframework.storage.options.StorageRemoveOptions
24+
import com.amplifyframework.storage.options.StorageUploadFileOptions
25+
import com.amplifyframework.storage.options.SubpathStrategy
26+
import com.amplifyframework.storage.s3.options.AWSS3StoragePagedListOptions
27+
import com.amplifyframework.storage.s3.test.R
28+
import com.amplifyframework.storage.s3.util.WorkmanagerTestUtils
29+
import com.amplifyframework.testutils.random.RandomTempFile
30+
import com.amplifyframework.testutils.sync.SynchronousAuth
31+
import com.amplifyframework.testutils.sync.SynchronousStorage
32+
import io.kotest.matchers.collections.shouldContainExactly
33+
import io.kotest.matchers.ints.shouldBeExactly
34+
import io.kotest.matchers.nulls.shouldBeNull
35+
import java.io.File
36+
import org.junit.After
37+
import org.junit.BeforeClass
38+
import org.junit.Test
39+
40+
/**
41+
* Integration tests for using SubpathStrategy with Storage List API
42+
*/
43+
class AWSS3StorageSubPathStrategyListTest {
44+
companion object {
45+
private const val SMALL_FILE_SIZE = 100L
46+
private const val FIRST_FILE_NAME = "01"
47+
private const val SECOND_FILE_NAME = "02"
48+
private const val THIRD_FILE_NAME = "03"
49+
private const val FOURTH_FILE_NAME = "04"
50+
private const val FIFTH_FILE_NAME = "05"
51+
private const val CUSTOM_FILE_NAME = "custom"
52+
private const val FIRST_FILE_STRING_PATH = "public/photos/2023/$FIRST_FILE_NAME"
53+
private val FIRST_FILE_PATH = StoragePath.fromString(FIRST_FILE_STRING_PATH)
54+
private const val SECOND_FILE_STRING_PATH = "public/photos/2023/$SECOND_FILE_NAME"
55+
private val SECOND_FILE_PATH = StoragePath.fromString(SECOND_FILE_STRING_PATH)
56+
private const val THIRD_FILE_STRING_PATH = "public/photos/2024/$THIRD_FILE_NAME"
57+
private val THIRD_FILE_PATH = StoragePath.fromString(THIRD_FILE_STRING_PATH)
58+
private const val FOURTH_FILE_STRING_PATH = "public/photos/2024/$FOURTH_FILE_NAME"
59+
private val FOURTH_FILE_PATH = StoragePath.fromString(FOURTH_FILE_STRING_PATH)
60+
private const val FIFTH_FILE_STRING_PATH = "public/photos/$FIFTH_FILE_NAME"
61+
private val FIFTH_FILE_PATH = StoragePath.fromString(FIFTH_FILE_STRING_PATH)
62+
private const val CUSTOM_FILE_STRING_PATH = "public/photos/202$/$CUSTOM_FILE_NAME"
63+
private val CUSTOM_FILE_PATH = StoragePath.fromString(CUSTOM_FILE_STRING_PATH)
64+
65+
lateinit var storageCategory: StorageCategory
66+
lateinit var synchronousStorage: SynchronousStorage
67+
lateinit var synchronousAuth: SynchronousAuth
68+
private lateinit var first: File
69+
private lateinit var second: File
70+
private lateinit var third: File
71+
private lateinit var fourth: File
72+
private lateinit var fifth: File
73+
private lateinit var customFile: File
74+
75+
/**
76+
* Initialize mobile client and configure the storage.
77+
* Upload the test files ahead of time.
78+
*/
79+
@JvmStatic
80+
@BeforeClass
81+
fun setUpOnce() {
82+
val context = ApplicationProvider.getApplicationContext<Context>()
83+
WorkmanagerTestUtils.initializeWorkmanagerTestUtil(context)
84+
85+
synchronousAuth = SynchronousAuth.delegatingToCognito(context, AWSCognitoAuthPlugin())
86+
87+
// Get a handle to storage
88+
storageCategory = TestStorageCategory.create(context, R.raw.amplifyconfiguration)
89+
synchronousStorage = SynchronousStorage.delegatingTo(storageCategory)
90+
91+
// Upload test files
92+
first = RandomTempFile(FIRST_FILE_NAME, SMALL_FILE_SIZE)
93+
synchronousStorage.uploadFile(FIRST_FILE_PATH, first, StorageUploadFileOptions.defaultInstance())
94+
second = RandomTempFile(SECOND_FILE_NAME, SMALL_FILE_SIZE)
95+
synchronousStorage.uploadFile(SECOND_FILE_PATH, second, StorageUploadFileOptions.defaultInstance())
96+
third = RandomTempFile(THIRD_FILE_NAME, SMALL_FILE_SIZE)
97+
synchronousStorage.uploadFile(THIRD_FILE_PATH, third, StorageUploadFileOptions.defaultInstance())
98+
fourth = RandomTempFile(FOURTH_FILE_NAME, SMALL_FILE_SIZE)
99+
synchronousStorage.uploadFile(FOURTH_FILE_PATH, fourth, StorageUploadFileOptions.defaultInstance())
100+
fifth = RandomTempFile(FIFTH_FILE_NAME, SMALL_FILE_SIZE)
101+
synchronousStorage.uploadFile(FIFTH_FILE_PATH, fifth, StorageUploadFileOptions.defaultInstance())
102+
103+
customFile = RandomTempFile(CUSTOM_FILE_NAME, SMALL_FILE_SIZE)
104+
synchronousStorage.uploadFile(CUSTOM_FILE_PATH, customFile, StorageUploadFileOptions.defaultInstance())
105+
}
106+
}
107+
108+
@After
109+
fun tearDown() {
110+
synchronousStorage.remove("photos/2023/$FIRST_FILE_NAME", StorageRemoveOptions.defaultInstance())
111+
synchronousStorage.remove("photos/2023/$SECOND_FILE_NAME", StorageRemoveOptions.defaultInstance())
112+
synchronousStorage.remove("photos/2024/$THIRD_FILE_NAME", StorageRemoveOptions.defaultInstance())
113+
synchronousStorage.remove("photos/2024/$FOURTH_FILE_NAME", StorageRemoveOptions.defaultInstance())
114+
synchronousStorage.remove("photos/$FIFTH_FILE_NAME", StorageRemoveOptions.defaultInstance())
115+
synchronousStorage.remove("photos/$CUSTOM_FILE_NAME", StorageRemoveOptions.defaultInstance())
116+
}
117+
118+
@Test
119+
fun testListWithIncludeStrategyAndStoragePath() {
120+
val path = StoragePath.fromString("public/photos/")
121+
val options = AWSS3StoragePagedListOptions
122+
.builder()
123+
.setPageSize(10)
124+
.setSubpathStrategy(SubpathStrategy.Include)
125+
.build()
126+
127+
val result = synchronousStorage.list(path, options)
128+
129+
result.items.size shouldBeExactly(6)
130+
result.items.mapNotNull { it.path } shouldContainExactly listOf(
131+
"public/photos/05",
132+
"public/photos/202$/custom",
133+
"public/photos/2023/01",
134+
"public/photos/2023/02",
135+
"public/photos/2024/03",
136+
"public/photos/2024/04"
137+
)
138+
}
139+
140+
@Test
141+
fun testListWithExcludeStrategyAndStoragePath() {
142+
val options = AWSS3StoragePagedListOptions
143+
.builder()
144+
.setPageSize(10)
145+
.setSubpathStrategy(SubpathStrategy.Exclude())
146+
.build()
147+
148+
var result = synchronousStorage.list(StoragePath.fromString("public/photos/"), options)
149+
150+
result.items.size shouldBeExactly(1)
151+
result.items.mapNotNull { it.path } shouldContainExactly listOf("public/photos/05")
152+
153+
result.excludedSubpaths.size shouldBeExactly(3)
154+
result.excludedSubpaths shouldContainExactly listOf(
155+
"public/photos/202$/",
156+
"public/photos/2023/",
157+
"public/photos/2024/"
158+
)
159+
160+
result = synchronousStorage.list(StoragePath.fromString("public/photos/2023/"), options)
161+
162+
result.items.size shouldBeExactly(2)
163+
result.items.mapNotNull { it.path } shouldContainExactly listOf(
164+
"public/photos/2023/01",
165+
"public/photos/2023/02"
166+
)
167+
168+
result.excludedSubpaths.shouldBeNull()
169+
}
170+
171+
@Test
172+
fun testListWithExcludeCustomDelimiterStrategyAndStoragePath() {
173+
val options = AWSS3StoragePagedListOptions
174+
.builder()
175+
.setPageSize(10)
176+
.setSubpathStrategy(SubpathStrategy.Exclude("$"))
177+
.build()
178+
179+
var result = synchronousStorage.list(StoragePath.fromString("public/photos/"), options)
180+
181+
result.items.size shouldBeExactly(5)
182+
result.items.mapNotNull { it.path } shouldContainExactly listOf(
183+
"public/photos/05",
184+
"public/photos/2023/01",
185+
"public/photos/2023/02",
186+
"public/photos/2024/03",
187+
"public/photos/2024/04"
188+
)
189+
190+
result.excludedSubpaths.size shouldBeExactly(1)
191+
result.excludedSubpaths shouldContainExactly listOf("public/photos/202$")
192+
193+
result = synchronousStorage.list(StoragePath.fromString("public/photos/2023/"), options)
194+
195+
result.items.size shouldBeExactly(2)
196+
result.items.mapNotNull { it.path } shouldContainExactly listOf(
197+
"public/photos/2023/01",
198+
"public/photos/2023/02",
199+
)
200+
}
201+
}

‎aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/AWSS3StoragePlugin.java

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -955,7 +955,8 @@ public StorageListOperation<?> list(@NonNull String path,
955955
options.getAccessLevel() != null ? options.getAccessLevel() : defaultAccessLevel,
956956
options.getTargetIdentityId(),
957957
options.getPageSize(),
958-
options.getNextToken());
958+
options.getNextToken(),
959+
options.getSubpathStrategy());
959960

960961
AWSS3StorageListOperation operation =
961962
new AWSS3StorageListOperation(
@@ -983,7 +984,8 @@ public StorageListOperation<?> list(
983984
AWSS3StoragePathListRequest request = new AWSS3StoragePathListRequest(
984985
path,
985986
options.getPageSize(),
986-
options.getNextToken());
987+
options.getNextToken(),
988+
options.getSubpathStrategy());
987989

988990
AWSS3StoragePathListOperation operation =
989991
new AWSS3StoragePathListOperation(

‎aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/operation/AWSS3StorageListOperation.java

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,15 +20,14 @@
2020
import com.amplifyframework.auth.AuthCredentialsProvider;
2121
import com.amplifyframework.core.Consumer;
2222
import com.amplifyframework.storage.StorageException;
23-
import com.amplifyframework.storage.StorageItem;
2423
import com.amplifyframework.storage.operation.StorageListOperation;
24+
import com.amplifyframework.storage.options.SubpathStrategy;
2525
import com.amplifyframework.storage.result.StorageListResult;
2626
import com.amplifyframework.storage.s3.configuration.AWSS3StoragePluginConfiguration;
2727
import com.amplifyframework.storage.s3.options.AWSS3StoragePagedListOptions;
2828
import com.amplifyframework.storage.s3.request.AWSS3StorageListRequest;
2929
import com.amplifyframework.storage.s3.service.StorageService;
3030

31-
import java.util.List;
3231
import java.util.concurrent.ExecutorService;
3332

3433
/**
@@ -86,14 +85,14 @@ public void start() {
8685
prefix -> {
8786
try {
8887
String serviceKey = prefix.concat(getRequest().getPath());
88+
SubpathStrategy subpathStrategy = getRequest().getSubpathStrategy();
8989
if (getRequest().getPageSize() == AWSS3StoragePagedListOptions.ALL_PAGE_SIZE) {
9090
// fetch all the keys
91-
List<StorageItem> listedItems = storageService.listFiles(serviceKey, prefix);
92-
onSuccess.accept(StorageListResult.fromItems(listedItems, null));
91+
onSuccess.accept(storageService.listFiles(serviceKey, prefix, subpathStrategy));
9392
} else {
9493
onSuccess.accept(
9594
storageService.listFiles(serviceKey, prefix, getRequest().getPageSize(),
96-
getRequest().getNextToken()));
95+
getRequest().getNextToken(), subpathStrategy));
9796
}
9897
} catch (Exception exception) {
9998
onError.accept(new StorageException(

‎aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/operation/AWSS3StoragePathListOperation.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ internal class AWSS3StoragePathListOperation(
4949

5050
try {
5151
onSuccess.accept(
52-
storageService.listFiles(serviceKey, request.pageSize, request.nextToken)
52+
storageService.listFiles(serviceKey, request.pageSize, request.nextToken, request.subpathStrategy)
5353
)
5454
} catch (exception: Exception) {
5555
onError.accept(

‎aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/request/AWSS3StorageListRequest.java

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import androidx.annotation.Nullable;
2020

2121
import com.amplifyframework.storage.StorageAccessLevel;
22+
import com.amplifyframework.storage.options.SubpathStrategy;
2223
import com.amplifyframework.storage.s3.options.AWSS3StoragePagedListOptions;
2324

2425
/**
@@ -33,6 +34,7 @@ public final class AWSS3StorageListRequest {
3334
private final String targetIdentityId;
3435
private final int pageSize;
3536
private final String nextToken;
37+
private final SubpathStrategy subpathStrategy;
3638

3739
/**
3840
* Constructs a new AWSS3StorageListRequest.
@@ -55,6 +57,7 @@ public AWSS3StorageListRequest(
5557
this.targetIdentityId = targetIdentityId;
5658
this.pageSize = AWSS3StoragePagedListOptions.ALL_PAGE_SIZE;
5759
this.nextToken = null;
60+
this.subpathStrategy = null;
5861
}
5962

6063
/**
@@ -82,6 +85,37 @@ public AWSS3StorageListRequest(
8285
this.targetIdentityId = targetIdentityId;
8386
this.pageSize = pageSize;
8487
this.nextToken = nextToken;
88+
this.subpathStrategy = null;
89+
}
90+
91+
/**
92+
* Constructs a new AWSS3StorageListRequest.
93+
* Although this has public access, it is intended for internal use and should not be used directly by host
94+
* applications. The behavior of this may change without warning.
95+
*
96+
* @param path the path in S3 to list items from
97+
* @param accessLevel Storage access level
98+
* @param targetIdentityId If set, this should override the current user's identity ID.
99+
* If null, the operation will fetch the current identity ID.
100+
* @param pageSize number of keys to be retrieved from s3
101+
* @param nextToken next continuation token to be passed to s3
102+
* @param subpathStrategy strategy to include or exclude sub-paths in s3 path
103+
*/
104+
@SuppressWarnings("deprecation")
105+
public AWSS3StorageListRequest(
106+
@NonNull String path,
107+
@NonNull StorageAccessLevel accessLevel,
108+
@Nullable String targetIdentityId,
109+
int pageSize,
110+
@Nullable String nextToken,
111+
@Nullable SubpathStrategy subpathStrategy
112+
) {
113+
this.path = path;
114+
this.accessLevel = accessLevel;
115+
this.targetIdentityId = targetIdentityId;
116+
this.pageSize = pageSize;
117+
this.nextToken = nextToken;
118+
this.subpathStrategy = subpathStrategy;
85119
}
86120

87121
/**
@@ -128,5 +162,14 @@ public int getPageSize() {
128162
public String getNextToken() {
129163
return nextToken;
130164
}
165+
166+
/**
167+
* Get SubpathStrategy to include/exclude sub-paths.
168+
* @return SubpathStrategy to include/exclude sub-paths.
169+
* */
170+
@Nullable
171+
public SubpathStrategy getSubpathStrategy() {
172+
return subpathStrategy;
173+
}
131174
}
132175

‎aws-storage-s3/src/main/java/com/amplifyframework/storage/s3/request/AWSS3StoragePathListRequest.kt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,14 @@
1515
package com.amplifyframework.storage.s3.request
1616

1717
import com.amplifyframework.storage.StoragePath
18+
import com.amplifyframework.storage.options.SubpathStrategy
1819

1920
/**
2021
* Parameters to provide to S3 that describe a request to list files.
2122
*/
2223
internal data class AWSS3StoragePathListRequest(
2324
val path: StoragePath,
2425
val pageSize: Int,
25-
val nextToken: String?
26+
val nextToken: String?,
27+
val subpathStrategy: SubpathStrategy?
2628
)

0 commit comments

Comments
 (0)