From d5a6777d19727674bba798ed051f1a29459fde48 Mon Sep 17 00:00:00 2001 From: haverchuck Date: Mon, 1 Jul 2024 14:08:45 -0700 Subject: [PATCH 01/13] feat(js): list api subpathStrategy parameter --- .../storage/list-files/index.mdx | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 37d01b40b9c..7ba780b3e4c 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -109,6 +109,43 @@ function processStorageList(response) { This places each item's data inside a special `__data` key. +### Subpath Strategies + +When objects are organized using a prefix you can return only those objects within the `path` you've provided, and not those within nested subpaths. + +For example, given the following keys in your `path` you may want to return only the jpg objects, and not the "vacation" subpath: + +``` +"photos/photo1.jpg", +"photos/photo2.jpg", +"photos/vacation/" +``` + +This can be accomplished with the `subpathStrategy` option: + +```js +const result = await list({ + path: "photos/", + options:{ + subpathStrategy: { strategy:'exclude' } + } +}); +``` + +The default delimiter character is '/', but this can be changed by supplying a custom delimiter: + +```js +const result = await list({ + path: "photos-", + options:{ + subpathStrategy: { + strategy:'exclude', + delimiter: '-' + } + } +}); +``` + ### More `list` options Option | Type | Description | From 81ea5dc49c2ffbcdd2299b9eb23bdc9e9ff430a2 Mon Sep 17 00:00:00 2001 From: haverchuck Date: Mon, 1 Jul 2024 14:16:29 -0700 Subject: [PATCH 02/13] fix: adds style guide suggestions --- .../build-a-backend/storage/list-files/index.mdx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 7ba780b3e4c..c79b106a548 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -109,17 +109,17 @@ function processStorageList(response) { This places each item's data inside a special `__data` key. -### Subpath Strategies +### Exclude Subpaths When objects are organized using a prefix you can return only those objects within the `path` you've provided, and not those within nested subpaths. For example, given the following keys in your `path` you may want to return only the jpg objects, and not the "vacation" subpath: -``` -"photos/photo1.jpg", -"photos/photo2.jpg", -"photos/vacation/" -``` + +** `photos/photo1.jpg` +** `photos/photo2.jpg` +** `photos/vacation/` + This can be accomplished with the `subpathStrategy` option: From 67dbe34e8da53eb4fd43948d121afc733e9412a7 Mon Sep 17 00:00:00 2001 From: haverchuck Date: Mon, 1 Jul 2024 14:40:09 -0700 Subject: [PATCH 03/13] fix: adds result structure --- .../storage/list-files/index.mdx | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index c79b106a548..e3166721029 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -113,13 +113,12 @@ This places each item's data inside a special `__data` key. When objects are organized using a prefix you can return only those objects within the `path` you've provided, and not those within nested subpaths. -For example, given the following keys in your `path` you may want to return only the jpg objects, and not the "vacation" subpath: - - -** `photos/photo1.jpg` -** `photos/photo2.jpg` -** `photos/vacation/` +For example, given the following keys in your `path` you may want to return only the jpg object, and not the "vacation" subpath and its contents: +``` +photos/photo1.jpg +photos/vacation/` +``` This can be accomplished with the `subpathStrategy` option: @@ -132,10 +131,29 @@ const result = await list({ }); ``` +The response will include only the objects within the `photos/` path and will also communicate any excluded subpaths: + +```js +{ + excludedSubpaths: [ + 'photos/vacation/' + ], + items: [ + { + path: "photos/photo1.jpg", + eTag: "30074401292215403a42b0739f3b5262", + lastModified: "Thu Oct 08 2020 23:59:31 GMT+0800 (Singapore Standard Time)", + size: 138256 + }, + ] +} +``` + The default delimiter character is '/', but this can be changed by supplying a custom delimiter: ```js const result = await list({ + // Path uses '-' character to organize files rather than '/' path: "photos-", options:{ subpathStrategy: { From 515181c9699b206035cd9f6957547fec9e9571ac Mon Sep 17 00:00:00 2001 From: haverchuck Date: Tue, 2 Jul 2024 13:15:28 -0700 Subject: [PATCH 04/13] fix: tried to delineate between recursive exploration of entire path and subpathStrategy option --- .../build-a-backend/storage/list-files/index.mdx | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index e3166721029..5b92f61382c 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -29,7 +29,7 @@ export function getStaticProps(context) { }; } -You can list files without having to download all the files. You can do this by using the listFile API from the Amplify Library for Storage. You can also get properties individually for a file using the getProperties API. +You can list files without having to download all the files. You can do this by using the `list` API from the Amplify Library for Storage. You can also get properties individually for a file using the getProperties API. ## List Files @@ -65,7 +65,13 @@ The format of the response will look similar to the below example: } ``` {/* in other files we're referring to paths instead of folders, can we be consistent on terminology? */} -Manually created folders will show up as files with a `size` of 0, but you can also match keys against a regex like `file.key.match(/\.[0-9a-z]+$/i)` to distinguish files from folders. Since "folders" are a virtual concept in Amazon S3, any file may declare any depth of folder just by having a `/` in its name. If you need to list all the folders, you'll have to parse them accordingly to get an authoritative list of files and folders: +Manually created folders will show up as files with a `size` of 0, but you can also match keys against a regex like `file.key.match(/\.[0-9a-z]+$/i)` to distinguish files from folders. Since "folders" are a virtual concept in Amazon S3, any file may declare any depth of folder just by having a `/` in its name. + +You can explore the contents and subpaths of each "folder" by either requesting the contents of the entire path and parsing it, or by using the `subpathStrategy` option to limit your query to the path's file contents only. + +### Explore entire contents of a path + +If you need to list all the contents folders, you'll have to parse them accordingly to get an authoritative list of files and folders: ```js function processStorageList(response) { @@ -109,9 +115,9 @@ function processStorageList(response) { This places each item's data inside a special `__data` key. -### Exclude Subpaths +### Excluding subpaths -When objects are organized using a prefix you can return only those objects within the `path` you've provided, and not those within nested subpaths. +In addition to using the `list` API to explore the complete contents of a path, you can also use it to explore the immediate contents of a path while excluding additional subpaths. For example, given the following keys in your `path` you may want to return only the jpg object, and not the "vacation" subpath and its contents: From 70eb98d376d77eee18cb75b0de64e17779b4bee1 Mon Sep 17 00:00:00 2001 From: Dustin Noyes Date: Tue, 2 Jul 2024 14:06:05 -0700 Subject: [PATCH 05/13] Update src/pages/[platform]/build-a-backend/storage/list-files/index.mdx Co-authored-by: Harshita Daddala --- .../[platform]/build-a-backend/storage/list-files/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 5b92f61382c..a02778cbe5b 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -71,7 +71,7 @@ You can explore the contents and subpaths of each "folder" by either requesting ### Explore entire contents of a path -If you need to list all the contents folders, you'll have to parse them accordingly to get an authoritative list of files and folders: +This retrieves all files and folders under a given path. You may need to parse the result to get only the files within the specified path. ```js function processStorageList(response) { From a42836dcc9e9acf6d6f192bb46ca16e059e14c32 Mon Sep 17 00:00:00 2001 From: Dustin Noyes Date: Tue, 2 Jul 2024 14:06:13 -0700 Subject: [PATCH 06/13] Update src/pages/[platform]/build-a-backend/storage/list-files/index.mdx Co-authored-by: Harshita Daddala --- .../[platform]/build-a-backend/storage/list-files/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index a02778cbe5b..1fb11cae1b0 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -117,7 +117,7 @@ This places each item's data inside a special `__data` key. ### Excluding subpaths -In addition to using the `list` API to explore the complete contents of a path, you can also use it to explore the immediate contents of a path while excluding additional subpaths. +In addition to using the `list` API to get all the contents of a path, you can also use it to get only the files within a path while excluding files under subpaths. For example, given the following keys in your `path` you may want to return only the jpg object, and not the "vacation" subpath and its contents: From 31d99710d4d7148bdc1d3513d3c21381a97f593e Mon Sep 17 00:00:00 2001 From: Dustin Noyes Date: Tue, 2 Jul 2024 14:06:24 -0700 Subject: [PATCH 07/13] Update src/pages/[platform]/build-a-backend/storage/list-files/index.mdx Co-authored-by: Harshita Daddala --- .../[platform]/build-a-backend/storage/list-files/index.mdx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 1fb11cae1b0..72eb3597d49 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -67,7 +67,10 @@ The format of the response will look similar to the below example: {/* in other files we're referring to paths instead of folders, can we be consistent on terminology? */} Manually created folders will show up as files with a `size` of 0, but you can also match keys against a regex like `file.key.match(/\.[0-9a-z]+$/i)` to distinguish files from folders. Since "folders" are a virtual concept in Amazon S3, any file may declare any depth of folder just by having a `/` in its name. -You can explore the contents and subpaths of each "folder" by either requesting the contents of the entire path and parsing it, or by using the `subpathStrategy` option to limit your query to the path's file contents only. +To access the contents and subpaths of a "folder", you have two options: + +1. Request the entire path and parse the contents. +2. Use the subpathStrategy option to retrieve only the files within the specified path (i.e. exclude files under subpaths). ### Explore entire contents of a path From c4fcf49b1743d2b7535187ed31bdbe303f481dd7 Mon Sep 17 00:00:00 2001 From: Dustin Noyes Date: Tue, 2 Jul 2024 14:06:30 -0700 Subject: [PATCH 08/13] Update src/pages/[platform]/build-a-backend/storage/list-files/index.mdx Co-authored-by: Harshita Daddala --- .../[platform]/build-a-backend/storage/list-files/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 72eb3597d49..6f37d72dfd1 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -72,7 +72,7 @@ To access the contents and subpaths of a "folder", you have two options: 1. Request the entire path and parse the contents. 2. Use the subpathStrategy option to retrieve only the files within the specified path (i.e. exclude files under subpaths). -### Explore entire contents of a path +### Get all nested files within a path This retrieves all files and folders under a given path. You may need to parse the result to get only the files within the specified path. From 058653f65108fb9cd7917a5ef074b4da06f6a844 Mon Sep 17 00:00:00 2001 From: Dustin Noyes Date: Tue, 2 Jul 2024 14:06:37 -0700 Subject: [PATCH 09/13] Update src/pages/[platform]/build-a-backend/storage/list-files/index.mdx Co-authored-by: ashika112 <155593080+ashika112@users.noreply.github.com> --- .../[platform]/build-a-backend/storage/list-files/index.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 6f37d72dfd1..1d5a0628140 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -126,7 +126,7 @@ For example, given the following keys in your `path` you may want to return only ``` photos/photo1.jpg -photos/vacation/` +photos/vacation/ ``` This can be accomplished with the `subpathStrategy` option: From 8dc83cfe79a2d7366a23eb8a2c4b24b065160389 Mon Sep 17 00:00:00 2001 From: Dustin Noyes Date: Tue, 9 Jul 2024 07:37:33 -0700 Subject: [PATCH 10/13] Update src/pages/[platform]/build-a-backend/storage/list-files/index.mdx Co-authored-by: josef --- .../[platform]/build-a-backend/storage/list-files/index.mdx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 1d5a0628140..4646fefae35 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -131,7 +131,8 @@ photos/vacation/ This can be accomplished with the `subpathStrategy` option: -```js +```ts title="src/main.ts" +import { list } from "aws-amplify/storage"; const result = await list({ path: "photos/", options:{ From 7eabdd6ba62f16663b8a70a95ef208a5b680650a Mon Sep 17 00:00:00 2001 From: haverchuck Date: Tue, 9 Jul 2024 07:43:10 -0700 Subject: [PATCH 11/13] comments and prettier --- .../storage/list-files/index.mdx | 60 +++++++++---------- 1 file changed, 28 insertions(+), 32 deletions(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 4646fefae35..4cb17d4ce4e 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -37,15 +37,9 @@ You can list files without having to download all the files. You can do this by ```javascript import { list } from 'aws-amplify/storage'; -try { - const result = await list({ - path: 'photos/', - // Alternatively, path: ({identityId}) => `album/{identityId}/photos/` - }); -} catch (error) { - console.log(error); -} -``` +try { const result = await list({ path: 'photos/', // Alternatively, path: ({identityId}) => `album/{identityId}/photos/` }); } catch (error) { console.log(error); } + +```` Note the trailing slash `/` - if you had requested `list({ path : 'photos' })` it would also match against files like `photos123.jpg` alongside `photos/123.jpg`. @@ -63,9 +57,9 @@ The format of the response will look similar to the below example: // ... ], } -``` -{/* in other files we're referring to paths instead of folders, can we be consistent on terminology? */} -Manually created folders will show up as files with a `size` of 0, but you can also match keys against a regex like `file.key.match(/\.[0-9a-z]+$/i)` to distinguish files from folders. Since "folders" are a virtual concept in Amazon S3, any file may declare any depth of folder just by having a `/` in its name. +```` + +{/* in other files we're referring to paths instead of folders, can we be consistent on terminology? */} Manually created folders will show up as files with a `size` of 0, but you can also match keys against a regex like `file.key.match(/\.[0-9a-z]+$/i)` to distinguish files from folders. Since "folders" are a virtual concept in Amazon S3, any file may declare any depth of folder just by having a `/` in its name. To access the contents and subpaths of a "folder", you have two options: @@ -161,13 +155,13 @@ The response will include only the objects within the `photos/` path and will al The default delimiter character is '/', but this can be changed by supplying a custom delimiter: -```js -const result = await list({ +```ts title="src/main.ts" +const result = await list({ // Path uses '-' character to organize files rather than '/' - path: "photos-", - options:{ - subpathStrategy: { - strategy:'exclude', + path: 'photos-', + options: { + subpathStrategy: { + strategy: 'exclude', delimiter: '-' } } @@ -176,8 +170,8 @@ const result = await list({ ### More `list` options -Option | Type | Description | -| -- | -- | ----------- | +| Option | Type | Description | +| --- | --- | --- | | listAll | boolean | Set to true to list all files within the specified `path` | | pageSize | number | Sets the maximum number of files to be return. The range is 0 - 1000 | | nextToken | string | Indicates whether the list is being continued on this bucket with a token | @@ -307,7 +301,7 @@ You can list all of the objects uploaded under a given path by setting the `page ```swift let options = StorageListRequest.Options(pageSize: 1000) let listResult = try await Amplify.Storage.list( - path: .fromString("public/example/path"), + path: .fromString("public/example/path"), options: options ) listResult.items.forEach { item in @@ -323,7 +317,7 @@ listResult.items.forEach { item in let sink = Amplify.Publisher.create { let options = StorageListRequest.Options(pageSize: 1000) try await Amplify.Storage.list( - path: .fromString("public/example/path"), + path: .fromString("public/example/path"), options: options ) }.sink { @@ -345,18 +339,19 @@ receiveValue: { listResult in ### More `list` options -Option | Type | Description | -| -- | -- | ----------- | +| Option | Type | Description | +| --- | --- | --- | | pageSize | UInt | Number between 1 and 1,000 that indicates the limit of how many entries to fetch when retrieving file lists from the server | | nextToken | String | String indicating the page offset at which to resume a listing. | - + This will list all files located under path `album` that: -* have `private` access level -* are in the root of `album/` (the result doesn't include files under any sub path) + +- have `private` access level +- are in the root of `album/` (the result doesn't include files under any sub path) ```dart Future listAlbum() async { @@ -412,8 +407,8 @@ Future listAllUnderPublicPath() async { ### More `list` options -Option | Type | Description | -| -- | -- | ----------- | +| Option | Type | Description | +| --- | --- | --- | | excludeSubPaths | boolean | Whether to exclude objects under the sub paths of the path to list. Defaults to false. | | delimiter | String | The delimiter to use when evaluating sub paths. If excludeSubPaths is false, this value has no impact on behavior. | @@ -430,7 +425,7 @@ import { getProperties } from 'aws-amplify/storage'; try { const result = await getProperties({ - path: "album/2024/1.jpg", + path: 'album/2024/1.jpg' // Alternatively, path: ({ identityId }) => `album/{identityId}/1.jpg` }); console.log('File Properties ', result); @@ -438,6 +433,7 @@ try { console.log('Error ', error); } ``` + The properties and metadata will look similar to the below example ```js @@ -453,8 +449,8 @@ The properties and metadata will look similar to the below example ### More `getProperties` options -Option | Type | Description | -| -- | -- | ----------- | +| Option | Type | Description | +| --------------------- | ------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | | useAccelerateEndpoint | boolean | Whether to use accelerate endpoint. | [Transfer Acceleration](/[platform]/build-a-backend/storage/extend-s3-resources/#example---enable-transfer-acceleration) | From a2cadec883a38dbe1d7125ec065dcab8ac2b4564 Mon Sep 17 00:00:00 2001 From: haverchuck Date: Tue, 9 Jul 2024 08:05:07 -0700 Subject: [PATCH 12/13] undid prettier table change --- .../[platform]/build-a-backend/storage/list-files/index.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 4cb17d4ce4e..26da1ebf15f 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -449,8 +449,8 @@ The properties and metadata will look similar to the below example ### More `getProperties` options -| Option | Type | Description | -| --------------------- | ------- | ----------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | +Option | Type | Description | +| -- | -- | ----------- | | useAccelerateEndpoint | boolean | Whether to use accelerate endpoint. | [Transfer Acceleration](/[platform]/build-a-backend/storage/extend-s3-resources/#example---enable-transfer-acceleration) | From baf7af6b8b08c5d3dd7cb07fe76ed22ba558936f Mon Sep 17 00:00:00 2001 From: Dustin Noyes Date: Tue, 9 Jul 2024 08:30:29 -0700 Subject: [PATCH 13/13] Update src/pages/[platform]/build-a-backend/storage/list-files/index.mdx Co-authored-by: josef --- .../build-a-backend/storage/list-files/index.mdx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx index 26da1ebf15f..3eb8a2613a3 100644 --- a/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx +++ b/src/pages/[platform]/build-a-backend/storage/list-files/index.mdx @@ -37,9 +37,11 @@ You can list files without having to download all the files. You can do this by ```javascript import { list } from 'aws-amplify/storage'; -try { const result = await list({ path: 'photos/', // Alternatively, path: ({identityId}) => `album/{identityId}/photos/` }); } catch (error) { console.log(error); } - -```` +const result = await list({ + path: 'photos/', + // Alternatively, path: ({identityId}) => `album/{identityId}/photos/` +}); +``` Note the trailing slash `/` - if you had requested `list({ path : 'photos' })` it would also match against files like `photos123.jpg` alongside `photos/123.jpg`.