From 62243262f2d0a6132ea427892c9ea2a9b24e7e31 Mon Sep 17 00:00:00 2001 From: Allan Zheng Date: Thu, 6 Jun 2024 17:06:58 -0700 Subject: [PATCH 01/40] chore: enable storage-browser preid release --- .github/workflows/push-preid-release.yml | 3 +-- package.json | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push-preid-release.yml b/.github/workflows/push-preid-release.yml index 9837290ed14..d4867df08d0 100644 --- a/.github/workflows/push-preid-release.yml +++ b/.github/workflows/push-preid-release.yml @@ -8,8 +8,7 @@ concurrency: on: push: branches: - # Change this to your branch name where "example-preid" corresponds to the preid you want your changes released on - - feat/example-preid-branch/main + - storage-browser/main jobs: e2e: diff --git a/package.json b/package.json index 7970a9f5d26..a3fba5f6d4f 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "publish:release": "lerna publish --conventional-commits --message 'chore(release): Publish [skip release]' --yes", "publish:v5-stable": "lerna publish --conventional-commits --yes --dist-tag=stable-5 --message 'chore(release): Publish [ci skip]' --no-verify-access", "publish:verdaccio": "lerna publish --canary --force-publish --no-push --dist-tag=unstable --preid=unstable --yes", + "publish:storage-browser/main": "lerna publish --canary --force-publish --dist-tag=storage-browser --preid=storage-browser --yes", "ts-coverage": "lerna run ts-coverage", "prepare": "husky && ./scripts/set-preid-versions.sh" }, From 092e25c3e1427cfb82e32eec6fb2f6795f81c474 Mon Sep 17 00:00:00 2001 From: israx <70438514+israx@users.noreply.github.com> Date: Fri, 7 Jun 2024 11:55:24 -0400 Subject: [PATCH 02/40] chore: sync main (#13478) * release(required): Parsing custom oAuth in amplify_outputs (#13474) * update parseAmplify logic * revert custom oAuth from gen1 config * update bundle size * chore(release): Publish [skip release] - @aws-amplify/adapter-nextjs@1.2.3 - @aws-amplify/analytics@7.0.35 - @aws-amplify/api@6.0.37 - @aws-amplify/api-graphql@4.1.6 - @aws-amplify/api-rest@4.0.35 - @aws-amplify/auth@6.3.5 - aws-amplify@6.3.6 - @aws-amplify/core@6.3.2 - @aws-amplify/datastore@5.0.37 - @aws-amplify/datastore-storage-adapter@2.1.37 - @aws-amplify/geo@3.0.35 - @aws-amplify/interactions@6.0.34 - @aws-amplify/notifications@2.0.35 - @aws-amplify/predictions@6.1.10 - @aws-amplify/pubsub@6.1.9 - @aws-amplify/storage@6.4.6 - tsc-compliance-test@0.1.39 * chore(release): update API docs [skip release] --------- Co-authored-by: ashika112 <155593080+ashika112@users.noreply.github.com> Co-authored-by: aws-amplify-bot Co-authored-by: Jim Blanchard --- .../modules/_aws_amplify_adapter_nextjs.html | 4 ++-- ...aws_amplify_datastore_storage_adapter.html | 4 ++-- docs/api/modules/_aws_amplify_geo.html | 4 ++-- .../modules/_aws_amplify_interactions.html | 4 ++-- .../api/modules/_aws_amplify_predictions.html | 4 ++-- docs/api/modules/_aws_amplify_pubsub.html | 4 ++-- docs/api/modules/aws_amplify.html | 4 ++-- packages/adapter-nextjs/CHANGELOG.md | 4 ++++ packages/adapter-nextjs/package.json | 4 ++-- packages/analytics/CHANGELOG.md | 4 ++++ packages/analytics/package.json | 4 ++-- packages/api-graphql/CHANGELOG.md | 4 ++++ packages/api-graphql/package.json | 6 ++--- packages/api-rest/CHANGELOG.md | 4 ++++ packages/api-rest/package.json | 4 ++-- packages/api/CHANGELOG.md | 4 ++++ packages/api/package.json | 6 ++--- packages/auth/CHANGELOG.md | 4 ++++ packages/auth/package.json | 4 ++-- packages/aws-amplify/CHANGELOG.md | 4 ++++ packages/aws-amplify/package.json | 24 +++++++++---------- packages/core/CHANGELOG.md | 4 ++++ .../configMocks/amplify_outputs.json | 2 +- .../configMocks/amplifyconfiguration.json | 18 +++++--------- .../core/__tests__/parseAWSExports.test.ts | 18 +++++++++++--- packages/core/package.json | 2 +- packages/core/src/parseAmplifyOutputs.ts | 8 ++++++- .../datastore-storage-adapter/CHANGELOG.md | 4 ++++ .../datastore-storage-adapter/package.json | 6 ++--- packages/datastore/CHANGELOG.md | 4 ++++ packages/datastore/package.json | 6 ++--- packages/geo/CHANGELOG.md | 4 ++++ packages/geo/package.json | 4 ++-- packages/interactions/CHANGELOG.md | 4 ++++ packages/interactions/package.json | 4 ++-- packages/notifications/CHANGELOG.md | 4 ++++ packages/notifications/package.json | 4 ++-- packages/predictions/CHANGELOG.md | 4 ++++ packages/predictions/package.json | 6 ++--- packages/pubsub/CHANGELOG.md | 4 ++++ packages/pubsub/package.json | 6 ++--- packages/storage/CHANGELOG.md | 4 ++++ packages/storage/package.json | 4 ++-- scripts/tsc-compliance-test/CHANGELOG.md | 4 ++++ scripts/tsc-compliance-test/package.json | 4 ++-- 45 files changed, 160 insertions(+), 80 deletions(-) diff --git a/docs/api/modules/_aws_amplify_adapter_nextjs.html b/docs/api/modules/_aws_amplify_adapter_nextjs.html index f95e9bd7d4e..2b30637060c 100644 --- a/docs/api/modules/_aws_amplify_adapter_nextjs.html +++ b/docs/api/modules/_aws_amplify_adapter_nextjs.html @@ -1,5 +1,5 @@ -@aws-amplify/adapter-nextjs - v1.2.2 | Amplify JS API Documentation -

Module @aws-amplify/adapter-nextjs - v1.2.2

This package contains the AWS Amplify Next.js Adapter. For more information on using Next.js in your application please reference the Amplify Dev Center.

+@aws-amplify/adapter-nextjs - v1.2.3 | Amplify JS API Documentation +

Module @aws-amplify/adapter-nextjs - v1.2.3

This package contains the AWS Amplify Next.js Adapter. For more information on using Next.js in your application please reference the Amplify Dev Center.

Index

Modules

api index utils diff --git a/docs/api/modules/_aws_amplify_datastore_storage_adapter.html b/docs/api/modules/_aws_amplify_datastore_storage_adapter.html index ec5be85c921..3965fa02687 100644 --- a/docs/api/modules/_aws_amplify_datastore_storage_adapter.html +++ b/docs/api/modules/_aws_amplify_datastore_storage_adapter.html @@ -1,5 +1,5 @@ -@aws-amplify/datastore-storage-adapter - v2.1.36 | Amplify JS API Documentation -

Module @aws-amplify/datastore-storage-adapter - v2.1.36

This package contains the AWS Amplify DataStore storage adapter. For more information on using the DataStore storage adapter in your application please reference the Amplify Dev Center.

+@aws-amplify/datastore-storage-adapter - v2.1.37 | Amplify JS API Documentation +

Module @aws-amplify/datastore-storage-adapter - v2.1.37

This package contains the AWS Amplify DataStore storage adapter. For more information on using the DataStore storage adapter in your application please reference the Amplify Dev Center.

Index

Modules

ExpoSQLiteAdapter/ExpoSQLiteAdapter SQLiteAdapter/SQLiteAdapter index diff --git a/docs/api/modules/_aws_amplify_geo.html b/docs/api/modules/_aws_amplify_geo.html index 745cd4b51c8..9870ca9910f 100644 --- a/docs/api/modules/_aws_amplify_geo.html +++ b/docs/api/modules/_aws_amplify_geo.html @@ -1,5 +1,5 @@ -@aws-amplify/geo - v3.0.34 | Amplify JS API Documentation -

Module @aws-amplify/geo - v3.0.34

This package contains the AWS Amplify Geo category. For more information on using Geo in your application please reference the Amplify Dev Center.

+@aws-amplify/geo - v3.0.35 | Amplify JS API Documentation +

Module @aws-amplify/geo - v3.0.35

This package contains the AWS Amplify Geo category. For more information on using Geo in your application please reference the Amplify Dev Center.

Index

Modules

index providers/location-service/AmazonLocationServiceProvider

Settings

Member Visibility

Theme

\ No newline at end of file diff --git a/docs/api/modules/_aws_amplify_interactions.html b/docs/api/modules/_aws_amplify_interactions.html index b750f7eebde..ecee0693c04 100644 --- a/docs/api/modules/_aws_amplify_interactions.html +++ b/docs/api/modules/_aws_amplify_interactions.html @@ -1,5 +1,5 @@ -@aws-amplify/interactions - v6.0.33 | Amplify JS API Documentation -

Module @aws-amplify/interactions - v6.0.33

This package contains the AWS Amplify Interactions category. For more information on using Interactions in your application please reference the Amplify Dev Center.

+@aws-amplify/interactions - v6.0.34 | Amplify JS API Documentation +

Module @aws-amplify/interactions - v6.0.34

This package contains the AWS Amplify Interactions category. For more information on using Interactions in your application please reference the Amplify Dev Center.

Index

Modules

index lex-v1 lex-v2 diff --git a/docs/api/modules/_aws_amplify_predictions.html b/docs/api/modules/_aws_amplify_predictions.html index 4415e3a0262..92bb635614c 100644 --- a/docs/api/modules/_aws_amplify_predictions.html +++ b/docs/api/modules/_aws_amplify_predictions.html @@ -1,5 +1,5 @@ -@aws-amplify/predictions - v6.1.9 | Amplify JS API Documentation -

Module @aws-amplify/predictions - v6.1.9

This package contains the AWS Amplify Predictions category. For more information on using Predictions in your application please reference the Amplify Dev Center.

+@aws-amplify/predictions - v6.1.10 | Amplify JS API Documentation +

Module @aws-amplify/predictions - v6.1.10

This package contains the AWS Amplify Predictions category. For more information on using Predictions in your application please reference the Amplify Dev Center.

Index

Modules

<Reference Types>

Interfaces

IdentifyEntitiesInput IdentifyEntitiesOutput diff --git a/docs/api/modules/_aws_amplify_pubsub.html b/docs/api/modules/_aws_amplify_pubsub.html index a2b6a608cf2..c176df20e1e 100644 --- a/docs/api/modules/_aws_amplify_pubsub.html +++ b/docs/api/modules/_aws_amplify_pubsub.html @@ -1,5 +1,5 @@ -@aws-amplify/pubsub - v6.1.8 | Amplify JS API Documentation -

Module @aws-amplify/pubsub - v6.1.8

This package contains the AWS Amplify PubSub category. For more information on using PubSub in your application please reference the Amplify Dev Center.

+@aws-amplify/pubsub - v6.1.9 | Amplify JS API Documentation +

Module @aws-amplify/pubsub - v6.1.9

This package contains the AWS Amplify PubSub category. For more information on using PubSub in your application please reference the Amplify Dev Center.

Index

Modules

clients/iot clients/mqtt index diff --git a/docs/api/modules/aws_amplify.html b/docs/api/modules/aws_amplify.html index 5b8769b3028..f05f4051a97 100644 --- a/docs/api/modules/aws_amplify.html +++ b/docs/api/modules/aws_amplify.html @@ -1,5 +1,5 @@ -aws-amplify - v6.3.5 | Amplify JS API Documentation -

Module aws-amplify - v6.3.5

AWS Amplify Package - aws-amplify

AWS Amplify is a JavaScript library for frontend and mobile developers building cloud-enabled applications. The library is a declarative interface across different categories of operations in order to make common tasks easier to add into your application. The default implementation works with Amazon Web Services (AWS) resources but is designed to be open and pluggable for usage with other cloud services that wish to provide an implementation or custom backends.

+aws-amplify - v6.3.6 | Amplify JS API Documentation +

Module aws-amplify - v6.3.6

AWS Amplify Package - aws-amplify

AWS Amplify is a JavaScript library for frontend and mobile developers building cloud-enabled applications. The library is a declarative interface across different categories of operations in order to make common tasks easier to add into your application. The default implementation works with Amazon Web Services (AWS) resources but is designed to be open and pluggable for usage with other cloud services that wish to provide an implementation or custom backends.

Documentation is available here.

Index

Modules

adapter-core analytics diff --git a/packages/adapter-nextjs/CHANGELOG.md b/packages/adapter-nextjs/CHANGELOG.md index 697fe57f670..6926211af83 100644 --- a/packages/adapter-nextjs/CHANGELOG.md +++ b/packages/adapter-nextjs/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [1.2.3](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/adapter-nextjs@1.2.2...@aws-amplify/adapter-nextjs@1.2.3) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/adapter-nextjs + ## [1.2.2](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/adapter-nextjs@1.2.1...@aws-amplify/adapter-nextjs@1.2.2) (2024-06-04) **Note:** Version bump only for package @aws-amplify/adapter-nextjs diff --git a/packages/adapter-nextjs/package.json b/packages/adapter-nextjs/package.json index 60992a73f7c..eff0c1113f6 100644 --- a/packages/adapter-nextjs/package.json +++ b/packages/adapter-nextjs/package.json @@ -1,7 +1,7 @@ { "author": "Amazon Web Services", "name": "@aws-amplify/adapter-nextjs", - "version": "1.2.2", + "version": "1.2.3", "description": "The adapter for the supporting of using Amplify APIs in Next.js.", "peerDependencies": { "aws-amplify": "^6.0.7", @@ -16,7 +16,7 @@ "@types/node": "^20.3.1", "@types/react": "^18.2.13", "@types/react-dom": "^18.2.6", - "aws-amplify": "6.3.5", + "aws-amplify": "6.3.6", "jest-fetch-mock": "3.0.3", "next": ">= 13.5.0 < 15.0.0", "typescript": "5.0.2" diff --git a/packages/analytics/CHANGELOG.md b/packages/analytics/CHANGELOG.md index ccb96ae1680..2581738efb7 100644 --- a/packages/analytics/CHANGELOG.md +++ b/packages/analytics/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [7.0.35](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/analytics@7.0.34...@aws-amplify/analytics@7.0.35) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/analytics + ## [7.0.34](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/analytics@7.0.33...@aws-amplify/analytics@7.0.34) (2024-06-04) **Note:** Version bump only for package @aws-amplify/analytics diff --git a/packages/analytics/package.json b/packages/analytics/package.json index f473bd81819..d9abe077155 100644 --- a/packages/analytics/package.json +++ b/packages/analytics/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/analytics", - "version": "7.0.34", + "version": "7.0.35", "description": "Analytics category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -103,7 +103,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "@aws-amplify/react-native": "1.1.1", "@aws-sdk/types": "3.398.0", "typescript": "5.0.2" diff --git a/packages/api-graphql/CHANGELOG.md b/packages/api-graphql/CHANGELOG.md index c42fbb70648..123719d7bd6 100644 --- a/packages/api-graphql/CHANGELOG.md +++ b/packages/api-graphql/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [4.1.6](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/api-graphql@4.1.5...@aws-amplify/api-graphql@4.1.6) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/api-graphql + ## [4.1.5](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/api-graphql@4.1.4...@aws-amplify/api-graphql@4.1.5) (2024-06-04) **Note:** Version bump only for package @aws-amplify/api-graphql diff --git a/packages/api-graphql/package.json b/packages/api-graphql/package.json index 2df9e837e92..8b225a92e49 100644 --- a/packages/api-graphql/package.json +++ b/packages/api-graphql/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/api-graphql", - "version": "4.1.5", + "version": "4.1.6", "description": "Api-graphql category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -84,8 +84,8 @@ "server" ], "dependencies": { - "@aws-amplify/api-rest": "4.0.34", - "@aws-amplify/core": "6.3.1", + "@aws-amplify/api-rest": "4.0.35", + "@aws-amplify/core": "6.3.2", "@aws-amplify/data-schema": "^1.0.0", "@aws-sdk/types": "3.387.0", "graphql": "15.8.0", diff --git a/packages/api-rest/CHANGELOG.md b/packages/api-rest/CHANGELOG.md index d29aece3370..6201cb6d06e 100644 --- a/packages/api-rest/CHANGELOG.md +++ b/packages/api-rest/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [4.0.35](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/api-rest@4.0.34...@aws-amplify/api-rest@4.0.35) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/api-rest + ## [4.0.34](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/api-rest@4.0.33...@aws-amplify/api-rest@4.0.34) (2024-06-04) **Note:** Version bump only for package @aws-amplify/api-rest diff --git a/packages/api-rest/package.json b/packages/api-rest/package.json index 933cb212408..470aca7ecf6 100644 --- a/packages/api-rest/package.json +++ b/packages/api-rest/package.json @@ -1,7 +1,7 @@ { "name": "@aws-amplify/api-rest", "private": false, - "version": "4.0.34", + "version": "4.0.35", "description": "Api-rest category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -87,7 +87,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "@aws-amplify/react-native": "1.1.1", "typescript": "5.0.2" }, diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index a822fb8525b..27fa15b9548 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.0.37](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/api@6.0.36...@aws-amplify/api@6.0.37) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/api + ## [6.0.36](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/api@6.0.35...@aws-amplify/api@6.0.36) (2024-06-04) **Note:** Version bump only for package @aws-amplify/api diff --git a/packages/api/package.json b/packages/api/package.json index d0e08aaf1ee..6aa55513f11 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/api", - "version": "6.0.36", + "version": "6.0.37", "description": "Api category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -79,8 +79,8 @@ "server" ], "dependencies": { - "@aws-amplify/api-graphql": "4.1.5", - "@aws-amplify/api-rest": "4.0.34", + "@aws-amplify/api-graphql": "4.1.6", + "@aws-amplify/api-rest": "4.0.35", "tslib": "^2.5.0" } } diff --git a/packages/auth/CHANGELOG.md b/packages/auth/CHANGELOG.md index dcb376d0a7c..87fdf81fcbc 100644 --- a/packages/auth/CHANGELOG.md +++ b/packages/auth/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.3.5](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/auth@6.3.4...@aws-amplify/auth@6.3.5) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/auth + ## [6.3.4](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/auth@6.3.3...@aws-amplify/auth@6.3.4) (2024-05-23) **Note:** Version bump only for package @aws-amplify/auth diff --git a/packages/auth/package.json b/packages/auth/package.json index baa4d781006..de95b0653f2 100644 --- a/packages/auth/package.json +++ b/packages/auth/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/auth", - "version": "6.3.4", + "version": "6.3.5", "description": "Auth category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -97,7 +97,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "@aws-amplify/react-native": "1.1.1", "@jest/test-sequencer": "^29.7.0", "typescript": "5.0.2" diff --git a/packages/aws-amplify/CHANGELOG.md b/packages/aws-amplify/CHANGELOG.md index 8af708e40c0..9ea9429d19f 100644 --- a/packages/aws-amplify/CHANGELOG.md +++ b/packages/aws-amplify/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.3.6](https://github.com/aws-amplify/amplify-js/compare/aws-amplify@6.3.5...aws-amplify@6.3.6) (2024-06-07) + +**Note:** Version bump only for package aws-amplify + ## [6.3.5](https://github.com/aws-amplify/amplify-js/compare/aws-amplify@6.3.4...aws-amplify@6.3.5) (2024-06-04) **Note:** Version bump only for package aws-amplify diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 088c00feae9..a3c44a89861 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -1,6 +1,6 @@ { "name": "aws-amplify", - "version": "6.3.5", + "version": "6.3.6", "description": "AWS Amplify is a JavaScript library for Frontend and mobile developers building cloud-enabled applications.", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -276,13 +276,13 @@ "utils" ], "dependencies": { - "@aws-amplify/analytics": "7.0.34", - "@aws-amplify/api": "6.0.36", - "@aws-amplify/auth": "6.3.4", - "@aws-amplify/core": "6.3.1", - "@aws-amplify/datastore": "5.0.36", - "@aws-amplify/notifications": "2.0.34", - "@aws-amplify/storage": "6.4.5", + "@aws-amplify/analytics": "7.0.35", + "@aws-amplify/api": "6.0.37", + "@aws-amplify/auth": "6.3.5", + "@aws-amplify/core": "6.3.2", + "@aws-amplify/datastore": "5.0.37", + "@aws-amplify/notifications": "2.0.35", + "@aws-amplify/storage": "6.4.6", "tslib": "^2.5.0" }, "devDependencies": { @@ -293,7 +293,7 @@ "name": "[Analytics] record (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ record }", - "limit": "17.08 kB" + "limit": "17.09 kB" }, { "name": "[Analytics] record (Kinesis)", @@ -317,7 +317,7 @@ "name": "[Analytics] identifyUser (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ identifyUser }", - "limit": "15.57 kB" + "limit": "15.59 kB" }, { "name": "[Analytics] enable", @@ -383,7 +383,7 @@ "name": "[Auth] confirmSignIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignIn }", - "limit": "28.26 kB" + "limit": "28.27 kB" }, { "name": "[Auth] updateMFAPreference (Cognito)", @@ -401,7 +401,7 @@ "name": "[Auth] verifyTOTPSetup (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ verifyTOTPSetup }", - "limit": "12.59 kB" + "limit": "12.6 kB" }, { "name": "[Auth] updatePassword (Cognito)", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index 0cbcb99a260..79488956493 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.3.2](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/core@6.3.1...@aws-amplify/core@6.3.2) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/core + ## [6.3.1](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/core@6.3.0...@aws-amplify/core@6.3.1) (2024-05-23) **Note:** Version bump only for package @aws-amplify/core diff --git a/packages/core/__mocks__/configMocks/amplify_outputs.json b/packages/core/__mocks__/configMocks/amplify_outputs.json index 432a6247e91..54269112f17 100644 --- a/packages/core/__mocks__/configMocks/amplify_outputs.json +++ b/packages/core/__mocks__/configMocks/amplify_outputs.json @@ -7,7 +7,7 @@ "user_pool_client_id": "mock-cup-client-id", "identity_pool_id": "mock-idp-id", "oauth": { - "identity_providers": ["FACEBOOK", "SIGN_IN_WITH_APPLE", "GOOGLE"], + "identity_providers": ["FACEBOOK", "SIGN_IN_WITH_APPLE", "GOOGLE", "Auth0"], "domain": "mock-oauth-domain", "scopes": ["phone"], "redirect_sign_in_uri": ["mock-sign-in-uri"], diff --git a/packages/core/__mocks__/configMocks/amplifyconfiguration.json b/packages/core/__mocks__/configMocks/amplifyconfiguration.json index 452ab0c64f0..d32597e9bc5 100644 --- a/packages/core/__mocks__/configMocks/amplifyconfiguration.json +++ b/packages/core/__mocks__/configMocks/amplifyconfiguration.json @@ -29,31 +29,25 @@ "aws_user_files_s3_bucket_region": "us-west-2", "aws_user_pools_id": "mock-cup-id", "aws_user_pools_web_client_id": "mock-cup-client-id", - "geo": { + "geo": { "amazon_location_service": { "search_indices": { - "items": [ - "mock-geo-search-item", - "mock-geo-search-item-alt" - ], + "items": ["mock-geo-search-item", "mock-geo-search-item-alt"], "default": "mock-geo-search-item" }, "geofenceCollections": { - "items": [ - "mock-geo-fence-item", - "mock-geo-fence-item-alt" - ], + "items": ["mock-geo-fence-item", "mock-geo-fence-item-alt"], "default": "mock-geo-fence-item" }, "region": "us-west-2" - } + } }, "aws_appsync_graphqlEndpoint": "mock-data-url", "aws_appsync_apiKey": "mock-data-api-key", "aws_appsync_region": "us-west-2", "aws_appsync_authenticationType": "API_KEY", "Notifications": { - "InAppMessaging": { + "InAppMessaging": { "AWSPinpoint": { "appId": "mock-pinpoint-app-id", "region": "us-west-2" @@ -66,4 +60,4 @@ } } } -} \ No newline at end of file +} diff --git a/packages/core/__tests__/parseAWSExports.test.ts b/packages/core/__tests__/parseAWSExports.test.ts index 1ad4520f7d5..d021792bd2f 100644 --- a/packages/core/__tests__/parseAWSExports.test.ts +++ b/packages/core/__tests__/parseAWSExports.test.ts @@ -1,5 +1,5 @@ import { parseAWSExports } from '../src/parseAWSExports'; -import { OAuthScope } from '../src/singleton/Auth/types'; +import { OAuthProvider, OAuthScope } from '../src/singleton/Auth/types'; import { ResourcesConfig } from '../src/singleton/types'; // TODO: Add API category tests @@ -91,7 +91,13 @@ describe('parseAWSExports', () => { email: false, oauth: { domain: oAuthDomain, - providers: ['Google', 'Apple', 'Facebook', 'Amazon'], + providers: [ + 'Google', + 'Apple', + 'Facebook', + 'Amazon', + 'Auth0', + ] as OAuthProvider[], redirectSignIn: [oAuthSigninUrl], redirectSignOut: [oAuthSignoutUrl], responseType: oAuthResponseType, @@ -172,7 +178,13 @@ describe('parseAWSExports', () => { responseType: oAuthResponseType, }, aws_cognito_verification_mechanisms: ['EMAIL'], - aws_cognito_social_providers: ['GOOGLE', 'APPLE', 'FACEBOOK', 'AMAZON'], + aws_cognito_social_providers: [ + 'GOOGLE', + 'APPLE', + 'FACEBOOK', + 'AMAZON', + 'Auth0', + ], aws_mandatory_sign_in: 'enable', aws_mobile_analytics_app_id: appId, aws_mobile_analytics_app_region: region, diff --git a/packages/core/package.json b/packages/core/package.json index c3ee2ff4a42..4621c987898 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/core", - "version": "6.3.1", + "version": "6.3.2", "description": "Core category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", diff --git a/packages/core/src/parseAmplifyOutputs.ts b/packages/core/src/parseAmplifyOutputs.ts index fcabd1aac06..ed742189266 100644 --- a/packages/core/src/parseAmplifyOutputs.ts +++ b/packages/core/src/parseAmplifyOutputs.ts @@ -316,7 +316,13 @@ const providerNames: Record = { }; function getOAuthProviders(providers: string[] = []): OAuthProvider[] { - return providers.map(provider => providerNames[provider]); + return providers.reduce((oAuthProviders, provider) => { + if (providerNames[provider] !== undefined) { + oAuthProviders.push(providerNames[provider]); + } + + return oAuthProviders; + }, []); } function getMfaStatus( diff --git a/packages/datastore-storage-adapter/CHANGELOG.md b/packages/datastore-storage-adapter/CHANGELOG.md index d17588fcd4c..f7b35dc7a30 100644 --- a/packages/datastore-storage-adapter/CHANGELOG.md +++ b/packages/datastore-storage-adapter/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [2.1.37](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/datastore-storage-adapter@2.1.36...@aws-amplify/datastore-storage-adapter@2.1.37) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/datastore-storage-adapter + ## [2.1.36](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/datastore-storage-adapter@2.1.35...@aws-amplify/datastore-storage-adapter@2.1.36) (2024-06-04) **Note:** Version bump only for package @aws-amplify/datastore-storage-adapter diff --git a/packages/datastore-storage-adapter/package.json b/packages/datastore-storage-adapter/package.json index 8cbfaa22d80..507876a97bc 100644 --- a/packages/datastore-storage-adapter/package.json +++ b/packages/datastore-storage-adapter/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/datastore-storage-adapter", - "version": "2.1.36", + "version": "2.1.37", "description": "SQLite storage adapter for Amplify DataStore ", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -36,8 +36,8 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", - "@aws-amplify/datastore": "5.0.36", + "@aws-amplify/core": "6.3.2", + "@aws-amplify/datastore": "5.0.37", "@types/react-native-sqlite-storage": "5.0.1", "expo-file-system": "13.1.4", "expo-sqlite": "10.1.0", diff --git a/packages/datastore/CHANGELOG.md b/packages/datastore/CHANGELOG.md index 2ab57e74df8..94b2fc06ee8 100644 --- a/packages/datastore/CHANGELOG.md +++ b/packages/datastore/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [5.0.37](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/datastore@5.0.36...@aws-amplify/datastore@5.0.37) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/datastore + ## [5.0.36](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/datastore@5.0.35...@aws-amplify/datastore@5.0.36) (2024-06-04) **Note:** Version bump only for package @aws-amplify/datastore diff --git a/packages/datastore/package.json b/packages/datastore/package.json index 7c5ad0eb9af..d8deb5d9a60 100644 --- a/packages/datastore/package.json +++ b/packages/datastore/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/datastore", - "version": "5.0.36", + "version": "5.0.37", "description": "AppSyncLocal support for aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -44,7 +44,7 @@ "src" ], "dependencies": { - "@aws-amplify/api": "6.0.36", + "@aws-amplify/api": "6.0.37", "buffer": "4.9.2", "idb": "5.0.6", "immer": "9.0.6", @@ -55,7 +55,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "@aws-amplify/react-native": "1.1.1", "@types/uuid-validate": "^0.0.1", "dexie": "3.2.2", diff --git a/packages/geo/CHANGELOG.md b/packages/geo/CHANGELOG.md index 566f54f26c8..21ce83c6457 100644 --- a/packages/geo/CHANGELOG.md +++ b/packages/geo/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [3.0.35](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/geo@3.0.34...@aws-amplify/geo@3.0.35) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/geo + ## [3.0.34](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/geo@3.0.33...@aws-amplify/geo@3.0.34) (2024-06-04) **Note:** Version bump only for package @aws-amplify/geo diff --git a/packages/geo/package.json b/packages/geo/package.json index c1fa991bbb6..ad9c3a3e583 100644 --- a/packages/geo/package.json +++ b/packages/geo/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/geo", - "version": "3.0.34", + "version": "3.0.35", "description": "Geo category for aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -76,7 +76,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "typescript": "5.0.2" }, "size-limit": [ diff --git a/packages/interactions/CHANGELOG.md b/packages/interactions/CHANGELOG.md index 3ae645bcbb5..4b1374637cd 100644 --- a/packages/interactions/CHANGELOG.md +++ b/packages/interactions/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.0.34](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/interactions@6.0.33...@aws-amplify/interactions@6.0.34) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/interactions + ## [6.0.33](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/interactions@6.0.32...@aws-amplify/interactions@6.0.33) (2024-05-23) **Note:** Version bump only for package @aws-amplify/interactions diff --git a/packages/interactions/package.json b/packages/interactions/package.json index 62fd955fc1a..faebbd94fae 100644 --- a/packages/interactions/package.json +++ b/packages/interactions/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/interactions", - "version": "6.0.33", + "version": "6.0.34", "description": "Interactions category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -81,7 +81,7 @@ "uuid": "^9.0.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "typescript": "^5.0.2" }, "size-limit": [ diff --git a/packages/notifications/CHANGELOG.md b/packages/notifications/CHANGELOG.md index 1874c2f26d9..f131a4fd4e0 100644 --- a/packages/notifications/CHANGELOG.md +++ b/packages/notifications/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [2.0.35](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/notifications@2.0.34...@aws-amplify/notifications@2.0.35) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/notifications + ## [2.0.34](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/notifications@2.0.33...@aws-amplify/notifications@2.0.34) (2024-06-04) **Note:** Version bump only for package @aws-amplify/notifications diff --git a/packages/notifications/package.json b/packages/notifications/package.json index b400ee1c704..4b97a5acd46 100644 --- a/packages/notifications/package.json +++ b/packages/notifications/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/notifications", - "version": "2.0.34", + "version": "2.0.35", "description": "Notifications category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -98,7 +98,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "@aws-amplify/react-native": "1.1.1", "typescript": "5.0.2" } diff --git a/packages/predictions/CHANGELOG.md b/packages/predictions/CHANGELOG.md index 71eefbdae5e..f7fdb22b7a6 100644 --- a/packages/predictions/CHANGELOG.md +++ b/packages/predictions/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.1.10](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/predictions@6.1.9...@aws-amplify/predictions@6.1.10) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/predictions + ## [6.1.9](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/predictions@6.1.8...@aws-amplify/predictions@6.1.9) (2024-06-04) **Note:** Version bump only for package @aws-amplify/predictions diff --git a/packages/predictions/package.json b/packages/predictions/package.json index 02458990541..6d642b6fa60 100644 --- a/packages/predictions/package.json +++ b/packages/predictions/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/predictions", - "version": "6.1.9", + "version": "6.1.10", "description": "Machine learning category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -43,7 +43,7 @@ "src" ], "dependencies": { - "@aws-amplify/storage": "6.4.5", + "@aws-amplify/storage": "6.4.6", "@aws-sdk/client-comprehend": "3.398.0", "@aws-sdk/client-polly": "3.398.0", "@aws-sdk/client-rekognition": "3.398.0", @@ -59,7 +59,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "typescript": "5.0.2" }, "size-limit": [ diff --git a/packages/pubsub/CHANGELOG.md b/packages/pubsub/CHANGELOG.md index 01acd740d44..a617997caac 100644 --- a/packages/pubsub/CHANGELOG.md +++ b/packages/pubsub/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.1.9](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/pubsub@6.1.8...@aws-amplify/pubsub@6.1.9) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/pubsub + ## [6.1.8](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/pubsub@6.1.7...@aws-amplify/pubsub@6.1.8) (2024-05-23) **Note:** Version bump only for package @aws-amplify/pubsub diff --git a/packages/pubsub/package.json b/packages/pubsub/package.json index f14ce5f411a..99873590cb6 100644 --- a/packages/pubsub/package.json +++ b/packages/pubsub/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/pubsub", - "version": "6.1.8", + "version": "6.1.9", "description": "Pubsub category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -73,7 +73,7 @@ "mqtt" ], "dependencies": { - "@aws-amplify/auth": "6.3.4", + "@aws-amplify/auth": "6.3.5", "buffer": "4.9.2", "graphql": "15.8.0", "rxjs": "^7.8.1", @@ -84,7 +84,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "typescript": "5.0.2" }, "size-limit": [ diff --git a/packages/storage/CHANGELOG.md b/packages/storage/CHANGELOG.md index c82db9f2f7d..04e7f32592c 100644 --- a/packages/storage/CHANGELOG.md +++ b/packages/storage/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [6.4.6](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/storage@6.4.5...@aws-amplify/storage@6.4.6) (2024-06-07) + +**Note:** Version bump only for package @aws-amplify/storage + ## [6.4.5](https://github.com/aws-amplify/amplify-js/compare/@aws-amplify/storage@6.4.4...@aws-amplify/storage@6.4.5) (2024-06-04) ### Bug Fixes diff --git a/packages/storage/package.json b/packages/storage/package.json index 1e5950d58ad..3de800029e0 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -1,6 +1,6 @@ { "name": "@aws-amplify/storage", - "version": "6.4.5", + "version": "6.4.6", "description": "Storage category of aws-amplify", "main": "./dist/cjs/index.js", "module": "./dist/esm/index.mjs", @@ -101,7 +101,7 @@ "@aws-amplify/core": "^6.1.0" }, "devDependencies": { - "@aws-amplify/core": "6.3.1", + "@aws-amplify/core": "6.3.2", "@aws-amplify/react-native": "1.1.1", "typescript": "5.0.2" } diff --git a/scripts/tsc-compliance-test/CHANGELOG.md b/scripts/tsc-compliance-test/CHANGELOG.md index 0b0b94d84d0..3a77f018286 100644 --- a/scripts/tsc-compliance-test/CHANGELOG.md +++ b/scripts/tsc-compliance-test/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [0.1.39](https://github.com/aws-amplify/amplify-js/compare/tsc-compliance-test@0.1.38...tsc-compliance-test@0.1.39) (2024-06-07) + +**Note:** Version bump only for package tsc-compliance-test + ## [0.1.38](https://github.com/aws-amplify/amplify-js/compare/tsc-compliance-test@0.1.37...tsc-compliance-test@0.1.38) (2024-06-04) **Note:** Version bump only for package tsc-compliance-test diff --git a/scripts/tsc-compliance-test/package.json b/scripts/tsc-compliance-test/package.json index d9e75580bbf..258250b3d01 100644 --- a/scripts/tsc-compliance-test/package.json +++ b/scripts/tsc-compliance-test/package.json @@ -1,11 +1,11 @@ { "name": "tsc-compliance-test", - "version": "0.1.38", + "version": "0.1.39", "license": "MIT", "private": true, "devDependencies": { "@types/node": "16.18.82", - "aws-amplify": "6.3.5", + "aws-amplify": "6.3.6", "typescript": "4.2.x" }, "scripts": { From 92e63479a7adae728cb2d6b06ff80dfec7bc510a Mon Sep 17 00:00:00 2001 From: israx <70438514+israx@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:17:40 -0400 Subject: [PATCH 03/40] feat(storage): add delimiter support (#13480) * feat: add delimiter input/output types * feat: pass Delimiter parameter to ListObjectsV2 API * chore: add unit tests * chore: bump bunde size * chore: address feedback * chore: fix build * chore: address feedback * chore: address feedback * chore: address feedback --- packages/aws-amplify/package.json | 2 +- .../__tests__/providers/s3/apis/list.test.ts | 112 ++++++++++++++++++ .../src/providers/s3/apis/internal/list.ts | 54 +++++++-- packages/storage/src/types/options.ts | 2 + packages/storage/src/types/outputs.ts | 5 + 5 files changed, 161 insertions(+), 14 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index a3c44a89861..6104db0a485 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -485,7 +485,7 @@ "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "14.94 kB" + "limit": "15.04 kB" }, { "name": "[Storage] remove (S3)", diff --git a/packages/storage/__tests__/providers/s3/apis/list.test.ts b/packages/storage/__tests__/providers/s3/apis/list.test.ts index 9629129d7a2..348719732c0 100644 --- a/packages/storage/__tests__/providers/s3/apis/list.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/list.test.ts @@ -512,4 +512,116 @@ describe('list API', () => { } }); }); + + describe('with delimiter', () => { + const mockedContents = [ + { + Key: 'photos/', + ...listObjectClientBaseResultItem, + }, + { + Key: 'photos/2023.png', + ...listObjectClientBaseResultItem, + }, + { + Key: 'photos/2024.png', + ...listObjectClientBaseResultItem, + }, + ]; + const mockedCommonPrefixes = [ + { Prefix: 'photos/2023/' }, + { Prefix: 'photos/2024/' }, + { Prefix: 'photos/2025/' }, + ]; + + const mockedPath = 'photos/'; + + beforeEach(() => { + mockListObject.mockResolvedValueOnce({ + Contents: mockedContents, + CommonPrefixes: mockedCommonPrefixes, + }); + }); + afterEach(() => { + jest.clearAllMocks(); + mockListObject.mockClear(); + }); + + it('should return subpaths when delimiter is passed in the request', async () => { + const { items, subpaths } = await list({ + path: mockedPath, + options: { + delimiter: '/', + }, + }); + expect(items).toHaveLength(3); + expect(subpaths).toEqual([ + 'photos/2023/', + 'photos/2024/', + 'photos/2025/', + ]); + expect(listObjectsV2).toHaveBeenCalledTimes(1); + await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + listObjectClientConfig, + { + Bucket: bucket, + MaxKeys: 1000, + Prefix: mockedPath, + Delimiter: '/', + }, + ); + }); + + it('should return subpaths when delimiter and listAll are passed in the request', async () => { + const { items, subpaths } = await list({ + path: mockedPath, + options: { + delimiter: '/', + listAll: true, + }, + }); + expect(items).toHaveLength(3); + expect(subpaths).toEqual([ + 'photos/2023/', + 'photos/2024/', + 'photos/2025/', + ]); + expect(listObjectsV2).toHaveBeenCalledTimes(1); + await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + listObjectClientConfig, + { + Bucket: bucket, + MaxKeys: 1000, + Prefix: mockedPath, + Delimiter: '/', + }, + ); + }); + + it('should return subpaths when delimiter is pageSize are passed in the request', async () => { + const { items, subpaths } = await list({ + path: mockedPath, + options: { + delimiter: '/', + pageSize: 3, + }, + }); + expect(items).toHaveLength(3); + expect(subpaths).toEqual([ + 'photos/2023/', + 'photos/2024/', + 'photos/2025/', + ]); + expect(listObjectsV2).toHaveBeenCalledTimes(1); + await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + listObjectClientConfig, + { + Bucket: bucket, + MaxKeys: 3, + Prefix: mockedPath, + Delimiter: '/', + }, + ); + }); + }); }); diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index f180dfe5247..7b625263a84 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -29,6 +29,7 @@ import { import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { STORAGE_INPUT_PREFIX } from '../../utils/constants'; +import { CommonPrefix } from '../../utils/client/types'; const MAX_PAGE_SIZE = 1000; @@ -79,6 +80,7 @@ export const list = async ( Prefix: isInputWithPrefix ? `${generatedPrefix}${objectKey}` : objectKey, MaxKeys: options?.listAll ? undefined : options?.pageSize, ContinuationToken: options?.listAll ? undefined : options?.nextToken, + Delimiter: options?.delimiter, }; logger.debug(`listing items from "${listParams.Prefix}"`); @@ -176,23 +178,29 @@ const _listAllWithPath = async ({ listParams, }: ListInputArgs): Promise => { const listResult: ListOutputItemWithPath[] = []; + const subpaths: string[] = []; let continuationToken = listParams.ContinuationToken; do { - const { items: pageResults, nextToken: pageNextToken } = - await _listWithPath({ - s3Config, - listParams: { - ...listParams, - ContinuationToken: continuationToken, - MaxKeys: MAX_PAGE_SIZE, - }, - }); + const { + items: pageResults, + subpaths: pageSubpaths, + nextToken: pageNextToken, + } = await _listWithPath({ + s3Config, + listParams: { + ...listParams, + ContinuationToken: continuationToken, + MaxKeys: MAX_PAGE_SIZE, + }, + }); listResult.push(...pageResults); + subpaths.push(...(pageSubpaths ?? [])); continuationToken = pageNextToken; } while (continuationToken); return { items: listResult, + ...parseSubpaths(subpaths), }; }; @@ -206,7 +214,11 @@ const _listWithPath = async ({ listParamsClone.MaxKeys = MAX_PAGE_SIZE; } - const response: ListObjectsV2Output = await listObjectsV2( + const { + Contents: contents, + NextContinuationToken: nextContinuationToken, + CommonPrefixes: commonPrefixes, + }: ListObjectsV2Output = await listObjectsV2( { ...s3Config, userAgentValue: getStorageUserAgentValue(StorageAction.List), @@ -214,19 +226,35 @@ const _listWithPath = async ({ listParamsClone, ); - if (!response?.Contents) { + const subpaths = mapCommonPrefixesToSubpaths(commonPrefixes); + + if (!contents) { return { items: [], + ...parseSubpaths(subpaths), }; } return { - items: response.Contents.map(item => ({ + items: contents.map(item => ({ path: item.Key!, eTag: item.ETag, lastModified: item.LastModified, size: item.Size, })), - nextToken: response.NextContinuationToken, + nextToken: nextContinuationToken, + ...parseSubpaths(subpaths), }; }; + +function mapCommonPrefixesToSubpaths( + commonPrefixes?: CommonPrefix[], +): string[] | undefined { + const mappedSubpaths = commonPrefixes?.map(({ Prefix }) => Prefix); + + return mappedSubpaths?.filter((subpath): subpath is string => !!subpath); +} + +function parseSubpaths(subpaths?: string[]) { + return subpaths && subpaths.length > 0 ? { subpaths } : {}; +} diff --git a/packages/storage/src/types/options.ts b/packages/storage/src/types/options.ts index b9c74590ba6..31e371593f5 100644 --- a/packages/storage/src/types/options.ts +++ b/packages/storage/src/types/options.ts @@ -10,12 +10,14 @@ export interface StorageOptions { export type StorageListAllOptions = StorageOptions & { listAll: true; + delimiter?: string; }; export type StorageListPaginateOptions = StorageOptions & { listAll?: false; pageSize?: number; nextToken?: string; + delimiter?: string; }; export type StorageRemoveOptions = StorageOptions; diff --git a/packages/storage/src/types/outputs.ts b/packages/storage/src/types/outputs.ts index e38482729b8..312966b087b 100644 --- a/packages/storage/src/types/outputs.ts +++ b/packages/storage/src/types/outputs.ts @@ -70,4 +70,9 @@ export interface StorageListOutput { * List of items returned by the list API. */ items: Item[]; + /** + * List of subpaths returned by the list API when a delimiter option is passed + * in the request of the list API. + */ + subpaths?: string[]; } From 614e38e478032fbed44f6cd2e841b913c0c03817 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Thu, 20 Jun 2024 08:45:12 -0700 Subject: [PATCH 04/40] chore: enable storage-browser preid release (#13524) chore: fix npm dist tag to be storage-browser --- .github/workflows/push-preid-release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/push-preid-release.yml b/.github/workflows/push-preid-release.yml index d4867df08d0..c24a87da0a3 100644 --- a/.github/workflows/push-preid-release.yml +++ b/.github/workflows/push-preid-release.yml @@ -34,4 +34,5 @@ jobs: # The preid should be detected from the branch name recommending feat/{PREID}/whatever as branch naming pattern # if your branch doesn't follow this pattern, you can override it here for your branch. with: - preid: ${{ needs.parse-preid.outputs.preid }} + preid: storage-browser + # preid: ${{ needs.parse-preid.outputs.preid }} From 3fc169e2044cf83ed7acb839fb46c32324c6f372 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Wed, 26 Jun 2024 09:38:40 -0700 Subject: [PATCH 05/40] feat(storage): add base types for storage browser (#13528) * feat(storage): add base types * fix(storage): address feedbacks Co-authored-by: israx <70438514+israx@users.noreply.github.com> --- .../storage/src/providers/s3/types/options.ts | 23 +++++ .../createLocationCredentialsHandler.ts | 18 ++++ packages/storage/src/storage-browser/index.ts | 10 ++ .../storage-browser/listCallerAccessGrants.ts | 25 +++++ .../createLocationCredentialsStore.ts | 12 +++ packages/storage/src/storage-browser/types.ts | 91 +++++++++++++++++++ 6 files changed, 179 insertions(+) create mode 100644 packages/storage/src/storage-browser/createLocationCredentialsHandler.ts create mode 100644 packages/storage/src/storage-browser/index.ts create mode 100644 packages/storage/src/storage-browser/listCallerAccessGrants.ts create mode 100644 packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts create mode 100644 packages/storage/src/storage-browser/types.ts diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index b2b7dfd0ddc..278e91adb9c 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { StorageAccessLevel } from '@aws-amplify/core'; +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { SigningOptions } from '@aws-amplify/core/internals/aws-client-utils'; import { TransferProgressEvent } from '../../../types'; @@ -10,12 +11,34 @@ import { StorageListPaginateOptions, } from '../../../types/options'; +/** + * @internal + */ +export type Permission = 'READ' | 'READWRITE' | 'WRITE'; + +/** + * @internal + */ +export type LocationCredentialsProvider = (options: { + forceRefresh?: boolean; + locations: { bucket: string; path: string }[]; + permission: Permission; +}) => Promise<{ credentials: AWSCredentials }>; + interface CommonOptions { /** * Whether to use accelerate endpoint. * @default false */ useAccelerateEndpoint?: boolean; + + /** + * Async function returning AWS credentials for an API call. This function + * is invoked with S3 locations(bucket and path). + * If omitted, the global credentials configured in Amplify Auth + * would be used. + */ + locationCredentialsProvider?: LocationCredentialsProvider; } /** @deprecated This may be removed in the next major version. */ diff --git a/packages/storage/src/storage-browser/createLocationCredentialsHandler.ts b/packages/storage/src/storage-browser/createLocationCredentialsHandler.ts new file mode 100644 index 00000000000..5da981d71bc --- /dev/null +++ b/packages/storage/src/storage-browser/createLocationCredentialsHandler.ts @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CredentialsProvider, LocationCredentialsHandler } from './types'; + +interface CreateLocationCredentialsHandlerInput { + accountId: string; + credentialsProvider: CredentialsProvider; + region: string; +} + +export const createLocationCredentialsHandler = ( + // eslint-disable-next-line unused-imports/no-unused-vars + input: CreateLocationCredentialsHandlerInput, +): LocationCredentialsHandler => { + // TODO(@AllanZhengYP) + throw new Error('Not Implemented'); +}; diff --git a/packages/storage/src/storage-browser/index.ts b/packages/storage/src/storage-browser/index.ts new file mode 100644 index 00000000000..ddd423173b4 --- /dev/null +++ b/packages/storage/src/storage-browser/index.ts @@ -0,0 +1,10 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + listCallerAccessGrants, + ListCallerAccessGrantsInput, + ListCallerAccessGrantsOutput, +} from './listCallerAccessGrants'; +export { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; +export { createLocationCredentialsStore } from './locationCredentialsStore/createLocationCredentialsStore'; diff --git a/packages/storage/src/storage-browser/listCallerAccessGrants.ts b/packages/storage/src/storage-browser/listCallerAccessGrants.ts new file mode 100644 index 00000000000..cd13e6a5dc5 --- /dev/null +++ b/packages/storage/src/storage-browser/listCallerAccessGrants.ts @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AccessGrant, CredentialsProvider, ListLocationsOutput } from './types'; + +export interface ListCallerAccessGrantsInput { + accountId: string; + credentialsProvider: CredentialsProvider; + region: string; + options?: { + nextToken?: string; + // Default to 100; If > 1000, API will make multiple API calls. + pageSize?: number; + }; +} + +export type ListCallerAccessGrantsOutput = ListLocationsOutput; + +export const listCallerAccessGrants = ( + // eslint-disable-next-line unused-imports/no-unused-vars + input: ListCallerAccessGrantsInput, +): Promise => { + // TODO(@AllanZhengYP) + throw new Error('Not Implemented'); +}; diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts b/packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts new file mode 100644 index 00000000000..3f73857fc5b --- /dev/null +++ b/packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts @@ -0,0 +1,12 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { LocationCredentialsHandler, LocationCredentialsStore } from '../types'; + +// eslint-disable-next-line unused-imports/no-unused-vars +export const createLocationCredentialsStore = (input: { + handler: LocationCredentialsHandler; +}): LocationCredentialsStore => { + // TODO(@AllanZhengYP) + throw new Error('Not Implemented'); +}; diff --git a/packages/storage/src/storage-browser/types.ts b/packages/storage/src/storage-browser/types.ts new file mode 100644 index 00000000000..7367f025235 --- /dev/null +++ b/packages/storage/src/storage-browser/types.ts @@ -0,0 +1,91 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { + LocationCredentialsProvider, + Permission, +} from '../providers/s3/types/options'; + +/** + * @internal + */ +export type CredentialsProvider = (options?: { + forceRefresh?: boolean; +}) => Promise<{ credentials: AWSCredentials }>; + +type LocationAccessType = 'BUCKET' | 'PREFIX' | 'OBJECT'; + +/** + * @internal + */ +export interface LocationAccess { + /** + * Scope of storage location. For S3 service, it's the S3 path of the data to + * which the access is granted. + * + * @example 's3://MY-BUCKET-NAME/prefix/*' + */ + readonly scope: string; + /** + * The type of access granted to your Storage data. Can be either of READ, + * WRITE or READWRITE + */ + readonly permission: Permission; + /** + * parse location type parsed from scope format: + * * bucket: `'s3:///*'` + * * prefix: `'s3:///*'` + * * object: `'s3:////'` + */ + readonly type: LocationAccessType; +} + +export interface AccessGrant extends LocationAccess { + /** + * The Amazon Resource Name (ARN) of an AWS IAM Identity Center application + * associated with your Identity Center instance. If the grant includes an + * application ARN, the grantee can only access the S3 data through this + * application. + */ + readonly applicationArn: string | undefined; +} + +/** + * @internal + */ +export interface ListLocationsOutput { + locations: T[]; + nextToken?: string; +} + +// Interface for listLocations() handler +export type ListLocations = () => Promise>; + +// Interface for getLocationCredentials() handler. +export type LocationCredentialsHandler = (input: { + scope: string; + permission: Permission; +}) => Promise<{ credentials: AWSCredentials; scope?: string }>; + +export interface LocationCredentialsStore { + /** + * Get location-specific credentials. It uses a cache internally to optimize performance when + * getting credentials for the same location. It will refresh credentials if they expire or + * when forced to. + * + * If specific credentials scope `option` is omitted, the store will attempt to resolve + * locations-specific credentials from the input bucket and full path. + */ + getProvider(option?: { + scope: string; + permission: Permission; + }): LocationCredentialsProvider; + /** + * Invalidate cached credentials and force subsequent calls to get location-specific + * credentials to throw. It also makes subsequent calls to `getCredentialsProviderForLocation` + * to throw. + */ + destroy(): void; +} From 401dbc7634ceba5bdd333100496936ea8e4bab3d Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Wed, 3 Jul 2024 09:24:20 -0700 Subject: [PATCH 06/40] feat(storage): add creds store scaffolding and update types (#13558) --- .../locationCredentialsStore/create.test.ts | 17 +++++ .../locationCredentialsStore/registry.test.ts | 29 ++++++++ .../locationCredentialsStore/store.test.ts | 21 ++++++ .../storage/src/providers/s3/types/options.ts | 10 ++- packages/storage/src/storage-browser/index.ts | 6 +- .../locationCredentialsStore/create.ts | 45 +++++++++++++ .../createLocationCredentialsStore.ts | 12 ---- .../locationCredentialsStore/registry.ts | 39 +++++++++++ .../locationCredentialsStore/store.ts | 67 +++++++++++++++++++ .../src/storage-browser/managedAuthAdapter.ts | 27 ++++++++ packages/storage/src/storage-browser/types.ts | 43 ++++++------ 11 files changed, 281 insertions(+), 35 deletions(-) create mode 100644 packages/storage/__tests__/storage-browser/locationCredentialsStore/create.test.ts create mode 100644 packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts create mode 100644 packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts create mode 100644 packages/storage/src/storage-browser/locationCredentialsStore/create.ts delete mode 100644 packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts create mode 100644 packages/storage/src/storage-browser/locationCredentialsStore/registry.ts create mode 100644 packages/storage/src/storage-browser/locationCredentialsStore/store.ts create mode 100644 packages/storage/src/storage-browser/managedAuthAdapter.ts diff --git a/packages/storage/__tests__/storage-browser/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/storage-browser/locationCredentialsStore/create.test.ts new file mode 100644 index 00000000000..1bd8f08c6a2 --- /dev/null +++ b/packages/storage/__tests__/storage-browser/locationCredentialsStore/create.test.ts @@ -0,0 +1,17 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +describe('createLocationCredentialsStore', () => { + it.todo('should create a store'); + describe('created store', () => { + describe('getValue()', () => { + it.todo('should call getValue() from store'); + it.todo( + 'should validate credentials location with resolved common scope', + ); + }); + describe('destroy()', () => { + it.todo('should call removeStore() from store'); + }); + }); +}); diff --git a/packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts b/packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts new file mode 100644 index 00000000000..759457ec207 --- /dev/null +++ b/packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts @@ -0,0 +1,29 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +describe('createStore', () => { + it.todo('should create a store with given capacity, refresh Handler'); + it.todo('should return a symbol to refer the store instance'); +}); + +describe('getValue', () => { + it.todo('should look up a cache value for given location and permission'); + it.todo( + 'should look up a cache value for given location and READWRITE permission', + ); + it.todo('should invoke the refresh handler if look up returns null'); + it.todo( + 'should invoke refresh handler only once if multiple look up for same location returns null', + ); + it.todo('should throw if refresh handler throws'); + it.todo( + 'should invoke the refresh handler if the refresh handler previously fails', + ); +}); + +describe('removeStore', () => { + it.todo('should remove the store with given symbol'); + it.todo('should not throw if store with given symbol does not exist'); +}); diff --git a/packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts b/packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts new file mode 100644 index 00000000000..1bcc003b137 --- /dev/null +++ b/packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +describe('initStore', () => { + it.todo( + 'should create a store with given capacity, refresh Handler and cache', + ); + it.todo('should create a store with default capacity if not provided'); + it.todo('should throw if capacity is not > 0'); +}); + +describe('getCacheKey', () => { + it.todo('should return a cache key for given location and permission'); +}); + +describe('getCacheValue', () => { + it.todo('should return a cache value for given location and permission'); + it.todo('should return null if cache value is not found'); + it.todo('should return null if cache value is expired'); + it.todo('should return null if cache value is not valid'); +}); diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 278e91adb9c..633366a4628 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -16,12 +16,20 @@ import { */ export type Permission = 'READ' | 'READWRITE' | 'WRITE'; +/** + * @internal + */ +export interface BucketLocation { + bucket: string; + path: string; +} + /** * @internal */ export type LocationCredentialsProvider = (options: { forceRefresh?: boolean; - locations: { bucket: string; path: string }[]; + locations: BucketLocation[]; permission: Permission; }) => Promise<{ credentials: AWSCredentials }>; diff --git a/packages/storage/src/storage-browser/index.ts b/packages/storage/src/storage-browser/index.ts index ddd423173b4..66022db1ccc 100644 --- a/packages/storage/src/storage-browser/index.ts +++ b/packages/storage/src/storage-browser/index.ts @@ -7,4 +7,8 @@ export { ListCallerAccessGrantsOutput, } from './listCallerAccessGrants'; export { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; -export { createLocationCredentialsStore } from './locationCredentialsStore/createLocationCredentialsStore'; +export { createLocationCredentialsStore } from './locationCredentialsStore/create'; +export { + managedAuthAdapter, + ManagedAuthAdapterInput, +} from './managedAuthAdapter'; diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/create.ts b/packages/storage/src/storage-browser/locationCredentialsStore/create.ts new file mode 100644 index 00000000000..22221bf9ee9 --- /dev/null +++ b/packages/storage/src/storage-browser/locationCredentialsStore/create.ts @@ -0,0 +1,45 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + CredentialsLocation, + LocationCredentialsHandler, + LocationCredentialsStore, +} from '../types'; +import { LocationCredentialsProvider } from '../../providers/s3/types/options'; + +import { createStore, getValue, removeStore } from './registry'; + +export const createLocationCredentialsStore = (input: { + handler: LocationCredentialsHandler; +}): LocationCredentialsStore => { + const storeReference = createStore(input.handler); + + const store = { + getProvider(providerLocation: CredentialsLocation) { + const locationCredentialsProvider = async ({ + permission, + locations, + forceRefresh = false, + }: Parameters[0]) => { + // TODO(@AllanZhengYP) validate input + + return getValue({ + storeReference, + location: { ...providerLocation }, + forceRefresh, + }); + }; + + return locationCredentialsProvider; + }, + + destroy() { + removeStore(storeReference); + }, + }; + + return store; +}; diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts b/packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts deleted file mode 100644 index 3f73857fc5b..00000000000 --- a/packages/storage/src/storage-browser/locationCredentialsStore/createLocationCredentialsStore.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { LocationCredentialsHandler, LocationCredentialsStore } from '../types'; - -// eslint-disable-next-line unused-imports/no-unused-vars -export const createLocationCredentialsStore = (input: { - handler: LocationCredentialsHandler; -}): LocationCredentialsStore => { - // TODO(@AllanZhengYP) - throw new Error('Not Implemented'); -}; diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/registry.ts b/packages/storage/src/storage-browser/locationCredentialsStore/registry.ts new file mode 100644 index 00000000000..a76c511cab0 --- /dev/null +++ b/packages/storage/src/storage-browser/locationCredentialsStore/registry.ts @@ -0,0 +1,39 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { CredentialsLocation, LocationCredentialsHandler } from '../types'; + +import { LocationCredentialsStore, initStore } from './store'; + +/** + * Keep all cache records for all instances of credentials store in a singleton + * so we can reliably de-reference from the memory when we destroy a store + * instance. + */ +const storeRegistry = new Map(); + +export const createStore = ( + refreshHandler: LocationCredentialsHandler, + size?: number, +) => { + const storeInstanceSymbol = Symbol('LocationCredentialsStore'); + storeRegistry.set(storeInstanceSymbol, initStore(refreshHandler, size)); + + return storeInstanceSymbol; +}; + +export const getValue = async (input: { + storeReference: symbol; + location: CredentialsLocation; + forceRefresh: boolean; +}): Promise<{ credentials: AWSCredentials }> => { + // TODO(@AllanZhengYP): get location credentials from store. + throw new Error('Not implemented'); +}; + +export const removeStore = (storeReference: symbol) => { + storeRegistry.delete(storeReference); +}; diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/store.ts b/packages/storage/src/storage-browser/locationCredentialsStore/store.ts new file mode 100644 index 00000000000..97d45196620 --- /dev/null +++ b/packages/storage/src/storage-browser/locationCredentialsStore/store.ts @@ -0,0 +1,67 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { Permission } from '../../providers/s3/types/options'; +import { CredentialsLocation, LocationCredentialsHandler } from '../types'; + +const CREDENTIALS_STORE_DEFAULT_SIZE = 10; + +export interface StoreValue extends CredentialsLocation { + credentials?: AWSCredentials; + inflightCredentials?: Promise<{ credentials: AWSCredentials }>; +} + +type S3Url = string; + +/** + * @internal + */ +export type CacheKey = `${S3Url}_${Permission}`; + +/** + * @internal + */ +export const getCacheKey = (compositeKey: CredentialsLocation): CacheKey => + `${compositeKey.scope}_${compositeKey.permission}`; + +/** + * LRU implementation for Location Credentials Store + * O(n) for get and set for simplicity. + * + * @internal + */ +export interface LocationCredentialsStore { + capacity: number; + refreshHandler: LocationCredentialsHandler; + values: Map; +} + +/** + * @internal + */ +export const initStore = ( + refreshHandler: LocationCredentialsHandler, + size = CREDENTIALS_STORE_DEFAULT_SIZE, +): LocationCredentialsStore => { + // TODO(@AllanZhengYP) create StorageError + if (size <= 0) { + throw new Error('Invalid Cache size'); + } + + return { + capacity: size, + refreshHandler, + values: new Map(), + }; +}; + +export const getCacheValue = ( + store: LocationCredentialsStore, + key: CacheKey, +): AWSCredentials | null => { + throw new Error('Not implemented'); +}; diff --git a/packages/storage/src/storage-browser/managedAuthAdapter.ts b/packages/storage/src/storage-browser/managedAuthAdapter.ts new file mode 100644 index 00000000000..9f3214b3158 --- /dev/null +++ b/packages/storage/src/storage-browser/managedAuthAdapter.ts @@ -0,0 +1,27 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { + CredentialsProvider, + ListLocations, + LocationCredentialsHandler, +} from './types'; + +export interface ManagedAuthAdapterInput { + accountId: string; + region: string; + credentialsProvider: CredentialsProvider; +} + +export interface ManagedAuthAdapterOutput { + listLocations: ListLocations; + getLocationCredentials: LocationCredentialsHandler; + region: string; +} + +export const managedAuthAdapter = ( + // eslint-disable-next-line unused-imports/no-unused-vars + input: ManagedAuthAdapterInput, +): ManagedAuthAdapterOutput => { + // TODO(@AllanZhengYP) + throw new Error('Not implemented'); +}; diff --git a/packages/storage/src/storage-browser/types.ts b/packages/storage/src/storage-browser/types.ts index 7367f025235..2525560369f 100644 --- a/packages/storage/src/storage-browser/types.ts +++ b/packages/storage/src/storage-browser/types.ts @@ -15,17 +15,19 @@ export type CredentialsProvider = (options?: { forceRefresh?: boolean; }) => Promise<{ credentials: AWSCredentials }>; -type LocationAccessType = 'BUCKET' | 'PREFIX' | 'OBJECT'; - /** * @internal */ -export interface LocationAccess { +export type LocationType = 'BUCKET' | 'PREFIX' | 'OBJECT'; + +export interface CredentialsLocation { /** * Scope of storage location. For S3 service, it's the S3 path of the data to - * which the access is granted. + * which the access is granted. It can be in following formats: * - * @example 's3://MY-BUCKET-NAME/prefix/*' + * @example Bucket 's3:///*' + * @example Prefix 's3:///*' + * @example Object 's3:////' */ readonly scope: string; /** @@ -33,13 +35,19 @@ export interface LocationAccess { * WRITE or READWRITE */ readonly permission: Permission; +} + +/** + * @internal + */ +export interface LocationAccess extends CredentialsLocation { /** - * parse location type parsed from scope format: - * * bucket: `'s3:///*'` - * * prefix: `'s3:///*'` - * * object: `'s3:////'` + * Parse location type parsed from scope format: + * * BUCKET: `'s3:///*'` + * * PREFIX: `'s3:///*'` + * * OBJECT: `'s3:////'` */ - readonly type: LocationAccessType; + readonly type: LocationType; } export interface AccessGrant extends LocationAccess { @@ -64,24 +72,17 @@ export interface ListLocationsOutput { export type ListLocations = () => Promise>; // Interface for getLocationCredentials() handler. -export type LocationCredentialsHandler = (input: { - scope: string; - permission: Permission; -}) => Promise<{ credentials: AWSCredentials; scope?: string }>; +export type LocationCredentialsHandler = ( + input: CredentialsLocation, +) => Promise<{ credentials: AWSCredentials }>; export interface LocationCredentialsStore { /** * Get location-specific credentials. It uses a cache internally to optimize performance when * getting credentials for the same location. It will refresh credentials if they expire or * when forced to. - * - * If specific credentials scope `option` is omitted, the store will attempt to resolve - * locations-specific credentials from the input bucket and full path. */ - getProvider(option?: { - scope: string; - permission: Permission; - }): LocationCredentialsProvider; + getProvider(option: CredentialsLocation): LocationCredentialsProvider; /** * Invalidate cached credentials and force subsequent calls to get location-specific * credentials to throw. It also makes subsequent calls to `getCredentialsProviderForLocation` From 6f17f6cbb88067bea6a03a8814b1abc568251460 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Tue, 9 Jul 2024 16:13:17 -0700 Subject: [PATCH 07/40] feat(storage): add cred store lru implementation (#13561) --- packages/aws-amplify/package.json | 14 +- .../locationCredentialsStore/registry.test.ts | 29 --- .../locationCredentialsStore/store.test.ts | 21 -- .../locationCredentialsStore/create.test.ts | 0 .../locationCredentialsStore/registry.test.ts | 188 +++++++++++++++ .../locationCredentialsStore/store.test.ts | 222 ++++++++++++++++++ .../storage/src/errors/types/validation.ts | 8 + .../locationCredentialsStore/registry.ts | 39 --- .../locationCredentialsStore/store.ts | 67 ------ .../createLocationCredentialsHandler.ts | 0 .../index.ts | 2 +- .../listCallerAccessGrants.ts | 0 .../locationCredentialsStore/create.ts | 6 +- .../locationCredentialsStore/index.ts | 4 + .../locationCredentialsStore/registry.ts | 82 +++++++ .../locationCredentialsStore/store.ts | 159 +++++++++++++ .../managedAuthAdapter.ts | 0 .../types.ts | 0 18 files changed, 674 insertions(+), 167 deletions(-) delete mode 100644 packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts delete mode 100644 packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts rename packages/storage/__tests__/{storage-browser => storageBrowser}/locationCredentialsStore/create.test.ts (100%) create mode 100644 packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts create mode 100644 packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts delete mode 100644 packages/storage/src/storage-browser/locationCredentialsStore/registry.ts delete mode 100644 packages/storage/src/storage-browser/locationCredentialsStore/store.ts rename packages/storage/src/{storage-browser => storageBrowser}/createLocationCredentialsHandler.ts (100%) rename packages/storage/src/{storage-browser => storageBrowser}/index.ts (95%) rename packages/storage/src/{storage-browser => storageBrowser}/listCallerAccessGrants.ts (100%) rename packages/storage/src/{storage-browser => storageBrowser}/locationCredentialsStore/create.ts (90%) create mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/index.ts create mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts create mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/store.ts rename packages/storage/src/{storage-browser => storageBrowser}/managedAuthAdapter.ts (100%) rename packages/storage/src/{storage-browser => storageBrowser}/types.ts (100%) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 6104db0a485..3ff5e5aa69d 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "14.54 kB" + "limit": "14.64 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.17 kB" + "limit": "15.27 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "14.43 kB" + "limit": "14.52 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "15.51 kB" + "limit": "15.62 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.04 kB" + "limit": "15.12 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "14.29 kB" + "limit": "14.38 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "19.64 kB" + "limit": "19.69 kB" } ] } diff --git a/packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts b/packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts deleted file mode 100644 index 759457ec207..00000000000 --- a/packages/storage/__tests__/storage-browser/locationCredentialsStore/registry.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* eslint-disable unused-imports/no-unused-vars */ - -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -describe('createStore', () => { - it.todo('should create a store with given capacity, refresh Handler'); - it.todo('should return a symbol to refer the store instance'); -}); - -describe('getValue', () => { - it.todo('should look up a cache value for given location and permission'); - it.todo( - 'should look up a cache value for given location and READWRITE permission', - ); - it.todo('should invoke the refresh handler if look up returns null'); - it.todo( - 'should invoke refresh handler only once if multiple look up for same location returns null', - ); - it.todo('should throw if refresh handler throws'); - it.todo( - 'should invoke the refresh handler if the refresh handler previously fails', - ); -}); - -describe('removeStore', () => { - it.todo('should remove the store with given symbol'); - it.todo('should not throw if store with given symbol does not exist'); -}); diff --git a/packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts b/packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts deleted file mode 100644 index 1bcc003b137..00000000000 --- a/packages/storage/__tests__/storage-browser/locationCredentialsStore/store.test.ts +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -describe('initStore', () => { - it.todo( - 'should create a store with given capacity, refresh Handler and cache', - ); - it.todo('should create a store with default capacity if not provided'); - it.todo('should throw if capacity is not > 0'); -}); - -describe('getCacheKey', () => { - it.todo('should return a cache key for given location and permission'); -}); - -describe('getCacheValue', () => { - it.todo('should return a cache value for given location and permission'); - it.todo('should return null if cache value is not found'); - it.todo('should return null if cache value is expired'); - it.todo('should return null if cache value is not valid'); -}); diff --git a/packages/storage/__tests__/storage-browser/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts similarity index 100% rename from packages/storage/__tests__/storage-browser/locationCredentialsStore/create.test.ts rename to packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts new file mode 100644 index 00000000000..19adc4576d9 --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts @@ -0,0 +1,188 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; +import { + createStore, + getValue, + removeStore, +} from '../../../src/storageBrowser/locationCredentialsStore/registry'; +import { + LruLocationCredentialsStore, + fetchNewValue, + getCacheValue, + initStore, +} from '../../../src/storageBrowser/locationCredentialsStore/store'; + +jest.mock('../../../src/storageBrowser/locationCredentialsStore/store'); + +const mockedStore = 'MOCKED_STORE' as any as LruLocationCredentialsStore; + +afterEach(() => { + jest.clearAllMocks(); +}); + +beforeEach(() => { + jest.mocked(initStore).mockReturnValue(mockedStore); +}); + +describe('createStore', () => { + it('should create a store with given capacity, refresh Handler', () => { + const refreshHandler = jest.fn(); + createStore(refreshHandler, 20); + expect(initStore).toHaveBeenCalledWith(refreshHandler, 20); + }); + + it('should return a symbol to refer the store instance', () => { + const storeReference = createStore(jest.fn(), 20); + expect(Object.prototype.toString.call(storeReference)).toBe( + '[object Symbol]', + ); + }); +}); + +describe('getValue', () => { + const mockCachedValue = 'CACHED_VALUE' as any as AWSCredentials; + let storeReference: symbol; + beforeEach(() => { + storeReference = createStore(jest.fn(), 20); + }); + afterEach(() => { + removeStore(storeReference); + jest.clearAllMocks(); + }); + + it('should throw if a store instance cannot be found from registry', async () => { + expect.assertions(1); + await expect( + getValue({ + storeSymbol: Symbol('invalid'), + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).rejects.toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsStoreDestroyed + ].message, + ); + }); + + it('should look up a cache value for given location and permission', async () => { + expect.assertions(2); + jest.mocked(getCacheValue).mockReturnValueOnce(mockCachedValue); + expect( + await getValue({ + storeSymbol: storeReference, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).toEqual({ credentials: mockCachedValue }); + expect(getCacheValue).toHaveBeenCalledWith(mockedStore, { + scope: 'abc', + permission: 'READ', + }); + }); + + it('should look up a cache value for given location and READWRITE permission', async () => { + expect.assertions(4); + + jest.mocked(getCacheValue).mockReturnValueOnce(null); + jest.mocked(getCacheValue).mockReturnValueOnce(mockCachedValue); + expect( + await getValue({ + storeSymbol: storeReference, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).toEqual({ credentials: mockCachedValue }); + expect(getCacheValue).toHaveBeenCalledTimes(2); + expect(getCacheValue).toHaveBeenNthCalledWith(1, mockedStore, { + scope: 'abc', + permission: 'READ', + }); + expect(getCacheValue).toHaveBeenNthCalledWith(2, mockedStore, { + scope: 'abc', + permission: 'READWRITE', + }); + }); + + it('should invoke the refresh handler if look up returns null', async () => { + expect.assertions(3); + jest.mocked(getCacheValue).mockReturnValue(null); + jest.mocked(fetchNewValue).mockResolvedValue('NEW_VALUE' as any); + expect( + await getValue({ + storeSymbol: storeReference, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).toEqual('NEW_VALUE'); + expect(fetchNewValue).toHaveBeenCalledTimes(1); + expect(fetchNewValue).toHaveBeenCalledWith(mockedStore, { + scope: 'abc', + permission: 'READ', + }); + }); + + it('should invoke the refresh handler regardless of cache if forceRefresh is true', async () => { + expect.assertions(3); + jest.mocked(getCacheValue).mockReturnValue(mockCachedValue); + jest.mocked(fetchNewValue).mockResolvedValue('NEW_VALUE' as any); + expect( + await getValue({ + storeSymbol: storeReference, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: true, + }), + ).toEqual('NEW_VALUE'); + expect(fetchNewValue).toHaveBeenCalledTimes(1); + expect(fetchNewValue).toHaveBeenCalledWith(mockedStore, { + scope: 'abc', + permission: 'READ', + }); + }); + + it('should throw if refresh handler throws', async () => { + expect.assertions(1); + jest + .mocked(fetchNewValue) + .mockRejectedValueOnce(new Error('Network error')); + await expect( + getValue({ + storeSymbol: storeReference, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: true, + }), + ).rejects.toThrow('Network error'); + }); +}); + +describe('removeStore', () => { + it('should remove the store with given symbol', async () => { + expect.assertions(1); + const storeReference = createStore(jest.fn(), 20); + removeStore(storeReference); + await expect( + getValue({ + storeSymbol: storeReference, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).rejects.toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsStoreDestroyed + ].message, + ); + }); + + it('should not throw if store with given symbol does not exist', () => { + expect(() => { + removeStore(Symbol('invalid')); + }).not.toThrow(); + }); +}); diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts new file mode 100644 index 00000000000..096b50f07b5 --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts @@ -0,0 +1,222 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; +import { + fetchNewValue, + getCacheValue, + initStore, +} from '../../../src/storageBrowser/locationCredentialsStore/store'; +import { CredentialsLocation } from '../../../src/storageBrowser/types'; + +describe('initStore', () => { + it('should create a store with given capacity, refresh Handler and values', () => { + const refreshHandler = jest.fn(); + const store = initStore(refreshHandler, 20); + expect(store).toEqual({ + capacity: 20, + refreshHandler, + values: expect.any(Map), + }); + }); + + it('should create a store with default capacity if not provided', () => { + const store = initStore(jest.fn()); + expect(store).toMatchObject({ + capacity: 10, + }); + }); + + it('should throw if capacity is not > 0', () => { + expect(() => initStore(jest.fn(), 0)).toThrow( + validationErrorMap[ + StorageValidationErrorCode.InvalidLocationCredentialsCacheSize + ].message, + ); + }); +}); + +describe('getCacheValue', () => { + it('should return a cache value for given location and permission', () => { + const cachedValue = { + credentials: 'MOCK_CREDS', + scope: 'abc', + permission: 'READ', + } as any; + const store = initStore(jest.fn()); + store.values.set('abc_READ', cachedValue); + expect( + getCacheValue(store, { + scope: 'abc', + permission: 'READ', + }), + ).toEqual(cachedValue.credentials); + }); + + it('should return null if cache value is not found', () => { + expect( + getCacheValue(initStore(jest.fn()), { + scope: 'abc', + permission: 'READ', + }), + ).toBeNull(); + }); + + it('should return null if cache value is expired', () => { + const expiredValue = { + credentials: { + expiration: new Date(), + }, + scope: 'abc', + permission: 'READ', + } as any; + const store = initStore(jest.fn()); + store.values.set('abc_READ', expiredValue); + expect( + getCacheValue(store, { + scope: 'abc', + permission: 'READ', + }), + ).toBeNull(); + expect(store.values.size).toBe(0); + }); + + it('should return null if cache value is expiring soon', () => { + const expiringValue = { + credentials: { + expiration: new Date(Date.now() + 1000 * 20), // 20 seconds + }, + scope: 'abc', + permission: 'READ', + } as any; + const store = initStore(jest.fn()); + store.values.set('abc_READ', expiringValue); + expect( + getCacheValue(store, { + scope: 'abc', + permission: 'READ', + }), + ).toBeNull(); + expect(store.values.size).toBe(0); + }); +}); + +describe('fetchNewValue', () => { + const mockCacheLocation = { + scope: 'abc', + permission: 'READ', + } as CredentialsLocation; + const createCacheKey = (location: CredentialsLocation) => + `${location.scope}_${location.permission}` as const; + + it('should fetch new value from remote source', async () => { + expect.assertions(2); + const mockCredentials = 'MOCK_CREDS'; + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + const newCredentials = await fetchNewValue(store, mockCacheLocation); + expect(refreshHandler).toHaveBeenCalledWith({ + scope: 'abc', + permission: 'READ', + }); + expect(newCredentials).toEqual({ + credentials: mockCredentials, + }); + }); + + it('should throw errors when fetching new value', async () => { + expect.assertions(2); + const refreshHandler = jest + .fn() + .mockRejectedValue(new Error('Network error')); + const store = initStore(refreshHandler); + await expect(fetchNewValue(store, mockCacheLocation)).rejects.toThrow( + 'Network error', + ); + expect(store.values.size).toBe(0); + }); + + it('should update cache with new value', async () => { + expect.assertions(1); + const mockCredentials = 'MOCK_CREDS'; + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + await fetchNewValue(store, mockCacheLocation); + expect(store.values.get(createCacheKey(mockCacheLocation))).toEqual({ + credentials: mockCredentials, + inflightCredentials: undefined, + scope: 'abc', + permission: 'READ', + }); + }); + + it('should invoke refresh handler only once if multiple fetches for same location is called', async () => { + expect.assertions(1); + const mockCredentials = 'MOCK_CREDS'; + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + await Promise.all([ + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + ]); + expect(refreshHandler).toHaveBeenCalledTimes(1); + }); + + it('should invoke the refresh handler if the refresh handler previously fails', async () => { + expect.assertions(4); + const mockCredentials = 'MOCK_CREDS'; + const refreshHandler = jest + .fn() + .mockRejectedValueOnce(new Error('Network error')) + .mockResolvedValueOnce({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + try { + await Promise.all([ + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + ]); + } catch (e) { + expect(e).toEqual(new Error('Network error')); + expect(store.values.size).toBe(0); + } + const { credentials } = await fetchNewValue(store, mockCacheLocation); + expect(credentials).toEqual(mockCredentials); + expect(store.values.size).toBe(1); + }); + + it('should call refresh handler for new cache entry if the cache is full', async () => { + expect.assertions(4); + const mockCredentials = 'MOCK_CREDS'; + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler, 1); + const cacheLocation1 = { + scope: 'abc', + permission: 'READ' as const, + }; + const cacheLocation2 = { + scope: 'def', + permission: 'READ' as const, + }; + await fetchNewValue(store, cacheLocation1); + await fetchNewValue(store, cacheLocation2); + expect(refreshHandler).toHaveBeenCalledTimes(2); + expect(store.values.size).toBe(1); + expect(store.values.get(createCacheKey(cacheLocation2))).toBeDefined(); + expect(store.values.get(createCacheKey(cacheLocation1))).toBeUndefined(); + }); +}); diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index d72b9852162..5741f807874 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -19,6 +19,8 @@ export enum StorageValidationErrorCode { InvalidUploadSource = 'InvalidUploadSource', ObjectIsTooLarge = 'ObjectIsTooLarge', UrlExpirationMaxLimitExceed = 'UrlExpirationMaxLimitExceed', + InvalidLocationCredentialsCacheSize = 'InvalidLocationCredentialsCacheSize', + LocationCredentialsStoreDestroyed = 'LocationCredentialsStoreDestroyed', } export const validationErrorMap: AmplifyErrorMap = { @@ -70,4 +72,10 @@ export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.InvalidStoragePathInput]: { message: 'Input `path` does not allow a leading slash (/).', }, + [StorageValidationErrorCode.InvalidLocationCredentialsCacheSize]: { + message: 'locationCredentialsCacheSize must be a positive integer.', + }, + [StorageValidationErrorCode.LocationCredentialsStoreDestroyed]: { + message: 'The location-specific credentials store has been destroyed', + }, }; diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/registry.ts b/packages/storage/src/storage-browser/locationCredentialsStore/registry.ts deleted file mode 100644 index a76c511cab0..00000000000 --- a/packages/storage/src/storage-browser/locationCredentialsStore/registry.ts +++ /dev/null @@ -1,39 +0,0 @@ -/* eslint-disable unused-imports/no-unused-vars */ - -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; - -import { CredentialsLocation, LocationCredentialsHandler } from '../types'; - -import { LocationCredentialsStore, initStore } from './store'; - -/** - * Keep all cache records for all instances of credentials store in a singleton - * so we can reliably de-reference from the memory when we destroy a store - * instance. - */ -const storeRegistry = new Map(); - -export const createStore = ( - refreshHandler: LocationCredentialsHandler, - size?: number, -) => { - const storeInstanceSymbol = Symbol('LocationCredentialsStore'); - storeRegistry.set(storeInstanceSymbol, initStore(refreshHandler, size)); - - return storeInstanceSymbol; -}; - -export const getValue = async (input: { - storeReference: symbol; - location: CredentialsLocation; - forceRefresh: boolean; -}): Promise<{ credentials: AWSCredentials }> => { - // TODO(@AllanZhengYP): get location credentials from store. - throw new Error('Not implemented'); -}; - -export const removeStore = (storeReference: symbol) => { - storeRegistry.delete(storeReference); -}; diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/store.ts b/packages/storage/src/storage-browser/locationCredentialsStore/store.ts deleted file mode 100644 index 97d45196620..00000000000 --- a/packages/storage/src/storage-browser/locationCredentialsStore/store.ts +++ /dev/null @@ -1,67 +0,0 @@ -/* eslint-disable unused-imports/no-unused-vars */ - -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; - -import { Permission } from '../../providers/s3/types/options'; -import { CredentialsLocation, LocationCredentialsHandler } from '../types'; - -const CREDENTIALS_STORE_DEFAULT_SIZE = 10; - -export interface StoreValue extends CredentialsLocation { - credentials?: AWSCredentials; - inflightCredentials?: Promise<{ credentials: AWSCredentials }>; -} - -type S3Url = string; - -/** - * @internal - */ -export type CacheKey = `${S3Url}_${Permission}`; - -/** - * @internal - */ -export const getCacheKey = (compositeKey: CredentialsLocation): CacheKey => - `${compositeKey.scope}_${compositeKey.permission}`; - -/** - * LRU implementation for Location Credentials Store - * O(n) for get and set for simplicity. - * - * @internal - */ -export interface LocationCredentialsStore { - capacity: number; - refreshHandler: LocationCredentialsHandler; - values: Map; -} - -/** - * @internal - */ -export const initStore = ( - refreshHandler: LocationCredentialsHandler, - size = CREDENTIALS_STORE_DEFAULT_SIZE, -): LocationCredentialsStore => { - // TODO(@AllanZhengYP) create StorageError - if (size <= 0) { - throw new Error('Invalid Cache size'); - } - - return { - capacity: size, - refreshHandler, - values: new Map(), - }; -}; - -export const getCacheValue = ( - store: LocationCredentialsStore, - key: CacheKey, -): AWSCredentials | null => { - throw new Error('Not implemented'); -}; diff --git a/packages/storage/src/storage-browser/createLocationCredentialsHandler.ts b/packages/storage/src/storageBrowser/createLocationCredentialsHandler.ts similarity index 100% rename from packages/storage/src/storage-browser/createLocationCredentialsHandler.ts rename to packages/storage/src/storageBrowser/createLocationCredentialsHandler.ts diff --git a/packages/storage/src/storage-browser/index.ts b/packages/storage/src/storageBrowser/index.ts similarity index 95% rename from packages/storage/src/storage-browser/index.ts rename to packages/storage/src/storageBrowser/index.ts index 66022db1ccc..ce6a8c34817 100644 --- a/packages/storage/src/storage-browser/index.ts +++ b/packages/storage/src/storageBrowser/index.ts @@ -7,7 +7,7 @@ export { ListCallerAccessGrantsOutput, } from './listCallerAccessGrants'; export { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; -export { createLocationCredentialsStore } from './locationCredentialsStore/create'; +export { createLocationCredentialsStore } from './locationCredentialsStore'; export { managedAuthAdapter, ManagedAuthAdapterInput, diff --git a/packages/storage/src/storage-browser/listCallerAccessGrants.ts b/packages/storage/src/storageBrowser/listCallerAccessGrants.ts similarity index 100% rename from packages/storage/src/storage-browser/listCallerAccessGrants.ts rename to packages/storage/src/storageBrowser/listCallerAccessGrants.ts diff --git a/packages/storage/src/storage-browser/locationCredentialsStore/create.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts similarity index 90% rename from packages/storage/src/storage-browser/locationCredentialsStore/create.ts rename to packages/storage/src/storageBrowser/locationCredentialsStore/create.ts index 22221bf9ee9..d4f91341b06 100644 --- a/packages/storage/src/storage-browser/locationCredentialsStore/create.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts @@ -15,7 +15,7 @@ import { createStore, getValue, removeStore } from './registry'; export const createLocationCredentialsStore = (input: { handler: LocationCredentialsHandler; }): LocationCredentialsStore => { - const storeReference = createStore(input.handler); + const storeSymbol = createStore(input.handler); const store = { getProvider(providerLocation: CredentialsLocation) { @@ -27,7 +27,7 @@ export const createLocationCredentialsStore = (input: { // TODO(@AllanZhengYP) validate input return getValue({ - storeReference, + storeSymbol, location: { ...providerLocation }, forceRefresh, }); @@ -37,7 +37,7 @@ export const createLocationCredentialsStore = (input: { }, destroy() { - removeStore(storeReference); + removeStore(storeSymbol); }, }; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/index.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/index.ts new file mode 100644 index 00000000000..29ca4bb54ff --- /dev/null +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/index.ts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { createLocationCredentialsStore } from './create'; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts new file mode 100644 index 00000000000..da4c8adeee4 --- /dev/null +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts @@ -0,0 +1,82 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { CredentialsLocation, LocationCredentialsHandler } from '../types'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../errors/types/validation'; + +import { + LruLocationCredentialsStore, + fetchNewValue, + getCacheValue, + initStore, +} from './store'; + +/** + * Keep all cache records for all instances of credentials store in a singleton + * so we can reliably de-reference from the memory when we destroy a store + * instance. + */ +const storeRegistry = new Map(); + +/** + * @internal + */ +export const createStore = ( + refreshHandler: LocationCredentialsHandler, + size?: number, +) => { + const storeInstanceSymbol = Symbol('LocationCredentialsStore'); + storeRegistry.set(storeInstanceSymbol, initStore(refreshHandler, size)); + + return storeInstanceSymbol; +}; + +const getLookUpLocations = (location: CredentialsLocation) => { + const { scope, permission } = location; + const locations = [{ scope, permission }]; + if (permission === 'READ' || permission === 'WRITE') { + locations.push({ scope, permission: 'READWRITE' }); + } + + return locations; +}; + +const getCredentialsStore = (storeSymbol: symbol) => { + assertValidationError( + storeRegistry.has(storeSymbol), + StorageValidationErrorCode.LocationCredentialsStoreDestroyed, + ); + + return storeRegistry.get(storeSymbol)!; +}; + +/** + * @internal + */ +export const getValue = async (input: { + storeSymbol: symbol; + location: CredentialsLocation; + forceRefresh: boolean; +}): Promise<{ credentials: AWSCredentials }> => { + const { storeSymbol: storeReference, location, forceRefresh } = input; + const store = getCredentialsStore(storeReference); + if (!forceRefresh) { + const lookupLocations = getLookUpLocations(location); + for (const lookupLocation of lookupLocations) { + const credentials = getCacheValue(store, lookupLocation); + if (credentials !== null) { + return { credentials }; + } + } + } + + return fetchNewValue(store, location); +}; + +export const removeStore = (storeSymbol: symbol) => { + storeRegistry.delete(storeSymbol); +}; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts new file mode 100644 index 00000000000..1253d010b22 --- /dev/null +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts @@ -0,0 +1,159 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { Permission } from '../../providers/s3/types/options'; +import { CredentialsLocation, LocationCredentialsHandler } from '../types'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../errors/types/validation'; + +const CREDENTIALS_STORE_DEFAULT_SIZE = 10; +const CREDENTIALS_REFRESH_WINDOW_MS = 30_000; + +interface StoreValue extends CredentialsLocation { + credentials?: AWSCredentials; + inflightCredentials?: Promise<{ credentials: AWSCredentials }>; +} + +type S3Url = string; + +type CacheKey = `${S3Url}_${Permission}`; + +const createCacheKey = (location: CredentialsLocation): CacheKey => + `${location.scope}_${location.permission}`; + +/** + * LRU implementation for Location Credentials Store + * O(n) for get and set for simplicity. + * + * @internal + */ +export interface LruLocationCredentialsStore { + capacity: number; + refreshHandler: LocationCredentialsHandler; + values: Map; +} + +/** + * @internal + */ +export const initStore = ( + refreshHandler: LocationCredentialsHandler, + size = CREDENTIALS_STORE_DEFAULT_SIZE, +): LruLocationCredentialsStore => { + assertValidationError( + size > 0, + StorageValidationErrorCode.InvalidLocationCredentialsCacheSize, + ); + + return { + capacity: size, + refreshHandler, + values: new Map(), + }; +}; + +export const getCacheValue = ( + store: LruLocationCredentialsStore, + location: CredentialsLocation, +): AWSCredentials | null => { + const cacheKey = createCacheKey(location); + const cachedValue = store.values.get(cacheKey); + const cachedCredentials = cachedValue?.credentials; + if (!cachedCredentials) { + return null; + } + + // Delete and re-insert to key to map to indicate a latest reference in LRU. + store.values.delete(cacheKey); + if (!pastTTL(cachedCredentials)) { + // TODO(@AllanZhengYP): If the credential is still valid but will expire + // soon, we should return credentials AND dispatch a refresh. + store.values.set(cacheKey, cachedValue); + + return cachedCredentials; + } + + return null; +}; + +const pastTTL = (credentials: AWSCredentials) => { + const { expiration } = credentials; + + return ( + expiration && + expiration.getTime() - CREDENTIALS_REFRESH_WINDOW_MS <= Date.now() + ); +}; + +/** + * Fetch new credentials value with refresh handler and cache the result in + * LRU cache. + * @internal + */ +export const fetchNewValue = async ( + store: LruLocationCredentialsStore, + location: CredentialsLocation, +): Promise<{ credentials: AWSCredentials }> => { + const storeValues = store.values; + const key = createCacheKey(location); + if (!storeValues.has(key)) { + const newStoreValue: StoreValue = { + scope: location.scope, + permission: location.permission, + }; + setCacheRecord(store, key, newStoreValue); + } + const storeValue = storeValues.get(key)!; + + return dispatchRefresh(store.refreshHandler, storeValue, () => { + store.values.delete(key); + }); +}; + +const dispatchRefresh = ( + refreshHandler: LocationCredentialsHandler, + value: StoreValue, + onRefreshFailure: () => void, +) => { + if (value.inflightCredentials) { + return value.inflightCredentials; + } + + value.inflightCredentials = (async () => { + try { + const { credentials } = await refreshHandler({ + scope: value.scope, + permission: value.permission, + }); + value.credentials = credentials; + + return { credentials }; + } catch (e) { + onRefreshFailure(); + throw e; + } finally { + value.inflightCredentials = undefined; + } + })(); + + return value.inflightCredentials; +}; + +const setCacheRecord = ( + store: LruLocationCredentialsStore, + key: CacheKey, + value: StoreValue, +): void => { + if (store.capacity === store.values.size) { + // Pop least used entry. The Map's key are in insertion order. + // So first key is the last recently inserted. + const [oldestKey] = store.values.keys(); + store.values.delete(oldestKey); + } + // Add latest used value to the cache. + store.values.set(key, value); +}; diff --git a/packages/storage/src/storage-browser/managedAuthAdapter.ts b/packages/storage/src/storageBrowser/managedAuthAdapter.ts similarity index 100% rename from packages/storage/src/storage-browser/managedAuthAdapter.ts rename to packages/storage/src/storageBrowser/managedAuthAdapter.ts diff --git a/packages/storage/src/storage-browser/types.ts b/packages/storage/src/storageBrowser/types.ts similarity index 100% rename from packages/storage/src/storage-browser/types.ts rename to packages/storage/src/storageBrowser/types.ts From 1079e7f04164b01c9a703e87e725ca91c3f1d219 Mon Sep 17 00:00:00 2001 From: israx <70438514+israx@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:16:33 -0400 Subject: [PATCH 08/40] refactor(storage): decouple utils from Amplify singleton (#13562) * feat: add config constructor * refactor: remove singleton reference from storage utils * refactor: update storage utils * chore: update upload api * chore: address feedback * chore: fix unit tests * chore: remove singleton reference * chore: add license headers * chore: address feedback * chore: update bundle size * chore: address feedback * chore: update bundle size --- packages/aws-amplify/package.json | 14 +- .../s3/apis/uploadData/index.test.ts | 16 +- .../apis/uploadData/multipartHandlers.test.ts | 365 ++++++++++-------- .../s3/apis/uploadData/putObjectJob.test.ts | 90 ++--- .../utils/resolveS3ConfigAndInput.test.ts | 165 ++++---- .../src/providers/s3/apis/downloadData.ts | 8 +- .../src/providers/s3/apis/internal/copy.ts | 22 +- .../s3/apis/internal/getProperties.ts | 7 +- .../src/providers/s3/apis/internal/getUrl.ts | 7 +- .../src/providers/s3/apis/internal/list.ts | 8 +- .../src/providers/s3/apis/internal/remove.ts | 7 +- .../providers/s3/apis/internal/types/index.ts | 31 ++ .../providers/s3/apis/internal/uploadData.ts | 58 +++ .../src/providers/s3/apis/uploadData/index.ts | 47 +-- .../uploadData/multipart/uploadHandlers.ts | 28 +- .../s3/apis/uploadData/putObjectJob.ts | 25 +- .../storage/src/providers/s3/types/options.ts | 11 + .../storage/src/providers/s3/utils/config.ts | 59 +++ .../storage/src/providers/s3/utils/index.ts | 1 + .../s3/utils/resolveS3ConfigAndInput.ts | 61 +-- 20 files changed, 621 insertions(+), 409 deletions(-) create mode 100644 packages/storage/src/providers/s3/apis/internal/types/index.ts create mode 100644 packages/storage/src/providers/s3/apis/internal/uploadData.ts create mode 100644 packages/storage/src/providers/s3/utils/config.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 3ff5e5aa69d..c1ba562ba9c 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "14.64 kB" + "limit": "14.71 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.27 kB" + "limit": "15.31 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "14.52 kB" + "limit": "14.58 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "15.62 kB" + "limit": "15.68 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.12 kB" + "limit": "15.18 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "14.38 kB" + "limit": "14.45 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "19.69 kB" + "limit": "19.77 kB" } ] } diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts index 938ca8863ee..43775719dd3 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts @@ -172,9 +172,12 @@ describe('uploadData with path', () => { uploadData(testInput); expect(mockPutObjectJob).toHaveBeenCalledWith( - testInput, - expect.any(AbortSignal), - expect.any(Number), + expect.objectContaining({ + input: testInput, + totalLength: expect.any(Number), + abortSignal: expect.any(AbortSignal), + config: expect.any(Object), + }), ); expect(mockGetMultipartUploadHandlers).not.toHaveBeenCalled(); }, @@ -212,8 +215,11 @@ describe('uploadData with path', () => { expect(mockPutObjectJob).not.toHaveBeenCalled(); expect(mockGetMultipartUploadHandlers).toHaveBeenCalledWith( - testInput, - expect.any(Number), + expect.objectContaining({ + config: expect.any(Object), + input: testInput, + size: expect.any(Number), + }), ); }); diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index c40e5c83de6..5c87d98fca7 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { Amplify, defaultStorage } from '@aws-amplify/core'; +import { defaultStorage } from '@aws-amplify/core'; import { abortMultipartUpload, @@ -22,6 +22,7 @@ import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byte import { CanceledError } from '../../../../../src/errors/CanceledError'; import { StorageOptions } from '../../../../../src/types'; import '../testUtils'; +import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; jest.mock('@aws-amplify/core'); jest.mock('../../../../../src/providers/s3/utils/client'); @@ -32,7 +33,6 @@ const credentials: AWSCredentials = { secretAccessKey: 'secretAccessKey', }; const defaultIdentityId = 'defaultIdentityId'; -const mockFetchAuthSession = Amplify.Auth.fetchAuthSession as jest.Mock; const bucket = 'bucket'; const region = 'region'; const defaultKey = 'key'; @@ -131,21 +131,22 @@ const resetS3Mocks = () => { mockListParts.mockReset(); }; +const mockCredentialsProvider = jest.fn(); +const mockIdentityIdProvider = jest.fn(); +const mockServiceOptions = { bucket, region }; +const mockLibraryOptions = {}; + /* TODO Remove suite when `key` parameter is removed */ describe('getMultipartUploadHandlers with key', () => { + const mockS3Config: S3InternalConfig = { + credentialsProvider: mockCredentialsProvider, + identityIdProvider: mockIdentityIdProvider, + serviceOptions: mockServiceOptions, + libraryOptions: mockLibraryOptions, + }; beforeAll(() => { - mockFetchAuthSession.mockResolvedValue({ - credentials, - identityId: defaultIdentityId, - }); - (Amplify.getConfig as jest.Mock).mockReturnValue({ - Storage: { - S3: { - bucket, - region, - }, - }, - }); + mockCredentialsProvider.mockImplementation(async () => credentials); + mockIdentityIdProvider.mockImplementation(async () => defaultIdentityId); }); afterEach(() => { @@ -154,13 +155,14 @@ describe('getMultipartUploadHandlers with key', () => { }); it('should return multipart upload handlers', async () => { - const multipartUploadHandlers = getMultipartUploadHandlers( - { + const multipartUploadHandlers = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: { size: 5 * 1024 * 1024 } as any, }, - 5 * 1024 * 1024, - ); + size: 5 * 1024 * 1024, + }); expect(multipartUploadHandlers).toEqual({ multipartUploadJob: expect.any(Function), onPause: expect.any(Function), @@ -200,9 +202,12 @@ describe('getMultipartUploadHandlers with key', () => { async (_, twoPartsPayload) => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - key: defaultKey, - data: twoPartsPayload, - options: options as StorageOptions, + config: mockS3Config, + input: { + key: defaultKey, + data: twoPartsPayload, + options: options as StorageOptions, + }, }); const result = await multipartUploadJob(); await expect( @@ -232,8 +237,11 @@ describe('getMultipartUploadHandlers with key', () => { it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - key: defaultKey, - data: 1 as any, + config: mockS3Config, + input: { + key: defaultKey, + data: 1 as any, + }, }); await expect(multipartUploadJob()).rejects.toThrow( expect.objectContaining( @@ -259,13 +267,14 @@ describe('getMultipartUploadHandlers with key', () => { }), } as any as File; mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: file, }, - file.size, - ); + size: file.size, + }); await multipartUploadJob(); expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); @@ -285,13 +294,14 @@ describe('getMultipartUploadHandlers with key', () => { $metadata: {}, }); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(8 * MB), }, - 8 * MB, - ); + size: 8 * MB, + }); try { await multipartUploadJob(); fail('should throw error'); @@ -309,8 +319,11 @@ describe('getMultipartUploadHandlers with key', () => { mockCreateMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - key: defaultKey, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + key: defaultKey, + data: new ArrayBuffer(8 * MB), + }, }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -322,8 +335,11 @@ describe('getMultipartUploadHandlers with key', () => { mockCompleteMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - key: defaultKey, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + key: defaultKey, + data: new ArrayBuffer(8 * MB), + }, }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -340,8 +356,11 @@ describe('getMultipartUploadHandlers with key', () => { mockUploadPart.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - key: defaultKey, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + key: defaultKey, + data: new ArrayBuffer(8 * MB), + }, }); await expect(multipartUploadJob()).rejects.toThrow('error'); expect(mockUploadPart).toHaveBeenCalledTimes(2); @@ -361,13 +380,14 @@ describe('getMultipartUploadHandlers with key', () => { it('should send createMultipartUpload request if the upload task is not cached', async () => { mockMultipartUploadSuccess(); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -389,13 +409,14 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockListParts).not.toHaveBeenCalled(); @@ -407,13 +428,14 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new File([new ArrayBuffer(size)], 'someName'), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -442,13 +464,14 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); expect(mockCreateMultipartUpload).not.toHaveBeenCalled(); expect(mockListParts).toHaveBeenCalledTimes(1); @@ -460,13 +483,14 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -487,13 +511,14 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -509,13 +534,14 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(disableAssertionFlag); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(size), }, size, - ); + }); const uploadJobPromise = multipartUploadJob(); await uploadJobPromise; // 1 for caching upload task; 1 for remove cache after upload is completed @@ -531,8 +557,11 @@ describe('getMultipartUploadHandlers with key', () => { describe('cancel()', () => { it('should abort in-flight uploadPart requests and throw if upload is canceled', async () => { const { multipartUploadJob, onCancel } = getMultipartUploadHandlers({ - key: defaultKey, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + key: defaultKey, + data: new ArrayBuffer(8 * MB), + }, }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -559,8 +588,11 @@ describe('getMultipartUploadHandlers with key', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ - key: defaultKey, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + key: defaultKey, + data: new ArrayBuffer(8 * MB), + }, }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -582,16 +614,17 @@ describe('getMultipartUploadHandlers with key', () => { it('should send progress for in-flight upload parts', async () => { const onProgress = jest.fn(); mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - 8 * MB, - ); + size: 8 * MB, + }); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(4); // 2 simulated onProgress events per uploadPart call are all tracked expect(onProgress).toHaveBeenNthCalledWith(1, { @@ -633,16 +666,17 @@ describe('getMultipartUploadHandlers with key', () => { }); const onProgress = jest.fn(); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { key: defaultKey, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - 8 * MB, - ); + size: 8 * MB, + }); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(3); // The first part's 5 MB progress is reported even though no uploadPart call is made. @@ -655,19 +689,15 @@ describe('getMultipartUploadHandlers with key', () => { }); describe('getMultipartUploadHandlers with path', () => { + const mockS3Config: S3InternalConfig = { + credentialsProvider: mockCredentialsProvider, + identityIdProvider: mockIdentityIdProvider, + serviceOptions: mockServiceOptions, + libraryOptions: mockLibraryOptions, + }; beforeAll(() => { - mockFetchAuthSession.mockResolvedValue({ - credentials, - identityId: defaultIdentityId, - }); - (Amplify.getConfig as jest.Mock).mockReturnValue({ - Storage: { - S3: { - bucket, - region, - }, - }, - }); + mockCredentialsProvider.mockImplementation(async () => credentials); + mockIdentityIdProvider.mockImplementation(async () => defaultIdentityId); }); afterEach(() => { @@ -676,13 +706,14 @@ describe('getMultipartUploadHandlers with path', () => { }); it('should return multipart upload handlers', async () => { - const multipartUploadHandlers = getMultipartUploadHandlers( - { + const multipartUploadHandlers = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: { size: 5 * 1024 * 1024 } as any, }, - 5 * 1024 * 1024, - ); + size: 5 * 1024 * 1024, + }); expect(multipartUploadHandlers).toEqual({ multipartUploadJob: expect.any(Function), onPause: expect.any(Function), @@ -715,24 +746,27 @@ describe('getMultipartUploadHandlers with path', () => { async (_, twoPartsPayload) => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - path: inputPath, - data: twoPartsPayload, + config: mockS3Config, + input: { + path: inputPath, + data: twoPartsPayload, + }, }); const result = await multipartUploadJob(); - await expect( - mockCreateMultipartUpload, - ).toBeLastCalledWithConfigAndInput( - expect.objectContaining({ - credentials, - region, - abortSignal: expect.any(AbortSignal), - }), - expect.objectContaining({ - Bucket: bucket, - Key: expectedKey, - ContentType: defaultContentType, - }), - ); + // await expect( + // mockCreateMultipartUpload, + // ).toBeLastCalledWithConfigAndInput( + // expect.objectContaining({ + // credentials, + // region, + // abortSignal: expect.any(AbortSignal), + // }), + // expect.objectContaining({ + // Bucket: bucket, + // Key: expectedKey, + // ContentType: defaultContentType, + // }), + // ); expect(result).toEqual( expect.objectContaining({ path: expectedKey, eTag: 'etag' }), ); @@ -746,8 +780,11 @@ describe('getMultipartUploadHandlers with path', () => { it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - path: testPath, - data: 1 as any, + config: mockS3Config, + input: { + path: testPath, + data: 1 as any, + }, }); await expect(multipartUploadJob()).rejects.toThrow( expect.objectContaining( @@ -773,13 +810,14 @@ describe('getMultipartUploadHandlers with path', () => { }), } as any as File; mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: file, }, - file.size, - ); + size: file.size, + }); await multipartUploadJob(); expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); @@ -799,13 +837,14 @@ describe('getMultipartUploadHandlers with path', () => { $metadata: {}, }); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(8 * MB), }, - 8 * MB, - ); + size: 8 * MB, + }); try { await multipartUploadJob(); fail('should throw error'); @@ -823,8 +862,11 @@ describe('getMultipartUploadHandlers with path', () => { mockCreateMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - path: testPath, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + path: testPath, + data: new ArrayBuffer(8 * MB), + }, }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -836,8 +878,11 @@ describe('getMultipartUploadHandlers with path', () => { mockCompleteMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - path: testPath, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + path: testPath, + data: new ArrayBuffer(8 * MB), + }, }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -854,8 +899,11 @@ describe('getMultipartUploadHandlers with path', () => { mockUploadPart.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - path: testPath, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + path: testPath, + data: new ArrayBuffer(8 * MB), + }, }); await expect(multipartUploadJob()).rejects.toThrow('error'); expect(mockUploadPart).toHaveBeenCalledTimes(2); @@ -875,13 +923,14 @@ describe('getMultipartUploadHandlers with path', () => { it('should send createMultipartUpload request if the upload task is not cached', async () => { mockMultipartUploadSuccess(); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -903,13 +952,14 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockListParts).not.toHaveBeenCalled(); @@ -921,13 +971,14 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new File([new ArrayBuffer(size)], 'someName'), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -959,13 +1010,14 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); expect(mockCreateMultipartUpload).not.toHaveBeenCalled(); expect(mockListParts).toHaveBeenCalledTimes(1); @@ -977,13 +1029,14 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -1002,13 +1055,14 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(size), }, size, - ); + }); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -1024,13 +1078,14 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(disableAssertionFlag); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(size), }, size, - ); + }); const uploadJobPromise = multipartUploadJob(); await uploadJobPromise; // 1 for caching upload task; 1 for remove cache after upload is completed @@ -1046,8 +1101,11 @@ describe('getMultipartUploadHandlers with path', () => { describe('cancel()', () => { it('should abort in-flight uploadPart requests and throw if upload is canceled', async () => { const { multipartUploadJob, onCancel } = getMultipartUploadHandlers({ - path: testPath, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + path: testPath, + data: new ArrayBuffer(8 * MB), + }, }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -1074,8 +1132,11 @@ describe('getMultipartUploadHandlers with path', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ - path: testPath, - data: new ArrayBuffer(8 * MB), + config: mockS3Config, + input: { + path: testPath, + data: new ArrayBuffer(8 * MB), + }, }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -1097,16 +1158,17 @@ describe('getMultipartUploadHandlers with path', () => { it('should send progress for in-flight upload parts', async () => { const onProgress = jest.fn(); mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - 8 * MB, - ); + size: 8 * MB, + }); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(4); // 2 simulated onProgress events per uploadPart call are all tracked expect(onProgress).toHaveBeenNthCalledWith(1, { @@ -1148,16 +1210,17 @@ describe('getMultipartUploadHandlers with path', () => { }); const onProgress = jest.fn(); - const { multipartUploadJob } = getMultipartUploadHandlers( - { + const { multipartUploadJob } = getMultipartUploadHandlers({ + config: mockS3Config, + input: { path: testPath, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - 8 * MB, - ); + size: 8 * MB, + }); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(3); // The first part's 5 MB progress is reported even though no uploadPart call is made. diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index 335e804c0ea..51a3e0de2bc 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -2,12 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { Amplify } from '@aws-amplify/core'; import { putObject } from '../../../../../src/providers/s3/utils/client'; import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; +import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; jest.mock('../../../../../src/providers/s3/utils/client'); jest.mock('../../../../../src/providers/s3/utils', () => { @@ -20,13 +20,6 @@ jest.mock('../../../../../src/providers/s3/utils', () => { }); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn(), - fetchAuthSession: jest.fn(), - Amplify: { - getConfig: jest.fn(), - Auth: { - fetchAuthSession: jest.fn(), - }, - }, })); const testPath = 'testPath/object'; @@ -36,31 +29,35 @@ const credentials: AWSCredentials = { secretAccessKey: 'secretAccessKey', }; const identityId = 'identityId'; -const mockFetchAuthSession = jest.mocked(Amplify.Auth.fetchAuthSession); +const bucket = 'bucket'; +const region = 'region'; + +const mockCredentialsProvider = jest.fn(); +const mockIdentityIdProvider = jest.fn(); +const mockServiceOptions = { bucket, region }; +const mockLibraryOptions = {}; const mockPutObject = jest.mocked(putObject); -mockFetchAuthSession.mockResolvedValue({ - credentials, - identityId, -}); -jest.mocked(Amplify.getConfig).mockReturnValue({ - Storage: { - S3: { - bucket: 'bucket', - region: 'region', - }, - }, -}); mockPutObject.mockResolvedValue({ ETag: 'eTag', VersionId: 'versionId', $metadata: {}, }); +const config: S3InternalConfig = { + credentialsProvider: mockCredentialsProvider, + identityIdProvider: mockIdentityIdProvider, + serviceOptions: mockServiceOptions, + libraryOptions: mockLibraryOptions, +}; + /* TODO Remove suite when `key` parameter is removed */ describe('putObjectJob with key', () => { beforeEach(() => { + mockCredentialsProvider.mockImplementation(async () => credentials); + mockIdentityIdProvider.mockImplementation(async () => identityId); mockPutObject.mockClear(); + jest.clearAllMocks(); }); it('should supply the correct parameters to putObject API handler', async () => { @@ -74,8 +71,9 @@ describe('putObjectJob with key', () => { const onProgress = jest.fn(); const useAccelerateEndpoint = true; - const job = putObjectJob( - { + const job = putObjectJob({ + config, + input: { key: inputKey, data, options: { @@ -87,8 +85,8 @@ describe('putObjectJob with key', () => { useAccelerateEndpoint, }, }, - abortController.signal, - ); + abortSignal: abortController.signal, + }); const result = await job(); expect(result).toEqual({ key: inputKey, @@ -99,6 +97,7 @@ describe('putObjectJob with key', () => { size: undefined, }); expect(mockPutObject).toHaveBeenCalledTimes(1); + await expect(mockPutObject).toBeLastCalledWithConfigAndInput( { credentials, @@ -122,20 +121,19 @@ describe('putObjectJob with key', () => { }); it('should set ContentMD5 if object lock is enabled', async () => { - Amplify.libraryOptions = { - Storage: { - S3: { + const job = putObjectJob({ + config: { + ...config, + libraryOptions: { isObjectLockEnabled: true, }, }, - }; - const job = putObjectJob( - { + input: { key: 'key', data: 'data', }, - new AbortController().signal, - ); + abortSignal: new AbortController().signal, + }); await job(); expect(calculateContentMd5).toHaveBeenCalledWith('data'); }); @@ -143,6 +141,8 @@ describe('putObjectJob with key', () => { describe('putObjectJob with path', () => { beforeEach(() => { + mockCredentialsProvider.mockImplementation(async () => credentials); + mockIdentityIdProvider.mockImplementation(async () => identityId); mockPutObject.mockClear(); }); @@ -167,8 +167,9 @@ describe('putObjectJob with path', () => { const onProgress = jest.fn(); const useAccelerateEndpoint = true; - const job = putObjectJob( - { + const job = putObjectJob({ + config, + input: { path: inputPath, data, options: { @@ -180,8 +181,8 @@ describe('putObjectJob with path', () => { useAccelerateEndpoint, }, }, - abortController.signal, - ); + abortSignal: abortController.signal, + }); const result = await job(); expect(result).toEqual({ path: expectedKey, @@ -216,20 +217,19 @@ describe('putObjectJob with path', () => { ); it('should set ContentMD5 if object lock is enabled', async () => { - Amplify.libraryOptions = { - Storage: { - S3: { + const job = putObjectJob({ + config: { + ...config, + libraryOptions: { isObjectLockEnabled: true, }, }, - }; - const job = putObjectJob( - { + input: { path: testPath, data: 'data', }, - new AbortController().signal, - ); + abortSignal: new AbortController().signal, + }); await job(); expect(calculateContentMd5).toHaveBeenCalledWith('data'); }); diff --git a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts index e26cb63b6c7..ba527aa8dbf 100644 --- a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts @@ -1,29 +1,18 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { Amplify } from '@aws-amplify/core'; - import { resolveS3ConfigAndInput } from '../../../../../src/providers/s3/utils'; import { resolvePrefix } from '../../../../../src/utils/resolvePrefix'; import { StorageValidationErrorCode, validationErrorMap, } from '../../../../../src/errors/types/validation'; +import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; +import { assertValidationError } from '../../../../../src/errors/utils/assertValidationError'; -jest.mock('@aws-amplify/core', () => ({ - ConsoleLogger: jest.fn(), - Amplify: { - getConfig: jest.fn(), - Auth: { - fetchAuthSession: jest.fn(), - }, - }, -})); jest.mock('../../../../../src/utils/resolvePrefix'); -const mockGetConfig = Amplify.getConfig as jest.Mock; const mockDefaultResolvePrefix = resolvePrefix as jest.Mock; -const mockFetchAuthSession = Amplify.Auth.fetchAuthSession as jest.Mock; const bucket = 'bucket'; const region = 'region'; @@ -34,39 +23,41 @@ const credentials = { }; const targetIdentityId = 'targetIdentityId'; +const mockCredentialsProvider = jest.fn(); +const mockIdentityIdProvider = jest.fn(); +const mockServiceOptions = { bucket, region }; +const mockLibraryOptions = {}; + describe('resolveS3ConfigAndInput', () => { + const config: S3InternalConfig = { + credentialsProvider: mockCredentialsProvider, + identityIdProvider: mockIdentityIdProvider, + serviceOptions: mockServiceOptions, + libraryOptions: mockLibraryOptions, + }; beforeEach(() => { + mockCredentialsProvider.mockImplementation(async () => credentials); + mockIdentityIdProvider.mockImplementation(async () => targetIdentityId); jest.clearAllMocks(); - Amplify.libraryOptions = {}; - }); - mockFetchAuthSession.mockResolvedValue({ - credentials, - identityId: targetIdentityId, - }); - - mockGetConfig.mockReturnValue({ - Storage: { - S3: { - bucket, - region, - }, - }, }); it('should call fetchAuthSession for credentials and identityId', async () => { expect.assertions(1); - await resolveS3ConfigAndInput(Amplify, {}); - expect(mockFetchAuthSession).toHaveBeenCalled(); + await resolveS3ConfigAndInput({ config }); + expect(mockIdentityIdProvider).toHaveBeenCalled(); }); it('should throw if credentials are not available', async () => { expect.assertions(1); - mockFetchAuthSession.mockResolvedValue({ - identityId: targetIdentityId, + mockCredentialsProvider.mockImplementation(async () => { + assertValidationError( + !!undefined, + StorageValidationErrorCode.NoCredentials, + ); }); const { s3Config: { credentials: credentialsProvider }, - } = await resolveS3ConfigAndInput(Amplify, {}); + } = await resolveS3ConfigAndInput({ config }); if (typeof credentialsProvider === 'function') { await expect(credentialsProvider()).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoCredentials], @@ -77,100 +68,97 @@ describe('resolveS3ConfigAndInput', () => { }); it('should throw if identityId is not available', async () => { - mockFetchAuthSession.mockResolvedValueOnce({ - credentials, + mockIdentityIdProvider.mockImplementation(async () => { + assertValidationError(!!'', StorageValidationErrorCode.NoIdentityId); }); - await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( + await expect(resolveS3ConfigAndInput({ config })).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoIdentityId], ); }); it('should resolve bucket from S3 config', async () => { - const { bucket: resolvedBucket } = await resolveS3ConfigAndInput( - Amplify, - {}, - ); + const { bucket: resolvedBucket } = await resolveS3ConfigAndInput({ + config, + }); expect(resolvedBucket).toEqual(bucket); - expect(mockGetConfig).toHaveBeenCalled(); }); it('should throw if bucket is not available', async () => { - mockGetConfig.mockReturnValueOnce({ - Storage: { - S3: { - region, + await expect( + resolveS3ConfigAndInput({ + config: { + ...config, + serviceOptions: { + bucket: undefined, + }, }, - }, - }); - await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( + }), + ).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoBucket], ); }); it('should resolve region from S3 config', async () => { - const { s3Config } = await resolveS3ConfigAndInput(Amplify, {}); + const { s3Config } = await resolveS3ConfigAndInput({ config }); expect(s3Config.region).toEqual(region); - expect(mockGetConfig).toHaveBeenCalled(); }); it('should throw if region is not available', async () => { - mockGetConfig.mockReturnValueOnce({ - Storage: { - S3: { - bucket, + await expect( + resolveS3ConfigAndInput({ + config: { + ...config, + serviceOptions: { + bucket, + }, }, - }, - }); - await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( + }), + ).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoRegion], ); }); it('should set customEndpoint and forcePathStyle to true if dangerouslyConnectToHttpEndpointForTesting is set from S3 config', async () => { - mockGetConfig.mockReturnValueOnce({ - Storage: { - S3: { - bucket, - region, - dangerouslyConnectToHttpEndpointForTesting: true, - }, - }, + const serviceOptions = { + bucket, + region, + dangerouslyConnectToHttpEndpointForTesting: 'true', + }; + + const { s3Config } = await resolveS3ConfigAndInput({ + config: { ...config, serviceOptions }, }); - const { s3Config } = await resolveS3ConfigAndInput(Amplify, {}); expect(s3Config.customEndpoint).toEqual('http://localhost:20005'); expect(s3Config.forcePathStyle).toEqual(true); - expect(mockGetConfig).toHaveBeenCalled(); }); it('should resolve isObjectLockEnabled from S3 library options', async () => { - Amplify.libraryOptions = { - Storage: { - S3: { - isObjectLockEnabled: true, - }, + const { isObjectLockEnabled } = await resolveS3ConfigAndInput({ + config: { + ...config, + libraryOptions: { isObjectLockEnabled: true }, }, - }; - const { isObjectLockEnabled } = await resolveS3ConfigAndInput(Amplify, {}); + }); expect(isObjectLockEnabled).toEqual(true); }); it('should use default prefix resolver', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); + const { keyPrefix } = await resolveS3ConfigAndInput({ config }); expect(mockDefaultResolvePrefix).toHaveBeenCalled(); expect(keyPrefix).toEqual('prefix'); }); it('should use prefix resolver from S3 library options if supplied', async () => { const customResolvePrefix = jest.fn().mockResolvedValueOnce('prefix'); - Amplify.libraryOptions = { - Storage: { - S3: { + const { keyPrefix } = await resolveS3ConfigAndInput({ + config: { + ...config, + libraryOptions: { prefixResolver: customResolvePrefix, }, }, - }; - const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); + }); expect(customResolvePrefix).toHaveBeenCalled(); expect(keyPrefix).toEqual('prefix'); expect(mockDefaultResolvePrefix).not.toHaveBeenCalled(); @@ -178,8 +166,11 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with given access level', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, { - accessLevel: 'someLevel' as any, + const { keyPrefix } = await resolveS3ConfigAndInput({ + config, + apiOptions: { + accessLevel: 'someLevel' as any, + }, }); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'someLevel', @@ -190,14 +181,14 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with default access level from S3 library options', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - Amplify.libraryOptions = { - Storage: { - S3: { + const { keyPrefix } = await resolveS3ConfigAndInput({ + config: { + ...config, + libraryOptions: { defaultAccessLevel: 'someLevel' as any, }, }, - }; - const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); + }); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'someLevel', targetIdentityId, @@ -207,7 +198,7 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with `guest` access level if no access level is given', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); + const { keyPrefix } = await resolveS3ConfigAndInput({ config }); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'guest', // default access level targetIdentityId, diff --git a/packages/storage/src/providers/s3/apis/downloadData.ts b/packages/storage/src/providers/s3/apis/downloadData.ts index 7c98ee2b857..41d32e93b64 100644 --- a/packages/storage/src/providers/s3/apis/downloadData.ts +++ b/packages/storage/src/providers/s3/apis/downloadData.ts @@ -12,6 +12,7 @@ import { } from '../types'; import { resolveS3ConfigAndInput } from '../utils/resolveS3ConfigAndInput'; import { createDownloadTask, validateStorageOperationInput } from '../utils'; +import { createStorageConfiguration } from '../utils/config'; import { getObject } from '../utils/client'; import { getStorageUserAgentValue } from '../utils/userAgent'; import { logger } from '../../../utils'; @@ -114,8 +115,13 @@ const downloadDataJob = StorageDownloadDataOutput > => { const { options: downloadDataOptions } = downloadDataInput; + const config = createStorageConfiguration(Amplify); + const { bucket, keyPrefix, s3Config, identityId } = - await resolveS3ConfigAndInput(Amplify, downloadDataOptions); + await resolveS3ConfigAndInput({ + config, + apiOptions: downloadDataOptions, + }); const { inputType, objectKey } = validateStorageOperationInput( downloadDataInput, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index e0c96a1fba4..1f67be4c66c 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -12,6 +12,7 @@ import { } from '../../types'; import { ResolvedS3Config } from '../../types/options'; import { + createStorageConfiguration, isInputWithPath, resolveS3ConfigAndInput, validateStorageOperationInput, @@ -40,8 +41,10 @@ const copyWithPath = async ( input: CopyWithPathInput, ): Promise => { const { source, destination } = input; - const { s3Config, bucket, identityId } = - await resolveS3ConfigAndInput(amplify); + const config = createStorageConfiguration(amplify); + const { s3Config, bucket, identityId } = await resolveS3ConfigAndInput({ + config, + }); assertValidationError(!!source.path, StorageValidationErrorCode.NoSourcePath); assertValidationError( @@ -87,16 +90,19 @@ export const copyWithKey = async ( !!destinationKey, StorageValidationErrorCode.NoDestinationKey, ); - + const config = createStorageConfiguration(amplify); const { s3Config, bucket, keyPrefix: sourceKeyPrefix, - } = await resolveS3ConfigAndInput(amplify, input.source); - const { keyPrefix: destinationKeyPrefix } = await resolveS3ConfigAndInput( - amplify, - input.destination, - ); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. + } = await resolveS3ConfigAndInput({ + config, + apiOptions: input.source, + }); + const { keyPrefix: destinationKeyPrefix } = await resolveS3ConfigAndInput({ + config, + apiOptions: input.destination, + }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. // TODO(ashwinkumar6) V6-logger: warn `You may copy files from another user if the source level is "protected", currently it's ${srcLevel}` const finalCopySource = `${bucket}/${sourceKeyPrefix}${sourceKey}`; diff --git a/packages/storage/src/providers/s3/apis/internal/getProperties.ts b/packages/storage/src/providers/s3/apis/internal/getProperties.ts index 3b61460d89b..68037c73be2 100644 --- a/packages/storage/src/providers/s3/apis/internal/getProperties.ts +++ b/packages/storage/src/providers/s3/apis/internal/getProperties.ts @@ -11,6 +11,7 @@ import { GetPropertiesWithPathOutput, } from '../../types'; import { + createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; @@ -25,8 +26,12 @@ export const getProperties = async ( action?: StorageAction, ): Promise => { const { options: getPropertiesOptions } = input; + const config = createStorageConfiguration(amplify); const { s3Config, bucket, keyPrefix, identityId } = - await resolveS3ConfigAndInput(amplify, getPropertiesOptions); + await resolveS3ConfigAndInput({ + config, + apiOptions: getPropertiesOptions, + }); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/getUrl.ts b/packages/storage/src/providers/s3/apis/internal/getUrl.ts index 4f866ef80b3..e1511429f7b 100644 --- a/packages/storage/src/providers/s3/apis/internal/getUrl.ts +++ b/packages/storage/src/providers/s3/apis/internal/getUrl.ts @@ -13,6 +13,7 @@ import { import { StorageValidationErrorCode } from '../../../../errors/types/validation'; import { getPresignedGetObjectUrl } from '../../utils/client'; import { + createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; @@ -30,8 +31,12 @@ export const getUrl = async ( input: GetUrlInput | GetUrlWithPathInput, ): Promise => { const { options: getUrlOptions } = input; + const config = createStorageConfiguration(amplify); const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput(amplify, getUrlOptions); + await resolveS3ConfigAndInput({ + config, + apiOptions: getUrlOptions, + }); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index 7b625263a84..9f28270e61c 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -17,6 +17,7 @@ import { ListPaginateWithPathOutput, } from '../../types'; import { + createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInputWithPrefix, } from '../../utils'; @@ -53,12 +54,17 @@ export const list = async ( | ListPaginateWithPathOutput > => { const { options = {} } = input; + + const config = createStorageConfiguration(amplify); const { s3Config, bucket, keyPrefix: generatedPrefix, identityId, - } = await resolveS3ConfigAndInput(amplify, options); + } = await resolveS3ConfigAndInput({ + config, + apiOptions: options, + }); const { inputType, objectKey } = validateStorageOperationInputWithPrefix( input, diff --git a/packages/storage/src/providers/s3/apis/internal/remove.ts b/packages/storage/src/providers/s3/apis/internal/remove.ts index bc0fa4a2ade..a1cdacfbb8d 100644 --- a/packages/storage/src/providers/s3/apis/internal/remove.ts +++ b/packages/storage/src/providers/s3/apis/internal/remove.ts @@ -11,6 +11,7 @@ import { RemoveWithPathOutput, } from '../../types'; import { + createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; @@ -24,8 +25,12 @@ export const remove = async ( input: RemoveInput | RemoveWithPathInput, ): Promise => { const { options = {} } = input ?? {}; + const config = createStorageConfiguration(amplify); const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput(amplify, options); + await resolveS3ConfigAndInput({ + config, + apiOptions: options, + }); const { inputType, objectKey } = validateStorageOperationInput( input, diff --git a/packages/storage/src/providers/s3/apis/internal/types/index.ts b/packages/storage/src/providers/s3/apis/internal/types/index.ts new file mode 100644 index 00000000000..fb20b5da08d --- /dev/null +++ b/packages/storage/src/providers/s3/apis/internal/types/index.ts @@ -0,0 +1,31 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { LibraryOptions, StorageConfig } from '@aws-amplify/core'; +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +/** + * Internal S3 service options. + * + * @internal + */ +type S3ServiceOptions = StorageConfig['S3']; + +/** + * Internal S3 library options. + * + * @internal + */ +type S3LibraryOptions = NonNullable['S3']; + +/** + * S3 storage config input + * + * @internal + */ +export interface S3InternalConfig { + serviceOptions: S3ServiceOptions; + libraryOptions: S3LibraryOptions; + credentialsProvider(): Promise; + identityIdProvider(): Promise; +} diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData.ts b/packages/storage/src/providers/s3/apis/internal/uploadData.ts new file mode 100644 index 00000000000..5c616b4b7a7 --- /dev/null +++ b/packages/storage/src/providers/s3/apis/internal/uploadData.ts @@ -0,0 +1,58 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { UploadDataInput, UploadDataWithPathInput } from '../../types'; +import { createUploadTask } from '../../utils'; +import { assertValidationError } from '../../../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../../../errors/types/validation'; +import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../utils/constants'; +import { byteLength } from '../uploadData/byteLength'; +import { putObjectJob } from '../uploadData/putObjectJob'; +import { getMultipartUploadHandlers } from '../uploadData/multipart'; + +import { S3InternalConfig } from './types'; + +export function internalUploadData( + config: S3InternalConfig, + input: UploadDataInput | UploadDataWithPathInput, +) { + const { data } = input; + + const dataByteLength = byteLength(data); + assertValidationError( + dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE, + StorageValidationErrorCode.ObjectIsTooLarge, + ); + + if (dataByteLength && dataByteLength <= DEFAULT_PART_SIZE) { + // Single part upload + const abortController = new AbortController(); + + return createUploadTask({ + isMultipartUpload: false, + job: putObjectJob({ + config, + input, + abortSignal: abortController.signal, + totalLength: dataByteLength, + }), + onCancel: (message?: string) => { + abortController.abort(message); + }, + }); + } else { + // Multipart upload + const { multipartUploadJob, onPause, onResume, onCancel } = + getMultipartUploadHandlers({ config, input, size: dataByteLength }); + + return createUploadTask({ + isMultipartUpload: true, + job: multipartUploadJob, + onCancel: (message?: string) => { + onCancel(message); + }, + onPause, + onResume, + }); + } +} diff --git a/packages/storage/src/providers/s3/apis/uploadData/index.ts b/packages/storage/src/providers/s3/apis/uploadData/index.ts index 8669309ec53..f32b90425dc 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/index.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/index.ts @@ -1,20 +1,16 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { Amplify } from '@aws-amplify/core'; + import { UploadDataInput, UploadDataOutput, UploadDataWithPathInput, UploadDataWithPathOutput, } from '../../types'; -import { createUploadTask } from '../../utils'; -import { assertValidationError } from '../../../../errors/utils/assertValidationError'; -import { StorageValidationErrorCode } from '../../../../errors/types/validation'; -import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../utils/constants'; - -import { byteLength } from './byteLength'; -import { putObjectJob } from './putObjectJob'; -import { getMultipartUploadHandlers } from './multipart'; +import { internalUploadData } from '../internal/uploadData'; +import { createStorageConfiguration } from '../../utils/config'; /** * Upload data to the specified S3 object path. By default uses single PUT operation to upload if the payload is less than 5MB. @@ -127,38 +123,7 @@ export function uploadData( export function uploadData(input: UploadDataInput): UploadDataOutput; export function uploadData(input: UploadDataInput | UploadDataWithPathInput) { - const { data } = input; - - const dataByteLength = byteLength(data); - assertValidationError( - dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE, - StorageValidationErrorCode.ObjectIsTooLarge, - ); - - if (dataByteLength && dataByteLength <= DEFAULT_PART_SIZE) { - // Single part upload - const abortController = new AbortController(); - - return createUploadTask({ - isMultipartUpload: false, - job: putObjectJob(input, abortController.signal, dataByteLength), - onCancel: (message?: string) => { - abortController.abort(message); - }, - }); - } else { - // Multipart upload - const { multipartUploadJob, onPause, onResume, onCancel } = - getMultipartUploadHandlers(input, dataByteLength); + const config = createStorageConfiguration(Amplify); - return createUploadTask({ - isMultipartUpload: true, - job: multipartUploadJob, - onCancel: (message?: string) => { - onCancel(message); - }, - onPause, - onResume, - }); - } + return internalUploadData(config, input); } diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index e216feeede7..d164a09dac8 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -29,6 +29,7 @@ import { } from '../../../utils/client'; import { getStorageUserAgentValue } from '../../../utils/userAgent'; import { logger } from '../../../../../utils'; +import { S3InternalConfig } from '../../internal/types'; import { uploadPartExecutor } from './uploadPartExecutor'; import { getUploadsCacheKey, removeCachedUpload } from './uploadCache'; @@ -42,10 +43,17 @@ import { getDataChunker } from './getDataChunker'; * * @internal */ -export const getMultipartUploadHandlers = ( - uploadDataInput: UploadDataInput | UploadDataWithPathInput, - size?: number, -) => { + +interface GetMultipartUploadHandlersProps { + config: S3InternalConfig; + input: UploadDataInput | UploadDataWithPathInput; + size?: number; +} +export const getMultipartUploadHandlers = ({ + config, + input, + size, +}: GetMultipartUploadHandlersProps) => { let resolveCallback: | ((value: ItemWithKey | ItemWithPath) => void) | undefined; @@ -70,11 +78,11 @@ export const getMultipartUploadHandlers = ( let isAbortSignalFromPause = false; const startUpload = async (): Promise => { - const { options: uploadDataOptions, data } = uploadDataInput; - const resolvedS3Options = await resolveS3ConfigAndInput( - Amplify, - uploadDataOptions, - ); + const { options: uploadDataOptions, data } = input; + const resolvedS3Options = await resolveS3ConfigAndInput({ + config, + apiOptions: uploadDataOptions, + }); abortController = new AbortController(); isAbortSignalFromPause = false; @@ -83,7 +91,7 @@ export const getMultipartUploadHandlers = ( resolvedIdentityId = resolvedS3Options.identityId; const { inputType, objectKey } = validateStorageOperationInput( - uploadDataInput, + input, resolvedIdentityId, ); diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index bb9b5ec4519..6ab0ebb9012 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -1,7 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { Amplify } from '@aws-amplify/core'; import { StorageAction } from '@aws-amplify/core/internals/utils'; import { UploadDataInput, UploadDataWithPathInput } from '../../types'; @@ -14,6 +13,14 @@ import { ItemWithKey, ItemWithPath } from '../../types/outputs'; import { putObject } from '../../utils/client'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; +import { S3InternalConfig } from '../internal/types'; + +interface PutObjectJobProps { + config: S3InternalConfig; + input: UploadDataInput | UploadDataWithPathInput; + abortSignal: AbortSignal; + totalLength?: number; +} /** * Get a function the returns a promise to call putObject API to S3. @@ -21,17 +28,17 @@ import { STORAGE_INPUT_KEY } from '../../utils/constants'; * @internal */ export const putObjectJob = - ( - uploadDataInput: UploadDataInput | UploadDataWithPathInput, - abortSignal: AbortSignal, - totalLength?: number, - ) => + ({ config, input, abortSignal, totalLength }: PutObjectJobProps) => async (): Promise => { - const { options: uploadDataOptions, data } = uploadDataInput; + const { options: uploadDataOptions, data } = input; + const { bucket, keyPrefix, s3Config, isObjectLockEnabled, identityId } = - await resolveS3ConfigAndInput(Amplify, uploadDataOptions); + await resolveS3ConfigAndInput({ + config, + apiOptions: uploadDataOptions, + }); const { inputType, objectKey } = validateStorageOperationInput( - uploadDataInput, + input, identityId, ); diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 633366a4628..9a908890352 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -213,3 +213,14 @@ export interface ResolvedS3Config forcePathStyle?: boolean; useAccelerateEndpoint?: boolean; } + +/** + * Internal S3 API options. + * + * @internal + */ +export interface S3ApiOptions { + accessLevel?: StorageAccessLevel; + targetIdentityId?: string; + useAccelerateEndpoint?: boolean; +} diff --git a/packages/storage/src/providers/s3/utils/config.ts b/packages/storage/src/providers/s3/utils/config.ts new file mode 100644 index 00000000000..49258d0e04d --- /dev/null +++ b/packages/storage/src/providers/s3/utils/config.ts @@ -0,0 +1,59 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { AmplifyClassV6 } from '@aws-amplify/core'; + +import { StorageValidationErrorCode } from '../../../errors/types/validation'; +import { assertValidationError } from '../../../errors/utils/assertValidationError'; +import { S3InternalConfig } from '../apis/internal/types'; + +const createDefaultCredentialsProvider = (amplify: AmplifyClassV6) => { + /** + * A credentials provider function instead of a static credentials object is + * used because the long-running tasks like multipart upload may span over the + * credentials expiry. Auth.fetchAuthSession() automatically refreshes the + * credentials if they are expired. + */ + return async () => { + const { credentials } = await amplify.Auth.fetchAuthSession(); + assertValidationError( + !!credentials, + StorageValidationErrorCode.NoCredentials, + ); + + return credentials; + }; +}; + +const createDefaultIdentityIdProvider = (amplify: AmplifyClassV6) => { + return async () => { + const { identityId } = await amplify.Auth.fetchAuthSession(); + assertValidationError( + !!identityId, + StorageValidationErrorCode.NoIdentityId, + ); + + return identityId; + }; +}; + +/** + * It will return a Storage configuration used by lower level utils and APIs. + * + * @internal + */ +export const createStorageConfiguration = ( + amplify: AmplifyClassV6, +): S3InternalConfig => { + const libraryOptions = amplify.libraryOptions?.Storage?.S3 ?? {}; + const serviceOptions = amplify.getConfig()?.Storage?.S3 ?? {}; + const credentialsProvider = createDefaultCredentialsProvider(amplify); + const identityIdProvider = createDefaultIdentityIdProvider(amplify); + + return { + libraryOptions, + serviceOptions, + credentialsProvider, + identityIdProvider, + }; +}; diff --git a/packages/storage/src/providers/s3/utils/index.ts b/packages/storage/src/providers/s3/utils/index.ts index cd6b9753019..1f43bb3f5d9 100644 --- a/packages/storage/src/providers/s3/utils/index.ts +++ b/packages/storage/src/providers/s3/utils/index.ts @@ -7,3 +7,4 @@ export { createDownloadTask, createUploadTask } from './transferTask'; export { validateStorageOperationInput } from './validateStorageOperationInput'; export { validateStorageOperationInputWithPrefix } from './validateStorageOperationInputWithPrefix'; export { isInputWithPath } from './isInputWithPath'; +export { createStorageConfiguration } from './config'; diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index ae7a185c93c..ece08ea9223 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -1,21 +1,14 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { AmplifyClassV6, StorageAccessLevel } from '@aws-amplify/core'; - import { assertValidationError } from '../../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { resolvePrefix as defaultPrefixResolver } from '../../../utils/resolvePrefix'; -import { ResolvedS3Config } from '../types/options'; +import { ResolvedS3Config, S3ApiOptions } from '../types/options'; +import { S3InternalConfig } from '../apis/internal/types'; import { DEFAULT_ACCESS_LEVEL, LOCAL_TESTING_S3_ENDPOINT } from './constants'; -interface S3ApiOptions { - accessLevel?: StorageAccessLevel; - targetIdentityId?: string; - useAccelerateEndpoint?: boolean; -} - interface ResolvedS3ConfigAndInput { s3Config: ResolvedS3Config; bucket: string; @@ -24,6 +17,10 @@ interface ResolvedS3ConfigAndInput { identityId?: string; } +interface ResolveS3ConfigAndInputParams { + config: S3InternalConfig; + apiOptions?: S3ApiOptions; +} /** * resolve the common input options for S3 API handlers from Amplify configuration and library options. * @@ -35,44 +32,26 @@ interface ResolvedS3ConfigAndInput { * * @internal */ -export const resolveS3ConfigAndInput = async ( - amplify: AmplifyClassV6, - apiOptions?: S3ApiOptions, -): Promise => { - /** - * IdentityId is always cached in memory so we can safely make calls here. It - * should be stable even for unauthenticated users, regardless of credentials. - */ - const { identityId } = await amplify.Auth.fetchAuthSession(); - assertValidationError(!!identityId, StorageValidationErrorCode.NoIdentityId); - - /** - * A credentials provider function instead of a static credentials object is - * used because the long-running tasks like multipart upload may span over the - * credentials expiry. Auth.fetchAuthSession() automatically refreshes the - * credentials if they are expired. - */ - const credentialsProvider = async () => { - const { credentials } = await amplify.Auth.fetchAuthSession(); - assertValidationError( - !!credentials, - StorageValidationErrorCode.NoCredentials, - ); - - return credentials; - }; - +export const resolveS3ConfigAndInput = async ({ + config, + apiOptions, +}: ResolveS3ConfigAndInputParams): Promise => { + const { + credentialsProvider, + serviceOptions, + libraryOptions, + identityIdProvider, + } = config; const { bucket, region, dangerouslyConnectToHttpEndpointForTesting } = - amplify.getConfig()?.Storage?.S3 ?? {}; + serviceOptions ?? {}; assertValidationError(!!bucket, StorageValidationErrorCode.NoBucket); assertValidationError(!!region, StorageValidationErrorCode.NoRegion); - + const identityId = await identityIdProvider(); const { defaultAccessLevel, prefixResolver = defaultPrefixResolver, isObjectLockEnabled, - } = amplify.libraryOptions?.Storage?.S3 ?? {}; - + } = libraryOptions ?? {}; const keyPrefix = await prefixResolver({ accessLevel: apiOptions?.accessLevel ?? defaultAccessLevel ?? DEFAULT_ACCESS_LEVEL, @@ -97,7 +76,7 @@ export const resolveS3ConfigAndInput = async ( }, bucket, keyPrefix, - identityId, isObjectLockEnabled, + identityId, }; }; From 084b4ff078211397eb99055d59c03982acf44467 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Thu, 11 Jul 2024 11:15:39 -0700 Subject: [PATCH 09/40] feat(storage): add cred store creation implementation (#13575) Co-authored-by: israx <70438514+israx@users.noreply.github.com> --- packages/aws-amplify/package.json | 14 +- .../locationCredentialsStore/create.test.ts | 152 ++++++++++++- .../validators.test.ts | 208 ++++++++++++++++++ .../storage/src/errors/types/validation.ts | 26 ++- .../locationCredentialsStore/constants.ts | 5 + .../locationCredentialsStore/create.ts | 68 +++++- .../locationCredentialsStore/store.ts | 11 +- .../locationCredentialsStore/validators.ts | 86 ++++++++ 8 files changed, 548 insertions(+), 22 deletions(-) create mode 100644 packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts create mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts create mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index c1ba562ba9c..d4cfecc01d7 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "14.71 kB" + "limit": "14.9 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.31 kB" + "limit": "15.49 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "14.58 kB" + "limit": "14.77 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "15.68 kB" + "limit": "15.87 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.18 kB" + "limit": "15.36 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "14.45 kB" + "limit": "14.63 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "19.77 kB" + "limit": "19.94 kB" } ] } diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts index 1bd8f08c6a2..fa1ba483ada 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts @@ -1,17 +1,157 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; + +import { createLocationCredentialsStore } from '../../../src/storageBrowser/locationCredentialsStore/create'; +import { + createStore, + getValue, + removeStore, +} from '../../../src/storageBrowser/locationCredentialsStore/registry'; +import { validateCredentialsProviderLocation } from '../../../src/storageBrowser/locationCredentialsStore/validators'; +import { LocationCredentialsStore } from '../../../src/storageBrowser/types'; +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; + +jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); +jest.mock('../../../src/storageBrowser/locationCredentialsStore/validators'); + +const mockedCredentials = 'MOCK_CREDS' as any as AWSCredentials; describe('createLocationCredentialsStore', () => { - it.todo('should create a store'); + it('should create a store', () => { + const refreshHandler = jest.fn(); + const store = createLocationCredentialsStore({ handler: refreshHandler }); + + expect(createStore).toHaveBeenCalledWith(refreshHandler); + expect(store.getProvider).toBeDefined(); + expect(store.destroy).toBeDefined(); + }); + describe('created store', () => { - describe('getValue()', () => { - it.todo('should call getValue() from store'); - it.todo( - 'should validate credentials location with resolved common scope', + describe('getProvider()', () => { + let store: LocationCredentialsStore; + + beforeEach(() => { + store = createLocationCredentialsStore({ handler: jest.fn() }); + }); + + afterEach(() => { + jest.clearAllMocks(); + store.destroy(); + }); + + it('should call getValue() from store', async () => { + expect.assertions(2); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + + const locationCredentialsProvider = store.getProvider({ + scope: 's3://bucket/path/*', + permission: 'READ', + }); + const { credentials } = await locationCredentialsProvider({ + locations: [{ bucket: 'bucket', path: 'path/to/object' }], + permission: 'READ', + }); + expect(credentials).toEqual(mockedCredentials); + expect(getValue).toHaveBeenCalledWith( + expect.objectContaining({ + location: { + scope: 's3://bucket/path/*', + permission: 'READ', + }, + forceRefresh: false, + }), + ); + }); + + it('should validate credentials location with resolved common scope', async () => { + expect.assertions(1); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + const locationCredentialsProvider = store.getProvider({ + scope: 's3://bucket/path/*', + permission: 'READWRITE', + }); + await locationCredentialsProvider({ + locations: [ + { bucket: 'bucket', path: 'path/to/object' }, + { bucket: 'bucket', path: 'path/to/object2' }, + { bucket: 'bucket', path: 'path/folder' }, + ], + permission: 'READ', + }); + expect(validateCredentialsProviderLocation).toHaveBeenCalledWith( + { bucket: 'bucket', path: 'path/', permission: 'READ' }, + { bucket: 'bucket', path: 'path/*', permission: 'READWRITE' }, + ); + }); + + it('should throw if action requires cross-bucket permission', async () => { + expect.assertions(1); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + const locationCredentialsProvider = store.getProvider({ + scope: 's3://bucket/path/*', + permission: 'READWRITE', + }); + try { + await locationCredentialsProvider({ + locations: [ + { bucket: 'bucket-1', path: 'path/to/object' }, + { bucket: 'bucket-2', path: 'path/to/object2' }, + ], + permission: 'READ', + }); + } catch (e: any) { + expect(e.message).toEqual( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsCrossBucket + ].message, + ); + } + }); + + it.each(['invalid-s3-uri', 's3://', 's3:///'])( + 'should throw if location credentials provider scope is not a valid S3 URI "%s"', + async invalidScope => { + expect.assertions(1); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + const locationCredentialsProvider = store.getProvider({ + scope: invalidScope, + permission: 'READWRITE', + }); + try { + await locationCredentialsProvider({ + locations: [{ bucket: 'XXXXXXXX', path: 'path/to/object' }], + permission: 'READ', + }); + } catch (e: any) { + expect(e.message).toEqual( + validationErrorMap[StorageValidationErrorCode.InvalidS3Uri] + .message, + ); + } + }, ); }); + describe('destroy()', () => { - it.todo('should call removeStore() from store'); + it('should call removeStore() from store', () => { + const store = createLocationCredentialsStore({ + handler: jest.fn(), + }); + store.destroy(); + expect(removeStore).toHaveBeenCalled(); + }); }); }); }); diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts new file mode 100644 index 00000000000..774b5663d34 --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts @@ -0,0 +1,208 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { validateCredentialsProviderLocation } from '../../../src/storageBrowser/locationCredentialsStore/validators'; +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; + +jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); + +const mockBucket = 'MOCK_BUCKET'; + +describe('validateCredentialsProviderLocation', () => { + it('should NOT throw if action path matches credentials path prefix', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READ', + }, + ); + }).not.toThrow(); + }); + + it('should throw if action path does not path credentials path prefix', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/other/*', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPathMismatch + ].message, + ); + }); + + it('should NOT throw if action path matches credentials object path', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + ); + }).not.toThrow(); + }); + + it('should throw if action path does not match credentials object path', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/object2', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPathMismatch + ].message, + ); + }); + + it('should throw if action bucket and credentials bucket does not match', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: 'bucket-1', + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: 'bucket-2', + path: 'path/to/object', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsBucketMismatch + ].message, + ); + }); + + it('should not throw if READ action permission matches READWRITE credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READWRITE', + }, + ); + }).not.toThrow(); + }); + + it('should not throw if WRITE action permission matches READWRITE credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'WRITE', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READWRITE', + }, + ); + }).not.toThrow(); + }); + + it('should throw if READ action permission does not match WRITE credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READ', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'WRITE', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPermissionMismatch + ].message, + ); + }); + + it('should throw if WRITE action permission does not match READ credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'WRITE', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPermissionMismatch + ].message, + ); + }); + + it('should throw if READWRITE action permission does not match READ credentials permission', () => { + expect(() => { + validateCredentialsProviderLocation( + { + bucket: mockBucket, + path: 'path/to/object', + permission: 'READWRITE', + }, + { + bucket: mockBucket, + path: 'path/to/*', + permission: 'READ', + }, + ); + }).toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsPermissionMismatch + ].message, + ); + }); +}); diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index 5741f807874..281272863de 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -21,8 +21,17 @@ export enum StorageValidationErrorCode { UrlExpirationMaxLimitExceed = 'UrlExpirationMaxLimitExceed', InvalidLocationCredentialsCacheSize = 'InvalidLocationCredentialsCacheSize', LocationCredentialsStoreDestroyed = 'LocationCredentialsStoreDestroyed', + LocationCredentialsBucketMismatch = 'LocationCredentialsBucketMismatch', + LocationCredentialsCrossBucket = 'LocationCredentialsCrossBucket', + LocationCredentialsPathMismatch = 'LocationCredentialsPathMismatch', + LocationCredentialsPermissionMismatch = 'LocationCredentialsPermissionMismatch', + InvalidS3Uri = 'InvalidS3Uri', } +// Common error message strings to save some bytes +const LOCATION_SPECIFIC_CREDENTIALS = 'Location-specific credentials'; +const DOES_NOT_MATCH = 'does not match that required for the API call'; + export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.NoCredentials]: { message: 'Credentials should not be empty.', @@ -76,6 +85,21 @@ export const validationErrorMap: AmplifyErrorMap = { message: 'locationCredentialsCacheSize must be a positive integer.', }, [StorageValidationErrorCode.LocationCredentialsStoreDestroyed]: { - message: 'The location-specific credentials store has been destroyed', + message: `${LOCATION_SPECIFIC_CREDENTIALS} store has been destroyed.`, + }, + [StorageValidationErrorCode.InvalidS3Uri]: { + message: 'Invalid S3 URI.', + }, + [StorageValidationErrorCode.LocationCredentialsCrossBucket]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} cannot be used across buckets.`, + }, + [StorageValidationErrorCode.LocationCredentialsBucketMismatch]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} bucket ${DOES_NOT_MATCH}.`, + }, + [StorageValidationErrorCode.LocationCredentialsPathMismatch]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} path ${DOES_NOT_MATCH}.`, + }, + [StorageValidationErrorCode.LocationCredentialsPermissionMismatch]: { + message: `${LOCATION_SPECIFIC_CREDENTIALS} permission ${DOES_NOT_MATCH}.`, }, }; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts new file mode 100644 index 00000000000..67b22505b95 --- /dev/null +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts @@ -0,0 +1,5 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const CREDENTIALS_STORE_DEFAULT_SIZE = 10; +export const CREDENTIALS_REFRESH_WINDOW_MS = 30_000; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts index d4f91341b06..dc8da21bff5 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts @@ -1,5 +1,3 @@ -/* eslint-disable unused-imports/no-unused-vars */ - // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 @@ -8,9 +6,15 @@ import { LocationCredentialsHandler, LocationCredentialsStore, } from '../types'; -import { LocationCredentialsProvider } from '../../providers/s3/types/options'; +import { StorageValidationErrorCode } from '../../errors/types/validation'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { + BucketLocation, + LocationCredentialsProvider, +} from '../../providers/s3/types/options'; import { createStore, getValue, removeStore } from './registry'; +import { validateCredentialsProviderLocation } from './validators'; export const createLocationCredentialsStore = (input: { handler: LocationCredentialsHandler; @@ -24,7 +28,18 @@ export const createLocationCredentialsStore = (input: { locations, forceRefresh = false, }: Parameters[0]) => { - // TODO(@AllanZhengYP) validate input + const actionBucketLocation = resolveCommonBucketLocation(locations); + const providerBucketLocation = parseS3Uri(providerLocation.scope); + validateCredentialsProviderLocation( + { + ...actionBucketLocation, + permission, + }, + { + ...providerBucketLocation, + permission: providerLocation.permission, + }, + ); return getValue({ storeSymbol, @@ -43,3 +58,48 @@ export const createLocationCredentialsStore = (input: { return store; }; + +type S3Uri = string; + +const parseS3Uri = (uri: S3Uri): BucketLocation => { + const s3UrlSchemaRegex = /^s3:\/\//; + // TODO(@AllanZhengYP): Provide more info to error message: url + assertValidationError( + s3UrlSchemaRegex.test(uri), + StorageValidationErrorCode.InvalidS3Uri, + ); + const [bucket, ...pathParts] = uri.replace(s3UrlSchemaRegex, '').split('/'); + assertValidationError(!!bucket, StorageValidationErrorCode.InvalidS3Uri); + const path = pathParts.join('/'); + + return { + bucket, + path, + }; +}; + +/** + * Given a list of bucket and path combinations, verify they have the same + * bucket and resolves the longest common prefix for multiple given paths. + */ +const resolveCommonBucketLocation = ( + locations: BucketLocation[], +): BucketLocation => { + let { bucket: commonBucket, path: commonPath } = locations[0]; + + for (const location of locations) { + const { bucket, path } = location; + assertValidationError( + bucket === commonBucket, + StorageValidationErrorCode.LocationCredentialsCrossBucket, + ); + while (commonPath !== '' && !path.startsWith(commonPath)) { + commonPath = commonPath.slice(0, -1); + } + } + + return { + bucket: commonBucket, + path: commonPath, + }; +}; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts index 1253d010b22..ac1901d669b 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts @@ -10,17 +10,19 @@ import { CredentialsLocation, LocationCredentialsHandler } from '../types'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; -const CREDENTIALS_STORE_DEFAULT_SIZE = 10; -const CREDENTIALS_REFRESH_WINDOW_MS = 30_000; +import { + CREDENTIALS_REFRESH_WINDOW_MS, + CREDENTIALS_STORE_DEFAULT_SIZE, +} from './constants'; interface StoreValue extends CredentialsLocation { credentials?: AWSCredentials; inflightCredentials?: Promise<{ credentials: AWSCredentials }>; } -type S3Url = string; +type S3Uri = string; -type CacheKey = `${S3Url}_${Permission}`; +type CacheKey = `${S3Uri}_${Permission}`; const createCacheKey = (location: CredentialsLocation): CacheKey => `${location.scope}_${location.permission}`; @@ -153,6 +155,7 @@ const setCacheRecord = ( // So first key is the last recently inserted. const [oldestKey] = store.values.keys(); store.values.delete(oldestKey); + // TODO(@AllanZhengYP): Add log info when record is evicted. } // Add latest used value to the cache. store.values.set(key, value); diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts new file mode 100644 index 00000000000..3409a899fdf --- /dev/null +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts @@ -0,0 +1,86 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageValidationErrorCode } from '../../errors/types/validation'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { BucketLocation, Permission } from '../../providers/s3/types/options'; + +interface CredentialsBucketLocation extends BucketLocation { + permission: Permission; +} + +/** + * @internal + */ +export const validateCredentialsProviderLocation = ( + actionLocation: CredentialsBucketLocation, + providerLocation: CredentialsBucketLocation, +): void => { + validateLocationBucket({ + actionBucket: actionLocation.bucket, + providerBucket: providerLocation.bucket, + }); + validateLocationPath({ + actionPath: actionLocation.path, + providerPath: providerLocation.path, + }); + validateLocationPermission({ + actionPermission: actionLocation.permission, + providerPermission: providerLocation.permission, + }); +}; + +const validateLocationBucket = (input: { + actionBucket: string; + providerBucket?: string; +}): void => { + const { actionBucket, providerBucket } = input; + if (!providerBucket) { + return; + } + assertValidationError( + actionBucket === providerBucket, + StorageValidationErrorCode.LocationCredentialsBucketMismatch, + ); +}; + +const validateLocationPath = (input: { + actionPath: string; + providerPath?: string; +}): void => { + const { actionPath, providerPath } = input; + if (!providerPath) { + return; + } + if (providerPath.endsWith('*')) { + // Verify if the action path has prefix required by the provider; + const providerPathPrefix = providerPath.replace(/\*$/, ''); + assertValidationError( + actionPath.startsWith(providerPathPrefix), + StorageValidationErrorCode.LocationCredentialsPathMismatch, + ); + } else { + // If provider path is scoped to an object, verify if the action path points to the same object. + // TODO(@AllanZhengYP) Provider more info in error message: actionPath, providerPath. + assertValidationError( + actionPath === providerPath, + StorageValidationErrorCode.LocationCredentialsPathMismatch, + ); + } +}; + +const validateLocationPermission = (input: { + actionPermission: Permission; + providerPermission?: Permission; +}) => { + const { actionPermission, providerPermission } = input; + if (!providerPermission) { + return; + } + // TODO(@AllanZhengYP) Provide more info in error message: `API needs permission ${actionPermission}, but provided + // location credentials provider with permission ${providerPermission}.` + assertValidationError( + providerPermission.includes(actionPermission), + StorageValidationErrorCode.LocationCredentialsPermissionMismatch, + ); +}; From 16e3c52a360a7577f114ee3f1a51ac25bbfad5cb Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Fri, 12 Jul 2024 15:03:08 -0700 Subject: [PATCH 10/40] feat(storage): use WeakMap for store registry (#13586) * feat(storage): use WeakMap for store registry * chore(storage): export storage browser utils from @aws-amplify/storage/storage-browser * doc(storage): add disclaimer --- .../locationCredentialsStore/registry.test.ts | 22 +++++++++---------- packages/storage/src/storageBrowser/index.ts | 6 +++++ .../locationCredentialsStore/registry.ts | 21 ++++++++++++------ packages/storage/storage-browser/package.json | 7 ++++++ 4 files changed, 38 insertions(+), 18 deletions(-) create mode 100644 packages/storage/storage-browser/package.json diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts index 19adc4576d9..0d36aaba8b9 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts @@ -40,7 +40,7 @@ describe('createStore', () => { it('should return a symbol to refer the store instance', () => { const storeReference = createStore(jest.fn(), 20); - expect(Object.prototype.toString.call(storeReference)).toBe( + expect(Object.prototype.toString.call(storeReference.value)).toBe( '[object Symbol]', ); }); @@ -48,12 +48,12 @@ describe('createStore', () => { describe('getValue', () => { const mockCachedValue = 'CACHED_VALUE' as any as AWSCredentials; - let storeReference: symbol; + let storeSymbol: { value: symbol }; beforeEach(() => { - storeReference = createStore(jest.fn(), 20); + storeSymbol = createStore(jest.fn(), 20); }); afterEach(() => { - removeStore(storeReference); + removeStore(storeSymbol); jest.clearAllMocks(); }); @@ -61,7 +61,7 @@ describe('getValue', () => { expect.assertions(1); await expect( getValue({ - storeSymbol: Symbol('invalid'), + storeSymbol: { value: Symbol('invalid') }, location: { scope: 'abc', permission: 'READ' }, forceRefresh: false, }), @@ -77,7 +77,7 @@ describe('getValue', () => { jest.mocked(getCacheValue).mockReturnValueOnce(mockCachedValue); expect( await getValue({ - storeSymbol: storeReference, + storeSymbol, location: { scope: 'abc', permission: 'READ' }, forceRefresh: false, }), @@ -95,7 +95,7 @@ describe('getValue', () => { jest.mocked(getCacheValue).mockReturnValueOnce(mockCachedValue); expect( await getValue({ - storeSymbol: storeReference, + storeSymbol, location: { scope: 'abc', permission: 'READ' }, forceRefresh: false, }), @@ -117,7 +117,7 @@ describe('getValue', () => { jest.mocked(fetchNewValue).mockResolvedValue('NEW_VALUE' as any); expect( await getValue({ - storeSymbol: storeReference, + storeSymbol, location: { scope: 'abc', permission: 'READ' }, forceRefresh: false, }), @@ -135,7 +135,7 @@ describe('getValue', () => { jest.mocked(fetchNewValue).mockResolvedValue('NEW_VALUE' as any); expect( await getValue({ - storeSymbol: storeReference, + storeSymbol, location: { scope: 'abc', permission: 'READ' }, forceRefresh: true, }), @@ -154,7 +154,7 @@ describe('getValue', () => { .mockRejectedValueOnce(new Error('Network error')); await expect( getValue({ - storeSymbol: storeReference, + storeSymbol, location: { scope: 'abc', permission: 'READ' }, forceRefresh: true, }), @@ -182,7 +182,7 @@ describe('removeStore', () => { it('should not throw if store with given symbol does not exist', () => { expect(() => { - removeStore(Symbol('invalid')); + removeStore({ value: Symbol('invalid') }); }).not.toThrow(); }); }); diff --git a/packages/storage/src/storageBrowser/index.ts b/packages/storage/src/storageBrowser/index.ts index ce6a8c34817..7d36d419558 100644 --- a/packages/storage/src/storageBrowser/index.ts +++ b/packages/storage/src/storageBrowser/index.ts @@ -1,6 +1,12 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +/** + * NOTE: The APIs exported from this file are ONLY intended for usage by + * Amplify UI. To use location-related features, please use + * @aws-amplify/ui-react-storage + */ + export { listCallerAccessGrants, ListCallerAccessGrantsInput, diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts index da4c8adeee4..6c3d41780f9 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts @@ -15,12 +15,19 @@ import { initStore, } from './store'; +interface StoreRegistrySymbol { + readonly value: symbol; +} + /** * Keep all cache records for all instances of credentials store in a singleton * so we can reliably de-reference from the memory when we destroy a store * instance. */ -const storeRegistry = new Map(); +const storeRegistry = new WeakMap< + StoreRegistrySymbol, + LruLocationCredentialsStore +>(); /** * @internal @@ -29,10 +36,10 @@ export const createStore = ( refreshHandler: LocationCredentialsHandler, size?: number, ) => { - const storeInstanceSymbol = Symbol('LocationCredentialsStore'); - storeRegistry.set(storeInstanceSymbol, initStore(refreshHandler, size)); + const storeSymbol = { value: Symbol('LocationCredentialsStore') }; + storeRegistry.set(storeSymbol, initStore(refreshHandler, size)); - return storeInstanceSymbol; + return storeSymbol; }; const getLookUpLocations = (location: CredentialsLocation) => { @@ -45,7 +52,7 @@ const getLookUpLocations = (location: CredentialsLocation) => { return locations; }; -const getCredentialsStore = (storeSymbol: symbol) => { +const getCredentialsStore = (storeSymbol: StoreRegistrySymbol) => { assertValidationError( storeRegistry.has(storeSymbol), StorageValidationErrorCode.LocationCredentialsStoreDestroyed, @@ -58,7 +65,7 @@ const getCredentialsStore = (storeSymbol: symbol) => { * @internal */ export const getValue = async (input: { - storeSymbol: symbol; + storeSymbol: StoreRegistrySymbol; location: CredentialsLocation; forceRefresh: boolean; }): Promise<{ credentials: AWSCredentials }> => { @@ -77,6 +84,6 @@ export const getValue = async (input: { return fetchNewValue(store, location); }; -export const removeStore = (storeSymbol: symbol) => { +export const removeStore = (storeSymbol: StoreRegistrySymbol) => { storeRegistry.delete(storeSymbol); }; diff --git a/packages/storage/storage-browser/package.json b/packages/storage/storage-browser/package.json new file mode 100644 index 00000000000..f492f32d490 --- /dev/null +++ b/packages/storage/storage-browser/package.json @@ -0,0 +1,7 @@ +{ + "name": "@aws-amplify/storage/storage-browser", + "main": "../dist/cjs/storageBrowser/index.js", + "browser": "../dist/esm/storageBrowser/index.mjs", + "module": "../dist/esm/storageBrowser/index.mjs", + "typings": "../dist/esm/storageBrowser/index.d.ts" +} From bcd8c9d3558cfeca91475160b29766e0513f9422 Mon Sep 17 00:00:00 2001 From: Jim Blanchard Date: Mon, 15 Jul 2024 08:27:13 -0500 Subject: [PATCH 11/40] feat(storage): Added getDataAccess & listCallerAccessGrant clients (#13582) --- .../__tests__/providers/s3/apis/copy.test.ts | 4 +- .../providers/s3/apis/downloadData.test.ts | 4 +- .../providers/s3/apis/getProperties.test.ts | 4 +- .../providers/s3/apis/getUrl.test.ts | 4 +- .../__tests__/providers/s3/apis/list.test.ts | 4 +- .../providers/s3/apis/remove.test.ts | 4 +- .../apis/uploadData/multipartHandlers.test.ts | 4 +- .../s3/apis/uploadData/putObjectJob.test.ts | 4 +- .../client/S3/cases/abortMultipartUpload.ts | 2 +- .../S3/cases/completeMultipartUpload.ts | 2 +- .../s3/utils/client/S3/cases/copyObject.ts | 2 +- .../client/S3/cases/createMultipartUpload.ts | 2 +- .../s3/utils/client/S3/cases/deleteObject.ts | 2 +- .../s3/utils/client/S3/cases/getDataAccess.ts | 100 ++++++++ .../s3/utils/client/S3/cases/getObject.ts | 2 +- .../s3/utils/client/S3/cases/headObject.ts | 2 +- .../s3/utils/client/S3/cases/index.ts | 4 + .../client/S3/cases/listCallerAccessGrants.ts | 105 +++++++++ .../s3/utils/client/S3/cases/listObjectsV2.ts | 2 +- .../s3/utils/client/S3/cases/listParts.ts | 2 +- .../s3/utils/client/S3/cases/putObject.ts | 2 +- .../s3/utils/client/S3/cases/uploadPart.ts | 2 +- .../utils/client/S3/functional-apis.test.ts | 4 +- .../S3/getPresignedGetObjectUrl.test.ts | 2 +- .../src/providers/s3/apis/downloadData.ts | 2 +- .../src/providers/s3/apis/internal/copy.ts | 2 +- .../s3/apis/internal/getProperties.ts | 2 +- .../src/providers/s3/apis/internal/getUrl.ts | 2 +- .../src/providers/s3/apis/internal/list.ts | 4 +- .../src/providers/s3/apis/internal/remove.ts | 2 +- .../uploadData/multipart/initialUpload.ts | 2 +- .../apis/uploadData/multipart/uploadCache.ts | 2 +- .../uploadData/multipart/uploadHandlers.ts | 2 +- .../multipart/uploadPartExecutor.ts | 2 +- .../s3/apis/uploadData/putObjectJob.ts | 2 +- .../s3/utils/client/s3control/base.ts | 71 ++++++ .../utils/client/s3control/getDataAccess.ts | 97 ++++++++ .../s3/utils/client/s3control/index.ts | 13 ++ .../s3control/listCallerAccessGrants.ts | 100 ++++++++ .../s3/utils/client/s3control/types.ts | 213 ++++++++++++++++++ .../{ => s3data}/abortMultipartUpload.ts | 7 +- .../s3/utils/client/{ => s3data}/base.ts | 2 +- .../{ => s3data}/completeMultipartUpload.ts | 17 +- .../utils/client/{ => s3data}/copyObject.ts | 7 +- .../{ => s3data}/createMultipartUpload.ts | 15 +- .../utils/client/{ => s3data}/deleteObject.ts | 13 +- .../s3/utils/client/{ => s3data}/getObject.ts | 15 +- .../utils/client/{ => s3data}/headObject.ts | 7 +- .../s3/utils/client/{ => s3data}/index.ts | 0 .../client/{ => s3data}/listObjectsV2.ts | 13 +- .../s3/utils/client/{ => s3data}/listParts.ts | 15 +- .../s3/utils/client/{ => s3data}/putObject.ts | 7 +- .../s3/utils/client/{ => s3data}/types.ts | 0 .../utils/client/{ => s3data}/uploadPart.ts | 7 +- scripts/dts-bundler/dts-bundler.config.js | 2 +- 55 files changed, 814 insertions(+), 100 deletions(-) create mode 100644 packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts create mode 100644 packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts create mode 100644 packages/storage/src/providers/s3/utils/client/s3control/base.ts create mode 100644 packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts create mode 100644 packages/storage/src/providers/s3/utils/client/s3control/index.ts create mode 100644 packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts create mode 100644 packages/storage/src/providers/s3/utils/client/s3control/types.ts rename packages/storage/src/providers/s3/utils/client/{ => s3data}/abortMultipartUpload.ts (98%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/base.ts (98%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/completeMultipartUpload.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/copyObject.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/createMultipartUpload.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/deleteObject.ts (98%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/getObject.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/headObject.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/index.ts (100%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/listObjectsV2.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/listParts.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/putObject.ts (99%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/types.ts (100%) rename packages/storage/src/providers/s3/utils/client/{ => s3data}/uploadPart.ts (99%) diff --git a/packages/storage/__tests__/providers/s3/apis/copy.test.ts b/packages/storage/__tests__/providers/s3/apis/copy.test.ts index 55547ae8e7c..ca56b8da5a4 100644 --- a/packages/storage/__tests__/providers/s3/apis/copy.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/copy.test.ts @@ -6,7 +6,7 @@ import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; import { StorageError } from '../../../../src/errors/StorageError'; import { StorageValidationErrorCode } from '../../../../src/errors/types/validation'; -import { copyObject } from '../../../../src/providers/s3/utils/client'; +import { copyObject } from '../../../../src/providers/s3/utils/client/s3data'; import { copy } from '../../../../src/providers/s3/apis'; import { CopyInput, @@ -16,7 +16,7 @@ import { } from '../../../../src/providers/s3/types'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts b/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts index 57d402b1f24..b7bac2438ed 100644 --- a/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { getObject } from '../../../../src/providers/s3/utils/client'; +import { getObject } from '../../../../src/providers/s3/utils/client/s3data'; import { downloadData } from '../../../../src/providers/s3'; import { createDownloadTask, @@ -25,7 +25,7 @@ import { } from '../../../../src/providers/s3/types/outputs'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('../../../../src/providers/s3/utils'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { diff --git a/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts b/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts index bb5a5b957a7..11297c3ad13 100644 --- a/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { headObject } from '../../../../src/providers/s3/utils/client'; +import { headObject } from '../../../../src/providers/s3/utils/client/s3data'; import { getProperties } from '../../../../src/providers/s3'; import { GetPropertiesInput, @@ -14,7 +14,7 @@ import { } from '../../../../src/providers/s3/types'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts b/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts index 994f4a0b648..09052b6bd30 100644 --- a/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts @@ -8,7 +8,7 @@ import { getUrl } from '../../../../src/providers/s3/apis'; import { getPresignedGetObjectUrl, headObject, -} from '../../../../src/providers/s3/utils/client'; +} from '../../../../src/providers/s3/utils/client/s3data'; import { GetUrlInput, GetUrlOutput, @@ -17,7 +17,7 @@ import { } from '../../../../src/providers/s3/types'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/list.test.ts b/packages/storage/__tests__/providers/s3/apis/list.test.ts index 348719732c0..004e3750639 100644 --- a/packages/storage/__tests__/providers/s3/apis/list.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/list.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { listObjectsV2 } from '../../../../src/providers/s3/utils/client'; +import { listObjectsV2 } from '../../../../src/providers/s3/utils/client/s3data'; import { list } from '../../../../src/providers/s3'; import { ListAllInput, @@ -18,7 +18,7 @@ import { } from '../../../../src/providers/s3/types'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/remove.test.ts b/packages/storage/__tests__/providers/s3/apis/remove.test.ts index ca1107f0912..aafe374f1aa 100644 --- a/packages/storage/__tests__/providers/s3/apis/remove.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/remove.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { deleteObject } from '../../../../src/providers/s3/utils/client'; +import { deleteObject } from '../../../../src/providers/s3/utils/client/s3data'; import { remove } from '../../../../src/providers/s3/apis'; import { StorageValidationErrorCode } from '../../../../src/errors/types/validation'; import { @@ -15,7 +15,7 @@ import { } from '../../../../src/providers/s3/types'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index 5c87d98fca7..ab049042afd 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -11,7 +11,7 @@ import { headObject, listParts, uploadPart, -} from '../../../../../src/providers/s3/utils/client'; +} from '../../../../../src/providers/s3/utils/client/s3data'; import { getMultipartUploadHandlers } from '../../../../../src/providers/s3/apis/uploadData/multipart'; import { StorageValidationErrorCode, @@ -25,7 +25,7 @@ import '../testUtils'; import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; jest.mock('@aws-amplify/core'); -jest.mock('../../../../../src/providers/s3/utils/client'); +jest.mock('../../../../../src/providers/s3/utils/client/s3data'); const credentials: AWSCredentials = { accessKeyId: 'accessKeyId', diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index 51a3e0de2bc..f6e06fa1140 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -3,13 +3,13 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { putObject } from '../../../../../src/providers/s3/utils/client'; +import { putObject } from '../../../../../src/providers/s3/utils/client/s3data'; import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; -jest.mock('../../../../../src/providers/s3/utils/client'); +jest.mock('../../../../../src/providers/s3/utils/client/s3data'); jest.mock('../../../../../src/providers/s3/utils', () => { const utils = jest.requireActual('../../../../../src/providers/s3/utils'); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts index 4628c433e51..cc81a2be88f 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { abortMultipartUpload } from '../../../../../../../src/providers/s3/utils/client'; +import { abortMultipartUpload } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts index 125cb505e4c..d94e6b94d34 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { completeMultipartUpload } from '../../../../../../../src/providers/s3/utils/client'; +import { completeMultipartUpload } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts index 746ca373057..cad0ad74cf4 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { copyObject } from '../../../../../../../src/providers/s3/utils/client'; +import { copyObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts index df13908e715..e027397e569 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { createMultipartUpload } from '../../../../../../../src/providers/s3/utils/client'; +import { createMultipartUpload } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts index f0a4439e13f..614a3c1fff6 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { deleteObject } from '../../../../../../../src/providers/s3/utils/client'; +import { deleteObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts new file mode 100644 index 00000000000..b0a0d174a7a --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts @@ -0,0 +1,100 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { getDataAccess } from '../../../../../../../src/providers/s3/utils/client/s3control'; +import { ApiFunctionalTestCase } from '../../testUtils/types'; + +import { + DEFAULT_RESPONSE_HEADERS, + defaultConfig, + expectedMetadata, +} from './shared'; + +const MOCK_ACCOUNT_ID = 'accountId'; +const MOCK_ACCESS_ID = 'accessId'; +const MOCK_SECRET_ACCESS_KEY = 'secretAccessKey'; +const MOCK_SESSION_TOKEN = 'sessionToken'; +const MOCK_EXPIRATION = '2013-09-17T18:07:53.000Z'; +const MOCK_EXPIRATION_DATE = new Date(MOCK_EXPIRATION); +const MOCK_GRANT_TARGET = 'matchedGrantTarget'; + +// API Reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_GetDataAccess.html +const getDataAccessHappyCase: ApiFunctionalTestCase = [ + 'happy case', + 'getDataAccess', + getDataAccess, + defaultConfig, + { + AccountId: MOCK_ACCOUNT_ID, + Target: 's3://my-bucket/path/to/object.md', + TargetType: 'Object', + DurationSeconds: 100, + Permission: 'READWRITE', + Privilege: 'Default', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.s3-control.us-east-1.amazonaws.com/v20180820/accessgrantsinstance/dataaccess?durationSeconds=100&permission=READWRITE&privilege=Default&target=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2Fobject.md&targetType=Object', + }), + method: 'GET', + headers: expect.objectContaining({ + 'x-amz-account-id': MOCK_ACCOUNT_ID, + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: ` + + + + ${MOCK_ACCESS_ID} + ${MOCK_SECRET_ACCESS_KEY} + ${MOCK_SESSION_TOKEN} + ${MOCK_EXPIRATION} + + ${MOCK_GRANT_TARGET} + + `, + }, + { + $metadata: expect.objectContaining(expectedMetadata), + Credentials: { + AccessKeyId: MOCK_ACCESS_ID, + SecretAccessKey: MOCK_SECRET_ACCESS_KEY, + SessionToken: MOCK_SESSION_TOKEN, + Expiration: MOCK_EXPIRATION_DATE, + }, + MatchedGrantTarget: MOCK_GRANT_TARGET, + }, +]; + +const getDataAccessErrorCase: ApiFunctionalTestCase = [ + 'error case', + 'getDataAccess', + getDataAccess, + defaultConfig, + getDataAccessHappyCase[4], + getDataAccessHappyCase[5], + { + status: 403, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + + AccessDenied + Access Denied + 656c76696e6727732072657175657374 + Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== + + `, + }, + { + message: 'Access Denied', + name: 'AccessDenied', + }, +]; + +export default [getDataAccessHappyCase, getDataAccessErrorCase]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts index c6b1e038926..a35c813f3d8 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { getObject } from '../../../../../../../src/providers/s3/utils/client'; +import { getObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts index 2275d7ac850..0cc016a7813 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { headObject } from '../../../../../../../src/providers/s3/utils/client'; +import { headObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts index 56a4e1719ae..b5688b18c78 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts @@ -12,6 +12,8 @@ import copyObjectCases from './copyObject'; import deleteObjectCases from './deleteObject'; import getObjectCases from './getObject'; import headObjectCases from './headObject'; +import getDataAccess from './getDataAccess'; +import listCallerAccessGrants from './listCallerAccessGrants'; export default [ ...listObjectsV2Cases, @@ -25,4 +27,6 @@ export default [ ...deleteObjectCases, ...getObjectCases, ...headObjectCases, + ...listCallerAccessGrants, + ...getDataAccess, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts new file mode 100644 index 00000000000..29a8e3f4516 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -0,0 +1,105 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { listCallerAccessGrants } from '../../../../../../../src/providers/s3/utils/client/s3control'; +import { ApiFunctionalTestCase } from '../../testUtils/types'; + +import { + DEFAULT_RESPONSE_HEADERS, + defaultConfig, + expectedMetadata, +} from './shared'; + +const MOCK_ACCOUNT_ID = 'accountId'; +const MOCK_NEXT_TOKEN = 'nextToken'; +const MOCK_APP_ARN = 'appArn'; +const MOCK_GRANT_SCOPE = 's3://my-bucket/path/to/object.md'; +const MOCK_PERMISSION = 'READWRITE'; + +// API Reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_ListAccessGrants.html +const listCallerAccessGrantsHappyCase: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'happy case', + 'listCallerAccessGrants', + listCallerAccessGrants, + defaultConfig, + { + AccountId: MOCK_ACCOUNT_ID, + GrantScope: 's3://my-bucket/path/to/', + MaxResults: 50, + NextToken: 'mockToken', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.s3-control.us-east-1.amazonaws.com/v20180820/accessgrantsinstance/caller/grants?grantscope=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2F&maxResults=50&nextToken=mockToken', + }), + method: 'GET', + headers: expect.objectContaining({ + 'x-amz-account-id': MOCK_ACCOUNT_ID, + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: ` + + + ${MOCK_NEXT_TOKEN} + + + ${MOCK_APP_ARN} + ${MOCK_GRANT_SCOPE} + ${MOCK_PERMISSION} + + + + `, + }, + { + $metadata: expect.objectContaining(expectedMetadata), + CallerAccessGrantsList: [ + { + ApplicationArn: MOCK_APP_ARN, + GrantScope: MOCK_GRANT_SCOPE, + Permission: MOCK_PERMISSION, + }, + ], + NextToken: MOCK_NEXT_TOKEN, + }, +]; + +const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'error case', + 'listCallerAccessGrants', + listCallerAccessGrants, + defaultConfig, + listCallerAccessGrantsHappyCase[4], + listCallerAccessGrantsHappyCase[5], + { + status: 403, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + + AccessDenied + Access Denied + 656c76696e6727732072657175657374 + Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== + + `, + }, + { + message: 'Access Denied', + name: 'AccessDenied', + }, +]; + +export default [ + listCallerAccessGrantsHappyCase, + listCallerAccessGrantsErrorCase, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index 7524a8daeb6..80ce6ea9a08 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { listObjectsV2 } from '../../../../../../../src/providers/s3/utils/client'; +import { listObjectsV2 } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts index 3e809d12bdc..396035c09dd 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { listParts } from '../../../../../../../src/providers/s3/utils/client'; +import { listParts } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts index 930870a7c15..867ee3f0af2 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { putObject } from '../../../../../../../src/providers/s3/utils/client'; +import { putObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts index b4906b223c2..4a46891c849 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { uploadPart } from '../../../../../../../src/providers/s3/utils/client'; +import { uploadPart } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts index 62b4aff0cf5..656f8d45ed7 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts @@ -68,11 +68,11 @@ describe('S3 APIs functional test', () => { expect.anything(), ); } else { - fail(`${name} ${caseType} should fail`); + throw new Error(`${name} ${caseType} should fail`); } } catch (e) { if (caseType === 'happy case') { - fail(`${name} ${caseType} should succeed: ${e}`); + throw new Error(`${name} ${caseType} should succeed: ${e}`); } else { expect(e).toBeInstanceOf(StorageError); expect(e).toEqual(expect.objectContaining(outputOrError)); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts index 93bd3963606..1f7f89aa03c 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts @@ -3,7 +3,7 @@ import { presignUrl } from '@aws-amplify/core/internals/aws-client-utils'; -import { getPresignedGetObjectUrl } from '../../../../../../src/providers/s3/utils/client'; +import { getPresignedGetObjectUrl } from '../../../../../../src/providers/s3/utils/client/s3data'; import { defaultConfigWithStaticCredentials } from './cases/shared'; diff --git a/packages/storage/src/providers/s3/apis/downloadData.ts b/packages/storage/src/providers/s3/apis/downloadData.ts index 41d32e93b64..e67ab7f6b25 100644 --- a/packages/storage/src/providers/s3/apis/downloadData.ts +++ b/packages/storage/src/providers/s3/apis/downloadData.ts @@ -12,8 +12,8 @@ import { } from '../types'; import { resolveS3ConfigAndInput } from '../utils/resolveS3ConfigAndInput'; import { createDownloadTask, validateStorageOperationInput } from '../utils'; +import { getObject } from '../utils/client/s3data'; import { createStorageConfiguration } from '../utils/config'; -import { getObject } from '../utils/client'; import { getStorageUserAgentValue } from '../utils/userAgent'; import { logger } from '../../../utils'; import { diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index 1f67be4c66c..22bdc1bac6a 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -19,7 +19,7 @@ import { } from '../../utils'; import { StorageValidationErrorCode } from '../../../../errors/types/validation'; import { assertValidationError } from '../../../../errors/utils/assertValidationError'; -import { copyObject } from '../../utils/client'; +import { copyObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; diff --git a/packages/storage/src/providers/s3/apis/internal/getProperties.ts b/packages/storage/src/providers/s3/apis/internal/getProperties.ts index 68037c73be2..f06f6bcab69 100644 --- a/packages/storage/src/providers/s3/apis/internal/getProperties.ts +++ b/packages/storage/src/providers/s3/apis/internal/getProperties.ts @@ -15,7 +15,7 @@ import { resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; -import { headObject } from '../../utils/client'; +import { headObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; diff --git a/packages/storage/src/providers/s3/apis/internal/getUrl.ts b/packages/storage/src/providers/s3/apis/internal/getUrl.ts index e1511429f7b..755a2028e4c 100644 --- a/packages/storage/src/providers/s3/apis/internal/getUrl.ts +++ b/packages/storage/src/providers/s3/apis/internal/getUrl.ts @@ -11,7 +11,7 @@ import { GetUrlWithPathOutput, } from '../../types'; import { StorageValidationErrorCode } from '../../../../errors/types/validation'; -import { getPresignedGetObjectUrl } from '../../utils/client'; +import { getPresignedGetObjectUrl } from '../../utils/client/s3data'; import { createStorageConfiguration, resolveS3ConfigAndInput, diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index 9f28270e61c..0da1742aac3 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -26,11 +26,11 @@ import { ListObjectsV2Input, ListObjectsV2Output, listObjectsV2, -} from '../../utils/client'; +} from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { STORAGE_INPUT_PREFIX } from '../../utils/constants'; -import { CommonPrefix } from '../../utils/client/types'; +import { CommonPrefix } from '../../utils/client/s3data/types'; const MAX_PAGE_SIZE = 1000; diff --git a/packages/storage/src/providers/s3/apis/internal/remove.ts b/packages/storage/src/providers/s3/apis/internal/remove.ts index a1cdacfbb8d..5a6add5ce67 100644 --- a/packages/storage/src/providers/s3/apis/internal/remove.ts +++ b/packages/storage/src/providers/s3/apis/internal/remove.ts @@ -15,7 +15,7 @@ import { resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; -import { deleteObject } from '../../utils/client'; +import { deleteObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts index 1179b89c08b..7307a90e007 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts @@ -5,7 +5,7 @@ import { StorageAccessLevel } from '@aws-amplify/core'; import { ResolvedS3Config } from '../../../types/options'; import { StorageUploadDataPayload } from '../../../../../types'; -import { Part, createMultipartUpload } from '../../../utils/client'; +import { Part, createMultipartUpload } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; import { diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts index e5619655f3b..ce14939be7f 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts @@ -9,7 +9,7 @@ import { import { UPLOADS_STORAGE_KEY } from '../../../utils/constants'; import { ResolvedS3Config } from '../../../types/options'; -import { Part, listParts } from '../../../utils/client'; +import { Part, listParts } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; const ONE_HOUR = 1000 * 60 * 60; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index d164a09dac8..e84de95619b 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -26,7 +26,7 @@ import { abortMultipartUpload, completeMultipartUpload, headObject, -} from '../../../utils/client'; +} from '../../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../../utils/userAgent'; import { logger } from '../../../../../utils'; import { S3InternalConfig } from '../../internal/types'; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts index c93d791aad3..224fce0e210 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts @@ -4,7 +4,7 @@ import { TransferProgressEvent } from '../../../../../types'; import { ResolvedS3Config } from '../../../types/options'; import { calculateContentMd5 } from '../../../utils'; -import { uploadPart } from '../../../utils/client'; +import { uploadPart } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; import { PartToUpload } from './getDataChunker'; diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index 6ab0ebb9012..4da8bf328b5 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -10,7 +10,7 @@ import { validateStorageOperationInput, } from '../../utils'; import { ItemWithKey, ItemWithPath } from '../../types/outputs'; -import { putObject } from '../../utils/client'; +import { putObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; import { S3InternalConfig } from '../internal/types'; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts new file mode 100644 index 00000000000..380488bf4ac --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -0,0 +1,71 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + AmplifyUrl, + getAmplifyUserAgent, +} from '@aws-amplify/core/internals/utils'; +import { + EndpointResolverOptions, + getDnsSuffix, + getRetryDecider, + jitteredBackoff, +} from '@aws-amplify/core/internals/aws-client-utils'; + +import { parseXmlError } from '../utils'; + +/** + * The service name used to sign requests if the API requires authentication. + */ +export const SERVICE_NAME = 's3'; + +/** + * Options for endpoint resolver. + * + * @internal + */ +export type S3EndpointResolverOptions = EndpointResolverOptions & { + /** + * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region. + */ + customEndpoint?: string; +}; + +/** + * The endpoint resolver function that returns the endpoint URL for a given region, and input parameters. + */ +const endpointResolver = ( + options: S3EndpointResolverOptions, + apiInput?: { AccountId?: string }, +) => { + const { region, customEndpoint } = options; + const { AccountId: accountId } = apiInput || {}; + let endpoint: URL; + // 1. get base endpoint + if (customEndpoint) { + endpoint = new AmplifyUrl(customEndpoint); + } else if (accountId) { + // Control plane operations + endpoint = new AmplifyUrl( + `https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`, + ); + } else { + endpoint = new AmplifyUrl( + `https://s3-control.${region}.${getDnsSuffix(region)}`, + ); + } + + return { url: endpoint }; +}; + +/** + * @internal + */ +export const defaultConfig = { + service: SERVICE_NAME, + endpointResolver, + retryDecider: getRetryDecider(parseXmlError), + computeDelay: jitteredBackoff, + userAgentValue: getAmplifyUserAgent(), + uriEscapePath: false, // Required by S3. See https://github.com/aws/aws-sdk-js-v3/blob/9ba012dfa3a3429aa2db0f90b3b0b3a7a31f9bc3/packages/signature-v4/src/SignatureV4.ts#L76-L83 +}; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts b/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts new file mode 100644 index 00000000000..f1053d8ddd7 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts @@ -0,0 +1,97 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + Endpoint, + HttpRequest, + HttpResponse, + parseMetadata, +} from '@aws-amplify/core/internals/aws-client-utils'; +import { + AmplifyUrl, + AmplifyUrlSearchParams, +} from '@aws-amplify/core/internals/utils'; +import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; + +import { + assignStringVariables, + buildStorageServiceError, + deserializeTimestamp, + map, + parseXmlBody, + parseXmlError, + s3TransferHandler, +} from '../utils'; + +import type { + GetDataAccessCommandInput, + GetDataAccessCommandOutput, +} from './types'; +import { defaultConfig } from './base'; + +export type GetDataAccessInput = GetDataAccessCommandInput; + +export type GetDataAccessOutput = GetDataAccessCommandOutput; + +const getDataAccessSerializer = ( + input: GetDataAccessInput, + endpoint: Endpoint, +): HttpRequest => { + const headers = assignStringVariables({ + 'x-amz-account-id': input.AccountId, + }); + const query = assignStringVariables({ + durationSeconds: input.DurationSeconds, + permission: input.Permission, + privilege: input.Privilege, + target: input.Target, + targetType: input.TargetType, + }); + const url = new AmplifyUrl(endpoint.url.toString()); + url.search = new AmplifyUrlSearchParams(query).toString(); + + // Ref: https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_GetDataAccess.html + url.pathname = '/v20180820/accessgrantsinstance/dataaccess'; + + return { + method: 'GET', + headers, + url, + }; +}; + +const getDataAccessDeserializer = async ( + response: HttpResponse, +): Promise => { + if (response.statusCode >= 300) { + // error is always set when statusCode >= 300 + const error = (await parseXmlError(response)) as Error; + throw buildStorageServiceError(error, response.statusCode); + } else { + const parsed = await parseXmlBody(response); + const contents = map(parsed, { + Credentials: ['Credentials', deserializeCredentials], + MatchedGrantTarget: 'MatchedGrantTarget', + }); + + return { + $metadata: parseMetadata(response), + ...contents, + }; + } +}; + +const deserializeCredentials = (output: any) => + map(output, { + AccessKeyId: 'AccessKeyId', + Expiration: ['Expiration', deserializeTimestamp], + SecretAccessKey: 'SecretAccessKey', + SessionToken: 'SessionToken', + }); + +export const getDataAccess = composeServiceApi( + s3TransferHandler, + getDataAccessSerializer, + getDataAccessDeserializer, + { ...defaultConfig, responseType: 'text' }, +); diff --git a/packages/storage/src/providers/s3/utils/client/s3control/index.ts b/packages/storage/src/providers/s3/utils/client/s3control/index.ts new file mode 100644 index 00000000000..b9ae5230334 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/index.ts @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + getDataAccess, + GetDataAccessInput, + GetDataAccessOutput, +} from '../s3control/getDataAccess'; +export { + listCallerAccessGrants, + ListCallerAccessGrantsInput, + ListCallerAccessGrantsOutput, +} from '../s3control/listCallerAccessGrants'; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts new file mode 100644 index 00000000000..3c95369fb99 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts @@ -0,0 +1,100 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + Endpoint, + HttpRequest, + HttpResponse, + parseMetadata, +} from '@aws-amplify/core/internals/aws-client-utils'; +import { + AmplifyUrl, + AmplifyUrlSearchParams, +} from '@aws-amplify/core/internals/utils'; +import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; + +import { + assignStringVariables, + buildStorageServiceError, + emptyArrayGuard, + map, + parseXmlBody, + parseXmlError, + s3TransferHandler, +} from '../utils'; + +import type { + ListCallerAccessGrantsCommandInput, + ListCallerAccessGrantsCommandOutput, +} from './types'; +import { defaultConfig } from './base'; + +export type ListCallerAccessGrantsInput = ListCallerAccessGrantsCommandInput; + +export type ListCallerAccessGrantsOutput = ListCallerAccessGrantsCommandOutput; + +const listCallerAccessGrantsSerializer = ( + input: ListCallerAccessGrantsInput, + endpoint: Endpoint, +): HttpRequest => { + const headers = assignStringVariables({ + 'x-amz-account-id': input.AccountId, + }); + const query = assignStringVariables({ + grantscope: input.GrantScope, + maxResults: input.MaxResults, + nextToken: input.NextToken, + }); + const url = new AmplifyUrl(endpoint.url.toString()); + url.search = new AmplifyUrlSearchParams(query).toString(); + + // Ref: NA + url.pathname = '/v20180820/accessgrantsinstance/caller/grants'; + + return { + method: 'GET', + headers, + url, + }; +}; + +const listCallerAccessGrantsDeserializer = async ( + response: HttpResponse, +): Promise => { + if (response.statusCode >= 300) { + // error is always set when statusCode >= 300 + const error = (await parseXmlError(response)) as Error; + throw buildStorageServiceError(error, response.statusCode); + } else { + const parsed = await parseXmlBody(response); + const contents = map(parsed, { + CallerAccessGrantsList: [ + 'CallerAccessGrantsList', + value => emptyArrayGuard(value, deserializeAccessGrantsList), + ], + NextToken: 'NextToken', + }); + + return { + $metadata: parseMetadata(response), + ...contents, + }; + } +}; + +const deserializeAccessGrantsList = (output: any[]) => + output.map(deserializeCallerAccessGrant); + +const deserializeCallerAccessGrant = (output: any) => + map(output.AccessGrantsInstance, { + ApplicationArn: 'ApplicationArn', + GrantScope: 'GrantScope', + Permission: 'Permission', + }); + +export const listCallerAccessGrants = composeServiceApi( + s3TransferHandler, + listCallerAccessGrantsSerializer, + listCallerAccessGrantsDeserializer, + { ...defaultConfig, responseType: 'text' }, +); diff --git a/packages/storage/src/providers/s3/utils/client/s3control/types.ts b/packages/storage/src/providers/s3/utils/client/s3control/types.ts new file mode 100644 index 00000000000..e88e2438707 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/types.ts @@ -0,0 +1,213 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/* +This file contains manually curated AWS service types that are not yet available via the AWS SDK and the `dts-bundler` +script. Once these APIs have been released to the AWS SDK, this file can be removed in favor of the `dts-bundler` +types. + +These types were harvested from Trebuchet. + +@TODO(jimblanc) Unify types & integrate with `dts-bundler` +*/ + +import { MetadataBearer as __MetadataBearer } from '@aws-sdk/types'; + +declare const Permission: { + readonly READ: 'READ'; + readonly READWRITE: 'READWRITE'; + readonly WRITE: 'WRITE'; +}; +declare const Privilege: { + readonly Default: 'Default'; + readonly Minimal: 'Minimal'; +}; +declare const S3PrefixType: { + readonly Object: 'Object'; +}; + +/** + * @public + */ +export type Permission = (typeof Permission)[keyof typeof Permission]; + +/** + * @public + */ +export type Privilege = (typeof Privilege)[keyof typeof Privilege]; + +/** + * @public + */ +export type S3PrefixType = (typeof S3PrefixType)[keyof typeof S3PrefixType]; + +/** + * @public + * + * The input for {@link ListCallerAccessGrantsCommand}. + */ +export type ListCallerAccessGrantsCommandInput = ListCallerAccessGrantsRequest; + +/** + * @public + * + * The output of {@link ListCallerAccessGrantsCommand}. + */ +export interface ListCallerAccessGrantsCommandOutput + extends ListCallerAccessGrantsResult, + __MetadataBearer {} + +/** + * @public + */ +export interface ListCallerAccessGrantsRequest { + AccountId?: string; + GrantScope?: string; + NextToken?: string; + MaxResults?: number; +} + +/** + * @public + */ +export interface ListCallerAccessGrantsEntry { + Permission?: Permission | string; + GrantScope?: string; + ApplicationArn?: string; +} + +/** + * @public + */ +export interface ListCallerAccessGrantsResult { + NextToken?: string; + CallerAccessGrantsList?: ListCallerAccessGrantsEntry[]; +} + +/** + * @public + * + * The input for {@link GetDataAccessCommand}. + */ +export type GetDataAccessCommandInput = GetDataAccessRequest; + +/** + * @public + * + * The output of {@link GetDataAccessCommand}. + */ +export interface GetDataAccessCommandOutput + extends GetDataAccessResult, + __MetadataBearer {} + +/** + *

The Amazon Web Services Security Token Service temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ +export interface Credentials { + /** + *

The unique access key ID of the Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ + AccessKeyId?: string; + + /** + *

The secret access key of the Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ + SecretAccessKey?: string; + + /** + *

The Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ + SessionToken?: string; + + /** + *

The expiration date and time of the temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ + Expiration?: Date; +} + +/** + * @public + */ +export interface GetDataAccessRequest { + /** + *

The ID of the Amazon Web Services account that is making this request.

+ * @public + */ + AccountId?: string; + + /** + *

The S3 URI path of the data to which you are requesting temporary access credentials. If the requesting account has an access grant for this data, S3 Access Grants vends temporary access credentials in the response.

+ * @public + */ + Target: string | undefined; + + /** + *

The type of permission granted to your S3 data, which can be set to one of the following values:

+ *
    + *
  • + *

    + * READ – Grant read-only access to the S3 data.

    + *
    • + *
  • + *

    + * WRITE – Grant write-only access to the S3 data.

    + *
    • + *
  • + *

    + * READWRITE – Grant both read and write access to the S3 data.

    + *
    • + *
+ * @public + */ + Permission: Permission | undefined; + + /** + *

The session duration, in seconds, of the temporary access credential that S3 Access Grants vends to the grantee or client application. The default value is 1 hour, but the grantee can specify a range from 900 seconds (15 minutes) up to 43200 seconds (12 hours). If the grantee requests a value higher than this maximum, the operation fails.

+ * @public + */ + DurationSeconds?: number; + + /** + *

The scope of the temporary access credential that S3 Access Grants vends to the grantee or client application.

+ *
    + *
  • + *

    + * Default – The scope of the returned temporary access token is the scope of the grant that is closest to the target scope.

    + *
    • + *
  • + *

    + * Minimal – The scope of the returned temporary access token is the same as the requested target scope as long as the requested scope is the same as or a subset of the grant scope.

    + *
    • + *
+ * @public + */ + Privilege?: Privilege; + + /** + *

The type of Target. The only possible value is Object. Pass this value if the target data that you would like to access is a path to an object. Do not pass this value if the target data is a bucket or a bucket and a prefix.

+ * @public + */ + TargetType?: S3PrefixType; +} + +/** + * @public + */ +export interface GetDataAccessResult { + /** + *

The temporary credential token that S3 Access Grants vends.

+ * @public + */ + Credentials?: Credentials; + + /** + *

The S3 URI path of the data to which you are being granted temporary access credentials.

+ * @public + */ + MatchedGrantTarget?: string; +} diff --git a/packages/storage/src/providers/s3/utils/client/abortMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts similarity index 98% rename from packages/storage/src/providers/s3/utils/client/abortMultipartUpload.ts rename to packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts index bddaf570d0e..fb541e803a0 100644 --- a/packages/storage/src/providers/s3/utils/client/abortMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts @@ -14,15 +14,16 @@ import { } from '@aws-amplify/core/internals/utils'; import { MetadataBearer } from '@aws-sdk/types'; -import type { AbortMultipartUploadCommandInput } from './types'; -import { defaultConfig } from './base'; import { buildStorageServiceError, parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { AbortMultipartUploadCommandInput } from './types'; +import { defaultConfig } from './base'; export type AbortMultipartUploadInput = Pick< AbortMultipartUploadCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts similarity index 98% rename from packages/storage/src/providers/s3/utils/client/base.ts rename to packages/storage/src/providers/s3/utils/client/s3data/base.ts index 96f0e5958ef..a31d6d5a2f1 100644 --- a/packages/storage/src/providers/s3/utils/client/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -12,7 +12,7 @@ import { jitteredBackoff, } from '@aws-amplify/core/internals/aws-client-utils'; -import { parseXmlError } from './utils'; +import { parseXmlError } from '../utils'; const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/; const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; diff --git a/packages/storage/src/providers/s3/utils/client/completeMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/completeMultipartUpload.ts rename to packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts index 36dd9f59a52..59a8e029afc 100644 --- a/packages/storage/src/providers/s3/utils/client/completeMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts @@ -13,13 +13,6 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - CompleteMultipartUploadCommandInput, - CompleteMultipartUploadCommandOutput, - CompletedMultipartUpload, - CompletedPart, -} from './types'; -import { defaultConfig } from './base'; import { buildStorageServiceError, map, @@ -28,7 +21,15 @@ import { s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + CompleteMultipartUploadCommandInput, + CompleteMultipartUploadCommandOutput, + CompletedMultipartUpload, + CompletedPart, +} from './types'; +import { defaultConfig } from './base'; const INVALID_PARAMETER_ERROR_MSG = 'Invalid parameter for ComplteMultipartUpload API'; diff --git a/packages/storage/src/providers/s3/utils/client/copyObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/copyObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts index a08301d9f7e..f56745680fa 100644 --- a/packages/storage/src/providers/s3/utils/client/copyObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts @@ -10,8 +10,6 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { CopyObjectCommandInput, CopyObjectCommandOutput } from './types'; -import { defaultConfig } from './base'; import { assignStringVariables, buildStorageServiceError, @@ -21,7 +19,10 @@ import { serializeObjectConfigsToHeaders, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { CopyObjectCommandInput, CopyObjectCommandOutput } from './types'; +import { defaultConfig } from './base'; export type CopyObjectInput = Pick< CopyObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/createMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/createMultipartUpload.ts rename to packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts index 5a2b79a9635..2b8669eb44c 100644 --- a/packages/storage/src/providers/s3/utils/client/createMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts @@ -10,12 +10,6 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - CreateMultipartUploadCommandInput, - CreateMultipartUploadCommandOutput, -} from './types'; -import type { PutObjectInput } from './putObject'; -import { defaultConfig } from './base'; import { buildStorageServiceError, map, @@ -25,7 +19,14 @@ import { serializeObjectConfigsToHeaders, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + CreateMultipartUploadCommandInput, + CreateMultipartUploadCommandOutput, +} from './types'; +import type { PutObjectInput } from './putObject'; +import { defaultConfig } from './base'; export type CreateMultipartUploadInput = Extract< CreateMultipartUploadCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/deleteObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts similarity index 98% rename from packages/storage/src/providers/s3/utils/client/deleteObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts index 290a3e5ebf0..f8843881668 100644 --- a/packages/storage/src/providers/s3/utils/client/deleteObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts @@ -10,11 +10,6 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - DeleteObjectCommandInput, - DeleteObjectCommandOutput, -} from './types'; -import { defaultConfig } from './base'; import { buildStorageServiceError, deserializeBoolean, @@ -23,7 +18,13 @@ import { s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + DeleteObjectCommandInput, + DeleteObjectCommandOutput, +} from './types'; +import { defaultConfig } from './base'; export type DeleteObjectInput = Pick< DeleteObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/getObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/getObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/getObject.ts index 4af6a32a39c..f01de5018ae 100644 --- a/packages/storage/src/providers/s3/utils/client/getObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts @@ -14,12 +14,6 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import { S3EndpointResolverOptions, defaultConfig } from './base'; -import type { - CompatibleHttpResponse, - GetObjectCommandInput, - GetObjectCommandOutput, -} from './types'; import { CONTENT_SHA256_HEADER, buildStorageServiceError, @@ -32,7 +26,14 @@ import { s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { S3EndpointResolverOptions, defaultConfig } from './base'; +import type { + CompatibleHttpResponse, + GetObjectCommandInput, + GetObjectCommandOutput, +} from './types'; const USER_AGENT_HEADER = 'x-amz-user-agent'; diff --git a/packages/storage/src/providers/s3/utils/client/headObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/headObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/headObject.ts index 109263def26..1ffdeedeb55 100644 --- a/packages/storage/src/providers/s3/utils/client/headObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts @@ -10,8 +10,6 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import { defaultConfig } from './base'; -import type { HeadObjectCommandInput, HeadObjectCommandOutput } from './types'; import { buildStorageServiceError, deserializeMetadata, @@ -22,7 +20,10 @@ import { s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { defaultConfig } from './base'; +import type { HeadObjectCommandInput, HeadObjectCommandOutput } from './types'; export type HeadObjectInput = Pick; diff --git a/packages/storage/src/providers/s3/utils/client/index.ts b/packages/storage/src/providers/s3/utils/client/s3data/index.ts similarity index 100% rename from packages/storage/src/providers/s3/utils/client/index.ts rename to packages/storage/src/providers/s3/utils/client/s3data/index.ts diff --git a/packages/storage/src/providers/s3/utils/client/listObjectsV2.ts b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/listObjectsV2.ts rename to packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts index 232499931c5..680111cf0e9 100644 --- a/packages/storage/src/providers/s3/utils/client/listObjectsV2.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts @@ -13,11 +13,6 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - ListObjectsV2CommandInput, - ListObjectsV2CommandOutput, -} from './types'; -import { defaultConfig } from './base'; import { assignStringVariables, buildStorageServiceError, @@ -29,7 +24,13 @@ import { parseXmlBody, parseXmlError, s3TransferHandler, -} from './utils'; +} from '../utils'; + +import type { + ListObjectsV2CommandInput, + ListObjectsV2CommandOutput, +} from './types'; +import { defaultConfig } from './base'; export type ListObjectsV2Input = ListObjectsV2CommandInput; diff --git a/packages/storage/src/providers/s3/utils/client/listParts.ts b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/listParts.ts rename to packages/storage/src/providers/s3/utils/client/s3data/listParts.ts index 86899ad4e9d..1a9deb3e4b9 100644 --- a/packages/storage/src/providers/s3/utils/client/listParts.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts @@ -13,12 +13,6 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - CompletedPart, - ListPartsCommandInput, - ListPartsCommandOutput, -} from './types'; -import { defaultConfig } from './base'; import { buildStorageServiceError, deserializeNumber, @@ -29,7 +23,14 @@ import { s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + CompletedPart, + ListPartsCommandInput, + ListPartsCommandOutput, +} from './types'; +import { defaultConfig } from './base'; export type ListPartsInput = Pick< ListPartsCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/putObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/putObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/putObject.ts index 86755f1c703..0feac108508 100644 --- a/packages/storage/src/providers/s3/utils/client/putObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts @@ -10,8 +10,6 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import { defaultConfig } from './base'; -import type { PutObjectCommandInput, PutObjectCommandOutput } from './types'; import { assignStringVariables, buildStorageServiceError, @@ -21,7 +19,10 @@ import { serializeObjectConfigsToHeaders, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { defaultConfig } from './base'; +import type { PutObjectCommandInput, PutObjectCommandOutput } from './types'; export type PutObjectInput = Pick< PutObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/types.ts b/packages/storage/src/providers/s3/utils/client/s3data/types.ts similarity index 100% rename from packages/storage/src/providers/s3/utils/client/types.ts rename to packages/storage/src/providers/s3/utils/client/s3data/types.ts diff --git a/packages/storage/src/providers/s3/utils/client/uploadPart.ts b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts similarity index 99% rename from packages/storage/src/providers/s3/utils/client/uploadPart.ts rename to packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts index 3bcacc6236f..f27bbdd5a32 100644 --- a/packages/storage/src/providers/s3/utils/client/uploadPart.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts @@ -13,8 +13,6 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import { defaultConfig } from './base'; -import type { UploadPartCommandInput, UploadPartCommandOutput } from './types'; import { assignStringVariables, buildStorageServiceError, @@ -23,7 +21,10 @@ import { s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { defaultConfig } from './base'; +import type { UploadPartCommandInput, UploadPartCommandOutput } from './types'; // Content-length is ignored here because it's forbidden header // and will be set by browser or fetch polyfill. diff --git a/scripts/dts-bundler/dts-bundler.config.js b/scripts/dts-bundler/dts-bundler.config.js index 77671630aeb..b72769224d2 100644 --- a/scripts/dts-bundler/dts-bundler.config.js +++ b/scripts/dts-bundler/dts-bundler.config.js @@ -74,7 +74,7 @@ const config = { }, { filePath: './s3.d.ts', - outFile: join(storagePackageSrcClientsPath, 'client', 'types.ts'), + outFile: join(storagePackageSrcClientsPath, 'client', 's3data', 'types.ts'), libraries: { inlinedLibraries: ['@aws-sdk/client-s3'], }, From 335315a3265697b80092325837b4c1fec6cd38a5 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Mon, 15 Jul 2024 11:43:35 -0700 Subject: [PATCH 12/40] feat(storage): add an adapter interface for storage browser (#13576) --------- Co-authored-by: Jim Blanchard --- .../createManagedAuthConfigAdapter.test.ts | 72 +++++++++++++++++++ packages/storage/src/storageBrowser/index.ts | 18 +---- .../storageBrowser/listCallerAccessGrants.ts | 5 -- .../locationCredentialsStore/create.ts | 4 +- .../locationCredentialsStore/registry.ts | 4 +- .../locationCredentialsStore/store.ts | 8 +-- .../src/storageBrowser/managedAuthAdapter.ts | 27 ------- .../createListLocationsHandler.ts | 18 +++++ .../createLocationCredentialsHandler.ts | 4 +- .../createManagedAuthConfigAdapter.ts | 52 ++++++++++++++ .../managedAuthConfigAdapter/index.ts | 4 ++ packages/storage/src/storageBrowser/types.ts | 16 +++-- 12 files changed, 170 insertions(+), 62 deletions(-) create mode 100644 packages/storage/__tests__/storageBrowser/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts delete mode 100644 packages/storage/src/storageBrowser/managedAuthAdapter.ts create mode 100644 packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts rename packages/storage/src/storageBrowser/{ => managedAuthConfigAdapter}/createLocationCredentialsHandler.ts (80%) create mode 100644 packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts create mode 100644 packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts diff --git a/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts b/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts new file mode 100644 index 00000000000..124fc16cd7a --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts @@ -0,0 +1,72 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createManagedAuthConfigAdapter } from '../../../src/storageBrowser/managedAuthConfigAdapter'; +import { createListLocationsHandler } from '../../../src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler'; +import { createLocationCredentialsHandler } from '../../../src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler'; + +jest.mock( + '../../../src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler', +); +jest.mock( + '../../../src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler', +); + +describe('createManagedAuthConfigAdapter', () => { + const region = 'us-foo-2'; + const accountId = 'XXXXXXXXXXXX'; + const credentialsProvider = jest.fn(); + + beforeEach(() => { + jest + .mocked(createListLocationsHandler) + .mockReturnValue('LIST_LOCATIONS_FN' as any); + jest + .mocked(createLocationCredentialsHandler) + .mockReturnValue('GET_LOCATION_CREDENTIALS_FN' as any); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should pass region to the adapter', () => { + expect(createManagedAuthConfigAdapter({ region } as any)).toMatchObject({ + region, + }); + }); + + it('should create list locations handler', () => { + expect( + createManagedAuthConfigAdapter({ + region, + accountId, + credentialsProvider, + }), + ).toMatchObject({ + listLocations: 'LIST_LOCATIONS_FN', + }); + expect(createListLocationsHandler).toHaveBeenCalledWith({ + region, + accountId, + credentialsProvider, + }); + }); + + it('should create get location credentials handler', () => { + expect( + createManagedAuthConfigAdapter({ + region, + accountId, + credentialsProvider, + }), + ).toMatchObject({ + getLocationCredentials: 'GET_LOCATION_CREDENTIALS_FN', + }); + expect(createLocationCredentialsHandler).toHaveBeenCalledWith({ + region, + accountId, + credentialsProvider, + }); + }); +}); diff --git a/packages/storage/src/storageBrowser/index.ts b/packages/storage/src/storageBrowser/index.ts index 7d36d419558..bde6d9bfe82 100644 --- a/packages/storage/src/storageBrowser/index.ts +++ b/packages/storage/src/storageBrowser/index.ts @@ -1,20 +1,6 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -/** - * NOTE: The APIs exported from this file are ONLY intended for usage by - * Amplify UI. To use location-related features, please use - * @aws-amplify/ui-react-storage - */ - -export { - listCallerAccessGrants, - ListCallerAccessGrantsInput, - ListCallerAccessGrantsOutput, -} from './listCallerAccessGrants'; -export { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; export { createLocationCredentialsStore } from './locationCredentialsStore'; -export { - managedAuthAdapter, - ManagedAuthAdapterInput, -} from './managedAuthAdapter'; +export { createManagedAuthConfigAdapter } from './managedAuthConfigAdapter/createManagedAuthConfigAdapter'; +export { GetLocationCredentials, ListLocations } from './types'; diff --git a/packages/storage/src/storageBrowser/listCallerAccessGrants.ts b/packages/storage/src/storageBrowser/listCallerAccessGrants.ts index cd13e6a5dc5..c55b0ea75e8 100644 --- a/packages/storage/src/storageBrowser/listCallerAccessGrants.ts +++ b/packages/storage/src/storageBrowser/listCallerAccessGrants.ts @@ -7,11 +7,6 @@ export interface ListCallerAccessGrantsInput { accountId: string; credentialsProvider: CredentialsProvider; region: string; - options?: { - nextToken?: string; - // Default to 100; If > 1000, API will make multiple API calls. - pageSize?: number; - }; } export type ListCallerAccessGrantsOutput = ListLocationsOutput; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts index dc8da21bff5..a54ae32a354 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts @@ -3,7 +3,7 @@ import { CredentialsLocation, - LocationCredentialsHandler, + GetLocationCredentials, LocationCredentialsStore, } from '../types'; import { StorageValidationErrorCode } from '../../errors/types/validation'; @@ -17,7 +17,7 @@ import { createStore, getValue, removeStore } from './registry'; import { validateCredentialsProviderLocation } from './validators'; export const createLocationCredentialsStore = (input: { - handler: LocationCredentialsHandler; + handler: GetLocationCredentials; }): LocationCredentialsStore => { const storeSymbol = createStore(input.handler); diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts index 6c3d41780f9..54ed8e4db83 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts @@ -4,7 +4,7 @@ // SPDX-License-Identifier: Apache-2.0 import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { CredentialsLocation, LocationCredentialsHandler } from '../types'; +import { CredentialsLocation, GetLocationCredentials } from '../types'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; @@ -33,7 +33,7 @@ const storeRegistry = new WeakMap< * @internal */ export const createStore = ( - refreshHandler: LocationCredentialsHandler, + refreshHandler: GetLocationCredentials, size?: number, ) => { const storeSymbol = { value: Symbol('LocationCredentialsStore') }; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts index ac1901d669b..04e800365c7 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts @@ -6,7 +6,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Permission } from '../../providers/s3/types/options'; -import { CredentialsLocation, LocationCredentialsHandler } from '../types'; +import { CredentialsLocation, GetLocationCredentials } from '../types'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; @@ -35,7 +35,7 @@ const createCacheKey = (location: CredentialsLocation): CacheKey => */ export interface LruLocationCredentialsStore { capacity: number; - refreshHandler: LocationCredentialsHandler; + refreshHandler: GetLocationCredentials; values: Map; } @@ -43,7 +43,7 @@ export interface LruLocationCredentialsStore { * @internal */ export const initStore = ( - refreshHandler: LocationCredentialsHandler, + refreshHandler: GetLocationCredentials, size = CREDENTIALS_STORE_DEFAULT_SIZE, ): LruLocationCredentialsStore => { assertValidationError( @@ -117,7 +117,7 @@ export const fetchNewValue = async ( }; const dispatchRefresh = ( - refreshHandler: LocationCredentialsHandler, + refreshHandler: GetLocationCredentials, value: StoreValue, onRefreshFailure: () => void, ) => { diff --git a/packages/storage/src/storageBrowser/managedAuthAdapter.ts b/packages/storage/src/storageBrowser/managedAuthAdapter.ts deleted file mode 100644 index 9f3214b3158..00000000000 --- a/packages/storage/src/storageBrowser/managedAuthAdapter.ts +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 -import { - CredentialsProvider, - ListLocations, - LocationCredentialsHandler, -} from './types'; - -export interface ManagedAuthAdapterInput { - accountId: string; - region: string; - credentialsProvider: CredentialsProvider; -} - -export interface ManagedAuthAdapterOutput { - listLocations: ListLocations; - getLocationCredentials: LocationCredentialsHandler; - region: string; -} - -export const managedAuthAdapter = ( - // eslint-disable-next-line unused-imports/no-unused-vars - input: ManagedAuthAdapterInput, -): ManagedAuthAdapterOutput => { - // TODO(@AllanZhengYP) - throw new Error('Not implemented'); -}; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts new file mode 100644 index 00000000000..c3e9c3c1a4a --- /dev/null +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts @@ -0,0 +1,18 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CredentialsProvider, ListLocations } from '../types'; + +export interface CreateListLocationsHandlerInput { + accountId: string; + credentialsProvider: CredentialsProvider; + region: string; +} + +export const createListLocationsHandler = ( + // eslint-disable-next-line unused-imports/no-unused-vars + input: CreateListLocationsHandlerInput, +): ListLocations => { + // TODO(@AllanZhengYP) + throw new Error('Not Implemented'); +}; diff --git a/packages/storage/src/storageBrowser/createLocationCredentialsHandler.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts similarity index 80% rename from packages/storage/src/storageBrowser/createLocationCredentialsHandler.ts rename to packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts index 5da981d71bc..9cded212928 100644 --- a/packages/storage/src/storageBrowser/createLocationCredentialsHandler.ts +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { CredentialsProvider, LocationCredentialsHandler } from './types'; +import { CredentialsProvider, GetLocationCredentials } from '../types'; interface CreateLocationCredentialsHandlerInput { accountId: string; @@ -12,7 +12,7 @@ interface CreateLocationCredentialsHandlerInput { export const createLocationCredentialsHandler = ( // eslint-disable-next-line unused-imports/no-unused-vars input: CreateLocationCredentialsHandlerInput, -): LocationCredentialsHandler => { +): GetLocationCredentials => { // TODO(@AllanZhengYP) throw new Error('Not Implemented'); }; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts new file mode 100644 index 00000000000..55d334b47c7 --- /dev/null +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts @@ -0,0 +1,52 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { + CredentialsProvider, + GetLocationCredentials, + ListLocations, +} from '../types'; + +import { createListLocationsHandler } from './createListLocationsHandler'; +import { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; + +interface CreateManagedAuthConfigAdapterInput { + accountId: string; + region: string; + credentialsProvider: CredentialsProvider; +} + +interface AuthConfigAdapter { + listLocations: ListLocations; + getLocationCredentials: GetLocationCredentials; + region: string; +} + +/** + * Create configuration including handlers to call S3 Access Grant APIs to list and get + * credentials for different locations. + * + * @param options - Configuration options for the adapter. + * @returns - An object containing the handlers to call S3 Access Grant APIs and region + */ +export const createManagedAuthConfigAdapter = ({ + credentialsProvider, + region, + accountId, +}: CreateManagedAuthConfigAdapterInput): AuthConfigAdapter => { + const listLocations = createListLocationsHandler({ + credentialsProvider, + accountId, + region, + }); + const getLocationCredentials = createLocationCredentialsHandler({ + credentialsProvider, + accountId, + region, + }); + + return { + listLocations, + getLocationCredentials, + region, + }; +}; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts new file mode 100644 index 00000000000..4e04b1a2a77 --- /dev/null +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { createManagedAuthConfigAdapter } from './createManagedAuthConfigAdapter'; diff --git a/packages/storage/src/storageBrowser/types.ts b/packages/storage/src/storageBrowser/types.ts index 2525560369f..3f184dcc313 100644 --- a/packages/storage/src/storageBrowser/types.ts +++ b/packages/storage/src/storageBrowser/types.ts @@ -68,11 +68,19 @@ export interface ListLocationsOutput { nextToken?: string; } -// Interface for listLocations() handler -export type ListLocations = () => Promise>; +/** + * @internal + */ +export interface ListLocationsInput { + pageSize?: number; + nextToken?: string; +} + +export type ListLocations = ( + input?: ListLocationsInput, +) => Promise>; -// Interface for getLocationCredentials() handler. -export type LocationCredentialsHandler = ( +export type GetLocationCredentials = ( input: CredentialsLocation, ) => Promise<{ credentials: AWSCredentials }>; From dcba3d7768f4daffff66a0fe02f1143edd4107eb Mon Sep 17 00:00:00 2001 From: israx <70438514+israx@users.noreply.github.com> Date: Mon, 15 Jul 2024 15:08:06 -0400 Subject: [PATCH 13/40] Revert "refactor(storage): decouple utils from Amplify singleton (#13562)" (#13597) This reverts commit 1079e7f04164b01c9a703e87e725ca91c3f1d219. --- .../s3/apis/uploadData/index.test.ts | 16 +- .../apis/uploadData/multipartHandlers.test.ts | 365 ++++++++---------- .../s3/apis/uploadData/putObjectJob.test.ts | 90 ++--- .../utils/resolveS3ConfigAndInput.test.ts | 165 ++++---- .../src/providers/s3/apis/downloadData.ts | 8 +- .../src/providers/s3/apis/internal/copy.ts | 22 +- .../s3/apis/internal/getProperties.ts | 7 +- .../src/providers/s3/apis/internal/getUrl.ts | 7 +- .../src/providers/s3/apis/internal/list.ts | 8 +- .../src/providers/s3/apis/internal/remove.ts | 7 +- .../providers/s3/apis/internal/types/index.ts | 31 -- .../providers/s3/apis/internal/uploadData.ts | 58 --- .../src/providers/s3/apis/uploadData/index.ts | 47 ++- .../uploadData/multipart/uploadHandlers.ts | 28 +- .../s3/apis/uploadData/putObjectJob.ts | 25 +- .../storage/src/providers/s3/types/options.ts | 11 - .../storage/src/providers/s3/utils/config.ts | 59 --- .../storage/src/providers/s3/utils/index.ts | 1 - .../s3/utils/resolveS3ConfigAndInput.ts | 61 ++- 19 files changed, 402 insertions(+), 614 deletions(-) delete mode 100644 packages/storage/src/providers/s3/apis/internal/types/index.ts delete mode 100644 packages/storage/src/providers/s3/apis/internal/uploadData.ts delete mode 100644 packages/storage/src/providers/s3/utils/config.ts diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts index 43775719dd3..938ca8863ee 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/index.test.ts @@ -172,12 +172,9 @@ describe('uploadData with path', () => { uploadData(testInput); expect(mockPutObjectJob).toHaveBeenCalledWith( - expect.objectContaining({ - input: testInput, - totalLength: expect.any(Number), - abortSignal: expect.any(AbortSignal), - config: expect.any(Object), - }), + testInput, + expect.any(AbortSignal), + expect.any(Number), ); expect(mockGetMultipartUploadHandlers).not.toHaveBeenCalled(); }, @@ -215,11 +212,8 @@ describe('uploadData with path', () => { expect(mockPutObjectJob).not.toHaveBeenCalled(); expect(mockGetMultipartUploadHandlers).toHaveBeenCalledWith( - expect.objectContaining({ - config: expect.any(Object), - input: testInput, - size: expect.any(Number), - }), + testInput, + expect.any(Number), ); }); diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index ab049042afd..04c6bf6522a 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { defaultStorage } from '@aws-amplify/core'; +import { Amplify, defaultStorage } from '@aws-amplify/core'; import { abortMultipartUpload, @@ -22,7 +22,6 @@ import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byte import { CanceledError } from '../../../../../src/errors/CanceledError'; import { StorageOptions } from '../../../../../src/types'; import '../testUtils'; -import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; jest.mock('@aws-amplify/core'); jest.mock('../../../../../src/providers/s3/utils/client/s3data'); @@ -33,6 +32,7 @@ const credentials: AWSCredentials = { secretAccessKey: 'secretAccessKey', }; const defaultIdentityId = 'defaultIdentityId'; +const mockFetchAuthSession = Amplify.Auth.fetchAuthSession as jest.Mock; const bucket = 'bucket'; const region = 'region'; const defaultKey = 'key'; @@ -131,22 +131,21 @@ const resetS3Mocks = () => { mockListParts.mockReset(); }; -const mockCredentialsProvider = jest.fn(); -const mockIdentityIdProvider = jest.fn(); -const mockServiceOptions = { bucket, region }; -const mockLibraryOptions = {}; - /* TODO Remove suite when `key` parameter is removed */ describe('getMultipartUploadHandlers with key', () => { - const mockS3Config: S3InternalConfig = { - credentialsProvider: mockCredentialsProvider, - identityIdProvider: mockIdentityIdProvider, - serviceOptions: mockServiceOptions, - libraryOptions: mockLibraryOptions, - }; beforeAll(() => { - mockCredentialsProvider.mockImplementation(async () => credentials); - mockIdentityIdProvider.mockImplementation(async () => defaultIdentityId); + mockFetchAuthSession.mockResolvedValue({ + credentials, + identityId: defaultIdentityId, + }); + (Amplify.getConfig as jest.Mock).mockReturnValue({ + Storage: { + S3: { + bucket, + region, + }, + }, + }); }); afterEach(() => { @@ -155,14 +154,13 @@ describe('getMultipartUploadHandlers with key', () => { }); it('should return multipart upload handlers', async () => { - const multipartUploadHandlers = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const multipartUploadHandlers = getMultipartUploadHandlers( + { key: defaultKey, data: { size: 5 * 1024 * 1024 } as any, }, - size: 5 * 1024 * 1024, - }); + 5 * 1024 * 1024, + ); expect(multipartUploadHandlers).toEqual({ multipartUploadJob: expect.any(Function), onPause: expect.any(Function), @@ -202,12 +200,9 @@ describe('getMultipartUploadHandlers with key', () => { async (_, twoPartsPayload) => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - key: defaultKey, - data: twoPartsPayload, - options: options as StorageOptions, - }, + key: defaultKey, + data: twoPartsPayload, + options: options as StorageOptions, }); const result = await multipartUploadJob(); await expect( @@ -237,11 +232,8 @@ describe('getMultipartUploadHandlers with key', () => { it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - key: defaultKey, - data: 1 as any, - }, + key: defaultKey, + data: 1 as any, }); await expect(multipartUploadJob()).rejects.toThrow( expect.objectContaining( @@ -267,14 +259,13 @@ describe('getMultipartUploadHandlers with key', () => { }), } as any as File; mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: file, }, - size: file.size, - }); + file.size, + ); await multipartUploadJob(); expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); @@ -294,14 +285,13 @@ describe('getMultipartUploadHandlers with key', () => { $metadata: {}, }); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(8 * MB), }, - size: 8 * MB, - }); + 8 * MB, + ); try { await multipartUploadJob(); fail('should throw error'); @@ -319,11 +309,8 @@ describe('getMultipartUploadHandlers with key', () => { mockCreateMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - key: defaultKey, - data: new ArrayBuffer(8 * MB), - }, + key: defaultKey, + data: new ArrayBuffer(8 * MB), }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -335,11 +322,8 @@ describe('getMultipartUploadHandlers with key', () => { mockCompleteMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - key: defaultKey, - data: new ArrayBuffer(8 * MB), - }, + key: defaultKey, + data: new ArrayBuffer(8 * MB), }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -356,11 +340,8 @@ describe('getMultipartUploadHandlers with key', () => { mockUploadPart.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - key: defaultKey, - data: new ArrayBuffer(8 * MB), - }, + key: defaultKey, + data: new ArrayBuffer(8 * MB), }); await expect(multipartUploadJob()).rejects.toThrow('error'); expect(mockUploadPart).toHaveBeenCalledTimes(2); @@ -380,14 +361,13 @@ describe('getMultipartUploadHandlers with key', () => { it('should send createMultipartUpload request if the upload task is not cached', async () => { mockMultipartUploadSuccess(); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -409,14 +389,13 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockListParts).not.toHaveBeenCalled(); @@ -428,14 +407,13 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new File([new ArrayBuffer(size)], 'someName'), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -464,14 +442,13 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); expect(mockCreateMultipartUpload).not.toHaveBeenCalled(); expect(mockListParts).toHaveBeenCalledTimes(1); @@ -483,14 +460,13 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -511,14 +487,13 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -534,14 +509,13 @@ describe('getMultipartUploadHandlers with key', () => { mockMultipartUploadSuccess(disableAssertionFlag); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(size), }, size, - }); + ); const uploadJobPromise = multipartUploadJob(); await uploadJobPromise; // 1 for caching upload task; 1 for remove cache after upload is completed @@ -557,11 +531,8 @@ describe('getMultipartUploadHandlers with key', () => { describe('cancel()', () => { it('should abort in-flight uploadPart requests and throw if upload is canceled', async () => { const { multipartUploadJob, onCancel } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - key: defaultKey, - data: new ArrayBuffer(8 * MB), - }, + key: defaultKey, + data: new ArrayBuffer(8 * MB), }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -588,11 +559,8 @@ describe('getMultipartUploadHandlers with key', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - key: defaultKey, - data: new ArrayBuffer(8 * MB), - }, + key: defaultKey, + data: new ArrayBuffer(8 * MB), }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -614,17 +582,16 @@ describe('getMultipartUploadHandlers with key', () => { it('should send progress for in-flight upload parts', async () => { const onProgress = jest.fn(); mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - size: 8 * MB, - }); + 8 * MB, + ); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(4); // 2 simulated onProgress events per uploadPart call are all tracked expect(onProgress).toHaveBeenNthCalledWith(1, { @@ -666,17 +633,16 @@ describe('getMultipartUploadHandlers with key', () => { }); const onProgress = jest.fn(); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { key: defaultKey, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - size: 8 * MB, - }); + 8 * MB, + ); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(3); // The first part's 5 MB progress is reported even though no uploadPart call is made. @@ -689,15 +655,19 @@ describe('getMultipartUploadHandlers with key', () => { }); describe('getMultipartUploadHandlers with path', () => { - const mockS3Config: S3InternalConfig = { - credentialsProvider: mockCredentialsProvider, - identityIdProvider: mockIdentityIdProvider, - serviceOptions: mockServiceOptions, - libraryOptions: mockLibraryOptions, - }; beforeAll(() => { - mockCredentialsProvider.mockImplementation(async () => credentials); - mockIdentityIdProvider.mockImplementation(async () => defaultIdentityId); + mockFetchAuthSession.mockResolvedValue({ + credentials, + identityId: defaultIdentityId, + }); + (Amplify.getConfig as jest.Mock).mockReturnValue({ + Storage: { + S3: { + bucket, + region, + }, + }, + }); }); afterEach(() => { @@ -706,14 +676,13 @@ describe('getMultipartUploadHandlers with path', () => { }); it('should return multipart upload handlers', async () => { - const multipartUploadHandlers = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const multipartUploadHandlers = getMultipartUploadHandlers( + { path: testPath, data: { size: 5 * 1024 * 1024 } as any, }, - size: 5 * 1024 * 1024, - }); + 5 * 1024 * 1024, + ); expect(multipartUploadHandlers).toEqual({ multipartUploadJob: expect.any(Function), onPause: expect.any(Function), @@ -746,27 +715,24 @@ describe('getMultipartUploadHandlers with path', () => { async (_, twoPartsPayload) => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - path: inputPath, - data: twoPartsPayload, - }, + path: inputPath, + data: twoPartsPayload, }); const result = await multipartUploadJob(); - // await expect( - // mockCreateMultipartUpload, - // ).toBeLastCalledWithConfigAndInput( - // expect.objectContaining({ - // credentials, - // region, - // abortSignal: expect.any(AbortSignal), - // }), - // expect.objectContaining({ - // Bucket: bucket, - // Key: expectedKey, - // ContentType: defaultContentType, - // }), - // ); + await expect( + mockCreateMultipartUpload, + ).toBeLastCalledWithConfigAndInput( + expect.objectContaining({ + credentials, + region, + abortSignal: expect.any(AbortSignal), + }), + expect.objectContaining({ + Bucket: bucket, + Key: expectedKey, + ContentType: defaultContentType, + }), + ); expect(result).toEqual( expect.objectContaining({ path: expectedKey, eTag: 'etag' }), ); @@ -780,11 +746,8 @@ describe('getMultipartUploadHandlers with path', () => { it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - path: testPath, - data: 1 as any, - }, + path: testPath, + data: 1 as any, }); await expect(multipartUploadJob()).rejects.toThrow( expect.objectContaining( @@ -810,14 +773,13 @@ describe('getMultipartUploadHandlers with path', () => { }), } as any as File; mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: file, }, - size: file.size, - }); + file.size, + ); await multipartUploadJob(); expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); @@ -837,14 +799,13 @@ describe('getMultipartUploadHandlers with path', () => { $metadata: {}, }); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(8 * MB), }, - size: 8 * MB, - }); + 8 * MB, + ); try { await multipartUploadJob(); fail('should throw error'); @@ -862,11 +823,8 @@ describe('getMultipartUploadHandlers with path', () => { mockCreateMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - path: testPath, - data: new ArrayBuffer(8 * MB), - }, + path: testPath, + data: new ArrayBuffer(8 * MB), }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -878,11 +836,8 @@ describe('getMultipartUploadHandlers with path', () => { mockCompleteMultipartUpload.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - path: testPath, - data: new ArrayBuffer(8 * MB), - }, + path: testPath, + data: new ArrayBuffer(8 * MB), }); await expect(multipartUploadJob()).rejects.toThrow('error'); }); @@ -899,11 +854,8 @@ describe('getMultipartUploadHandlers with path', () => { mockUploadPart.mockRejectedValueOnce(new Error('error')); const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - path: testPath, - data: new ArrayBuffer(8 * MB), - }, + path: testPath, + data: new ArrayBuffer(8 * MB), }); await expect(multipartUploadJob()).rejects.toThrow('error'); expect(mockUploadPart).toHaveBeenCalledTimes(2); @@ -923,14 +875,13 @@ describe('getMultipartUploadHandlers with path', () => { it('should send createMultipartUpload request if the upload task is not cached', async () => { mockMultipartUploadSuccess(); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -952,14 +903,13 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockListParts).not.toHaveBeenCalled(); @@ -971,14 +921,13 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new File([new ArrayBuffer(size)], 'someName'), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -1010,14 +959,13 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); expect(mockCreateMultipartUpload).not.toHaveBeenCalled(); expect(mockListParts).toHaveBeenCalledTimes(1); @@ -1029,14 +977,13 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -1055,14 +1002,13 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(size), }, size, - }); + ); await multipartUploadJob(); // 1 for caching upload task; 1 for remove cache after upload is completed expect(mockDefaultStorage.setItem).toHaveBeenCalledTimes(2); @@ -1078,14 +1024,13 @@ describe('getMultipartUploadHandlers with path', () => { mockMultipartUploadSuccess(disableAssertionFlag); mockListParts.mockResolvedValueOnce({ Parts: [], $metadata: {} }); const size = 8 * MB; - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(size), }, size, - }); + ); const uploadJobPromise = multipartUploadJob(); await uploadJobPromise; // 1 for caching upload task; 1 for remove cache after upload is completed @@ -1101,11 +1046,8 @@ describe('getMultipartUploadHandlers with path', () => { describe('cancel()', () => { it('should abort in-flight uploadPart requests and throw if upload is canceled', async () => { const { multipartUploadJob, onCancel } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - path: testPath, - data: new ArrayBuffer(8 * MB), - }, + path: testPath, + data: new ArrayBuffer(8 * MB), }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -1132,11 +1074,8 @@ describe('getMultipartUploadHandlers with path', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { - path: testPath, - data: new ArrayBuffer(8 * MB), - }, + path: testPath, + data: new ArrayBuffer(8 * MB), }); let partCount = 0; mockMultipartUploadCancellation(() => { @@ -1158,17 +1097,16 @@ describe('getMultipartUploadHandlers with path', () => { it('should send progress for in-flight upload parts', async () => { const onProgress = jest.fn(); mockMultipartUploadSuccess(); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - size: 8 * MB, - }); + 8 * MB, + ); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(4); // 2 simulated onProgress events per uploadPart call are all tracked expect(onProgress).toHaveBeenNthCalledWith(1, { @@ -1210,17 +1148,16 @@ describe('getMultipartUploadHandlers with path', () => { }); const onProgress = jest.fn(); - const { multipartUploadJob } = getMultipartUploadHandlers({ - config: mockS3Config, - input: { + const { multipartUploadJob } = getMultipartUploadHandlers( + { path: testPath, data: new ArrayBuffer(8 * MB), options: { onProgress, }, }, - size: 8 * MB, - }); + 8 * MB, + ); await multipartUploadJob(); expect(onProgress).toHaveBeenCalledTimes(3); // The first part's 5 MB progress is reported even though no uploadPart call is made. diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index f6e06fa1140..df1b92113a1 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -2,12 +2,12 @@ // SPDX-License-Identifier: Apache-2.0 import { AWSCredentials } from '@aws-amplify/core/internals/utils'; +import { Amplify } from '@aws-amplify/core'; import { putObject } from '../../../../../src/providers/s3/utils/client/s3data'; import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; -import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; jest.mock('../../../../../src/providers/s3/utils/client/s3data'); jest.mock('../../../../../src/providers/s3/utils', () => { @@ -20,6 +20,13 @@ jest.mock('../../../../../src/providers/s3/utils', () => { }); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn(), + fetchAuthSession: jest.fn(), + Amplify: { + getConfig: jest.fn(), + Auth: { + fetchAuthSession: jest.fn(), + }, + }, })); const testPath = 'testPath/object'; @@ -29,35 +36,31 @@ const credentials: AWSCredentials = { secretAccessKey: 'secretAccessKey', }; const identityId = 'identityId'; -const bucket = 'bucket'; -const region = 'region'; - -const mockCredentialsProvider = jest.fn(); -const mockIdentityIdProvider = jest.fn(); -const mockServiceOptions = { bucket, region }; -const mockLibraryOptions = {}; +const mockFetchAuthSession = jest.mocked(Amplify.Auth.fetchAuthSession); const mockPutObject = jest.mocked(putObject); +mockFetchAuthSession.mockResolvedValue({ + credentials, + identityId, +}); +jest.mocked(Amplify.getConfig).mockReturnValue({ + Storage: { + S3: { + bucket: 'bucket', + region: 'region', + }, + }, +}); mockPutObject.mockResolvedValue({ ETag: 'eTag', VersionId: 'versionId', $metadata: {}, }); -const config: S3InternalConfig = { - credentialsProvider: mockCredentialsProvider, - identityIdProvider: mockIdentityIdProvider, - serviceOptions: mockServiceOptions, - libraryOptions: mockLibraryOptions, -}; - /* TODO Remove suite when `key` parameter is removed */ describe('putObjectJob with key', () => { beforeEach(() => { - mockCredentialsProvider.mockImplementation(async () => credentials); - mockIdentityIdProvider.mockImplementation(async () => identityId); mockPutObject.mockClear(); - jest.clearAllMocks(); }); it('should supply the correct parameters to putObject API handler', async () => { @@ -71,9 +74,8 @@ describe('putObjectJob with key', () => { const onProgress = jest.fn(); const useAccelerateEndpoint = true; - const job = putObjectJob({ - config, - input: { + const job = putObjectJob( + { key: inputKey, data, options: { @@ -85,8 +87,8 @@ describe('putObjectJob with key', () => { useAccelerateEndpoint, }, }, - abortSignal: abortController.signal, - }); + abortController.signal, + ); const result = await job(); expect(result).toEqual({ key: inputKey, @@ -97,7 +99,6 @@ describe('putObjectJob with key', () => { size: undefined, }); expect(mockPutObject).toHaveBeenCalledTimes(1); - await expect(mockPutObject).toBeLastCalledWithConfigAndInput( { credentials, @@ -121,19 +122,20 @@ describe('putObjectJob with key', () => { }); it('should set ContentMD5 if object lock is enabled', async () => { - const job = putObjectJob({ - config: { - ...config, - libraryOptions: { + Amplify.libraryOptions = { + Storage: { + S3: { isObjectLockEnabled: true, }, }, - input: { + }; + const job = putObjectJob( + { key: 'key', data: 'data', }, - abortSignal: new AbortController().signal, - }); + new AbortController().signal, + ); await job(); expect(calculateContentMd5).toHaveBeenCalledWith('data'); }); @@ -141,8 +143,6 @@ describe('putObjectJob with key', () => { describe('putObjectJob with path', () => { beforeEach(() => { - mockCredentialsProvider.mockImplementation(async () => credentials); - mockIdentityIdProvider.mockImplementation(async () => identityId); mockPutObject.mockClear(); }); @@ -167,9 +167,8 @@ describe('putObjectJob with path', () => { const onProgress = jest.fn(); const useAccelerateEndpoint = true; - const job = putObjectJob({ - config, - input: { + const job = putObjectJob( + { path: inputPath, data, options: { @@ -181,8 +180,8 @@ describe('putObjectJob with path', () => { useAccelerateEndpoint, }, }, - abortSignal: abortController.signal, - }); + abortController.signal, + ); const result = await job(); expect(result).toEqual({ path: expectedKey, @@ -217,19 +216,20 @@ describe('putObjectJob with path', () => { ); it('should set ContentMD5 if object lock is enabled', async () => { - const job = putObjectJob({ - config: { - ...config, - libraryOptions: { + Amplify.libraryOptions = { + Storage: { + S3: { isObjectLockEnabled: true, }, }, - input: { + }; + const job = putObjectJob( + { path: testPath, data: 'data', }, - abortSignal: new AbortController().signal, - }); + new AbortController().signal, + ); await job(); expect(calculateContentMd5).toHaveBeenCalledWith('data'); }); diff --git a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts index ba527aa8dbf..e26cb63b6c7 100644 --- a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts @@ -1,18 +1,29 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { Amplify } from '@aws-amplify/core'; + import { resolveS3ConfigAndInput } from '../../../../../src/providers/s3/utils'; import { resolvePrefix } from '../../../../../src/utils/resolvePrefix'; import { StorageValidationErrorCode, validationErrorMap, } from '../../../../../src/errors/types/validation'; -import { S3InternalConfig } from '../../../../../src/providers/s3/apis/internal/types'; -import { assertValidationError } from '../../../../../src/errors/utils/assertValidationError'; +jest.mock('@aws-amplify/core', () => ({ + ConsoleLogger: jest.fn(), + Amplify: { + getConfig: jest.fn(), + Auth: { + fetchAuthSession: jest.fn(), + }, + }, +})); jest.mock('../../../../../src/utils/resolvePrefix'); +const mockGetConfig = Amplify.getConfig as jest.Mock; const mockDefaultResolvePrefix = resolvePrefix as jest.Mock; +const mockFetchAuthSession = Amplify.Auth.fetchAuthSession as jest.Mock; const bucket = 'bucket'; const region = 'region'; @@ -23,41 +34,39 @@ const credentials = { }; const targetIdentityId = 'targetIdentityId'; -const mockCredentialsProvider = jest.fn(); -const mockIdentityIdProvider = jest.fn(); -const mockServiceOptions = { bucket, region }; -const mockLibraryOptions = {}; - describe('resolveS3ConfigAndInput', () => { - const config: S3InternalConfig = { - credentialsProvider: mockCredentialsProvider, - identityIdProvider: mockIdentityIdProvider, - serviceOptions: mockServiceOptions, - libraryOptions: mockLibraryOptions, - }; beforeEach(() => { - mockCredentialsProvider.mockImplementation(async () => credentials); - mockIdentityIdProvider.mockImplementation(async () => targetIdentityId); jest.clearAllMocks(); + Amplify.libraryOptions = {}; + }); + mockFetchAuthSession.mockResolvedValue({ + credentials, + identityId: targetIdentityId, + }); + + mockGetConfig.mockReturnValue({ + Storage: { + S3: { + bucket, + region, + }, + }, }); it('should call fetchAuthSession for credentials and identityId', async () => { expect.assertions(1); - await resolveS3ConfigAndInput({ config }); - expect(mockIdentityIdProvider).toHaveBeenCalled(); + await resolveS3ConfigAndInput(Amplify, {}); + expect(mockFetchAuthSession).toHaveBeenCalled(); }); it('should throw if credentials are not available', async () => { expect.assertions(1); - mockCredentialsProvider.mockImplementation(async () => { - assertValidationError( - !!undefined, - StorageValidationErrorCode.NoCredentials, - ); + mockFetchAuthSession.mockResolvedValue({ + identityId: targetIdentityId, }); const { s3Config: { credentials: credentialsProvider }, - } = await resolveS3ConfigAndInput({ config }); + } = await resolveS3ConfigAndInput(Amplify, {}); if (typeof credentialsProvider === 'function') { await expect(credentialsProvider()).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoCredentials], @@ -68,97 +77,100 @@ describe('resolveS3ConfigAndInput', () => { }); it('should throw if identityId is not available', async () => { - mockIdentityIdProvider.mockImplementation(async () => { - assertValidationError(!!'', StorageValidationErrorCode.NoIdentityId); + mockFetchAuthSession.mockResolvedValueOnce({ + credentials, }); - await expect(resolveS3ConfigAndInput({ config })).rejects.toMatchObject( + await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoIdentityId], ); }); it('should resolve bucket from S3 config', async () => { - const { bucket: resolvedBucket } = await resolveS3ConfigAndInput({ - config, - }); + const { bucket: resolvedBucket } = await resolveS3ConfigAndInput( + Amplify, + {}, + ); expect(resolvedBucket).toEqual(bucket); + expect(mockGetConfig).toHaveBeenCalled(); }); it('should throw if bucket is not available', async () => { - await expect( - resolveS3ConfigAndInput({ - config: { - ...config, - serviceOptions: { - bucket: undefined, - }, + mockGetConfig.mockReturnValueOnce({ + Storage: { + S3: { + region, }, - }), - ).rejects.toMatchObject( + }, + }); + await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoBucket], ); }); it('should resolve region from S3 config', async () => { - const { s3Config } = await resolveS3ConfigAndInput({ config }); + const { s3Config } = await resolveS3ConfigAndInput(Amplify, {}); expect(s3Config.region).toEqual(region); + expect(mockGetConfig).toHaveBeenCalled(); }); it('should throw if region is not available', async () => { - await expect( - resolveS3ConfigAndInput({ - config: { - ...config, - serviceOptions: { - bucket, - }, + mockGetConfig.mockReturnValueOnce({ + Storage: { + S3: { + bucket, }, - }), - ).rejects.toMatchObject( + }, + }); + await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( validationErrorMap[StorageValidationErrorCode.NoRegion], ); }); it('should set customEndpoint and forcePathStyle to true if dangerouslyConnectToHttpEndpointForTesting is set from S3 config', async () => { - const serviceOptions = { - bucket, - region, - dangerouslyConnectToHttpEndpointForTesting: 'true', - }; - - const { s3Config } = await resolveS3ConfigAndInput({ - config: { ...config, serviceOptions }, + mockGetConfig.mockReturnValueOnce({ + Storage: { + S3: { + bucket, + region, + dangerouslyConnectToHttpEndpointForTesting: true, + }, + }, }); + const { s3Config } = await resolveS3ConfigAndInput(Amplify, {}); expect(s3Config.customEndpoint).toEqual('http://localhost:20005'); expect(s3Config.forcePathStyle).toEqual(true); + expect(mockGetConfig).toHaveBeenCalled(); }); it('should resolve isObjectLockEnabled from S3 library options', async () => { - const { isObjectLockEnabled } = await resolveS3ConfigAndInput({ - config: { - ...config, - libraryOptions: { isObjectLockEnabled: true }, + Amplify.libraryOptions = { + Storage: { + S3: { + isObjectLockEnabled: true, + }, }, - }); + }; + const { isObjectLockEnabled } = await resolveS3ConfigAndInput(Amplify, {}); expect(isObjectLockEnabled).toEqual(true); }); it('should use default prefix resolver', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput({ config }); + const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); expect(mockDefaultResolvePrefix).toHaveBeenCalled(); expect(keyPrefix).toEqual('prefix'); }); it('should use prefix resolver from S3 library options if supplied', async () => { const customResolvePrefix = jest.fn().mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput({ - config: { - ...config, - libraryOptions: { + Amplify.libraryOptions = { + Storage: { + S3: { prefixResolver: customResolvePrefix, }, }, - }); + }; + const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); expect(customResolvePrefix).toHaveBeenCalled(); expect(keyPrefix).toEqual('prefix'); expect(mockDefaultResolvePrefix).not.toHaveBeenCalled(); @@ -166,11 +178,8 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with given access level', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput({ - config, - apiOptions: { - accessLevel: 'someLevel' as any, - }, + const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, { + accessLevel: 'someLevel' as any, }); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'someLevel', @@ -181,14 +190,14 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with default access level from S3 library options', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput({ - config: { - ...config, - libraryOptions: { + Amplify.libraryOptions = { + Storage: { + S3: { defaultAccessLevel: 'someLevel' as any, }, }, - }); + }; + const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'someLevel', targetIdentityId, @@ -198,7 +207,7 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with `guest` access level if no access level is given', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); - const { keyPrefix } = await resolveS3ConfigAndInput({ config }); + const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, {}); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'guest', // default access level targetIdentityId, diff --git a/packages/storage/src/providers/s3/apis/downloadData.ts b/packages/storage/src/providers/s3/apis/downloadData.ts index e67ab7f6b25..009e75ae95b 100644 --- a/packages/storage/src/providers/s3/apis/downloadData.ts +++ b/packages/storage/src/providers/s3/apis/downloadData.ts @@ -13,7 +13,6 @@ import { import { resolveS3ConfigAndInput } from '../utils/resolveS3ConfigAndInput'; import { createDownloadTask, validateStorageOperationInput } from '../utils'; import { getObject } from '../utils/client/s3data'; -import { createStorageConfiguration } from '../utils/config'; import { getStorageUserAgentValue } from '../utils/userAgent'; import { logger } from '../../../utils'; import { @@ -115,13 +114,8 @@ const downloadDataJob = StorageDownloadDataOutput > => { const { options: downloadDataOptions } = downloadDataInput; - const config = createStorageConfiguration(Amplify); - const { bucket, keyPrefix, s3Config, identityId } = - await resolveS3ConfigAndInput({ - config, - apiOptions: downloadDataOptions, - }); + await resolveS3ConfigAndInput(Amplify, downloadDataOptions); const { inputType, objectKey } = validateStorageOperationInput( downloadDataInput, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index 22bdc1bac6a..0e1fc01eadf 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -12,7 +12,6 @@ import { } from '../../types'; import { ResolvedS3Config } from '../../types/options'; import { - createStorageConfiguration, isInputWithPath, resolveS3ConfigAndInput, validateStorageOperationInput, @@ -41,10 +40,8 @@ const copyWithPath = async ( input: CopyWithPathInput, ): Promise => { const { source, destination } = input; - const config = createStorageConfiguration(amplify); - const { s3Config, bucket, identityId } = await resolveS3ConfigAndInput({ - config, - }); + const { s3Config, bucket, identityId } = + await resolveS3ConfigAndInput(amplify); assertValidationError(!!source.path, StorageValidationErrorCode.NoSourcePath); assertValidationError( @@ -90,19 +87,16 @@ export const copyWithKey = async ( !!destinationKey, StorageValidationErrorCode.NoDestinationKey, ); - const config = createStorageConfiguration(amplify); + const { s3Config, bucket, keyPrefix: sourceKeyPrefix, - } = await resolveS3ConfigAndInput({ - config, - apiOptions: input.source, - }); - const { keyPrefix: destinationKeyPrefix } = await resolveS3ConfigAndInput({ - config, - apiOptions: input.destination, - }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. + } = await resolveS3ConfigAndInput(amplify, input.source); + const { keyPrefix: destinationKeyPrefix } = await resolveS3ConfigAndInput( + amplify, + input.destination, + ); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. // TODO(ashwinkumar6) V6-logger: warn `You may copy files from another user if the source level is "protected", currently it's ${srcLevel}` const finalCopySource = `${bucket}/${sourceKeyPrefix}${sourceKey}`; diff --git a/packages/storage/src/providers/s3/apis/internal/getProperties.ts b/packages/storage/src/providers/s3/apis/internal/getProperties.ts index f06f6bcab69..915f02db495 100644 --- a/packages/storage/src/providers/s3/apis/internal/getProperties.ts +++ b/packages/storage/src/providers/s3/apis/internal/getProperties.ts @@ -11,7 +11,6 @@ import { GetPropertiesWithPathOutput, } from '../../types'; import { - createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; @@ -26,12 +25,8 @@ export const getProperties = async ( action?: StorageAction, ): Promise => { const { options: getPropertiesOptions } = input; - const config = createStorageConfiguration(amplify); const { s3Config, bucket, keyPrefix, identityId } = - await resolveS3ConfigAndInput({ - config, - apiOptions: getPropertiesOptions, - }); + await resolveS3ConfigAndInput(amplify, getPropertiesOptions); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/getUrl.ts b/packages/storage/src/providers/s3/apis/internal/getUrl.ts index 755a2028e4c..e8440ce80eb 100644 --- a/packages/storage/src/providers/s3/apis/internal/getUrl.ts +++ b/packages/storage/src/providers/s3/apis/internal/getUrl.ts @@ -13,7 +13,6 @@ import { import { StorageValidationErrorCode } from '../../../../errors/types/validation'; import { getPresignedGetObjectUrl } from '../../utils/client/s3data'; import { - createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; @@ -31,12 +30,8 @@ export const getUrl = async ( input: GetUrlInput | GetUrlWithPathInput, ): Promise => { const { options: getUrlOptions } = input; - const config = createStorageConfiguration(amplify); const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput({ - config, - apiOptions: getUrlOptions, - }); + await resolveS3ConfigAndInput(amplify, getUrlOptions); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index 0da1742aac3..6f074858738 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -17,7 +17,6 @@ import { ListPaginateWithPathOutput, } from '../../types'; import { - createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInputWithPrefix, } from '../../utils'; @@ -54,17 +53,12 @@ export const list = async ( | ListPaginateWithPathOutput > => { const { options = {} } = input; - - const config = createStorageConfiguration(amplify); const { s3Config, bucket, keyPrefix: generatedPrefix, identityId, - } = await resolveS3ConfigAndInput({ - config, - apiOptions: options, - }); + } = await resolveS3ConfigAndInput(amplify, options); const { inputType, objectKey } = validateStorageOperationInputWithPrefix( input, diff --git a/packages/storage/src/providers/s3/apis/internal/remove.ts b/packages/storage/src/providers/s3/apis/internal/remove.ts index 5a6add5ce67..e2a9377f39e 100644 --- a/packages/storage/src/providers/s3/apis/internal/remove.ts +++ b/packages/storage/src/providers/s3/apis/internal/remove.ts @@ -11,7 +11,6 @@ import { RemoveWithPathOutput, } from '../../types'; import { - createStorageConfiguration, resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; @@ -25,12 +24,8 @@ export const remove = async ( input: RemoveInput | RemoveWithPathInput, ): Promise => { const { options = {} } = input ?? {}; - const config = createStorageConfiguration(amplify); const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput({ - config, - apiOptions: options, - }); + await resolveS3ConfigAndInput(amplify, options); const { inputType, objectKey } = validateStorageOperationInput( input, diff --git a/packages/storage/src/providers/s3/apis/internal/types/index.ts b/packages/storage/src/providers/s3/apis/internal/types/index.ts deleted file mode 100644 index fb20b5da08d..00000000000 --- a/packages/storage/src/providers/s3/apis/internal/types/index.ts +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { LibraryOptions, StorageConfig } from '@aws-amplify/core'; -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; - -/** - * Internal S3 service options. - * - * @internal - */ -type S3ServiceOptions = StorageConfig['S3']; - -/** - * Internal S3 library options. - * - * @internal - */ -type S3LibraryOptions = NonNullable['S3']; - -/** - * S3 storage config input - * - * @internal - */ -export interface S3InternalConfig { - serviceOptions: S3ServiceOptions; - libraryOptions: S3LibraryOptions; - credentialsProvider(): Promise; - identityIdProvider(): Promise; -} diff --git a/packages/storage/src/providers/s3/apis/internal/uploadData.ts b/packages/storage/src/providers/s3/apis/internal/uploadData.ts deleted file mode 100644 index 5c616b4b7a7..00000000000 --- a/packages/storage/src/providers/s3/apis/internal/uploadData.ts +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { UploadDataInput, UploadDataWithPathInput } from '../../types'; -import { createUploadTask } from '../../utils'; -import { assertValidationError } from '../../../../errors/utils/assertValidationError'; -import { StorageValidationErrorCode } from '../../../../errors/types/validation'; -import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../utils/constants'; -import { byteLength } from '../uploadData/byteLength'; -import { putObjectJob } from '../uploadData/putObjectJob'; -import { getMultipartUploadHandlers } from '../uploadData/multipart'; - -import { S3InternalConfig } from './types'; - -export function internalUploadData( - config: S3InternalConfig, - input: UploadDataInput | UploadDataWithPathInput, -) { - const { data } = input; - - const dataByteLength = byteLength(data); - assertValidationError( - dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE, - StorageValidationErrorCode.ObjectIsTooLarge, - ); - - if (dataByteLength && dataByteLength <= DEFAULT_PART_SIZE) { - // Single part upload - const abortController = new AbortController(); - - return createUploadTask({ - isMultipartUpload: false, - job: putObjectJob({ - config, - input, - abortSignal: abortController.signal, - totalLength: dataByteLength, - }), - onCancel: (message?: string) => { - abortController.abort(message); - }, - }); - } else { - // Multipart upload - const { multipartUploadJob, onPause, onResume, onCancel } = - getMultipartUploadHandlers({ config, input, size: dataByteLength }); - - return createUploadTask({ - isMultipartUpload: true, - job: multipartUploadJob, - onCancel: (message?: string) => { - onCancel(message); - }, - onPause, - onResume, - }); - } -} diff --git a/packages/storage/src/providers/s3/apis/uploadData/index.ts b/packages/storage/src/providers/s3/apis/uploadData/index.ts index f32b90425dc..8669309ec53 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/index.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/index.ts @@ -1,16 +1,20 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { Amplify } from '@aws-amplify/core'; - import { UploadDataInput, UploadDataOutput, UploadDataWithPathInput, UploadDataWithPathOutput, } from '../../types'; -import { internalUploadData } from '../internal/uploadData'; -import { createStorageConfiguration } from '../../utils/config'; +import { createUploadTask } from '../../utils'; +import { assertValidationError } from '../../../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../../../errors/types/validation'; +import { DEFAULT_PART_SIZE, MAX_OBJECT_SIZE } from '../../utils/constants'; + +import { byteLength } from './byteLength'; +import { putObjectJob } from './putObjectJob'; +import { getMultipartUploadHandlers } from './multipart'; /** * Upload data to the specified S3 object path. By default uses single PUT operation to upload if the payload is less than 5MB. @@ -123,7 +127,38 @@ export function uploadData( export function uploadData(input: UploadDataInput): UploadDataOutput; export function uploadData(input: UploadDataInput | UploadDataWithPathInput) { - const config = createStorageConfiguration(Amplify); + const { data } = input; + + const dataByteLength = byteLength(data); + assertValidationError( + dataByteLength === undefined || dataByteLength <= MAX_OBJECT_SIZE, + StorageValidationErrorCode.ObjectIsTooLarge, + ); + + if (dataByteLength && dataByteLength <= DEFAULT_PART_SIZE) { + // Single part upload + const abortController = new AbortController(); + + return createUploadTask({ + isMultipartUpload: false, + job: putObjectJob(input, abortController.signal, dataByteLength), + onCancel: (message?: string) => { + abortController.abort(message); + }, + }); + } else { + // Multipart upload + const { multipartUploadJob, onPause, onResume, onCancel } = + getMultipartUploadHandlers(input, dataByteLength); - return internalUploadData(config, input); + return createUploadTask({ + isMultipartUpload: true, + job: multipartUploadJob, + onCancel: (message?: string) => { + onCancel(message); + }, + onPause, + onResume, + }); + } } diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index e84de95619b..886a769648b 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -29,7 +29,6 @@ import { } from '../../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../../utils/userAgent'; import { logger } from '../../../../../utils'; -import { S3InternalConfig } from '../../internal/types'; import { uploadPartExecutor } from './uploadPartExecutor'; import { getUploadsCacheKey, removeCachedUpload } from './uploadCache'; @@ -43,17 +42,10 @@ import { getDataChunker } from './getDataChunker'; * * @internal */ - -interface GetMultipartUploadHandlersProps { - config: S3InternalConfig; - input: UploadDataInput | UploadDataWithPathInput; - size?: number; -} -export const getMultipartUploadHandlers = ({ - config, - input, - size, -}: GetMultipartUploadHandlersProps) => { +export const getMultipartUploadHandlers = ( + uploadDataInput: UploadDataInput | UploadDataWithPathInput, + size?: number, +) => { let resolveCallback: | ((value: ItemWithKey | ItemWithPath) => void) | undefined; @@ -78,11 +70,11 @@ export const getMultipartUploadHandlers = ({ let isAbortSignalFromPause = false; const startUpload = async (): Promise => { - const { options: uploadDataOptions, data } = input; - const resolvedS3Options = await resolveS3ConfigAndInput({ - config, - apiOptions: uploadDataOptions, - }); + const { options: uploadDataOptions, data } = uploadDataInput; + const resolvedS3Options = await resolveS3ConfigAndInput( + Amplify, + uploadDataOptions, + ); abortController = new AbortController(); isAbortSignalFromPause = false; @@ -91,7 +83,7 @@ export const getMultipartUploadHandlers = ({ resolvedIdentityId = resolvedS3Options.identityId; const { inputType, objectKey } = validateStorageOperationInput( - input, + uploadDataInput, resolvedIdentityId, ); diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index 4da8bf328b5..76f9ebf5638 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -1,6 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { Amplify } from '@aws-amplify/core'; import { StorageAction } from '@aws-amplify/core/internals/utils'; import { UploadDataInput, UploadDataWithPathInput } from '../../types'; @@ -13,14 +14,6 @@ import { ItemWithKey, ItemWithPath } from '../../types/outputs'; import { putObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; -import { S3InternalConfig } from '../internal/types'; - -interface PutObjectJobProps { - config: S3InternalConfig; - input: UploadDataInput | UploadDataWithPathInput; - abortSignal: AbortSignal; - totalLength?: number; -} /** * Get a function the returns a promise to call putObject API to S3. @@ -28,17 +21,17 @@ interface PutObjectJobProps { * @internal */ export const putObjectJob = - ({ config, input, abortSignal, totalLength }: PutObjectJobProps) => + ( + uploadDataInput: UploadDataInput | UploadDataWithPathInput, + abortSignal: AbortSignal, + totalLength?: number, + ) => async (): Promise => { - const { options: uploadDataOptions, data } = input; - + const { options: uploadDataOptions, data } = uploadDataInput; const { bucket, keyPrefix, s3Config, isObjectLockEnabled, identityId } = - await resolveS3ConfigAndInput({ - config, - apiOptions: uploadDataOptions, - }); + await resolveS3ConfigAndInput(Amplify, uploadDataOptions); const { inputType, objectKey } = validateStorageOperationInput( - input, + uploadDataInput, identityId, ); diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 9a908890352..633366a4628 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -213,14 +213,3 @@ export interface ResolvedS3Config forcePathStyle?: boolean; useAccelerateEndpoint?: boolean; } - -/** - * Internal S3 API options. - * - * @internal - */ -export interface S3ApiOptions { - accessLevel?: StorageAccessLevel; - targetIdentityId?: string; - useAccelerateEndpoint?: boolean; -} diff --git a/packages/storage/src/providers/s3/utils/config.ts b/packages/storage/src/providers/s3/utils/config.ts deleted file mode 100644 index 49258d0e04d..00000000000 --- a/packages/storage/src/providers/s3/utils/config.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { AmplifyClassV6 } from '@aws-amplify/core'; - -import { StorageValidationErrorCode } from '../../../errors/types/validation'; -import { assertValidationError } from '../../../errors/utils/assertValidationError'; -import { S3InternalConfig } from '../apis/internal/types'; - -const createDefaultCredentialsProvider = (amplify: AmplifyClassV6) => { - /** - * A credentials provider function instead of a static credentials object is - * used because the long-running tasks like multipart upload may span over the - * credentials expiry. Auth.fetchAuthSession() automatically refreshes the - * credentials if they are expired. - */ - return async () => { - const { credentials } = await amplify.Auth.fetchAuthSession(); - assertValidationError( - !!credentials, - StorageValidationErrorCode.NoCredentials, - ); - - return credentials; - }; -}; - -const createDefaultIdentityIdProvider = (amplify: AmplifyClassV6) => { - return async () => { - const { identityId } = await amplify.Auth.fetchAuthSession(); - assertValidationError( - !!identityId, - StorageValidationErrorCode.NoIdentityId, - ); - - return identityId; - }; -}; - -/** - * It will return a Storage configuration used by lower level utils and APIs. - * - * @internal - */ -export const createStorageConfiguration = ( - amplify: AmplifyClassV6, -): S3InternalConfig => { - const libraryOptions = amplify.libraryOptions?.Storage?.S3 ?? {}; - const serviceOptions = amplify.getConfig()?.Storage?.S3 ?? {}; - const credentialsProvider = createDefaultCredentialsProvider(amplify); - const identityIdProvider = createDefaultIdentityIdProvider(amplify); - - return { - libraryOptions, - serviceOptions, - credentialsProvider, - identityIdProvider, - }; -}; diff --git a/packages/storage/src/providers/s3/utils/index.ts b/packages/storage/src/providers/s3/utils/index.ts index 1f43bb3f5d9..cd6b9753019 100644 --- a/packages/storage/src/providers/s3/utils/index.ts +++ b/packages/storage/src/providers/s3/utils/index.ts @@ -7,4 +7,3 @@ export { createDownloadTask, createUploadTask } from './transferTask'; export { validateStorageOperationInput } from './validateStorageOperationInput'; export { validateStorageOperationInputWithPrefix } from './validateStorageOperationInputWithPrefix'; export { isInputWithPath } from './isInputWithPath'; -export { createStorageConfiguration } from './config'; diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index ece08ea9223..ae7a185c93c 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -1,14 +1,21 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { AmplifyClassV6, StorageAccessLevel } from '@aws-amplify/core'; + import { assertValidationError } from '../../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { resolvePrefix as defaultPrefixResolver } from '../../../utils/resolvePrefix'; -import { ResolvedS3Config, S3ApiOptions } from '../types/options'; -import { S3InternalConfig } from '../apis/internal/types'; +import { ResolvedS3Config } from '../types/options'; import { DEFAULT_ACCESS_LEVEL, LOCAL_TESTING_S3_ENDPOINT } from './constants'; +interface S3ApiOptions { + accessLevel?: StorageAccessLevel; + targetIdentityId?: string; + useAccelerateEndpoint?: boolean; +} + interface ResolvedS3ConfigAndInput { s3Config: ResolvedS3Config; bucket: string; @@ -17,10 +24,6 @@ interface ResolvedS3ConfigAndInput { identityId?: string; } -interface ResolveS3ConfigAndInputParams { - config: S3InternalConfig; - apiOptions?: S3ApiOptions; -} /** * resolve the common input options for S3 API handlers from Amplify configuration and library options. * @@ -32,26 +35,44 @@ interface ResolveS3ConfigAndInputParams { * * @internal */ -export const resolveS3ConfigAndInput = async ({ - config, - apiOptions, -}: ResolveS3ConfigAndInputParams): Promise => { - const { - credentialsProvider, - serviceOptions, - libraryOptions, - identityIdProvider, - } = config; +export const resolveS3ConfigAndInput = async ( + amplify: AmplifyClassV6, + apiOptions?: S3ApiOptions, +): Promise => { + /** + * IdentityId is always cached in memory so we can safely make calls here. It + * should be stable even for unauthenticated users, regardless of credentials. + */ + const { identityId } = await amplify.Auth.fetchAuthSession(); + assertValidationError(!!identityId, StorageValidationErrorCode.NoIdentityId); + + /** + * A credentials provider function instead of a static credentials object is + * used because the long-running tasks like multipart upload may span over the + * credentials expiry. Auth.fetchAuthSession() automatically refreshes the + * credentials if they are expired. + */ + const credentialsProvider = async () => { + const { credentials } = await amplify.Auth.fetchAuthSession(); + assertValidationError( + !!credentials, + StorageValidationErrorCode.NoCredentials, + ); + + return credentials; + }; + const { bucket, region, dangerouslyConnectToHttpEndpointForTesting } = - serviceOptions ?? {}; + amplify.getConfig()?.Storage?.S3 ?? {}; assertValidationError(!!bucket, StorageValidationErrorCode.NoBucket); assertValidationError(!!region, StorageValidationErrorCode.NoRegion); - const identityId = await identityIdProvider(); + const { defaultAccessLevel, prefixResolver = defaultPrefixResolver, isObjectLockEnabled, - } = libraryOptions ?? {}; + } = amplify.libraryOptions?.Storage?.S3 ?? {}; + const keyPrefix = await prefixResolver({ accessLevel: apiOptions?.accessLevel ?? defaultAccessLevel ?? DEFAULT_ACCESS_LEVEL, @@ -76,7 +97,7 @@ export const resolveS3ConfigAndInput = async ({ }, bucket, keyPrefix, - isObjectLockEnabled, identityId, + isObjectLockEnabled, }; }; From 06c093b3efc6d847fb13c8bde5943aeeea8029da Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Tue, 16 Jul 2024 12:45:02 -0700 Subject: [PATCH 14/40] feat(storage): simplify the location cred provider option input (#13601) Remove the un-essetnial validation of per-API's location credentials provider input scope and permission for now. --- .../locationCredentialsStore/create.test.ts | 61 +---- .../validators.test.ts | 208 ------------------ .../storage/src/errors/types/validation.ts | 22 +- .../storage/src/providers/s3/types/options.ts | 17 +- .../locationCredentialsStore/create.ts | 63 +----- .../locationCredentialsStore/store.ts | 7 +- .../locationCredentialsStore/validators.ts | 86 -------- packages/storage/src/storageBrowser/types.ts | 10 +- 8 files changed, 21 insertions(+), 453 deletions(-) delete mode 100644 packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts delete mode 100644 packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts index fa1ba483ada..6405cdb5487 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts @@ -8,7 +8,6 @@ import { getValue, removeStore, } from '../../../src/storageBrowser/locationCredentialsStore/registry'; -import { validateCredentialsProviderLocation } from '../../../src/storageBrowser/locationCredentialsStore/validators'; import { LocationCredentialsStore } from '../../../src/storageBrowser/types'; import { StorageValidationErrorCode, @@ -16,7 +15,6 @@ import { } from '../../../src/errors/types/validation'; jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); -jest.mock('../../../src/storageBrowser/locationCredentialsStore/validators'); const mockedCredentials = 'MOCK_CREDS' as any as AWSCredentials; @@ -53,10 +51,7 @@ describe('createLocationCredentialsStore', () => { scope: 's3://bucket/path/*', permission: 'READ', }); - const { credentials } = await locationCredentialsProvider({ - locations: [{ bucket: 'bucket', path: 'path/to/object' }], - permission: 'READ', - }); + const { credentials } = await locationCredentialsProvider(); expect(credentials).toEqual(mockedCredentials); expect(getValue).toHaveBeenCalledWith( expect.objectContaining({ @@ -69,55 +64,6 @@ describe('createLocationCredentialsStore', () => { ); }); - it('should validate credentials location with resolved common scope', async () => { - expect.assertions(1); - jest - .mocked(getValue) - .mockResolvedValue({ credentials: mockedCredentials }); - const locationCredentialsProvider = store.getProvider({ - scope: 's3://bucket/path/*', - permission: 'READWRITE', - }); - await locationCredentialsProvider({ - locations: [ - { bucket: 'bucket', path: 'path/to/object' }, - { bucket: 'bucket', path: 'path/to/object2' }, - { bucket: 'bucket', path: 'path/folder' }, - ], - permission: 'READ', - }); - expect(validateCredentialsProviderLocation).toHaveBeenCalledWith( - { bucket: 'bucket', path: 'path/', permission: 'READ' }, - { bucket: 'bucket', path: 'path/*', permission: 'READWRITE' }, - ); - }); - - it('should throw if action requires cross-bucket permission', async () => { - expect.assertions(1); - jest - .mocked(getValue) - .mockResolvedValue({ credentials: mockedCredentials }); - const locationCredentialsProvider = store.getProvider({ - scope: 's3://bucket/path/*', - permission: 'READWRITE', - }); - try { - await locationCredentialsProvider({ - locations: [ - { bucket: 'bucket-1', path: 'path/to/object' }, - { bucket: 'bucket-2', path: 'path/to/object2' }, - ], - permission: 'READ', - }); - } catch (e: any) { - expect(e.message).toEqual( - validationErrorMap[ - StorageValidationErrorCode.LocationCredentialsCrossBucket - ].message, - ); - } - }); - it.each(['invalid-s3-uri', 's3://', 's3:///'])( 'should throw if location credentials provider scope is not a valid S3 URI "%s"', async invalidScope => { @@ -130,10 +76,7 @@ describe('createLocationCredentialsStore', () => { permission: 'READWRITE', }); try { - await locationCredentialsProvider({ - locations: [{ bucket: 'XXXXXXXX', path: 'path/to/object' }], - permission: 'READ', - }); + await locationCredentialsProvider(); } catch (e: any) { expect(e.message).toEqual( validationErrorMap[StorageValidationErrorCode.InvalidS3Uri] diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts deleted file mode 100644 index 774b5663d34..00000000000 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/validators.test.ts +++ /dev/null @@ -1,208 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { validateCredentialsProviderLocation } from '../../../src/storageBrowser/locationCredentialsStore/validators'; -import { - StorageValidationErrorCode, - validationErrorMap, -} from '../../../src/errors/types/validation'; - -jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); - -const mockBucket = 'MOCK_BUCKET'; - -describe('validateCredentialsProviderLocation', () => { - it('should NOT throw if action path matches credentials path prefix', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READ', - }, - { - bucket: mockBucket, - path: 'path/to/*', - permission: 'READ', - }, - ); - }).not.toThrow(); - }); - - it('should throw if action path does not path credentials path prefix', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READ', - }, - { - bucket: mockBucket, - path: 'path/to/other/*', - permission: 'READ', - }, - ); - }).toThrow( - validationErrorMap[ - StorageValidationErrorCode.LocationCredentialsPathMismatch - ].message, - ); - }); - - it('should NOT throw if action path matches credentials object path', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READ', - }, - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READ', - }, - ); - }).not.toThrow(); - }); - - it('should throw if action path does not match credentials object path', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READ', - }, - { - bucket: mockBucket, - path: 'path/to/object2', - permission: 'READ', - }, - ); - }).toThrow( - validationErrorMap[ - StorageValidationErrorCode.LocationCredentialsPathMismatch - ].message, - ); - }); - - it('should throw if action bucket and credentials bucket does not match', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: 'bucket-1', - path: 'path/to/object', - permission: 'READ', - }, - { - bucket: 'bucket-2', - path: 'path/to/object', - permission: 'READ', - }, - ); - }).toThrow( - validationErrorMap[ - StorageValidationErrorCode.LocationCredentialsBucketMismatch - ].message, - ); - }); - - it('should not throw if READ action permission matches READWRITE credentials permission', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READ', - }, - { - bucket: mockBucket, - path: 'path/to/*', - permission: 'READWRITE', - }, - ); - }).not.toThrow(); - }); - - it('should not throw if WRITE action permission matches READWRITE credentials permission', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'WRITE', - }, - { - bucket: mockBucket, - path: 'path/to/*', - permission: 'READWRITE', - }, - ); - }).not.toThrow(); - }); - - it('should throw if READ action permission does not match WRITE credentials permission', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READ', - }, - { - bucket: mockBucket, - path: 'path/to/*', - permission: 'WRITE', - }, - ); - }).toThrow( - validationErrorMap[ - StorageValidationErrorCode.LocationCredentialsPermissionMismatch - ].message, - ); - }); - - it('should throw if WRITE action permission does not match READ credentials permission', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'WRITE', - }, - { - bucket: mockBucket, - path: 'path/to/*', - permission: 'READ', - }, - ); - }).toThrow( - validationErrorMap[ - StorageValidationErrorCode.LocationCredentialsPermissionMismatch - ].message, - ); - }); - - it('should throw if READWRITE action permission does not match READ credentials permission', () => { - expect(() => { - validateCredentialsProviderLocation( - { - bucket: mockBucket, - path: 'path/to/object', - permission: 'READWRITE', - }, - { - bucket: mockBucket, - path: 'path/to/*', - permission: 'READ', - }, - ); - }).toThrow( - validationErrorMap[ - StorageValidationErrorCode.LocationCredentialsPermissionMismatch - ].message, - ); - }); -}); diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index 281272863de..deefdb1700b 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -21,17 +21,9 @@ export enum StorageValidationErrorCode { UrlExpirationMaxLimitExceed = 'UrlExpirationMaxLimitExceed', InvalidLocationCredentialsCacheSize = 'InvalidLocationCredentialsCacheSize', LocationCredentialsStoreDestroyed = 'LocationCredentialsStoreDestroyed', - LocationCredentialsBucketMismatch = 'LocationCredentialsBucketMismatch', - LocationCredentialsCrossBucket = 'LocationCredentialsCrossBucket', - LocationCredentialsPathMismatch = 'LocationCredentialsPathMismatch', - LocationCredentialsPermissionMismatch = 'LocationCredentialsPermissionMismatch', InvalidS3Uri = 'InvalidS3Uri', } -// Common error message strings to save some bytes -const LOCATION_SPECIFIC_CREDENTIALS = 'Location-specific credentials'; -const DOES_NOT_MATCH = 'does not match that required for the API call'; - export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.NoCredentials]: { message: 'Credentials should not be empty.', @@ -85,21 +77,9 @@ export const validationErrorMap: AmplifyErrorMap = { message: 'locationCredentialsCacheSize must be a positive integer.', }, [StorageValidationErrorCode.LocationCredentialsStoreDestroyed]: { - message: `${LOCATION_SPECIFIC_CREDENTIALS} store has been destroyed.`, + message: `Location-specific credentials store has been destroyed.`, }, [StorageValidationErrorCode.InvalidS3Uri]: { message: 'Invalid S3 URI.', }, - [StorageValidationErrorCode.LocationCredentialsCrossBucket]: { - message: `${LOCATION_SPECIFIC_CREDENTIALS} cannot be used across buckets.`, - }, - [StorageValidationErrorCode.LocationCredentialsBucketMismatch]: { - message: `${LOCATION_SPECIFIC_CREDENTIALS} bucket ${DOES_NOT_MATCH}.`, - }, - [StorageValidationErrorCode.LocationCredentialsPathMismatch]: { - message: `${LOCATION_SPECIFIC_CREDENTIALS} path ${DOES_NOT_MATCH}.`, - }, - [StorageValidationErrorCode.LocationCredentialsPermissionMismatch]: { - message: `${LOCATION_SPECIFIC_CREDENTIALS} permission ${DOES_NOT_MATCH}.`, - }, }; diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 633366a4628..e37bcca8bbc 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -14,23 +14,8 @@ import { /** * @internal */ -export type Permission = 'READ' | 'READWRITE' | 'WRITE'; - -/** - * @internal - */ -export interface BucketLocation { - bucket: string; - path: string; -} - -/** - * @internal - */ -export type LocationCredentialsProvider = (options: { +export type LocationCredentialsProvider = (options?: { forceRefresh?: boolean; - locations: BucketLocation[]; - permission: Permission; }) => Promise<{ credentials: AWSCredentials }>; interface CommonOptions { diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts index a54ae32a354..ce4e9126612 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts @@ -8,13 +8,9 @@ import { } from '../types'; import { StorageValidationErrorCode } from '../../errors/types/validation'; import { assertValidationError } from '../../errors/utils/assertValidationError'; -import { - BucketLocation, - LocationCredentialsProvider, -} from '../../providers/s3/types/options'; +import { LocationCredentialsProvider } from '../../providers/s3/types/options'; import { createStore, getValue, removeStore } from './registry'; -import { validateCredentialsProviderLocation } from './validators'; export const createLocationCredentialsStore = (input: { handler: GetLocationCredentials; @@ -24,23 +20,11 @@ export const createLocationCredentialsStore = (input: { const store = { getProvider(providerLocation: CredentialsLocation) { const locationCredentialsProvider = async ({ - permission, - locations, forceRefresh = false, - }: Parameters[0]) => { - const actionBucketLocation = resolveCommonBucketLocation(locations); - const providerBucketLocation = parseS3Uri(providerLocation.scope); - validateCredentialsProviderLocation( - { - ...actionBucketLocation, - permission, - }, - { - ...providerBucketLocation, - permission: providerLocation.permission, - }, - ); + }: Parameters[0] = {}) => { + validateS3Uri(providerLocation.scope); + // TODO(@AllanZhengYP): validate the action bucket and paths matches provider scope. return getValue({ storeSymbol, location: { ...providerLocation }, @@ -61,45 +45,10 @@ export const createLocationCredentialsStore = (input: { type S3Uri = string; -const parseS3Uri = (uri: S3Uri): BucketLocation => { - const s3UrlSchemaRegex = /^s3:\/\//; - // TODO(@AllanZhengYP): Provide more info to error message: url +const validateS3Uri = (uri: S3Uri): void => { + const s3UrlSchemaRegex = /^s3:\/\/[^/]+/; assertValidationError( s3UrlSchemaRegex.test(uri), StorageValidationErrorCode.InvalidS3Uri, ); - const [bucket, ...pathParts] = uri.replace(s3UrlSchemaRegex, '').split('/'); - assertValidationError(!!bucket, StorageValidationErrorCode.InvalidS3Uri); - const path = pathParts.join('/'); - - return { - bucket, - path, - }; -}; - -/** - * Given a list of bucket and path combinations, verify they have the same - * bucket and resolves the longest common prefix for multiple given paths. - */ -const resolveCommonBucketLocation = ( - locations: BucketLocation[], -): BucketLocation => { - let { bucket: commonBucket, path: commonPath } = locations[0]; - - for (const location of locations) { - const { bucket, path } = location; - assertValidationError( - bucket === commonBucket, - StorageValidationErrorCode.LocationCredentialsCrossBucket, - ); - while (commonPath !== '' && !path.startsWith(commonPath)) { - commonPath = commonPath.slice(0, -1); - } - } - - return { - bucket: commonBucket, - path: commonPath, - }; }; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts index 04e800365c7..63b88008b71 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts @@ -5,8 +5,11 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { Permission } from '../../providers/s3/types/options'; -import { CredentialsLocation, GetLocationCredentials } from '../types'; +import { + CredentialsLocation, + GetLocationCredentials, + Permission, +} from '../types'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts deleted file mode 100644 index 3409a899fdf..00000000000 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/validators.ts +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -import { StorageValidationErrorCode } from '../../errors/types/validation'; -import { assertValidationError } from '../../errors/utils/assertValidationError'; -import { BucketLocation, Permission } from '../../providers/s3/types/options'; - -interface CredentialsBucketLocation extends BucketLocation { - permission: Permission; -} - -/** - * @internal - */ -export const validateCredentialsProviderLocation = ( - actionLocation: CredentialsBucketLocation, - providerLocation: CredentialsBucketLocation, -): void => { - validateLocationBucket({ - actionBucket: actionLocation.bucket, - providerBucket: providerLocation.bucket, - }); - validateLocationPath({ - actionPath: actionLocation.path, - providerPath: providerLocation.path, - }); - validateLocationPermission({ - actionPermission: actionLocation.permission, - providerPermission: providerLocation.permission, - }); -}; - -const validateLocationBucket = (input: { - actionBucket: string; - providerBucket?: string; -}): void => { - const { actionBucket, providerBucket } = input; - if (!providerBucket) { - return; - } - assertValidationError( - actionBucket === providerBucket, - StorageValidationErrorCode.LocationCredentialsBucketMismatch, - ); -}; - -const validateLocationPath = (input: { - actionPath: string; - providerPath?: string; -}): void => { - const { actionPath, providerPath } = input; - if (!providerPath) { - return; - } - if (providerPath.endsWith('*')) { - // Verify if the action path has prefix required by the provider; - const providerPathPrefix = providerPath.replace(/\*$/, ''); - assertValidationError( - actionPath.startsWith(providerPathPrefix), - StorageValidationErrorCode.LocationCredentialsPathMismatch, - ); - } else { - // If provider path is scoped to an object, verify if the action path points to the same object. - // TODO(@AllanZhengYP) Provider more info in error message: actionPath, providerPath. - assertValidationError( - actionPath === providerPath, - StorageValidationErrorCode.LocationCredentialsPathMismatch, - ); - } -}; - -const validateLocationPermission = (input: { - actionPermission: Permission; - providerPermission?: Permission; -}) => { - const { actionPermission, providerPermission } = input; - if (!providerPermission) { - return; - } - // TODO(@AllanZhengYP) Provide more info in error message: `API needs permission ${actionPermission}, but provided - // location credentials provider with permission ${providerPermission}.` - assertValidationError( - providerPermission.includes(actionPermission), - StorageValidationErrorCode.LocationCredentialsPermissionMismatch, - ); -}; diff --git a/packages/storage/src/storageBrowser/types.ts b/packages/storage/src/storageBrowser/types.ts index 3f184dcc313..68fe71aadaf 100644 --- a/packages/storage/src/storageBrowser/types.ts +++ b/packages/storage/src/storageBrowser/types.ts @@ -3,10 +3,12 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { - LocationCredentialsProvider, - Permission, -} from '../providers/s3/types/options'; +import { LocationCredentialsProvider } from '../providers/s3/types/options'; + +/** + * @internal + */ +export type Permission = 'READ' | 'READWRITE' | 'WRITE'; /** * @internal From e44d290313a4b2f7443d55380282193499a7abde Mon Sep 17 00:00:00 2001 From: Jim Blanchard Date: Tue, 16 Jul 2024 20:05:09 -0500 Subject: [PATCH 15/40] feat: Implement getLocationCredentials handler & integrate with adapter (#13600) --- packages/aws-amplify/package.json | 12 +- packages/core/src/Platform/types.ts | 1 + .../storageBrowser/apis/getDataAccess.test.ts | 116 ++++++++++++++++++ .../src/storageBrowser/apis/constants.ts | 4 + .../src/storageBrowser/apis/getDataAccess.ts | 66 ++++++++++ .../{ => apis}/listCallerAccessGrants.ts | 13 +- .../storage/src/storageBrowser/apis/types.ts | 33 +++++ .../createLocationCredentialsHandler.ts | 32 ++++- .../createManagedAuthConfigAdapter.ts | 1 + packages/storage/src/storageBrowser/types.ts | 29 ++++- 10 files changed, 284 insertions(+), 23 deletions(-) create mode 100644 packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts create mode 100644 packages/storage/src/storageBrowser/apis/constants.ts create mode 100644 packages/storage/src/storageBrowser/apis/getDataAccess.ts rename packages/storage/src/storageBrowser/{ => apis}/listCallerAccessGrants.ts (55%) create mode 100644 packages/storage/src/storageBrowser/apis/types.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index d4cfecc01d7..ee7960ade9a 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -293,7 +293,7 @@ "name": "[Analytics] record (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ record }", - "limit": "17.09 kB" + "limit": "17.11 kB" }, { "name": "[Analytics] record (Kinesis)", @@ -317,7 +317,7 @@ "name": "[Analytics] identifyUser (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ identifyUser }", - "limit": "15.59 kB" + "limit": "15.60 kB" }, { "name": "[Analytics] enable", @@ -383,7 +383,7 @@ "name": "[Auth] confirmSignIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignIn }", - "limit": "28.27 kB" + "limit": "28.28 kB" }, { "name": "[Auth] updateMFAPreference (Cognito)", @@ -401,7 +401,7 @@ "name": "[Auth] verifyTOTPSetup (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ verifyTOTPSetup }", - "limit": "12.6 kB" + "limit": "12.7 kB" }, { "name": "[Auth] updatePassword (Cognito)", @@ -449,13 +449,13 @@ "name": "[Auth] Basic Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signIn, signOut, fetchAuthSession, confirmSignIn }", - "limit": "30.06 kB" + "limit": "30.07 kB" }, { "name": "[Auth] OAuth Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect, signOut, fetchAuthSession }", - "limit": "21.47 kB" + "limit": "21.49 kB" }, { "name": "[Storage] copy (S3)", diff --git a/packages/core/src/Platform/types.ts b/packages/core/src/Platform/types.ts index 96ca1de77ec..aa49d1f06b6 100644 --- a/packages/core/src/Platform/types.ts +++ b/packages/core/src/Platform/types.ts @@ -120,6 +120,7 @@ export enum StorageAction { Remove = '5', GetProperties = '6', GetUrl = '7', + GetDataAccess = '8', } interface ActionMap { diff --git a/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts b/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts new file mode 100644 index 00000000000..0753e0ae334 --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts @@ -0,0 +1,116 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { getDataAccess } from '../../../src/storageBrowser/apis/getDataAccess'; +import { getDataAccess as getDataAccessClient } from '../../../src/providers/s3/utils/client/s3control'; +import { GetDataAccessInput } from '../../../src/storageBrowser/apis/types'; + +jest.mock('../../../src/providers/s3/utils/client/s3control'); + +const MOCK_ACCOUNT_ID = 'accountId'; +const MOCK_REGION = 'us-east-2'; +const MOCK_ACCESS_ID = 'accessId'; +const MOCK_SECRET_ACCESS_KEY = 'secretAccessKey'; +const MOCK_SESSION_TOKEN = 'sessionToken'; +const MOCK_EXPIRATION = '2013-09-17T18:07:53.000Z'; +const MOCK_EXPIRATION_DATE = new Date(MOCK_EXPIRATION); +const MOCK_SCOPE = 's3://mybucket/files/*'; +const MOCK_CREDENTIALS = { + credentials: { + accessKeyId: MOCK_ACCESS_ID, + secretAccessKey: MOCK_SECRET_ACCESS_KEY, + sessionToken: MOCK_SESSION_TOKEN, + expiration: MOCK_EXPIRATION_DATE, + }, +}; +const MOCK_ACCESS_CREDENTIALS = { + AccessKeyId: MOCK_ACCESS_ID, + SecretAccessKey: MOCK_SECRET_ACCESS_KEY, + SessionToken: MOCK_SESSION_TOKEN, + Expiration: MOCK_EXPIRATION_DATE, +}; +const MOCK_CREDENTIAL_PROVIDER = async () => MOCK_CREDENTIALS; + +const sharedGetDataAccessParams: GetDataAccessInput = { + accountId: MOCK_ACCOUNT_ID, + credentialsProvider: MOCK_CREDENTIAL_PROVIDER, + durationSeconds: 900, + permission: 'READWRITE', + region: MOCK_REGION, + scope: MOCK_SCOPE, +}; + +describe('getDataAccess', () => { + const getDataAccessClientMock = getDataAccessClient as jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + + getDataAccessClientMock.mockResolvedValue({ + Credentials: MOCK_ACCESS_CREDENTIALS, + MatchedGrantTarget: MOCK_SCOPE, + }); + }); + + it('should invoke the getDataAccess client correctly', async () => { + const result = await getDataAccess(sharedGetDataAccessParams); + + expect(getDataAccessClientMock).toHaveBeenCalledWith( + expect.objectContaining({ + credentials: MOCK_CREDENTIALS.credentials, + region: MOCK_REGION, + userAgentValue: expect.stringContaining('storage/8'), + }), + expect.objectContaining({ + AccountId: MOCK_ACCOUNT_ID, + Target: MOCK_SCOPE, + Permission: 'READWRITE', + TargetType: undefined, + DurationSeconds: 900, + }), + ); + + expect(result.credentials).toEqual(MOCK_CREDENTIALS.credentials); + expect(result.scope).toEqual(MOCK_SCOPE); + }); + + it('should throw an error if the service does not return credentials', async () => { + expect.assertions(1); + + getDataAccessClientMock.mockResolvedValue({ + Credentials: undefined, + MatchedGrantTarget: MOCK_SCOPE, + }); + + expect(getDataAccess(sharedGetDataAccessParams)).rejects.toThrow( + 'Service did not return credentials.', + ); + }); + + it('should set the correct target type when accessing an object', async () => { + const MOCK_OBJECT_SCOPE = 's3://mybucket/files/file.md'; + + getDataAccessClientMock.mockResolvedValue({ + Credentials: MOCK_ACCESS_CREDENTIALS, + MatchedGrantTarget: MOCK_OBJECT_SCOPE, + }); + + const result = await getDataAccess({ + ...sharedGetDataAccessParams, + scope: MOCK_OBJECT_SCOPE, + }); + + expect(getDataAccessClientMock).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + AccountId: MOCK_ACCOUNT_ID, + Target: MOCK_OBJECT_SCOPE, + Permission: 'READWRITE', + TargetType: 'Object', + DurationSeconds: 900, + }), + ); + + expect(result.scope).toEqual(MOCK_OBJECT_SCOPE); + }); +}); diff --git a/packages/storage/src/storageBrowser/apis/constants.ts b/packages/storage/src/storageBrowser/apis/constants.ts new file mode 100644 index 00000000000..e333ac5a5e2 --- /dev/null +++ b/packages/storage/src/storageBrowser/apis/constants.ts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const DEFAULT_CRED_TTL = 15 * 60; // 15 minutes diff --git a/packages/storage/src/storageBrowser/apis/getDataAccess.ts b/packages/storage/src/storageBrowser/apis/getDataAccess.ts new file mode 100644 index 00000000000..5e5bec23540 --- /dev/null +++ b/packages/storage/src/storageBrowser/apis/getDataAccess.ts @@ -0,0 +1,66 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + AmplifyErrorCode, + StorageAction, +} from '@aws-amplify/core/internals/utils'; + +import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; +import { getDataAccess as getDataAccessClient } from '../../providers/s3/utils/client/s3control'; +import { StorageError } from '../../errors/StorageError'; +import { logger } from '../../utils'; + +import { GetDataAccessInput, GetDataAccessOutput } from './types'; +import { DEFAULT_CRED_TTL } from './constants'; + +export const getDataAccess = async ( + input: GetDataAccessInput, +): Promise => { + const targetType = input.scope.endsWith('*') ? undefined : 'Object'; + const { credentials } = await input.credentialsProvider(); + + const result = await getDataAccessClient( + { + credentials, + region: input.region, + userAgentValue: getStorageUserAgentValue(StorageAction.GetDataAccess), + }, + { + AccountId: input.accountId, + Target: input.scope, + Permission: input.permission, + TargetType: targetType, + DurationSeconds: DEFAULT_CRED_TTL, + }, + ); + + const grantCredentials = result.Credentials; + + // Ensure that S3 returned credentials (this shouldn't happen) + if (!grantCredentials) { + throw new StorageError({ + name: AmplifyErrorCode.Unknown, + message: 'Service did not return credentials.', + }); + } else { + logger.debug(`Retrieved credentials for: ${result.MatchedGrantTarget}`); + } + + const { + AccessKeyId: accessKeyId, + SecretAccessKey: secretAccessKey, + SessionToken: sessionToken, + Expiration: expiration, + } = grantCredentials; + + return { + credentials: { + accessKeyId: accessKeyId!, + secretAccessKey: secretAccessKey!, + sessionToken, + expiration, + }, + scope: result.MatchedGrantTarget, + }; +}; diff --git a/packages/storage/src/storageBrowser/listCallerAccessGrants.ts b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts similarity index 55% rename from packages/storage/src/storageBrowser/listCallerAccessGrants.ts rename to packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts index c55b0ea75e8..dbe8b305b36 100644 --- a/packages/storage/src/storageBrowser/listCallerAccessGrants.ts +++ b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts @@ -1,15 +1,10 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { AccessGrant, CredentialsProvider, ListLocationsOutput } from './types'; - -export interface ListCallerAccessGrantsInput { - accountId: string; - credentialsProvider: CredentialsProvider; - region: string; -} - -export type ListCallerAccessGrantsOutput = ListLocationsOutput; +import { + ListCallerAccessGrantsInput, + ListCallerAccessGrantsOutput, +} from './types'; export const listCallerAccessGrants = ( // eslint-disable-next-line unused-imports/no-unused-vars diff --git a/packages/storage/src/storageBrowser/apis/types.ts b/packages/storage/src/storageBrowser/apis/types.ts new file mode 100644 index 00000000000..2928cfd1f38 --- /dev/null +++ b/packages/storage/src/storageBrowser/apis/types.ts @@ -0,0 +1,33 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + AccessGrant, + CredentialsProvider, + ListLocationsOutput, + LocationCredentials, + Permission, + PrefixType, + Privilege, +} from '../types'; + +export interface ListCallerAccessGrantsInput { + accountId: string; + credentialsProvider: CredentialsProvider; + region: string; +} + +export type ListCallerAccessGrantsOutput = ListLocationsOutput; + +export interface GetDataAccessInput { + accountId: string; + credentialsProvider: CredentialsProvider; + durationSeconds?: number; + permission: Permission; + prefixType?: PrefixType; + privilege?: Privilege; + region: string; + scope: string; +} + +export type GetDataAccessOutput = LocationCredentials; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts index 9cded212928..248e81882ac 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts @@ -1,7 +1,12 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { CredentialsProvider, GetLocationCredentials } from '../types'; +import { getDataAccess } from '../apis/getDataAccess'; +import { + CredentialsProvider, + GetLocationCredentials, + GetLocationCredentialsInput, +} from '../types'; interface CreateLocationCredentialsHandlerInput { accountId: string; @@ -10,9 +15,26 @@ interface CreateLocationCredentialsHandlerInput { } export const createLocationCredentialsHandler = ( - // eslint-disable-next-line unused-imports/no-unused-vars - input: CreateLocationCredentialsHandlerInput, + handlerInput: CreateLocationCredentialsHandlerInput, ): GetLocationCredentials => { - // TODO(@AllanZhengYP) - throw new Error('Not Implemented'); + const { accountId, region, credentialsProvider } = handlerInput; + + /** + * Retrieves credentials for the specified scope & permission. + * + * @param input - An object specifying the requested scope & permission. + * + * @returns A promise which will resolve with the requested credentials. + */ + return (input: GetLocationCredentialsInput) => { + const { scope, permission } = input; + + return getDataAccess({ + accountId, + credentialsProvider, + permission, + region, + scope, + }); + }; }; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts index 55d334b47c7..267fee96c21 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts @@ -38,6 +38,7 @@ export const createManagedAuthConfigAdapter = ({ accountId, region, }); + const getLocationCredentials = createLocationCredentialsHandler({ credentialsProvider, accountId, diff --git a/packages/storage/src/storageBrowser/types.ts b/packages/storage/src/storageBrowser/types.ts index 68fe71aadaf..c770b7472a3 100644 --- a/packages/storage/src/storageBrowser/types.ts +++ b/packages/storage/src/storageBrowser/types.ts @@ -22,7 +22,17 @@ export type CredentialsProvider = (options?: { */ export type LocationType = 'BUCKET' | 'PREFIX' | 'OBJECT'; -export interface CredentialsLocation { +/** + * @internal + */ +export type Privilege = 'Default' | 'Minimal'; + +/** + * @internal + */ +export type PrefixType = 'Object'; + +export interface LocationScope { /** * Scope of storage location. For S3 service, it's the S3 path of the data to * which the access is granted. It can be in following formats: @@ -32,6 +42,9 @@ export interface CredentialsLocation { * @example Object 's3:////' */ readonly scope: string; +} + +export interface CredentialsLocation extends LocationScope { /** * The type of access granted to your Storage data. Can be either of READ, * WRITE or READWRITE @@ -52,6 +65,13 @@ export interface LocationAccess extends CredentialsLocation { readonly type: LocationType; } +export interface LocationCredentials extends Partial { + /** + * AWS credentials which can be used to access the specified location. + */ + readonly credentials: AWSCredentials; +} + export interface AccessGrant extends LocationAccess { /** * The Amazon Resource Name (ARN) of an AWS IAM Identity Center application @@ -82,9 +102,12 @@ export type ListLocations = ( input?: ListLocationsInput, ) => Promise>; +export type GetLocationCredentialsInput = CredentialsLocation; +export type GetLocationCredentialsOutput = LocationCredentials; + export type GetLocationCredentials = ( - input: CredentialsLocation, -) => Promise<{ credentials: AWSCredentials }>; + input: GetLocationCredentialsInput, +) => Promise; export interface LocationCredentialsStore { /** From a8f8e6e747e40e3efc9ea4e08c549a79132b8c95 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Wed, 17 Jul 2024 15:24:06 -0700 Subject: [PATCH 16/40] feat(storage): implement listLocations API and creation handler (#13602) --- packages/aws-amplify/package.json | 28 ++--- packages/core/src/Platform/types.ts | 1 + packages/interactions/package.json | 6 +- .../apis/listCallerAccessGrants.test.ts | 116 ++++++++++++++++++ .../createListLocationsHandler.test.ts | 34 +++++ .../src/storageBrowser/apis/constants.ts | 1 + .../apis/listCallerAccessGrants.ts | 98 ++++++++++++++- .../storage/src/storageBrowser/apis/types.ts | 3 +- .../createListLocationsHandler.ts | 25 +++- 9 files changed, 285 insertions(+), 27 deletions(-) create mode 100644 packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts create mode 100644 packages/storage/__tests__/storageBrowser/managedAuthAdapter/createListLocationsHandler.test.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index ee7960ade9a..ccf5001d233 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -293,7 +293,7 @@ "name": "[Analytics] record (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ record }", - "limit": "17.11 kB" + "limit": "17.14 kB" }, { "name": "[Analytics] record (Kinesis)", @@ -311,13 +311,13 @@ "name": "[Analytics] record (Personalize)", "path": "./dist/esm/analytics/personalize/index.mjs", "import": "{ record }", - "limit": "49.50 kB" + "limit": "49.53 kB" }, { "name": "[Analytics] identifyUser (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ identifyUser }", - "limit": "15.60 kB" + "limit": "15.64 kB" }, { "name": "[Analytics] enable", @@ -353,13 +353,13 @@ "name": "[Auth] resetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resetPassword }", - "limit": "12.44 kB" + "limit": "12.48 kB" }, { "name": "[Auth] confirmResetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmResetPassword }", - "limit": "12.39 kB" + "limit": "12.42 kB" }, { "name": "[Auth] signIn (Cognito)", @@ -371,7 +371,7 @@ "name": "[Auth] resendSignUpCode (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resendSignUpCode }", - "limit": "12.40 kB" + "limit": "12.44 kB" }, { "name": "[Auth] confirmSignUp (Cognito)", @@ -383,19 +383,19 @@ "name": "[Auth] confirmSignIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignIn }", - "limit": "28.28 kB" + "limit": "28.32 kB" }, { "name": "[Auth] updateMFAPreference (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updateMFAPreference }", - "limit": "11.74 kB" + "limit": "11.78 kB" }, { "name": "[Auth] fetchMFAPreference (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ fetchMFAPreference }", - "limit": "11.78 kB" + "limit": "11.81 kB" }, { "name": "[Auth] verifyTOTPSetup (Cognito)", @@ -407,7 +407,7 @@ "name": "[Auth] updatePassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updatePassword }", - "limit": "12.63 kB" + "limit": "12.67 kB" }, { "name": "[Auth] setUpTOTP (Cognito)", @@ -431,7 +431,7 @@ "name": "[Auth] confirmUserAttribute (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmUserAttribute }", - "limit": "12.61 kB" + "limit": "12.64 kB" }, { "name": "[Auth] signInWithRedirect (Cognito)", @@ -443,19 +443,19 @@ "name": "[Auth] fetchUserAttributes (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ fetchUserAttributes }", - "limit": "11.69 kB" + "limit": "11.72 kB" }, { "name": "[Auth] Basic Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signIn, signOut, fetchAuthSession, confirmSignIn }", - "limit": "30.07 kB" + "limit": "30.11 kB" }, { "name": "[Auth] OAuth Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect, signOut, fetchAuthSession }", - "limit": "21.49 kB" + "limit": "21.52 kB" }, { "name": "[Storage] copy (S3)", diff --git a/packages/core/src/Platform/types.ts b/packages/core/src/Platform/types.ts index aa49d1f06b6..1d430706608 100644 --- a/packages/core/src/Platform/types.ts +++ b/packages/core/src/Platform/types.ts @@ -121,6 +121,7 @@ export enum StorageAction { GetProperties = '6', GetUrl = '7', GetDataAccess = '8', + ListCallerAccessGrants = '9', } interface ActionMap { diff --git a/packages/interactions/package.json b/packages/interactions/package.json index faebbd94fae..89f29d4b005 100644 --- a/packages/interactions/package.json +++ b/packages/interactions/package.json @@ -89,19 +89,19 @@ "name": "Interactions (default to Lex v2)", "path": "./dist/esm/index.mjs", "import": "{ Interactions }", - "limit": "52.52 kB" + "limit": "52.55 kB" }, { "name": "Interactions (Lex v2)", "path": "./dist/esm/lex-v2/index.mjs", "import": "{ Interactions }", - "limit": "52.52 kB" + "limit": "52.55 kB" }, { "name": "Interactions (Lex v1)", "path": "./dist/esm/lex-v1/index.mjs", "import": "{ Interactions }", - "limit": "47.33 kB" + "limit": "47.37 kB" } ] } diff --git a/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts new file mode 100644 index 00000000000..3e0051f7461 --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts @@ -0,0 +1,116 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import { listCallerAccessGrants } from '../../../src/storageBrowser/apis/listCallerAccessGrants'; +import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../../src/providers/s3/utils/client/s3control'; + +jest.mock('../../../src/providers/s3/utils/client/s3control'); + +const mockAccountId = '1234567890'; +const mockRegion = 'us-foo-2'; +const mockCredentialsProvider = jest.fn(); +const mockNextToken = '123'; +const mockPageSize = 123; + +describe('listCallerAccessGrants', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should invoke the listCallerAccessGrants client with expected parameters', async () => { + expect.assertions(1); + jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ + NextToken: undefined, + CallerAccessGrantsList: [], + $metadata: {} as any, + }); + await listCallerAccessGrants({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + nextToken: mockNextToken, + pageSize: mockPageSize, + }); + expect(listCallerAccessGrantsClient).toHaveBeenCalledWith( + expect.objectContaining({ + region: mockRegion, + credentials: expect.any(Function), + }), + expect.objectContaining({ + AccountId: mockAccountId, + NextToken: mockNextToken, + MaxResults: mockPageSize, + }), + ); + }); + + it('should set a default page size', async () => { + expect.assertions(1); + jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ + NextToken: undefined, + CallerAccessGrantsList: [], + $metadata: {} as any, + }); + await listCallerAccessGrants({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + }); + expect(listCallerAccessGrantsClient).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + MaxResults: 1000, + }), + ); + }); + + it('should set response location type correctly', async () => { + expect.assertions(2); + jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ + NextToken: undefined, + CallerAccessGrantsList: [ + { + GrantScope: 's3://bucket/*', + Permission: 'READ', + }, + { + GrantScope: 's3://bucket/path/*', + Permission: 'READWRITE', + }, + { + GrantScope: 's3://bucket/path/to/object', + Permission: 'READ', + ApplicationArn: 'arn:123', + }, + ], + $metadata: {} as any, + }); + const { locations, nextToken } = await listCallerAccessGrants({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + }); + + expect(locations).toEqual([ + { + scope: 's3://bucket/*', + type: 'BUCKET', + permission: 'READ', + applicationArn: undefined, + }, + { + scope: 's3://bucket/path/*', + type: 'PREFIX', + permission: 'READWRITE', + applicationArn: undefined, + }, + { + scope: 's3://bucket/path/to/object', + type: 'OBJECT', + permission: 'READ', + applicationArn: 'arn:123', + }, + ]); + expect(nextToken).toBeUndefined(); + }); +}); diff --git a/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createListLocationsHandler.test.ts b/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createListLocationsHandler.test.ts new file mode 100644 index 00000000000..c2104ce728a --- /dev/null +++ b/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createListLocationsHandler.test.ts @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createListLocationsHandler } from '../../../src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler'; +import { listCallerAccessGrants } from '../../../src/storageBrowser/apis/listCallerAccessGrants'; + +jest.mock('../../../src/storageBrowser/apis/listCallerAccessGrants'); + +jest.mocked(listCallerAccessGrants).mockResolvedValue({ + locations: [], +}); + +describe('createListLocationsHandler', () => { + it('should parse the underlying API with right parameters', async () => { + const mockAccountId = '1234567890'; + const mockRegion = 'us-foo-1'; + const mockCredentialsProvider = jest.fn(); + const mockNextToken = '123'; + const mockPageSize = 123; + const handler = createListLocationsHandler({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + }); + await handler({ nextToken: mockNextToken, pageSize: mockPageSize }); + expect(listCallerAccessGrants).toHaveBeenCalledWith({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + nextToken: mockNextToken, + pageSize: mockPageSize, + }); + }); +}); diff --git a/packages/storage/src/storageBrowser/apis/constants.ts b/packages/storage/src/storageBrowser/apis/constants.ts index e333ac5a5e2..4c322de94f1 100644 --- a/packages/storage/src/storageBrowser/apis/constants.ts +++ b/packages/storage/src/storageBrowser/apis/constants.ts @@ -2,3 +2,4 @@ // SPDX-License-Identifier: Apache-2.0 export const DEFAULT_CRED_TTL = 15 * 60; // 15 minutes +export const MAX_PAGE_SIZE = 1000; diff --git a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts index dbe8b305b36..957e6eb1fcb 100644 --- a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts @@ -1,15 +1,105 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { StorageAction } from '@aws-amplify/core/internals/utils'; + +import { logger } from '../../utils'; +import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../providers/s3/utils/client/s3control'; +import { AccessGrant, LocationType, Permission } from '../types'; +import { StorageError } from '../../errors/StorageError'; +import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; + import { ListCallerAccessGrantsInput, ListCallerAccessGrantsOutput, } from './types'; +import { MAX_PAGE_SIZE } from './constants'; -export const listCallerAccessGrants = ( - // eslint-disable-next-line unused-imports/no-unused-vars +export const listCallerAccessGrants = async ( input: ListCallerAccessGrantsInput, ): Promise => { - // TODO(@AllanZhengYP) - throw new Error('Not Implemented'); + const { credentialsProvider, accountId, region, nextToken, pageSize } = input; + + logger.debug(`listing available locations from account ${input.accountId}`); + + if (!!pageSize && pageSize > MAX_PAGE_SIZE) { + logger.debug(`defaulting pageSize to ${MAX_PAGE_SIZE}.`); + } + + const clientCredentialsProvider = async () => { + const { credentials } = await credentialsProvider(); + + return credentials; + }; + + const { CallerAccessGrantsList, NextToken } = + await listCallerAccessGrantsClient( + { + credentials: clientCredentialsProvider, + region, + userAgentValue: getStorageUserAgentValue( + StorageAction.ListCallerAccessGrants, + ), + }, + { + AccountId: accountId, + NextToken: nextToken, + MaxResults: pageSize ?? MAX_PAGE_SIZE, + }, + ); + + const accessGrants: AccessGrant[] = + CallerAccessGrantsList?.map(grant => { + // These values are correct from service mostly, but we add assertions to make TSC happy. + assertPermission(grant.Permission); + assertGrantScope(grant.GrantScope); + + return { + scope: grant.GrantScope, + permission: grant.Permission, + applicationArn: grant.ApplicationArn, + type: parseGrantType(grant.GrantScope!), + }; + }) ?? []; + + return { + locations: accessGrants, + nextToken: NextToken, + }; }; + +const parseGrantType = (grantScope: string): LocationType => { + const bucketScopeReg = /^s3:\/\/(.*)\/\*$/; + const possibleBucketName = grantScope.match(bucketScopeReg)?.[1]; + if (!grantScope.endsWith('*')) { + return 'OBJECT'; + } else if ( + grantScope.endsWith('/*') && + possibleBucketName && + possibleBucketName.indexOf('/') === -1 + ) { + return 'BUCKET'; + } else { + return 'PREFIX'; + } +}; + +function assertPermission( + permissionValue: string | undefined, +): asserts permissionValue is Permission { + if (!['READ', 'READWRITE', 'WRITE'].includes(permissionValue ?? '')) { + throw new StorageError({ + name: 'InvalidPermission', + message: `Invalid permission: ${permissionValue}`, + }); + } +} + +function assertGrantScope(value: unknown): asserts value is string { + if (typeof value !== 'string' || !value.startsWith('s3://')) { + throw new StorageError({ + name: 'InvalidGrantScope', + message: `Expected a valid grant scope, got ${value}`, + }); + } +} diff --git a/packages/storage/src/storageBrowser/apis/types.ts b/packages/storage/src/storageBrowser/apis/types.ts index 2928cfd1f38..c97a7c4bbdd 100644 --- a/packages/storage/src/storageBrowser/apis/types.ts +++ b/packages/storage/src/storageBrowser/apis/types.ts @@ -4,6 +4,7 @@ import { AccessGrant, CredentialsProvider, + ListLocationsInput, ListLocationsOutput, LocationCredentials, Permission, @@ -11,7 +12,7 @@ import { Privilege, } from '../types'; -export interface ListCallerAccessGrantsInput { +export interface ListCallerAccessGrantsInput extends ListLocationsInput { accountId: string; credentialsProvider: CredentialsProvider; region: string; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts index c3e9c3c1a4a..eb224b6ab3c 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts @@ -2,17 +2,32 @@ // SPDX-License-Identifier: Apache-2.0 import { CredentialsProvider, ListLocations } from '../types'; +import { listCallerAccessGrants } from '../apis/listCallerAccessGrants'; -export interface CreateListLocationsHandlerInput { +interface CreateListLocationsHandlerInput { accountId: string; credentialsProvider: CredentialsProvider; region: string; } export const createListLocationsHandler = ( - // eslint-disable-next-line unused-imports/no-unused-vars - input: CreateListLocationsHandlerInput, + handlerInput: CreateListLocationsHandlerInput, ): ListLocations => { - // TODO(@AllanZhengYP) - throw new Error('Not Implemented'); + return async (input = {}) => { + const { nextToken, pageSize } = input; + const { locations, nextToken: newNextToken } = await listCallerAccessGrants( + { + accountId: handlerInput.accountId, + credentialsProvider: handlerInput.credentialsProvider, + region: handlerInput.region, + pageSize, + nextToken, + }, + ); + + return { + locations, + nextToken: newNextToken || undefined, + }; + }; }; From 5d5bea15744a6e943da30009e3eca62668653985 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Thu, 18 Jul 2024 12:01:09 -0700 Subject: [PATCH 17/40] chore: expose path storage-browser from scoped package (#13611) chore: expose path storage-browser from scoped package Co-authored-by: Ashwin Kumar --- packages/storage/package.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/storage/package.json b/packages/storage/package.json index 3de800029e0..264c20c0191 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -95,6 +95,11 @@ "import": "./dist/esm/providers/s3/server.mjs", "require": "./dist/cjs/providers/s3/server.js" }, + "./storage-browser": { + "types": "./dist/esm/storageBrowser/index.d.ts", + "import": "./dist/esm/storageBrowser/index.mjs", + "require": "./dist/cjs/storageBrowser/index.js" + }, "./package.json": "./package.json" }, "peerDependencies": { From c5464ac4c8962d365853c3c7d0d41aaca9881bf8 Mon Sep 17 00:00:00 2001 From: israx <70438514+israx@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:55:38 -0400 Subject: [PATCH 18/40] feat(storage): enables location credentials provider (#13605) * feat: add location credentials provider * chore: add unit tests * chore: address feedback * chore: add locationCredentialsOption to copy * chore: remove casting types * chore: assert idenitity id * chore: avoid export common options interface * chore: address feedback * chore: fix test * chore: address feedback * address feedback * chore: clean-up types * chore: add test --- .../utils/resolveS3ConfigAndInput.test.ts | 100 +++++++++++++++++- packages/storage/src/errors/constants.ts | 4 + .../src/providers/s3/apis/downloadData.ts | 2 +- .../src/providers/s3/apis/internal/copy.ts | 13 ++- .../s3/apis/internal/getProperties.ts | 3 +- .../src/providers/s3/apis/internal/getUrl.ts | 2 +- .../src/providers/s3/apis/internal/list.ts | 2 +- .../src/providers/s3/apis/internal/remove.ts | 3 +- .../uploadData/multipart/uploadHandlers.ts | 2 +- .../s3/apis/uploadData/putObjectJob.ts | 2 +- .../storage/src/providers/s3/types/inputs.ts | 6 +- .../providers/s3/utils/resolveIdentityId.ts | 11 ++ .../s3/utils/resolveS3ConfigAndInput.ts | 96 ++++++++++++++++- .../s3/utils/validateStorageOperationInput.ts | 6 +- ...validateStorageOperationInputWithPrefix.ts | 6 +- packages/storage/src/types/inputs.ts | 3 +- 16 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 packages/storage/src/errors/constants.ts create mode 100644 packages/storage/src/providers/s3/utils/resolveIdentityId.ts diff --git a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts index e26cb63b6c7..efae2febaaf 100644 --- a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts @@ -9,6 +9,11 @@ import { StorageValidationErrorCode, validationErrorMap, } from '../../../../../src/errors/types/validation'; +import { + CallbackPathStorageInput, + DeprecatedStorageInput, +} from '../../../../../src/providers/s3/utils/resolveS3ConfigAndInput'; +import { INVALID_STORAGE_INPUT } from '../../../../../src/errors/constants'; jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn(), @@ -76,13 +81,11 @@ describe('resolveS3ConfigAndInput', () => { } }); - it('should throw if identityId is not available', async () => { + it('should not throw if identityId is not available', async () => { mockFetchAuthSession.mockResolvedValueOnce({ credentials, }); - await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( - validationErrorMap[StorageValidationErrorCode.NoIdentityId], - ); + expect(async () => resolveS3ConfigAndInput(Amplify, {})).not.toThrow(); }); it('should resolve bucket from S3 config', async () => { @@ -179,7 +182,7 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with given access level', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, { - accessLevel: 'someLevel' as any, + options: { accessLevel: 'someLevel' as any }, }); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'someLevel', @@ -214,4 +217,91 @@ describe('resolveS3ConfigAndInput', () => { }); expect(keyPrefix).toEqual('prefix'); }); + + describe('with locationCredentialsProvider', () => { + const mockLocationCredentialsProvider = jest + .fn() + .mockReturnValue({ credentials }); + it('should resolve credentials without Amplify singleton', async () => { + mockGetConfig.mockReturnValue({ + Storage: { + S3: { + bucket, + region, + }, + }, + }); + const { s3Config } = await resolveS3ConfigAndInput(Amplify, { + options: { + locationCredentialsProvider: mockLocationCredentialsProvider, + }, + }); + + if (typeof s3Config.credentials === 'function') { + const result = await s3Config.credentials(); + expect(mockLocationCredentialsProvider).toHaveBeenCalled(); + expect(result).toEqual(credentials); + } else { + throw new Error('Expect credentials to be a function'); + } + }); + + it('should not throw when path is pass as a string', async () => { + const { s3Config } = await resolveS3ConfigAndInput(Amplify, { + path: 'my-path', + options: { + locationCredentialsProvider: mockLocationCredentialsProvider, + }, + }); + + if (typeof s3Config.credentials === 'function') { + const result = await s3Config.credentials(); + expect(mockLocationCredentialsProvider).toHaveBeenCalled(); + expect(result).toEqual(credentials); + } else { + throw new Error('Expect credentials to be a function'); + } + }); + + describe('with deprecated or callback paths as inputs', () => { + const key = 'mock-value'; + const prefix = 'mock-value'; + const path = () => 'path'; + const deprecatedInputs: DeprecatedStorageInput[] = [ + { prefix }, + { key }, + { + source: { key }, + destination: { key }, + }, + ]; + const callbackPathInputs: CallbackPathStorageInput[] = [ + { path }, + { + destination: { path }, + source: { path }, + }, + ]; + + const testCases = [...deprecatedInputs, ...callbackPathInputs]; + + it.each(testCases)('should throw when input is %s', async input => { + const { s3Config } = await resolveS3ConfigAndInput(Amplify, { + ...input, + options: { + locationCredentialsProvider: mockLocationCredentialsProvider, + }, + }); + if (typeof s3Config.credentials === 'function') { + await expect(s3Config.credentials()).rejects.toThrow( + expect.objectContaining({ + name: INVALID_STORAGE_INPUT, + }), + ); + } else { + throw new Error('Expect credentials to be a function'); + } + }); + }); + }); }); diff --git a/packages/storage/src/errors/constants.ts b/packages/storage/src/errors/constants.ts new file mode 100644 index 00000000000..ca127c2e623 --- /dev/null +++ b/packages/storage/src/errors/constants.ts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const INVALID_STORAGE_INPUT = 'InvalidStorageInput'; diff --git a/packages/storage/src/providers/s3/apis/downloadData.ts b/packages/storage/src/providers/s3/apis/downloadData.ts index 009e75ae95b..32f0c558642 100644 --- a/packages/storage/src/providers/s3/apis/downloadData.ts +++ b/packages/storage/src/providers/s3/apis/downloadData.ts @@ -115,7 +115,7 @@ const downloadDataJob = > => { const { options: downloadDataOptions } = downloadDataInput; const { bucket, keyPrefix, s3Config, identityId } = - await resolveS3ConfigAndInput(Amplify, downloadDataOptions); + await resolveS3ConfigAndInput(Amplify, downloadDataInput); const { inputType, objectKey } = validateStorageOperationInput( downloadDataInput, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index 0e1fc01eadf..44906a816dc 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -40,8 +40,10 @@ const copyWithPath = async ( input: CopyWithPathInput, ): Promise => { const { source, destination } = input; - const { s3Config, bucket, identityId } = - await resolveS3ConfigAndInput(amplify); + const { s3Config, bucket, identityId } = await resolveS3ConfigAndInput( + amplify, + input, + ); assertValidationError(!!source.path, StorageValidationErrorCode.NoSourcePath); assertValidationError( @@ -92,10 +94,13 @@ export const copyWithKey = async ( s3Config, bucket, keyPrefix: sourceKeyPrefix, - } = await resolveS3ConfigAndInput(amplify, input.source); + } = await resolveS3ConfigAndInput(amplify, { + ...input, + options: input.source, + }); const { keyPrefix: destinationKeyPrefix } = await resolveS3ConfigAndInput( amplify, - input.destination, + { ...input, options: input.destination }, ); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. // TODO(ashwinkumar6) V6-logger: warn `You may copy files from another user if the source level is "protected", currently it's ${srcLevel}` diff --git a/packages/storage/src/providers/s3/apis/internal/getProperties.ts b/packages/storage/src/providers/s3/apis/internal/getProperties.ts index 915f02db495..ac04b2dbe6e 100644 --- a/packages/storage/src/providers/s3/apis/internal/getProperties.ts +++ b/packages/storage/src/providers/s3/apis/internal/getProperties.ts @@ -24,9 +24,8 @@ export const getProperties = async ( input: GetPropertiesInput | GetPropertiesWithPathInput, action?: StorageAction, ): Promise => { - const { options: getPropertiesOptions } = input; const { s3Config, bucket, keyPrefix, identityId } = - await resolveS3ConfigAndInput(amplify, getPropertiesOptions); + await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/getUrl.ts b/packages/storage/src/providers/s3/apis/internal/getUrl.ts index e8440ce80eb..7ea49a448c9 100644 --- a/packages/storage/src/providers/s3/apis/internal/getUrl.ts +++ b/packages/storage/src/providers/s3/apis/internal/getUrl.ts @@ -31,7 +31,7 @@ export const getUrl = async ( ): Promise => { const { options: getUrlOptions } = input; const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput(amplify, getUrlOptions); + await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index 6f074858738..7a1ab47007a 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -58,7 +58,7 @@ export const list = async ( bucket, keyPrefix: generatedPrefix, identityId, - } = await resolveS3ConfigAndInput(amplify, options); + } = await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInputWithPrefix( input, diff --git a/packages/storage/src/providers/s3/apis/internal/remove.ts b/packages/storage/src/providers/s3/apis/internal/remove.ts index e2a9377f39e..d73a13346e4 100644 --- a/packages/storage/src/providers/s3/apis/internal/remove.ts +++ b/packages/storage/src/providers/s3/apis/internal/remove.ts @@ -23,9 +23,8 @@ export const remove = async ( amplify: AmplifyClassV6, input: RemoveInput | RemoveWithPathInput, ): Promise => { - const { options = {} } = input ?? {}; const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput(amplify, options); + await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInput( input, diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index 886a769648b..00d3903914c 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -73,7 +73,7 @@ export const getMultipartUploadHandlers = ( const { options: uploadDataOptions, data } = uploadDataInput; const resolvedS3Options = await resolveS3ConfigAndInput( Amplify, - uploadDataOptions, + uploadDataInput, ); abortController = new AbortController(); diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index 76f9ebf5638..b7c930ea563 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -29,7 +29,7 @@ export const putObjectJob = async (): Promise => { const { options: uploadDataOptions, data } = uploadDataInput; const { bucket, keyPrefix, s3Config, isObjectLockEnabled, identityId } = - await resolveS3ConfigAndInput(Amplify, uploadDataOptions); + await resolveS3ConfigAndInput(Amplify, uploadDataInput); const { inputType, objectKey } = validateStorageOperationInput( uploadDataInput, identityId, diff --git a/packages/storage/src/providers/s3/types/inputs.ts b/packages/storage/src/providers/s3/types/inputs.ts index f7bd6c5db44..3158cc0fbba 100644 --- a/packages/storage/src/providers/s3/types/inputs.ts +++ b/packages/storage/src/providers/s3/types/inputs.ts @@ -35,6 +35,8 @@ import { UploadDataOptionsWithPath, } from '../types'; +import { LocationCredentialsProvider } from './options'; + // TODO: support use accelerate endpoint option /** * @deprecated Use {@link CopyWithPathInput} instead. @@ -47,7 +49,9 @@ export type CopyInput = StorageCopyInputWithKey< /** * Input type with path for S3 copy API. */ -export type CopyWithPathInput = StorageCopyInputWithPath; +export type CopyWithPathInput = StorageCopyInputWithPath<{ + locationCredentialsProvider?: LocationCredentialsProvider; +}>; /** * @deprecated Use {@link GetPropertiesWithPathInput} instead. diff --git a/packages/storage/src/providers/s3/utils/resolveIdentityId.ts b/packages/storage/src/providers/s3/utils/resolveIdentityId.ts new file mode 100644 index 00000000000..c4831ae88c4 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/resolveIdentityId.ts @@ -0,0 +1,11 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageValidationErrorCode } from '../../../errors/types/validation'; +import { assertValidationError } from '../../../errors/utils/assertValidationError'; + +export const resolveIdentityId = (identityId?: string): string => { + assertValidationError(!!identityId, StorageValidationErrorCode.NoIdentityId); + + return identityId; +}; diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index ae7a185c93c..af6b75f36ec 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -6,7 +6,18 @@ import { AmplifyClassV6, StorageAccessLevel } from '@aws-amplify/core'; import { assertValidationError } from '../../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { resolvePrefix as defaultPrefixResolver } from '../../../utils/resolvePrefix'; -import { ResolvedS3Config } from '../types/options'; +import { + LocationCredentialsProvider, + ResolvedS3Config, +} from '../types/options'; +import { + StorageOperationInputWithKey, + StorageOperationInputWithPath, + StorageOperationInputWithPrefix, +} from '../../../types/inputs'; +import { StorageError } from '../../../errors/StorageError'; +import { CopyInput, CopyWithPathInput } from '../types'; +import { INVALID_STORAGE_INPUT } from '../../../errors/constants'; import { DEFAULT_ACCESS_LEVEL, LOCAL_TESTING_S3_ENDPOINT } from './constants'; @@ -14,6 +25,7 @@ interface S3ApiOptions { accessLevel?: StorageAccessLevel; targetIdentityId?: string; useAccelerateEndpoint?: boolean; + locationCredentialsProvider?: LocationCredentialsProvider; } interface ResolvedS3ConfigAndInput { @@ -23,6 +35,16 @@ interface ResolvedS3ConfigAndInput { isObjectLockEnabled?: boolean; identityId?: string; } +export type DeprecatedStorageInput = + | StorageOperationInputWithKey + | StorageOperationInputWithPrefix + | CopyInput; + +export type CallbackPathStorageInput = + | StorageOperationInputWithPath + | CopyWithPathInput; + +type StorageInput = DeprecatedStorageInput | CallbackPathStorageInput; /** * resolve the common input options for S3 API handlers from Amplify configuration and library options. @@ -37,14 +59,14 @@ interface ResolvedS3ConfigAndInput { */ export const resolveS3ConfigAndInput = async ( amplify: AmplifyClassV6, - apiOptions?: S3ApiOptions, + apiInput?: StorageInput & { options?: S3ApiOptions }, ): Promise => { + const { options: apiOptions } = apiInput ?? {}; /** * IdentityId is always cached in memory so we can safely make calls here. It * should be stable even for unauthenticated users, regardless of credentials. */ const { identityId } = await amplify.Auth.fetchAuthSession(); - assertValidationError(!!identityId, StorageValidationErrorCode.NoIdentityId); /** * A credentials provider function instead of a static credentials object is @@ -53,7 +75,13 @@ export const resolveS3ConfigAndInput = async ( * credentials if they are expired. */ const credentialsProvider = async () => { - const { credentials } = await amplify.Auth.fetchAuthSession(); + if (isLocationCredentialsProvider(apiOptions)) { + assertStorageInput(apiInput); + } + + const { credentials } = isLocationCredentialsProvider(apiOptions) + ? await apiOptions.locationCredentialsProvider() + : await amplify.Auth.fetchAuthSession(); assertValidationError( !!credentials, StorageValidationErrorCode.NoCredentials, @@ -101,3 +129,63 @@ export const resolveS3ConfigAndInput = async ( isObjectLockEnabled, }; }; + +const isLocationCredentialsProvider = ( + options?: S3ApiOptions, +): options is S3ApiOptions & { + locationCredentialsProvider: LocationCredentialsProvider; +} => { + return !!options?.locationCredentialsProvider; +}; + +const isInputWithCallbackPath = (input?: CallbackPathStorageInput) => { + return ( + ((input as StorageOperationInputWithPath)?.path && + typeof (input as StorageOperationInputWithPath).path === 'function') || + ((input as CopyWithPathInput)?.destination?.path && + typeof (input as CopyWithPathInput).destination?.path === 'function') || + ((input as CopyWithPathInput)?.source?.path && + typeof (input as CopyWithPathInput).source?.path === 'function') + ); +}; + +const isDeprecatedInput = ( + input?: StorageInput, +): input is DeprecatedStorageInput => { + return ( + isInputWithKey(input) || + isInputWithPrefix(input) || + isInputWithCopySourceOrDestination(input) + ); +}; +const assertStorageInput = (input?: StorageInput) => { + if (isDeprecatedInput(input) || isInputWithCallbackPath(input)) { + throw new StorageError({ + name: INVALID_STORAGE_INPUT, + message: 'The input needs to have a path as a string value.', + recoverySuggestion: + 'Please provide a valid path as a string value for the input.', + }); + } +}; + +const isInputWithKey = ( + input?: StorageInput, +): input is StorageOperationInputWithKey => { + return !!(typeof (input as StorageOperationInputWithKey).key === 'string'); +}; +const isInputWithPrefix = ( + input?: StorageInput, +): input is StorageOperationInputWithPrefix => { + return !!( + typeof (input as StorageOperationInputWithPrefix).prefix === 'string' + ); +}; +const isInputWithCopySourceOrDestination = ( + input?: StorageInput, +): input is CopyInput => { + return !!( + typeof (input as CopyInput).source?.key === 'string' || + typeof (input as CopyInput).destination?.key === 'string' + ); +}; diff --git a/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts b/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts index 585701c81e9..fa423b45913 100644 --- a/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts +++ b/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts @@ -7,6 +7,7 @@ import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { isInputWithPath } from './isInputWithPath'; import { STORAGE_INPUT_KEY, STORAGE_INPUT_PATH } from './constants'; +import { resolveIdentityId } from './resolveIdentityId'; export const validateStorageOperationInput = ( input: Input, @@ -22,7 +23,10 @@ export const validateStorageOperationInput = ( if (isInputWithPath(input)) { const { path } = input; - const objectKey = typeof path === 'string' ? path : path({ identityId }); + const objectKey = + typeof path === 'string' + ? path + : path({ identityId: resolveIdentityId(identityId) }); assertValidationError( !objectKey.startsWith('/'), diff --git a/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts b/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts index da1068af010..1c2efce19f7 100644 --- a/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts +++ b/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts @@ -9,6 +9,7 @@ import { assertValidationError } from '../../../errors/utils/assertValidationErr import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { STORAGE_INPUT_PATH, STORAGE_INPUT_PREFIX } from './constants'; +import { resolveIdentityId } from './resolveIdentityId'; // Local assertion function with StorageOperationInputWithPrefixPath as Input const _isInputWithPath = ( @@ -28,7 +29,10 @@ export const validateStorageOperationInputWithPrefix = ( ); if (_isInputWithPath(input)) { const { path } = input; - const objectKey = typeof path === 'string' ? path : path({ identityId }); + const objectKey = + typeof path === 'string' + ? path + : path({ identityId: resolveIdentityId(identityId) }); // Assert on no leading slash in the path parameter assertValidationError( diff --git a/packages/storage/src/types/inputs.ts b/packages/storage/src/types/inputs.ts index 403a2a14332..4e369824ca4 100644 --- a/packages/storage/src/types/inputs.ts +++ b/packages/storage/src/types/inputs.ts @@ -90,7 +90,8 @@ export interface StorageCopyInputWithKey< destination: DestinationOptions; } -export interface StorageCopyInputWithPath { +export interface StorageCopyInputWithPath + extends StorageOperationOptionsInput { source: StorageOperationInputWithPath; destination: StorageOperationInputWithPath; } From 603e1247ca47b59e79ec5fbe637f1a0a0065b490 Mon Sep 17 00:00:00 2001 From: Allan Zheng Date: Tue, 23 Jul 2024 14:09:33 -0700 Subject: [PATCH 19/40] chore: update api bundlesize --- packages/aws-amplify/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index a3c65b1ae25..b925e10b7db 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -305,7 +305,7 @@ "name": "[Analytics] record (Kinesis Firehose)", "path": "./dist/esm/analytics/kinesis-firehose/index.mjs", "import": "{ record }", - "limit": "45.68 kB" + "limit": "45.71 kB" }, { "name": "[Analytics] record (Personalize)", @@ -485,7 +485,7 @@ "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.36 kB" + "limit": "15.39 kB" }, { "name": "[Storage] remove (S3)", From 6e5b3c58fe2d1c3e01eed99fd53036ddeb075a35 Mon Sep 17 00:00:00 2001 From: Allan Zheng Date: Tue, 23 Jul 2024 18:25:30 -0700 Subject: [PATCH 20/40] feat(storage): resolve merge issue with multibucket --- .../src/providers/s3/apis/internal/copy.ts | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index ae0117334c1..a8562ac3beb 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -56,20 +56,27 @@ const copyWithPath = async ( input: CopyWithPathInput, ): Promise => { const { source, destination } = input; - // TODO(@AllanZhengYP) - await resolveS3ConfigAndInput(amplify, input); storageBucketAssertion(source.bucket, destination.bucket); - const { bucket: sourceBucket, identityId } = await resolveS3ConfigAndInput( - amplify, - input.source, - ); + const { bucket: sourceBucket } = await resolveS3ConfigAndInput(amplify, { + path: input.source.path, + options: { ...input.source }, + }); - const { s3Config, bucket: destBucket } = await resolveS3ConfigAndInput( - amplify, - input.destination, - ); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. + // The bucket, region, credentials of s3 client are resolved from destination. + // Whereas the source bucket and path are a input parameter of S3 copy operation. + const { + s3Config, + bucket: destBucket, + identityId, + } = await resolveS3ConfigAndInput(amplify, { + path: input.destination.path, + options: { + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.destination, + }, + }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. assertValidationError(!!source.path, StorageValidationErrorCode.NoSourcePath); assertValidationError( @@ -121,6 +128,8 @@ export const copyWithKey = async ( options: input.source, }); + // The bucket, region, credentials of s3 client are resolved from destination. + // Whereas the source bucket and path are a input parameter of S3 copy operation. const { s3Config, bucket: destBucket, From 048ea00a8233bae5eec06894664fe02bd49ffc53 Mon Sep 17 00:00:00 2001 From: Allan Zheng Date: Wed, 24 Jul 2024 09:14:56 -0700 Subject: [PATCH 21/40] chore: update bundle size for config change and s3 multibucket --- packages/aws-amplify/package.json | 54 +++++++++++++++--------------- packages/interactions/package.json | 6 ++-- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 35c982c2a71..47c23fac960 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -293,31 +293,31 @@ "name": "[Analytics] record (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ record }", - "limit": "17.18 kB" + "limit": "17.23 kB" }, { "name": "[Analytics] record (Kinesis)", "path": "./dist/esm/analytics/kinesis/index.mjs", "import": "{ record }", - "limit": "48.61 kB" + "limit": "48.65 kB" }, { "name": "[Analytics] record (Kinesis Firehose)", "path": "./dist/esm/analytics/kinesis-firehose/index.mjs", "import": "{ record }", - "limit": "45.76 kB" + "limit": "45.81 kB" }, { "name": "[Analytics] record (Personalize)", "path": "./dist/esm/analytics/personalize/index.mjs", "import": "{ record }", - "limit": "49.58 kB" + "limit": "49.63 kB" }, { "name": "[Analytics] identifyUser (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ identifyUser }", - "limit": "15.68 kB" + "limit": "15.73 kB" }, { "name": "[Analytics] enable", @@ -335,7 +335,7 @@ "name": "[API] generateClient (AppSync)", "path": "./dist/esm/api/index.mjs", "import": "{ generateClient }", - "limit": "40.14 kB" + "limit": "40.19 kB" }, { "name": "[API] REST API handlers", @@ -353,13 +353,13 @@ "name": "[Auth] resetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resetPassword }", - "limit": "12.53 kB" + "limit": "12.58 kB" }, { "name": "[Auth] confirmResetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmResetPassword }", - "limit": "12.47 kB" + "limit": "12.52 kB" }, { "name": "[Auth] signIn (Cognito)", @@ -371,7 +371,7 @@ "name": "[Auth] resendSignUpCode (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resendSignUpCode }", - "limit": "12.49 kB" + "limit": "12.53 kB" }, { "name": "[Auth] confirmSignUp (Cognito)", @@ -383,31 +383,31 @@ "name": "[Auth] confirmSignIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignIn }", - "limit": "28.38 kB" + "limit": "28.42 kB" }, { "name": "[Auth] updateMFAPreference (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updateMFAPreference }", - "limit": "11.83 kB" + "limit": "11.87 kB" }, { "name": "[Auth] fetchMFAPreference (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ fetchMFAPreference }", - "limit": "11.86 kB" + "limit": "11.91 kB" }, { "name": "[Auth] verifyTOTPSetup (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ verifyTOTPSetup }", - "limit": "12.71 kB" + "limit": "12.75 kB" }, { "name": "[Auth] updatePassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updatePassword }", - "limit": "12.73 kB" + "limit": "12.76 kB" }, { "name": "[Auth] setUpTOTP (Cognito)", @@ -419,7 +419,7 @@ "name": "[Auth] updateUserAttributes (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updateUserAttributes }", - "limit": "11.95 kB" + "limit": "11.99 kB" }, { "name": "[Auth] getCurrentUser (Cognito)", @@ -431,73 +431,73 @@ "name": "[Auth] confirmUserAttribute (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmUserAttribute }", - "limit": "12.71 kB" + "limit": "12.75 kB" }, { "name": "[Auth] signInWithRedirect (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect }", - "limit": "21.15 kB" + "limit": "21.19 kB" }, { "name": "[Auth] fetchUserAttributes (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ fetchUserAttributes }", - "limit": "11.77 kB" + "limit": "11.82 kB" }, { "name": "[Auth] Basic Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signIn, signOut, fetchAuthSession, confirmSignIn }", - "limit": "30.15 kB" + "limit": "30.20 kB" }, { "name": "[Auth] OAuth Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect, signOut, fetchAuthSession }", - "limit": "21.58 kB" + "limit": "21.62 kB" }, { "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "14.86 kB" + "limit": "15.24 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.45 kB" + "limit": "15.76 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "14.70 kB" + "limit": "15.03 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "15.79 kB" + "limit": "16.09 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.30 kB" + "limit": "15.65 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "14.56 kB" + "limit": "14.88 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "19.85 kB" + "limit": "20.18 kB" } ] } diff --git a/packages/interactions/package.json b/packages/interactions/package.json index 9194df2c3b6..98359327196 100644 --- a/packages/interactions/package.json +++ b/packages/interactions/package.json @@ -89,19 +89,19 @@ "name": "Interactions (default to Lex v2)", "path": "./dist/esm/index.mjs", "import": "{ Interactions }", - "limit": "52.61 kB" + "limit": "52.66 kB" }, { "name": "Interactions (Lex v2)", "path": "./dist/esm/lex-v2/index.mjs", "import": "{ Interactions }", - "limit": "52.61 kB" + "limit": "52.66 kB" }, { "name": "Interactions (Lex v1)", "path": "./dist/esm/lex-v1/index.mjs", "import": "{ Interactions }", - "limit": "47.41 kB" + "limit": "47.46 kB" } ] } From 4f9f9d8e8c0d7bfa0fa9877cd46378ac94b0fb31 Mon Sep 17 00:00:00 2001 From: Allan Zheng Date: Wed, 24 Jul 2024 10:43:44 -0700 Subject: [PATCH 22/40] chore: address feedbacks --- .../src/providers/s3/apis/internal/copy.ts | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index a8562ac3beb..9119917efa1 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -61,7 +61,10 @@ const copyWithPath = async ( const { bucket: sourceBucket } = await resolveS3ConfigAndInput(amplify, { path: input.source.path, - options: { ...input.source }, + options: { + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.source, + }, }); // The bucket, region, credentials of s3 client are resolved from destination. @@ -125,7 +128,12 @@ export const copyWithKey = async ( const { bucket: sourceBucket, keyPrefix: sourceKeyPrefix } = await resolveS3ConfigAndInput(amplify, { ...input, - options: input.source, + options: { + // @ts-expect-error: 'options' does not exist on type 'CopyInput'. In case of JS users set the location + // credentials provider option, resolveS3ConfigAndInput will throw validation error. + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.source, + }, }); // The bucket, region, credentials of s3 client are resolved from destination. @@ -136,7 +144,12 @@ export const copyWithKey = async ( keyPrefix: destinationKeyPrefix, } = await resolveS3ConfigAndInput(amplify, { ...input, - options: input.destination, + options: { + // @ts-expect-error: 'options' does not exist on type 'CopyInput'. In case of JS users set the location + // credentials provider option, resolveS3ConfigAndInput will throw validation error. + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.destination, + }, }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. // TODO(ashwinkumar6) V6-logger: warn `You may copy files from another user if the source level is "protected", currently it's ${srcLevel}` From 0b1ec2e4a00098caadf4f509e7821b5aa00cdc4a Mon Sep 17 00:00:00 2001 From: Jamie Epp <168486127+eppjame@users.noreply.github.com> Date: Wed, 24 Jul 2024 14:41:59 -0700 Subject: [PATCH 23/40] feat(storage): introduce preventOverwrite option to uploadData via HeadObject (#13640) * feat(storage): introduce preventOverwrite operation to uploadData via HeadObject * fix: add missing license and remove dependency on core in preventOverwrite validator * chore: update storage:uploadData bundle size * feat: move existing object validation to before completeMultipartUpload --- packages/aws-amplify/package.json | 2 +- .../apis/uploadData/multipartHandlers.test.ts | 78 ++++++++++++++++++ .../s3/apis/uploadData/putObjectJob.test.ts | 81 ++++++++++++++++++- .../uploadData/multipart/uploadHandlers.ts | 9 +++ .../s3/apis/uploadData/putObjectJob.ts | 10 +++ .../uploadData/validateObjectNotExists.ts | 25 ++++++ .../storage/src/providers/s3/types/options.ts | 5 ++ 7 files changed, 208 insertions(+), 2 deletions(-) create mode 100644 packages/storage/src/providers/s3/apis/uploadData/validateObjectNotExists.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index ccf5001d233..8358fa48e44 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "19.94 kB" + "limit": "20.05 kB" } ] } diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index 04c6bf6522a..7e499147405 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -861,6 +861,84 @@ describe('getMultipartUploadHandlers with path', () => { expect(mockUploadPart).toHaveBeenCalledTimes(2); expect(mockCompleteMultipartUpload).not.toHaveBeenCalled(); }); + + describe('overwrite prevention', () => { + beforeEach(() => { + mockHeadObject.mockReset(); + mockUploadPart.mockReset(); + }); + + it('should upload if target key is not found', async () => { + expect.assertions(7); + const notFoundError = new Error('mock message'); + notFoundError.name = 'NotFound'; + mockHeadObject.mockRejectedValueOnce(notFoundError); + mockMultipartUploadSuccess(); + + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new ArrayBuffer(8 * MB), + options: { preventOverwrite: true }, + }); + await multipartUploadJob(); + + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + await expect(mockHeadObject).toBeLastCalledWithConfigAndInput( + expect.objectContaining({ + credentials, + region, + }), + expect.objectContaining({ + Bucket: bucket, + Key: testPath, + }), + ); + expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1); + }); + + it('should not upload if target key already exists', async () => { + expect.assertions(6); + mockHeadObject.mockResolvedValueOnce({ + ContentLength: 0, + $metadata: {}, + }); + mockMultipartUploadSuccess(); + + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new ArrayBuffer(8 * MB), + options: { preventOverwrite: true }, + }); + + await expect(multipartUploadJob()).rejects.toThrow( + 'At least one of the pre-conditions you specified did not hold', + ); + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockCompleteMultipartUpload).not.toHaveBeenCalled(); + }); + + it('should not upload if HeadObject fails with other error', async () => { + expect.assertions(6); + const accessDeniedError = new Error('mock error'); + accessDeniedError.name = 'AccessDenied'; + mockHeadObject.mockRejectedValueOnce(accessDeniedError); + mockMultipartUploadSuccess(); + + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new ArrayBuffer(8 * MB), + options: { preventOverwrite: true }, + }); + + await expect(multipartUploadJob()).rejects.toThrow('mock error'); + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockCompleteMultipartUpload).not.toHaveBeenCalled(); + }); + }); }); describe('upload caching', () => { diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index df1b92113a1..c1bfff5c7a3 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -4,7 +4,10 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify } from '@aws-amplify/core'; -import { putObject } from '../../../../../src/providers/s3/utils/client/s3data'; +import { + headObject, + putObject, +} from '../../../../../src/providers/s3/utils/client/s3data'; import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; @@ -38,6 +41,7 @@ const credentials: AWSCredentials = { const identityId = 'identityId'; const mockFetchAuthSession = jest.mocked(Amplify.Auth.fetchAuthSession); const mockPutObject = jest.mocked(putObject); +const mockHeadObject = jest.mocked(headObject); mockFetchAuthSession.mockResolvedValue({ credentials, @@ -233,4 +237,79 @@ describe('putObjectJob with path', () => { await job(); expect(calculateContentMd5).toHaveBeenCalledWith('data'); }); + + describe('overwrite prevention', () => { + beforeEach(() => { + mockHeadObject.mockClear(); + }); + + it('should upload if target key is not found', async () => { + expect.assertions(3); + const notFoundError = new Error('mock message'); + notFoundError.name = 'NotFound'; + mockHeadObject.mockRejectedValueOnce(notFoundError); + + const job = putObjectJob( + { + path: testPath, + data: 'data', + options: { preventOverwrite: true }, + }, + new AbortController().signal, + ); + await job(); + + await expect(mockHeadObject).toBeLastCalledWithConfigAndInput( + { + credentials, + region: 'region', + }, + { + Bucket: 'bucket', + Key: testPath, + }, + ); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + expect(mockPutObject).toHaveBeenCalledTimes(1); + }); + + it('should not upload if target key already exists', async () => { + expect.assertions(3); + mockHeadObject.mockResolvedValueOnce({ + ContentLength: 0, + $metadata: {}, + }); + const job = putObjectJob( + { + path: testPath, + data: 'data', + options: { preventOverwrite: true }, + }, + new AbortController().signal, + ); + await expect(job()).rejects.toThrow( + 'At least one of the pre-conditions you specified did not hold', + ); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + expect(mockPutObject).not.toHaveBeenCalled(); + }); + + it('should not upload if HeadObject fails with other error', async () => { + expect.assertions(3); + const accessDeniedError = new Error('mock error'); + accessDeniedError.name = 'AccessDenied'; + mockHeadObject.mockRejectedValueOnce(accessDeniedError); + const job = putObjectJob( + { + path: testPath, + data: 'data', + options: { preventOverwrite: true }, + }, + new AbortController().signal, + ); + await expect(job()).rejects.toThrow('mock error'); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + expect(mockPutObject).not.toHaveBeenCalled(); + }); + }); }); diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index 00d3903914c..cb7d8dc1348 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -29,6 +29,7 @@ import { } from '../../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../../utils/userAgent'; import { logger } from '../../../../../utils'; +import { validateObjectNotExists } from '../validateObjectNotExists'; import { uploadPartExecutor } from './uploadPartExecutor'; import { getUploadsCacheKey, removeCachedUpload } from './uploadCache'; @@ -92,6 +93,7 @@ export const getMultipartUploadHandlers = ( contentEncoding, contentType = 'application/octet-stream', metadata, + preventOverwrite, onProgress, } = uploadDataOptions ?? {}; @@ -175,6 +177,13 @@ export const getMultipartUploadHandlers = ( await Promise.all(concurrentUploadPartExecutors); + if (preventOverwrite) { + await validateObjectNotExists(resolvedS3Config, { + Bucket: resolvedBucket, + Key: finalKey, + }); + } + const { ETag: eTag } = await completeMultipartUpload( { ...resolvedS3Config, diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index b7c930ea563..cecd5f736e3 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -15,6 +15,8 @@ import { putObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; +import { validateObjectNotExists } from './validateObjectNotExists'; + /** * Get a function the returns a promise to call putObject API to S3. * @@ -41,10 +43,18 @@ export const putObjectJob = contentDisposition, contentEncoding, contentType = 'application/octet-stream', + preventOverwrite, metadata, onProgress, } = uploadDataOptions ?? {}; + if (preventOverwrite) { + await validateObjectNotExists(s3Config, { + Bucket: bucket, + Key: finalKey, + }); + } + const { ETag: eTag, VersionId: versionId } = await putObject( { ...s3Config, diff --git a/packages/storage/src/providers/s3/apis/uploadData/validateObjectNotExists.ts b/packages/storage/src/providers/s3/apis/uploadData/validateObjectNotExists.ts new file mode 100644 index 00000000000..d2d68e3bdb0 --- /dev/null +++ b/packages/storage/src/providers/s3/apis/uploadData/validateObjectNotExists.ts @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageError } from '../../../../errors/StorageError'; +import { ResolvedS3Config } from '../../types/options'; +import { HeadObjectInput, headObject } from '../../utils/client/s3data'; + +export const validateObjectNotExists = async ( + s3Config: ResolvedS3Config, + input: HeadObjectInput, +): Promise => { + try { + await headObject(s3Config, input); + + throw new StorageError({ + name: 'PreconditionFailed', + message: 'At least one of the pre-conditions you specified did not hold', + }); + } catch (error) { + const serviceError = error as StorageError; + if (serviceError.name !== 'NotFound') { + throw error; + } + } +}; diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index e37bcca8bbc..a2d5fc474ad 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -169,6 +169,11 @@ export type UploadDataOptions = CommonOptions & * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata */ metadata?: Record; + /** + * Enforces target key does not already exist in S3 before committing upload. + * @default false + */ + preventOverwrite?: boolean; }; /** @deprecated Use {@link UploadDataOptionsWithPath} instead. */ From 247871d9ff9aca2f41f034172079c782fe208864 Mon Sep 17 00:00:00 2001 From: Jamie Epp Date: Wed, 24 Jul 2024 17:20:36 -0700 Subject: [PATCH 24/40] fix: increase storage:uploadData bundle size --- packages/aws-amplify/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 47c23fac960..6f7e403e420 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "20.18 kB" + "limit": "20.30 kB" } ] } From 9ba67823d9b33efb13058be6ea45adbc95bc1aed Mon Sep 17 00:00:00 2001 From: Caleb Pollman Date: Thu, 25 Jul 2024 23:02:03 -0700 Subject: [PATCH 25/40] fix(storage): export storage-browser types for TS v4.2+ (#13647) --- packages/storage/package.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/storage/package.json b/packages/storage/package.json index 7bb7c3dfb7d..e6963b64fe4 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -45,7 +45,10 @@ ], "s3/server": [ "./dist/esm/providers/s3/server.d.ts" - ] + ], + "storage-browser": [ + "./dist/esm/storageBrowser/index.d.ts" + ] } }, "repository": { From a17ed4d4770550e56a9287f73f846a5f64ebc464 Mon Sep 17 00:00:00 2001 From: Caleb Pollman Date: Tue, 30 Jul 2024 09:45:41 -0700 Subject: [PATCH 26/40] chore(storage-browser): export store and credentials related types, update createListLocationsHandler (#13660) --- packages/storage/src/storageBrowser/index.ts | 15 +++++++++++++-- .../createListLocationsHandler.ts | 18 +++--------------- .../createManagedAuthConfigAdapter.ts | 4 ++-- .../managedAuthConfigAdapter/index.ts | 6 +++++- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/packages/storage/src/storageBrowser/index.ts b/packages/storage/src/storageBrowser/index.ts index bde6d9bfe82..72878852a09 100644 --- a/packages/storage/src/storageBrowser/index.ts +++ b/packages/storage/src/storageBrowser/index.ts @@ -1,6 +1,17 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +export { LocationCredentialsProvider } from '../providers/s3/types/options'; +export { StorageSubpathStrategy } from '../types/options'; + export { createLocationCredentialsStore } from './locationCredentialsStore'; -export { createManagedAuthConfigAdapter } from './managedAuthConfigAdapter/createManagedAuthConfigAdapter'; -export { GetLocationCredentials, ListLocations } from './types'; +export { + AuthConfigAdapter, + createManagedAuthConfigAdapter, + CreateManagedAuthConfigAdapterInput, +} from './managedAuthConfigAdapter'; +export { + GetLocationCredentials, + ListLocations, + LocationCredentialsStore, +} from './types'; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts index eb224b6ab3c..e246a2bdc27 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts @@ -13,21 +13,9 @@ interface CreateListLocationsHandlerInput { export const createListLocationsHandler = ( handlerInput: CreateListLocationsHandlerInput, ): ListLocations => { - return async (input = {}) => { - const { nextToken, pageSize } = input; - const { locations, nextToken: newNextToken } = await listCallerAccessGrants( - { - accountId: handlerInput.accountId, - credentialsProvider: handlerInput.credentialsProvider, - region: handlerInput.region, - pageSize, - nextToken, - }, - ); + return async function listLocations(input = {}) { + const result = await listCallerAccessGrants({ ...input, ...handlerInput }); - return { - locations, - nextToken: newNextToken || undefined, - }; + return result; }; }; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts index 267fee96c21..3d41d93e2b3 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts @@ -9,13 +9,13 @@ import { import { createListLocationsHandler } from './createListLocationsHandler'; import { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; -interface CreateManagedAuthConfigAdapterInput { +export interface CreateManagedAuthConfigAdapterInput { accountId: string; region: string; credentialsProvider: CredentialsProvider; } -interface AuthConfigAdapter { +export interface AuthConfigAdapter { listLocations: ListLocations; getLocationCredentials: GetLocationCredentials; region: string; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts index 4e04b1a2a77..3dbf18c8849 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts +++ b/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts @@ -1,4 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -export { createManagedAuthConfigAdapter } from './createManagedAuthConfigAdapter'; +export { + AuthConfigAdapter, + createManagedAuthConfigAdapter, + CreateManagedAuthConfigAdapterInput, +} from './createManagedAuthConfigAdapter'; From ed29226194d72e8fcf6f756f3f0e6caf9e82dbeb Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Tue, 30 Jul 2024 14:09:03 -0700 Subject: [PATCH 27/40] feat(storage): support force refresh location credentials (#13589) --- packages/aws-amplify/package.json | 60 ++++---- .../retry/defaultRetryDecider.test.ts | 136 ++++++++++++++++++ .../middleware/retry/middleware.test.ts | 58 ++++++-- .../middleware/signing/middleware.test.ts | 26 ++++ packages/core/src/clients/index.ts | 7 +- .../middleware/retry/defaultRetryDecider.ts | 15 +- .../src/clients/middleware/retry/index.ts | 1 + .../clients/middleware/retry/middleware.ts | 18 ++- .../src/clients/middleware/retry/types.ts | 7 + .../src/clients/middleware/signing/index.ts | 6 +- .../clients/middleware/signing/middleware.ts | 24 +++- packages/core/src/clients/types/core.ts | 5 + packages/core/src/clients/types/index.ts | 1 + .../__tests__/providers/s3/apis/copy.test.ts | 2 - .../utils/resolveS3ConfigAndInput.test.ts | 6 +- .../S3/cases/completeMultipartUpload.ts | 7 +- .../client/S3/utils/retryDecider.test.ts | 103 +++++++++++++ .../storageBrowser/apis/getDataAccess.test.ts | 21 ++- .../apis/listCallerAccessGrants.test.ts | 25 +++- .../storage/src/providers/s3/types/options.ts | 11 +- .../s3/utils/client/s3control/base.ts | 5 +- .../providers/s3/utils/client/s3data/base.ts | 5 +- .../client/s3data/completeMultipartUpload.ts | 18 +-- .../providers/s3/utils/client/utils/index.ts | 1 + .../s3/utils/client/utils/retryDecider.ts | 81 +++++++++++ .../s3/utils/resolveS3ConfigAndInput.ts | 11 +- .../src/storageBrowser/apis/getDataAccess.ts | 11 +- .../apis/listCallerAccessGrants.ts | 7 +- packages/storage/src/storageBrowser/types.ts | 7 +- 29 files changed, 595 insertions(+), 90 deletions(-) create mode 100644 packages/core/__tests__/clients/middleware/retry/defaultRetryDecider.test.ts create mode 100644 packages/core/src/clients/middleware/retry/types.ts create mode 100644 packages/storage/__tests__/providers/s3/utils/client/S3/utils/retryDecider.test.ts create mode 100644 packages/storage/src/providers/s3/utils/client/utils/retryDecider.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 6f7e403e420..3d1f5cef58a 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -293,31 +293,31 @@ "name": "[Analytics] record (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ record }", - "limit": "17.23 kB" + "limit": "17.28 kB" }, { "name": "[Analytics] record (Kinesis)", "path": "./dist/esm/analytics/kinesis/index.mjs", "import": "{ record }", - "limit": "48.65 kB" + "limit": "48.69 kB" }, { "name": "[Analytics] record (Kinesis Firehose)", "path": "./dist/esm/analytics/kinesis-firehose/index.mjs", "import": "{ record }", - "limit": "45.81 kB" + "limit": "45.85 kB" }, { "name": "[Analytics] record (Personalize)", "path": "./dist/esm/analytics/personalize/index.mjs", "import": "{ record }", - "limit": "49.63 kB" + "limit": "49.67 kB" }, { "name": "[Analytics] identifyUser (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ identifyUser }", - "limit": "15.73 kB" + "limit": "15.79 kB" }, { "name": "[Analytics] enable", @@ -335,7 +335,7 @@ "name": "[API] generateClient (AppSync)", "path": "./dist/esm/api/index.mjs", "import": "{ generateClient }", - "limit": "40.19 kB" + "limit": "40.23 kB" }, { "name": "[API] REST API handlers", @@ -353,61 +353,61 @@ "name": "[Auth] resetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resetPassword }", - "limit": "12.58 kB" + "limit": "12.62 kB" }, { "name": "[Auth] confirmResetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmResetPassword }", - "limit": "12.52 kB" + "limit": "12.56 kB" }, { "name": "[Auth] signIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signIn }", - "limit": "30.00 kB" + "limit": "28.78 kB" }, { "name": "[Auth] resendSignUpCode (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resendSignUpCode }", - "limit": "12.53 kB" + "limit": "12.57 kB" }, { "name": "[Auth] confirmSignUp (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignUp }", - "limit": "31.00 kB" + "limit": "29.40 kB" }, { "name": "[Auth] confirmSignIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignIn }", - "limit": "28.42 kB" + "limit": "28.46 kB" }, { "name": "[Auth] updateMFAPreference (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updateMFAPreference }", - "limit": "11.87 kB" + "limit": "11.92 kB" }, { "name": "[Auth] fetchMFAPreference (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ fetchMFAPreference }", - "limit": "11.91 kB" + "limit": "11.94 kB" }, { "name": "[Auth] verifyTOTPSetup (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ verifyTOTPSetup }", - "limit": "12.75 kB" + "limit": "12.78 kB" }, { "name": "[Auth] updatePassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updatePassword }", - "limit": "12.76 kB" + "limit": "12.80 kB" }, { "name": "[Auth] setUpTOTP (Cognito)", @@ -419,85 +419,85 @@ "name": "[Auth] updateUserAttributes (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updateUserAttributes }", - "limit": "11.99 kB" + "limit": "12.03 kB" }, { "name": "[Auth] getCurrentUser (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ getCurrentUser }", - "limit": "7.85 kB" + "limit": "7.86 kB" }, { "name": "[Auth] confirmUserAttribute (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmUserAttribute }", - "limit": "12.75 kB" + "limit": "12.79 kB" }, { "name": "[Auth] signInWithRedirect (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect }", - "limit": "21.19 kB" + "limit": "21.21 kB" }, { "name": "[Auth] fetchUserAttributes (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ fetchUserAttributes }", - "limit": "11.82 kB" + "limit": "11.86 kB" }, { "name": "[Auth] Basic Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signIn, signOut, fetchAuthSession, confirmSignIn }", - "limit": "30.20 kB" + "limit": "30.23 kB" }, { "name": "[Auth] OAuth Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect, signOut, fetchAuthSession }", - "limit": "21.62 kB" + "limit": "21.64 kB" }, { "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "15.24 kB" + "limit": "15.42 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.76 kB" + "limit": "15.93 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "15.03 kB" + "limit": "15.20 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "16.09 kB" + "limit": "16.26 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.65 kB" + "limit": "15.82 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "14.88 kB" + "limit": "15.05 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "20.30 kB" + "limit": "20.48 kB" } ] } diff --git a/packages/core/__tests__/clients/middleware/retry/defaultRetryDecider.test.ts b/packages/core/__tests__/clients/middleware/retry/defaultRetryDecider.test.ts new file mode 100644 index 00000000000..b33b4e27f4d --- /dev/null +++ b/packages/core/__tests__/clients/middleware/retry/defaultRetryDecider.test.ts @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { HttpResponse } from '../../../../src/clients'; +import { getRetryDecider } from '../../../../src/clients/middleware/retry'; +import { isClockSkewError } from '../../../../src/clients/middleware/retry/isClockSkewError'; + +jest.mock('../../../../src/clients/middleware/retry/isClockSkewError'); + +const mockIsClockSkewError = jest.mocked(isClockSkewError); + +describe('getRetryDecider', () => { + const mockErrorParser = jest.fn(); + const mockHttpResponse: HttpResponse = { + statusCode: 200, + headers: {}, + body: 'body' as any, + }; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should handle network errors', async () => { + expect.assertions(2); + const retryDecider = getRetryDecider(mockErrorParser); + const connectionError = Object.assign(new Error(), { + name: 'Network error', + }); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + connectionError, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }); + + describe('handling throttling errors', () => { + it.each([ + 'BandwidthLimitExceeded', + 'EC2ThrottledException', + 'LimitExceededException', + 'PriorRequestNotComplete', + 'ProvisionedThroughputExceededException', + 'RequestLimitExceeded', + 'RequestThrottled', + 'RequestThrottledException', + 'SlowDown', + 'ThrottledException', + 'Throttling', + 'ThrottlingException', + 'TooManyRequestsException', + ])('should return retryable at %s error', async errorCode => { + expect.assertions(2); + mockErrorParser.mockResolvedValueOnce({ + code: errorCode, + }); + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }); + + it('should set retryable for 402 error', async () => { + expect.assertions(2); + const retryDecider = getRetryDecider(mockErrorParser); + const { + retryable, + isCredentialsExpiredError: isInvalidCredentialsError, + } = await retryDecider( + { + ...mockHttpResponse, + statusCode: 429, + }, + undefined, + ); + expect(retryable).toBe(true); + expect(isInvalidCredentialsError).toBeFalsy(); + }); + }); + + describe('handling clockskew error', () => { + it.each([{ code: 'ClockSkew' }, { name: 'ClockSkew' }])( + 'should handle clockskew error %o', + async parsedError => { + expect.assertions(3); + mockErrorParser.mockResolvedValue(parsedError); + mockIsClockSkewError.mockReturnValue(true); + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + expect(mockIsClockSkewError).toHaveBeenCalledWith( + Object.values(parsedError)[0], + ); + }, + ); + }); + + it.each([500, 502, 503, 504])( + 'should handle server-side status code %s', + async statusCode => { + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { + ...mockHttpResponse, + statusCode, + }, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }, + ); + + it.each(['TimeoutError', 'RequestTimeout', 'RequestTimeoutException'])( + 'should handle server-side timeout error code %s', + async errorCode => { + expect.assertions(2); + mockErrorParser.mockResolvedValue({ code: errorCode }); + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }, + ); +}); diff --git a/packages/core/__tests__/clients/middleware/retry/middleware.test.ts b/packages/core/__tests__/clients/middleware/retry/middleware.test.ts index 1391f010d23..05f1b0f8de9 100644 --- a/packages/core/__tests__/clients/middleware/retry/middleware.test.ts +++ b/packages/core/__tests__/clients/middleware/retry/middleware.test.ts @@ -11,13 +11,13 @@ import { jest.spyOn(global, 'setTimeout'); jest.spyOn(global, 'clearTimeout'); -describe(`${retryMiddlewareFactory.name} middleware`, () => { +describe(`retry middleware`, () => { beforeEach(() => { jest.clearAllMocks(); }); const defaultRetryOptions = { - retryDecider: async () => true, + retryDecider: async () => ({ retryable: true }), computeDelay: () => 1, }; const defaultRequest = { url: new URL('https://a.b') }; @@ -72,7 +72,7 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { const retryableHandler = getRetryableHandler(nextHandler); const retryDecider = jest .fn() - .mockImplementation(response => response.body !== 'foo'); // retry if response is not foo + .mockImplementation(response => ({ retryable: response.body !== 'foo' })); // retry if response is not foo const resp = await retryableHandler(defaultRequest, { ...defaultRetryOptions, retryDecider, @@ -88,11 +88,9 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { .fn() .mockRejectedValue(new Error('UnretryableError')); const retryableHandler = getRetryableHandler(nextHandler); - const retryDecider = jest - .fn() - .mockImplementation( - (resp, error) => error.message !== 'UnretryableError', - ); + const retryDecider = jest.fn().mockImplementation((resp, error) => ({ + retryable: error.message !== 'UnretryableError', + })); try { await retryableHandler(defaultRequest, { ...defaultRetryOptions, @@ -103,11 +101,46 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { expect(e.message).toBe('UnretryableError'); expect(nextHandler).toHaveBeenCalledTimes(1); expect(retryDecider).toHaveBeenCalledTimes(1); - expect(retryDecider).toHaveBeenCalledWith(undefined, expect.any(Error)); + expect(retryDecider).toHaveBeenCalledWith( + undefined, + expect.any(Error), + expect.anything(), + ); } expect.assertions(4); }); + test('should set isCredentialsExpired in middleware context if retry decider returns the flag', async () => { + expect.assertions(4); + const coreHandler = jest + .fn() + .mockRejectedValueOnce(new Error('InvalidSignature')) + .mockResolvedValueOnce(defaultResponse); + + const nextMiddleware = jest.fn( + (next: MiddlewareHandler) => (request: any) => next(request), + ); + const retryableHandler = composeTransferHandler<[RetryOptions, any]>( + coreHandler, + [retryMiddlewareFactory, () => nextMiddleware], + ); + const retryDecider = jest.fn().mockImplementation((resp, error) => ({ + retryable: error?.message === 'InvalidSignature', + isCredentialsExpiredError: error?.message === 'InvalidSignature', + })); + const response = await retryableHandler(defaultRequest, { + ...defaultRetryOptions, + retryDecider, + }); + expect(response).toEqual(expect.objectContaining(defaultResponse)); + expect(coreHandler).toHaveBeenCalledTimes(2); + expect(retryDecider).toHaveBeenCalledTimes(2); + expect(nextMiddleware).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ isCredentialsExpired: true }), + ); + }); + test('should call computeDelay for intervals', async () => { const nextHandler = jest.fn().mockResolvedValue(defaultResponse); const retryableHandler = getRetryableHandler(nextHandler); @@ -152,7 +185,7 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { const nextHandler = jest.fn().mockResolvedValue(defaultResponse); const retryableHandler = getRetryableHandler(nextHandler); const controller = new AbortController(); - const retryDecider = async () => true; + const retryDecider = async () => ({ retryable: true }); const computeDelay = jest.fn().mockImplementation(attempt => { if (attempt === 1) { setTimeout(() => { @@ -204,9 +237,10 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { const retryDecider = jest .fn() .mockImplementation((response, error: Error) => { - if (error && error.message.endsWith('RetryableError')) return true; + if (error && error.message.endsWith('RetryableError')) + return { retryable: true }; - return false; + return { retryable: false }; }); const computeDelay = jest.fn().mockReturnValue(0); const response = await doubleRetryableHandler(defaultRequest, { diff --git a/packages/core/__tests__/clients/middleware/signing/middleware.test.ts b/packages/core/__tests__/clients/middleware/signing/middleware.test.ts index a3183ebcdb5..874d82e2282 100644 --- a/packages/core/__tests__/clients/middleware/signing/middleware.test.ts +++ b/packages/core/__tests__/clients/middleware/signing/middleware.test.ts @@ -11,6 +11,7 @@ import { getUpdatedSystemClockOffset } from '../../../../src/clients/middleware/ import { HttpRequest, HttpResponse, + Middleware, MiddlewareHandler, } from '../../../../src/clients/types'; @@ -113,6 +114,30 @@ describe('Signing middleware', () => { expect(credentialsProvider).toHaveBeenCalledTimes(1); }); + test('should forceRefresh credentials provider if middleware context isCredentialsInvalid flag is set', async () => { + expect.assertions(2); + const credentialsProvider = jest.fn().mockResolvedValue(credentials); + const nextHandler = jest.fn().mockResolvedValue(defaultResponse); + const setInvalidCredsMiddleware: Middleware = + () => (next, context) => request => { + context.isCredentialsExpired = true; + + return next(request); + }; + const signableHandler = composeTransferHandler< + [any, SigningOptions], + HttpRequest, + HttpResponse + >(nextHandler, [setInvalidCredsMiddleware, signingMiddlewareFactory]); + const config = { + ...defaultSigningOptions, + credentials: credentialsProvider, + }; + await signableHandler(defaultRequest, config); + expect(credentialsProvider).toHaveBeenCalledTimes(1); + expect(credentialsProvider).toHaveBeenCalledWith({ forceRefresh: true }); + }); + test.each([ ['response with Date header', 'Date'], ['response with date header', 'date'], @@ -128,6 +153,7 @@ describe('Signing middleware', () => { const middlewareFunction = signingMiddlewareFactory(defaultSigningOptions)( nextHandler, + {}, ); await middlewareFunction(defaultRequest); diff --git a/packages/core/src/clients/index.ts b/packages/core/src/clients/index.ts index a06067604bc..31abf267c77 100644 --- a/packages/core/src/clients/index.ts +++ b/packages/core/src/clients/index.ts @@ -15,9 +15,14 @@ export { } from './middleware/signing/signer/signatureV4'; export { EMPTY_HASH as EMPTY_SHA256_HASH } from './middleware/signing/signer/signatureV4/constants'; export { extendedEncodeURIComponent } from './middleware/signing/utils/extendedEncodeURIComponent'; -export { signingMiddlewareFactory, SigningOptions } from './middleware/signing'; +export { + signingMiddlewareFactory, + SigningOptions, + CredentialsProviderOptions, +} from './middleware/signing'; export { getRetryDecider, + RetryDeciderOutput, jitteredBackoff, retryMiddlewareFactory, RetryOptions, diff --git a/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts b/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts index 874cc74314e..edec193ebf1 100644 --- a/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts +++ b/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts @@ -4,6 +4,7 @@ import { ErrorParser, HttpResponse } from '../../types'; import { isClockSkewError } from './isClockSkewError'; +import { RetryDeciderOutput } from './types'; /** * Get retry decider function @@ -11,7 +12,10 @@ import { isClockSkewError } from './isClockSkewError'; */ export const getRetryDecider = (errorParser: ErrorParser) => - async (response?: HttpResponse, error?: unknown): Promise => { + async ( + response?: HttpResponse, + error?: unknown, + ): Promise => { const parsedError = (error as Error & { code: string }) ?? (await errorParser(response)) ?? @@ -19,12 +23,15 @@ export const getRetryDecider = const errorCode = parsedError?.code || parsedError?.name; const statusCode = response?.statusCode; - return ( + const isRetryable = isConnectionError(error) || isThrottlingError(statusCode, errorCode) || isClockSkewError(errorCode) || - isServerSideError(statusCode, errorCode) - ); + isServerSideError(statusCode, errorCode); + + return { + retryable: isRetryable, + }; }; // reference: https://github.com/aws/aws-sdk-js-v3/blob/ab0e7be36e7e7f8a0c04834357aaad643c7912c3/packages/service-error-classification/src/constants.ts#L22-L37 diff --git a/packages/core/src/clients/middleware/retry/index.ts b/packages/core/src/clients/middleware/retry/index.ts index 4c82c603508..fdf34552fa7 100644 --- a/packages/core/src/clients/middleware/retry/index.ts +++ b/packages/core/src/clients/middleware/retry/index.ts @@ -4,3 +4,4 @@ export { RetryOptions, retryMiddlewareFactory } from './middleware'; export { jitteredBackoff } from './jitteredBackoff'; export { getRetryDecider } from './defaultRetryDecider'; +export { RetryDeciderOutput } from './types'; diff --git a/packages/core/src/clients/middleware/retry/middleware.ts b/packages/core/src/clients/middleware/retry/middleware.ts index 1c8d88bc4fd..8d9a9c2cd9b 100644 --- a/packages/core/src/clients/middleware/retry/middleware.ts +++ b/packages/core/src/clients/middleware/retry/middleware.ts @@ -8,6 +8,8 @@ import { Response, } from '../../types/core'; +import { RetryDeciderOutput } from './types'; + const DEFAULT_RETRY_ATTEMPTS = 3; /** @@ -19,9 +21,14 @@ export interface RetryOptions { * * @param response Optional response of the request. * @param error Optional error thrown from previous attempts. + * @param middlewareContext Optional context object to store data between retries. * @returns True if the request should be retried. */ - retryDecider(response?: TResponse, error?: unknown): Promise; + retryDecider( + response?: TResponse, + error?: unknown, + middlewareContext?: MiddlewareContext, + ): Promise; /** * Function to compute the delay in milliseconds before the next retry based * on the number of attempts. @@ -87,7 +94,14 @@ export const retryMiddlewareFactory = ({ ? context.attemptsCount ?? 0 : attemptsCount + 1; context.attemptsCount = attemptsCount; - if (await retryDecider(response, error)) { + const { isCredentialsExpiredError, retryable } = await retryDecider( + response, + error, + context, + ); + if (retryable) { + // Setting isCredentialsInvalid flag to notify signing middleware to forceRefresh credentials provider. + context.isCredentialsExpired = !!isCredentialsExpiredError; if (!abortSignal?.aborted && attemptsCount < maxAttempts) { // prevent sleep for last attempt or cancelled request; const delay = computeDelay(attemptsCount); diff --git a/packages/core/src/clients/middleware/retry/types.ts b/packages/core/src/clients/middleware/retry/types.ts new file mode 100644 index 00000000000..a229216edee --- /dev/null +++ b/packages/core/src/clients/middleware/retry/types.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface RetryDeciderOutput { + retryable: boolean; + isCredentialsExpiredError?: boolean; +} diff --git a/packages/core/src/clients/middleware/signing/index.ts b/packages/core/src/clients/middleware/signing/index.ts index a1458bca3e4..1ce90db4b7e 100644 --- a/packages/core/src/clients/middleware/signing/index.ts +++ b/packages/core/src/clients/middleware/signing/index.ts @@ -1,4 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -export { signingMiddlewareFactory, SigningOptions } from './middleware'; +export { + signingMiddlewareFactory, + SigningOptions, + CredentialsProviderOptions, +} from './middleware'; diff --git a/packages/core/src/clients/middleware/signing/middleware.ts b/packages/core/src/clients/middleware/signing/middleware.ts index a7bed1e6b7f..1b36519729e 100644 --- a/packages/core/src/clients/middleware/signing/middleware.ts +++ b/packages/core/src/clients/middleware/signing/middleware.ts @@ -7,16 +7,27 @@ import { HttpResponse, MiddlewareHandler, } from '../../types'; +import { MiddlewareContext } from '../../types/core'; import { signRequest } from './signer/signatureV4'; import { getSkewCorrectedDate } from './utils/getSkewCorrectedDate'; import { getUpdatedSystemClockOffset } from './utils/getUpdatedSystemClockOffset'; +/** + * Options type for the async callback function returning aws credentials. This + * function is used by SigV4 signer to resolve the aws credentials + */ +export interface CredentialsProviderOptions { + forceRefresh?: boolean; +} + /** * Configuration of the signing middleware */ export interface SigningOptions { - credentials: Credentials | (() => Promise); + credentials: + | Credentials + | ((options?: CredentialsProviderOptions) => Promise); region: string; service: string; @@ -41,12 +52,19 @@ export const signingMiddlewareFactory = ({ }: SigningOptions) => { let currentSystemClockOffset: number; - return (next: MiddlewareHandler) => + return ( + next: MiddlewareHandler, + context: MiddlewareContext, + ) => async function signingMiddleware(request: HttpRequest) { currentSystemClockOffset = currentSystemClockOffset ?? 0; const signRequestOptions = { credentials: - typeof credentials === 'function' ? await credentials() : credentials, + typeof credentials === 'function' + ? await credentials({ + forceRefresh: !!context?.isCredentialsExpired, + }) + : credentials, signingDate: getSkewCorrectedDate(currentSystemClockOffset), signingRegion: region, signingService: service, diff --git a/packages/core/src/clients/types/core.ts b/packages/core/src/clients/types/core.ts index 1fa122250b6..a6348655899 100644 --- a/packages/core/src/clients/types/core.ts +++ b/packages/core/src/clients/types/core.ts @@ -30,6 +30,11 @@ export type MiddlewareHandler = ( * The context object to store states across the middleware chain. */ export interface MiddlewareContext { + /** + * Whether an error indicating expired credentials has been returned from server-side. + * This is set by the retry middleware. + */ + isCredentialsExpired?: boolean; /** * The number of times the request has been attempted. This is set by retry middleware */ diff --git a/packages/core/src/clients/types/index.ts b/packages/core/src/clients/types/index.ts index e2b8953a4d2..0ee905fb162 100644 --- a/packages/core/src/clients/types/index.ts +++ b/packages/core/src/clients/types/index.ts @@ -4,6 +4,7 @@ export { Middleware, MiddlewareHandler, + MiddlewareContext, Request, Response, TransferHandler, diff --git a/packages/storage/__tests__/providers/s3/apis/copy.test.ts b/packages/storage/__tests__/providers/s3/apis/copy.test.ts index 56f46927b30..56104e84d17 100644 --- a/packages/storage/__tests__/providers/s3/apis/copy.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/copy.test.ts @@ -384,7 +384,6 @@ describe('copy API', () => { }, }); } catch (error: any) { - console.log(error); expect(error).toBeInstanceOf(StorageError); expect(error.name).toBe( StorageValidationErrorCode.InvalidCopyOperationStorageBucket, @@ -403,7 +402,6 @@ describe('copy API', () => { }, }); } catch (error: any) { - console.log(error); expect(error).toBeInstanceOf(StorageError); expect(error.name).toBe( StorageValidationErrorCode.InvalidCopyOperationStorageBucket, diff --git a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts index 527e66dce18..662640e3340 100644 --- a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts @@ -241,8 +241,10 @@ describe('resolveS3ConfigAndInput', () => { }); if (typeof s3Config.credentials === 'function') { - const result = await s3Config.credentials(); - expect(mockLocationCredentialsProvider).toHaveBeenCalled(); + const result = await s3Config.credentials({ forceRefresh: true }); + expect(mockLocationCredentialsProvider).toHaveBeenCalledWith({ + forceRefresh: true, + }); expect(result).toEqual(credentials); } else { throw new Error('Expect credentials to be a function'); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts index d94e6b94d34..6b20ab56254 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts @@ -109,7 +109,12 @@ const completeMultipartUploadErrorWith200CodeCase: ApiFunctionalTestCase< 'error case', 'completeMultipartUpload with 200 status', completeMultipartUpload, - { ...defaultConfig, retryDecider: async () => false }, // disable retry + { + ...defaultConfig, + retryDecider: async () => ({ + retryable: false, + }), + }, // disable retry completeMultipartUploadHappyCase[4], completeMultipartUploadHappyCase[5], { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/utils/retryDecider.test.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/utils/retryDecider.test.ts new file mode 100644 index 00000000000..5e1801c07db --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/utils/retryDecider.test.ts @@ -0,0 +1,103 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { + HttpResponse, + getRetryDecider as getDefaultRetryDecider, +} from '@aws-amplify/core/internals/aws-client-utils'; + +import { retryDecider } from '../../../../../../../src/providers/s3/utils/client/utils'; +import { parseXmlError } from '../../../../../../../src/providers/s3/utils/client/utils/parsePayload'; + +jest.mock( + '../../../../../../../src/providers/s3/utils/client/utils/parsePayload', +); +jest.mock('@aws-amplify/core/internals/aws-client-utils'); + +const mockErrorParser = jest.mocked(parseXmlError); + +describe('retryDecider', () => { + const mockHttpResponse: HttpResponse = { + statusCode: 200, + headers: {}, + body: 'body' as any, + }; + + beforeEach(() => { + jest.mocked(getDefaultRetryDecider).mockReturnValue(async () => { + return { retryable: false }; + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should invoke the default retry decider', async () => { + expect.assertions(3); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + {}, + ); + expect(getDefaultRetryDecider).toHaveBeenCalledWith(mockErrorParser); + expect(retryable).toBe(false); + expect(isCredentialsExpiredError).toBeFalsy(); + }); + + describe('handling expired token errors', () => { + const mockErrorMessage = 'Token expired'; + it.each(['RequestExpired', 'ExpiredTokenException', 'ExpiredToken'])( + 'should retry if expired credentials error name %s', + async errorName => { + expect.assertions(2); + const parsedError = { + name: errorName, + message: mockErrorMessage, + $metadata: {}, + }; + mockErrorParser.mockResolvedValue(parsedError); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { ...mockHttpResponse, statusCode: 400 }, + undefined, + {}, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBe(true); + }, + ); + + it('should retry if error message indicates invalid credentials', async () => { + expect.assertions(2); + const parsedError = { + name: 'InvalidSignature', + message: 'Auth token in request is expired.', + $metadata: {}, + }; + mockErrorParser.mockResolvedValue(parsedError); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { ...mockHttpResponse, statusCode: 400 }, + undefined, + {}, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBe(true); + }); + + it('should not retry if invalid credentials error has been retried previously', async () => { + expect.assertions(2); + const parsedError = { + name: 'RequestExpired', + message: mockErrorMessage, + $metadata: {}, + }; + mockErrorParser.mockResolvedValue(parsedError); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { ...mockHttpResponse, statusCode: 400 }, + undefined, + { isCredentialsExpired: true }, + ); + expect(retryable).toBe(false); + expect(isCredentialsExpiredError).toBe(true); + }); + }); +}); diff --git a/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts b/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts index 0753e0ae334..91a3fd12556 100644 --- a/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts +++ b/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts @@ -1,6 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; + import { getDataAccess } from '../../../src/storageBrowser/apis/getDataAccess'; import { getDataAccess as getDataAccessClient } from '../../../src/providers/s3/utils/client/s3control'; import { GetDataAccessInput } from '../../../src/storageBrowser/apis/types'; @@ -29,7 +31,7 @@ const MOCK_ACCESS_CREDENTIALS = { SessionToken: MOCK_SESSION_TOKEN, Expiration: MOCK_EXPIRATION_DATE, }; -const MOCK_CREDENTIAL_PROVIDER = async () => MOCK_CREDENTIALS; +const MOCK_CREDENTIAL_PROVIDER = jest.fn().mockResolvedValue(MOCK_CREDENTIALS); const sharedGetDataAccessParams: GetDataAccessInput = { accountId: MOCK_ACCOUNT_ID, @@ -41,7 +43,7 @@ const sharedGetDataAccessParams: GetDataAccessInput = { }; describe('getDataAccess', () => { - const getDataAccessClientMock = getDataAccessClient as jest.Mock; + const getDataAccessClientMock = jest.mocked(getDataAccessClient); beforeEach(() => { jest.clearAllMocks(); @@ -49,15 +51,17 @@ describe('getDataAccess', () => { getDataAccessClientMock.mockResolvedValue({ Credentials: MOCK_ACCESS_CREDENTIALS, MatchedGrantTarget: MOCK_SCOPE, + $metadata: {}, }); }); it('should invoke the getDataAccess client correctly', async () => { + expect.assertions(6); const result = await getDataAccess(sharedGetDataAccessParams); expect(getDataAccessClientMock).toHaveBeenCalledWith( expect.objectContaining({ - credentials: MOCK_CREDENTIALS.credentials, + credentials: expect.any(Function), region: MOCK_REGION, userAgentValue: expect.stringContaining('storage/8'), }), @@ -69,6 +73,15 @@ describe('getDataAccess', () => { DurationSeconds: 900, }), ); + const inputCredentialsProvider = getDataAccessClientMock.mock.calls[0][0] + .credentials as (input: CredentialsProviderOptions) => any; + expect(inputCredentialsProvider).toBeInstanceOf(Function); + await expect( + inputCredentialsProvider({ forceRefresh: true }), + ).resolves.toEqual(MOCK_CREDENTIALS.credentials); + expect(MOCK_CREDENTIAL_PROVIDER).toHaveBeenCalledWith({ + forceRefresh: true, + }); expect(result.credentials).toEqual(MOCK_CREDENTIALS.credentials); expect(result.scope).toEqual(MOCK_SCOPE); @@ -80,6 +93,7 @@ describe('getDataAccess', () => { getDataAccessClientMock.mockResolvedValue({ Credentials: undefined, MatchedGrantTarget: MOCK_SCOPE, + $metadata: {}, }); expect(getDataAccess(sharedGetDataAccessParams)).rejects.toThrow( @@ -93,6 +107,7 @@ describe('getDataAccess', () => { getDataAccessClientMock.mockResolvedValue({ Credentials: MOCK_ACCESS_CREDENTIALS, MatchedGrantTarget: MOCK_OBJECT_SCOPE, + $metadata: {}, }); const result = await getDataAccess({ diff --git a/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts index 3e0051f7461..bff4b4e07bd 100644 --- a/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts +++ b/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; + import { listCallerAccessGrants } from '../../../src/storageBrowser/apis/listCallerAccessGrants'; import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../../src/providers/s3/utils/client/s3control'; @@ -8,7 +10,15 @@ jest.mock('../../../src/providers/s3/utils/client/s3control'); const mockAccountId = '1234567890'; const mockRegion = 'us-foo-2'; -const mockCredentialsProvider = jest.fn(); +const mockCredentials = { + accessKeyId: 'key', + secretAccessKey: 'secret', + sessionToken: 'session', + expiration: new Date(), +}; +const mockCredentialsProvider = jest + .fn() + .mockResolvedValue({ credentials: mockCredentials }); const mockNextToken = '123'; const mockPageSize = 123; @@ -18,7 +28,7 @@ describe('listCallerAccessGrants', () => { }); it('should invoke the listCallerAccessGrants client with expected parameters', async () => { - expect.assertions(1); + expect.assertions(4); jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ NextToken: undefined, CallerAccessGrantsList: [], @@ -42,6 +52,17 @@ describe('listCallerAccessGrants', () => { MaxResults: mockPageSize, }), ); + const inputCredentialsProvider = jest.mocked(listCallerAccessGrantsClient) + .mock.calls[0][0].credentials as ( + input: CredentialsProviderOptions, + ) => any; + expect(inputCredentialsProvider).toBeInstanceOf(Function); + await expect( + inputCredentialsProvider({ forceRefresh: true }), + ).resolves.toEqual(mockCredentials); + expect(mockCredentialsProvider).toHaveBeenCalledWith({ + forceRefresh: true, + }); }); it('should set a default page size', async () => { diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 170cafd7313..0852c761891 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -3,7 +3,10 @@ import { StorageAccessLevel } from '@aws-amplify/core'; import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { SigningOptions } from '@aws-amplify/core/internals/aws-client-utils'; +import { + CredentialsProviderOptions, + SigningOptions, +} from '@aws-amplify/core/internals/aws-client-utils'; import { TransferProgressEvent } from '../../../types'; import { @@ -15,9 +18,9 @@ import { /** * @internal */ -export type LocationCredentialsProvider = (options?: { - forceRefresh?: boolean; -}) => Promise<{ credentials: AWSCredentials }>; +export type LocationCredentialsProvider = ( + input?: CredentialsProviderOptions, +) => Promise<{ credentials: AWSCredentials }>; export interface BucketInfo { bucketName: string; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts index 380488bf4ac..a40f9f6a5dd 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -8,11 +8,10 @@ import { import { EndpointResolverOptions, getDnsSuffix, - getRetryDecider, jitteredBackoff, } from '@aws-amplify/core/internals/aws-client-utils'; -import { parseXmlError } from '../utils'; +import { retryDecider } from '../utils'; /** * The service name used to sign requests if the API requires authentication. @@ -64,7 +63,7 @@ const endpointResolver = ( export const defaultConfig = { service: SERVICE_NAME, endpointResolver, - retryDecider: getRetryDecider(parseXmlError), + retryDecider, computeDelay: jitteredBackoff, userAgentValue: getAmplifyUserAgent(), uriEscapePath: false, // Required by S3. See https://github.com/aws/aws-sdk-js-v3/blob/9ba012dfa3a3429aa2db0f90b3b0b3a7a31f9bc3/packages/signature-v4/src/SignatureV4.ts#L76-L83 diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index a31d6d5a2f1..d51c3a18a11 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -8,11 +8,10 @@ import { import { EndpointResolverOptions, getDnsSuffix, - getRetryDecider, jitteredBackoff, } from '@aws-amplify/core/internals/aws-client-utils'; -import { parseXmlError } from '../utils'; +import { retryDecider } from '../utils'; const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/; const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; @@ -106,7 +105,7 @@ export const isDnsCompatibleBucketName = (bucketName: string): boolean => export const defaultConfig = { service: SERVICE_NAME, endpointResolver, - retryDecider: getRetryDecider(parseXmlError), + retryDecider, computeDelay: jitteredBackoff, userAgentValue: getAmplifyUserAgent(), useAccelerateEndpoint: false, diff --git a/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts index 59a8e029afc..1e399e824e7 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts @@ -5,6 +5,8 @@ import { Endpoint, HttpRequest, HttpResponse, + MiddlewareContext, + RetryDeciderOutput, parseMetadata, } from '@aws-amplify/core/internals/aws-client-utils'; import { @@ -18,6 +20,7 @@ import { map, parseXmlBody, parseXmlError, + retryDecider, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, @@ -136,25 +139,24 @@ const completeMultipartUploadDeserializer = async ( const retryWhenErrorWith200StatusCode = async ( response?: HttpResponse, error?: unknown, -): Promise => { + middlewareContext?: MiddlewareContext, +): Promise => { if (!response) { - return false; + return { retryable: false }; } if (response.statusCode === 200) { if (!response.body) { - return true; + return { retryable: true }; } const parsed = await parseXmlBody(response); if (parsed.Code !== undefined && parsed.Message !== undefined) { - return true; + return { retryable: true }; } - return false; + return { retryable: false }; } - const defaultRetryDecider = defaultConfig.retryDecider; - - return defaultRetryDecider(response, error); + return retryDecider(response, error, middlewareContext); }; export const completeMultipartUpload = composeServiceApi( diff --git a/packages/storage/src/providers/s3/utils/client/utils/index.ts b/packages/storage/src/providers/s3/utils/client/utils/index.ts index abfe9328d45..423987699f8 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/index.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/index.ts @@ -25,3 +25,4 @@ export { serializePathnameObjectKey, validateS3RequiredParameter, } from './serializeHelpers'; +export { retryDecider } from './retryDecider'; diff --git a/packages/storage/src/providers/s3/utils/client/utils/retryDecider.ts b/packages/storage/src/providers/s3/utils/client/utils/retryDecider.ts new file mode 100644 index 00000000000..3e1e0fcc3da --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/utils/retryDecider.ts @@ -0,0 +1,81 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { + HttpResponse, + MiddlewareContext, + RetryDeciderOutput, + getRetryDecider, +} from '@aws-amplify/core/internals/aws-client-utils'; + +import { LocationCredentialsProvider } from '../../../types/options'; + +import { parseXmlError } from './parsePayload'; + +/** + * Function to decide if the S3 request should be retried. For S3 APIs, we support forceRefresh option + * for {@link LocationCredentialsProvider | LocationCredentialsProvider } option. It's set when S3 returns + * credentials expired error. In the retry decider, we detect this response and set flag to signify a retry + * attempt. The retry attempt would invoke the LocationCredentialsProvider with forceRefresh option set. + * + * @param response Optional response of the request. + * @param error Optional error thrown from previous attempts. + * @param middlewareContext Optional context object to store data between retries. + * @returns True if the request should be retried. + */ +export const retryDecider = async ( + response?: HttpResponse, + error?: unknown, + middlewareContext?: MiddlewareContext, +): Promise => { + const defaultRetryDecider = getRetryDecider(parseXmlError); + const defaultRetryDecision = await defaultRetryDecider(response, error); + if (!response || response.statusCode < 300) { + return { retryable: false }; + } + const parsedError = await parseXmlError(response); + const errorCode = parsedError?.name; + const errorMessage = parsedError?.message; + const isCredentialsExpired = isCredentialsExpiredError( + errorCode, + errorMessage, + ); + + return { + retryable: + defaultRetryDecision.retryable || + // If we know the previous retry attempt sets isCredentialsExpired in the + // middleware context, we don't want to retry anymore. + !!(isCredentialsExpired && !middlewareContext?.isCredentialsExpired), + isCredentialsExpiredError: isCredentialsExpired, + }; +}; + +// Ref: https://github.com/aws/aws-sdk-js/blob/54829e341181b41573c419bd870dd0e0f8f10632/lib/event_listeners.js#L522-L541 +const INVALID_TOKEN_ERROR_CODES = [ + 'RequestExpired', + 'ExpiredTokenException', + 'ExpiredToken', +]; + +/** + * Given an error code, returns true if it is related to invalid credentials. + * + * @param errorCode String representation of some error. + * @returns True if given error indicates the credentials used to authorize request + * are invalid. + */ +const isCredentialsExpiredError = ( + errorCode?: string, + errorMessage?: string, +) => { + const isExpiredTokenError = + !!errorCode && INVALID_TOKEN_ERROR_CODES.includes(errorCode); + // Ref: https://github.com/aws/aws-sdk-js/blob/54829e341181b41573c419bd870dd0e0f8f10632/lib/event_listeners.js#L536-L539 + const isExpiredSignatureError = + !!errorCode && + !!errorMessage && + errorCode.includes('Signature') && + errorMessage.includes('expired'); + + return isExpiredTokenError || isExpiredSignatureError; +}; diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index ef3b4621c33..c07888871f6 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { AmplifyClassV6, StorageAccessLevel } from '@aws-amplify/core'; +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; import { assertValidationError } from '../../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../../errors/types/validation'; @@ -76,14 +77,20 @@ export const resolveS3ConfigAndInput = async ( * used because the long-running tasks like multipart upload may span over the * credentials expiry. Auth.fetchAuthSession() automatically refreshes the * credentials if they are expired. + * + * The optional forceRefresh option is set when the S3 service returns expired + * tokens error in the previous API call attempt. */ - const credentialsProvider = async () => { + const credentialsProvider = async (options?: CredentialsProviderOptions) => { if (isLocationCredentialsProvider(apiOptions)) { assertStorageInput(apiInput); } + // TODO: forceRefresh option of fetchAuthSession would refresh both tokens and + // AWS credentials. So we do not support forceRefreshing from the Auth until + // we support refreshing only the credentials. const { credentials } = isLocationCredentialsProvider(apiOptions) - ? await apiOptions.locationCredentialsProvider() + ? await apiOptions.locationCredentialsProvider(options) : await amplify.Auth.fetchAuthSession(); assertValidationError( !!credentials, diff --git a/packages/storage/src/storageBrowser/apis/getDataAccess.ts b/packages/storage/src/storageBrowser/apis/getDataAccess.ts index 5e5bec23540..440a83e08cc 100644 --- a/packages/storage/src/storageBrowser/apis/getDataAccess.ts +++ b/packages/storage/src/storageBrowser/apis/getDataAccess.ts @@ -5,6 +5,7 @@ import { AmplifyErrorCode, StorageAction, } from '@aws-amplify/core/internals/utils'; +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; import { getDataAccess as getDataAccessClient } from '../../providers/s3/utils/client/s3control'; @@ -18,11 +19,17 @@ export const getDataAccess = async ( input: GetDataAccessInput, ): Promise => { const targetType = input.scope.endsWith('*') ? undefined : 'Object'; - const { credentials } = await input.credentialsProvider(); + const clientCredentialsProvider = async ( + options?: CredentialsProviderOptions, + ) => { + const { credentials } = await input.credentialsProvider(options); + + return credentials; + }; const result = await getDataAccessClient( { - credentials, + credentials: clientCredentialsProvider, region: input.region, userAgentValue: getStorageUserAgentValue(StorageAction.GetDataAccess), }, diff --git a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts index 957e6eb1fcb..12836d59880 100644 --- a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { StorageAction } from '@aws-amplify/core/internals/utils'; +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; import { logger } from '../../utils'; import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../providers/s3/utils/client/s3control'; @@ -26,8 +27,10 @@ export const listCallerAccessGrants = async ( logger.debug(`defaulting pageSize to ${MAX_PAGE_SIZE}.`); } - const clientCredentialsProvider = async () => { - const { credentials } = await credentialsProvider(); + const clientCredentialsProvider = async ( + options?: CredentialsProviderOptions, + ) => { + const { credentials } = await credentialsProvider(options); return credentials; }; diff --git a/packages/storage/src/storageBrowser/types.ts b/packages/storage/src/storageBrowser/types.ts index c770b7472a3..a09492bfb65 100644 --- a/packages/storage/src/storageBrowser/types.ts +++ b/packages/storage/src/storageBrowser/types.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { AWSCredentials } from '@aws-amplify/core/internals/utils'; +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; import { LocationCredentialsProvider } from '../providers/s3/types/options'; @@ -13,9 +14,9 @@ export type Permission = 'READ' | 'READWRITE' | 'WRITE'; /** * @internal */ -export type CredentialsProvider = (options?: { - forceRefresh?: boolean; -}) => Promise<{ credentials: AWSCredentials }>; +export type CredentialsProvider = ( + options?: CredentialsProviderOptions, +) => Promise<{ credentials: AWSCredentials }>; /** * @internal From 30404f44e57cc4586b559de8577892c290889866 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Wed, 31 Jul 2024 18:00:28 -0700 Subject: [PATCH 28/40] feat(storage): require temporary creds for storage browser interfaces (#13664) --- .../storageBrowser/apis/getDataAccess.test.ts | 2 +- .../locationCredentialsStore/create.test.ts | 4 ++-- .../locationCredentialsStore/registry.test.ts | 5 ++--- .../locationCredentialsStore/store.test.ts | 11 +++++------ .../storage/src/providers/s3/types/options.ts | 14 ++++++++++++-- .../src/storageBrowser/apis/getDataAccess.ts | 14 ++++++++++---- .../locationCredentialsStore/registry.ts | 5 ++--- .../locationCredentialsStore/store.ts | 18 +++++++----------- packages/storage/src/storageBrowser/types.ts | 14 ++++++-------- 9 files changed, 47 insertions(+), 40 deletions(-) diff --git a/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts b/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts index 91a3fd12556..c43f9b004ba 100644 --- a/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts +++ b/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts @@ -97,7 +97,7 @@ describe('getDataAccess', () => { }); expect(getDataAccess(sharedGetDataAccessParams)).rejects.toThrow( - 'Service did not return credentials.', + 'Service did not return valid temporary credentials.', ); }); diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts index 6405cdb5487..1bebd3772bf 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts @@ -1,6 +1,5 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { createLocationCredentialsStore } from '../../../src/storageBrowser/locationCredentialsStore/create'; import { @@ -13,10 +12,11 @@ import { StorageValidationErrorCode, validationErrorMap, } from '../../../src/errors/types/validation'; +import { AWSTemporaryCredentials } from '../../../src/providers/s3/types/options'; jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); -const mockedCredentials = 'MOCK_CREDS' as any as AWSCredentials; +const mockedCredentials = 'MOCK_CREDS' as any as AWSTemporaryCredentials; describe('createLocationCredentialsStore', () => { it('should create a store', () => { diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts index 0d36aaba8b9..9f2cfcfc742 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts @@ -1,12 +1,11 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; - import { StorageValidationErrorCode, validationErrorMap, } from '../../../src/errors/types/validation'; +import { AWSTemporaryCredentials } from '../../../src/providers/s3/types/options'; import { createStore, getValue, @@ -47,7 +46,7 @@ describe('createStore', () => { }); describe('getValue', () => { - const mockCachedValue = 'CACHED_VALUE' as any as AWSCredentials; + const mockCachedValue = 'CACHED_VALUE' as any as AWSTemporaryCredentials; let storeSymbol: { value: symbol }; beforeEach(() => { storeSymbol = createStore(jest.fn(), 20); diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts index 096b50f07b5..81ca4702417 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts +++ b/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts @@ -12,6 +12,10 @@ import { } from '../../../src/storageBrowser/locationCredentialsStore/store'; import { CredentialsLocation } from '../../../src/storageBrowser/types'; +const mockCredentials = { + expiration: new Date(Date.now() + 60 * 60_1000), +}; + describe('initStore', () => { it('should create a store with given capacity, refresh Handler and values', () => { const refreshHandler = jest.fn(); @@ -42,7 +46,7 @@ describe('initStore', () => { describe('getCacheValue', () => { it('should return a cache value for given location and permission', () => { const cachedValue = { - credentials: 'MOCK_CREDS', + credentials: mockCredentials, scope: 'abc', permission: 'READ', } as any; @@ -114,7 +118,6 @@ describe('fetchNewValue', () => { it('should fetch new value from remote source', async () => { expect.assertions(2); - const mockCredentials = 'MOCK_CREDS'; const refreshHandler = jest.fn().mockResolvedValue({ credentials: mockCredentials, }); @@ -143,7 +146,6 @@ describe('fetchNewValue', () => { it('should update cache with new value', async () => { expect.assertions(1); - const mockCredentials = 'MOCK_CREDS'; const refreshHandler = jest.fn().mockResolvedValue({ credentials: mockCredentials, }); @@ -159,7 +161,6 @@ describe('fetchNewValue', () => { it('should invoke refresh handler only once if multiple fetches for same location is called', async () => { expect.assertions(1); - const mockCredentials = 'MOCK_CREDS'; const refreshHandler = jest.fn().mockResolvedValue({ credentials: mockCredentials, }); @@ -174,7 +175,6 @@ describe('fetchNewValue', () => { it('should invoke the refresh handler if the refresh handler previously fails', async () => { expect.assertions(4); - const mockCredentials = 'MOCK_CREDS'; const refreshHandler = jest .fn() .mockRejectedValueOnce(new Error('Network error')) @@ -199,7 +199,6 @@ describe('fetchNewValue', () => { it('should call refresh handler for new cache entry if the cache is full', async () => { expect.assertions(4); - const mockCredentials = 'MOCK_CREDS'; const refreshHandler = jest.fn().mockResolvedValue({ credentials: mockCredentials, }); diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 0852c761891..26d529cde64 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -15,12 +15,22 @@ import { StorageSubpathStrategy, } from '../../../types/options'; +/** + * @internal + */ +export type AWSTemporaryCredentials = Required< + Pick< + AWSCredentials, + 'accessKeyId' | 'secretAccessKey' | 'sessionToken' | 'expiration' + > +>; + /** * @internal */ export type LocationCredentialsProvider = ( - input?: CredentialsProviderOptions, -) => Promise<{ credentials: AWSCredentials }>; + options?: CredentialsProviderOptions, +) => Promise<{ credentials: AWSTemporaryCredentials }>; export interface BucketInfo { bucketName: string; diff --git a/packages/storage/src/storageBrowser/apis/getDataAccess.ts b/packages/storage/src/storageBrowser/apis/getDataAccess.ts index 440a83e08cc..b5d511c8e0d 100644 --- a/packages/storage/src/storageBrowser/apis/getDataAccess.ts +++ b/packages/storage/src/storageBrowser/apis/getDataAccess.ts @@ -45,10 +45,16 @@ export const getDataAccess = async ( const grantCredentials = result.Credentials; // Ensure that S3 returned credentials (this shouldn't happen) - if (!grantCredentials) { + if ( + !grantCredentials || + !grantCredentials.AccessKeyId || + !grantCredentials.SecretAccessKey || + !grantCredentials.SessionToken || + !grantCredentials.Expiration + ) { throw new StorageError({ name: AmplifyErrorCode.Unknown, - message: 'Service did not return credentials.', + message: 'Service did not return valid temporary credentials.', }); } else { logger.debug(`Retrieved credentials for: ${result.MatchedGrantTarget}`); @@ -63,8 +69,8 @@ export const getDataAccess = async ( return { credentials: { - accessKeyId: accessKeyId!, - secretAccessKey: secretAccessKey!, + accessKeyId, + secretAccessKey, sessionToken, expiration, }, diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts index 54ed8e4db83..41b7c101b82 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts @@ -2,8 +2,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; - +import { AWSTemporaryCredentials } from '../../providers/s3/types/options'; import { CredentialsLocation, GetLocationCredentials } from '../types'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; @@ -68,7 +67,7 @@ export const getValue = async (input: { storeSymbol: StoreRegistrySymbol; location: CredentialsLocation; forceRefresh: boolean; -}): Promise<{ credentials: AWSCredentials }> => { +}): Promise<{ credentials: AWSTemporaryCredentials }> => { const { storeSymbol: storeReference, location, forceRefresh } = input; const store = getCredentialsStore(storeReference); if (!forceRefresh) { diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts index 63b88008b71..c5effe0f91e 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts @@ -3,13 +3,12 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; - import { CredentialsLocation, GetLocationCredentials, Permission, } from '../types'; +import { AWSTemporaryCredentials } from '../../providers/s3/types/options'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; @@ -19,8 +18,8 @@ import { } from './constants'; interface StoreValue extends CredentialsLocation { - credentials?: AWSCredentials; - inflightCredentials?: Promise<{ credentials: AWSCredentials }>; + credentials?: AWSTemporaryCredentials; + inflightCredentials?: Promise<{ credentials: AWSTemporaryCredentials }>; } type S3Uri = string; @@ -64,7 +63,7 @@ export const initStore = ( export const getCacheValue = ( store: LruLocationCredentialsStore, location: CredentialsLocation, -): AWSCredentials | null => { +): AWSTemporaryCredentials | null => { const cacheKey = createCacheKey(location); const cachedValue = store.values.get(cacheKey); const cachedCredentials = cachedValue?.credentials; @@ -85,13 +84,10 @@ export const getCacheValue = ( return null; }; -const pastTTL = (credentials: AWSCredentials) => { +const pastTTL = (credentials: AWSTemporaryCredentials) => { const { expiration } = credentials; - return ( - expiration && - expiration.getTime() - CREDENTIALS_REFRESH_WINDOW_MS <= Date.now() - ); + return expiration.getTime() - CREDENTIALS_REFRESH_WINDOW_MS <= Date.now(); }; /** @@ -102,7 +98,7 @@ const pastTTL = (credentials: AWSCredentials) => { export const fetchNewValue = async ( store: LruLocationCredentialsStore, location: CredentialsLocation, -): Promise<{ credentials: AWSCredentials }> => { +): Promise<{ credentials: AWSTemporaryCredentials }> => { const storeValues = store.values; const key = createCacheKey(location); if (!storeValues.has(key)) { diff --git a/packages/storage/src/storageBrowser/types.ts b/packages/storage/src/storageBrowser/types.ts index a09492bfb65..1199cf0e851 100644 --- a/packages/storage/src/storageBrowser/types.ts +++ b/packages/storage/src/storageBrowser/types.ts @@ -1,10 +1,10 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { AWSCredentials } from '@aws-amplify/core/internals/utils'; -import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; - -import { LocationCredentialsProvider } from '../providers/s3/types/options'; +import { + AWSTemporaryCredentials, + LocationCredentialsProvider, +} from '../providers/s3/types/options'; /** * @internal @@ -14,9 +14,7 @@ export type Permission = 'READ' | 'READWRITE' | 'WRITE'; /** * @internal */ -export type CredentialsProvider = ( - options?: CredentialsProviderOptions, -) => Promise<{ credentials: AWSCredentials }>; +export type CredentialsProvider = LocationCredentialsProvider; /** * @internal @@ -70,7 +68,7 @@ export interface LocationCredentials extends Partial { /** * AWS credentials which can be used to access the specified location. */ - readonly credentials: AWSCredentials; + readonly credentials: AWSTemporaryCredentials; } export interface AccessGrant extends LocationAccess { From a06c2f99c5564546dd412691ca4f3c6500d2b306 Mon Sep 17 00:00:00 2001 From: Jamie Epp <168486127+eppjame@users.noreply.github.com> Date: Thu, 1 Aug 2024 08:32:43 -0700 Subject: [PATCH 29/40] feat: introduce CRC32 checksums to storage:uploadData API (#13649) Co-authored-by: Donny Wu --- packages/aws-amplify/package.json | 4 +- packages/interactions/package.json | 4 +- .../apis/uploadData/multipartHandlers.test.ts | 217 +++++++++++++++++- .../s3/apis/uploadData/putObjectJob.test.ts | 27 ++- .../S3/cases/completeMultipartUpload.ts | 4 + .../providers/s3/utils/crc32.native.test.ts | 13 ++ .../providers/s3/utils/crc32.test.ts | 136 +++++++++++ packages/storage/package.json | 2 + .../uploadData/multipart/initialUpload.ts | 29 +++ .../apis/uploadData/multipart/uploadCache.ts | 3 + .../uploadData/multipart/uploadHandlers.ts | 40 ++-- .../multipart/uploadPartExecutor.ts | 28 ++- .../s3/apis/uploadData/putObjectJob.ts | 13 +- .../client/s3data/completeMultipartUpload.ts | 12 +- .../client/s3data/createMultipartUpload.ts | 8 +- .../s3/utils/client/s3data/listParts.ts | 1 + .../s3/utils/client/s3data/putObject.ts | 2 + .../s3/utils/client/s3data/uploadPart.ts | 11 +- .../src/providers/s3/utils/crc32.native.ts | 11 + .../storage/src/providers/s3/utils/crc32.ts | 51 ++++ yarn.lock | 12 + 21 files changed, 584 insertions(+), 44 deletions(-) create mode 100644 packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts create mode 100644 packages/storage/__tests__/providers/s3/utils/crc32.test.ts create mode 100644 packages/storage/src/providers/s3/utils/crc32.native.ts create mode 100644 packages/storage/src/providers/s3/utils/crc32.ts diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 3d1f5cef58a..935da84a03a 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -299,7 +299,7 @@ "name": "[Analytics] record (Kinesis)", "path": "./dist/esm/analytics/kinesis/index.mjs", "import": "{ record }", - "limit": "48.69 kB" + "limit": "48.8 kB" }, { "name": "[Analytics] record (Kinesis Firehose)", @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "20.48 kB" + "limit": "21.63 kB" } ] } diff --git a/packages/interactions/package.json b/packages/interactions/package.json index 98359327196..edfb57ad2ba 100644 --- a/packages/interactions/package.json +++ b/packages/interactions/package.json @@ -89,13 +89,13 @@ "name": "Interactions (default to Lex v2)", "path": "./dist/esm/index.mjs", "import": "{ Interactions }", - "limit": "52.66 kB" + "limit": "54.05 kB" }, { "name": "Interactions (Lex v2)", "path": "./dist/esm/lex-v2/index.mjs", "import": "{ Interactions }", - "limit": "52.66 kB" + "limit": "54.05 kB" }, { "name": "Interactions (Lex v1)", diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index 30f8dfdfe1d..9098388e5f4 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -1,6 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; + import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, defaultStorage } from '@aws-amplify/core'; @@ -22,9 +25,16 @@ import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byte import { CanceledError } from '../../../../../src/errors/CanceledError'; import { StorageOptions } from '../../../../../src/types'; import '../testUtils'; +import { calculateContentCRC32 } from '../../../../../src/providers/s3/utils/crc32'; +import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; jest.mock('@aws-amplify/core'); jest.mock('../../../../../src/providers/s3/utils/client/s3data'); +jest.mock('../../../../../src/providers/s3/utils/crc32'); const credentials: AWSCredentials = { accessKeyId: 'accessKeyId', @@ -47,11 +57,44 @@ const mockCompleteMultipartUpload = jest.mocked(completeMultipartUpload); const mockAbortMultipartUpload = jest.mocked(abortMultipartUpload); const mockListParts = jest.mocked(listParts); const mockHeadObject = jest.mocked(headObject); +const mockCalculateContentCRC32 = jest.mocked(calculateContentCRC32); const disableAssertionFlag = true; const MB = 1024 * 1024; +jest.mock('../../../../../src/providers/s3/utils', () => ({ + ...jest.requireActual('../../../../../src/providers/s3/utils'), + calculateContentMd5: jest.fn(), +})); + +const getZeroDelayTimeout = () => + new Promise(resolve => { + setTimeout(() => { + resolve(); + }, 0); + }); + +const mockCalculateContentCRC32Mock = () => { + mockCalculateContentCRC32.mockReset(); + mockCalculateContentCRC32.mockResolvedValue({ + checksumArrayBuffer: new ArrayBuffer(0), + checksum: 'mockChecksum', + seed: 0, + }); +}; +const mockCalculateContentCRC32Undefined = () => { + mockCalculateContentCRC32.mockReset(); + mockCalculateContentCRC32.mockResolvedValue(undefined); +}; +const mockCalculateContentCRC32Reset = () => { + mockCalculateContentCRC32.mockReset(); + mockCalculateContentCRC32.mockImplementation( + jest.requireActual('../../../../../src/providers/s3/utils/crc32') + .calculateContentCRC32, + ); +}; + const mockMultipartUploadSuccess = (disableAssertion?: boolean) => { let totalSize = 0; mockCreateMultipartUpload.mockResolvedValueOnce({ @@ -149,9 +192,10 @@ describe('getMultipartUploadHandlers with key', () => { }); }); - afterEach(() => { + beforeEach(() => { jest.clearAllMocks(); resetS3Mocks(); + mockCalculateContentCRC32Reset(); }); it('should return multipart upload handlers', async () => { @@ -230,6 +274,69 @@ describe('getMultipartUploadHandlers with key', () => { ); }); + it.each([ + [ + 'file', + new File([getBlob(8 * MB)], 'someName'), + ['JCnBsQ==', 'HELzGQ=='], + ], + ['blob', getBlob(8 * MB), ['JCnBsQ==', 'HELzGQ==']], + ['string', 'Ü'.repeat(4 * MB), ['DL735w==', 'Akga7g==']], + ['arrayBuffer', new ArrayBuffer(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ['arrayBufferView', new Uint8Array(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ])( + `should create crc32 for %s type body`, + async (_, twoPartsPayload, expectedCrc32) => { + mockMultipartUploadSuccess(); + const { multipartUploadJob } = getMultipartUploadHandlers({ + key: defaultKey, + data: twoPartsPayload, + }); + await multipartUploadJob(); + + /** + * final crc32 calculation calls calculateContentCRC32 3 times + * 1 time for each of the 2 parts + * 1 time to combine the resulting hash for each of the two parts + * + * uploading each part calls calculateContentCRC32 1 time each + * + * these steps results in 5 calls in total + */ + expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentMd5).not.toHaveBeenCalled(); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[0] }), + ); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[1] }), + ); + }, + ); + + it('should use md5 if crc32 is returning undefined', async () => { + mockCalculateContentCRC32Undefined(); + mockMultipartUploadSuccess(); + Amplify.libraryOptions = { + Storage: { + S3: { + isObjectLockEnabled: true, + }, + }, + }; + const { multipartUploadJob } = getMultipartUploadHandlers({ + key: defaultKey, + data: new Uint8Array(8 * MB), + }); + await multipartUploadJob(); + expect(calculateContentCRC32).toHaveBeenCalledTimes(1); // (final crc32 calculation = 1 undefined) + expect(calculateContentMd5).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + }); + it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ @@ -244,6 +351,7 @@ describe('getMultipartUploadHandlers with key', () => { }); it('should upload a body that exceeds the size of default part size and parts count', async () => { + mockCalculateContentCRC32Mock(); let buffer: ArrayBuffer; const file = { __proto__: File.prototype, @@ -268,7 +376,7 @@ describe('getMultipartUploadHandlers with key', () => { file.size, ); await multipartUploadJob(); - expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count + expect(file.slice).toHaveBeenCalledTimes(10_000 * 2); // S3 limit of parts count double for crc32 calculations expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockUploadPart).toHaveBeenCalledTimes(10_000); expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1); @@ -621,6 +729,15 @@ describe('getMultipartUploadHandlers with key', () => { describe('pause() & resume()', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { + let pausedOnce = false; + + let resumeTest: () => void; + const waitForPause = new Promise(resolve => { + resumeTest = () => { + resolve(); + }; + }); + const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ key: defaultKey, @@ -629,16 +746,21 @@ describe('getMultipartUploadHandlers with key', () => { let partCount = 0; mockMultipartUploadCancellation(() => { partCount++; - if (partCount === 2) { + if (partCount === 2 && !pausedOnce) { onPause(); // Pause upload at the the last uploadPart call + resumeTest(); + pausedOnce = true; } }); const uploadPromise = multipartUploadJob(); + await waitForPause; + await getZeroDelayTimeout(); onResume(); await uploadPromise; - expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(3); expect(mockUploadPart.mock.calls[0][0].abortSignal?.aborted).toBe(true); expect(mockUploadPart.mock.calls[1][0].abortSignal?.aborted).toBe(true); + expect(mockUploadPart.mock.calls[2][0].abortSignal?.aborted).toBe(false); }); }); @@ -735,9 +857,10 @@ describe('getMultipartUploadHandlers with path', () => { }); }); - afterEach(() => { + beforeEach(() => { jest.clearAllMocks(); resetS3Mocks(); + mockCalculateContentCRC32Reset(); }); it('should return multipart upload handlers', async () => { @@ -808,6 +931,69 @@ describe('getMultipartUploadHandlers with path', () => { ); }); + it.each([ + [ + 'file', + new File([getBlob(8 * MB)], 'someName'), + ['JCnBsQ==', 'HELzGQ=='], + ], + ['blob', getBlob(8 * MB), ['JCnBsQ==', 'HELzGQ==']], + ['string', 'Ü'.repeat(4 * MB), ['DL735w==', 'Akga7g==']], + ['arrayBuffer', new ArrayBuffer(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ['arrayBufferView', new Uint8Array(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ])( + `should create crc32 for %s type body`, + async (_, twoPartsPayload, expectedCrc32) => { + mockMultipartUploadSuccess(); + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: twoPartsPayload, + }); + await multipartUploadJob(); + + /** + * final crc32 calculation calls calculateContentCRC32 3 times + * 1 time for each of the 2 parts + * 1 time to combine the resulting hash for each of the two parts + * + * uploading each part calls calculateContentCRC32 1 time each + * + * these steps results in 5 calls in total + */ + expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentMd5).not.toHaveBeenCalled(); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[0] }), + ); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[1] }), + ); + }, + ); + + it('should use md5 if crc32 is returning undefined', async () => { + mockCalculateContentCRC32Undefined(); + mockMultipartUploadSuccess(); + Amplify.libraryOptions = { + Storage: { + S3: { + isObjectLockEnabled: true, + }, + }, + }; + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new Uint8Array(8 * MB), + }); + await multipartUploadJob(); + expect(calculateContentCRC32).toHaveBeenCalledTimes(1); // (final crc32 calculation = 1 undefined) + expect(calculateContentMd5).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + }); + it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ @@ -822,6 +1008,7 @@ describe('getMultipartUploadHandlers with path', () => { }); it('should upload a body that exceeds the size of default part size and parts count', async () => { + mockCalculateContentCRC32Mock(); let buffer: ArrayBuffer; const file = { __proto__: File.prototype, @@ -846,7 +1033,7 @@ describe('getMultipartUploadHandlers with path', () => { file.size, ); await multipartUploadJob(); - expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count + expect(file.slice).toHaveBeenCalledTimes(10_000 * 2); // S3 limit of parts count double for crc32 calculations expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockUploadPart).toHaveBeenCalledTimes(10_000); expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1); @@ -1277,6 +1464,14 @@ describe('getMultipartUploadHandlers with path', () => { describe('pause() & resume()', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { + let pausedOnce = false; + let resumeTest: () => void; + const waitForPause = new Promise(resolve => { + resumeTest = () => { + resolve(); + }; + }); + const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ path: testPath, @@ -1285,16 +1480,22 @@ describe('getMultipartUploadHandlers with path', () => { let partCount = 0; mockMultipartUploadCancellation(() => { partCount++; - if (partCount === 2) { + if (partCount === 2 && !pausedOnce) { onPause(); // Pause upload at the the last uploadPart call + resumeTest(); + pausedOnce = true; } }); const uploadPromise = multipartUploadJob(); + await waitForPause; + await getZeroDelayTimeout(); + onResume(); await uploadPromise; - expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(3); expect(mockUploadPart.mock.calls[0][0].abortSignal?.aborted).toBe(true); expect(mockUploadPart.mock.calls[1][0].abortSignal?.aborted).toBe(true); + expect(mockUploadPart.mock.calls[2][0].abortSignal?.aborted).toBe(false); }); }); diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index fe28304fc43..a76871e2435 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -1,6 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; + import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify } from '@aws-amplify/core'; @@ -9,9 +12,14 @@ import { putObject, } from '../../../../../src/providers/s3/utils/client/s3data'; import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; +import * as CRC32 from '../../../../../src/providers/s3/utils/crc32'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; + jest.mock('../../../../../src/providers/s3/utils/client/s3data'); jest.mock('../../../../../src/providers/s3/utils', () => { const utils = jest.requireActual('../../../../../src/providers/s3/utils'); @@ -67,7 +75,7 @@ mockPutObject.mockResolvedValue({ /* TODO Remove suite when `key` parameter is removed */ describe('putObjectJob with key', () => { beforeEach(() => { - mockPutObject.mockClear(); + jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); it('should supply the correct parameters to putObject API handler', async () => { @@ -123,12 +131,16 @@ describe('putObjectJob with key', () => { ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: mockMetadata, - ContentMD5: undefined, + ChecksumCRC32: 'rfPzYw==', }, ); }); it('should set ContentMD5 if object lock is enabled', async () => { + jest + .spyOn(CRC32, 'calculateContentCRC32') + .mockResolvedValue(undefined as any); + Amplify.libraryOptions = { Storage: { S3: { @@ -181,6 +193,7 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -212,6 +225,7 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -221,6 +235,7 @@ describe('putObjectJob with key', () => { describe('putObjectJob with path', () => { beforeEach(() => { mockPutObject.mockClear(); + jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); test.each([ @@ -286,13 +301,17 @@ describe('putObjectJob with path', () => { ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: mockMetadata, - ContentMD5: undefined, + ChecksumCRC32: 'rfPzYw==', }, ); }, ); it('should set ContentMD5 if object lock is enabled', async () => { + jest + .spyOn(CRC32, 'calculateContentCRC32') + .mockResolvedValue(undefined as any); + Amplify.libraryOptions = { Storage: { S3: { @@ -420,6 +439,7 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -451,6 +471,7 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts index 6b20ab56254..c9cbbe8912f 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts @@ -26,10 +26,12 @@ const completeMultipartUploadHappyCase: ApiFunctionalTestCase< { ETag: 'etag1', PartNumber: 1, + ChecksumCRC32: 'test-checksum-1', }, { ETag: 'etag2', PartNumber: 2, + ChecksumCRC32: 'test-checksum-2', }, ], }, @@ -49,10 +51,12 @@ const completeMultipartUploadHappyCase: ApiFunctionalTestCase< '' + 'etag1' + '1' + + 'test-checksum-1' + '' + '' + 'etag2' + '2' + + 'test-checksum-2' + '' + '', }), diff --git a/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts new file mode 100644 index 00000000000..0f4c1adce27 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32.native'; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); + +describe('calculate crc32 native', () => { + it('should return undefined', async () => { + expect(await calculateContentCRC32(getBlob(8 * MB))).toEqual(undefined); + }); +}); diff --git a/packages/storage/__tests__/providers/s3/utils/crc32.test.ts b/packages/storage/__tests__/providers/s3/utils/crc32.test.ts new file mode 100644 index 00000000000..e2195ddf5e4 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/crc32.test.ts @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + +import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: encoder.encode('data').buffer, + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)).buffer, + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + const result = (await calculateContentCRC32(data))!; + expect(result.checksum).toEqual(expected.checksum); + expect(result.seed).toEqual(expected.seed); + expect(decoder.decode(result.checksumArrayBuffer)).toEqual( + decoder.decode(expected.checksumArrayBuffer), + ); + }); + }); +}); diff --git a/packages/storage/package.json b/packages/storage/package.json index e6963b64fe4..928db834660 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -72,6 +72,7 @@ "@aws-sdk/types": "3.398.0", "@smithy/md5-js": "2.0.7", "buffer": "4.9.2", + "crc-32": "^1.2.2", "fast-xml-parser": "^4.2.5", "tslib": "^2.5.0" }, @@ -111,6 +112,7 @@ "devDependencies": { "@aws-amplify/core": "6.3.6", "@aws-amplify/react-native": "1.1.3", + "@types/node": "20.14.12", "typescript": "5.0.2" } } diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts index 7307a90e007..1dd77ae2b20 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts @@ -7,12 +7,14 @@ import { ResolvedS3Config } from '../../../types/options'; import { StorageUploadDataPayload } from '../../../../../types'; import { Part, createMultipartUpload } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; +import { calculateContentCRC32 } from '../../../utils/crc32'; import { cacheMultipartUpload, findCachedUploadParts, getUploadsCacheKey, } from './uploadCache'; +import { getDataChunker } from './getDataChunker'; interface LoadOrCreateMultipartUploadOptions { s3Config: ResolvedS3Config; @@ -32,6 +34,7 @@ interface LoadOrCreateMultipartUploadOptions { interface LoadOrCreateMultipartUploadResult { uploadId: string; cachedParts: Part[]; + finalCrc32?: string; } /** @@ -61,6 +64,7 @@ export const loadOrCreateMultipartUpload = async ({ parts: Part[]; uploadId: string; uploadCacheKey: string; + finalCrc32?: string; } | undefined; if (size === undefined) { @@ -91,8 +95,11 @@ export const loadOrCreateMultipartUpload = async ({ return { uploadId: cachedUpload.uploadId, cachedParts: cachedUpload.parts, + finalCrc32: cachedUpload.finalCrc32, }; } else { + const finalCrc32 = await getCombinedCrc32(data, size); + const { UploadId } = await createMultipartUpload( { ...s3Config, @@ -105,14 +112,17 @@ export const loadOrCreateMultipartUpload = async ({ ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: metadata, + ChecksumAlgorithm: finalCrc32 ? 'CRC32' : undefined, }, ); + if (size === undefined) { logger.debug('uploaded data size cannot be determined, skipping cache.'); return { uploadId: UploadId!, cachedParts: [], + finalCrc32, }; } const uploadCacheKey = getUploadsCacheKey({ @@ -127,12 +137,31 @@ export const loadOrCreateMultipartUpload = async ({ uploadId: UploadId!, bucket, key, + finalCrc32, fileName: data instanceof File ? data.name : '', }); return { uploadId: UploadId!, cachedParts: [], + finalCrc32, }; } }; + +const getCombinedCrc32 = async ( + data: StorageUploadDataPayload, + size: number | undefined, +) => { + const crc32List: ArrayBuffer[] = []; + const dataChunker = getDataChunker(data, size); + for (const { data: checkData } of dataChunker) { + const checksumArrayBuffer = (await calculateContentCRC32(checkData)) + ?.checksumArrayBuffer; + if (checksumArrayBuffer === undefined) return undefined; + + crc32List.push(checksumArrayBuffer); + } + + return `${(await calculateContentCRC32(new Blob(crc32List)))?.checksum}-${crc32List.length}`; +}; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts index ce14939be7f..9761ee85732 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts @@ -33,6 +33,7 @@ export const findCachedUploadParts = async ({ }: FindCachedUploadPartsOptions): Promise<{ parts: Part[]; uploadId: string; + finalCrc32?: string; } | null> => { const cachedUploads = await listCachedUploadTasks(defaultStorage); if ( @@ -60,6 +61,7 @@ export const findCachedUploadParts = async ({ return { parts: Parts, uploadId: cachedUpload.uploadId, + finalCrc32: cachedUpload.finalCrc32, }; } catch (e) { logger.debug('failed to list cached parts, removing cached upload.'); @@ -74,6 +76,7 @@ interface FileMetadata { fileName: string; key: string; uploadId: string; + finalCrc32?: string; // Unix timestamp in ms lastTouched: number; } diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index cb7d8dc1348..0c3379d04d2 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -55,6 +55,7 @@ export const getMultipartUploadHandlers = ( | { uploadId: string; completedParts: Part[]; + finalCrc32?: string; } | undefined; let resolvedS3Config: ResolvedS3Config | undefined; @@ -110,23 +111,25 @@ export const getMultipartUploadHandlers = ( } if (!inProgressUpload) { - const { uploadId, cachedParts } = await loadOrCreateMultipartUpload({ - s3Config: resolvedS3Config, - accessLevel: resolvedAccessLevel, - bucket: resolvedBucket, - keyPrefix: resolvedKeyPrefix, - key: objectKey, - contentType, - contentDisposition, - contentEncoding, - metadata, - data, - size, - abortSignal: abortController.signal, - }); + const { uploadId, cachedParts, finalCrc32 } = + await loadOrCreateMultipartUpload({ + s3Config: resolvedS3Config, + accessLevel: resolvedAccessLevel, + bucket: resolvedBucket, + keyPrefix: resolvedKeyPrefix, + key: objectKey, + contentType, + contentDisposition, + contentEncoding, + metadata, + data, + size, + abortSignal: abortController.signal, + }); inProgressUpload = { uploadId, completedParts: cachedParts, + finalCrc32, }; } @@ -145,10 +148,15 @@ export const getMultipartUploadHandlers = ( const completedPartNumberSet = new Set( inProgressUpload.completedParts.map(({ PartNumber }) => PartNumber!), ); - const onPartUploadCompletion = (partNumber: number, eTag: string) => { + const onPartUploadCompletion = ( + partNumber: number, + eTag: string, + crc32: string | undefined, + ) => { inProgressUpload?.completedParts.push({ PartNumber: partNumber, ETag: eTag, + ChecksumCRC32: crc32, }); }; const concurrentUploadsProgressTracker = @@ -171,6 +179,7 @@ export const getMultipartUploadHandlers = ( onPartUploadCompletion, onProgress: concurrentUploadsProgressTracker.getOnProgressListener(), isObjectLockEnabled: resolvedS3Options.isObjectLockEnabled, + useCRC32Checksum: Boolean(inProgressUpload.finalCrc32), }), ); } @@ -194,6 +203,7 @@ export const getMultipartUploadHandlers = ( Bucket: resolvedBucket, Key: finalKey, UploadId: inProgressUpload.uploadId, + ChecksumCRC32: inProgressUpload.finalCrc32, MultipartUpload: { Parts: inProgressUpload.completedParts.sort( (partA, partB) => partA.PartNumber! - partB.PartNumber!, diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts index 224fce0e210..03d9ebaffc1 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts @@ -3,9 +3,10 @@ import { TransferProgressEvent } from '../../../../../types'; import { ResolvedS3Config } from '../../../types/options'; -import { calculateContentMd5 } from '../../../utils'; import { uploadPart } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; +import { CRC32Checksum, calculateContentCRC32 } from '../../../utils/crc32'; +import { calculateContentMd5 } from '../../../utils'; import { PartToUpload } from './getDataChunker'; @@ -18,7 +19,12 @@ interface UploadPartExecutorOptions { finalKey: string; uploadId: string; isObjectLockEnabled?: boolean; - onPartUploadCompletion(partNumber: number, eTag: string): void; + useCRC32Checksum?: boolean; + onPartUploadCompletion( + partNumber: number, + eTag: string, + crc32: string | undefined, + ): void; onProgress?(event: TransferProgressEvent): void; } @@ -33,6 +39,7 @@ export const uploadPartExecutor = async ({ onPartUploadCompletion, onProgress, isObjectLockEnabled, + useCRC32Checksum, }: UploadPartExecutorOptions) => { let transferredBytes = 0; for (const { data, partNumber, size } of dataChunkerGenerator) { @@ -49,6 +56,16 @@ export const uploadPartExecutor = async ({ }); } else { // handle cancel error + let checksumCRC32: CRC32Checksum | undefined; + if (useCRC32Checksum) { + checksumCRC32 = await calculateContentCRC32(data); + } + const contentMD5 = + // check if checksum exists. ex: should not exist in react native + !checksumCRC32 && isObjectLockEnabled + ? await calculateContentMd5(data) + : undefined; + const { ETag: eTag } = await uploadPart( { ...s3Config, @@ -66,14 +83,13 @@ export const uploadPartExecutor = async ({ UploadId: uploadId, Body: data, PartNumber: partNumber, - ContentMD5: isObjectLockEnabled - ? await calculateContentMd5(data) - : undefined, + ChecksumCRC32: checksumCRC32?.checksum, + ContentMD5: contentMD5, }, ); transferredBytes += size; // eTag will always be set even the S3 model interface marks it as optional. - onPartUploadCompletion(partNumber, eTag!); + onPartUploadCompletion(partNumber, eTag!, checksumCRC32?.checksum); } } }; diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index cecd5f736e3..1f0c8314750 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -14,6 +14,7 @@ import { ItemWithKey, ItemWithPath } from '../../types/outputs'; import { putObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; +import { calculateContentCRC32 } from '../../utils/crc32'; import { validateObjectNotExists } from './validateObjectNotExists'; @@ -48,6 +49,13 @@ export const putObjectJob = onProgress, } = uploadDataOptions ?? {}; + const checksumCRC32 = await calculateContentCRC32(data); + const contentMD5 = + // check if checksum exists. ex: should not exist in react native + !checksumCRC32 && isObjectLockEnabled + ? await calculateContentMd5(data) + : undefined; + if (preventOverwrite) { await validateObjectNotExists(s3Config, { Bucket: bucket, @@ -70,9 +78,8 @@ export const putObjectJob = ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: metadata, - ContentMD5: isObjectLockEnabled - ? await calculateContentMd5(data) - : undefined, + ContentMD5: contentMD5, + ChecksumCRC32: checksumCRC32?.checksum, }, ); diff --git a/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts index 1e399e824e7..d732044e953 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts @@ -16,6 +16,7 @@ import { import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; import { + assignStringVariables, buildStorageServiceError, map, parseXmlBody, @@ -39,7 +40,7 @@ const INVALID_PARAMETER_ERROR_MSG = export type CompleteMultipartUploadInput = Pick< CompleteMultipartUploadCommandInput, - 'Bucket' | 'Key' | 'UploadId' | 'MultipartUpload' + 'Bucket' | 'Key' | 'UploadId' | 'MultipartUpload' | 'ChecksumCRC32' >; export type CompleteMultipartUploadOutput = Pick< @@ -53,6 +54,7 @@ const completeMultipartUploadSerializer = async ( ): Promise => { const headers = { 'content-type': 'application/xml', + ...assignStringVariables({ 'x-amz-checksum-crc32': input.ChecksumCRC32 }), }; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); @@ -90,7 +92,13 @@ const serializeCompletedPartList = (input: CompletedPart): string => { throw new Error(`${INVALID_PARAMETER_ERROR_MSG}: ${input}`); } - return `${input.ETag}${input.PartNumber}`; + const eTag = `${input.ETag}`; + const partNumber = `${input.PartNumber}`; + const checksumCRC32 = input.ChecksumCRC32 + ? `${input.ChecksumCRC32}` + : ''; + + return `${eTag}${partNumber}${checksumCRC32}`; }; /** diff --git a/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts index 2b8669eb44c..45d755d2ef9 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts @@ -11,6 +11,7 @@ import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; import { + assignStringVariables, buildStorageServiceError, map, parseXmlBody, @@ -42,7 +43,12 @@ const createMultipartUploadSerializer = async ( input: CreateMultipartUploadInput, endpoint: Endpoint, ): Promise => { - const headers = await serializeObjectConfigsToHeaders(input); + const headers = { + ...(await serializeObjectConfigsToHeaders(input)), + ...assignStringVariables({ + 'x-amz-checksum-algorithm': input.ChecksumAlgorithm, + }), + }; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); url.pathname = serializePathnameObjectKey(url, input.Key); diff --git a/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts index 1a9deb3e4b9..392eff988d0 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts @@ -91,6 +91,7 @@ const deserializeCompletedPartList = (input: any[]): CompletedPart[] => PartNumber: ['PartNumber', deserializeNumber], ETag: 'ETag', Size: ['Size', deserializeNumber], + ChecksumCRC32: 'ChecksumCRC32', }), ); diff --git a/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts index 0feac108508..3330f260fba 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts @@ -38,6 +38,7 @@ export type PutObjectInput = Pick< | 'Expires' | 'Metadata' | 'Tagging' + | 'ChecksumCRC32' >; export type PutObjectOutput = Pick< @@ -57,6 +58,7 @@ const putObjectSerializer = async ( ContentType: input.ContentType ?? 'application/octet-stream', })), ...assignStringVariables({ 'content-md5': input.ContentMD5 }), + ...assignStringVariables({ 'x-amz-checksum-crc32': input.ChecksumCRC32 }), }; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); diff --git a/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts index f27bbdd5a32..6a37e143f7e 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts @@ -30,7 +30,13 @@ import type { UploadPartCommandInput, UploadPartCommandOutput } from './types'; // and will be set by browser or fetch polyfill. export type UploadPartInput = Pick< UploadPartCommandInput, - 'PartNumber' | 'Body' | 'UploadId' | 'Bucket' | 'Key' | 'ContentMD5' + | 'PartNumber' + | 'Body' + | 'UploadId' + | 'Bucket' + | 'Key' + | 'ContentMD5' + | 'ChecksumCRC32' >; export type UploadPartOutput = Pick< @@ -43,9 +49,10 @@ const uploadPartSerializer = async ( endpoint: Endpoint, ): Promise => { const headers = { + ...assignStringVariables({ 'x-amz-checksum-crc32': input.ChecksumCRC32 }), ...assignStringVariables({ 'content-md5': input.ContentMD5 }), + 'content-type': 'application/octet-stream', }; - headers['content-type'] = 'application/octet-stream'; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); url.pathname = serializePathnameObjectKey(url, input.Key); diff --git a/packages/storage/src/providers/s3/utils/crc32.native.ts b/packages/storage/src/providers/s3/utils/crc32.native.ts new file mode 100644 index 00000000000..389cb5fc87b --- /dev/null +++ b/packages/storage/src/providers/s3/utils/crc32.native.ts @@ -0,0 +1,11 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CRC32Checksum } from './crc32'; + +export const calculateContentCRC32 = async ( + content: Blob | string | ArrayBuffer | ArrayBufferView, + _seed = 0, +): Promise => { + return undefined; +}; diff --git a/packages/storage/src/providers/s3/utils/crc32.ts b/packages/storage/src/providers/s3/utils/crc32.ts new file mode 100644 index 00000000000..6d9e194c3af --- /dev/null +++ b/packages/storage/src/providers/s3/utils/crc32.ts @@ -0,0 +1,51 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crc32 from 'crc-32'; + +export interface CRC32Checksum { + checksumArrayBuffer: ArrayBuffer; + checksum: string; + seed: number; +} + +export const calculateContentCRC32 = async ( + content: Blob | string | ArrayBuffer | ArrayBufferView, + seed = 0, +): Promise => { + let internalSeed = seed; + let blob: Blob; + + if (content instanceof Blob) { + blob = content; + } else { + blob = new Blob([content]); + } + + await blob.stream().pipeTo( + new WritableStream({ + write(chunk) { + internalSeed = crc32.buf(chunk, internalSeed) >>> 0; + }, + }), + ); + const hex = internalSeed.toString(16).padStart(8, '0'); + + return { + checksumArrayBuffer: hexToArrayBuffer(hex), + checksum: hexToBase64(hex), + seed: internalSeed, + }; +}; + +const hexToArrayBuffer = (hexString: string) => + new Uint8Array((hexString.match(/\w{2}/g)! ?? []).map(h => parseInt(h, 16))) + .buffer; + +const hexToBase64 = (hexString: string) => + btoa( + hexString + .match(/\w{2}/g)! + .map((a: string) => String.fromCharCode(parseInt(a, 16))) + .join(''), + ); diff --git a/yarn.lock b/yarn.lock index e0bec61e208..d68896840e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5017,6 +5017,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.82.tgz#58d734b4acaa5be339864bbec9cd8024dd0b43d5" integrity sha512-pcDZtkx9z8XYV+ius2P3Ot2VVrcYOfXffBQUBuiszrlUzKSmoDYqo+mV+IoL8iIiIjjtOMvNSmH1hwJ+Q+f96Q== +"@types/node@20.14.12": + version "20.14.12" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.12.tgz#129d7c3a822cb49fc7ff661235f19cfefd422b49" + integrity sha512-r7wNXakLeSsGT0H1AU863vS2wa5wBOK4bWMjZz2wj+8nBx+m5PeIn0k8AloSLpRuiwdRQZwarZqHE4FNArPuJQ== + dependencies: + undici-types "~5.26.4" + "@types/node@^18.0.0": version "18.19.17" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.17.tgz#a581a9fb4b2cfdbc61f008804f4436b2d5c40354" @@ -6834,6 +6841,11 @@ cosmiconfig@^8.2.0: parse-json "^5.2.0" path-type "^4.0.0" +crc-32@^1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" + integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== + create-jest@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/create-jest/-/create-jest-29.7.0.tgz#a355c5b3cb1e1af02ba177fe7afd7feee49a5320" From 08ef01bef53c2baeefd8a01652f44ff099af015a Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Thu, 1 Aug 2024 15:51:47 -0700 Subject: [PATCH 30/40] fix(storage-browser): listCallerAccessGrantsDeserializer not parsing multiple AccessGrant instances (#13671) * fix(storage-browser): listCallerAccessGrantsDeserializernot parsing multiple AccessGrant instances * chore: add unit tests for single and multiple grants --------- Co-authored-by: Ashwin Kumar --- .../client/S3/cases/listCallerAccessGrants.ts | 75 +++++++++++++++++-- .../s3control/listCallerAccessGrants.ts | 8 +- 2 files changed, 76 insertions(+), 7 deletions(-) diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts index 29a8e3f4516..1a72a2049bc 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -17,11 +17,11 @@ const MOCK_GRANT_SCOPE = 's3://my-bucket/path/to/object.md'; const MOCK_PERMISSION = 'READWRITE'; // API Reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_ListAccessGrants.html -const listCallerAccessGrantsHappyCase: ApiFunctionalTestCase< +const listCallerAccessGrantsHappyCaseSingleGrant: ApiFunctionalTestCase< typeof listCallerAccessGrants > = [ 'happy case', - 'listCallerAccessGrants', + 'listCallerAccessGrantsHappyCaseSingleGrant', listCallerAccessGrants, defaultConfig, { @@ -71,6 +71,70 @@ const listCallerAccessGrantsHappyCase: ApiFunctionalTestCase< }, ]; +const listCallerAccessGrantsHappyCaseMultipleGrants: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'happy case', + 'listCallerAccessGrantsHappyCaseMultipleGrants', + listCallerAccessGrants, + defaultConfig, + { + AccountId: MOCK_ACCOUNT_ID, + GrantScope: 's3://my-bucket/path/to/', + MaxResults: 50, + NextToken: 'mockToken', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.s3-control.us-east-1.amazonaws.com/v20180820/accessgrantsinstance/caller/grants?grantscope=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2F&maxResults=50&nextToken=mockToken', + }), + method: 'GET', + headers: expect.objectContaining({ + 'x-amz-account-id': MOCK_ACCOUNT_ID, + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: ` + + + ${MOCK_NEXT_TOKEN} + + + ${MOCK_APP_ARN} + ${MOCK_GRANT_SCOPE} + ${MOCK_PERMISSION} + + + ${MOCK_APP_ARN} + ${MOCK_GRANT_SCOPE} + ${MOCK_PERMISSION} + + + + `, + }, + { + $metadata: expect.objectContaining(expectedMetadata), + CallerAccessGrantsList: [ + { + ApplicationArn: MOCK_APP_ARN, + GrantScope: MOCK_GRANT_SCOPE, + Permission: MOCK_PERMISSION, + }, + { + ApplicationArn: MOCK_APP_ARN, + GrantScope: MOCK_GRANT_SCOPE, + Permission: MOCK_PERMISSION, + }, + ], + NextToken: MOCK_NEXT_TOKEN, + }, +]; + const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< typeof listCallerAccessGrants > = [ @@ -78,8 +142,8 @@ const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< 'listCallerAccessGrants', listCallerAccessGrants, defaultConfig, - listCallerAccessGrantsHappyCase[4], - listCallerAccessGrantsHappyCase[5], + listCallerAccessGrantsHappyCaseSingleGrant[4], + listCallerAccessGrantsHappyCaseSingleGrant[5], { status: 403, headers: DEFAULT_RESPONSE_HEADERS, @@ -100,6 +164,7 @@ const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< ]; export default [ - listCallerAccessGrantsHappyCase, + listCallerAccessGrantsHappyCaseSingleGrant, + listCallerAccessGrantsHappyCaseMultipleGrants, listCallerAccessGrantsErrorCase, ]; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts index 3c95369fb99..8728ab19832 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts @@ -70,7 +70,11 @@ const listCallerAccessGrantsDeserializer = async ( const contents = map(parsed, { CallerAccessGrantsList: [ 'CallerAccessGrantsList', - value => emptyArrayGuard(value, deserializeAccessGrantsList), + value => + emptyArrayGuard( + value.AccessGrantsInstance, + deserializeAccessGrantsList, + ), ], NextToken: 'NextToken', }); @@ -86,7 +90,7 @@ const deserializeAccessGrantsList = (output: any[]) => output.map(deserializeCallerAccessGrant); const deserializeCallerAccessGrant = (output: any) => - map(output.AccessGrantsInstance, { + map(output, { ApplicationArn: 'ApplicationArn', GrantScope: 'GrantScope', Permission: 'Permission', From d69c58b46970455e1cf7e06dd6e4641ea065a720 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Wed, 7 Aug 2024 09:33:50 -0700 Subject: [PATCH 31/40] chore(storage-browser): expose additional input output types (#13682) * chore(storage-browser): expose additional internal types * address feedback * remove 'applicationArn' from listCallerAccessGrant unit test * Update packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts Co-authored-by: Caleb Pollman --------- Co-authored-by: Ashwin Kumar Co-authored-by: Caleb Pollman --- .../apis/listCallerAccessGrants.test.ts | 3 --- .../apis/listCallerAccessGrants.ts | 5 ++-- .../storage/src/storageBrowser/apis/types.ts | 3 +-- packages/storage/src/storageBrowser/index.ts | 8 ++++++ .../locationCredentialsStore/create.ts | 8 +++--- packages/storage/src/storageBrowser/types.ts | 25 +++++++------------ 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts index bff4b4e07bd..54a90d66206 100644 --- a/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts +++ b/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts @@ -117,19 +117,16 @@ describe('listCallerAccessGrants', () => { scope: 's3://bucket/*', type: 'BUCKET', permission: 'READ', - applicationArn: undefined, }, { scope: 's3://bucket/path/*', type: 'PREFIX', permission: 'READWRITE', - applicationArn: undefined, }, { scope: 's3://bucket/path/to/object', type: 'OBJECT', permission: 'READ', - applicationArn: 'arn:123', }, ]); expect(nextToken).toBeUndefined(); diff --git a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts index 12836d59880..157657b09d4 100644 --- a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts @@ -6,7 +6,7 @@ import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-clie import { logger } from '../../utils'; import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../providers/s3/utils/client/s3control'; -import { AccessGrant, LocationType, Permission } from '../types'; +import { LocationAccess, LocationType, Permission } from '../types'; import { StorageError } from '../../errors/StorageError'; import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; @@ -51,7 +51,7 @@ export const listCallerAccessGrants = async ( }, ); - const accessGrants: AccessGrant[] = + const accessGrants: LocationAccess[] = CallerAccessGrantsList?.map(grant => { // These values are correct from service mostly, but we add assertions to make TSC happy. assertPermission(grant.Permission); @@ -60,7 +60,6 @@ export const listCallerAccessGrants = async ( return { scope: grant.GrantScope, permission: grant.Permission, - applicationArn: grant.ApplicationArn, type: parseGrantType(grant.GrantScope!), }; }) ?? []; diff --git a/packages/storage/src/storageBrowser/apis/types.ts b/packages/storage/src/storageBrowser/apis/types.ts index c97a7c4bbdd..d0892c42b52 100644 --- a/packages/storage/src/storageBrowser/apis/types.ts +++ b/packages/storage/src/storageBrowser/apis/types.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { - AccessGrant, CredentialsProvider, ListLocationsInput, ListLocationsOutput, @@ -18,7 +17,7 @@ export interface ListCallerAccessGrantsInput extends ListLocationsInput { region: string; } -export type ListCallerAccessGrantsOutput = ListLocationsOutput; +export type ListCallerAccessGrantsOutput = ListLocationsOutput; export interface GetDataAccessInput { accountId: string; diff --git a/packages/storage/src/storageBrowser/index.ts b/packages/storage/src/storageBrowser/index.ts index 72878852a09..94c741b857c 100644 --- a/packages/storage/src/storageBrowser/index.ts +++ b/packages/storage/src/storageBrowser/index.ts @@ -14,4 +14,12 @@ export { GetLocationCredentials, ListLocations, LocationCredentialsStore, + CreateLocationCredentialsStoreInput, + LocationCredentials, + ListLocationsInput, + ListLocationsOutput, + GetLocationCredentialsInput, + GetLocationCredentialsOutput, + Permission, } from './types'; +export { AWSTemporaryCredentials } from '../providers/s3/types/options'; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts index ce4e9126612..05d68004f43 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts +++ b/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 import { + CreateLocationCredentialsStoreInput, CredentialsLocation, - GetLocationCredentials, LocationCredentialsStore, } from '../types'; import { StorageValidationErrorCode } from '../../errors/types/validation'; @@ -12,9 +12,9 @@ import { LocationCredentialsProvider } from '../../providers/s3/types/options'; import { createStore, getValue, removeStore } from './registry'; -export const createLocationCredentialsStore = (input: { - handler: GetLocationCredentials; -}): LocationCredentialsStore => { +export const createLocationCredentialsStore = ( + input: CreateLocationCredentialsStoreInput, +): LocationCredentialsStore => { const storeSymbol = createStore(input.handler); const store = { diff --git a/packages/storage/src/storageBrowser/types.ts b/packages/storage/src/storageBrowser/types.ts index 1199cf0e851..94d0c1f6eac 100644 --- a/packages/storage/src/storageBrowser/types.ts +++ b/packages/storage/src/storageBrowser/types.ts @@ -71,35 +71,24 @@ export interface LocationCredentials extends Partial { readonly credentials: AWSTemporaryCredentials; } -export interface AccessGrant extends LocationAccess { - /** - * The Amazon Resource Name (ARN) of an AWS IAM Identity Center application - * associated with your Identity Center instance. If the grant includes an - * application ARN, the grantee can only access the S3 data through this - * application. - */ - readonly applicationArn: string | undefined; -} - /** * @internal */ -export interface ListLocationsOutput { - locations: T[]; +export interface ListLocationsInput { + pageSize?: number; nextToken?: string; } - /** * @internal */ -export interface ListLocationsInput { - pageSize?: number; +export interface ListLocationsOutput { + locations: LocationAccess[]; nextToken?: string; } export type ListLocations = ( input?: ListLocationsInput, -) => Promise>; +) => Promise; export type GetLocationCredentialsInput = CredentialsLocation; export type GetLocationCredentialsOutput = LocationCredentials; @@ -108,6 +97,10 @@ export type GetLocationCredentials = ( input: GetLocationCredentialsInput, ) => Promise; +export interface CreateLocationCredentialsStoreInput { + handler: GetLocationCredentials; +} + export interface LocationCredentialsStore { /** * Get location-specific credentials. It uses a cache internally to optimize performance when From 0e837ec8241d3696680e2a011c9044643feb0bb0 Mon Sep 17 00:00:00 2001 From: Allan Zheng Date: Wed, 7 Aug 2024 17:54:11 -0700 Subject: [PATCH 32/40] chore: add ui to storage browser co-owner --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 78ed19747ec..3af1b2c1bd9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,8 @@ /packages/geo @aws-amplify/amplify-js @aws-amplify/amplify-ui /packages/pubsub @aws-amplify/amplify-js @aws-amplify/amplify-data /packages/aws-amplify/package.json @aws-amplify/amplify-js-admins +packages/storage/src/storageBrowser @aws-amplify/amplify-js @aws-amplify/amplify-ui +packages/storage/storage-browser @aws-amplify/amplify-js @aws-amplify/amplify-ui /.circleci/ @aws-amplify/amplify-js @aws-amplify/amplify-devops /.github/ @aws-amplify/amplify-js-admins From c5655a3b1e50e94ff1fd5129ad5073273b9adf97 Mon Sep 17 00:00:00 2001 From: Allan Zheng Date: Thu, 8 Aug 2024 10:38:49 -0700 Subject: [PATCH 33/40] chore: update bundle size --- packages/aws-amplify/package.json | 4 ++-- packages/core/src/clients/middleware/retry/middleware.ts | 2 +- packages/core/src/providers/pinpoint/apis/updateEndpoint.ts | 2 +- .../storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index b9e90d28fdb..75650dd833d 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -479,7 +479,7 @@ "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "16.26 kB" + "limit": "16.43 kB" }, { "name": "[Storage] list (S3)", @@ -497,7 +497,7 @@ "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "21.63 kB" + "limit": "21.68 kB" } ] } diff --git a/packages/core/src/clients/middleware/retry/middleware.ts b/packages/core/src/clients/middleware/retry/middleware.ts index 8d9a9c2cd9b..9bf7e093030 100644 --- a/packages/core/src/clients/middleware/retry/middleware.ts +++ b/packages/core/src/clients/middleware/retry/middleware.ts @@ -91,7 +91,7 @@ export const retryMiddlewareFactory = ({ // context.attemptsCount may be updated after calling next handler which may retry the request by itself. attemptsCount = (context.attemptsCount ?? 0) > attemptsCount - ? context.attemptsCount ?? 0 + ? (context.attemptsCount ?? 0) : attemptsCount + 1; context.attemptsCount = attemptsCount; const { isCredentialsExpiredError, retryable } = await retryDecider( diff --git a/packages/core/src/providers/pinpoint/apis/updateEndpoint.ts b/packages/core/src/providers/pinpoint/apis/updateEndpoint.ts index 100e1de4fe1..0a2b88edfc9 100644 --- a/packages/core/src/providers/pinpoint/apis/updateEndpoint.ts +++ b/packages/core/src/providers/pinpoint/apis/updateEndpoint.ts @@ -51,7 +51,7 @@ export const updateEndpoint = async ({ // only automatically populate the endpoint with client info and identity id upon endpoint creation to // avoid overwriting the endpoint with these values every time the endpoint is updated const demographicsFromClientInfo: UserProfile['demographic'] = {}; - const resolvedUserId = createdEndpointId ? userId ?? identityId : userId; + const resolvedUserId = createdEndpointId ? (userId ?? identityId) : userId; if (createdEndpointId) { const clientInfo = getClientInfo(); demographicsFromClientInfo.appVersion = clientInfo.appVersion; diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index c07888871f6..f19be5ddca7 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -123,7 +123,7 @@ export const resolveS3ConfigAndInput = async ( apiOptions?.accessLevel ?? defaultAccessLevel ?? DEFAULT_ACCESS_LEVEL; const targetIdentityId = accessLevel === 'protected' - ? apiOptions?.targetIdentityId ?? identityId + ? (apiOptions?.targetIdentityId ?? identityId) : identityId; const keyPrefix = await prefixResolver({ accessLevel, targetIdentityId }); From 7faa74e1cafbee1c867bcff0e4be1d98effcc421 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Thu, 8 Aug 2024 23:47:28 -0700 Subject: [PATCH 34/40] chore: enable storage-browser integ test (#13698) Co-authored-by: Ashwin Kumar --- .github/integ-config/integ-all.yml | 7 +++++++ .github/workflows/callable-e2e-test.yml | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/.github/integ-config/integ-all.yml b/.github/integ-config/integ-all.yml index d29ae41ba42..04a51139a19 100644 --- a/.github/integ-config/integ-all.yml +++ b/.github/integ-config/integ-all.yml @@ -813,6 +813,13 @@ tests: sample_name: [multi-part-copy] spec: multi-part-copy browser: *minimal_browser_list + - test_name: integ_react_storage_browser + desc: 'React Storage Browser' + framework: vite + category: storage + sample_name: [storage-browser] + spec: storage-browser + browser: *minimal_browser_list # GEN2 STORAGE - test_name: integ_react_storage diff --git a/.github/workflows/callable-e2e-test.yml b/.github/workflows/callable-e2e-test.yml index 18697cf5dc5..e80adccc8a7 100644 --- a/.github/workflows/callable-e2e-test.yml +++ b/.github/workflows/callable-e2e-test.yml @@ -43,6 +43,10 @@ env: CYPRESS_GOOGLE_CLIENTID: ${{ secrets.CYPRESS_GOOGLE_CLIENTID }} CYPRESS_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_GOOGLE_CLIENT_SECRET }} CYPRESS_GOOGLE_REFRESH_TOKEN: ${{ secrets.CYPRESS_GOOGLE_REFRESH_TOKEN }} + CYPRESS_AUTH0_CLIENTID: ${{ secrets.CYPRESS_AUTH0_CLIENTID }} + CYPRESS_AUTH0_SECRET: ${{ secrets.CYPRESS_AUTH0_SECRET }} + CYPRESS_AUTH0_AUDIENCE: ${{ secrets.CYPRESS_AUTH0_AUDIENCE }} + CYPRESS_AUTH0_DOMAIN: ${{ secrets.CYPRESS_AUTH0_DOMAIN }} jobs: e2e-test: From 0d31f3ba2fdd1373cf4285d1449de578b00cda3c Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Tue, 13 Aug 2024 09:02:50 -0700 Subject: [PATCH 35/40] chore(storage): update s3 control model (#13705) * chore(storage): update s3 control model * fix: move permission validation to custom client * chore: formatting code --- .../client/S3/cases/listCallerAccessGrants.ts | 12 +- .../s3control/listCallerAccessGrants.ts | 14 +- .../s3/utils/client/s3control/types.ts | 157 +++++++++--------- .../utils/client/utils/deserializeHelpers.ts | 41 +++++ .../apis/listCallerAccessGrants.ts | 17 +- 5 files changed, 138 insertions(+), 103 deletions(-) diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts index 1a72a2049bc..f63c80bf4bc 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -49,11 +49,11 @@ const listCallerAccessGrantsHappyCaseSingleGrant: ApiFunctionalTestCase< ${MOCK_NEXT_TOKEN} - + ${MOCK_APP_ARN} ${MOCK_GRANT_SCOPE} ${MOCK_PERMISSION} - + `, @@ -103,16 +103,16 @@ const listCallerAccessGrantsHappyCaseMultipleGrants: ApiFunctionalTestCase< ${MOCK_NEXT_TOKEN} - + ${MOCK_APP_ARN} ${MOCK_GRANT_SCOPE} ${MOCK_PERMISSION} - - + + ${MOCK_APP_ARN} ${MOCK_GRANT_SCOPE} ${MOCK_PERMISSION} - + `, diff --git a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts index 8728ab19832..a1e7038749d 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts @@ -22,6 +22,7 @@ import { parseXmlError, s3TransferHandler, } from '../utils'; +import { createStringEnumDeserializer } from '../utils/deserializeHelpers'; import type { ListCallerAccessGrantsCommandInput, @@ -71,10 +72,7 @@ const listCallerAccessGrantsDeserializer = async ( CallerAccessGrantsList: [ 'CallerAccessGrantsList', value => - emptyArrayGuard( - value.AccessGrantsInstance, - deserializeAccessGrantsList, - ), + emptyArrayGuard(value.AccessGrant, deserializeAccessGrantsList), ], NextToken: 'NextToken', }); @@ -93,7 +91,13 @@ const deserializeCallerAccessGrant = (output: any) => map(output, { ApplicationArn: 'ApplicationArn', GrantScope: 'GrantScope', - Permission: 'Permission', + Permission: [ + 'Permission', + createStringEnumDeserializer( + ['READ', 'READWRITE', 'WRITE'] as const, + 'Permission', + ), + ], }); export const listCallerAccessGrants = composeServiceApi( diff --git a/packages/storage/src/providers/s3/utils/client/s3control/types.ts b/packages/storage/src/providers/s3/utils/client/s3control/types.ts index e88e2438707..088bfc8b3ba 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/types.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/types.ts @@ -26,80 +26,6 @@ declare const S3PrefixType: { readonly Object: 'Object'; }; -/** - * @public - */ -export type Permission = (typeof Permission)[keyof typeof Permission]; - -/** - * @public - */ -export type Privilege = (typeof Privilege)[keyof typeof Privilege]; - -/** - * @public - */ -export type S3PrefixType = (typeof S3PrefixType)[keyof typeof S3PrefixType]; - -/** - * @public - * - * The input for {@link ListCallerAccessGrantsCommand}. - */ -export type ListCallerAccessGrantsCommandInput = ListCallerAccessGrantsRequest; - -/** - * @public - * - * The output of {@link ListCallerAccessGrantsCommand}. - */ -export interface ListCallerAccessGrantsCommandOutput - extends ListCallerAccessGrantsResult, - __MetadataBearer {} - -/** - * @public - */ -export interface ListCallerAccessGrantsRequest { - AccountId?: string; - GrantScope?: string; - NextToken?: string; - MaxResults?: number; -} - -/** - * @public - */ -export interface ListCallerAccessGrantsEntry { - Permission?: Permission | string; - GrantScope?: string; - ApplicationArn?: string; -} - -/** - * @public - */ -export interface ListCallerAccessGrantsResult { - NextToken?: string; - CallerAccessGrantsList?: ListCallerAccessGrantsEntry[]; -} - -/** - * @public - * - * The input for {@link GetDataAccessCommand}. - */ -export type GetDataAccessCommandInput = GetDataAccessRequest; - -/** - * @public - * - * The output of {@link GetDataAccessCommand}. - */ -export interface GetDataAccessCommandOutput - extends GetDataAccessResult, - __MetadataBearer {} - /** *

The Amazon Web Services Security Token Service temporary credential that S3 Access Grants vends to grantees and client applications.

* @public @@ -110,26 +36,50 @@ export interface Credentials { * @public */ AccessKeyId?: string; - /** *

The secret access key of the Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

* @public */ SecretAccessKey?: string; - /** *

The Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

* @public + * @public + * + * The input for {@link ListCallerAccessGrantsCommand}. + * @public + * + * The input for {@link ListCallerAccessGrantsCommand}. */ SessionToken?: string; - /** *

The expiration date and time of the temporary credential that S3 Access Grants vends to grantees and client applications.

* @public + * @public + * + * The output of {@link ListCallerAccessGrantsCommand}. + * @public + * + * The output of {@link ListCallerAccessGrantsCommand}. */ Expiration?: Date; } +/** + * @public + * + * The input for {@link GetDataAccessCommand}. + */ +export type GetDataAccessCommandInput = GetDataAccessRequest; +/** + * @public + * + * The output of {@link GetDataAccessCommand}. + */ +export interface GetDataAccessCommandOutput + extends GetDataAccessResult, + __MetadataBearer {} + /** * @public */ @@ -211,3 +161,56 @@ export interface GetDataAccessResult { */ MatchedGrantTarget?: string; } + +/** + * @public + * + * The input for {@link ListCallerAccessGrantsCommand}. + */ +export type ListCallerAccessGrantsCommandInput = ListCallerAccessGrantsRequest; +/** + * @public + * + * The output of {@link ListCallerAccessGrantsCommand}. + */ +export interface ListCallerAccessGrantsCommandOutput + extends ListCallerAccessGrantsResult, + __MetadataBearer {} +/** + * @public + */ +export interface ListCallerAccessGrantsEntry { + Permission?: Permission; + GrantScope?: string; + ApplicationArn?: string; +} +/** + * @public + */ +export interface ListCallerAccessGrantsRequest { + AccountId?: string; + GrantScope?: string; + NextToken?: string; + MaxResults?: number; +} +/** + * @public + */ +export interface ListCallerAccessGrantsResult { + NextToken?: string; + CallerAccessGrantsList?: ListCallerAccessGrantsEntry[]; +} +/** + * @public + */ +export type Permission = (typeof Permission)[keyof typeof Permission]; +/** + * @public + */ +export type Privilege = (typeof Privilege)[keyof typeof Privilege]; +/** + * @public + */ +export type S3PrefixType = (typeof S3PrefixType)[keyof typeof S3PrefixType]; + +export {}; diff --git a/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts b/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts index 0c06cbc60e7..8681d714372 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts @@ -104,6 +104,47 @@ export const deserializeTimestamp = (value: string): Date | undefined => { return value ? new Date(value) : undefined; }; +/** + * Create a function deserializing a string to an enum value. If the string is not a valid enum value, it throws a + * StorageError. + * + * @example + * ```typescript + * const deserializeStringEnum = createStringEnumDeserializer(['a', 'b', 'c'] as const, 'FieldName'); + * const deserializedArray = ['a', 'b', 'c'].map(deserializeStringEnum); + * // deserializedArray = ['a', 'b', 'c'] + * + * const invalidValue = deserializeStringEnum('d'); + * // Throws InvalidFieldName: Invalid FieldName: d + * ``` + * + * @internal + */ +export const createStringEnumDeserializer = ( + enumValues: T, + fieldName: string, +) => { + const deserializeStringEnum = ( + value: any, + ): T extends (infer E)[] ? E : never => { + const parsedEnumValue = value + ? (enumValues.find(enumValue => enumValue === value) as any) + : undefined; + if (!parsedEnumValue) { + throw new StorageError({ + name: `Invalid${fieldName}`, + message: `Invalid ${fieldName}: ${value}`, + recoverySuggestion: + 'This is likely to be a bug. Please reach out to library authors.', + }); + } + + return parsedEnumValue; + }; + + return deserializeStringEnum; +}; + /** * Function that makes sure the deserializer receives non-empty array. * diff --git a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts index 157657b09d4..255837dc25c 100644 --- a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts @@ -6,7 +6,7 @@ import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-clie import { logger } from '../../utils'; import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../providers/s3/utils/client/s3control'; -import { LocationAccess, LocationType, Permission } from '../types'; +import { LocationAccess, LocationType } from '../types'; import { StorageError } from '../../errors/StorageError'; import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; @@ -53,13 +53,11 @@ export const listCallerAccessGrants = async ( const accessGrants: LocationAccess[] = CallerAccessGrantsList?.map(grant => { - // These values are correct from service mostly, but we add assertions to make TSC happy. - assertPermission(grant.Permission); assertGrantScope(grant.GrantScope); return { scope: grant.GrantScope, - permission: grant.Permission, + permission: grant.Permission!, type: parseGrantType(grant.GrantScope!), }; }) ?? []; @@ -86,17 +84,6 @@ const parseGrantType = (grantScope: string): LocationType => { } }; -function assertPermission( - permissionValue: string | undefined, -): asserts permissionValue is Permission { - if (!['READ', 'READWRITE', 'WRITE'].includes(permissionValue ?? '')) { - throw new StorageError({ - name: 'InvalidPermission', - message: `Invalid permission: ${permissionValue}`, - }); - } -} - function assertGrantScope(value: unknown): asserts value is string { if (typeof value !== 'string' || !value.startsWith('s3://')) { throw new StorageError({ From 7fc23a708e3fbddccfbd39b1fa31da17d5b10a5b Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Tue, 27 Aug 2024 13:44:43 -0700 Subject: [PATCH 36/40] chore(storage-browser): pin crc-32 dep at 1.2.2 (#13752) * chore(storage-browser): pin crc-32 dep at 1.2.2 * chore: update lock file --- packages/storage/package.json | 2 +- yarn.lock | 33 ++++----------------------------- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/packages/storage/package.json b/packages/storage/package.json index 560be8bf799..275d7315f4c 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -72,7 +72,7 @@ "@aws-sdk/types": "3.398.0", "@smithy/md5-js": "2.0.7", "buffer": "4.9.2", - "crc-32": "^1.2.2", + "crc-32": "1.2.2", "fast-xml-parser": "^4.4.1", "tslib": "^2.5.0" }, diff --git a/yarn.lock b/yarn.lock index 0fb5ad99009..4e7468f5942 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7037,7 +7037,7 @@ cosmiconfig@^8.2.0: parse-json "^5.2.0" path-type "^4.0.0" -crc-32@^1.2.2: +crc-32@1.2.2: version "1.2.2" resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== @@ -14739,16 +14739,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -14826,7 +14817,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -14840,13 +14831,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -15985,7 +15969,7 @@ wordwrap@^1.0.0: resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-1.0.0.tgz#27584810891456a4171c8d0226441ade90cbcaeb" integrity sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -16012,15 +15996,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214" From 36607e8a89acd7df60959cb0a00762d89ba28634 Mon Sep 17 00:00:00 2001 From: AllanZhengYP Date: Wed, 4 Sep 2024 21:37:22 -0700 Subject: [PATCH 37/40] fix(storage-browser): missing error wrapping for s3 control responses (#13779) --- packages/aws-amplify/package.json | 14 ++--- .../s3/utils/client/S3/cases/getDataAccess.ts | 10 +-- .../client/S3/cases/listCallerAccessGrants.ts | 10 +-- ...der.test.ts => createRetryDecider.test.ts} | 14 ++--- .../s3/utils/client/testUtils/types.ts | 2 +- .../s3/utils/client/s3control/base.ts | 29 ++++++++- .../utils/client/s3control/getDataAccess.ts | 5 +- .../s3control/listCallerAccessGrants.ts | 3 +- .../client/s3data/abortMultipartUpload.ts | 3 +- .../providers/s3/utils/client/s3data/base.ts | 26 +++++++- .../client/s3data/completeMultipartUpload.ts | 4 +- .../s3/utils/client/s3data/copyObject.ts | 3 +- .../client/s3data/createMultipartUpload.ts | 3 +- .../s3/utils/client/s3data/deleteObject.ts | 3 +- .../s3/utils/client/s3data/getObject.ts | 9 ++- .../s3/utils/client/s3data/headObject.ts | 3 +- .../s3/utils/client/s3data/listObjectsV2.ts | 3 +- .../s3/utils/client/s3data/listParts.ts | 3 +- .../s3/utils/client/s3data/putObject.ts | 3 +- .../s3/utils/client/s3data/uploadPart.ts | 3 +- ...{retryDecider.ts => createRetryDecider.ts} | 63 ++++++++++++------- .../providers/s3/utils/client/utils/index.ts | 4 +- .../s3/utils/client/utils/parsePayload.ts | 54 ++++++++++------ 23 files changed, 176 insertions(+), 98 deletions(-) rename packages/storage/__tests__/providers/s3/utils/client/S3/utils/{retryDecider.test.ts => createRetryDecider.test.ts} (86%) rename packages/storage/src/providers/s3/utils/client/utils/{retryDecider.ts => createRetryDecider.ts} (60%) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 8af9fd02167..aa3203f6a3f 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -461,43 +461,43 @@ "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "15.42 kB" + "limit": "15.47 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.93 kB" + "limit": "15.98 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "15.20 kB" + "limit": "15.25 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "16.43 kB" + "limit": "16.47 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.82 kB" + "limit": "15.88 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "15.05 kB" + "limit": "15.13 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "21.68 kB" + "limit": "21.73 kB" } ] } diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts index b0a0d174a7a..851bc993a7c 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts @@ -83,12 +83,14 @@ const getDataAccessErrorCase: ApiFunctionalTestCase = [ headers: DEFAULT_RESPONSE_HEADERS, body: ` - - AccessDenied - Access Denied + + + AccessDenied + Access Denied + 656c76696e6727732072657175657374 Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== - + `, }, { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts index f63c80bf4bc..961ef27b3bf 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -149,12 +149,14 @@ const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< headers: DEFAULT_RESPONSE_HEADERS, body: ` - - AccessDenied - Access Denied + + + AccessDenied + Access Denied + 656c76696e6727732072657175657374 Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== - + `, }, { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/utils/retryDecider.test.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/utils/createRetryDecider.test.ts similarity index 86% rename from packages/storage/__tests__/providers/s3/utils/client/S3/utils/retryDecider.test.ts rename to packages/storage/__tests__/providers/s3/utils/client/S3/utils/createRetryDecider.test.ts index 5e1801c07db..7f30a9f0cab 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/utils/retryDecider.test.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/utils/createRetryDecider.test.ts @@ -5,17 +5,13 @@ import { getRetryDecider as getDefaultRetryDecider, } from '@aws-amplify/core/internals/aws-client-utils'; -import { retryDecider } from '../../../../../../../src/providers/s3/utils/client/utils'; -import { parseXmlError } from '../../../../../../../src/providers/s3/utils/client/utils/parsePayload'; +import { createRetryDecider } from '../../../../../../../src/providers/s3/utils/client/utils'; -jest.mock( - '../../../../../../../src/providers/s3/utils/client/utils/parsePayload', -); jest.mock('@aws-amplify/core/internals/aws-client-utils'); -const mockErrorParser = jest.mocked(parseXmlError); +const mockErrorParser = jest.fn(); -describe('retryDecider', () => { +describe('createRetryDecider', () => { const mockHttpResponse: HttpResponse = { statusCode: 200, headers: {}, @@ -34,6 +30,7 @@ describe('retryDecider', () => { it('should invoke the default retry decider', async () => { expect.assertions(3); + const retryDecider = createRetryDecider(mockErrorParser); const { retryable, isCredentialsExpiredError } = await retryDecider( mockHttpResponse, undefined, @@ -56,6 +53,7 @@ describe('retryDecider', () => { $metadata: {}, }; mockErrorParser.mockResolvedValue(parsedError); + const retryDecider = createRetryDecider(mockErrorParser); const { retryable, isCredentialsExpiredError } = await retryDecider( { ...mockHttpResponse, statusCode: 400 }, undefined, @@ -74,6 +72,7 @@ describe('retryDecider', () => { $metadata: {}, }; mockErrorParser.mockResolvedValue(parsedError); + const retryDecider = createRetryDecider(mockErrorParser); const { retryable, isCredentialsExpiredError } = await retryDecider( { ...mockHttpResponse, statusCode: 400 }, undefined, @@ -91,6 +90,7 @@ describe('retryDecider', () => { $metadata: {}, }; mockErrorParser.mockResolvedValue(parsedError); + const retryDecider = createRetryDecider(mockErrorParser); const { retryable, isCredentialsExpiredError } = await retryDecider( { ...mockHttpResponse, statusCode: 400 }, undefined, diff --git a/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts b/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts index b47d2ec7695..a3754b41707 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts @@ -3,7 +3,7 @@ import { HttpRequest } from '@aws-amplify/core/internals/aws-client-utils'; -interface MockFetchResponse { +export interface MockFetchResponse { body: BodyInit; headers: HeadersInit; status: number; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts index a40f9f6a5dd..590f2b26120 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -11,7 +11,7 @@ import { jitteredBackoff, } from '@aws-amplify/core/internals/aws-client-utils'; -import { retryDecider } from '../utils'; +import { createRetryDecider, createXmlErrorParser } from '../utils'; /** * The service name used to sign requests if the API requires authentication. @@ -57,6 +57,33 @@ const endpointResolver = ( return { url: endpoint }; }; +/** + * Error parser for the XML payload of S3 control plane error response. The + * error's `Code` and `Message` locates at the nested `Error` element instead of + * the XML root element. + * + * @example + * ``` + * + * + * + * AccessDenied + * Access Denied + * + * 656c76696e6727732072657175657374 + * Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== + * + * ``` + * + * @internal + */ +export const parseXmlError = createXmlErrorParser(); + +/** + * @internal + */ +export const retryDecider = createRetryDecider(parseXmlError); + /** * @internal */ diff --git a/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts b/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts index f1053d8ddd7..84adb14e8aa 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts @@ -7,11 +7,11 @@ import { HttpResponse, parseMetadata, } from '@aws-amplify/core/internals/aws-client-utils'; +import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; import { AmplifyUrl, AmplifyUrlSearchParams, } from '@aws-amplify/core/internals/utils'; -import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; import { assignStringVariables, @@ -19,7 +19,6 @@ import { deserializeTimestamp, map, parseXmlBody, - parseXmlError, s3TransferHandler, } from '../utils'; @@ -27,7 +26,7 @@ import type { GetDataAccessCommandInput, GetDataAccessCommandOutput, } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type GetDataAccessInput = GetDataAccessCommandInput; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts index a1e7038749d..81b0e62a9c8 100644 --- a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts +++ b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts @@ -19,7 +19,6 @@ import { emptyArrayGuard, map, parseXmlBody, - parseXmlError, s3TransferHandler, } from '../utils'; import { createStringEnumDeserializer } from '../utils/deserializeHelpers'; @@ -28,7 +27,7 @@ import type { ListCallerAccessGrantsCommandInput, ListCallerAccessGrantsCommandOutput, } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type ListCallerAccessGrantsInput = ListCallerAccessGrantsCommandInput; diff --git a/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts index fb541e803a0..fe2182e0c30 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts @@ -16,14 +16,13 @@ import { MetadataBearer } from '@aws-sdk/types'; import { buildStorageServiceError, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, } from '../utils'; import type { AbortMultipartUploadCommandInput } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type AbortMultipartUploadInput = Pick< AbortMultipartUploadCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/s3data/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts index d51c3a18a11..c7aef5c033c 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -11,7 +11,7 @@ import { jitteredBackoff, } from '@aws-amplify/core/internals/aws-client-utils'; -import { retryDecider } from '../utils'; +import { createRetryDecider, createXmlErrorParser } from '../utils'; const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/; const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; @@ -99,6 +99,30 @@ export const isDnsCompatibleBucketName = (bucketName: string): boolean => !IP_ADDRESS_PATTERN.test(bucketName) && !DOTS_PATTERN.test(bucketName); +/** + * Error parser for the XML payload of S3 data plane error response. The error's + * `Code` and `Message` locates directly at the XML root element. + * + * @example + * ``` + * + * + * NoSuchKey + * The resource you requested does not exist + * /mybucket/myfoto.jpg + * 4442587FB7D0A2F9 + * + * ``` + * + * @internal + */ +export const parseXmlError = createXmlErrorParser({ noErrorWrapping: true }); + +/** + * @internal + */ +export const retryDecider = createRetryDecider(parseXmlError); + /** * @internal */ diff --git a/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts index d732044e953..00267d21172 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts @@ -20,8 +20,6 @@ import { buildStorageServiceError, map, parseXmlBody, - parseXmlError, - retryDecider, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, @@ -33,7 +31,7 @@ import type { CompletedMultipartUpload, CompletedPart, } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError, retryDecider } from './base'; const INVALID_PARAMETER_ERROR_MSG = 'Invalid parameter for ComplteMultipartUpload API'; diff --git a/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts index f56745680fa..612e650aabb 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts @@ -14,7 +14,6 @@ import { assignStringVariables, buildStorageServiceError, parseXmlBody, - parseXmlError, s3TransferHandler, serializeObjectConfigsToHeaders, serializePathnameObjectKey, @@ -22,7 +21,7 @@ import { } from '../utils'; import type { CopyObjectCommandInput, CopyObjectCommandOutput } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type CopyObjectInput = Pick< CopyObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts index 45d755d2ef9..10549673a0e 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts @@ -15,7 +15,6 @@ import { buildStorageServiceError, map, parseXmlBody, - parseXmlError, s3TransferHandler, serializeObjectConfigsToHeaders, serializePathnameObjectKey, @@ -27,7 +26,7 @@ import type { CreateMultipartUploadCommandOutput, } from './types'; import type { PutObjectInput } from './putObject'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type CreateMultipartUploadInput = Extract< CreateMultipartUploadCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts index f8843881668..eb9c98b47ac 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts @@ -14,7 +14,6 @@ import { buildStorageServiceError, deserializeBoolean, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, @@ -24,7 +23,7 @@ import type { DeleteObjectCommandInput, DeleteObjectCommandOutput, } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type DeleteObjectInput = Pick< DeleteObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts index 4ef5afa6896..37ceade549d 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts @@ -11,8 +11,8 @@ import { parseMetadata, presignUrl, } from '@aws-amplify/core/internals/aws-client-utils'; -import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; +import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { CONTENT_SHA256_HEADER, @@ -22,13 +22,16 @@ import { deserializeNumber, deserializeTimestamp, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, } from '../utils'; -import { S3EndpointResolverOptions, defaultConfig } from './base'; +import { + S3EndpointResolverOptions, + defaultConfig, + parseXmlError, +} from './base'; import type { CompatibleHttpResponse, GetObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts index 1ffdeedeb55..ebf0c374ff5 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts @@ -16,13 +16,12 @@ import { deserializeNumber, deserializeTimestamp, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, } from '../utils'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; import type { HeadObjectCommandInput, HeadObjectCommandOutput } from './types'; export type HeadObjectInput = Pick; diff --git a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts index 680111cf0e9..b5db2c04ae8 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts @@ -22,7 +22,6 @@ import { emptyArrayGuard, map, parseXmlBody, - parseXmlError, s3TransferHandler, } from '../utils'; @@ -30,7 +29,7 @@ import type { ListObjectsV2CommandInput, ListObjectsV2CommandOutput, } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type ListObjectsV2Input = ListObjectsV2CommandInput; diff --git a/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts index 392eff988d0..5c4583d1ed2 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts @@ -19,7 +19,6 @@ import { emptyArrayGuard, map, parseXmlBody, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, @@ -30,7 +29,7 @@ import type { ListPartsCommandInput, ListPartsCommandOutput, } from './types'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; export type ListPartsInput = Pick< ListPartsCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts index 3330f260fba..3db17f66090 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts @@ -14,14 +14,13 @@ import { assignStringVariables, buildStorageServiceError, map, - parseXmlError, s3TransferHandler, serializeObjectConfigsToHeaders, serializePathnameObjectKey, validateS3RequiredParameter, } from '../utils'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; import type { PutObjectCommandInput, PutObjectCommandOutput } from './types'; export type PutObjectInput = Pick< diff --git a/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts index 6a37e143f7e..2ddea0dd836 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts @@ -17,13 +17,12 @@ import { assignStringVariables, buildStorageServiceError, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, } from '../utils'; -import { defaultConfig } from './base'; +import { defaultConfig, parseXmlError } from './base'; import type { UploadPartCommandInput, UploadPartCommandOutput } from './types'; // Content-length is ignored here because it's forbidden header diff --git a/packages/storage/src/providers/s3/utils/client/utils/retryDecider.ts b/packages/storage/src/providers/s3/utils/client/utils/createRetryDecider.ts similarity index 60% rename from packages/storage/src/providers/s3/utils/client/utils/retryDecider.ts rename to packages/storage/src/providers/s3/utils/client/utils/createRetryDecider.ts index 3e1e0fcc3da..0cfbc0eacde 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/retryDecider.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/createRetryDecider.ts @@ -1,6 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { + ErrorParser, HttpResponse, MiddlewareContext, RetryDeciderOutput, @@ -9,8 +10,6 @@ import { import { LocationCredentialsProvider } from '../../../types/options'; -import { parseXmlError } from './parsePayload'; - /** * Function to decide if the S3 request should be retried. For S3 APIs, we support forceRefresh option * for {@link LocationCredentialsProvider | LocationCredentialsProvider } option. It's set when S3 returns @@ -22,33 +21,49 @@ import { parseXmlError } from './parsePayload'; * @param middlewareContext Optional context object to store data between retries. * @returns True if the request should be retried. */ -export const retryDecider = async ( +export type RetryDecider = ( response?: HttpResponse, error?: unknown, middlewareContext?: MiddlewareContext, -): Promise => { - const defaultRetryDecider = getRetryDecider(parseXmlError); - const defaultRetryDecision = await defaultRetryDecider(response, error); - if (!response || response.statusCode < 300) { - return { retryable: false }; - } - const parsedError = await parseXmlError(response); - const errorCode = parsedError?.name; - const errorMessage = parsedError?.message; - const isCredentialsExpired = isCredentialsExpiredError( - errorCode, - errorMessage, - ); +) => Promise; + +/** + * Factory of a {@link RetryDecider} function. + * + * @param errorParser function to parse HTTP response wth XML payload to JS + * Error instance. + * @returns A structure indicating if the response is retryable; And if it is a + * CredentialsExpiredError + */ +export const createRetryDecider = + (errorParser: ErrorParser): RetryDecider => + async ( + response?: HttpResponse, + error?: unknown, + middlewareContext?: MiddlewareContext, + ): Promise => { + const defaultRetryDecider = getRetryDecider(errorParser); + const defaultRetryDecision = await defaultRetryDecider(response, error); + if (!response || response.statusCode < 300) { + return { retryable: false }; + } + const parsedError = await errorParser(response); + const errorCode = parsedError?.name; + const errorMessage = parsedError?.message; + const isCredentialsExpired = isCredentialsExpiredError( + errorCode, + errorMessage, + ); - return { - retryable: - defaultRetryDecision.retryable || - // If we know the previous retry attempt sets isCredentialsExpired in the - // middleware context, we don't want to retry anymore. - !!(isCredentialsExpired && !middlewareContext?.isCredentialsExpired), - isCredentialsExpiredError: isCredentialsExpired, + return { + retryable: + defaultRetryDecision.retryable || + // If we know the previous retry attempt sets isCredentialsExpired in the + // middleware context, we don't want to retry anymore. + !!(isCredentialsExpired && !middlewareContext?.isCredentialsExpired), + isCredentialsExpiredError: isCredentialsExpired, + }; }; -}; // Ref: https://github.com/aws/aws-sdk-js/blob/54829e341181b41573c419bd870dd0e0f8f10632/lib/event_listeners.js#L522-L541 const INVALID_TOKEN_ERROR_CODES = [ diff --git a/packages/storage/src/providers/s3/utils/client/utils/index.ts b/packages/storage/src/providers/s3/utils/client/utils/index.ts index 423987699f8..77042fafa0c 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/index.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/index.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -export { parseXmlBody, parseXmlError } from './parsePayload'; +export { parseXmlBody, createXmlErrorParser } from './parsePayload'; export { SEND_DOWNLOAD_PROGRESS_EVENT, SEND_UPLOAD_PROGRESS_EVENT, @@ -25,4 +25,4 @@ export { serializePathnameObjectKey, validateS3RequiredParameter, } from './serializeHelpers'; -export { retryDecider } from './retryDecider'; +export { createRetryDecider } from './createRetryDecider'; diff --git a/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts b/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts index 9da44dcbdd0..f0284d573d2 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts @@ -9,25 +9,43 @@ import { import { parser } from '../runtime'; -export const parseXmlError: ErrorParser = async (response?: HttpResponse) => { - if (!response || response.statusCode < 300) { - return; - } - const { statusCode } = response; - const body = await parseXmlBody(response); - const code = body?.Code - ? (body.Code as string) - : statusCode === 404 - ? 'NotFound' - : statusCode.toString(); - const message = body?.message ?? body?.Message ?? code; - const error = new Error(message); +/** + * Factory creating a parser that parses the JS Error object from the XML + * response payload. + * + * @param input Input object + * @param input.noErrorWrapping Whether the error code and message are located + * directly in the root XML element, or in a nested `` element. + * See: https://smithy.io/2.0/aws/protocols/aws-restxml-protocol.html#restxml-errors + * + * Default to false. + * + * @internal + */ +export const createXmlErrorParser = + ({ + noErrorWrapping = false, + }: { noErrorWrapping?: boolean } = {}): ErrorParser => + async (response?: HttpResponse) => { + if (!response || response.statusCode < 300) { + return; + } + const { statusCode } = response; + const body = await parseXmlBody(response); + const errorLocation = noErrorWrapping ? body : body.Error; + const code = errorLocation?.Code + ? (errorLocation.Code as string) + : statusCode === 404 + ? 'NotFound' + : statusCode.toString(); + const message = errorLocation?.message ?? errorLocation?.Message ?? code; + const error = new Error(message); - return Object.assign(error, { - name: code, - $metadata: parseMetadata(response), - }); -}; + return Object.assign(error, { + name: code, + $metadata: parseMetadata(response), + }); + }; export const parseXmlBody = async (response: HttpResponse): Promise => { if (!response.body) { From 090dad4cbdf2c78a44b3ddf80cd881c99fd5eab6 Mon Sep 17 00:00:00 2001 From: Ashwin Kumar Date: Mon, 30 Sep 2024 12:45:07 -0700 Subject: [PATCH 38/40] chore: increse bundle size --- packages/aws-amplify/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index c9fba3f55c7..5dc6691e9cb 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -335,7 +335,7 @@ "name": "[API] generateClient (AppSync)", "path": "./dist/esm/api/index.mjs", "import": "{ generateClient }", - "limit": "43.1 kB" + "limit": "43.17 kB" }, { "name": "[API] REST API handlers", From bda551b20aeb3f9f4e38c17d8e8c0e2b9761bd1c Mon Sep 17 00:00:00 2001 From: Jim Blanchard Date: Fri, 27 Sep 2024 15:04:19 -0500 Subject: [PATCH 39/40] chore: Setup storage internals route (#13858) --- packages/storage/internals/package.json | 7 +++++++ packages/storage/package.json | 9 +++++++++ packages/storage/src/internals/apis/index.ts | 2 ++ packages/storage/src/internals/index.ts | 2 ++ packages/storage/src/internals/types/index.ts | 2 ++ 5 files changed, 22 insertions(+) create mode 100644 packages/storage/internals/package.json create mode 100644 packages/storage/src/internals/apis/index.ts create mode 100644 packages/storage/src/internals/index.ts create mode 100644 packages/storage/src/internals/types/index.ts diff --git a/packages/storage/internals/package.json b/packages/storage/internals/package.json new file mode 100644 index 00000000000..169011166f3 --- /dev/null +++ b/packages/storage/internals/package.json @@ -0,0 +1,7 @@ +{ + "name": "@aws-amplify/storage/internals", + "types": "../dist/esm/internals/index.d.ts", + "main": "../dist/cjs/internals/index.js", + "module": "../dist/esm/internals/index.mjs", + "sideEffects": false +} diff --git a/packages/storage/package.json b/packages/storage/package.json index 47507f61619..e0a5afa1fb5 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -40,6 +40,9 @@ "s3": [ "./dist/esm/providers/s3/index.d.ts" ], + "internals": [ + "./dist/esm/internals/index.d.ts" + ], "server": [ "./dist/esm/server.d.ts" ], @@ -64,6 +67,7 @@ "files": [ "dist/cjs", "dist/esm", + "internals", "src", "server", "s3" @@ -83,6 +87,11 @@ "require": "./dist/cjs/index.js", "react-native": "./src/index.ts" }, + "./internals": { + "types": "./dist/esm/internals/index.d.ts", + "import": "./dist/esm/internals/index.mjs", + "require": "./dist/cjs/internals/index.js" + }, "./server": { "types": "./dist/esm/server.d.ts", "import": "./dist/esm/server.mjs", diff --git a/packages/storage/src/internals/apis/index.ts b/packages/storage/src/internals/apis/index.ts new file mode 100644 index 00000000000..cf1406c9425 --- /dev/null +++ b/packages/storage/src/internals/apis/index.ts @@ -0,0 +1,2 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 diff --git a/packages/storage/src/internals/index.ts b/packages/storage/src/internals/index.ts new file mode 100644 index 00000000000..cf1406c9425 --- /dev/null +++ b/packages/storage/src/internals/index.ts @@ -0,0 +1,2 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 diff --git a/packages/storage/src/internals/types/index.ts b/packages/storage/src/internals/types/index.ts new file mode 100644 index 00000000000..cf1406c9425 --- /dev/null +++ b/packages/storage/src/internals/types/index.ts @@ -0,0 +1,2 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 From a84bcf887be2053d18e83fe59de139e29b4aa4bd Mon Sep 17 00:00:00 2001 From: Jim Blanchard Date: Mon, 30 Sep 2024 14:53:05 -0500 Subject: [PATCH 40/40] chore: Refactor contents of `storage-browser/` path into `internals/` (#13859) --- .../apis/getDataAccess.test.ts | 4 +- .../apis/listCallerAccessGrants.test.ts | 2 +- .../locationCredentialsStore/create.test.ts | 8 +- .../locationCredentialsStore/registry.test.ts | 6 +- .../locationCredentialsStore/store.test.ts | 4 +- .../createListLocationsHandler.test.ts | 6 +- .../createManagedAuthConfigAdapter.test.ts | 10 +- packages/storage/package.json | 10 +- .../apis/getDataAccess.ts | 9 +- .../apis/listCallerAccessGrants.ts | 15 +-- packages/storage/src/internals/index.ts | 42 +++++++ .../locationCredentialsStore/constants.ts | 0 .../locationCredentialsStore/create.ts | 2 +- .../locationCredentialsStore/index.ts | 0 .../locationCredentialsStore/registry.ts | 5 +- .../locationCredentialsStore/store.ts | 4 +- .../createListLocationsHandler.ts | 2 +- .../createLocationCredentialsHandler.ts | 2 +- .../createManagedAuthConfigAdapter.ts | 2 +- .../managedAuthConfigAdapter/index.ts | 0 .../storage/src/internals/types/common.ts | 22 ++++ .../types/credentials.ts} | 110 ++++++++---------- packages/storage/src/internals/types/index.ts | 2 - .../types.ts => internals/types/inputs.ts} | 21 ++-- .../{apis/index.ts => types/options.ts} | 0 .../storage/src/internals/types/outputs.ts | 14 +++ .../apis => internals/utils}/constants.ts | 0 packages/storage/src/storageBrowser/index.ts | 25 ---- packages/storage/storage-browser/package.json | 7 -- 29 files changed, 182 insertions(+), 152 deletions(-) rename packages/storage/__tests__/{storageBrowser => internals}/apis/getDataAccess.test.ts (96%) rename packages/storage/__tests__/{storageBrowser => internals}/apis/listCallerAccessGrants.test.ts (97%) rename packages/storage/__tests__/{storageBrowser => internals}/locationCredentialsStore/create.test.ts (88%) rename packages/storage/__tests__/{storageBrowser => internals}/locationCredentialsStore/registry.test.ts (95%) rename packages/storage/__tests__/{storageBrowser => internals}/locationCredentialsStore/store.test.ts (97%) rename packages/storage/__tests__/{storageBrowser => internals}/managedAuthAdapter/createListLocationsHandler.test.ts (76%) rename packages/storage/__tests__/{storageBrowser => internals}/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts (72%) rename packages/storage/src/{storageBrowser => internals}/apis/getDataAccess.ts (91%) rename packages/storage/src/{storageBrowser => internals}/apis/listCallerAccessGrants.ts (89%) rename packages/storage/src/{storageBrowser => internals}/locationCredentialsStore/constants.ts (100%) rename packages/storage/src/{storageBrowser => internals}/locationCredentialsStore/create.ts (97%) rename packages/storage/src/{storageBrowser => internals}/locationCredentialsStore/index.ts (100%) rename packages/storage/src/{storageBrowser => internals}/locationCredentialsStore/registry.ts (96%) rename packages/storage/src/{storageBrowser => internals}/locationCredentialsStore/store.ts (98%) rename packages/storage/src/{storageBrowser => internals}/managedAuthConfigAdapter/createListLocationsHandler.ts (88%) rename packages/storage/src/{storageBrowser => internals}/managedAuthConfigAdapter/createLocationCredentialsHandler.ts (97%) rename packages/storage/src/{storageBrowser => internals}/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts (97%) rename packages/storage/src/{storageBrowser => internals}/managedAuthConfigAdapter/index.ts (100%) create mode 100644 packages/storage/src/internals/types/common.ts rename packages/storage/src/{storageBrowser/types.ts => internals/types/credentials.ts} (91%) delete mode 100644 packages/storage/src/internals/types/index.ts rename packages/storage/src/{storageBrowser/apis/types.ts => internals/types/inputs.ts} (64%) rename packages/storage/src/internals/{apis/index.ts => types/options.ts} (100%) create mode 100644 packages/storage/src/internals/types/outputs.ts rename packages/storage/src/{storageBrowser/apis => internals/utils}/constants.ts (100%) delete mode 100644 packages/storage/src/storageBrowser/index.ts delete mode 100644 packages/storage/storage-browser/package.json diff --git a/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts similarity index 96% rename from packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts rename to packages/storage/__tests__/internals/apis/getDataAccess.test.ts index c43f9b004ba..630e7835f9e 100644 --- a/packages/storage/__tests__/storageBrowser/apis/getDataAccess.test.ts +++ b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts @@ -3,9 +3,9 @@ import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; -import { getDataAccess } from '../../../src/storageBrowser/apis/getDataAccess'; +import { getDataAccess } from '../../../src/internals/apis/getDataAccess'; import { getDataAccess as getDataAccessClient } from '../../../src/providers/s3/utils/client/s3control'; -import { GetDataAccessInput } from '../../../src/storageBrowser/apis/types'; +import { GetDataAccessInput } from '../../../src/internals/types/inputs'; jest.mock('../../../src/providers/s3/utils/client/s3control'); diff --git a/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts similarity index 97% rename from packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts rename to packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts index 54a90d66206..f5067843129 100644 --- a/packages/storage/__tests__/storageBrowser/apis/listCallerAccessGrants.test.ts +++ b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts @@ -3,7 +3,7 @@ import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; -import { listCallerAccessGrants } from '../../../src/storageBrowser/apis/listCallerAccessGrants'; +import { listCallerAccessGrants } from '../../../src/internals/apis/listCallerAccessGrants'; import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../../src/providers/s3/utils/client/s3control'; jest.mock('../../../src/providers/s3/utils/client/s3control'); diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/internals/locationCredentialsStore/create.test.ts similarity index 88% rename from packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts rename to packages/storage/__tests__/internals/locationCredentialsStore/create.test.ts index 1bebd3772bf..206a2573fa5 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/create.test.ts +++ b/packages/storage/__tests__/internals/locationCredentialsStore/create.test.ts @@ -1,20 +1,20 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { createLocationCredentialsStore } from '../../../src/storageBrowser/locationCredentialsStore/create'; +import { createLocationCredentialsStore } from '../../../src/internals/locationCredentialsStore/create'; import { createStore, getValue, removeStore, -} from '../../../src/storageBrowser/locationCredentialsStore/registry'; -import { LocationCredentialsStore } from '../../../src/storageBrowser/types'; +} from '../../../src/internals/locationCredentialsStore/registry'; +import { LocationCredentialsStore } from '../../../src/internals/types/credentials'; import { StorageValidationErrorCode, validationErrorMap, } from '../../../src/errors/types/validation'; import { AWSTemporaryCredentials } from '../../../src/providers/s3/types/options'; -jest.mock('../../../src/storageBrowser/locationCredentialsStore/registry'); +jest.mock('../../../src/internals/locationCredentialsStore/registry'); const mockedCredentials = 'MOCK_CREDS' as any as AWSTemporaryCredentials; diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts b/packages/storage/__tests__/internals/locationCredentialsStore/registry.test.ts similarity index 95% rename from packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts rename to packages/storage/__tests__/internals/locationCredentialsStore/registry.test.ts index 9f2cfcfc742..9aa5b0c305a 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/registry.test.ts +++ b/packages/storage/__tests__/internals/locationCredentialsStore/registry.test.ts @@ -10,15 +10,15 @@ import { createStore, getValue, removeStore, -} from '../../../src/storageBrowser/locationCredentialsStore/registry'; +} from '../../../src/internals/locationCredentialsStore/registry'; import { LruLocationCredentialsStore, fetchNewValue, getCacheValue, initStore, -} from '../../../src/storageBrowser/locationCredentialsStore/store'; +} from '../../../src/internals/locationCredentialsStore/store'; -jest.mock('../../../src/storageBrowser/locationCredentialsStore/store'); +jest.mock('../../../src/internals/locationCredentialsStore/store'); const mockedStore = 'MOCKED_STORE' as any as LruLocationCredentialsStore; diff --git a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts b/packages/storage/__tests__/internals/locationCredentialsStore/store.test.ts similarity index 97% rename from packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts rename to packages/storage/__tests__/internals/locationCredentialsStore/store.test.ts index 81ca4702417..6bc34132147 100644 --- a/packages/storage/__tests__/storageBrowser/locationCredentialsStore/store.test.ts +++ b/packages/storage/__tests__/internals/locationCredentialsStore/store.test.ts @@ -9,8 +9,8 @@ import { fetchNewValue, getCacheValue, initStore, -} from '../../../src/storageBrowser/locationCredentialsStore/store'; -import { CredentialsLocation } from '../../../src/storageBrowser/types'; +} from '../../../src/internals/locationCredentialsStore/store'; +import { CredentialsLocation } from '../../../src/internals/types/credentials'; const mockCredentials = { expiration: new Date(Date.now() + 60 * 60_1000), diff --git a/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createListLocationsHandler.test.ts b/packages/storage/__tests__/internals/managedAuthAdapter/createListLocationsHandler.test.ts similarity index 76% rename from packages/storage/__tests__/storageBrowser/managedAuthAdapter/createListLocationsHandler.test.ts rename to packages/storage/__tests__/internals/managedAuthAdapter/createListLocationsHandler.test.ts index c2104ce728a..26a81b10c3d 100644 --- a/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createListLocationsHandler.test.ts +++ b/packages/storage/__tests__/internals/managedAuthAdapter/createListLocationsHandler.test.ts @@ -1,10 +1,10 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { createListLocationsHandler } from '../../../src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler'; -import { listCallerAccessGrants } from '../../../src/storageBrowser/apis/listCallerAccessGrants'; +import { createListLocationsHandler } from '../../../src/internals/managedAuthConfigAdapter/createListLocationsHandler'; +import { listCallerAccessGrants } from '../../../src/internals/apis/listCallerAccessGrants'; -jest.mock('../../../src/storageBrowser/apis/listCallerAccessGrants'); +jest.mock('../../../src/internals/apis/listCallerAccessGrants'); jest.mocked(listCallerAccessGrants).mockResolvedValue({ locations: [], diff --git a/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts b/packages/storage/__tests__/internals/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts similarity index 72% rename from packages/storage/__tests__/storageBrowser/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts rename to packages/storage/__tests__/internals/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts index 124fc16cd7a..74bf7e10bd7 100644 --- a/packages/storage/__tests__/storageBrowser/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts +++ b/packages/storage/__tests__/internals/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts @@ -1,15 +1,15 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { createManagedAuthConfigAdapter } from '../../../src/storageBrowser/managedAuthConfigAdapter'; -import { createListLocationsHandler } from '../../../src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler'; -import { createLocationCredentialsHandler } from '../../../src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler'; +import { createManagedAuthConfigAdapter } from '../../../src/internals/managedAuthConfigAdapter'; +import { createListLocationsHandler } from '../../../src/internals/managedAuthConfigAdapter/createListLocationsHandler'; +import { createLocationCredentialsHandler } from '../../../src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler'; jest.mock( - '../../../src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler', + '../../../src/internals/managedAuthConfigAdapter/createListLocationsHandler', ); jest.mock( - '../../../src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler', + '../../../src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler', ); describe('createManagedAuthConfigAdapter', () => { diff --git a/packages/storage/package.json b/packages/storage/package.json index e0a5afa1fb5..7e9fa894ac3 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -48,10 +48,7 @@ ], "s3/server": [ "./dist/esm/providers/s3/server.d.ts" - ], - "storage-browser": [ - "./dist/esm/storageBrowser/index.d.ts" - ] + ] } }, "repository": { @@ -108,11 +105,6 @@ "import": "./dist/esm/providers/s3/server.mjs", "require": "./dist/cjs/providers/s3/server.js" }, - "./storage-browser": { - "types": "./dist/esm/storageBrowser/index.d.ts", - "import": "./dist/esm/storageBrowser/index.mjs", - "require": "./dist/cjs/storageBrowser/index.js" - }, "./package.json": "./package.json" }, "peerDependencies": { diff --git a/packages/storage/src/storageBrowser/apis/getDataAccess.ts b/packages/storage/src/internals/apis/getDataAccess.ts similarity index 91% rename from packages/storage/src/storageBrowser/apis/getDataAccess.ts rename to packages/storage/src/internals/apis/getDataAccess.ts index b5d511c8e0d..3a6af14441a 100644 --- a/packages/storage/src/storageBrowser/apis/getDataAccess.ts +++ b/packages/storage/src/internals/apis/getDataAccess.ts @@ -10,11 +10,14 @@ import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-clie import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; import { getDataAccess as getDataAccessClient } from '../../providers/s3/utils/client/s3control'; import { StorageError } from '../../errors/StorageError'; +import { GetDataAccessInput } from '../types/inputs'; +import { GetDataAccessOutput } from '../types/outputs'; import { logger } from '../../utils'; +import { DEFAULT_CRED_TTL } from '../utils/constants'; -import { GetDataAccessInput, GetDataAccessOutput } from './types'; -import { DEFAULT_CRED_TTL } from './constants'; - +/** + * @internal + */ export const getDataAccess = async ( input: GetDataAccessInput, ): Promise => { diff --git a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts b/packages/storage/src/internals/apis/listCallerAccessGrants.ts similarity index 89% rename from packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts rename to packages/storage/src/internals/apis/listCallerAccessGrants.ts index 255837dc25c..3f8601fdaaa 100644 --- a/packages/storage/src/storageBrowser/apis/listCallerAccessGrants.ts +++ b/packages/storage/src/internals/apis/listCallerAccessGrants.ts @@ -6,16 +6,17 @@ import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-clie import { logger } from '../../utils'; import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../providers/s3/utils/client/s3control'; -import { LocationAccess, LocationType } from '../types'; import { StorageError } from '../../errors/StorageError'; import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; +import { LocationType } from '../types/common'; +import { LocationAccess } from '../types/credentials'; +import { ListCallerAccessGrantsInput } from '../types/inputs'; +import { ListCallerAccessGrantsOutput } from '../types/outputs'; +import { MAX_PAGE_SIZE } from '../utils/constants'; -import { - ListCallerAccessGrantsInput, - ListCallerAccessGrantsOutput, -} from './types'; -import { MAX_PAGE_SIZE } from './constants'; - +/** + * @internal + */ export const listCallerAccessGrants = async ( input: ListCallerAccessGrantsInput, ): Promise => { diff --git a/packages/storage/src/internals/index.ts b/packages/storage/src/internals/index.ts index cf1406c9425..279ef2afd3b 100644 --- a/packages/storage/src/internals/index.ts +++ b/packages/storage/src/internals/index.ts @@ -1,2 +1,44 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 + +export { LocationCredentialsProvider } from '../providers/s3/types/options'; +export { StorageSubpathStrategy } from '../types/options'; + +export { Permission } from './types/common'; + +/* +Internal APIs +*/ +export { + GetDataAccessInput, + ListCallerAccessGrantsInput, +} from './types/inputs'; +export { + GetDataAccessOutput, + ListCallerAccessGrantsOutput, +} from './types/outputs'; + +export { getDataAccess } from './apis/getDataAccess'; +export { listCallerAccessGrants } from './apis/listCallerAccessGrants'; + +/* +CredentialsStore exports +*/ +export { createLocationCredentialsStore } from './locationCredentialsStore'; +export { + AuthConfigAdapter, + createManagedAuthConfigAdapter, + CreateManagedAuthConfigAdapterInput, +} from './managedAuthConfigAdapter'; +export { + GetLocationCredentials, + ListLocations, + LocationCredentialsStore, + CreateLocationCredentialsStoreInput, + LocationCredentials, + ListLocationsInput, + ListLocationsOutput, + GetLocationCredentialsInput, + GetLocationCredentialsOutput, +} from './types/credentials'; +export { AWSTemporaryCredentials } from '../providers/s3/types/options'; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts b/packages/storage/src/internals/locationCredentialsStore/constants.ts similarity index 100% rename from packages/storage/src/storageBrowser/locationCredentialsStore/constants.ts rename to packages/storage/src/internals/locationCredentialsStore/constants.ts diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts b/packages/storage/src/internals/locationCredentialsStore/create.ts similarity index 97% rename from packages/storage/src/storageBrowser/locationCredentialsStore/create.ts rename to packages/storage/src/internals/locationCredentialsStore/create.ts index 05d68004f43..dab4539025e 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/create.ts +++ b/packages/storage/src/internals/locationCredentialsStore/create.ts @@ -5,7 +5,7 @@ import { CreateLocationCredentialsStoreInput, CredentialsLocation, LocationCredentialsStore, -} from '../types'; +} from '../types/credentials'; import { StorageValidationErrorCode } from '../../errors/types/validation'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { LocationCredentialsProvider } from '../../providers/s3/types/options'; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/index.ts b/packages/storage/src/internals/locationCredentialsStore/index.ts similarity index 100% rename from packages/storage/src/storageBrowser/locationCredentialsStore/index.ts rename to packages/storage/src/internals/locationCredentialsStore/index.ts diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts b/packages/storage/src/internals/locationCredentialsStore/registry.ts similarity index 96% rename from packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts rename to packages/storage/src/internals/locationCredentialsStore/registry.ts index 41b7c101b82..90d511c36fe 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/registry.ts +++ b/packages/storage/src/internals/locationCredentialsStore/registry.ts @@ -3,7 +3,10 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 import { AWSTemporaryCredentials } from '../../providers/s3/types/options'; -import { CredentialsLocation, GetLocationCredentials } from '../types'; +import { + CredentialsLocation, + GetLocationCredentials, +} from '../types/credentials'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; diff --git a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts b/packages/storage/src/internals/locationCredentialsStore/store.ts similarity index 98% rename from packages/storage/src/storageBrowser/locationCredentialsStore/store.ts rename to packages/storage/src/internals/locationCredentialsStore/store.ts index c5effe0f91e..64c26d5c5a4 100644 --- a/packages/storage/src/storageBrowser/locationCredentialsStore/store.ts +++ b/packages/storage/src/internals/locationCredentialsStore/store.ts @@ -6,8 +6,8 @@ import { CredentialsLocation, GetLocationCredentials, - Permission, -} from '../types'; +} from '../types/credentials'; +import { Permission } from '../types/common'; import { AWSTemporaryCredentials } from '../../providers/s3/types/options'; import { assertValidationError } from '../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../errors/types/validation'; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts b/packages/storage/src/internals/managedAuthConfigAdapter/createListLocationsHandler.ts similarity index 88% rename from packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts rename to packages/storage/src/internals/managedAuthConfigAdapter/createListLocationsHandler.ts index e246a2bdc27..b34dff9848c 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createListLocationsHandler.ts +++ b/packages/storage/src/internals/managedAuthConfigAdapter/createListLocationsHandler.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { CredentialsProvider, ListLocations } from '../types'; +import { CredentialsProvider, ListLocations } from '../types/credentials'; import { listCallerAccessGrants } from '../apis/listCallerAccessGrants'; interface CreateListLocationsHandlerInput { diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts b/packages/storage/src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler.ts similarity index 97% rename from packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts rename to packages/storage/src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler.ts index 248e81882ac..944a25206fa 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createLocationCredentialsHandler.ts +++ b/packages/storage/src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler.ts @@ -6,7 +6,7 @@ import { CredentialsProvider, GetLocationCredentials, GetLocationCredentialsInput, -} from '../types'; +} from '../types/credentials'; interface CreateLocationCredentialsHandlerInput { accountId: string; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts b/packages/storage/src/internals/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts similarity index 97% rename from packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts rename to packages/storage/src/internals/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts index 3d41d93e2b3..8b622baaee8 100644 --- a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts +++ b/packages/storage/src/internals/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts @@ -4,7 +4,7 @@ import { CredentialsProvider, GetLocationCredentials, ListLocations, -} from '../types'; +} from '../types/credentials'; import { createListLocationsHandler } from './createListLocationsHandler'; import { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; diff --git a/packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts b/packages/storage/src/internals/managedAuthConfigAdapter/index.ts similarity index 100% rename from packages/storage/src/storageBrowser/managedAuthConfigAdapter/index.ts rename to packages/storage/src/internals/managedAuthConfigAdapter/index.ts diff --git a/packages/storage/src/internals/types/common.ts b/packages/storage/src/internals/types/common.ts new file mode 100644 index 00000000000..2fd32ca6d8c --- /dev/null +++ b/packages/storage/src/internals/types/common.ts @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * @internal + */ +export type Permission = 'READ' | 'READWRITE' | 'WRITE'; + +/** + * @internal + */ +export type LocationType = 'BUCKET' | 'PREFIX' | 'OBJECT'; + +/** + * @internal + */ +export type Privilege = 'Default' | 'Minimal'; + +/** + * @internal + */ +export type PrefixType = 'Object'; diff --git a/packages/storage/src/storageBrowser/types.ts b/packages/storage/src/internals/types/credentials.ts similarity index 91% rename from packages/storage/src/storageBrowser/types.ts rename to packages/storage/src/internals/types/credentials.ts index 94d0c1f6eac..95d6de8a00e 100644 --- a/packages/storage/src/storageBrowser/types.ts +++ b/packages/storage/src/internals/types/credentials.ts @@ -4,33 +4,74 @@ import { AWSTemporaryCredentials, LocationCredentialsProvider, -} from '../providers/s3/types/options'; +} from '../../providers/s3/types/options'; + +import { LocationType, Permission } from './common'; /** * @internal */ -export type Permission = 'READ' | 'READWRITE' | 'WRITE'; +export type CredentialsProvider = LocationCredentialsProvider; + +export interface CreateLocationCredentialsStoreInput { + handler: GetLocationCredentials; +} + +export interface LocationCredentialsStore { + /** + * Get location-specific credentials. It uses a cache internally to optimize performance when + * getting credentials for the same location. It will refresh credentials if they expire or + * when forced to. + */ + getProvider(option: CredentialsLocation): LocationCredentialsProvider; + /** + * Invalidate cached credentials and force subsequent calls to get location-specific + * credentials to throw. It also makes subsequent calls to `getCredentialsProviderForLocation` + * to throw. + */ + destroy(): void; +} + +export interface LocationCredentials extends Partial { + /** + * AWS credentials which can be used to access the specified location. + */ + readonly credentials: AWSTemporaryCredentials; +} + +export type GetLocationCredentialsInput = CredentialsLocation; +export type GetLocationCredentialsOutput = LocationCredentials; + +export type GetLocationCredentials = ( + input: GetLocationCredentialsInput, +) => Promise; /** * @internal */ -export type CredentialsProvider = LocationCredentialsProvider; +export interface ListLocationsInput { + pageSize?: number; + nextToken?: string; +} /** * @internal */ -export type LocationType = 'BUCKET' | 'PREFIX' | 'OBJECT'; +export interface ListLocationsOutput { + locations: LocationAccess[]; + nextToken?: string; +} /** * @internal */ -export type Privilege = 'Default' | 'Minimal'; +export type ListLocations = ( + input?: ListLocationsInput, +) => Promise; /** * @internal */ -export type PrefixType = 'Object'; - export interface LocationScope { /** * Scope of storage location. For S3 service, it's the S3 path of the data to @@ -43,6 +84,9 @@ export interface LocationScope { readonly scope: string; } +/** + * @internal + */ export interface CredentialsLocation extends LocationScope { /** * The type of access granted to your Storage data. Can be either of READ, @@ -63,55 +107,3 @@ export interface LocationAccess extends CredentialsLocation { */ readonly type: LocationType; } - -export interface LocationCredentials extends Partial { - /** - * AWS credentials which can be used to access the specified location. - */ - readonly credentials: AWSTemporaryCredentials; -} - -/** - * @internal - */ -export interface ListLocationsInput { - pageSize?: number; - nextToken?: string; -} -/** - * @internal - */ -export interface ListLocationsOutput { - locations: LocationAccess[]; - nextToken?: string; -} - -export type ListLocations = ( - input?: ListLocationsInput, -) => Promise; - -export type GetLocationCredentialsInput = CredentialsLocation; -export type GetLocationCredentialsOutput = LocationCredentials; - -export type GetLocationCredentials = ( - input: GetLocationCredentialsInput, -) => Promise; - -export interface CreateLocationCredentialsStoreInput { - handler: GetLocationCredentials; -} - -export interface LocationCredentialsStore { - /** - * Get location-specific credentials. It uses a cache internally to optimize performance when - * getting credentials for the same location. It will refresh credentials if they expire or - * when forced to. - */ - getProvider(option: CredentialsLocation): LocationCredentialsProvider; - /** - * Invalidate cached credentials and force subsequent calls to get location-specific - * credentials to throw. It also makes subsequent calls to `getCredentialsProviderForLocation` - * to throw. - */ - destroy(): void; -} diff --git a/packages/storage/src/internals/types/index.ts b/packages/storage/src/internals/types/index.ts deleted file mode 100644 index cf1406c9425..00000000000 --- a/packages/storage/src/internals/types/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 diff --git a/packages/storage/src/storageBrowser/apis/types.ts b/packages/storage/src/internals/types/inputs.ts similarity index 64% rename from packages/storage/src/storageBrowser/apis/types.ts rename to packages/storage/src/internals/types/inputs.ts index d0892c42b52..51d04acf9c6 100644 --- a/packages/storage/src/storageBrowser/apis/types.ts +++ b/packages/storage/src/internals/types/inputs.ts @@ -1,24 +1,21 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { - CredentialsProvider, - ListLocationsInput, - ListLocationsOutput, - LocationCredentials, - Permission, - PrefixType, - Privilege, -} from '../types'; +import { CredentialsProvider, ListLocationsInput } from './credentials'; +import { Permission, PrefixType, Privilege } from './common'; +/** + * @internal + */ export interface ListCallerAccessGrantsInput extends ListLocationsInput { accountId: string; credentialsProvider: CredentialsProvider; region: string; } -export type ListCallerAccessGrantsOutput = ListLocationsOutput; - +/** + * @internal + */ export interface GetDataAccessInput { accountId: string; credentialsProvider: CredentialsProvider; @@ -29,5 +26,3 @@ export interface GetDataAccessInput { region: string; scope: string; } - -export type GetDataAccessOutput = LocationCredentials; diff --git a/packages/storage/src/internals/apis/index.ts b/packages/storage/src/internals/types/options.ts similarity index 100% rename from packages/storage/src/internals/apis/index.ts rename to packages/storage/src/internals/types/options.ts diff --git a/packages/storage/src/internals/types/outputs.ts b/packages/storage/src/internals/types/outputs.ts new file mode 100644 index 00000000000..09eecb7cfc2 --- /dev/null +++ b/packages/storage/src/internals/types/outputs.ts @@ -0,0 +1,14 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ListLocationsOutput, LocationCredentials } from './credentials'; + +/** + * @internal + */ +export type ListCallerAccessGrantsOutput = ListLocationsOutput; + +/** + * @internal + */ +export type GetDataAccessOutput = LocationCredentials; diff --git a/packages/storage/src/storageBrowser/apis/constants.ts b/packages/storage/src/internals/utils/constants.ts similarity index 100% rename from packages/storage/src/storageBrowser/apis/constants.ts rename to packages/storage/src/internals/utils/constants.ts diff --git a/packages/storage/src/storageBrowser/index.ts b/packages/storage/src/storageBrowser/index.ts deleted file mode 100644 index 94c741b857c..00000000000 --- a/packages/storage/src/storageBrowser/index.ts +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -export { LocationCredentialsProvider } from '../providers/s3/types/options'; -export { StorageSubpathStrategy } from '../types/options'; - -export { createLocationCredentialsStore } from './locationCredentialsStore'; -export { - AuthConfigAdapter, - createManagedAuthConfigAdapter, - CreateManagedAuthConfigAdapterInput, -} from './managedAuthConfigAdapter'; -export { - GetLocationCredentials, - ListLocations, - LocationCredentialsStore, - CreateLocationCredentialsStoreInput, - LocationCredentials, - ListLocationsInput, - ListLocationsOutput, - GetLocationCredentialsInput, - GetLocationCredentialsOutput, - Permission, -} from './types'; -export { AWSTemporaryCredentials } from '../providers/s3/types/options'; diff --git a/packages/storage/storage-browser/package.json b/packages/storage/storage-browser/package.json deleted file mode 100644 index f492f32d490..00000000000 --- a/packages/storage/storage-browser/package.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "@aws-amplify/storage/storage-browser", - "main": "../dist/cjs/storageBrowser/index.js", - "browser": "../dist/esm/storageBrowser/index.mjs", - "module": "../dist/esm/storageBrowser/index.mjs", - "typings": "../dist/esm/storageBrowser/index.d.ts" -}