diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8a945ce2b82..8ec26f3ab05 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,6 +8,8 @@ /packages/geo @aws-amplify/amplify-js @aws-amplify/amplify-ui /packages/pubsub @aws-amplify/amplify-js @aws-amplify/amplify-data /packages/aws-amplify/package.json @aws-amplify/amplify-js-admins +/packages/storage/src/storageBrowser @aws-amplify/amplify-js @aws-amplify/amplify-ui +/packages/storage/storage-browser @aws-amplify/amplify-js @aws-amplify/amplify-ui /.circleci/ @aws-amplify/amplify-js @aws-amplify/amplify-devops /.github/ @aws-amplify/amplify-js-admins diff --git a/.github/integ-config/integ-all.yml b/.github/integ-config/integ-all.yml index c7316c37c1f..211ebc76db1 100644 --- a/.github/integ-config/integ-all.yml +++ b/.github/integ-config/integ-all.yml @@ -827,6 +827,13 @@ tests: sample_name: [multi-part-copy] spec: multi-part-copy browser: *minimal_browser_list + - test_name: integ_react_storage_browser + desc: 'React Storage Browser' + framework: vite + category: storage + sample_name: [storage-browser] + spec: storage-browser + browser: *minimal_browser_list # GEN2 STORAGE - test_name: integ_react_storage diff --git a/.github/workflows/callable-e2e-test.yml b/.github/workflows/callable-e2e-test.yml index 18697cf5dc5..e80adccc8a7 100644 --- a/.github/workflows/callable-e2e-test.yml +++ b/.github/workflows/callable-e2e-test.yml @@ -43,6 +43,10 @@ env: CYPRESS_GOOGLE_CLIENTID: ${{ secrets.CYPRESS_GOOGLE_CLIENTID }} CYPRESS_GOOGLE_CLIENT_SECRET: ${{ secrets.CYPRESS_GOOGLE_CLIENT_SECRET }} CYPRESS_GOOGLE_REFRESH_TOKEN: ${{ secrets.CYPRESS_GOOGLE_REFRESH_TOKEN }} + CYPRESS_AUTH0_CLIENTID: ${{ secrets.CYPRESS_AUTH0_CLIENTID }} + CYPRESS_AUTH0_SECRET: ${{ secrets.CYPRESS_AUTH0_SECRET }} + CYPRESS_AUTH0_AUDIENCE: ${{ secrets.CYPRESS_AUTH0_AUDIENCE }} + CYPRESS_AUTH0_DOMAIN: ${{ secrets.CYPRESS_AUTH0_DOMAIN }} jobs: e2e-test: diff --git a/.github/workflows/push-preid-release.yml b/.github/workflows/push-preid-release.yml index 9837290ed14..c24a87da0a3 100644 --- a/.github/workflows/push-preid-release.yml +++ b/.github/workflows/push-preid-release.yml @@ -8,8 +8,7 @@ concurrency: on: push: branches: - # Change this to your branch name where "example-preid" corresponds to the preid you want your changes released on - - feat/example-preid-branch/main + - storage-browser/main jobs: e2e: @@ -35,4 +34,5 @@ jobs: # The preid should be detected from the branch name recommending feat/{PREID}/whatever as branch naming pattern # if your branch doesn't follow this pattern, you can override it here for your branch. with: - preid: ${{ needs.parse-preid.outputs.preid }} + preid: storage-browser + # preid: ${{ needs.parse-preid.outputs.preid }} diff --git a/package.json b/package.json index b50ecce61d2..e66105eae4f 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "publish:release": "yarn generate-metadata && lerna publish --conventional-commits --message 'chore(release): Publish [skip release]' --yes", "publish:v5-stable": "lerna publish --conventional-commits --yes --dist-tag=stable-5 --message 'chore(release): Publish [ci skip]' --no-verify-access", "publish:verdaccio": "lerna publish --canary --force-publish --no-push --dist-tag=unstable --preid=unstable --yes", + "publish:storage-browser/main": "lerna publish --canary --force-publish --dist-tag=storage-browser --preid=storage-browser --yes", "generate-metadata": "git rev-parse --short HEAD > packages/core/metadata && git commit -am 'chore(release): Set core metadata [skip release]'", "ts-coverage": "lerna run ts-coverage", "prepare": "husky && ./scripts/set-preid-versions.sh" diff --git a/packages/aws-amplify/package.json b/packages/aws-amplify/package.json index 7437c7e963c..d8227302885 100644 --- a/packages/aws-amplify/package.json +++ b/packages/aws-amplify/package.json @@ -293,31 +293,31 @@ "name": "[Analytics] record (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ record }", - "limit": "17.35 kB" + "limit": "17.42 kB" }, { "name": "[Analytics] record (Kinesis)", "path": "./dist/esm/analytics/kinesis/index.mjs", "import": "{ record }", - "limit": "48.74 kB" + "limit": "48.8 kB" }, { "name": "[Analytics] record (Kinesis Firehose)", "path": "./dist/esm/analytics/kinesis-firehose/index.mjs", "import": "{ record }", - "limit": "45.76 kB" + "limit": "45.85 kB" }, { "name": "[Analytics] record (Personalize)", "path": "./dist/esm/analytics/personalize/index.mjs", "import": "{ record }", - "limit": "49.58 kB" + "limit": "49.67 kB" }, { "name": "[Analytics] identifyUser (Pinpoint)", "path": "./dist/esm/analytics/index.mjs", "import": "{ identifyUser }", - "limit": "15.85 kB" + "limit": "15.91 kB" }, { "name": "[Analytics] enable", @@ -335,7 +335,7 @@ "name": "[API] generateClient (AppSync)", "path": "./dist/esm/api/index.mjs", "import": "{ generateClient }", - "limit": "43.40 kB" + "limit": "43.46 kB" }, { "name": "[API] REST API handlers", @@ -353,43 +353,43 @@ "name": "[Auth] resetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resetPassword }", - "limit": "12.57 kB" + "limit": "12.66 kB" }, { "name": "[Auth] confirmResetPassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmResetPassword }", - "limit": "12.51 kB" + "limit": "12.60 kB" }, { "name": "[Auth] signIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signIn }", - "limit": "30.00 kB" + "limit": "28.78 kB" }, { "name": "[Auth] resendSignUpCode (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ resendSignUpCode }", - "limit": "12.53 kB" + "limit": "12.61 kB" }, { "name": "[Auth] confirmSignUp (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignUp }", - "limit": "31.00 kB" + "limit": "29.40 kB" }, { "name": "[Auth] confirmSignIn (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmSignIn }", - "limit": "28.39 kB" + "limit": "28.46 kB" }, { "name": "[Auth] updateMFAPreference (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updateMFAPreference }", - "limit": "11.98 kB" + "limit": "12.07 kB" }, { "name": "[Auth] fetchMFAPreference (Cognito)", @@ -401,13 +401,13 @@ "name": "[Auth] verifyTOTPSetup (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ verifyTOTPSetup }", - "limit": "12.86 kB" + "limit": "12.94 kB" }, { "name": "[Auth] updatePassword (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updatePassword }", - "limit": "12.87 kB" + "limit": "12.96 kB" }, { "name": "[Auth] setUpTOTP (Cognito)", @@ -419,31 +419,31 @@ "name": "[Auth] updateUserAttributes (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ updateUserAttributes }", - "limit": "12.1 kB" + "limit": "12.19 kB" }, { "name": "[Auth] getCurrentUser (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ getCurrentUser }", - "limit": "7.97 kB" + "limit": "7.99 kB" }, { "name": "[Auth] confirmUserAttribute (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ confirmUserAttribute }", - "limit": "12.86 kB" + "limit": "12.93 kB" }, { "name": "[Auth] signInWithRedirect (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect }", - "limit": "21.19 kB" + "limit": "21.21 kB" }, { "name": "[Auth] fetchUserAttributes (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ fetchUserAttributes }", - "limit": "11.93 kB" + "limit": "12.01 kB" }, { "name": "[Auth] Basic Auth Flow (Cognito)", @@ -455,49 +455,49 @@ "name": "[Auth] OAuth Auth Flow (Cognito)", "path": "./dist/esm/auth/index.mjs", "import": "{ signInWithRedirect, signOut, fetchAuthSession }", - "limit": "21.66 kB" + "limit": "21.64 kB" }, { "name": "[Storage] copy (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ copy }", - "limit": "14.96 kB" + "limit": "15.59 kB" }, { "name": "[Storage] downloadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ downloadData }", - "limit": "15.55 kB" + "limit": "16.13 kB" }, { "name": "[Storage] getProperties (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getProperties }", - "limit": "14.81 kB" + "limit": "15.37 kB" }, { "name": "[Storage] getUrl (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ getUrl }", - "limit": "16.05 kB" + "limit": "16.59 kB" }, { "name": "[Storage] list (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ list }", - "limit": "15.41 kB" + "limit": "15.99 kB" }, { "name": "[Storage] remove (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ remove }", - "limit": "14.67 kB" + "limit": "15.23 kB" }, { "name": "[Storage] uploadData (S3)", "path": "./dist/esm/storage/index.mjs", "import": "{ uploadData }", - "limit": "20.05 kB" + "limit": "21.82 kB" } ] } diff --git a/packages/core/__tests__/clients/middleware/retry/defaultRetryDecider.test.ts b/packages/core/__tests__/clients/middleware/retry/defaultRetryDecider.test.ts new file mode 100644 index 00000000000..b33b4e27f4d --- /dev/null +++ b/packages/core/__tests__/clients/middleware/retry/defaultRetryDecider.test.ts @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { HttpResponse } from '../../../../src/clients'; +import { getRetryDecider } from '../../../../src/clients/middleware/retry'; +import { isClockSkewError } from '../../../../src/clients/middleware/retry/isClockSkewError'; + +jest.mock('../../../../src/clients/middleware/retry/isClockSkewError'); + +const mockIsClockSkewError = jest.mocked(isClockSkewError); + +describe('getRetryDecider', () => { + const mockErrorParser = jest.fn(); + const mockHttpResponse: HttpResponse = { + statusCode: 200, + headers: {}, + body: 'body' as any, + }; + + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('should handle network errors', async () => { + expect.assertions(2); + const retryDecider = getRetryDecider(mockErrorParser); + const connectionError = Object.assign(new Error(), { + name: 'Network error', + }); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + connectionError, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }); + + describe('handling throttling errors', () => { + it.each([ + 'BandwidthLimitExceeded', + 'EC2ThrottledException', + 'LimitExceededException', + 'PriorRequestNotComplete', + 'ProvisionedThroughputExceededException', + 'RequestLimitExceeded', + 'RequestThrottled', + 'RequestThrottledException', + 'SlowDown', + 'ThrottledException', + 'Throttling', + 'ThrottlingException', + 'TooManyRequestsException', + ])('should return retryable at %s error', async errorCode => { + expect.assertions(2); + mockErrorParser.mockResolvedValueOnce({ + code: errorCode, + }); + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }); + + it('should set retryable for 402 error', async () => { + expect.assertions(2); + const retryDecider = getRetryDecider(mockErrorParser); + const { + retryable, + isCredentialsExpiredError: isInvalidCredentialsError, + } = await retryDecider( + { + ...mockHttpResponse, + statusCode: 429, + }, + undefined, + ); + expect(retryable).toBe(true); + expect(isInvalidCredentialsError).toBeFalsy(); + }); + }); + + describe('handling clockskew error', () => { + it.each([{ code: 'ClockSkew' }, { name: 'ClockSkew' }])( + 'should handle clockskew error %o', + async parsedError => { + expect.assertions(3); + mockErrorParser.mockResolvedValue(parsedError); + mockIsClockSkewError.mockReturnValue(true); + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + expect(mockIsClockSkewError).toHaveBeenCalledWith( + Object.values(parsedError)[0], + ); + }, + ); + }); + + it.each([500, 502, 503, 504])( + 'should handle server-side status code %s', + async statusCode => { + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { + ...mockHttpResponse, + statusCode, + }, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }, + ); + + it.each(['TimeoutError', 'RequestTimeout', 'RequestTimeoutException'])( + 'should handle server-side timeout error code %s', + async errorCode => { + expect.assertions(2); + mockErrorParser.mockResolvedValue({ code: errorCode }); + const retryDecider = getRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBeFalsy(); + }, + ); +}); diff --git a/packages/core/__tests__/clients/middleware/retry/middleware.test.ts b/packages/core/__tests__/clients/middleware/retry/middleware.test.ts index 1391f010d23..05f1b0f8de9 100644 --- a/packages/core/__tests__/clients/middleware/retry/middleware.test.ts +++ b/packages/core/__tests__/clients/middleware/retry/middleware.test.ts @@ -11,13 +11,13 @@ import { jest.spyOn(global, 'setTimeout'); jest.spyOn(global, 'clearTimeout'); -describe(`${retryMiddlewareFactory.name} middleware`, () => { +describe(`retry middleware`, () => { beforeEach(() => { jest.clearAllMocks(); }); const defaultRetryOptions = { - retryDecider: async () => true, + retryDecider: async () => ({ retryable: true }), computeDelay: () => 1, }; const defaultRequest = { url: new URL('https://a.b') }; @@ -72,7 +72,7 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { const retryableHandler = getRetryableHandler(nextHandler); const retryDecider = jest .fn() - .mockImplementation(response => response.body !== 'foo'); // retry if response is not foo + .mockImplementation(response => ({ retryable: response.body !== 'foo' })); // retry if response is not foo const resp = await retryableHandler(defaultRequest, { ...defaultRetryOptions, retryDecider, @@ -88,11 +88,9 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { .fn() .mockRejectedValue(new Error('UnretryableError')); const retryableHandler = getRetryableHandler(nextHandler); - const retryDecider = jest - .fn() - .mockImplementation( - (resp, error) => error.message !== 'UnretryableError', - ); + const retryDecider = jest.fn().mockImplementation((resp, error) => ({ + retryable: error.message !== 'UnretryableError', + })); try { await retryableHandler(defaultRequest, { ...defaultRetryOptions, @@ -103,11 +101,46 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { expect(e.message).toBe('UnretryableError'); expect(nextHandler).toHaveBeenCalledTimes(1); expect(retryDecider).toHaveBeenCalledTimes(1); - expect(retryDecider).toHaveBeenCalledWith(undefined, expect.any(Error)); + expect(retryDecider).toHaveBeenCalledWith( + undefined, + expect.any(Error), + expect.anything(), + ); } expect.assertions(4); }); + test('should set isCredentialsExpired in middleware context if retry decider returns the flag', async () => { + expect.assertions(4); + const coreHandler = jest + .fn() + .mockRejectedValueOnce(new Error('InvalidSignature')) + .mockResolvedValueOnce(defaultResponse); + + const nextMiddleware = jest.fn( + (next: MiddlewareHandler) => (request: any) => next(request), + ); + const retryableHandler = composeTransferHandler<[RetryOptions, any]>( + coreHandler, + [retryMiddlewareFactory, () => nextMiddleware], + ); + const retryDecider = jest.fn().mockImplementation((resp, error) => ({ + retryable: error?.message === 'InvalidSignature', + isCredentialsExpiredError: error?.message === 'InvalidSignature', + })); + const response = await retryableHandler(defaultRequest, { + ...defaultRetryOptions, + retryDecider, + }); + expect(response).toEqual(expect.objectContaining(defaultResponse)); + expect(coreHandler).toHaveBeenCalledTimes(2); + expect(retryDecider).toHaveBeenCalledTimes(2); + expect(nextMiddleware).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ isCredentialsExpired: true }), + ); + }); + test('should call computeDelay for intervals', async () => { const nextHandler = jest.fn().mockResolvedValue(defaultResponse); const retryableHandler = getRetryableHandler(nextHandler); @@ -152,7 +185,7 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { const nextHandler = jest.fn().mockResolvedValue(defaultResponse); const retryableHandler = getRetryableHandler(nextHandler); const controller = new AbortController(); - const retryDecider = async () => true; + const retryDecider = async () => ({ retryable: true }); const computeDelay = jest.fn().mockImplementation(attempt => { if (attempt === 1) { setTimeout(() => { @@ -204,9 +237,10 @@ describe(`${retryMiddlewareFactory.name} middleware`, () => { const retryDecider = jest .fn() .mockImplementation((response, error: Error) => { - if (error && error.message.endsWith('RetryableError')) return true; + if (error && error.message.endsWith('RetryableError')) + return { retryable: true }; - return false; + return { retryable: false }; }); const computeDelay = jest.fn().mockReturnValue(0); const response = await doubleRetryableHandler(defaultRequest, { diff --git a/packages/core/__tests__/clients/middleware/signing/middleware.test.ts b/packages/core/__tests__/clients/middleware/signing/middleware.test.ts index a3183ebcdb5..874d82e2282 100644 --- a/packages/core/__tests__/clients/middleware/signing/middleware.test.ts +++ b/packages/core/__tests__/clients/middleware/signing/middleware.test.ts @@ -11,6 +11,7 @@ import { getUpdatedSystemClockOffset } from '../../../../src/clients/middleware/ import { HttpRequest, HttpResponse, + Middleware, MiddlewareHandler, } from '../../../../src/clients/types'; @@ -113,6 +114,30 @@ describe('Signing middleware', () => { expect(credentialsProvider).toHaveBeenCalledTimes(1); }); + test('should forceRefresh credentials provider if middleware context isCredentialsInvalid flag is set', async () => { + expect.assertions(2); + const credentialsProvider = jest.fn().mockResolvedValue(credentials); + const nextHandler = jest.fn().mockResolvedValue(defaultResponse); + const setInvalidCredsMiddleware: Middleware = + () => (next, context) => request => { + context.isCredentialsExpired = true; + + return next(request); + }; + const signableHandler = composeTransferHandler< + [any, SigningOptions], + HttpRequest, + HttpResponse + >(nextHandler, [setInvalidCredsMiddleware, signingMiddlewareFactory]); + const config = { + ...defaultSigningOptions, + credentials: credentialsProvider, + }; + await signableHandler(defaultRequest, config); + expect(credentialsProvider).toHaveBeenCalledTimes(1); + expect(credentialsProvider).toHaveBeenCalledWith({ forceRefresh: true }); + }); + test.each([ ['response with Date header', 'Date'], ['response with date header', 'date'], @@ -128,6 +153,7 @@ describe('Signing middleware', () => { const middlewareFunction = signingMiddlewareFactory(defaultSigningOptions)( nextHandler, + {}, ); await middlewareFunction(defaultRequest); diff --git a/packages/core/src/Platform/types.ts b/packages/core/src/Platform/types.ts index 628de42ef6a..1e4ef8dd6ae 100644 --- a/packages/core/src/Platform/types.ts +++ b/packages/core/src/Platform/types.ts @@ -132,6 +132,8 @@ export enum StorageAction { Remove = '5', GetProperties = '6', GetUrl = '7', + GetDataAccess = '8', + ListCallerAccessGrants = '9', } interface ActionMap { diff --git a/packages/core/src/clients/index.ts b/packages/core/src/clients/index.ts index a06067604bc..31abf267c77 100644 --- a/packages/core/src/clients/index.ts +++ b/packages/core/src/clients/index.ts @@ -15,9 +15,14 @@ export { } from './middleware/signing/signer/signatureV4'; export { EMPTY_HASH as EMPTY_SHA256_HASH } from './middleware/signing/signer/signatureV4/constants'; export { extendedEncodeURIComponent } from './middleware/signing/utils/extendedEncodeURIComponent'; -export { signingMiddlewareFactory, SigningOptions } from './middleware/signing'; +export { + signingMiddlewareFactory, + SigningOptions, + CredentialsProviderOptions, +} from './middleware/signing'; export { getRetryDecider, + RetryDeciderOutput, jitteredBackoff, retryMiddlewareFactory, RetryOptions, diff --git a/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts b/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts index 874cc74314e..edec193ebf1 100644 --- a/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts +++ b/packages/core/src/clients/middleware/retry/defaultRetryDecider.ts @@ -4,6 +4,7 @@ import { ErrorParser, HttpResponse } from '../../types'; import { isClockSkewError } from './isClockSkewError'; +import { RetryDeciderOutput } from './types'; /** * Get retry decider function @@ -11,7 +12,10 @@ import { isClockSkewError } from './isClockSkewError'; */ export const getRetryDecider = (errorParser: ErrorParser) => - async (response?: HttpResponse, error?: unknown): Promise => { + async ( + response?: HttpResponse, + error?: unknown, + ): Promise => { const parsedError = (error as Error & { code: string }) ?? (await errorParser(response)) ?? @@ -19,12 +23,15 @@ export const getRetryDecider = const errorCode = parsedError?.code || parsedError?.name; const statusCode = response?.statusCode; - return ( + const isRetryable = isConnectionError(error) || isThrottlingError(statusCode, errorCode) || isClockSkewError(errorCode) || - isServerSideError(statusCode, errorCode) - ); + isServerSideError(statusCode, errorCode); + + return { + retryable: isRetryable, + }; }; // reference: https://github.com/aws/aws-sdk-js-v3/blob/ab0e7be36e7e7f8a0c04834357aaad643c7912c3/packages/service-error-classification/src/constants.ts#L22-L37 diff --git a/packages/core/src/clients/middleware/retry/index.ts b/packages/core/src/clients/middleware/retry/index.ts index 4c82c603508..fdf34552fa7 100644 --- a/packages/core/src/clients/middleware/retry/index.ts +++ b/packages/core/src/clients/middleware/retry/index.ts @@ -4,3 +4,4 @@ export { RetryOptions, retryMiddlewareFactory } from './middleware'; export { jitteredBackoff } from './jitteredBackoff'; export { getRetryDecider } from './defaultRetryDecider'; +export { RetryDeciderOutput } from './types'; diff --git a/packages/core/src/clients/middleware/retry/middleware.ts b/packages/core/src/clients/middleware/retry/middleware.ts index bce886abb73..9bf7e093030 100644 --- a/packages/core/src/clients/middleware/retry/middleware.ts +++ b/packages/core/src/clients/middleware/retry/middleware.ts @@ -8,6 +8,8 @@ import { Response, } from '../../types/core'; +import { RetryDeciderOutput } from './types'; + const DEFAULT_RETRY_ATTEMPTS = 3; /** @@ -19,9 +21,14 @@ export interface RetryOptions { * * @param response Optional response of the request. * @param error Optional error thrown from previous attempts. + * @param middlewareContext Optional context object to store data between retries. * @returns True if the request should be retried. */ - retryDecider(response?: TResponse, error?: unknown): Promise; + retryDecider( + response?: TResponse, + error?: unknown, + middlewareContext?: MiddlewareContext, + ): Promise; /** * Function to compute the delay in milliseconds before the next retry based * on the number of attempts. @@ -87,7 +94,14 @@ export const retryMiddlewareFactory = ({ ? (context.attemptsCount ?? 0) : attemptsCount + 1; context.attemptsCount = attemptsCount; - if (await retryDecider(response, error)) { + const { isCredentialsExpiredError, retryable } = await retryDecider( + response, + error, + context, + ); + if (retryable) { + // Setting isCredentialsInvalid flag to notify signing middleware to forceRefresh credentials provider. + context.isCredentialsExpired = !!isCredentialsExpiredError; if (!abortSignal?.aborted && attemptsCount < maxAttempts) { // prevent sleep for last attempt or cancelled request; const delay = computeDelay(attemptsCount); diff --git a/packages/core/src/clients/middleware/retry/types.ts b/packages/core/src/clients/middleware/retry/types.ts new file mode 100644 index 00000000000..a229216edee --- /dev/null +++ b/packages/core/src/clients/middleware/retry/types.ts @@ -0,0 +1,7 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export interface RetryDeciderOutput { + retryable: boolean; + isCredentialsExpiredError?: boolean; +} diff --git a/packages/core/src/clients/middleware/signing/index.ts b/packages/core/src/clients/middleware/signing/index.ts index a1458bca3e4..1ce90db4b7e 100644 --- a/packages/core/src/clients/middleware/signing/index.ts +++ b/packages/core/src/clients/middleware/signing/index.ts @@ -1,4 +1,8 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -export { signingMiddlewareFactory, SigningOptions } from './middleware'; +export { + signingMiddlewareFactory, + SigningOptions, + CredentialsProviderOptions, +} from './middleware'; diff --git a/packages/core/src/clients/middleware/signing/middleware.ts b/packages/core/src/clients/middleware/signing/middleware.ts index a7bed1e6b7f..1b36519729e 100644 --- a/packages/core/src/clients/middleware/signing/middleware.ts +++ b/packages/core/src/clients/middleware/signing/middleware.ts @@ -7,16 +7,27 @@ import { HttpResponse, MiddlewareHandler, } from '../../types'; +import { MiddlewareContext } from '../../types/core'; import { signRequest } from './signer/signatureV4'; import { getSkewCorrectedDate } from './utils/getSkewCorrectedDate'; import { getUpdatedSystemClockOffset } from './utils/getUpdatedSystemClockOffset'; +/** + * Options type for the async callback function returning aws credentials. This + * function is used by SigV4 signer to resolve the aws credentials + */ +export interface CredentialsProviderOptions { + forceRefresh?: boolean; +} + /** * Configuration of the signing middleware */ export interface SigningOptions { - credentials: Credentials | (() => Promise); + credentials: + | Credentials + | ((options?: CredentialsProviderOptions) => Promise); region: string; service: string; @@ -41,12 +52,19 @@ export const signingMiddlewareFactory = ({ }: SigningOptions) => { let currentSystemClockOffset: number; - return (next: MiddlewareHandler) => + return ( + next: MiddlewareHandler, + context: MiddlewareContext, + ) => async function signingMiddleware(request: HttpRequest) { currentSystemClockOffset = currentSystemClockOffset ?? 0; const signRequestOptions = { credentials: - typeof credentials === 'function' ? await credentials() : credentials, + typeof credentials === 'function' + ? await credentials({ + forceRefresh: !!context?.isCredentialsExpired, + }) + : credentials, signingDate: getSkewCorrectedDate(currentSystemClockOffset), signingRegion: region, signingService: service, diff --git a/packages/core/src/clients/types/core.ts b/packages/core/src/clients/types/core.ts index 1fa122250b6..a6348655899 100644 --- a/packages/core/src/clients/types/core.ts +++ b/packages/core/src/clients/types/core.ts @@ -30,6 +30,11 @@ export type MiddlewareHandler = ( * The context object to store states across the middleware chain. */ export interface MiddlewareContext { + /** + * Whether an error indicating expired credentials has been returned from server-side. + * This is set by the retry middleware. + */ + isCredentialsExpired?: boolean; /** * The number of times the request has been attempted. This is set by retry middleware */ diff --git a/packages/core/src/clients/types/index.ts b/packages/core/src/clients/types/index.ts index e2b8953a4d2..0ee905fb162 100644 --- a/packages/core/src/clients/types/index.ts +++ b/packages/core/src/clients/types/index.ts @@ -4,6 +4,7 @@ export { Middleware, MiddlewareHandler, + MiddlewareContext, Request, Response, TransferHandler, diff --git a/packages/interactions/package.json b/packages/interactions/package.json index 9b13a17c4e5..99995624fa3 100644 --- a/packages/interactions/package.json +++ b/packages/interactions/package.json @@ -89,19 +89,19 @@ "name": "Interactions (default to Lex v2)", "path": "./dist/esm/index.mjs", "import": "{ Interactions }", - "limit": "52.61 kB" + "limit": "54.05 kB" }, { "name": "Interactions (Lex v2)", "path": "./dist/esm/lex-v2/index.mjs", "import": "{ Interactions }", - "limit": "52.61 kB" + "limit": "54.05 kB" }, { "name": "Interactions (Lex v1)", "path": "./dist/esm/lex-v1/index.mjs", "import": "{ Interactions }", - "limit": "47.41 kB" + "limit": "47.46 kB" } ] } diff --git a/packages/storage/__tests__/internals/apis/getDataAccess.test.ts b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts new file mode 100644 index 00000000000..630e7835f9e --- /dev/null +++ b/packages/storage/__tests__/internals/apis/getDataAccess.test.ts @@ -0,0 +1,131 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; + +import { getDataAccess } from '../../../src/internals/apis/getDataAccess'; +import { getDataAccess as getDataAccessClient } from '../../../src/providers/s3/utils/client/s3control'; +import { GetDataAccessInput } from '../../../src/internals/types/inputs'; + +jest.mock('../../../src/providers/s3/utils/client/s3control'); + +const MOCK_ACCOUNT_ID = 'accountId'; +const MOCK_REGION = 'us-east-2'; +const MOCK_ACCESS_ID = 'accessId'; +const MOCK_SECRET_ACCESS_KEY = 'secretAccessKey'; +const MOCK_SESSION_TOKEN = 'sessionToken'; +const MOCK_EXPIRATION = '2013-09-17T18:07:53.000Z'; +const MOCK_EXPIRATION_DATE = new Date(MOCK_EXPIRATION); +const MOCK_SCOPE = 's3://mybucket/files/*'; +const MOCK_CREDENTIALS = { + credentials: { + accessKeyId: MOCK_ACCESS_ID, + secretAccessKey: MOCK_SECRET_ACCESS_KEY, + sessionToken: MOCK_SESSION_TOKEN, + expiration: MOCK_EXPIRATION_DATE, + }, +}; +const MOCK_ACCESS_CREDENTIALS = { + AccessKeyId: MOCK_ACCESS_ID, + SecretAccessKey: MOCK_SECRET_ACCESS_KEY, + SessionToken: MOCK_SESSION_TOKEN, + Expiration: MOCK_EXPIRATION_DATE, +}; +const MOCK_CREDENTIAL_PROVIDER = jest.fn().mockResolvedValue(MOCK_CREDENTIALS); + +const sharedGetDataAccessParams: GetDataAccessInput = { + accountId: MOCK_ACCOUNT_ID, + credentialsProvider: MOCK_CREDENTIAL_PROVIDER, + durationSeconds: 900, + permission: 'READWRITE', + region: MOCK_REGION, + scope: MOCK_SCOPE, +}; + +describe('getDataAccess', () => { + const getDataAccessClientMock = jest.mocked(getDataAccessClient); + + beforeEach(() => { + jest.clearAllMocks(); + + getDataAccessClientMock.mockResolvedValue({ + Credentials: MOCK_ACCESS_CREDENTIALS, + MatchedGrantTarget: MOCK_SCOPE, + $metadata: {}, + }); + }); + + it('should invoke the getDataAccess client correctly', async () => { + expect.assertions(6); + const result = await getDataAccess(sharedGetDataAccessParams); + + expect(getDataAccessClientMock).toHaveBeenCalledWith( + expect.objectContaining({ + credentials: expect.any(Function), + region: MOCK_REGION, + userAgentValue: expect.stringContaining('storage/8'), + }), + expect.objectContaining({ + AccountId: MOCK_ACCOUNT_ID, + Target: MOCK_SCOPE, + Permission: 'READWRITE', + TargetType: undefined, + DurationSeconds: 900, + }), + ); + const inputCredentialsProvider = getDataAccessClientMock.mock.calls[0][0] + .credentials as (input: CredentialsProviderOptions) => any; + expect(inputCredentialsProvider).toBeInstanceOf(Function); + await expect( + inputCredentialsProvider({ forceRefresh: true }), + ).resolves.toEqual(MOCK_CREDENTIALS.credentials); + expect(MOCK_CREDENTIAL_PROVIDER).toHaveBeenCalledWith({ + forceRefresh: true, + }); + + expect(result.credentials).toEqual(MOCK_CREDENTIALS.credentials); + expect(result.scope).toEqual(MOCK_SCOPE); + }); + + it('should throw an error if the service does not return credentials', async () => { + expect.assertions(1); + + getDataAccessClientMock.mockResolvedValue({ + Credentials: undefined, + MatchedGrantTarget: MOCK_SCOPE, + $metadata: {}, + }); + + expect(getDataAccess(sharedGetDataAccessParams)).rejects.toThrow( + 'Service did not return valid temporary credentials.', + ); + }); + + it('should set the correct target type when accessing an object', async () => { + const MOCK_OBJECT_SCOPE = 's3://mybucket/files/file.md'; + + getDataAccessClientMock.mockResolvedValue({ + Credentials: MOCK_ACCESS_CREDENTIALS, + MatchedGrantTarget: MOCK_OBJECT_SCOPE, + $metadata: {}, + }); + + const result = await getDataAccess({ + ...sharedGetDataAccessParams, + scope: MOCK_OBJECT_SCOPE, + }); + + expect(getDataAccessClientMock).toHaveBeenCalledWith( + expect.any(Object), + expect.objectContaining({ + AccountId: MOCK_ACCOUNT_ID, + Target: MOCK_OBJECT_SCOPE, + Permission: 'READWRITE', + TargetType: 'Object', + DurationSeconds: 900, + }), + ); + + expect(result.scope).toEqual(MOCK_OBJECT_SCOPE); + }); +}); diff --git a/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts new file mode 100644 index 00000000000..f5067843129 --- /dev/null +++ b/packages/storage/__tests__/internals/apis/listCallerAccessGrants.test.ts @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; + +import { listCallerAccessGrants } from '../../../src/internals/apis/listCallerAccessGrants'; +import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../../src/providers/s3/utils/client/s3control'; + +jest.mock('../../../src/providers/s3/utils/client/s3control'); + +const mockAccountId = '1234567890'; +const mockRegion = 'us-foo-2'; +const mockCredentials = { + accessKeyId: 'key', + secretAccessKey: 'secret', + sessionToken: 'session', + expiration: new Date(), +}; +const mockCredentialsProvider = jest + .fn() + .mockResolvedValue({ credentials: mockCredentials }); +const mockNextToken = '123'; +const mockPageSize = 123; + +describe('listCallerAccessGrants', () => { + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should invoke the listCallerAccessGrants client with expected parameters', async () => { + expect.assertions(4); + jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ + NextToken: undefined, + CallerAccessGrantsList: [], + $metadata: {} as any, + }); + await listCallerAccessGrants({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + nextToken: mockNextToken, + pageSize: mockPageSize, + }); + expect(listCallerAccessGrantsClient).toHaveBeenCalledWith( + expect.objectContaining({ + region: mockRegion, + credentials: expect.any(Function), + }), + expect.objectContaining({ + AccountId: mockAccountId, + NextToken: mockNextToken, + MaxResults: mockPageSize, + }), + ); + const inputCredentialsProvider = jest.mocked(listCallerAccessGrantsClient) + .mock.calls[0][0].credentials as ( + input: CredentialsProviderOptions, + ) => any; + expect(inputCredentialsProvider).toBeInstanceOf(Function); + await expect( + inputCredentialsProvider({ forceRefresh: true }), + ).resolves.toEqual(mockCredentials); + expect(mockCredentialsProvider).toHaveBeenCalledWith({ + forceRefresh: true, + }); + }); + + it('should set a default page size', async () => { + expect.assertions(1); + jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ + NextToken: undefined, + CallerAccessGrantsList: [], + $metadata: {} as any, + }); + await listCallerAccessGrants({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + }); + expect(listCallerAccessGrantsClient).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ + MaxResults: 1000, + }), + ); + }); + + it('should set response location type correctly', async () => { + expect.assertions(2); + jest.mocked(listCallerAccessGrantsClient).mockResolvedValue({ + NextToken: undefined, + CallerAccessGrantsList: [ + { + GrantScope: 's3://bucket/*', + Permission: 'READ', + }, + { + GrantScope: 's3://bucket/path/*', + Permission: 'READWRITE', + }, + { + GrantScope: 's3://bucket/path/to/object', + Permission: 'READ', + ApplicationArn: 'arn:123', + }, + ], + $metadata: {} as any, + }); + const { locations, nextToken } = await listCallerAccessGrants({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + }); + + expect(locations).toEqual([ + { + scope: 's3://bucket/*', + type: 'BUCKET', + permission: 'READ', + }, + { + scope: 's3://bucket/path/*', + type: 'PREFIX', + permission: 'READWRITE', + }, + { + scope: 's3://bucket/path/to/object', + type: 'OBJECT', + permission: 'READ', + }, + ]); + expect(nextToken).toBeUndefined(); + }); +}); diff --git a/packages/storage/__tests__/internals/locationCredentialsStore/create.test.ts b/packages/storage/__tests__/internals/locationCredentialsStore/create.test.ts new file mode 100644 index 00000000000..206a2573fa5 --- /dev/null +++ b/packages/storage/__tests__/internals/locationCredentialsStore/create.test.ts @@ -0,0 +1,100 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createLocationCredentialsStore } from '../../../src/internals/locationCredentialsStore/create'; +import { + createStore, + getValue, + removeStore, +} from '../../../src/internals/locationCredentialsStore/registry'; +import { LocationCredentialsStore } from '../../../src/internals/types/credentials'; +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; +import { AWSTemporaryCredentials } from '../../../src/providers/s3/types/options'; + +jest.mock('../../../src/internals/locationCredentialsStore/registry'); + +const mockedCredentials = 'MOCK_CREDS' as any as AWSTemporaryCredentials; + +describe('createLocationCredentialsStore', () => { + it('should create a store', () => { + const refreshHandler = jest.fn(); + const store = createLocationCredentialsStore({ handler: refreshHandler }); + + expect(createStore).toHaveBeenCalledWith(refreshHandler); + expect(store.getProvider).toBeDefined(); + expect(store.destroy).toBeDefined(); + }); + + describe('created store', () => { + describe('getProvider()', () => { + let store: LocationCredentialsStore; + + beforeEach(() => { + store = createLocationCredentialsStore({ handler: jest.fn() }); + }); + + afterEach(() => { + jest.clearAllMocks(); + store.destroy(); + }); + + it('should call getValue() from store', async () => { + expect.assertions(2); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + + const locationCredentialsProvider = store.getProvider({ + scope: 's3://bucket/path/*', + permission: 'READ', + }); + const { credentials } = await locationCredentialsProvider(); + expect(credentials).toEqual(mockedCredentials); + expect(getValue).toHaveBeenCalledWith( + expect.objectContaining({ + location: { + scope: 's3://bucket/path/*', + permission: 'READ', + }, + forceRefresh: false, + }), + ); + }); + + it.each(['invalid-s3-uri', 's3://', 's3:///'])( + 'should throw if location credentials provider scope is not a valid S3 URI "%s"', + async invalidScope => { + expect.assertions(1); + jest + .mocked(getValue) + .mockResolvedValue({ credentials: mockedCredentials }); + const locationCredentialsProvider = store.getProvider({ + scope: invalidScope, + permission: 'READWRITE', + }); + try { + await locationCredentialsProvider(); + } catch (e: any) { + expect(e.message).toEqual( + validationErrorMap[StorageValidationErrorCode.InvalidS3Uri] + .message, + ); + } + }, + ); + }); + + describe('destroy()', () => { + it('should call removeStore() from store', () => { + const store = createLocationCredentialsStore({ + handler: jest.fn(), + }); + store.destroy(); + expect(removeStore).toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/packages/storage/__tests__/internals/locationCredentialsStore/registry.test.ts b/packages/storage/__tests__/internals/locationCredentialsStore/registry.test.ts new file mode 100644 index 00000000000..9aa5b0c305a --- /dev/null +++ b/packages/storage/__tests__/internals/locationCredentialsStore/registry.test.ts @@ -0,0 +1,187 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; +import { AWSTemporaryCredentials } from '../../../src/providers/s3/types/options'; +import { + createStore, + getValue, + removeStore, +} from '../../../src/internals/locationCredentialsStore/registry'; +import { + LruLocationCredentialsStore, + fetchNewValue, + getCacheValue, + initStore, +} from '../../../src/internals/locationCredentialsStore/store'; + +jest.mock('../../../src/internals/locationCredentialsStore/store'); + +const mockedStore = 'MOCKED_STORE' as any as LruLocationCredentialsStore; + +afterEach(() => { + jest.clearAllMocks(); +}); + +beforeEach(() => { + jest.mocked(initStore).mockReturnValue(mockedStore); +}); + +describe('createStore', () => { + it('should create a store with given capacity, refresh Handler', () => { + const refreshHandler = jest.fn(); + createStore(refreshHandler, 20); + expect(initStore).toHaveBeenCalledWith(refreshHandler, 20); + }); + + it('should return a symbol to refer the store instance', () => { + const storeReference = createStore(jest.fn(), 20); + expect(Object.prototype.toString.call(storeReference.value)).toBe( + '[object Symbol]', + ); + }); +}); + +describe('getValue', () => { + const mockCachedValue = 'CACHED_VALUE' as any as AWSTemporaryCredentials; + let storeSymbol: { value: symbol }; + beforeEach(() => { + storeSymbol = createStore(jest.fn(), 20); + }); + afterEach(() => { + removeStore(storeSymbol); + jest.clearAllMocks(); + }); + + it('should throw if a store instance cannot be found from registry', async () => { + expect.assertions(1); + await expect( + getValue({ + storeSymbol: { value: Symbol('invalid') }, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).rejects.toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsStoreDestroyed + ].message, + ); + }); + + it('should look up a cache value for given location and permission', async () => { + expect.assertions(2); + jest.mocked(getCacheValue).mockReturnValueOnce(mockCachedValue); + expect( + await getValue({ + storeSymbol, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).toEqual({ credentials: mockCachedValue }); + expect(getCacheValue).toHaveBeenCalledWith(mockedStore, { + scope: 'abc', + permission: 'READ', + }); + }); + + it('should look up a cache value for given location and READWRITE permission', async () => { + expect.assertions(4); + + jest.mocked(getCacheValue).mockReturnValueOnce(null); + jest.mocked(getCacheValue).mockReturnValueOnce(mockCachedValue); + expect( + await getValue({ + storeSymbol, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).toEqual({ credentials: mockCachedValue }); + expect(getCacheValue).toHaveBeenCalledTimes(2); + expect(getCacheValue).toHaveBeenNthCalledWith(1, mockedStore, { + scope: 'abc', + permission: 'READ', + }); + expect(getCacheValue).toHaveBeenNthCalledWith(2, mockedStore, { + scope: 'abc', + permission: 'READWRITE', + }); + }); + + it('should invoke the refresh handler if look up returns null', async () => { + expect.assertions(3); + jest.mocked(getCacheValue).mockReturnValue(null); + jest.mocked(fetchNewValue).mockResolvedValue('NEW_VALUE' as any); + expect( + await getValue({ + storeSymbol, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).toEqual('NEW_VALUE'); + expect(fetchNewValue).toHaveBeenCalledTimes(1); + expect(fetchNewValue).toHaveBeenCalledWith(mockedStore, { + scope: 'abc', + permission: 'READ', + }); + }); + + it('should invoke the refresh handler regardless of cache if forceRefresh is true', async () => { + expect.assertions(3); + jest.mocked(getCacheValue).mockReturnValue(mockCachedValue); + jest.mocked(fetchNewValue).mockResolvedValue('NEW_VALUE' as any); + expect( + await getValue({ + storeSymbol, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: true, + }), + ).toEqual('NEW_VALUE'); + expect(fetchNewValue).toHaveBeenCalledTimes(1); + expect(fetchNewValue).toHaveBeenCalledWith(mockedStore, { + scope: 'abc', + permission: 'READ', + }); + }); + + it('should throw if refresh handler throws', async () => { + expect.assertions(1); + jest + .mocked(fetchNewValue) + .mockRejectedValueOnce(new Error('Network error')); + await expect( + getValue({ + storeSymbol, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: true, + }), + ).rejects.toThrow('Network error'); + }); +}); + +describe('removeStore', () => { + it('should remove the store with given symbol', async () => { + expect.assertions(1); + const storeReference = createStore(jest.fn(), 20); + removeStore(storeReference); + await expect( + getValue({ + storeSymbol: storeReference, + location: { scope: 'abc', permission: 'READ' }, + forceRefresh: false, + }), + ).rejects.toThrow( + validationErrorMap[ + StorageValidationErrorCode.LocationCredentialsStoreDestroyed + ].message, + ); + }); + + it('should not throw if store with given symbol does not exist', () => { + expect(() => { + removeStore({ value: Symbol('invalid') }); + }).not.toThrow(); + }); +}); diff --git a/packages/storage/__tests__/internals/locationCredentialsStore/store.test.ts b/packages/storage/__tests__/internals/locationCredentialsStore/store.test.ts new file mode 100644 index 00000000000..6bc34132147 --- /dev/null +++ b/packages/storage/__tests__/internals/locationCredentialsStore/store.test.ts @@ -0,0 +1,221 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + StorageValidationErrorCode, + validationErrorMap, +} from '../../../src/errors/types/validation'; +import { + fetchNewValue, + getCacheValue, + initStore, +} from '../../../src/internals/locationCredentialsStore/store'; +import { CredentialsLocation } from '../../../src/internals/types/credentials'; + +const mockCredentials = { + expiration: new Date(Date.now() + 60 * 60_1000), +}; + +describe('initStore', () => { + it('should create a store with given capacity, refresh Handler and values', () => { + const refreshHandler = jest.fn(); + const store = initStore(refreshHandler, 20); + expect(store).toEqual({ + capacity: 20, + refreshHandler, + values: expect.any(Map), + }); + }); + + it('should create a store with default capacity if not provided', () => { + const store = initStore(jest.fn()); + expect(store).toMatchObject({ + capacity: 10, + }); + }); + + it('should throw if capacity is not > 0', () => { + expect(() => initStore(jest.fn(), 0)).toThrow( + validationErrorMap[ + StorageValidationErrorCode.InvalidLocationCredentialsCacheSize + ].message, + ); + }); +}); + +describe('getCacheValue', () => { + it('should return a cache value for given location and permission', () => { + const cachedValue = { + credentials: mockCredentials, + scope: 'abc', + permission: 'READ', + } as any; + const store = initStore(jest.fn()); + store.values.set('abc_READ', cachedValue); + expect( + getCacheValue(store, { + scope: 'abc', + permission: 'READ', + }), + ).toEqual(cachedValue.credentials); + }); + + it('should return null if cache value is not found', () => { + expect( + getCacheValue(initStore(jest.fn()), { + scope: 'abc', + permission: 'READ', + }), + ).toBeNull(); + }); + + it('should return null if cache value is expired', () => { + const expiredValue = { + credentials: { + expiration: new Date(), + }, + scope: 'abc', + permission: 'READ', + } as any; + const store = initStore(jest.fn()); + store.values.set('abc_READ', expiredValue); + expect( + getCacheValue(store, { + scope: 'abc', + permission: 'READ', + }), + ).toBeNull(); + expect(store.values.size).toBe(0); + }); + + it('should return null if cache value is expiring soon', () => { + const expiringValue = { + credentials: { + expiration: new Date(Date.now() + 1000 * 20), // 20 seconds + }, + scope: 'abc', + permission: 'READ', + } as any; + const store = initStore(jest.fn()); + store.values.set('abc_READ', expiringValue); + expect( + getCacheValue(store, { + scope: 'abc', + permission: 'READ', + }), + ).toBeNull(); + expect(store.values.size).toBe(0); + }); +}); + +describe('fetchNewValue', () => { + const mockCacheLocation = { + scope: 'abc', + permission: 'READ', + } as CredentialsLocation; + const createCacheKey = (location: CredentialsLocation) => + `${location.scope}_${location.permission}` as const; + + it('should fetch new value from remote source', async () => { + expect.assertions(2); + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + const newCredentials = await fetchNewValue(store, mockCacheLocation); + expect(refreshHandler).toHaveBeenCalledWith({ + scope: 'abc', + permission: 'READ', + }); + expect(newCredentials).toEqual({ + credentials: mockCredentials, + }); + }); + + it('should throw errors when fetching new value', async () => { + expect.assertions(2); + const refreshHandler = jest + .fn() + .mockRejectedValue(new Error('Network error')); + const store = initStore(refreshHandler); + await expect(fetchNewValue(store, mockCacheLocation)).rejects.toThrow( + 'Network error', + ); + expect(store.values.size).toBe(0); + }); + + it('should update cache with new value', async () => { + expect.assertions(1); + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + await fetchNewValue(store, mockCacheLocation); + expect(store.values.get(createCacheKey(mockCacheLocation))).toEqual({ + credentials: mockCredentials, + inflightCredentials: undefined, + scope: 'abc', + permission: 'READ', + }); + }); + + it('should invoke refresh handler only once if multiple fetches for same location is called', async () => { + expect.assertions(1); + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + await Promise.all([ + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + ]); + expect(refreshHandler).toHaveBeenCalledTimes(1); + }); + + it('should invoke the refresh handler if the refresh handler previously fails', async () => { + expect.assertions(4); + const refreshHandler = jest + .fn() + .mockRejectedValueOnce(new Error('Network error')) + .mockResolvedValueOnce({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler); + try { + await Promise.all([ + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + fetchNewValue(store, mockCacheLocation), + ]); + } catch (e) { + expect(e).toEqual(new Error('Network error')); + expect(store.values.size).toBe(0); + } + const { credentials } = await fetchNewValue(store, mockCacheLocation); + expect(credentials).toEqual(mockCredentials); + expect(store.values.size).toBe(1); + }); + + it('should call refresh handler for new cache entry if the cache is full', async () => { + expect.assertions(4); + const refreshHandler = jest.fn().mockResolvedValue({ + credentials: mockCredentials, + }); + const store = initStore(refreshHandler, 1); + const cacheLocation1 = { + scope: 'abc', + permission: 'READ' as const, + }; + const cacheLocation2 = { + scope: 'def', + permission: 'READ' as const, + }; + await fetchNewValue(store, cacheLocation1); + await fetchNewValue(store, cacheLocation2); + expect(refreshHandler).toHaveBeenCalledTimes(2); + expect(store.values.size).toBe(1); + expect(store.values.get(createCacheKey(cacheLocation2))).toBeDefined(); + expect(store.values.get(createCacheKey(cacheLocation1))).toBeUndefined(); + }); +}); diff --git a/packages/storage/__tests__/internals/managedAuthAdapter/createListLocationsHandler.test.ts b/packages/storage/__tests__/internals/managedAuthAdapter/createListLocationsHandler.test.ts new file mode 100644 index 00000000000..26a81b10c3d --- /dev/null +++ b/packages/storage/__tests__/internals/managedAuthAdapter/createListLocationsHandler.test.ts @@ -0,0 +1,34 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createListLocationsHandler } from '../../../src/internals/managedAuthConfigAdapter/createListLocationsHandler'; +import { listCallerAccessGrants } from '../../../src/internals/apis/listCallerAccessGrants'; + +jest.mock('../../../src/internals/apis/listCallerAccessGrants'); + +jest.mocked(listCallerAccessGrants).mockResolvedValue({ + locations: [], +}); + +describe('createListLocationsHandler', () => { + it('should parse the underlying API with right parameters', async () => { + const mockAccountId = '1234567890'; + const mockRegion = 'us-foo-1'; + const mockCredentialsProvider = jest.fn(); + const mockNextToken = '123'; + const mockPageSize = 123; + const handler = createListLocationsHandler({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + }); + await handler({ nextToken: mockNextToken, pageSize: mockPageSize }); + expect(listCallerAccessGrants).toHaveBeenCalledWith({ + accountId: mockAccountId, + region: mockRegion, + credentialsProvider: mockCredentialsProvider, + nextToken: mockNextToken, + pageSize: mockPageSize, + }); + }); +}); diff --git a/packages/storage/__tests__/internals/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts b/packages/storage/__tests__/internals/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts new file mode 100644 index 00000000000..74bf7e10bd7 --- /dev/null +++ b/packages/storage/__tests__/internals/managedAuthAdapter/createManagedAuthConfigAdapter.test.ts @@ -0,0 +1,72 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { createManagedAuthConfigAdapter } from '../../../src/internals/managedAuthConfigAdapter'; +import { createListLocationsHandler } from '../../../src/internals/managedAuthConfigAdapter/createListLocationsHandler'; +import { createLocationCredentialsHandler } from '../../../src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler'; + +jest.mock( + '../../../src/internals/managedAuthConfigAdapter/createListLocationsHandler', +); +jest.mock( + '../../../src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler', +); + +describe('createManagedAuthConfigAdapter', () => { + const region = 'us-foo-2'; + const accountId = 'XXXXXXXXXXXX'; + const credentialsProvider = jest.fn(); + + beforeEach(() => { + jest + .mocked(createListLocationsHandler) + .mockReturnValue('LIST_LOCATIONS_FN' as any); + jest + .mocked(createLocationCredentialsHandler) + .mockReturnValue('GET_LOCATION_CREDENTIALS_FN' as any); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should pass region to the adapter', () => { + expect(createManagedAuthConfigAdapter({ region } as any)).toMatchObject({ + region, + }); + }); + + it('should create list locations handler', () => { + expect( + createManagedAuthConfigAdapter({ + region, + accountId, + credentialsProvider, + }), + ).toMatchObject({ + listLocations: 'LIST_LOCATIONS_FN', + }); + expect(createListLocationsHandler).toHaveBeenCalledWith({ + region, + accountId, + credentialsProvider, + }); + }); + + it('should create get location credentials handler', () => { + expect( + createManagedAuthConfigAdapter({ + region, + accountId, + credentialsProvider, + }), + ).toMatchObject({ + getLocationCredentials: 'GET_LOCATION_CREDENTIALS_FN', + }); + expect(createLocationCredentialsHandler).toHaveBeenCalledWith({ + region, + accountId, + credentialsProvider, + }); + }); +}); diff --git a/packages/storage/__tests__/providers/s3/apis/copy.test.ts b/packages/storage/__tests__/providers/s3/apis/copy.test.ts index 7ddd0430dd8..56104e84d17 100644 --- a/packages/storage/__tests__/providers/s3/apis/copy.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/copy.test.ts @@ -6,7 +6,7 @@ import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; import { StorageError } from '../../../../src/errors/StorageError'; import { StorageValidationErrorCode } from '../../../../src/errors/types/validation'; -import { copyObject } from '../../../../src/providers/s3/utils/client'; +import { copyObject } from '../../../../src/providers/s3/utils/client/s3data'; import { copy } from '../../../../src/providers/s3/apis'; import { CopyInput, @@ -17,7 +17,7 @@ import { import './testUtils'; import { BucketInfo } from '../../../../src/providers/s3/types/options'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts b/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts index 35b790366bc..aab4d1b8260 100644 --- a/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/downloadData.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { getObject } from '../../../../src/providers/s3/utils/client'; +import { getObject } from '../../../../src/providers/s3/utils/client/s3data'; import { downloadData } from '../../../../src/providers/s3'; import { createDownloadTask, @@ -26,7 +26,7 @@ import { import './testUtils'; import { BucketInfo } from '../../../../src/providers/s3/types/options'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('../../../../src/providers/s3/utils'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { diff --git a/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts b/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts index 0fcd989453e..cfde93eb069 100644 --- a/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/getProperties.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { headObject } from '../../../../src/providers/s3/utils/client'; +import { headObject } from '../../../../src/providers/s3/utils/client/s3data'; import { getProperties } from '../../../../src/providers/s3'; import { GetPropertiesInput, @@ -15,7 +15,7 @@ import { import './testUtils'; import { BucketInfo } from '../../../../src/providers/s3/types/options'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts b/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts index 52e65ddd1b0..7383e34f531 100644 --- a/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/getUrl.test.ts @@ -8,7 +8,7 @@ import { getUrl } from '../../../../src/providers/s3/apis'; import { getPresignedGetObjectUrl, headObject, -} from '../../../../src/providers/s3/utils/client'; +} from '../../../../src/providers/s3/utils/client/s3data'; import { GetUrlInput, GetUrlOutput, @@ -18,7 +18,7 @@ import { import './testUtils'; import { BucketInfo } from '../../../../src/providers/s3/types/options'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/list.test.ts b/packages/storage/__tests__/providers/s3/apis/list.test.ts index a13ae54b5a4..a3ca6a375cb 100644 --- a/packages/storage/__tests__/providers/s3/apis/list.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/list.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { listObjectsV2 } from '../../../../src/providers/s3/utils/client'; +import { listObjectsV2 } from '../../../../src/providers/s3/utils/client/s3data'; import { list } from '../../../../src/providers/s3'; import { ListAllInput, @@ -18,7 +18,7 @@ import { } from '../../../../src/providers/s3/types'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/remove.test.ts b/packages/storage/__tests__/providers/s3/apis/remove.test.ts index eb3407eb610..16adafd3e7c 100644 --- a/packages/storage/__tests__/providers/s3/apis/remove.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/remove.test.ts @@ -4,7 +4,7 @@ import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, StorageAccessLevel } from '@aws-amplify/core'; -import { deleteObject } from '../../../../src/providers/s3/utils/client'; +import { deleteObject } from '../../../../src/providers/s3/utils/client/s3data'; import { remove } from '../../../../src/providers/s3/apis'; import { StorageValidationErrorCode } from '../../../../src/errors/types/validation'; import { @@ -15,7 +15,7 @@ import { } from '../../../../src/providers/s3/types'; import './testUtils'; -jest.mock('../../../../src/providers/s3/utils/client'); +jest.mock('../../../../src/providers/s3/utils/client/s3data'); jest.mock('@aws-amplify/core', () => ({ ConsoleLogger: jest.fn().mockImplementation(function ConsoleLogger() { return { debug: jest.fn() }; diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts index 8957c9ef764..695822f57c8 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/multipartHandlers.test.ts @@ -1,6 +1,9 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; + import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify, defaultStorage } from '@aws-amplify/core'; @@ -11,7 +14,7 @@ import { headObject, listParts, uploadPart, -} from '../../../../../src/providers/s3/utils/client'; +} from '../../../../../src/providers/s3/utils/client/s3data'; import { getMultipartUploadHandlers } from '../../../../../src/providers/s3/apis/uploadData/multipart'; import { StorageValidationErrorCode, @@ -22,9 +25,16 @@ import { byteLength } from '../../../../../src/providers/s3/apis/uploadData/byte import { CanceledError } from '../../../../../src/errors/CanceledError'; import { StorageOptions } from '../../../../../src/types'; import '../testUtils'; +import { calculateContentCRC32 } from '../../../../../src/providers/s3/utils/crc32'; +import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; jest.mock('@aws-amplify/core'); -jest.mock('../../../../../src/providers/s3/utils/client'); +jest.mock('../../../../../src/providers/s3/utils/client/s3data'); +jest.mock('../../../../../src/providers/s3/utils/crc32'); const credentials: AWSCredentials = { accessKeyId: 'accessKeyId', @@ -47,11 +57,44 @@ const mockCompleteMultipartUpload = jest.mocked(completeMultipartUpload); const mockAbortMultipartUpload = jest.mocked(abortMultipartUpload); const mockListParts = jest.mocked(listParts); const mockHeadObject = jest.mocked(headObject); +const mockCalculateContentCRC32 = jest.mocked(calculateContentCRC32); const disableAssertionFlag = true; const MB = 1024 * 1024; +jest.mock('../../../../../src/providers/s3/utils', () => ({ + ...jest.requireActual('../../../../../src/providers/s3/utils'), + calculateContentMd5: jest.fn(), +})); + +const getZeroDelayTimeout = () => + new Promise(resolve => { + setTimeout(() => { + resolve(); + }, 0); + }); + +const mockCalculateContentCRC32Mock = () => { + mockCalculateContentCRC32.mockReset(); + mockCalculateContentCRC32.mockResolvedValue({ + checksumArrayBuffer: new ArrayBuffer(0), + checksum: 'mockChecksum', + seed: 0, + }); +}; +const mockCalculateContentCRC32Undefined = () => { + mockCalculateContentCRC32.mockReset(); + mockCalculateContentCRC32.mockResolvedValue(undefined); +}; +const mockCalculateContentCRC32Reset = () => { + mockCalculateContentCRC32.mockReset(); + mockCalculateContentCRC32.mockImplementation( + jest.requireActual('../../../../../src/providers/s3/utils/crc32') + .calculateContentCRC32, + ); +}; + const mockMultipartUploadSuccess = (disableAssertion?: boolean) => { let totalSize = 0; mockCreateMultipartUpload.mockResolvedValueOnce({ @@ -149,9 +192,10 @@ describe('getMultipartUploadHandlers with key', () => { }); }); - afterEach(() => { + beforeEach(() => { jest.clearAllMocks(); resetS3Mocks(); + mockCalculateContentCRC32Reset(); }); it('should return multipart upload handlers', async () => { @@ -230,6 +274,69 @@ describe('getMultipartUploadHandlers with key', () => { ); }); + it.each([ + [ + 'file', + new File([getBlob(8 * MB)], 'someName'), + ['JCnBsQ==', 'HELzGQ=='], + ], + ['blob', getBlob(8 * MB), ['JCnBsQ==', 'HELzGQ==']], + ['string', 'Ü'.repeat(4 * MB), ['DL735w==', 'Akga7g==']], + ['arrayBuffer', new ArrayBuffer(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ['arrayBufferView', new Uint8Array(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ])( + `should create crc32 for %s type body`, + async (_, twoPartsPayload, expectedCrc32) => { + mockMultipartUploadSuccess(); + const { multipartUploadJob } = getMultipartUploadHandlers({ + key: defaultKey, + data: twoPartsPayload, + }); + await multipartUploadJob(); + + /** + * final crc32 calculation calls calculateContentCRC32 3 times + * 1 time for each of the 2 parts + * 1 time to combine the resulting hash for each of the two parts + * + * uploading each part calls calculateContentCRC32 1 time each + * + * these steps results in 5 calls in total + */ + expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentMd5).not.toHaveBeenCalled(); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[0] }), + ); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[1] }), + ); + }, + ); + + it('should use md5 if crc32 is returning undefined', async () => { + mockCalculateContentCRC32Undefined(); + mockMultipartUploadSuccess(); + Amplify.libraryOptions = { + Storage: { + S3: { + isObjectLockEnabled: true, + }, + }, + }; + const { multipartUploadJob } = getMultipartUploadHandlers({ + key: defaultKey, + data: new Uint8Array(8 * MB), + }); + await multipartUploadJob(); + expect(calculateContentCRC32).toHaveBeenCalledTimes(1); // (final crc32 calculation = 1 undefined) + expect(calculateContentMd5).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + }); + it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ @@ -244,6 +351,7 @@ describe('getMultipartUploadHandlers with key', () => { }); it('should upload a body that exceeds the size of default part size and parts count', async () => { + mockCalculateContentCRC32Mock(); let buffer: ArrayBuffer; const file = { __proto__: File.prototype, @@ -268,7 +376,7 @@ describe('getMultipartUploadHandlers with key', () => { file.size, ); await multipartUploadJob(); - expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count + expect(file.slice).toHaveBeenCalledTimes(10_000 * 2); // S3 limit of parts count double for crc32 calculations expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockUploadPart).toHaveBeenCalledTimes(10_000); expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1); @@ -615,6 +723,15 @@ describe('getMultipartUploadHandlers with key', () => { describe('pause() & resume()', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { + let pausedOnce = false; + + let resumeTest: () => void; + const waitForPause = new Promise(resolve => { + resumeTest = () => { + resolve(); + }; + }); + const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ key: defaultKey, @@ -623,16 +740,21 @@ describe('getMultipartUploadHandlers with key', () => { let partCount = 0; mockMultipartUploadCancellation(() => { partCount++; - if (partCount === 2) { + if (partCount === 2 && !pausedOnce) { onPause(); // Pause upload at the the last uploadPart call + resumeTest(); + pausedOnce = true; } }); const uploadPromise = multipartUploadJob(); + await waitForPause; + await getZeroDelayTimeout(); onResume(); await uploadPromise; - expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(3); expect(mockUploadPart.mock.calls[0][0].abortSignal?.aborted).toBe(true); expect(mockUploadPart.mock.calls[1][0].abortSignal?.aborted).toBe(true); + expect(mockUploadPart.mock.calls[2][0].abortSignal?.aborted).toBe(false); }); }); @@ -729,9 +851,10 @@ describe('getMultipartUploadHandlers with path', () => { }); }); - afterEach(() => { + beforeEach(() => { jest.clearAllMocks(); resetS3Mocks(); + mockCalculateContentCRC32Reset(); }); it('should return multipart upload handlers', async () => { @@ -802,6 +925,69 @@ describe('getMultipartUploadHandlers with path', () => { ); }); + it.each([ + [ + 'file', + new File([getBlob(8 * MB)], 'someName'), + ['JCnBsQ==', 'HELzGQ=='], + ], + ['blob', getBlob(8 * MB), ['JCnBsQ==', 'HELzGQ==']], + ['string', 'Ü'.repeat(4 * MB), ['DL735w==', 'Akga7g==']], + ['arrayBuffer', new ArrayBuffer(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ['arrayBufferView', new Uint8Array(8 * MB), ['yTuzdQ==', 'eXJPxg==']], + ])( + `should create crc32 for %s type body`, + async (_, twoPartsPayload, expectedCrc32) => { + mockMultipartUploadSuccess(); + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: twoPartsPayload, + }); + await multipartUploadJob(); + + /** + * final crc32 calculation calls calculateContentCRC32 3 times + * 1 time for each of the 2 parts + * 1 time to combine the resulting hash for each of the two parts + * + * uploading each part calls calculateContentCRC32 1 time each + * + * these steps results in 5 calls in total + */ + expect(calculateContentCRC32).toHaveBeenCalledTimes(5); + expect(calculateContentMd5).not.toHaveBeenCalled(); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[0] }), + ); + expect(mockUploadPart).toHaveBeenCalledWith( + expect.anything(), + expect.objectContaining({ ChecksumCRC32: expectedCrc32[1] }), + ); + }, + ); + + it('should use md5 if crc32 is returning undefined', async () => { + mockCalculateContentCRC32Undefined(); + mockMultipartUploadSuccess(); + Amplify.libraryOptions = { + Storage: { + S3: { + isObjectLockEnabled: true, + }, + }, + }; + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new Uint8Array(8 * MB), + }); + await multipartUploadJob(); + expect(calculateContentCRC32).toHaveBeenCalledTimes(1); // (final crc32 calculation = 1 undefined) + expect(calculateContentMd5).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + }); + it('should throw if unsupported payload type is provided', async () => { mockMultipartUploadSuccess(); const { multipartUploadJob } = getMultipartUploadHandlers({ @@ -816,6 +1002,7 @@ describe('getMultipartUploadHandlers with path', () => { }); it('should upload a body that exceeds the size of default part size and parts count', async () => { + mockCalculateContentCRC32Mock(); let buffer: ArrayBuffer; const file = { __proto__: File.prototype, @@ -840,7 +1027,7 @@ describe('getMultipartUploadHandlers with path', () => { file.size, ); await multipartUploadJob(); - expect(file.slice).toHaveBeenCalledTimes(10_000); // S3 limit of parts count + expect(file.slice).toHaveBeenCalledTimes(10_000 * 2); // S3 limit of parts count double for crc32 calculations expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); expect(mockUploadPart).toHaveBeenCalledTimes(10_000); expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1); @@ -921,6 +1108,84 @@ describe('getMultipartUploadHandlers with path', () => { expect(mockCompleteMultipartUpload).not.toHaveBeenCalled(); }); + describe('overwrite prevention', () => { + beforeEach(() => { + mockHeadObject.mockReset(); + mockUploadPart.mockReset(); + }); + + it('should upload if target key is not found', async () => { + expect.assertions(7); + const notFoundError = new Error('mock message'); + notFoundError.name = 'NotFound'; + mockHeadObject.mockRejectedValueOnce(notFoundError); + mockMultipartUploadSuccess(); + + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new ArrayBuffer(8 * MB), + options: { preventOverwrite: true }, + }); + await multipartUploadJob(); + + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + await expect(mockHeadObject).toBeLastCalledWithConfigAndInput( + expect.objectContaining({ + credentials, + region, + }), + expect.objectContaining({ + Bucket: bucket, + Key: testPath, + }), + ); + expect(mockCompleteMultipartUpload).toHaveBeenCalledTimes(1); + }); + + it('should not upload if target key already exists', async () => { + expect.assertions(6); + mockHeadObject.mockResolvedValueOnce({ + ContentLength: 0, + $metadata: {}, + }); + mockMultipartUploadSuccess(); + + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new ArrayBuffer(8 * MB), + options: { preventOverwrite: true }, + }); + + await expect(multipartUploadJob()).rejects.toThrow( + 'At least one of the pre-conditions you specified did not hold', + ); + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockCompleteMultipartUpload).not.toHaveBeenCalled(); + }); + + it('should not upload if HeadObject fails with other error', async () => { + expect.assertions(6); + const accessDeniedError = new Error('mock error'); + accessDeniedError.name = 'AccessDenied'; + mockHeadObject.mockRejectedValueOnce(accessDeniedError); + mockMultipartUploadSuccess(); + + const { multipartUploadJob } = getMultipartUploadHandlers({ + path: testPath, + data: new ArrayBuffer(8 * MB), + options: { preventOverwrite: true }, + }); + + await expect(multipartUploadJob()).rejects.toThrow('mock error'); + expect(mockCreateMultipartUpload).toHaveBeenCalledTimes(1); + expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockCompleteMultipartUpload).not.toHaveBeenCalled(); + }); + }); + describe('bucket passed in options', () => { const mockData = 'Ü'.repeat(4 * MB); it('should override bucket in putObject call when bucket as object', async () => { @@ -1193,6 +1458,14 @@ describe('getMultipartUploadHandlers with path', () => { describe('pause() & resume()', () => { it('should abort in-flight uploadPart requests if upload is paused', async () => { + let pausedOnce = false; + let resumeTest: () => void; + const waitForPause = new Promise(resolve => { + resumeTest = () => { + resolve(); + }; + }); + const { multipartUploadJob, onPause, onResume } = getMultipartUploadHandlers({ path: testPath, @@ -1201,16 +1474,22 @@ describe('getMultipartUploadHandlers with path', () => { let partCount = 0; mockMultipartUploadCancellation(() => { partCount++; - if (partCount === 2) { + if (partCount === 2 && !pausedOnce) { onPause(); // Pause upload at the the last uploadPart call + resumeTest(); + pausedOnce = true; } }); const uploadPromise = multipartUploadJob(); + await waitForPause; + await getZeroDelayTimeout(); + onResume(); await uploadPromise; - expect(mockUploadPart).toHaveBeenCalledTimes(2); + expect(mockUploadPart).toHaveBeenCalledTimes(3); expect(mockUploadPart.mock.calls[0][0].abortSignal?.aborted).toBe(true); expect(mockUploadPart.mock.calls[1][0].abortSignal?.aborted).toBe(true); + expect(mockUploadPart.mock.calls[2][0].abortSignal?.aborted).toBe(false); }); }); diff --git a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts index aa9cf2ff8cd..a76871e2435 100644 --- a/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/uploadData/putObjectJob.test.ts @@ -1,15 +1,26 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; + import { AWSCredentials } from '@aws-amplify/core/internals/utils'; import { Amplify } from '@aws-amplify/core'; -import { putObject } from '../../../../../src/providers/s3/utils/client'; +import { + headObject, + putObject, +} from '../../../../../src/providers/s3/utils/client/s3data'; import { calculateContentMd5 } from '../../../../../src/providers/s3/utils'; +import * as CRC32 from '../../../../../src/providers/s3/utils/crc32'; import { putObjectJob } from '../../../../../src/providers/s3/apis/uploadData/putObjectJob'; import '../testUtils'; -jest.mock('../../../../../src/providers/s3/utils/client'); +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; + +jest.mock('../../../../../src/providers/s3/utils/client/s3data'); jest.mock('../../../../../src/providers/s3/utils', () => { const utils = jest.requireActual('../../../../../src/providers/s3/utils'); @@ -38,6 +49,7 @@ const credentials: AWSCredentials = { const identityId = 'identityId'; const mockFetchAuthSession = jest.mocked(Amplify.Auth.fetchAuthSession); const mockPutObject = jest.mocked(putObject); +const mockHeadObject = jest.mocked(headObject); const bucket = 'bucket'; const region = 'region'; @@ -63,7 +75,7 @@ mockPutObject.mockResolvedValue({ /* TODO Remove suite when `key` parameter is removed */ describe('putObjectJob with key', () => { beforeEach(() => { - mockPutObject.mockClear(); + jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); it('should supply the correct parameters to putObject API handler', async () => { @@ -119,12 +131,16 @@ describe('putObjectJob with key', () => { ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: mockMetadata, - ContentMD5: undefined, + ChecksumCRC32: 'rfPzYw==', }, ); }); it('should set ContentMD5 if object lock is enabled', async () => { + jest + .spyOn(CRC32, 'calculateContentCRC32') + .mockResolvedValue(undefined as any); + Amplify.libraryOptions = { Storage: { S3: { @@ -177,6 +193,7 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -208,6 +225,7 @@ describe('putObjectJob with key', () => { Key: 'public/key', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -217,6 +235,7 @@ describe('putObjectJob with key', () => { describe('putObjectJob with path', () => { beforeEach(() => { mockPutObject.mockClear(); + jest.spyOn(CRC32, 'calculateContentCRC32').mockRestore(); }); test.each([ @@ -282,13 +301,17 @@ describe('putObjectJob with path', () => { ContentDisposition: contentDisposition, ContentEncoding: contentEncoding, Metadata: mockMetadata, - ContentMD5: undefined, + ChecksumCRC32: 'rfPzYw==', }, ); }, ); it('should set ContentMD5 if object lock is enabled', async () => { + jest + .spyOn(CRC32, 'calculateContentCRC32') + .mockResolvedValue(undefined as any); + Amplify.libraryOptions = { Storage: { S3: { @@ -307,6 +330,81 @@ describe('putObjectJob with path', () => { expect(calculateContentMd5).toHaveBeenCalledWith('data'); }); + describe('overwrite prevention', () => { + beforeEach(() => { + mockHeadObject.mockClear(); + }); + + it('should upload if target key is not found', async () => { + expect.assertions(3); + const notFoundError = new Error('mock message'); + notFoundError.name = 'NotFound'; + mockHeadObject.mockRejectedValueOnce(notFoundError); + + const job = putObjectJob( + { + path: testPath, + data: 'data', + options: { preventOverwrite: true }, + }, + new AbortController().signal, + ); + await job(); + + await expect(mockHeadObject).toBeLastCalledWithConfigAndInput( + { + credentials, + region: 'region', + }, + { + Bucket: 'bucket', + Key: testPath, + }, + ); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + expect(mockPutObject).toHaveBeenCalledTimes(1); + }); + + it('should not upload if target key already exists', async () => { + expect.assertions(3); + mockHeadObject.mockResolvedValueOnce({ + ContentLength: 0, + $metadata: {}, + }); + const job = putObjectJob( + { + path: testPath, + data: 'data', + options: { preventOverwrite: true }, + }, + new AbortController().signal, + ); + await expect(job()).rejects.toThrow( + 'At least one of the pre-conditions you specified did not hold', + ); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + expect(mockPutObject).not.toHaveBeenCalled(); + }); + + it('should not upload if HeadObject fails with other error', async () => { + expect.assertions(3); + const accessDeniedError = new Error('mock error'); + accessDeniedError.name = 'AccessDenied'; + mockHeadObject.mockRejectedValueOnce(accessDeniedError); + const job = putObjectJob( + { + path: testPath, + data: 'data', + options: { preventOverwrite: true }, + }, + new AbortController().signal, + ); + await expect(job()).rejects.toThrow('mock error'); + expect(mockHeadObject).toHaveBeenCalledTimes(1); + expect(mockPutObject).not.toHaveBeenCalled(); + }); + }); + describe('bucket passed in options', () => { it('should override bucket in putObject call when bucket as object', async () => { const abortController = new AbortController(); @@ -341,6 +439,7 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); @@ -372,6 +471,7 @@ describe('putObjectJob with path', () => { Key: 'path/', Body: data, ContentType: 'application/octet-stream', + ChecksumCRC32: 'rfPzYw==', }, ); }); diff --git a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts index 022c2f0c1fb..662640e3340 100644 --- a/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/utils/resolveS3ConfigAndInput.test.ts @@ -9,6 +9,11 @@ import { StorageValidationErrorCode, validationErrorMap, } from '../../../../../src/errors/types/validation'; +import { + CallbackPathStorageInput, + DeprecatedStorageInput, +} from '../../../../../src/providers/s3/utils/resolveS3ConfigAndInput'; +import { INVALID_STORAGE_INPUT } from '../../../../../src/errors/constants'; import { BucketInfo } from '../../../../../src/providers/s3/types/options'; import { StorageError } from '../../../../../src/errors/StorageError'; @@ -79,13 +84,11 @@ describe('resolveS3ConfigAndInput', () => { } }); - it('should throw if identityId is not available', async () => { + it('should not throw if identityId is not available', async () => { mockFetchAuthSession.mockResolvedValueOnce({ credentials, }); - await expect(resolveS3ConfigAndInput(Amplify, {})).rejects.toMatchObject( - validationErrorMap[StorageValidationErrorCode.NoIdentityId], - ); + expect(async () => resolveS3ConfigAndInput(Amplify, {})).not.toThrow(); }); it('should resolve bucket from S3 config', async () => { @@ -182,7 +185,7 @@ describe('resolveS3ConfigAndInput', () => { it('should resolve prefix with given access level', async () => { mockDefaultResolvePrefix.mockResolvedValueOnce('prefix'); const { keyPrefix } = await resolveS3ConfigAndInput(Amplify, { - accessLevel: 'someLevel' as any, + options: { accessLevel: 'someLevel' as any }, }); expect(mockDefaultResolvePrefix).toHaveBeenCalledWith({ accessLevel: 'someLevel', @@ -218,6 +221,95 @@ describe('resolveS3ConfigAndInput', () => { expect(keyPrefix).toEqual('prefix'); }); + describe('with locationCredentialsProvider', () => { + const mockLocationCredentialsProvider = jest + .fn() + .mockReturnValue({ credentials }); + it('should resolve credentials without Amplify singleton', async () => { + mockGetConfig.mockReturnValue({ + Storage: { + S3: { + bucket, + region, + }, + }, + }); + const { s3Config } = await resolveS3ConfigAndInput(Amplify, { + options: { + locationCredentialsProvider: mockLocationCredentialsProvider, + }, + }); + + if (typeof s3Config.credentials === 'function') { + const result = await s3Config.credentials({ forceRefresh: true }); + expect(mockLocationCredentialsProvider).toHaveBeenCalledWith({ + forceRefresh: true, + }); + expect(result).toEqual(credentials); + } else { + throw new Error('Expect credentials to be a function'); + } + }); + + it('should not throw when path is pass as a string', async () => { + const { s3Config } = await resolveS3ConfigAndInput(Amplify, { + path: 'my-path', + options: { + locationCredentialsProvider: mockLocationCredentialsProvider, + }, + }); + + if (typeof s3Config.credentials === 'function') { + const result = await s3Config.credentials(); + expect(mockLocationCredentialsProvider).toHaveBeenCalled(); + expect(result).toEqual(credentials); + } else { + throw new Error('Expect credentials to be a function'); + } + }); + + describe('with deprecated or callback paths as inputs', () => { + const key = 'mock-value'; + const prefix = 'mock-value'; + const path = () => 'path'; + const deprecatedInputs: DeprecatedStorageInput[] = [ + { prefix }, + { key }, + { + source: { key }, + destination: { key }, + }, + ]; + const callbackPathInputs: CallbackPathStorageInput[] = [ + { path }, + { + destination: { path }, + source: { path }, + }, + ]; + + const testCases = [...deprecatedInputs, ...callbackPathInputs]; + + it.each(testCases)('should throw when input is %s', async input => { + const { s3Config } = await resolveS3ConfigAndInput(Amplify, { + ...input, + options: { + locationCredentialsProvider: mockLocationCredentialsProvider, + }, + }); + if (typeof s3Config.credentials === 'function') { + await expect(s3Config.credentials()).rejects.toThrow( + expect.objectContaining({ + name: INVALID_STORAGE_INPUT, + }), + ); + } else { + throw new Error('Expect credentials to be a function'); + } + }); + }); + }); + it('should resolve bucket and region with overrides when bucket API option is passed', async () => { const bucketInfo: BucketInfo = { bucketName: 'bucket-2', @@ -228,7 +320,7 @@ describe('resolveS3ConfigAndInput', () => { bucket: resolvedBucket, s3Config: { region: resolvedRegion }, } = await resolveS3ConfigAndInput(Amplify, { - bucket: bucketInfo, + options: { bucket: bucketInfo }, }); expect(mockGetConfig).toHaveBeenCalled(); @@ -239,7 +331,7 @@ describe('resolveS3ConfigAndInput', () => { it('should throw when unable to lookup bucket from the config when bucket API option is passed', async () => { try { await resolveS3ConfigAndInput(Amplify, { - bucket: 'error-bucket', + options: { bucket: 'error-bucket' }, }); } catch (error: any) { expect(error).toBeInstanceOf(StorageError); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts index 4628c433e51..cc81a2be88f 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/abortMultipartUpload.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { abortMultipartUpload } from '../../../../../../../src/providers/s3/utils/client'; +import { abortMultipartUpload } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts index 125cb505e4c..c9cbbe8912f 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/completeMultipartUpload.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { completeMultipartUpload } from '../../../../../../../src/providers/s3/utils/client'; +import { completeMultipartUpload } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { @@ -26,10 +26,12 @@ const completeMultipartUploadHappyCase: ApiFunctionalTestCase< { ETag: 'etag1', PartNumber: 1, + ChecksumCRC32: 'test-checksum-1', }, { ETag: 'etag2', PartNumber: 2, + ChecksumCRC32: 'test-checksum-2', }, ], }, @@ -49,10 +51,12 @@ const completeMultipartUploadHappyCase: ApiFunctionalTestCase< '' + 'etag1' + '1' + + 'test-checksum-1' + '' + '' + 'etag2' + '2' + + 'test-checksum-2' + '' + '', }), @@ -109,7 +113,12 @@ const completeMultipartUploadErrorWith200CodeCase: ApiFunctionalTestCase< 'error case', 'completeMultipartUpload with 200 status', completeMultipartUpload, - { ...defaultConfig, retryDecider: async () => false }, // disable retry + { + ...defaultConfig, + retryDecider: async () => ({ + retryable: false, + }), + }, // disable retry completeMultipartUploadHappyCase[4], completeMultipartUploadHappyCase[5], { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts index 746ca373057..cad0ad74cf4 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/copyObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { copyObject } from '../../../../../../../src/providers/s3/utils/client'; +import { copyObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts index df13908e715..e027397e569 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/createMultipartUpload.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { createMultipartUpload } from '../../../../../../../src/providers/s3/utils/client'; +import { createMultipartUpload } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts index f0a4439e13f..614a3c1fff6 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/deleteObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { deleteObject } from '../../../../../../../src/providers/s3/utils/client'; +import { deleteObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts new file mode 100644 index 00000000000..851bc993a7c --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getDataAccess.ts @@ -0,0 +1,102 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { getDataAccess } from '../../../../../../../src/providers/s3/utils/client/s3control'; +import { ApiFunctionalTestCase } from '../../testUtils/types'; + +import { + DEFAULT_RESPONSE_HEADERS, + defaultConfig, + expectedMetadata, +} from './shared'; + +const MOCK_ACCOUNT_ID = 'accountId'; +const MOCK_ACCESS_ID = 'accessId'; +const MOCK_SECRET_ACCESS_KEY = 'secretAccessKey'; +const MOCK_SESSION_TOKEN = 'sessionToken'; +const MOCK_EXPIRATION = '2013-09-17T18:07:53.000Z'; +const MOCK_EXPIRATION_DATE = new Date(MOCK_EXPIRATION); +const MOCK_GRANT_TARGET = 'matchedGrantTarget'; + +// API Reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_GetDataAccess.html +const getDataAccessHappyCase: ApiFunctionalTestCase = [ + 'happy case', + 'getDataAccess', + getDataAccess, + defaultConfig, + { + AccountId: MOCK_ACCOUNT_ID, + Target: 's3://my-bucket/path/to/object.md', + TargetType: 'Object', + DurationSeconds: 100, + Permission: 'READWRITE', + Privilege: 'Default', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.s3-control.us-east-1.amazonaws.com/v20180820/accessgrantsinstance/dataaccess?durationSeconds=100&permission=READWRITE&privilege=Default&target=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2Fobject.md&targetType=Object', + }), + method: 'GET', + headers: expect.objectContaining({ + 'x-amz-account-id': MOCK_ACCOUNT_ID, + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: ` + + + + ${MOCK_ACCESS_ID} + ${MOCK_SECRET_ACCESS_KEY} + ${MOCK_SESSION_TOKEN} + ${MOCK_EXPIRATION} + + ${MOCK_GRANT_TARGET} + + `, + }, + { + $metadata: expect.objectContaining(expectedMetadata), + Credentials: { + AccessKeyId: MOCK_ACCESS_ID, + SecretAccessKey: MOCK_SECRET_ACCESS_KEY, + SessionToken: MOCK_SESSION_TOKEN, + Expiration: MOCK_EXPIRATION_DATE, + }, + MatchedGrantTarget: MOCK_GRANT_TARGET, + }, +]; + +const getDataAccessErrorCase: ApiFunctionalTestCase = [ + 'error case', + 'getDataAccess', + getDataAccess, + defaultConfig, + getDataAccessHappyCase[4], + getDataAccessHappyCase[5], + { + status: 403, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + + + AccessDenied + Access Denied + + 656c76696e6727732072657175657374 + Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== + + `, + }, + { + message: 'Access Denied', + name: 'AccessDenied', + }, +]; + +export default [getDataAccessHappyCase, getDataAccessErrorCase]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts index c6b1e038926..a35c813f3d8 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/getObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { getObject } from '../../../../../../../src/providers/s3/utils/client'; +import { getObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts index 2275d7ac850..0cc016a7813 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/headObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { headObject } from '../../../../../../../src/providers/s3/utils/client'; +import { headObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts index 56a4e1719ae..b5688b18c78 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/index.ts @@ -12,6 +12,8 @@ import copyObjectCases from './copyObject'; import deleteObjectCases from './deleteObject'; import getObjectCases from './getObject'; import headObjectCases from './headObject'; +import getDataAccess from './getDataAccess'; +import listCallerAccessGrants from './listCallerAccessGrants'; export default [ ...listObjectsV2Cases, @@ -25,4 +27,6 @@ export default [ ...deleteObjectCases, ...getObjectCases, ...headObjectCases, + ...listCallerAccessGrants, + ...getDataAccess, ]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts new file mode 100644 index 00000000000..961ef27b3bf --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listCallerAccessGrants.ts @@ -0,0 +1,172 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { listCallerAccessGrants } from '../../../../../../../src/providers/s3/utils/client/s3control'; +import { ApiFunctionalTestCase } from '../../testUtils/types'; + +import { + DEFAULT_RESPONSE_HEADERS, + defaultConfig, + expectedMetadata, +} from './shared'; + +const MOCK_ACCOUNT_ID = 'accountId'; +const MOCK_NEXT_TOKEN = 'nextToken'; +const MOCK_APP_ARN = 'appArn'; +const MOCK_GRANT_SCOPE = 's3://my-bucket/path/to/object.md'; +const MOCK_PERMISSION = 'READWRITE'; + +// API Reference: https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_ListAccessGrants.html +const listCallerAccessGrantsHappyCaseSingleGrant: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'happy case', + 'listCallerAccessGrantsHappyCaseSingleGrant', + listCallerAccessGrants, + defaultConfig, + { + AccountId: MOCK_ACCOUNT_ID, + GrantScope: 's3://my-bucket/path/to/', + MaxResults: 50, + NextToken: 'mockToken', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.s3-control.us-east-1.amazonaws.com/v20180820/accessgrantsinstance/caller/grants?grantscope=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2F&maxResults=50&nextToken=mockToken', + }), + method: 'GET', + headers: expect.objectContaining({ + 'x-amz-account-id': MOCK_ACCOUNT_ID, + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: ` + + + ${MOCK_NEXT_TOKEN} + + + ${MOCK_APP_ARN} + ${MOCK_GRANT_SCOPE} + ${MOCK_PERMISSION} + + + + `, + }, + { + $metadata: expect.objectContaining(expectedMetadata), + CallerAccessGrantsList: [ + { + ApplicationArn: MOCK_APP_ARN, + GrantScope: MOCK_GRANT_SCOPE, + Permission: MOCK_PERMISSION, + }, + ], + NextToken: MOCK_NEXT_TOKEN, + }, +]; + +const listCallerAccessGrantsHappyCaseMultipleGrants: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'happy case', + 'listCallerAccessGrantsHappyCaseMultipleGrants', + listCallerAccessGrants, + defaultConfig, + { + AccountId: MOCK_ACCOUNT_ID, + GrantScope: 's3://my-bucket/path/to/', + MaxResults: 50, + NextToken: 'mockToken', + }, + expect.objectContaining({ + url: expect.objectContaining({ + href: 'https://accountid.s3-control.us-east-1.amazonaws.com/v20180820/accessgrantsinstance/caller/grants?grantscope=s3%3A%2F%2Fmy-bucket%2Fpath%2Fto%2F&maxResults=50&nextToken=mockToken', + }), + method: 'GET', + headers: expect.objectContaining({ + 'x-amz-account-id': MOCK_ACCOUNT_ID, + }), + }), + { + status: 200, + headers: { + ...DEFAULT_RESPONSE_HEADERS, + }, + body: ` + + + ${MOCK_NEXT_TOKEN} + + + ${MOCK_APP_ARN} + ${MOCK_GRANT_SCOPE} + ${MOCK_PERMISSION} + + + ${MOCK_APP_ARN} + ${MOCK_GRANT_SCOPE} + ${MOCK_PERMISSION} + + + + `, + }, + { + $metadata: expect.objectContaining(expectedMetadata), + CallerAccessGrantsList: [ + { + ApplicationArn: MOCK_APP_ARN, + GrantScope: MOCK_GRANT_SCOPE, + Permission: MOCK_PERMISSION, + }, + { + ApplicationArn: MOCK_APP_ARN, + GrantScope: MOCK_GRANT_SCOPE, + Permission: MOCK_PERMISSION, + }, + ], + NextToken: MOCK_NEXT_TOKEN, + }, +]; + +const listCallerAccessGrantsErrorCase: ApiFunctionalTestCase< + typeof listCallerAccessGrants +> = [ + 'error case', + 'listCallerAccessGrants', + listCallerAccessGrants, + defaultConfig, + listCallerAccessGrantsHappyCaseSingleGrant[4], + listCallerAccessGrantsHappyCaseSingleGrant[5], + { + status: 403, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + + + AccessDenied + Access Denied + + 656c76696e6727732072657175657374 + Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== + + `, + }, + { + message: 'Access Denied', + name: 'AccessDenied', + }, +]; + +export default [ + listCallerAccessGrantsHappyCaseSingleGrant, + listCallerAccessGrantsHappyCaseMultipleGrants, + listCallerAccessGrantsErrorCase, +]; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index 7524a8daeb6..80ce6ea9a08 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { listObjectsV2 } from '../../../../../../../src/providers/s3/utils/client'; +import { listObjectsV2 } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts index 3e809d12bdc..396035c09dd 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listParts.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { listParts } from '../../../../../../../src/providers/s3/utils/client'; +import { listParts } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts index 930870a7c15..867ee3f0af2 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/putObject.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { putObject } from '../../../../../../../src/providers/s3/utils/client'; +import { putObject } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts index b4906b223c2..4a46891c849 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/uploadPart.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -import { uploadPart } from '../../../../../../../src/providers/s3/utils/client'; +import { uploadPart } from '../../../../../../../src/providers/s3/utils/client/s3data'; import { ApiFunctionalTestCase } from '../../testUtils/types'; import { diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts index 62b4aff0cf5..656f8d45ed7 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/functional-apis.test.ts @@ -68,11 +68,11 @@ describe('S3 APIs functional test', () => { expect.anything(), ); } else { - fail(`${name} ${caseType} should fail`); + throw new Error(`${name} ${caseType} should fail`); } } catch (e) { if (caseType === 'happy case') { - fail(`${name} ${caseType} should succeed: ${e}`); + throw new Error(`${name} ${caseType} should succeed: ${e}`); } else { expect(e).toBeInstanceOf(StorageError); expect(e).toEqual(expect.objectContaining(outputOrError)); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts index ab84fb03eb6..a208859a7c8 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/getPresignedGetObjectUrl.test.ts @@ -3,7 +3,7 @@ import { presignUrl } from '@aws-amplify/core/internals/aws-client-utils'; -import { getPresignedGetObjectUrl } from '../../../../../../src/providers/s3/utils/client'; +import { getPresignedGetObjectUrl } from '../../../../../../src/providers/s3/utils/client/s3data'; import { defaultConfigWithStaticCredentials } from './cases/shared'; diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/utils/createRetryDecider.test.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/utils/createRetryDecider.test.ts new file mode 100644 index 00000000000..7f30a9f0cab --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/utils/createRetryDecider.test.ts @@ -0,0 +1,103 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { + HttpResponse, + getRetryDecider as getDefaultRetryDecider, +} from '@aws-amplify/core/internals/aws-client-utils'; + +import { createRetryDecider } from '../../../../../../../src/providers/s3/utils/client/utils'; + +jest.mock('@aws-amplify/core/internals/aws-client-utils'); + +const mockErrorParser = jest.fn(); + +describe('createRetryDecider', () => { + const mockHttpResponse: HttpResponse = { + statusCode: 200, + headers: {}, + body: 'body' as any, + }; + + beforeEach(() => { + jest.mocked(getDefaultRetryDecider).mockReturnValue(async () => { + return { retryable: false }; + }); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + it('should invoke the default retry decider', async () => { + expect.assertions(3); + const retryDecider = createRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + mockHttpResponse, + undefined, + {}, + ); + expect(getDefaultRetryDecider).toHaveBeenCalledWith(mockErrorParser); + expect(retryable).toBe(false); + expect(isCredentialsExpiredError).toBeFalsy(); + }); + + describe('handling expired token errors', () => { + const mockErrorMessage = 'Token expired'; + it.each(['RequestExpired', 'ExpiredTokenException', 'ExpiredToken'])( + 'should retry if expired credentials error name %s', + async errorName => { + expect.assertions(2); + const parsedError = { + name: errorName, + message: mockErrorMessage, + $metadata: {}, + }; + mockErrorParser.mockResolvedValue(parsedError); + const retryDecider = createRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { ...mockHttpResponse, statusCode: 400 }, + undefined, + {}, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBe(true); + }, + ); + + it('should retry if error message indicates invalid credentials', async () => { + expect.assertions(2); + const parsedError = { + name: 'InvalidSignature', + message: 'Auth token in request is expired.', + $metadata: {}, + }; + mockErrorParser.mockResolvedValue(parsedError); + const retryDecider = createRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { ...mockHttpResponse, statusCode: 400 }, + undefined, + {}, + ); + expect(retryable).toBe(true); + expect(isCredentialsExpiredError).toBe(true); + }); + + it('should not retry if invalid credentials error has been retried previously', async () => { + expect.assertions(2); + const parsedError = { + name: 'RequestExpired', + message: mockErrorMessage, + $metadata: {}, + }; + mockErrorParser.mockResolvedValue(parsedError); + const retryDecider = createRetryDecider(mockErrorParser); + const { retryable, isCredentialsExpiredError } = await retryDecider( + { ...mockHttpResponse, statusCode: 400 }, + undefined, + { isCredentialsExpired: true }, + ); + expect(retryable).toBe(false); + expect(isCredentialsExpiredError).toBe(true); + }); + }); +}); diff --git a/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts b/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts index b47d2ec7695..a3754b41707 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/testUtils/types.ts @@ -3,7 +3,7 @@ import { HttpRequest } from '@aws-amplify/core/internals/aws-client-utils'; -interface MockFetchResponse { +export interface MockFetchResponse { body: BodyInit; headers: HeadersInit; status: number; diff --git a/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts new file mode 100644 index 00000000000..0f4c1adce27 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/crc32.native.test.ts @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32.native'; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); + +describe('calculate crc32 native', () => { + it('should return undefined', async () => { + expect(await calculateContentCRC32(getBlob(8 * MB))).toEqual(undefined); + }); +}); diff --git a/packages/storage/__tests__/providers/s3/utils/crc32.test.ts b/packages/storage/__tests__/providers/s3/utils/crc32.test.ts new file mode 100644 index 00000000000..e2195ddf5e4 --- /dev/null +++ b/packages/storage/__tests__/providers/s3/utils/crc32.test.ts @@ -0,0 +1,136 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { Blob as BlobPolyfill, File as FilePolyfill } from 'node:buffer'; +import { WritableStream as WritableStreamPolyfill } from 'node:stream/web'; +import { + TextDecoder as TextDecoderPolyfill, + TextEncoder as TextEncoderPolyfill, +} from 'node:util'; + +import { calculateContentCRC32 } from '../../../../src/providers/s3/utils/crc32'; + +global.Blob = BlobPolyfill as any; +global.File = FilePolyfill as any; +global.WritableStream = WritableStreamPolyfill as any; +global.TextEncoder = TextEncoderPolyfill as any; +global.TextDecoder = TextDecoderPolyfill as any; + +const MB = 1024 * 1024; +const getBlob = (size: number) => new Blob(['1'.repeat(size)]); +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +describe('calculate crc32', () => { + describe.each([ + { + type: 'file', + size: '4B', + data: new File(['data'], 'someName'), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'blob', + size: '4B', + data: new Blob(['data']), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'string', + size: '4B', + data: 'data', + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBuffer', + size: '4B', + data: encoder.encode('data').buffer, + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'arrayBufferView', + size: '4B', + data: new DataView(encoder.encode('1234 data 5678').buffer, 5, 4), + expected: { + checksum: 'rfPzYw==', + checksumArrayBuffer: new Uint8Array([173, 243, 243, 99]).buffer, + seed: 2918445923, + }, + }, + { + type: 'file', + size: '8MB', + data: new File([getBlob(8 * MB)], 'someName'), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'blob', + size: '8MB', + data: getBlob(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'string', + size: '8MB', + data: '1'.repeat(8 * MB), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBuffer', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)).buffer, + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + { + type: 'arrayBufferView', + size: '8MB', + data: encoder.encode('1'.repeat(8 * MB)), + expected: { + checksum: '/YBlgg==', + checksumArrayBuffer: new Uint8Array([253, 128, 101, 130]).buffer, + seed: 4253050242, + }, + }, + ])('output for data type of $type with size $size', ({ data, expected }) => { + it('should match expected checksum results', async () => { + const result = (await calculateContentCRC32(data))!; + expect(result.checksum).toEqual(expected.checksum); + expect(result.seed).toEqual(expected.seed); + expect(decoder.decode(result.checksumArrayBuffer)).toEqual( + decoder.decode(expected.checksumArrayBuffer), + ); + }); + }); +}); diff --git a/packages/storage/internals/package.json b/packages/storage/internals/package.json new file mode 100644 index 00000000000..169011166f3 --- /dev/null +++ b/packages/storage/internals/package.json @@ -0,0 +1,7 @@ +{ + "name": "@aws-amplify/storage/internals", + "types": "../dist/esm/internals/index.d.ts", + "main": "../dist/cjs/internals/index.js", + "module": "../dist/esm/internals/index.mjs", + "sideEffects": false +} diff --git a/packages/storage/package.json b/packages/storage/package.json index 70701547ce3..1f12a39a9c5 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -40,6 +40,9 @@ "s3": [ "./dist/esm/providers/s3/index.d.ts" ], + "internals": [ + "./dist/esm/internals/index.d.ts" + ], "server": [ "./dist/esm/server.d.ts" ], @@ -61,6 +64,7 @@ "files": [ "dist/cjs", "dist/esm", + "internals", "src", "server", "s3" @@ -69,6 +73,7 @@ "@aws-sdk/types": "3.398.0", "@smithy/md5-js": "2.0.7", "buffer": "4.9.2", + "crc-32": "1.2.2", "fast-xml-parser": "^4.4.1", "tslib": "^2.5.0" }, @@ -79,6 +84,11 @@ "require": "./dist/cjs/index.js", "react-native": "./src/index.ts" }, + "./internals": { + "types": "./dist/esm/internals/index.d.ts", + "import": "./dist/esm/internals/index.mjs", + "require": "./dist/cjs/internals/index.js" + }, "./server": { "types": "./dist/esm/server.d.ts", "import": "./dist/esm/server.mjs", @@ -102,6 +112,7 @@ }, "devDependencies": { "@aws-amplify/core": "6.4.3", + "@types/node": "20.14.12", "@aws-amplify/react-native": "1.1.6", "typescript": "5.0.2" } diff --git a/packages/storage/src/errors/constants.ts b/packages/storage/src/errors/constants.ts new file mode 100644 index 00000000000..ca127c2e623 --- /dev/null +++ b/packages/storage/src/errors/constants.ts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const INVALID_STORAGE_INPUT = 'InvalidStorageInput'; diff --git a/packages/storage/src/errors/types/validation.ts b/packages/storage/src/errors/types/validation.ts index 7fb1bd89765..75c90771461 100644 --- a/packages/storage/src/errors/types/validation.ts +++ b/packages/storage/src/errors/types/validation.ts @@ -21,6 +21,9 @@ export enum StorageValidationErrorCode { InvalidUploadSource = 'InvalidUploadSource', ObjectIsTooLarge = 'ObjectIsTooLarge', UrlExpirationMaxLimitExceed = 'UrlExpirationMaxLimitExceed', + InvalidLocationCredentialsCacheSize = 'InvalidLocationCredentialsCacheSize', + LocationCredentialsStoreDestroyed = 'LocationCredentialsStoreDestroyed', + InvalidS3Uri = 'InvalidS3Uri', } export const validationErrorMap: AmplifyErrorMap = { @@ -72,6 +75,15 @@ export const validationErrorMap: AmplifyErrorMap = { [StorageValidationErrorCode.InvalidStoragePathInput]: { message: 'Input `path` does not allow a leading slash (/).', }, + [StorageValidationErrorCode.InvalidLocationCredentialsCacheSize]: { + message: 'locationCredentialsCacheSize must be a positive integer.', + }, + [StorageValidationErrorCode.LocationCredentialsStoreDestroyed]: { + message: `Location-specific credentials store has been destroyed.`, + }, + [StorageValidationErrorCode.InvalidS3Uri]: { + message: 'Invalid S3 URI.', + }, [StorageValidationErrorCode.InvalidStorageBucket]: { message: 'Unable to lookup bucket from provided name in Amplify configuration.', diff --git a/packages/storage/src/internals/apis/getDataAccess.ts b/packages/storage/src/internals/apis/getDataAccess.ts new file mode 100644 index 00000000000..3a6af14441a --- /dev/null +++ b/packages/storage/src/internals/apis/getDataAccess.ts @@ -0,0 +1,82 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + AmplifyErrorCode, + StorageAction, +} from '@aws-amplify/core/internals/utils'; +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; + +import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; +import { getDataAccess as getDataAccessClient } from '../../providers/s3/utils/client/s3control'; +import { StorageError } from '../../errors/StorageError'; +import { GetDataAccessInput } from '../types/inputs'; +import { GetDataAccessOutput } from '../types/outputs'; +import { logger } from '../../utils'; +import { DEFAULT_CRED_TTL } from '../utils/constants'; + +/** + * @internal + */ +export const getDataAccess = async ( + input: GetDataAccessInput, +): Promise => { + const targetType = input.scope.endsWith('*') ? undefined : 'Object'; + const clientCredentialsProvider = async ( + options?: CredentialsProviderOptions, + ) => { + const { credentials } = await input.credentialsProvider(options); + + return credentials; + }; + + const result = await getDataAccessClient( + { + credentials: clientCredentialsProvider, + region: input.region, + userAgentValue: getStorageUserAgentValue(StorageAction.GetDataAccess), + }, + { + AccountId: input.accountId, + Target: input.scope, + Permission: input.permission, + TargetType: targetType, + DurationSeconds: DEFAULT_CRED_TTL, + }, + ); + + const grantCredentials = result.Credentials; + + // Ensure that S3 returned credentials (this shouldn't happen) + if ( + !grantCredentials || + !grantCredentials.AccessKeyId || + !grantCredentials.SecretAccessKey || + !grantCredentials.SessionToken || + !grantCredentials.Expiration + ) { + throw new StorageError({ + name: AmplifyErrorCode.Unknown, + message: 'Service did not return valid temporary credentials.', + }); + } else { + logger.debug(`Retrieved credentials for: ${result.MatchedGrantTarget}`); + } + + const { + AccessKeyId: accessKeyId, + SecretAccessKey: secretAccessKey, + SessionToken: sessionToken, + Expiration: expiration, + } = grantCredentials; + + return { + credentials: { + accessKeyId, + secretAccessKey, + sessionToken, + expiration, + }, + scope: result.MatchedGrantTarget, + }; +}; diff --git a/packages/storage/src/internals/apis/listCallerAccessGrants.ts b/packages/storage/src/internals/apis/listCallerAccessGrants.ts new file mode 100644 index 00000000000..3f8601fdaaa --- /dev/null +++ b/packages/storage/src/internals/apis/listCallerAccessGrants.ts @@ -0,0 +1,95 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageAction } from '@aws-amplify/core/internals/utils'; +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; + +import { logger } from '../../utils'; +import { listCallerAccessGrants as listCallerAccessGrantsClient } from '../../providers/s3/utils/client/s3control'; +import { StorageError } from '../../errors/StorageError'; +import { getStorageUserAgentValue } from '../../providers/s3/utils/userAgent'; +import { LocationType } from '../types/common'; +import { LocationAccess } from '../types/credentials'; +import { ListCallerAccessGrantsInput } from '../types/inputs'; +import { ListCallerAccessGrantsOutput } from '../types/outputs'; +import { MAX_PAGE_SIZE } from '../utils/constants'; + +/** + * @internal + */ +export const listCallerAccessGrants = async ( + input: ListCallerAccessGrantsInput, +): Promise => { + const { credentialsProvider, accountId, region, nextToken, pageSize } = input; + + logger.debug(`listing available locations from account ${input.accountId}`); + + if (!!pageSize && pageSize > MAX_PAGE_SIZE) { + logger.debug(`defaulting pageSize to ${MAX_PAGE_SIZE}.`); + } + + const clientCredentialsProvider = async ( + options?: CredentialsProviderOptions, + ) => { + const { credentials } = await credentialsProvider(options); + + return credentials; + }; + + const { CallerAccessGrantsList, NextToken } = + await listCallerAccessGrantsClient( + { + credentials: clientCredentialsProvider, + region, + userAgentValue: getStorageUserAgentValue( + StorageAction.ListCallerAccessGrants, + ), + }, + { + AccountId: accountId, + NextToken: nextToken, + MaxResults: pageSize ?? MAX_PAGE_SIZE, + }, + ); + + const accessGrants: LocationAccess[] = + CallerAccessGrantsList?.map(grant => { + assertGrantScope(grant.GrantScope); + + return { + scope: grant.GrantScope, + permission: grant.Permission!, + type: parseGrantType(grant.GrantScope!), + }; + }) ?? []; + + return { + locations: accessGrants, + nextToken: NextToken, + }; +}; + +const parseGrantType = (grantScope: string): LocationType => { + const bucketScopeReg = /^s3:\/\/(.*)\/\*$/; + const possibleBucketName = grantScope.match(bucketScopeReg)?.[1]; + if (!grantScope.endsWith('*')) { + return 'OBJECT'; + } else if ( + grantScope.endsWith('/*') && + possibleBucketName && + possibleBucketName.indexOf('/') === -1 + ) { + return 'BUCKET'; + } else { + return 'PREFIX'; + } +}; + +function assertGrantScope(value: unknown): asserts value is string { + if (typeof value !== 'string' || !value.startsWith('s3://')) { + throw new StorageError({ + name: 'InvalidGrantScope', + message: `Expected a valid grant scope, got ${value}`, + }); + } +} diff --git a/packages/storage/src/internals/index.ts b/packages/storage/src/internals/index.ts new file mode 100644 index 00000000000..279ef2afd3b --- /dev/null +++ b/packages/storage/src/internals/index.ts @@ -0,0 +1,44 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { LocationCredentialsProvider } from '../providers/s3/types/options'; +export { StorageSubpathStrategy } from '../types/options'; + +export { Permission } from './types/common'; + +/* +Internal APIs +*/ +export { + GetDataAccessInput, + ListCallerAccessGrantsInput, +} from './types/inputs'; +export { + GetDataAccessOutput, + ListCallerAccessGrantsOutput, +} from './types/outputs'; + +export { getDataAccess } from './apis/getDataAccess'; +export { listCallerAccessGrants } from './apis/listCallerAccessGrants'; + +/* +CredentialsStore exports +*/ +export { createLocationCredentialsStore } from './locationCredentialsStore'; +export { + AuthConfigAdapter, + createManagedAuthConfigAdapter, + CreateManagedAuthConfigAdapterInput, +} from './managedAuthConfigAdapter'; +export { + GetLocationCredentials, + ListLocations, + LocationCredentialsStore, + CreateLocationCredentialsStoreInput, + LocationCredentials, + ListLocationsInput, + ListLocationsOutput, + GetLocationCredentialsInput, + GetLocationCredentialsOutput, +} from './types/credentials'; +export { AWSTemporaryCredentials } from '../providers/s3/types/options'; diff --git a/packages/storage/src/internals/locationCredentialsStore/constants.ts b/packages/storage/src/internals/locationCredentialsStore/constants.ts new file mode 100644 index 00000000000..67b22505b95 --- /dev/null +++ b/packages/storage/src/internals/locationCredentialsStore/constants.ts @@ -0,0 +1,5 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const CREDENTIALS_STORE_DEFAULT_SIZE = 10; +export const CREDENTIALS_REFRESH_WINDOW_MS = 30_000; diff --git a/packages/storage/src/internals/locationCredentialsStore/create.ts b/packages/storage/src/internals/locationCredentialsStore/create.ts new file mode 100644 index 00000000000..dab4539025e --- /dev/null +++ b/packages/storage/src/internals/locationCredentialsStore/create.ts @@ -0,0 +1,54 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + CreateLocationCredentialsStoreInput, + CredentialsLocation, + LocationCredentialsStore, +} from '../types/credentials'; +import { StorageValidationErrorCode } from '../../errors/types/validation'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { LocationCredentialsProvider } from '../../providers/s3/types/options'; + +import { createStore, getValue, removeStore } from './registry'; + +export const createLocationCredentialsStore = ( + input: CreateLocationCredentialsStoreInput, +): LocationCredentialsStore => { + const storeSymbol = createStore(input.handler); + + const store = { + getProvider(providerLocation: CredentialsLocation) { + const locationCredentialsProvider = async ({ + forceRefresh = false, + }: Parameters[0] = {}) => { + validateS3Uri(providerLocation.scope); + + // TODO(@AllanZhengYP): validate the action bucket and paths matches provider scope. + return getValue({ + storeSymbol, + location: { ...providerLocation }, + forceRefresh, + }); + }; + + return locationCredentialsProvider; + }, + + destroy() { + removeStore(storeSymbol); + }, + }; + + return store; +}; + +type S3Uri = string; + +const validateS3Uri = (uri: S3Uri): void => { + const s3UrlSchemaRegex = /^s3:\/\/[^/]+/; + assertValidationError( + s3UrlSchemaRegex.test(uri), + StorageValidationErrorCode.InvalidS3Uri, + ); +}; diff --git a/packages/storage/src/internals/locationCredentialsStore/index.ts b/packages/storage/src/internals/locationCredentialsStore/index.ts new file mode 100644 index 00000000000..29ca4bb54ff --- /dev/null +++ b/packages/storage/src/internals/locationCredentialsStore/index.ts @@ -0,0 +1,4 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { createLocationCredentialsStore } from './create'; diff --git a/packages/storage/src/internals/locationCredentialsStore/registry.ts b/packages/storage/src/internals/locationCredentialsStore/registry.ts new file mode 100644 index 00000000000..90d511c36fe --- /dev/null +++ b/packages/storage/src/internals/locationCredentialsStore/registry.ts @@ -0,0 +1,91 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { AWSTemporaryCredentials } from '../../providers/s3/types/options'; +import { + CredentialsLocation, + GetLocationCredentials, +} from '../types/credentials'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../errors/types/validation'; + +import { + LruLocationCredentialsStore, + fetchNewValue, + getCacheValue, + initStore, +} from './store'; + +interface StoreRegistrySymbol { + readonly value: symbol; +} + +/** + * Keep all cache records for all instances of credentials store in a singleton + * so we can reliably de-reference from the memory when we destroy a store + * instance. + */ +const storeRegistry = new WeakMap< + StoreRegistrySymbol, + LruLocationCredentialsStore +>(); + +/** + * @internal + */ +export const createStore = ( + refreshHandler: GetLocationCredentials, + size?: number, +) => { + const storeSymbol = { value: Symbol('LocationCredentialsStore') }; + storeRegistry.set(storeSymbol, initStore(refreshHandler, size)); + + return storeSymbol; +}; + +const getLookUpLocations = (location: CredentialsLocation) => { + const { scope, permission } = location; + const locations = [{ scope, permission }]; + if (permission === 'READ' || permission === 'WRITE') { + locations.push({ scope, permission: 'READWRITE' }); + } + + return locations; +}; + +const getCredentialsStore = (storeSymbol: StoreRegistrySymbol) => { + assertValidationError( + storeRegistry.has(storeSymbol), + StorageValidationErrorCode.LocationCredentialsStoreDestroyed, + ); + + return storeRegistry.get(storeSymbol)!; +}; + +/** + * @internal + */ +export const getValue = async (input: { + storeSymbol: StoreRegistrySymbol; + location: CredentialsLocation; + forceRefresh: boolean; +}): Promise<{ credentials: AWSTemporaryCredentials }> => { + const { storeSymbol: storeReference, location, forceRefresh } = input; + const store = getCredentialsStore(storeReference); + if (!forceRefresh) { + const lookupLocations = getLookUpLocations(location); + for (const lookupLocation of lookupLocations) { + const credentials = getCacheValue(store, lookupLocation); + if (credentials !== null) { + return { credentials }; + } + } + } + + return fetchNewValue(store, location); +}; + +export const removeStore = (storeSymbol: StoreRegistrySymbol) => { + storeRegistry.delete(storeSymbol); +}; diff --git a/packages/storage/src/internals/locationCredentialsStore/store.ts b/packages/storage/src/internals/locationCredentialsStore/store.ts new file mode 100644 index 00000000000..64c26d5c5a4 --- /dev/null +++ b/packages/storage/src/internals/locationCredentialsStore/store.ts @@ -0,0 +1,161 @@ +/* eslint-disable unused-imports/no-unused-vars */ + +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + CredentialsLocation, + GetLocationCredentials, +} from '../types/credentials'; +import { Permission } from '../types/common'; +import { AWSTemporaryCredentials } from '../../providers/s3/types/options'; +import { assertValidationError } from '../../errors/utils/assertValidationError'; +import { StorageValidationErrorCode } from '../../errors/types/validation'; + +import { + CREDENTIALS_REFRESH_WINDOW_MS, + CREDENTIALS_STORE_DEFAULT_SIZE, +} from './constants'; + +interface StoreValue extends CredentialsLocation { + credentials?: AWSTemporaryCredentials; + inflightCredentials?: Promise<{ credentials: AWSTemporaryCredentials }>; +} + +type S3Uri = string; + +type CacheKey = `${S3Uri}_${Permission}`; + +const createCacheKey = (location: CredentialsLocation): CacheKey => + `${location.scope}_${location.permission}`; + +/** + * LRU implementation for Location Credentials Store + * O(n) for get and set for simplicity. + * + * @internal + */ +export interface LruLocationCredentialsStore { + capacity: number; + refreshHandler: GetLocationCredentials; + values: Map; +} + +/** + * @internal + */ +export const initStore = ( + refreshHandler: GetLocationCredentials, + size = CREDENTIALS_STORE_DEFAULT_SIZE, +): LruLocationCredentialsStore => { + assertValidationError( + size > 0, + StorageValidationErrorCode.InvalidLocationCredentialsCacheSize, + ); + + return { + capacity: size, + refreshHandler, + values: new Map(), + }; +}; + +export const getCacheValue = ( + store: LruLocationCredentialsStore, + location: CredentialsLocation, +): AWSTemporaryCredentials | null => { + const cacheKey = createCacheKey(location); + const cachedValue = store.values.get(cacheKey); + const cachedCredentials = cachedValue?.credentials; + if (!cachedCredentials) { + return null; + } + + // Delete and re-insert to key to map to indicate a latest reference in LRU. + store.values.delete(cacheKey); + if (!pastTTL(cachedCredentials)) { + // TODO(@AllanZhengYP): If the credential is still valid but will expire + // soon, we should return credentials AND dispatch a refresh. + store.values.set(cacheKey, cachedValue); + + return cachedCredentials; + } + + return null; +}; + +const pastTTL = (credentials: AWSTemporaryCredentials) => { + const { expiration } = credentials; + + return expiration.getTime() - CREDENTIALS_REFRESH_WINDOW_MS <= Date.now(); +}; + +/** + * Fetch new credentials value with refresh handler and cache the result in + * LRU cache. + * @internal + */ +export const fetchNewValue = async ( + store: LruLocationCredentialsStore, + location: CredentialsLocation, +): Promise<{ credentials: AWSTemporaryCredentials }> => { + const storeValues = store.values; + const key = createCacheKey(location); + if (!storeValues.has(key)) { + const newStoreValue: StoreValue = { + scope: location.scope, + permission: location.permission, + }; + setCacheRecord(store, key, newStoreValue); + } + const storeValue = storeValues.get(key)!; + + return dispatchRefresh(store.refreshHandler, storeValue, () => { + store.values.delete(key); + }); +}; + +const dispatchRefresh = ( + refreshHandler: GetLocationCredentials, + value: StoreValue, + onRefreshFailure: () => void, +) => { + if (value.inflightCredentials) { + return value.inflightCredentials; + } + + value.inflightCredentials = (async () => { + try { + const { credentials } = await refreshHandler({ + scope: value.scope, + permission: value.permission, + }); + value.credentials = credentials; + + return { credentials }; + } catch (e) { + onRefreshFailure(); + throw e; + } finally { + value.inflightCredentials = undefined; + } + })(); + + return value.inflightCredentials; +}; + +const setCacheRecord = ( + store: LruLocationCredentialsStore, + key: CacheKey, + value: StoreValue, +): void => { + if (store.capacity === store.values.size) { + // Pop least used entry. The Map's key are in insertion order. + // So first key is the last recently inserted. + const [oldestKey] = store.values.keys(); + store.values.delete(oldestKey); + // TODO(@AllanZhengYP): Add log info when record is evicted. + } + // Add latest used value to the cache. + store.values.set(key, value); +}; diff --git a/packages/storage/src/internals/managedAuthConfigAdapter/createListLocationsHandler.ts b/packages/storage/src/internals/managedAuthConfigAdapter/createListLocationsHandler.ts new file mode 100644 index 00000000000..b34dff9848c --- /dev/null +++ b/packages/storage/src/internals/managedAuthConfigAdapter/createListLocationsHandler.ts @@ -0,0 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CredentialsProvider, ListLocations } from '../types/credentials'; +import { listCallerAccessGrants } from '../apis/listCallerAccessGrants'; + +interface CreateListLocationsHandlerInput { + accountId: string; + credentialsProvider: CredentialsProvider; + region: string; +} + +export const createListLocationsHandler = ( + handlerInput: CreateListLocationsHandlerInput, +): ListLocations => { + return async function listLocations(input = {}) { + const result = await listCallerAccessGrants({ ...input, ...handlerInput }); + + return result; + }; +}; diff --git a/packages/storage/src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler.ts b/packages/storage/src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler.ts new file mode 100644 index 00000000000..944a25206fa --- /dev/null +++ b/packages/storage/src/internals/managedAuthConfigAdapter/createLocationCredentialsHandler.ts @@ -0,0 +1,40 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { getDataAccess } from '../apis/getDataAccess'; +import { + CredentialsProvider, + GetLocationCredentials, + GetLocationCredentialsInput, +} from '../types/credentials'; + +interface CreateLocationCredentialsHandlerInput { + accountId: string; + credentialsProvider: CredentialsProvider; + region: string; +} + +export const createLocationCredentialsHandler = ( + handlerInput: CreateLocationCredentialsHandlerInput, +): GetLocationCredentials => { + const { accountId, region, credentialsProvider } = handlerInput; + + /** + * Retrieves credentials for the specified scope & permission. + * + * @param input - An object specifying the requested scope & permission. + * + * @returns A promise which will resolve with the requested credentials. + */ + return (input: GetLocationCredentialsInput) => { + const { scope, permission } = input; + + return getDataAccess({ + accountId, + credentialsProvider, + permission, + region, + scope, + }); + }; +}; diff --git a/packages/storage/src/internals/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts b/packages/storage/src/internals/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts new file mode 100644 index 00000000000..8b622baaee8 --- /dev/null +++ b/packages/storage/src/internals/managedAuthConfigAdapter/createManagedAuthConfigAdapter.ts @@ -0,0 +1,53 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { + CredentialsProvider, + GetLocationCredentials, + ListLocations, +} from '../types/credentials'; + +import { createListLocationsHandler } from './createListLocationsHandler'; +import { createLocationCredentialsHandler } from './createLocationCredentialsHandler'; + +export interface CreateManagedAuthConfigAdapterInput { + accountId: string; + region: string; + credentialsProvider: CredentialsProvider; +} + +export interface AuthConfigAdapter { + listLocations: ListLocations; + getLocationCredentials: GetLocationCredentials; + region: string; +} + +/** + * Create configuration including handlers to call S3 Access Grant APIs to list and get + * credentials for different locations. + * + * @param options - Configuration options for the adapter. + * @returns - An object containing the handlers to call S3 Access Grant APIs and region + */ +export const createManagedAuthConfigAdapter = ({ + credentialsProvider, + region, + accountId, +}: CreateManagedAuthConfigAdapterInput): AuthConfigAdapter => { + const listLocations = createListLocationsHandler({ + credentialsProvider, + accountId, + region, + }); + + const getLocationCredentials = createLocationCredentialsHandler({ + credentialsProvider, + accountId, + region, + }); + + return { + listLocations, + getLocationCredentials, + region, + }; +}; diff --git a/packages/storage/src/internals/managedAuthConfigAdapter/index.ts b/packages/storage/src/internals/managedAuthConfigAdapter/index.ts new file mode 100644 index 00000000000..3dbf18c8849 --- /dev/null +++ b/packages/storage/src/internals/managedAuthConfigAdapter/index.ts @@ -0,0 +1,8 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + AuthConfigAdapter, + createManagedAuthConfigAdapter, + CreateManagedAuthConfigAdapterInput, +} from './createManagedAuthConfigAdapter'; diff --git a/packages/storage/src/internals/types/common.ts b/packages/storage/src/internals/types/common.ts new file mode 100644 index 00000000000..2fd32ca6d8c --- /dev/null +++ b/packages/storage/src/internals/types/common.ts @@ -0,0 +1,22 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * @internal + */ +export type Permission = 'READ' | 'READWRITE' | 'WRITE'; + +/** + * @internal + */ +export type LocationType = 'BUCKET' | 'PREFIX' | 'OBJECT'; + +/** + * @internal + */ +export type Privilege = 'Default' | 'Minimal'; + +/** + * @internal + */ +export type PrefixType = 'Object'; diff --git a/packages/storage/src/internals/types/credentials.ts b/packages/storage/src/internals/types/credentials.ts new file mode 100644 index 00000000000..95d6de8a00e --- /dev/null +++ b/packages/storage/src/internals/types/credentials.ts @@ -0,0 +1,109 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + AWSTemporaryCredentials, + LocationCredentialsProvider, +} from '../../providers/s3/types/options'; + +import { LocationType, Permission } from './common'; + +/** + * @internal + */ +export type CredentialsProvider = LocationCredentialsProvider; + +export interface CreateLocationCredentialsStoreInput { + handler: GetLocationCredentials; +} + +export interface LocationCredentialsStore { + /** + * Get location-specific credentials. It uses a cache internally to optimize performance when + * getting credentials for the same location. It will refresh credentials if they expire or + * when forced to. + */ + getProvider(option: CredentialsLocation): LocationCredentialsProvider; + /** + * Invalidate cached credentials and force subsequent calls to get location-specific + * credentials to throw. It also makes subsequent calls to `getCredentialsProviderForLocation` + * to throw. + */ + destroy(): void; +} + +export interface LocationCredentials extends Partial { + /** + * AWS credentials which can be used to access the specified location. + */ + readonly credentials: AWSTemporaryCredentials; +} + +export type GetLocationCredentialsInput = CredentialsLocation; +export type GetLocationCredentialsOutput = LocationCredentials; + +export type GetLocationCredentials = ( + input: GetLocationCredentialsInput, +) => Promise; + +/** + * @internal + */ +export interface ListLocationsInput { + pageSize?: number; + nextToken?: string; +} + +/** + * @internal + */ +export interface ListLocationsOutput { + locations: LocationAccess[]; + nextToken?: string; +} + +/** + * @internal + */ +export type ListLocations = ( + input?: ListLocationsInput, +) => Promise; + +/** + * @internal + */ +export interface LocationScope { + /** + * Scope of storage location. For S3 service, it's the S3 path of the data to + * which the access is granted. It can be in following formats: + * + * @example Bucket 's3:///*' + * @example Prefix 's3:///*' + * @example Object 's3:////' + */ + readonly scope: string; +} + +/** + * @internal + */ +export interface CredentialsLocation extends LocationScope { + /** + * The type of access granted to your Storage data. Can be either of READ, + * WRITE or READWRITE + */ + readonly permission: Permission; +} + +/** + * @internal + */ +export interface LocationAccess extends CredentialsLocation { + /** + * Parse location type parsed from scope format: + * * BUCKET: `'s3:///*'` + * * PREFIX: `'s3:///*'` + * * OBJECT: `'s3:////'` + */ + readonly type: LocationType; +} diff --git a/packages/storage/src/internals/types/inputs.ts b/packages/storage/src/internals/types/inputs.ts new file mode 100644 index 00000000000..51d04acf9c6 --- /dev/null +++ b/packages/storage/src/internals/types/inputs.ts @@ -0,0 +1,28 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CredentialsProvider, ListLocationsInput } from './credentials'; +import { Permission, PrefixType, Privilege } from './common'; + +/** + * @internal + */ +export interface ListCallerAccessGrantsInput extends ListLocationsInput { + accountId: string; + credentialsProvider: CredentialsProvider; + region: string; +} + +/** + * @internal + */ +export interface GetDataAccessInput { + accountId: string; + credentialsProvider: CredentialsProvider; + durationSeconds?: number; + permission: Permission; + prefixType?: PrefixType; + privilege?: Privilege; + region: string; + scope: string; +} diff --git a/packages/storage/src/internals/types/options.ts b/packages/storage/src/internals/types/options.ts new file mode 100644 index 00000000000..cf1406c9425 --- /dev/null +++ b/packages/storage/src/internals/types/options.ts @@ -0,0 +1,2 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 diff --git a/packages/storage/src/internals/types/outputs.ts b/packages/storage/src/internals/types/outputs.ts new file mode 100644 index 00000000000..09eecb7cfc2 --- /dev/null +++ b/packages/storage/src/internals/types/outputs.ts @@ -0,0 +1,14 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { ListLocationsOutput, LocationCredentials } from './credentials'; + +/** + * @internal + */ +export type ListCallerAccessGrantsOutput = ListLocationsOutput; + +/** + * @internal + */ +export type GetDataAccessOutput = LocationCredentials; diff --git a/packages/storage/src/internals/utils/constants.ts b/packages/storage/src/internals/utils/constants.ts new file mode 100644 index 00000000000..4c322de94f1 --- /dev/null +++ b/packages/storage/src/internals/utils/constants.ts @@ -0,0 +1,5 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const DEFAULT_CRED_TTL = 15 * 60; // 15 minutes +export const MAX_PAGE_SIZE = 1000; diff --git a/packages/storage/src/providers/s3/apis/downloadData.ts b/packages/storage/src/providers/s3/apis/downloadData.ts index 7c98ee2b857..32f0c558642 100644 --- a/packages/storage/src/providers/s3/apis/downloadData.ts +++ b/packages/storage/src/providers/s3/apis/downloadData.ts @@ -12,7 +12,7 @@ import { } from '../types'; import { resolveS3ConfigAndInput } from '../utils/resolveS3ConfigAndInput'; import { createDownloadTask, validateStorageOperationInput } from '../utils'; -import { getObject } from '../utils/client'; +import { getObject } from '../utils/client/s3data'; import { getStorageUserAgentValue } from '../utils/userAgent'; import { logger } from '../../../utils'; import { @@ -115,7 +115,7 @@ const downloadDataJob = > => { const { options: downloadDataOptions } = downloadDataInput; const { bucket, keyPrefix, s3Config, identityId } = - await resolveS3ConfigAndInput(Amplify, downloadDataOptions); + await resolveS3ConfigAndInput(Amplify, downloadDataInput); const { inputType, objectKey } = validateStorageOperationInput( downloadDataInput, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/copy.ts b/packages/storage/src/providers/s3/apis/internal/copy.ts index 5035f897017..9119917efa1 100644 --- a/packages/storage/src/providers/s3/apis/internal/copy.ts +++ b/packages/storage/src/providers/s3/apis/internal/copy.ts @@ -18,7 +18,7 @@ import { } from '../../utils'; import { StorageValidationErrorCode } from '../../../../errors/types/validation'; import { assertValidationError } from '../../../../errors/utils/assertValidationError'; -import { copyObject } from '../../utils/client'; +import { copyObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; @@ -59,15 +59,27 @@ const copyWithPath = async ( storageBucketAssertion(source.bucket, destination.bucket); - const { bucket: sourceBucket, identityId } = await resolveS3ConfigAndInput( - amplify, - input.source, - ); + const { bucket: sourceBucket } = await resolveS3ConfigAndInput(amplify, { + path: input.source.path, + options: { + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.source, + }, + }); - const { s3Config, bucket: destBucket } = await resolveS3ConfigAndInput( - amplify, - input.destination, - ); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. + // The bucket, region, credentials of s3 client are resolved from destination. + // Whereas the source bucket and path are a input parameter of S3 copy operation. + const { + s3Config, + bucket: destBucket, + identityId, + } = await resolveS3ConfigAndInput(amplify, { + path: input.destination.path, + options: { + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.destination, + }, + }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. assertValidationError(!!source.path, StorageValidationErrorCode.NoSourcePath); assertValidationError( @@ -114,13 +126,31 @@ export const copyWithKey = async ( ); const { bucket: sourceBucket, keyPrefix: sourceKeyPrefix } = - await resolveS3ConfigAndInput(amplify, source); - + await resolveS3ConfigAndInput(amplify, { + ...input, + options: { + // @ts-expect-error: 'options' does not exist on type 'CopyInput'. In case of JS users set the location + // credentials provider option, resolveS3ConfigAndInput will throw validation error. + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.source, + }, + }); + + // The bucket, region, credentials of s3 client are resolved from destination. + // Whereas the source bucket and path are a input parameter of S3 copy operation. const { s3Config, bucket: destBucket, keyPrefix: destinationKeyPrefix, - } = await resolveS3ConfigAndInput(amplify, destination); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. + } = await resolveS3ConfigAndInput(amplify, { + ...input, + options: { + // @ts-expect-error: 'options' does not exist on type 'CopyInput'. In case of JS users set the location + // credentials provider option, resolveS3ConfigAndInput will throw validation error. + locationCredentialsProvider: input.options?.locationCredentialsProvider, + ...input.destination, + }, + }); // resolveS3ConfigAndInput does not make extra API calls or storage access if called repeatedly. // TODO(ashwinkumar6) V6-logger: warn `You may copy files from another user if the source level is "protected", currently it's ${srcLevel}` const finalCopySource = `${sourceBucket}/${sourceKeyPrefix}${source.key}`; diff --git a/packages/storage/src/providers/s3/apis/internal/getProperties.ts b/packages/storage/src/providers/s3/apis/internal/getProperties.ts index 3b61460d89b..ac04b2dbe6e 100644 --- a/packages/storage/src/providers/s3/apis/internal/getProperties.ts +++ b/packages/storage/src/providers/s3/apis/internal/getProperties.ts @@ -14,7 +14,7 @@ import { resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; -import { headObject } from '../../utils/client'; +import { headObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; @@ -24,9 +24,8 @@ export const getProperties = async ( input: GetPropertiesInput | GetPropertiesWithPathInput, action?: StorageAction, ): Promise => { - const { options: getPropertiesOptions } = input; const { s3Config, bucket, keyPrefix, identityId } = - await resolveS3ConfigAndInput(amplify, getPropertiesOptions); + await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/getUrl.ts b/packages/storage/src/providers/s3/apis/internal/getUrl.ts index a5c319a1389..98e7a198cc3 100644 --- a/packages/storage/src/providers/s3/apis/internal/getUrl.ts +++ b/packages/storage/src/providers/s3/apis/internal/getUrl.ts @@ -11,7 +11,7 @@ import { GetUrlWithPathOutput, } from '../../types'; import { StorageValidationErrorCode } from '../../../../errors/types/validation'; -import { getPresignedGetObjectUrl } from '../../utils/client'; +import { getPresignedGetObjectUrl } from '../../utils/client/s3data'; import { resolveS3ConfigAndInput, validateStorageOperationInput, @@ -32,7 +32,7 @@ export const getUrl = async ( ): Promise => { const { options: getUrlOptions } = input; const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput(amplify, getUrlOptions); + await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInput( input, identityId, diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index bbcb342a603..4a2fcc0b9e8 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -29,11 +29,11 @@ import { ListObjectsV2Input, ListObjectsV2Output, listObjectsV2, -} from '../../utils/client'; +} from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { DEFAULT_DELIMITER, STORAGE_INPUT_PREFIX } from '../../utils/constants'; -import { CommonPrefix } from '../../utils/client/types'; +import { CommonPrefix } from '../../utils/client/s3data/types'; const MAX_PAGE_SIZE = 1000; @@ -62,7 +62,7 @@ export const list = async ( bucket, keyPrefix: generatedPrefix, identityId, - } = await resolveS3ConfigAndInput(amplify, options); + } = await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInputWithPrefix( input, diff --git a/packages/storage/src/providers/s3/apis/internal/remove.ts b/packages/storage/src/providers/s3/apis/internal/remove.ts index bc0fa4a2ade..d73a13346e4 100644 --- a/packages/storage/src/providers/s3/apis/internal/remove.ts +++ b/packages/storage/src/providers/s3/apis/internal/remove.ts @@ -14,7 +14,7 @@ import { resolveS3ConfigAndInput, validateStorageOperationInput, } from '../../utils'; -import { deleteObject } from '../../utils/client'; +import { deleteObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { logger } from '../../../../utils'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; @@ -23,9 +23,8 @@ export const remove = async ( amplify: AmplifyClassV6, input: RemoveInput | RemoveWithPathInput, ): Promise => { - const { options = {} } = input ?? {}; const { s3Config, keyPrefix, bucket, identityId } = - await resolveS3ConfigAndInput(amplify, options); + await resolveS3ConfigAndInput(amplify, input); const { inputType, objectKey } = validateStorageOperationInput( input, diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts index 25338b2003f..32462a83545 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/initialUpload.ts @@ -5,8 +5,9 @@ import { StorageAccessLevel } from '@aws-amplify/core'; import { ContentDisposition, ResolvedS3Config } from '../../../types/options'; import { StorageUploadDataPayload } from '../../../../../types'; -import { Part, createMultipartUpload } from '../../../utils/client'; +import { Part, createMultipartUpload } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; +import { calculateContentCRC32 } from '../../../utils/crc32'; import { constructContentDisposition } from '../../../utils/constructContentDisposition'; import { @@ -14,6 +15,7 @@ import { findCachedUploadParts, getUploadsCacheKey, } from './uploadCache'; +import { getDataChunker } from './getDataChunker'; interface LoadOrCreateMultipartUploadOptions { s3Config: ResolvedS3Config; @@ -33,6 +35,7 @@ interface LoadOrCreateMultipartUploadOptions { interface LoadOrCreateMultipartUploadResult { uploadId: string; cachedParts: Part[]; + finalCrc32?: string; } /** @@ -62,6 +65,7 @@ export const loadOrCreateMultipartUpload = async ({ parts: Part[]; uploadId: string; uploadCacheKey: string; + finalCrc32?: string; } | undefined; if (size === undefined) { @@ -92,8 +96,11 @@ export const loadOrCreateMultipartUpload = async ({ return { uploadId: cachedUpload.uploadId, cachedParts: cachedUpload.parts, + finalCrc32: cachedUpload.finalCrc32, }; } else { + const finalCrc32 = await getCombinedCrc32(data, size); + const { UploadId } = await createMultipartUpload( { ...s3Config, @@ -106,14 +113,17 @@ export const loadOrCreateMultipartUpload = async ({ ContentDisposition: constructContentDisposition(contentDisposition), ContentEncoding: contentEncoding, Metadata: metadata, + ChecksumAlgorithm: finalCrc32 ? 'CRC32' : undefined, }, ); + if (size === undefined) { logger.debug('uploaded data size cannot be determined, skipping cache.'); return { uploadId: UploadId!, cachedParts: [], + finalCrc32, }; } const uploadCacheKey = getUploadsCacheKey({ @@ -128,12 +138,31 @@ export const loadOrCreateMultipartUpload = async ({ uploadId: UploadId!, bucket, key, + finalCrc32, fileName: data instanceof File ? data.name : '', }); return { uploadId: UploadId!, cachedParts: [], + finalCrc32, }; } }; + +const getCombinedCrc32 = async ( + data: StorageUploadDataPayload, + size: number | undefined, +) => { + const crc32List: ArrayBuffer[] = []; + const dataChunker = getDataChunker(data, size); + for (const { data: checkData } of dataChunker) { + const checksumArrayBuffer = (await calculateContentCRC32(checkData)) + ?.checksumArrayBuffer; + if (checksumArrayBuffer === undefined) return undefined; + + crc32List.push(checksumArrayBuffer); + } + + return `${(await calculateContentCRC32(new Blob(crc32List)))?.checksum}-${crc32List.length}`; +}; diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts index e5619655f3b..9761ee85732 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadCache.ts @@ -9,7 +9,7 @@ import { import { UPLOADS_STORAGE_KEY } from '../../../utils/constants'; import { ResolvedS3Config } from '../../../types/options'; -import { Part, listParts } from '../../../utils/client'; +import { Part, listParts } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; const ONE_HOUR = 1000 * 60 * 60; @@ -33,6 +33,7 @@ export const findCachedUploadParts = async ({ }: FindCachedUploadPartsOptions): Promise<{ parts: Part[]; uploadId: string; + finalCrc32?: string; } | null> => { const cachedUploads = await listCachedUploadTasks(defaultStorage); if ( @@ -60,6 +61,7 @@ export const findCachedUploadParts = async ({ return { parts: Parts, uploadId: cachedUpload.uploadId, + finalCrc32: cachedUpload.finalCrc32, }; } catch (e) { logger.debug('failed to list cached parts, removing cached upload.'); @@ -74,6 +76,7 @@ interface FileMetadata { fileName: string; key: string; uploadId: string; + finalCrc32?: string; // Unix timestamp in ms lastTouched: number; } diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts index 8d002df37db..f1869ac618a 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadHandlers.ts @@ -26,9 +26,10 @@ import { abortMultipartUpload, completeMultipartUpload, headObject, -} from '../../../utils/client'; +} from '../../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../../utils/userAgent'; import { logger } from '../../../../../utils'; +import { validateObjectNotExists } from '../validateObjectNotExists'; import { uploadPartExecutor } from './uploadPartExecutor'; import { getUploadsCacheKey, removeCachedUpload } from './uploadCache'; @@ -54,6 +55,7 @@ export const getMultipartUploadHandlers = ( | { uploadId: string; completedParts: Part[]; + finalCrc32?: string; } | undefined; let resolvedS3Config: ResolvedS3Config | undefined; @@ -73,7 +75,7 @@ export const getMultipartUploadHandlers = ( const { options: uploadDataOptions, data } = uploadDataInput; const resolvedS3Options = await resolveS3ConfigAndInput( Amplify, - uploadDataOptions, + uploadDataInput, ); abortController = new AbortController(); @@ -92,6 +94,7 @@ export const getMultipartUploadHandlers = ( contentEncoding, contentType = 'application/octet-stream', metadata, + preventOverwrite, onProgress, } = uploadDataOptions ?? {}; @@ -108,23 +111,25 @@ export const getMultipartUploadHandlers = ( } if (!inProgressUpload) { - const { uploadId, cachedParts } = await loadOrCreateMultipartUpload({ - s3Config: resolvedS3Config, - accessLevel: resolvedAccessLevel, - bucket: resolvedBucket, - keyPrefix: resolvedKeyPrefix, - key: objectKey, - contentType, - contentDisposition, - contentEncoding, - metadata, - data, - size, - abortSignal: abortController.signal, - }); + const { uploadId, cachedParts, finalCrc32 } = + await loadOrCreateMultipartUpload({ + s3Config: resolvedS3Config, + accessLevel: resolvedAccessLevel, + bucket: resolvedBucket, + keyPrefix: resolvedKeyPrefix, + key: objectKey, + contentType, + contentDisposition, + contentEncoding, + metadata, + data, + size, + abortSignal: abortController.signal, + }); inProgressUpload = { uploadId, completedParts: cachedParts, + finalCrc32, }; } @@ -143,10 +148,15 @@ export const getMultipartUploadHandlers = ( const completedPartNumberSet = new Set( inProgressUpload.completedParts.map(({ PartNumber }) => PartNumber!), ); - const onPartUploadCompletion = (partNumber: number, eTag: string) => { + const onPartUploadCompletion = ( + partNumber: number, + eTag: string, + crc32: string | undefined, + ) => { inProgressUpload?.completedParts.push({ PartNumber: partNumber, ETag: eTag, + ChecksumCRC32: crc32, }); }; const concurrentUploadsProgressTracker = @@ -169,12 +179,20 @@ export const getMultipartUploadHandlers = ( onPartUploadCompletion, onProgress: concurrentUploadsProgressTracker.getOnProgressListener(), isObjectLockEnabled: resolvedS3Options.isObjectLockEnabled, + useCRC32Checksum: Boolean(inProgressUpload.finalCrc32), }), ); } await Promise.all(concurrentUploadPartExecutors); + if (preventOverwrite) { + await validateObjectNotExists(resolvedS3Config, { + Bucket: resolvedBucket, + Key: finalKey, + }); + } + const { ETag: eTag } = await completeMultipartUpload( { ...resolvedS3Config, @@ -185,6 +203,7 @@ export const getMultipartUploadHandlers = ( Bucket: resolvedBucket, Key: finalKey, UploadId: inProgressUpload.uploadId, + ChecksumCRC32: inProgressUpload.finalCrc32, MultipartUpload: { Parts: inProgressUpload.completedParts.sort( (partA, partB) => partA.PartNumber! - partB.PartNumber!, diff --git a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts index c93d791aad3..03d9ebaffc1 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/multipart/uploadPartExecutor.ts @@ -3,9 +3,10 @@ import { TransferProgressEvent } from '../../../../../types'; import { ResolvedS3Config } from '../../../types/options'; -import { calculateContentMd5 } from '../../../utils'; -import { uploadPart } from '../../../utils/client'; +import { uploadPart } from '../../../utils/client/s3data'; import { logger } from '../../../../../utils'; +import { CRC32Checksum, calculateContentCRC32 } from '../../../utils/crc32'; +import { calculateContentMd5 } from '../../../utils'; import { PartToUpload } from './getDataChunker'; @@ -18,7 +19,12 @@ interface UploadPartExecutorOptions { finalKey: string; uploadId: string; isObjectLockEnabled?: boolean; - onPartUploadCompletion(partNumber: number, eTag: string): void; + useCRC32Checksum?: boolean; + onPartUploadCompletion( + partNumber: number, + eTag: string, + crc32: string | undefined, + ): void; onProgress?(event: TransferProgressEvent): void; } @@ -33,6 +39,7 @@ export const uploadPartExecutor = async ({ onPartUploadCompletion, onProgress, isObjectLockEnabled, + useCRC32Checksum, }: UploadPartExecutorOptions) => { let transferredBytes = 0; for (const { data, partNumber, size } of dataChunkerGenerator) { @@ -49,6 +56,16 @@ export const uploadPartExecutor = async ({ }); } else { // handle cancel error + let checksumCRC32: CRC32Checksum | undefined; + if (useCRC32Checksum) { + checksumCRC32 = await calculateContentCRC32(data); + } + const contentMD5 = + // check if checksum exists. ex: should not exist in react native + !checksumCRC32 && isObjectLockEnabled + ? await calculateContentMd5(data) + : undefined; + const { ETag: eTag } = await uploadPart( { ...s3Config, @@ -66,14 +83,13 @@ export const uploadPartExecutor = async ({ UploadId: uploadId, Body: data, PartNumber: partNumber, - ContentMD5: isObjectLockEnabled - ? await calculateContentMd5(data) - : undefined, + ChecksumCRC32: checksumCRC32?.checksum, + ContentMD5: contentMD5, }, ); transferredBytes += size; // eTag will always be set even the S3 model interface marks it as optional. - onPartUploadCompletion(partNumber, eTag!); + onPartUploadCompletion(partNumber, eTag!, checksumCRC32?.checksum); } } }; diff --git a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts index 262a046ac71..cbd602580a0 100644 --- a/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts +++ b/packages/storage/src/providers/s3/apis/uploadData/putObjectJob.ts @@ -11,11 +11,14 @@ import { validateStorageOperationInput, } from '../../utils'; import { ItemWithKey, ItemWithPath } from '../../types/outputs'; -import { putObject } from '../../utils/client'; +import { putObject } from '../../utils/client/s3data'; import { getStorageUserAgentValue } from '../../utils/userAgent'; import { STORAGE_INPUT_KEY } from '../../utils/constants'; +import { calculateContentCRC32 } from '../../utils/crc32'; import { constructContentDisposition } from '../../utils/constructContentDisposition'; +import { validateObjectNotExists } from './validateObjectNotExists'; + /** * Get a function the returns a promise to call putObject API to S3. * @@ -30,7 +33,7 @@ export const putObjectJob = async (): Promise => { const { options: uploadDataOptions, data } = uploadDataInput; const { bucket, keyPrefix, s3Config, isObjectLockEnabled, identityId } = - await resolveS3ConfigAndInput(Amplify, uploadDataOptions); + await resolveS3ConfigAndInput(Amplify, uploadDataInput); const { inputType, objectKey } = validateStorageOperationInput( uploadDataInput, identityId, @@ -42,10 +45,25 @@ export const putObjectJob = contentDisposition, contentEncoding, contentType = 'application/octet-stream', + preventOverwrite, metadata, onProgress, } = uploadDataOptions ?? {}; + const checksumCRC32 = await calculateContentCRC32(data); + const contentMD5 = + // check if checksum exists. ex: should not exist in react native + !checksumCRC32 && isObjectLockEnabled + ? await calculateContentMd5(data) + : undefined; + + if (preventOverwrite) { + await validateObjectNotExists(s3Config, { + Bucket: bucket, + Key: finalKey, + }); + } + const { ETag: eTag, VersionId: versionId } = await putObject( { ...s3Config, @@ -61,9 +79,8 @@ export const putObjectJob = ContentDisposition: constructContentDisposition(contentDisposition), ContentEncoding: contentEncoding, Metadata: metadata, - ContentMD5: isObjectLockEnabled - ? await calculateContentMd5(data) - : undefined, + ContentMD5: contentMD5, + ChecksumCRC32: checksumCRC32?.checksum, }, ); diff --git a/packages/storage/src/providers/s3/apis/uploadData/validateObjectNotExists.ts b/packages/storage/src/providers/s3/apis/uploadData/validateObjectNotExists.ts new file mode 100644 index 00000000000..d2d68e3bdb0 --- /dev/null +++ b/packages/storage/src/providers/s3/apis/uploadData/validateObjectNotExists.ts @@ -0,0 +1,25 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageError } from '../../../../errors/StorageError'; +import { ResolvedS3Config } from '../../types/options'; +import { HeadObjectInput, headObject } from '../../utils/client/s3data'; + +export const validateObjectNotExists = async ( + s3Config: ResolvedS3Config, + input: HeadObjectInput, +): Promise => { + try { + await headObject(s3Config, input); + + throw new StorageError({ + name: 'PreconditionFailed', + message: 'At least one of the pre-conditions you specified did not hold', + }); + } catch (error) { + const serviceError = error as StorageError; + if (serviceError.name !== 'NotFound') { + throw error; + } + } +}; diff --git a/packages/storage/src/providers/s3/types/inputs.ts b/packages/storage/src/providers/s3/types/inputs.ts index 7b405964fb6..041451fbfd5 100644 --- a/packages/storage/src/providers/s3/types/inputs.ts +++ b/packages/storage/src/providers/s3/types/inputs.ts @@ -17,6 +17,7 @@ import { StorageUploadDataInputWithKey, StorageUploadDataInputWithPath, } from '../../../types'; +import { StorageOperationOptionsInput } from '../../../types/inputs'; import { CopyDestinationWithKeyOptions, CopySourceWithKeyOptions, @@ -35,6 +36,8 @@ import { UploadDataWithPathOptions, } from '../types'; +import { LocationCredentialsProvider } from './options'; + // TODO: support use accelerate endpoint option /** * @deprecated Use {@link CopyWithPathInput} instead. @@ -47,7 +50,10 @@ export type CopyInput = StorageCopyInputWithKey< /** * Input type with path for S3 copy API. */ -export type CopyWithPathInput = StorageCopyInputWithPath; +export type CopyWithPathInput = StorageCopyInputWithPath & + StorageOperationOptionsInput<{ + locationCredentialsProvider?: LocationCredentialsProvider; + }>; /** * @deprecated Use {@link GetPropertiesWithPathInput} instead. diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 9a608c6dd2b..3afeded1b0c 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -2,7 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { StorageAccessLevel } from '@aws-amplify/core'; -import { SigningOptions } from '@aws-amplify/core/internals/aws-client-utils'; +import { AWSCredentials } from '@aws-amplify/core/internals/utils'; +import { + CredentialsProviderOptions, + SigningOptions, +} from '@aws-amplify/core/internals/aws-client-utils'; import { TransferProgressEvent } from '../../../types'; import { @@ -11,6 +15,23 @@ import { StorageSubpathStrategy, } from '../../../types/options'; +/** + * @internal + */ +export type AWSTemporaryCredentials = Required< + Pick< + AWSCredentials, + 'accessKeyId' | 'secretAccessKey' | 'sessionToken' | 'expiration' + > +>; + +/** + * @internal + */ +export type LocationCredentialsProvider = ( + options?: CredentialsProviderOptions, +) => Promise<{ credentials: AWSTemporaryCredentials }>; + export interface BucketInfo { bucketName: string; region: string; @@ -23,6 +44,14 @@ interface CommonOptions { * @default false */ useAccelerateEndpoint?: boolean; + + /** + * Async function returning AWS credentials for an API call. This function + * is invoked with S3 locations(bucket and path). + * If omitted, the global credentials configured in Amplify Auth + * would be used. + */ + locationCredentialsProvider?: LocationCredentialsProvider; bucket?: StorageBucket; } @@ -190,6 +219,11 @@ export type UploadDataOptions = CommonOptions & * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html#UserMetadata */ metadata?: Record; + /** + * Enforces target key does not already exist in S3 before committing upload. + * @default false + */ + preventOverwrite?: boolean; }; /** @deprecated Use {@link UploadDataWithPathOptions} instead. */ diff --git a/packages/storage/src/providers/s3/utils/client/s3control/base.ts b/packages/storage/src/providers/s3/utils/client/s3control/base.ts new file mode 100644 index 00000000000..590f2b26120 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/base.ts @@ -0,0 +1,97 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + AmplifyUrl, + getAmplifyUserAgent, +} from '@aws-amplify/core/internals/utils'; +import { + EndpointResolverOptions, + getDnsSuffix, + jitteredBackoff, +} from '@aws-amplify/core/internals/aws-client-utils'; + +import { createRetryDecider, createXmlErrorParser } from '../utils'; + +/** + * The service name used to sign requests if the API requires authentication. + */ +export const SERVICE_NAME = 's3'; + +/** + * Options for endpoint resolver. + * + * @internal + */ +export type S3EndpointResolverOptions = EndpointResolverOptions & { + /** + * Fully qualified custom endpoint for S3. If this is set, this endpoint will be used regardless of region. + */ + customEndpoint?: string; +}; + +/** + * The endpoint resolver function that returns the endpoint URL for a given region, and input parameters. + */ +const endpointResolver = ( + options: S3EndpointResolverOptions, + apiInput?: { AccountId?: string }, +) => { + const { region, customEndpoint } = options; + const { AccountId: accountId } = apiInput || {}; + let endpoint: URL; + // 1. get base endpoint + if (customEndpoint) { + endpoint = new AmplifyUrl(customEndpoint); + } else if (accountId) { + // Control plane operations + endpoint = new AmplifyUrl( + `https://${accountId}.s3-control.${region}.${getDnsSuffix(region)}`, + ); + } else { + endpoint = new AmplifyUrl( + `https://s3-control.${region}.${getDnsSuffix(region)}`, + ); + } + + return { url: endpoint }; +}; + +/** + * Error parser for the XML payload of S3 control plane error response. The + * error's `Code` and `Message` locates at the nested `Error` element instead of + * the XML root element. + * + * @example + * ``` + * + * + * + * AccessDenied + * Access Denied + * + * 656c76696e6727732072657175657374 + * Uuag1LuByRx9e6j5Onimru9pO4ZVKnJ2Qz7/C1NPcfTWAtRPfTaOFg== + * + * ``` + * + * @internal + */ +export const parseXmlError = createXmlErrorParser(); + +/** + * @internal + */ +export const retryDecider = createRetryDecider(parseXmlError); + +/** + * @internal + */ +export const defaultConfig = { + service: SERVICE_NAME, + endpointResolver, + retryDecider, + computeDelay: jitteredBackoff, + userAgentValue: getAmplifyUserAgent(), + uriEscapePath: false, // Required by S3. See https://github.com/aws/aws-sdk-js-v3/blob/9ba012dfa3a3429aa2db0f90b3b0b3a7a31f9bc3/packages/signature-v4/src/SignatureV4.ts#L76-L83 +}; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts b/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts new file mode 100644 index 00000000000..84adb14e8aa --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/getDataAccess.ts @@ -0,0 +1,96 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + Endpoint, + HttpRequest, + HttpResponse, + parseMetadata, +} from '@aws-amplify/core/internals/aws-client-utils'; +import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; +import { + AmplifyUrl, + AmplifyUrlSearchParams, +} from '@aws-amplify/core/internals/utils'; + +import { + assignStringVariables, + buildStorageServiceError, + deserializeTimestamp, + map, + parseXmlBody, + s3TransferHandler, +} from '../utils'; + +import type { + GetDataAccessCommandInput, + GetDataAccessCommandOutput, +} from './types'; +import { defaultConfig, parseXmlError } from './base'; + +export type GetDataAccessInput = GetDataAccessCommandInput; + +export type GetDataAccessOutput = GetDataAccessCommandOutput; + +const getDataAccessSerializer = ( + input: GetDataAccessInput, + endpoint: Endpoint, +): HttpRequest => { + const headers = assignStringVariables({ + 'x-amz-account-id': input.AccountId, + }); + const query = assignStringVariables({ + durationSeconds: input.DurationSeconds, + permission: input.Permission, + privilege: input.Privilege, + target: input.Target, + targetType: input.TargetType, + }); + const url = new AmplifyUrl(endpoint.url.toString()); + url.search = new AmplifyUrlSearchParams(query).toString(); + + // Ref: https://docs.aws.amazon.com/AmazonS3/latest/API/API_control_GetDataAccess.html + url.pathname = '/v20180820/accessgrantsinstance/dataaccess'; + + return { + method: 'GET', + headers, + url, + }; +}; + +const getDataAccessDeserializer = async ( + response: HttpResponse, +): Promise => { + if (response.statusCode >= 300) { + // error is always set when statusCode >= 300 + const error = (await parseXmlError(response)) as Error; + throw buildStorageServiceError(error, response.statusCode); + } else { + const parsed = await parseXmlBody(response); + const contents = map(parsed, { + Credentials: ['Credentials', deserializeCredentials], + MatchedGrantTarget: 'MatchedGrantTarget', + }); + + return { + $metadata: parseMetadata(response), + ...contents, + }; + } +}; + +const deserializeCredentials = (output: any) => + map(output, { + AccessKeyId: 'AccessKeyId', + Expiration: ['Expiration', deserializeTimestamp], + SecretAccessKey: 'SecretAccessKey', + SessionToken: 'SessionToken', + }); + +export const getDataAccess = composeServiceApi( + s3TransferHandler, + getDataAccessSerializer, + getDataAccessDeserializer, + { ...defaultConfig, responseType: 'text' }, +); diff --git a/packages/storage/src/providers/s3/utils/client/s3control/index.ts b/packages/storage/src/providers/s3/utils/client/s3control/index.ts new file mode 100644 index 00000000000..b9ae5230334 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/index.ts @@ -0,0 +1,13 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +export { + getDataAccess, + GetDataAccessInput, + GetDataAccessOutput, +} from '../s3control/getDataAccess'; +export { + listCallerAccessGrants, + ListCallerAccessGrantsInput, + ListCallerAccessGrantsOutput, +} from '../s3control/listCallerAccessGrants'; diff --git a/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts new file mode 100644 index 00000000000..81b0e62a9c8 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/listCallerAccessGrants.ts @@ -0,0 +1,107 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + Endpoint, + HttpRequest, + HttpResponse, + parseMetadata, +} from '@aws-amplify/core/internals/aws-client-utils'; +import { + AmplifyUrl, + AmplifyUrlSearchParams, +} from '@aws-amplify/core/internals/utils'; +import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; + +import { + assignStringVariables, + buildStorageServiceError, + emptyArrayGuard, + map, + parseXmlBody, + s3TransferHandler, +} from '../utils'; +import { createStringEnumDeserializer } from '../utils/deserializeHelpers'; + +import type { + ListCallerAccessGrantsCommandInput, + ListCallerAccessGrantsCommandOutput, +} from './types'; +import { defaultConfig, parseXmlError } from './base'; + +export type ListCallerAccessGrantsInput = ListCallerAccessGrantsCommandInput; + +export type ListCallerAccessGrantsOutput = ListCallerAccessGrantsCommandOutput; + +const listCallerAccessGrantsSerializer = ( + input: ListCallerAccessGrantsInput, + endpoint: Endpoint, +): HttpRequest => { + const headers = assignStringVariables({ + 'x-amz-account-id': input.AccountId, + }); + const query = assignStringVariables({ + grantscope: input.GrantScope, + maxResults: input.MaxResults, + nextToken: input.NextToken, + }); + const url = new AmplifyUrl(endpoint.url.toString()); + url.search = new AmplifyUrlSearchParams(query).toString(); + + // Ref: NA + url.pathname = '/v20180820/accessgrantsinstance/caller/grants'; + + return { + method: 'GET', + headers, + url, + }; +}; + +const listCallerAccessGrantsDeserializer = async ( + response: HttpResponse, +): Promise => { + if (response.statusCode >= 300) { + // error is always set when statusCode >= 300 + const error = (await parseXmlError(response)) as Error; + throw buildStorageServiceError(error, response.statusCode); + } else { + const parsed = await parseXmlBody(response); + const contents = map(parsed, { + CallerAccessGrantsList: [ + 'CallerAccessGrantsList', + value => + emptyArrayGuard(value.AccessGrant, deserializeAccessGrantsList), + ], + NextToken: 'NextToken', + }); + + return { + $metadata: parseMetadata(response), + ...contents, + }; + } +}; + +const deserializeAccessGrantsList = (output: any[]) => + output.map(deserializeCallerAccessGrant); + +const deserializeCallerAccessGrant = (output: any) => + map(output, { + ApplicationArn: 'ApplicationArn', + GrantScope: 'GrantScope', + Permission: [ + 'Permission', + createStringEnumDeserializer( + ['READ', 'READWRITE', 'WRITE'] as const, + 'Permission', + ), + ], + }); + +export const listCallerAccessGrants = composeServiceApi( + s3TransferHandler, + listCallerAccessGrantsSerializer, + listCallerAccessGrantsDeserializer, + { ...defaultConfig, responseType: 'text' }, +); diff --git a/packages/storage/src/providers/s3/utils/client/s3control/types.ts b/packages/storage/src/providers/s3/utils/client/s3control/types.ts new file mode 100644 index 00000000000..088bfc8b3ba --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/s3control/types.ts @@ -0,0 +1,216 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/* +This file contains manually curated AWS service types that are not yet available via the AWS SDK and the `dts-bundler` +script. Once these APIs have been released to the AWS SDK, this file can be removed in favor of the `dts-bundler` +types. + +These types were harvested from Trebuchet. + +@TODO(jimblanc) Unify types & integrate with `dts-bundler` +*/ + +import { MetadataBearer as __MetadataBearer } from '@aws-sdk/types'; + +declare const Permission: { + readonly READ: 'READ'; + readonly READWRITE: 'READWRITE'; + readonly WRITE: 'WRITE'; +}; +declare const Privilege: { + readonly Default: 'Default'; + readonly Minimal: 'Minimal'; +}; +declare const S3PrefixType: { + readonly Object: 'Object'; +}; + +/** + *

The Amazon Web Services Security Token Service temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ +export interface Credentials { + /** + *

The unique access key ID of the Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ + AccessKeyId?: string; + /** + *

The secret access key of the Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + */ + SecretAccessKey?: string; + /** + *

The Amazon Web Services STS temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + * @public + * + * The input for {@link ListCallerAccessGrantsCommand}. + * @public + * + * The input for {@link ListCallerAccessGrantsCommand}. + */ + SessionToken?: string; + /** + *

The expiration date and time of the temporary credential that S3 Access Grants vends to grantees and client applications.

+ * @public + * @public + * + * The output of {@link ListCallerAccessGrantsCommand}. + * @public + * + * The output of {@link ListCallerAccessGrantsCommand}. + */ + Expiration?: Date; +} + +/** + * @public + * + * The input for {@link GetDataAccessCommand}. + */ +export type GetDataAccessCommandInput = GetDataAccessRequest; +/** + * @public + * + * The output of {@link GetDataAccessCommand}. + */ +export interface GetDataAccessCommandOutput + extends GetDataAccessResult, + __MetadataBearer {} + +/** + * @public + */ +export interface GetDataAccessRequest { + /** + *

The ID of the Amazon Web Services account that is making this request.

+ * @public + */ + AccountId?: string; + + /** + *

The S3 URI path of the data to which you are requesting temporary access credentials. If the requesting account has an access grant for this data, S3 Access Grants vends temporary access credentials in the response.

+ * @public + */ + Target: string | undefined; + + /** + *

The type of permission granted to your S3 data, which can be set to one of the following values:

+ *
    + *
  • + *

    + * READ – Grant read-only access to the S3 data.

    + *
    • + *
  • + *

    + * WRITE – Grant write-only access to the S3 data.

    + *
    • + *
  • + *

    + * READWRITE – Grant both read and write access to the S3 data.

    + *
    • + *
+ * @public + */ + Permission: Permission | undefined; + + /** + *

The session duration, in seconds, of the temporary access credential that S3 Access Grants vends to the grantee or client application. The default value is 1 hour, but the grantee can specify a range from 900 seconds (15 minutes) up to 43200 seconds (12 hours). If the grantee requests a value higher than this maximum, the operation fails.

+ * @public + */ + DurationSeconds?: number; + + /** + *

The scope of the temporary access credential that S3 Access Grants vends to the grantee or client application.

+ *
    + *
  • + *

    + * Default – The scope of the returned temporary access token is the scope of the grant that is closest to the target scope.

    + *
    • + *
  • + *

    + * Minimal – The scope of the returned temporary access token is the same as the requested target scope as long as the requested scope is the same as or a subset of the grant scope.

    + *
    • + *
+ * @public + */ + Privilege?: Privilege; + + /** + *

The type of Target. The only possible value is Object. Pass this value if the target data that you would like to access is a path to an object. Do not pass this value if the target data is a bucket or a bucket and a prefix.

+ * @public + */ + TargetType?: S3PrefixType; +} + +/** + * @public + */ +export interface GetDataAccessResult { + /** + *

The temporary credential token that S3 Access Grants vends.

+ * @public + */ + Credentials?: Credentials; + + /** + *

The S3 URI path of the data to which you are being granted temporary access credentials.

+ * @public + */ + MatchedGrantTarget?: string; +} + +/** + * @public + * + * The input for {@link ListCallerAccessGrantsCommand}. + */ +export type ListCallerAccessGrantsCommandInput = ListCallerAccessGrantsRequest; +/** + * @public + * + * The output of {@link ListCallerAccessGrantsCommand}. + */ +export interface ListCallerAccessGrantsCommandOutput + extends ListCallerAccessGrantsResult, + __MetadataBearer {} +/** + * @public + */ +export interface ListCallerAccessGrantsEntry { + Permission?: Permission; + GrantScope?: string; + ApplicationArn?: string; +} +/** + * @public + */ +export interface ListCallerAccessGrantsRequest { + AccountId?: string; + GrantScope?: string; + NextToken?: string; + MaxResults?: number; +} +/** + * @public + */ +export interface ListCallerAccessGrantsResult { + NextToken?: string; + CallerAccessGrantsList?: ListCallerAccessGrantsEntry[]; +} +/** + * @public + */ +export type Permission = (typeof Permission)[keyof typeof Permission]; +/** + * @public + */ +export type Privilege = (typeof Privilege)[keyof typeof Privilege]; +/** + * @public + */ +export type S3PrefixType = (typeof S3PrefixType)[keyof typeof S3PrefixType]; + +export {}; diff --git a/packages/storage/src/providers/s3/utils/client/abortMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts similarity index 96% rename from packages/storage/src/providers/s3/utils/client/abortMultipartUpload.ts rename to packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts index bddaf570d0e..fe2182e0c30 100644 --- a/packages/storage/src/providers/s3/utils/client/abortMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/abortMultipartUpload.ts @@ -14,15 +14,15 @@ import { } from '@aws-amplify/core/internals/utils'; import { MetadataBearer } from '@aws-sdk/types'; -import type { AbortMultipartUploadCommandInput } from './types'; -import { defaultConfig } from './base'; import { buildStorageServiceError, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { AbortMultipartUploadCommandInput } from './types'; +import { defaultConfig, parseXmlError } from './base'; export type AbortMultipartUploadInput = Pick< AbortMultipartUploadCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/base.ts b/packages/storage/src/providers/s3/utils/client/s3data/base.ts similarity index 82% rename from packages/storage/src/providers/s3/utils/client/base.ts rename to packages/storage/src/providers/s3/utils/client/s3data/base.ts index 96f0e5958ef..c7aef5c033c 100644 --- a/packages/storage/src/providers/s3/utils/client/base.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/base.ts @@ -8,11 +8,10 @@ import { import { EndpointResolverOptions, getDnsSuffix, - getRetryDecider, jitteredBackoff, } from '@aws-amplify/core/internals/aws-client-utils'; -import { parseXmlError } from './utils'; +import { createRetryDecider, createXmlErrorParser } from '../utils'; const DOMAIN_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/; const IP_ADDRESS_PATTERN = /(\d+\.){3}\d+/; @@ -100,13 +99,37 @@ export const isDnsCompatibleBucketName = (bucketName: string): boolean => !IP_ADDRESS_PATTERN.test(bucketName) && !DOTS_PATTERN.test(bucketName); +/** + * Error parser for the XML payload of S3 data plane error response. The error's + * `Code` and `Message` locates directly at the XML root element. + * + * @example + * ``` + * + * + * NoSuchKey + * The resource you requested does not exist + * /mybucket/myfoto.jpg + * 4442587FB7D0A2F9 + * + * ``` + * + * @internal + */ +export const parseXmlError = createXmlErrorParser({ noErrorWrapping: true }); + +/** + * @internal + */ +export const retryDecider = createRetryDecider(parseXmlError); + /** * @internal */ export const defaultConfig = { service: SERVICE_NAME, endpointResolver, - retryDecider: getRetryDecider(parseXmlError), + retryDecider, computeDelay: jitteredBackoff, userAgentValue: getAmplifyUserAgent(), useAccelerateEndpoint: false, diff --git a/packages/storage/src/providers/s3/utils/client/completeMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts similarity index 84% rename from packages/storage/src/providers/s3/utils/client/completeMultipartUpload.ts rename to packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts index 36dd9f59a52..00267d21172 100644 --- a/packages/storage/src/providers/s3/utils/client/completeMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/completeMultipartUpload.ts @@ -5,6 +5,8 @@ import { Endpoint, HttpRequest, HttpResponse, + MiddlewareContext, + RetryDeciderOutput, parseMetadata, } from '@aws-amplify/core/internals/aws-client-utils'; import { @@ -13,29 +15,30 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - CompleteMultipartUploadCommandInput, - CompleteMultipartUploadCommandOutput, - CompletedMultipartUpload, - CompletedPart, -} from './types'; -import { defaultConfig } from './base'; import { + assignStringVariables, buildStorageServiceError, map, parseXmlBody, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + CompleteMultipartUploadCommandInput, + CompleteMultipartUploadCommandOutput, + CompletedMultipartUpload, + CompletedPart, +} from './types'; +import { defaultConfig, parseXmlError, retryDecider } from './base'; const INVALID_PARAMETER_ERROR_MSG = 'Invalid parameter for ComplteMultipartUpload API'; export type CompleteMultipartUploadInput = Pick< CompleteMultipartUploadCommandInput, - 'Bucket' | 'Key' | 'UploadId' | 'MultipartUpload' + 'Bucket' | 'Key' | 'UploadId' | 'MultipartUpload' | 'ChecksumCRC32' >; export type CompleteMultipartUploadOutput = Pick< @@ -49,6 +52,7 @@ const completeMultipartUploadSerializer = async ( ): Promise => { const headers = { 'content-type': 'application/xml', + ...assignStringVariables({ 'x-amz-checksum-crc32': input.ChecksumCRC32 }), }; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); @@ -86,7 +90,13 @@ const serializeCompletedPartList = (input: CompletedPart): string => { throw new Error(`${INVALID_PARAMETER_ERROR_MSG}: ${input}`); } - return `${input.ETag}${input.PartNumber}`; + const eTag = `${input.ETag}`; + const partNumber = `${input.PartNumber}`; + const checksumCRC32 = input.ChecksumCRC32 + ? `${input.ChecksumCRC32}` + : ''; + + return `${eTag}${partNumber}${checksumCRC32}`; }; /** @@ -135,25 +145,24 @@ const completeMultipartUploadDeserializer = async ( const retryWhenErrorWith200StatusCode = async ( response?: HttpResponse, error?: unknown, -): Promise => { + middlewareContext?: MiddlewareContext, +): Promise => { if (!response) { - return false; + return { retryable: false }; } if (response.statusCode === 200) { if (!response.body) { - return true; + return { retryable: true }; } const parsed = await parseXmlBody(response); if (parsed.Code !== undefined && parsed.Message !== undefined) { - return true; + return { retryable: true }; } - return false; + return { retryable: false }; } - const defaultRetryDecider = defaultConfig.retryDecider; - - return defaultRetryDecider(response, error); + return retryDecider(response, error, middlewareContext); }; export const completeMultipartUpload = composeServiceApi( diff --git a/packages/storage/src/providers/s3/utils/client/copyObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts similarity index 96% rename from packages/storage/src/providers/s3/utils/client/copyObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts index a08301d9f7e..612e650aabb 100644 --- a/packages/storage/src/providers/s3/utils/client/copyObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/copyObject.ts @@ -10,18 +10,18 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { CopyObjectCommandInput, CopyObjectCommandOutput } from './types'; -import { defaultConfig } from './base'; import { assignStringVariables, buildStorageServiceError, parseXmlBody, - parseXmlError, s3TransferHandler, serializeObjectConfigsToHeaders, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { CopyObjectCommandInput, CopyObjectCommandOutput } from './types'; +import { defaultConfig, parseXmlError } from './base'; export type CopyObjectInput = Pick< CopyObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/createMultipartUpload.ts b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts similarity index 88% rename from packages/storage/src/providers/s3/utils/client/createMultipartUpload.ts rename to packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts index 5a2b79a9635..10549673a0e 100644 --- a/packages/storage/src/providers/s3/utils/client/createMultipartUpload.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/createMultipartUpload.ts @@ -10,22 +10,23 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - CreateMultipartUploadCommandInput, - CreateMultipartUploadCommandOutput, -} from './types'; -import type { PutObjectInput } from './putObject'; -import { defaultConfig } from './base'; import { + assignStringVariables, buildStorageServiceError, map, parseXmlBody, - parseXmlError, s3TransferHandler, serializeObjectConfigsToHeaders, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + CreateMultipartUploadCommandInput, + CreateMultipartUploadCommandOutput, +} from './types'; +import type { PutObjectInput } from './putObject'; +import { defaultConfig, parseXmlError } from './base'; export type CreateMultipartUploadInput = Extract< CreateMultipartUploadCommandInput, @@ -41,7 +42,12 @@ const createMultipartUploadSerializer = async ( input: CreateMultipartUploadInput, endpoint: Endpoint, ): Promise => { - const headers = await serializeObjectConfigsToHeaders(input); + const headers = { + ...(await serializeObjectConfigsToHeaders(input)), + ...assignStringVariables({ + 'x-amz-checksum-algorithm': input.ChecksumAlgorithm, + }), + }; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); url.pathname = serializePathnameObjectKey(url, input.Key); diff --git a/packages/storage/src/providers/s3/utils/client/deleteObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts similarity index 96% rename from packages/storage/src/providers/s3/utils/client/deleteObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts index 290a3e5ebf0..eb9c98b47ac 100644 --- a/packages/storage/src/providers/s3/utils/client/deleteObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/deleteObject.ts @@ -10,20 +10,20 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - DeleteObjectCommandInput, - DeleteObjectCommandOutput, -} from './types'; -import { defaultConfig } from './base'; import { buildStorageServiceError, deserializeBoolean, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + DeleteObjectCommandInput, + DeleteObjectCommandOutput, +} from './types'; +import { defaultConfig, parseXmlError } from './base'; export type DeleteObjectInput = Pick< DeleteObjectCommandInput, diff --git a/packages/storage/src/providers/s3/utils/client/getObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts similarity index 98% rename from packages/storage/src/providers/s3/utils/client/getObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/getObject.ts index 2b4153541cd..37ceade549d 100644 --- a/packages/storage/src/providers/s3/utils/client/getObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/getObject.ts @@ -11,15 +11,9 @@ import { parseMetadata, presignUrl, } from '@aws-amplify/core/internals/aws-client-utils'; -import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; +import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; -import { S3EndpointResolverOptions, defaultConfig } from './base'; -import type { - CompatibleHttpResponse, - GetObjectCommandInput, - GetObjectCommandOutput, -} from './types'; import { CONTENT_SHA256_HEADER, buildStorageServiceError, @@ -28,11 +22,21 @@ import { deserializeNumber, deserializeTimestamp, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { + S3EndpointResolverOptions, + defaultConfig, + parseXmlError, +} from './base'; +import type { + CompatibleHttpResponse, + GetObjectCommandInput, + GetObjectCommandOutput, +} from './types'; const USER_AGENT_HEADER = 'x-amz-user-agent'; diff --git a/packages/storage/src/providers/s3/utils/client/headObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts similarity index 96% rename from packages/storage/src/providers/s3/utils/client/headObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/headObject.ts index 109263def26..ebf0c374ff5 100644 --- a/packages/storage/src/providers/s3/utils/client/headObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/headObject.ts @@ -10,19 +10,19 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import { defaultConfig } from './base'; -import type { HeadObjectCommandInput, HeadObjectCommandOutput } from './types'; import { buildStorageServiceError, deserializeMetadata, deserializeNumber, deserializeTimestamp, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { defaultConfig, parseXmlError } from './base'; +import type { HeadObjectCommandInput, HeadObjectCommandOutput } from './types'; export type HeadObjectInput = Pick; diff --git a/packages/storage/src/providers/s3/utils/client/index.ts b/packages/storage/src/providers/s3/utils/client/s3data/index.ts similarity index 100% rename from packages/storage/src/providers/s3/utils/client/index.ts rename to packages/storage/src/providers/s3/utils/client/s3data/index.ts diff --git a/packages/storage/src/providers/s3/utils/client/listObjectsV2.ts b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts similarity index 97% rename from packages/storage/src/providers/s3/utils/client/listObjectsV2.ts rename to packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts index 232499931c5..b5db2c04ae8 100644 --- a/packages/storage/src/providers/s3/utils/client/listObjectsV2.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts @@ -13,11 +13,6 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - ListObjectsV2CommandInput, - ListObjectsV2CommandOutput, -} from './types'; -import { defaultConfig } from './base'; import { assignStringVariables, buildStorageServiceError, @@ -27,9 +22,14 @@ import { emptyArrayGuard, map, parseXmlBody, - parseXmlError, s3TransferHandler, -} from './utils'; +} from '../utils'; + +import type { + ListObjectsV2CommandInput, + ListObjectsV2CommandOutput, +} from './types'; +import { defaultConfig, parseXmlError } from './base'; export type ListObjectsV2Input = ListObjectsV2CommandInput; diff --git a/packages/storage/src/providers/s3/utils/client/listParts.ts b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts similarity index 95% rename from packages/storage/src/providers/s3/utils/client/listParts.ts rename to packages/storage/src/providers/s3/utils/client/s3data/listParts.ts index 86899ad4e9d..5c4583d1ed2 100644 --- a/packages/storage/src/providers/s3/utils/client/listParts.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listParts.ts @@ -13,23 +13,23 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import type { - CompletedPart, - ListPartsCommandInput, - ListPartsCommandOutput, -} from './types'; -import { defaultConfig } from './base'; import { buildStorageServiceError, deserializeNumber, emptyArrayGuard, map, parseXmlBody, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import type { + CompletedPart, + ListPartsCommandInput, + ListPartsCommandOutput, +} from './types'; +import { defaultConfig, parseXmlError } from './base'; export type ListPartsInput = Pick< ListPartsCommandInput, @@ -90,6 +90,7 @@ const deserializeCompletedPartList = (input: any[]): CompletedPart[] => PartNumber: ['PartNumber', deserializeNumber], ETag: 'ETag', Size: ['Size', deserializeNumber], + ChecksumCRC32: 'ChecksumCRC32', }), ); diff --git a/packages/storage/src/providers/s3/utils/client/putObject.ts b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts similarity index 93% rename from packages/storage/src/providers/s3/utils/client/putObject.ts rename to packages/storage/src/providers/s3/utils/client/s3data/putObject.ts index 86755f1c703..3db17f66090 100644 --- a/packages/storage/src/providers/s3/utils/client/putObject.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/putObject.ts @@ -10,18 +10,18 @@ import { import { AmplifyUrl } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import { defaultConfig } from './base'; -import type { PutObjectCommandInput, PutObjectCommandOutput } from './types'; import { assignStringVariables, buildStorageServiceError, map, - parseXmlError, s3TransferHandler, serializeObjectConfigsToHeaders, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { defaultConfig, parseXmlError } from './base'; +import type { PutObjectCommandInput, PutObjectCommandOutput } from './types'; export type PutObjectInput = Pick< PutObjectCommandInput, @@ -37,6 +37,7 @@ export type PutObjectInput = Pick< | 'Expires' | 'Metadata' | 'Tagging' + | 'ChecksumCRC32' >; export type PutObjectOutput = Pick< @@ -56,6 +57,7 @@ const putObjectSerializer = async ( ContentType: input.ContentType ?? 'application/octet-stream', })), ...assignStringVariables({ 'content-md5': input.ContentMD5 }), + ...assignStringVariables({ 'x-amz-checksum-crc32': input.ChecksumCRC32 }), }; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); diff --git a/packages/storage/src/providers/s3/utils/client/types.ts b/packages/storage/src/providers/s3/utils/client/s3data/types.ts similarity index 100% rename from packages/storage/src/providers/s3/utils/client/types.ts rename to packages/storage/src/providers/s3/utils/client/s3data/types.ts diff --git a/packages/storage/src/providers/s3/utils/client/uploadPart.ts b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts similarity index 87% rename from packages/storage/src/providers/s3/utils/client/uploadPart.ts rename to packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts index 3bcacc6236f..2ddea0dd836 100644 --- a/packages/storage/src/providers/s3/utils/client/uploadPart.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/uploadPart.ts @@ -13,23 +13,29 @@ import { } from '@aws-amplify/core/internals/utils'; import { composeServiceApi } from '@aws-amplify/core/internals/aws-client-utils/composers'; -import { defaultConfig } from './base'; -import type { UploadPartCommandInput, UploadPartCommandOutput } from './types'; import { assignStringVariables, buildStorageServiceError, map, - parseXmlError, s3TransferHandler, serializePathnameObjectKey, validateS3RequiredParameter, -} from './utils'; +} from '../utils'; + +import { defaultConfig, parseXmlError } from './base'; +import type { UploadPartCommandInput, UploadPartCommandOutput } from './types'; // Content-length is ignored here because it's forbidden header // and will be set by browser or fetch polyfill. export type UploadPartInput = Pick< UploadPartCommandInput, - 'PartNumber' | 'Body' | 'UploadId' | 'Bucket' | 'Key' | 'ContentMD5' + | 'PartNumber' + | 'Body' + | 'UploadId' + | 'Bucket' + | 'Key' + | 'ContentMD5' + | 'ChecksumCRC32' >; export type UploadPartOutput = Pick< @@ -42,9 +48,10 @@ const uploadPartSerializer = async ( endpoint: Endpoint, ): Promise => { const headers = { + ...assignStringVariables({ 'x-amz-checksum-crc32': input.ChecksumCRC32 }), ...assignStringVariables({ 'content-md5': input.ContentMD5 }), + 'content-type': 'application/octet-stream', }; - headers['content-type'] = 'application/octet-stream'; const url = new AmplifyUrl(endpoint.url.toString()); validateS3RequiredParameter(!!input.Key, 'Key'); url.pathname = serializePathnameObjectKey(url, input.Key); diff --git a/packages/storage/src/providers/s3/utils/client/utils/createRetryDecider.ts b/packages/storage/src/providers/s3/utils/client/utils/createRetryDecider.ts new file mode 100644 index 00000000000..0cfbc0eacde --- /dev/null +++ b/packages/storage/src/providers/s3/utils/client/utils/createRetryDecider.ts @@ -0,0 +1,96 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +import { + ErrorParser, + HttpResponse, + MiddlewareContext, + RetryDeciderOutput, + getRetryDecider, +} from '@aws-amplify/core/internals/aws-client-utils'; + +import { LocationCredentialsProvider } from '../../../types/options'; + +/** + * Function to decide if the S3 request should be retried. For S3 APIs, we support forceRefresh option + * for {@link LocationCredentialsProvider | LocationCredentialsProvider } option. It's set when S3 returns + * credentials expired error. In the retry decider, we detect this response and set flag to signify a retry + * attempt. The retry attempt would invoke the LocationCredentialsProvider with forceRefresh option set. + * + * @param response Optional response of the request. + * @param error Optional error thrown from previous attempts. + * @param middlewareContext Optional context object to store data between retries. + * @returns True if the request should be retried. + */ +export type RetryDecider = ( + response?: HttpResponse, + error?: unknown, + middlewareContext?: MiddlewareContext, +) => Promise; + +/** + * Factory of a {@link RetryDecider} function. + * + * @param errorParser function to parse HTTP response wth XML payload to JS + * Error instance. + * @returns A structure indicating if the response is retryable; And if it is a + * CredentialsExpiredError + */ +export const createRetryDecider = + (errorParser: ErrorParser): RetryDecider => + async ( + response?: HttpResponse, + error?: unknown, + middlewareContext?: MiddlewareContext, + ): Promise => { + const defaultRetryDecider = getRetryDecider(errorParser); + const defaultRetryDecision = await defaultRetryDecider(response, error); + if (!response || response.statusCode < 300) { + return { retryable: false }; + } + const parsedError = await errorParser(response); + const errorCode = parsedError?.name; + const errorMessage = parsedError?.message; + const isCredentialsExpired = isCredentialsExpiredError( + errorCode, + errorMessage, + ); + + return { + retryable: + defaultRetryDecision.retryable || + // If we know the previous retry attempt sets isCredentialsExpired in the + // middleware context, we don't want to retry anymore. + !!(isCredentialsExpired && !middlewareContext?.isCredentialsExpired), + isCredentialsExpiredError: isCredentialsExpired, + }; + }; + +// Ref: https://github.com/aws/aws-sdk-js/blob/54829e341181b41573c419bd870dd0e0f8f10632/lib/event_listeners.js#L522-L541 +const INVALID_TOKEN_ERROR_CODES = [ + 'RequestExpired', + 'ExpiredTokenException', + 'ExpiredToken', +]; + +/** + * Given an error code, returns true if it is related to invalid credentials. + * + * @param errorCode String representation of some error. + * @returns True if given error indicates the credentials used to authorize request + * are invalid. + */ +const isCredentialsExpiredError = ( + errorCode?: string, + errorMessage?: string, +) => { + const isExpiredTokenError = + !!errorCode && INVALID_TOKEN_ERROR_CODES.includes(errorCode); + // Ref: https://github.com/aws/aws-sdk-js/blob/54829e341181b41573c419bd870dd0e0f8f10632/lib/event_listeners.js#L536-L539 + const isExpiredSignatureError = + !!errorCode && + !!errorMessage && + errorCode.includes('Signature') && + errorMessage.includes('expired'); + + return isExpiredTokenError || isExpiredSignatureError; +}; diff --git a/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts b/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts index 0c06cbc60e7..8681d714372 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/deserializeHelpers.ts @@ -104,6 +104,47 @@ export const deserializeTimestamp = (value: string): Date | undefined => { return value ? new Date(value) : undefined; }; +/** + * Create a function deserializing a string to an enum value. If the string is not a valid enum value, it throws a + * StorageError. + * + * @example + * ```typescript + * const deserializeStringEnum = createStringEnumDeserializer(['a', 'b', 'c'] as const, 'FieldName'); + * const deserializedArray = ['a', 'b', 'c'].map(deserializeStringEnum); + * // deserializedArray = ['a', 'b', 'c'] + * + * const invalidValue = deserializeStringEnum('d'); + * // Throws InvalidFieldName: Invalid FieldName: d + * ``` + * + * @internal + */ +export const createStringEnumDeserializer = ( + enumValues: T, + fieldName: string, +) => { + const deserializeStringEnum = ( + value: any, + ): T extends (infer E)[] ? E : never => { + const parsedEnumValue = value + ? (enumValues.find(enumValue => enumValue === value) as any) + : undefined; + if (!parsedEnumValue) { + throw new StorageError({ + name: `Invalid${fieldName}`, + message: `Invalid ${fieldName}: ${value}`, + recoverySuggestion: + 'This is likely to be a bug. Please reach out to library authors.', + }); + } + + return parsedEnumValue; + }; + + return deserializeStringEnum; +}; + /** * Function that makes sure the deserializer receives non-empty array. * diff --git a/packages/storage/src/providers/s3/utils/client/utils/index.ts b/packages/storage/src/providers/s3/utils/client/utils/index.ts index abfe9328d45..77042fafa0c 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/index.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/index.ts @@ -1,7 +1,7 @@ // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -export { parseXmlBody, parseXmlError } from './parsePayload'; +export { parseXmlBody, createXmlErrorParser } from './parsePayload'; export { SEND_DOWNLOAD_PROGRESS_EVENT, SEND_UPLOAD_PROGRESS_EVENT, @@ -25,3 +25,4 @@ export { serializePathnameObjectKey, validateS3RequiredParameter, } from './serializeHelpers'; +export { createRetryDecider } from './createRetryDecider'; diff --git a/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts b/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts index 9da44dcbdd0..f0284d573d2 100644 --- a/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts +++ b/packages/storage/src/providers/s3/utils/client/utils/parsePayload.ts @@ -9,25 +9,43 @@ import { import { parser } from '../runtime'; -export const parseXmlError: ErrorParser = async (response?: HttpResponse) => { - if (!response || response.statusCode < 300) { - return; - } - const { statusCode } = response; - const body = await parseXmlBody(response); - const code = body?.Code - ? (body.Code as string) - : statusCode === 404 - ? 'NotFound' - : statusCode.toString(); - const message = body?.message ?? body?.Message ?? code; - const error = new Error(message); +/** + * Factory creating a parser that parses the JS Error object from the XML + * response payload. + * + * @param input Input object + * @param input.noErrorWrapping Whether the error code and message are located + * directly in the root XML element, or in a nested `` element. + * See: https://smithy.io/2.0/aws/protocols/aws-restxml-protocol.html#restxml-errors + * + * Default to false. + * + * @internal + */ +export const createXmlErrorParser = + ({ + noErrorWrapping = false, + }: { noErrorWrapping?: boolean } = {}): ErrorParser => + async (response?: HttpResponse) => { + if (!response || response.statusCode < 300) { + return; + } + const { statusCode } = response; + const body = await parseXmlBody(response); + const errorLocation = noErrorWrapping ? body : body.Error; + const code = errorLocation?.Code + ? (errorLocation.Code as string) + : statusCode === 404 + ? 'NotFound' + : statusCode.toString(); + const message = errorLocation?.message ?? errorLocation?.Message ?? code; + const error = new Error(message); - return Object.assign(error, { - name: code, - $metadata: parseMetadata(response), - }); -}; + return Object.assign(error, { + name: code, + $metadata: parseMetadata(response), + }); + }; export const parseXmlBody = async (response: HttpResponse): Promise => { if (!response.body) { diff --git a/packages/storage/src/providers/s3/utils/crc32.native.ts b/packages/storage/src/providers/s3/utils/crc32.native.ts new file mode 100644 index 00000000000..389cb5fc87b --- /dev/null +++ b/packages/storage/src/providers/s3/utils/crc32.native.ts @@ -0,0 +1,11 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { CRC32Checksum } from './crc32'; + +export const calculateContentCRC32 = async ( + content: Blob | string | ArrayBuffer | ArrayBufferView, + _seed = 0, +): Promise => { + return undefined; +}; diff --git a/packages/storage/src/providers/s3/utils/crc32.ts b/packages/storage/src/providers/s3/utils/crc32.ts new file mode 100644 index 00000000000..6d9e194c3af --- /dev/null +++ b/packages/storage/src/providers/s3/utils/crc32.ts @@ -0,0 +1,51 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crc32 from 'crc-32'; + +export interface CRC32Checksum { + checksumArrayBuffer: ArrayBuffer; + checksum: string; + seed: number; +} + +export const calculateContentCRC32 = async ( + content: Blob | string | ArrayBuffer | ArrayBufferView, + seed = 0, +): Promise => { + let internalSeed = seed; + let blob: Blob; + + if (content instanceof Blob) { + blob = content; + } else { + blob = new Blob([content]); + } + + await blob.stream().pipeTo( + new WritableStream({ + write(chunk) { + internalSeed = crc32.buf(chunk, internalSeed) >>> 0; + }, + }), + ); + const hex = internalSeed.toString(16).padStart(8, '0'); + + return { + checksumArrayBuffer: hexToArrayBuffer(hex), + checksum: hexToBase64(hex), + seed: internalSeed, + }; +}; + +const hexToArrayBuffer = (hexString: string) => + new Uint8Array((hexString.match(/\w{2}/g)! ?? []).map(h => parseInt(h, 16))) + .buffer; + +const hexToBase64 = (hexString: string) => + btoa( + hexString + .match(/\w{2}/g)! + .map((a: string) => String.fromCharCode(parseInt(a, 16))) + .join(''), + ); diff --git a/packages/storage/src/providers/s3/utils/resolveIdentityId.ts b/packages/storage/src/providers/s3/utils/resolveIdentityId.ts new file mode 100644 index 00000000000..c4831ae88c4 --- /dev/null +++ b/packages/storage/src/providers/s3/utils/resolveIdentityId.ts @@ -0,0 +1,11 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { StorageValidationErrorCode } from '../../../errors/types/validation'; +import { assertValidationError } from '../../../errors/utils/assertValidationError'; + +export const resolveIdentityId = (identityId?: string): string => { + assertValidationError(!!identityId, StorageValidationErrorCode.NoIdentityId); + + return identityId; +}; diff --git a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts index 1e731ec2a12..f19be5ddca7 100644 --- a/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts +++ b/packages/storage/src/providers/s3/utils/resolveS3ConfigAndInput.ts @@ -2,11 +2,25 @@ // SPDX-License-Identifier: Apache-2.0 import { AmplifyClassV6, StorageAccessLevel } from '@aws-amplify/core'; +import { CredentialsProviderOptions } from '@aws-amplify/core/internals/aws-client-utils'; import { assertValidationError } from '../../../errors/utils/assertValidationError'; import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { resolvePrefix as defaultPrefixResolver } from '../../../utils/resolvePrefix'; -import { BucketInfo, ResolvedS3Config, StorageBucket } from '../types/options'; +import { + StorageOperationInputWithKey, + StorageOperationInputWithPath, + StorageOperationInputWithPrefix, +} from '../../../types/inputs'; +import { StorageError } from '../../../errors/StorageError'; +import { CopyInput, CopyWithPathInput } from '../types'; +import { INVALID_STORAGE_INPUT } from '../../../errors/constants'; +import { + BucketInfo, + LocationCredentialsProvider, + ResolvedS3Config, + StorageBucket, +} from '../types/options'; import { DEFAULT_ACCESS_LEVEL, LOCAL_TESTING_S3_ENDPOINT } from './constants'; @@ -14,6 +28,7 @@ interface S3ApiOptions { accessLevel?: StorageAccessLevel; targetIdentityId?: string; useAccelerateEndpoint?: boolean; + locationCredentialsProvider?: LocationCredentialsProvider; bucket?: StorageBucket; } @@ -24,6 +39,16 @@ interface ResolvedS3ConfigAndInput { isObjectLockEnabled?: boolean; identityId?: string; } +export type DeprecatedStorageInput = + | StorageOperationInputWithKey + | StorageOperationInputWithPrefix + | CopyInput; + +export type CallbackPathStorageInput = + | StorageOperationInputWithPath + | CopyWithPathInput; + +type StorageInput = DeprecatedStorageInput | CallbackPathStorageInput; /** * resolve the common input options for S3 API handlers from Amplify configuration and library options. @@ -38,23 +63,35 @@ interface ResolvedS3ConfigAndInput { */ export const resolveS3ConfigAndInput = async ( amplify: AmplifyClassV6, - apiOptions?: S3ApiOptions, + apiInput?: StorageInput & { options?: S3ApiOptions }, ): Promise => { + const { options: apiOptions } = apiInput ?? {}; /** * IdentityId is always cached in memory so we can safely make calls here. It * should be stable even for unauthenticated users, regardless of credentials. */ const { identityId } = await amplify.Auth.fetchAuthSession(); - assertValidationError(!!identityId, StorageValidationErrorCode.NoIdentityId); /** * A credentials provider function instead of a static credentials object is * used because the long-running tasks like multipart upload may span over the * credentials expiry. Auth.fetchAuthSession() automatically refreshes the * credentials if they are expired. + * + * The optional forceRefresh option is set when the S3 service returns expired + * tokens error in the previous API call attempt. */ - const credentialsProvider = async () => { - const { credentials } = await amplify.Auth.fetchAuthSession(); + const credentialsProvider = async (options?: CredentialsProviderOptions) => { + if (isLocationCredentialsProvider(apiOptions)) { + assertStorageInput(apiInput); + } + + // TODO: forceRefresh option of fetchAuthSession would refresh both tokens and + // AWS credentials. So we do not support forceRefreshing from the Auth until + // we support refreshing only the credentials. + const { credentials } = isLocationCredentialsProvider(apiOptions) + ? await apiOptions.locationCredentialsProvider(options) + : await amplify.Auth.fetchAuthSession(); assertValidationError( !!credentials, StorageValidationErrorCode.NoCredentials, @@ -82,15 +119,14 @@ export const resolveS3ConfigAndInput = async ( isObjectLockEnabled, } = amplify.libraryOptions?.Storage?.S3 ?? {}; - const keyPrefix = await prefixResolver({ - accessLevel: - apiOptions?.accessLevel ?? defaultAccessLevel ?? DEFAULT_ACCESS_LEVEL, - // use conditional assign to make tsc happy because StorageOptions is a union type that may not have targetIdentityId - targetIdentityId: - apiOptions?.accessLevel === 'protected' - ? (apiOptions?.targetIdentityId ?? identityId) - : identityId, - }); + const accessLevel = + apiOptions?.accessLevel ?? defaultAccessLevel ?? DEFAULT_ACCESS_LEVEL; + const targetIdentityId = + accessLevel === 'protected' + ? (apiOptions?.targetIdentityId ?? identityId) + : identityId; + + const keyPrefix = await prefixResolver({ accessLevel, targetIdentityId }); return { s3Config: { @@ -111,6 +147,65 @@ export const resolveS3ConfigAndInput = async ( }; }; +const isLocationCredentialsProvider = ( + options?: S3ApiOptions, +): options is S3ApiOptions & { + locationCredentialsProvider: LocationCredentialsProvider; +} => { + return !!options?.locationCredentialsProvider; +}; + +const isInputWithCallbackPath = (input?: CallbackPathStorageInput) => { + return ( + ((input as StorageOperationInputWithPath)?.path && + typeof (input as StorageOperationInputWithPath).path === 'function') || + ((input as CopyWithPathInput)?.destination?.path && + typeof (input as CopyWithPathInput).destination?.path === 'function') || + ((input as CopyWithPathInput)?.source?.path && + typeof (input as CopyWithPathInput).source?.path === 'function') + ); +}; + +const isDeprecatedInput = ( + input?: StorageInput, +): input is DeprecatedStorageInput => { + return ( + isInputWithKey(input) || + isInputWithPrefix(input) || + isInputWithCopySourceOrDestination(input) + ); +}; +const assertStorageInput = (input?: StorageInput) => { + if (isDeprecatedInput(input) || isInputWithCallbackPath(input)) { + throw new StorageError({ + name: INVALID_STORAGE_INPUT, + message: 'The input needs to have a path as a string value.', + recoverySuggestion: + 'Please provide a valid path as a string value for the input.', + }); + } +}; + +const isInputWithKey = ( + input?: StorageInput, +): input is StorageOperationInputWithKey => { + return !!(typeof (input as StorageOperationInputWithKey).key === 'string'); +}; +const isInputWithPrefix = ( + input?: StorageInput, +): input is StorageOperationInputWithPrefix => { + return !!( + typeof (input as StorageOperationInputWithPrefix).prefix === 'string' + ); +}; +const isInputWithCopySourceOrDestination = ( + input?: StorageInput, +): input is CopyInput => { + return !!( + typeof (input as CopyInput).source?.key === 'string' || + typeof (input as CopyInput).destination?.key === 'string' + ); +}; const resolveBucketConfig = ( apiOptions: S3ApiOptions, buckets: Record | undefined, diff --git a/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts b/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts index 585701c81e9..fa423b45913 100644 --- a/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts +++ b/packages/storage/src/providers/s3/utils/validateStorageOperationInput.ts @@ -7,6 +7,7 @@ import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { isInputWithPath } from './isInputWithPath'; import { STORAGE_INPUT_KEY, STORAGE_INPUT_PATH } from './constants'; +import { resolveIdentityId } from './resolveIdentityId'; export const validateStorageOperationInput = ( input: Input, @@ -22,7 +23,10 @@ export const validateStorageOperationInput = ( if (isInputWithPath(input)) { const { path } = input; - const objectKey = typeof path === 'string' ? path : path({ identityId }); + const objectKey = + typeof path === 'string' + ? path + : path({ identityId: resolveIdentityId(identityId) }); assertValidationError( !objectKey.startsWith('/'), diff --git a/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts b/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts index da1068af010..1c2efce19f7 100644 --- a/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts +++ b/packages/storage/src/providers/s3/utils/validateStorageOperationInputWithPrefix.ts @@ -9,6 +9,7 @@ import { assertValidationError } from '../../../errors/utils/assertValidationErr import { StorageValidationErrorCode } from '../../../errors/types/validation'; import { STORAGE_INPUT_PATH, STORAGE_INPUT_PREFIX } from './constants'; +import { resolveIdentityId } from './resolveIdentityId'; // Local assertion function with StorageOperationInputWithPrefixPath as Input const _isInputWithPath = ( @@ -28,7 +29,10 @@ export const validateStorageOperationInputWithPrefix = ( ); if (_isInputWithPath(input)) { const { path } = input; - const objectKey = typeof path === 'string' ? path : path({ identityId }); + const objectKey = + typeof path === 'string' + ? path + : path({ identityId: resolveIdentityId(identityId) }); // Assert on no leading slash in the path parameter assertValidationError( diff --git a/scripts/dts-bundler/dts-bundler.config.js b/scripts/dts-bundler/dts-bundler.config.js index e4ac1a24d61..3851ca452bb 100644 --- a/scripts/dts-bundler/dts-bundler.config.js +++ b/scripts/dts-bundler/dts-bundler.config.js @@ -76,7 +76,7 @@ const config = { }, { filePath: './s3.d.ts', - outFile: join(storagePackageSrcClientsPath, 'client', 'types.ts'), + outFile: join(storagePackageSrcClientsPath, 'client', 's3data', 'types.ts'), libraries: { inlinedLibraries: ['@aws-sdk/client-s3'], }, diff --git a/yarn.lock b/yarn.lock index c2c865e8aad..01d73035280 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5148,6 +5148,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.18.82.tgz#58d734b4acaa5be339864bbec9cd8024dd0b43d5" integrity sha512-pcDZtkx9z8XYV+ius2P3Ot2VVrcYOfXffBQUBuiszrlUzKSmoDYqo+mV+IoL8iIiIjjtOMvNSmH1hwJ+Q+f96Q== +"@types/node@20.14.12": + version "20.14.12" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.12.tgz#129d7c3a822cb49fc7ff661235f19cfefd422b49" + integrity sha512-r7wNXakLeSsGT0H1AU863vS2wa5wBOK4bWMjZz2wj+8nBx+m5PeIn0k8AloSLpRuiwdRQZwarZqHE4FNArPuJQ== + dependencies: + undici-types "~5.26.4" + "@types/node@^18.0.0": version "18.19.55" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.55.tgz#29c3f8e1485a92ec96636957ddec55aabc6e856e" @@ -6961,6 +6968,11 @@ cosmiconfig@^9.0.0: js-yaml "^4.1.0" parse-json "^5.2.0" +crc-32@1.2.2: + version "1.2.2" + resolved "https://registry.yarnpkg.com/crc-32/-/crc-32-1.2.2.tgz#3cad35a934b8bf71f25ca524b6da51fb7eace2ff" + integrity sha512-ROmzCKrTnOwybPcJApAA6WBWij23HVfGVNKqqrZpuyZOHqK2CwHSvpGuyt/UNNvaIjEd8X5IFGp4Mh+Ie1IHJQ== + create-jest@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/create-jest/-/create-jest-29.7.0.tgz#a355c5b3cb1e1af02ba177fe7afd7feee49a5320"