feat(storage-s3): presigned URLs for file downloads (#12307)

Adds pre-signed URLs support file downloads with the S3 adapter. Can be
enabled per-collection:
```ts
s3Storage({
  collections: {
    media: { signedDownloads: true }, // or { signedDownloads: { expiresIn: 3600 }} for custom expiresIn (default 7200)
  },
  bucket: process.env.S3_BUCKET,
  config: {
    credentials: {
      accessKeyId: process.env.S3_ACCESS_KEY_ID,
      secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,
    },
    endpoint: process.env.S3_ENDPOINT,
    forcePathStyle: process.env.S3_FORCE_PATH_STYLE === 'true',
    region: process.env.S3_REGION,
  },
}),
```

The main use case is when you care about the Payload access control (so
you don't want to use `disablePayloadAccessControl: true` but you don't
want your files to be served through Payload (which can affect
performance with large videos for example).
This feature instead generates a signed URL (after verifying the access
control) and redirects you directly to the S3 provider.

This is an addition to #11382
which added pre-signed URLs for file uploads.

Author: r1tsuu

docs/upload/storage-adapters.mdx

@@ -84,6 +84,7 @@ pnpm add @payloadcms/storage-s3
 - The `config` object can be any [`S3ClientConfig`](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3) object (from [`@aws-sdk/client-s3`](https://github.com/aws/aws-sdk-js-v3)). _This is highly dependent on your AWS setup_. Check the AWS documentation for more information.
 - When enabled, this package will automatically set `disableLocalStorage` to `true` for each collection.
 - When deploying to Vercel, server uploads are limited with 4.5MB. Set `clientUploads` to `true` to do uploads directly on the client. You must allow CORS PUT method for the bucket to your website.
+- Configure `signedDownloads` (either globally of per-collection in `collections`) to use [presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html) for files downloading. This can improve performance for large files (like videos) while still respecting your access control.
 
 ```ts
 import { s3Storage } from '@payloadcms/storage-s3'

packages/storage-s3/src/index.ts

@@ -12,6 +12,8 @@ import * as AWS from '@aws-sdk/client-s3'
 import { cloudStoragePlugin } from '@payloadcms/plugin-cloud-storage'
 import { initClientUploads } from '@payloadcms/plugin-cloud-storage/utilities'
 
+import type { SignedDownloadsConfig } from './staticHandler.js'
+
 import { getGenerateSignedURLHandler } from './generateSignedURL.js'
 import { getGenerateURL } from './generateURL.js'
 import { getHandleDelete } from './handleDelete.js'
@@ -24,6 +26,7 @@ export type S3StorageOptions = {
    */
 
   acl?: 'private' | 'public-read'
+
   /**
    * Bucket name to upload files to.
    *
@@ -39,8 +42,15 @@ export type S3StorageOptions = {
   /**
    * Collection options to apply the S3 adapter to.
    */
-  collections: Partial<Record<UploadCollectionSlug, Omit<CollectionOptions, 'adapter'> | true>>
-
+  collections: Partial<
+    Record<
+      UploadCollectionSlug,
+      | ({
+          signedDownloads?: SignedDownloadsConfig
+        } & Omit<CollectionOptions, 'adapter'>)
+      | true
+    >
+  >
   /**
    * AWS S3 client configuration. Highly dependent on your AWS setup.
    *
@@ -61,6 +71,10 @@ export type S3StorageOptions = {
    * Default: true
    */
   enabled?: boolean
+  /**
+   * Use pre-signed URLs for files downloading. Can be overriden per-collection.
+   */
+  signedDownloads?: SignedDownloadsConfig
 }
 
 type S3StoragePlugin = (storageS3Args: S3StorageOptions) => Plugin
@@ -158,9 +172,27 @@ export const s3Storage: S3StoragePlugin =
 
 function s3StorageInternal(
   getStorageClient: () => AWS.S3,
-  { acl, bucket, clientUploads, config = {} }: S3StorageOptions,
+  {
+    acl,
+    bucket,
+    clientUploads,
+    collections,
+    config = {},
+    signedDownloads: topLevelSignedDownloads,
+  }: S3StorageOptions,
 ): Adapter {
   return ({ collection, prefix }): GeneratedAdapter => {
+    const collectionStorageConfig = collections[collection.slug]
+
+    let signedDownloads: null | SignedDownloadsConfig =
+      typeof collectionStorageConfig === 'object'
+        ? (collectionStorageConfig.signedDownloads ?? false)
+        : null
+
+    if (signedDownloads === null) {
+      signedDownloads = topLevelSignedDownloads ?? null
+    }
+
     return {
       name: 's3',
       clientUploads,
@@ -173,7 +205,12 @@ function s3StorageInternal(
         getStorageClient,
         prefix,
       }),
-      staticHandler: getHandler({ bucket, collection, getStorageClient }),
+      staticHandler: getHandler({
+        bucket,
+        collection,
+        getStorageClient,
+        signedDownloads: signedDownloads ?? false,
+      }),
     }
   }
 }

packages/storage-s3/src/staticHandler.ts

@@ -3,13 +3,23 @@ import type { StaticHandler } from '@payloadcms/plugin-cloud-storage/types'
 import type { CollectionConfig } from 'payload'
 import type { Readable } from 'stream'
 
+import { GetObjectCommand } from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
 import { getFilePrefix } from '@payloadcms/plugin-cloud-storage/utilities'
 import path from 'path'
 
+export type SignedDownloadsConfig =
+  | {
+      /** @default 7200 */
+      expiresIn?: number
+    }
+  | boolean
+
 interface Args {
   bucket: string
   collection: CollectionConfig
   getStorageClient: () => AWS.S3
+  signedDownloads?: SignedDownloadsConfig
 }
 
 // Type guard for NodeJS.Readable streams
@@ -40,14 +50,30 @@ const streamToBuffer = async (readableStream: any) => {
   return Buffer.concat(chunks)
 }
 
-export const getHandler = ({ bucket, collection, getStorageClient }: Args): StaticHandler => {
+export const getHandler = ({
+  bucket,
+  collection,
+  getStorageClient,
+  signedDownloads,
+}: Args): StaticHandler => {
   return async (req, { params: { clientUploadContext, filename } }) => {
     let object: AWS.GetObjectOutput | undefined = undefined
     try {
       const prefix = await getFilePrefix({ clientUploadContext, collection, filename, req })
 
       const key = path.posix.join(prefix, filename)
 
+      if (signedDownloads && !clientUploadContext) {
+        const command = new GetObjectCommand({ Bucket: bucket, Key: key })
+        const signedUrl = await getSignedUrl(
+          // @ts-expect-error mismatch versions
+          getStorageClient(),
+          command,
+          typeof signedDownloads === 'object' ? signedDownloads : { expiresIn: 7200 },
+        )
+        return Response.redirect(signedUrl)
+      }
+
       object = await getStorageClient().getObject({
         Bucket: bucket,
         Key: key,

test/storage-s3/collections/MediaWithSignedDownloads.ts

@@ -0,0 +1,9 @@
+import type { CollectionConfig } from 'payload'
+
+import { mediaWithSignedDownloadsSlug } from '../shared.js'
+
+export const MediaWithSignedDownloads: CollectionConfig = {
+  slug: mediaWithSignedDownloadsSlug,
+  upload: true,
+  fields: [],
+}

test/storage-s3/config.ts

@@ -7,8 +7,9 @@ import { buildConfigWithDefaults } from '../buildConfigWithDefaults.js'
 import { devUser } from '../credentials.js'
 import { Media } from './collections/Media.js'
 import { MediaWithPrefix } from './collections/MediaWithPrefix.js'
+import { MediaWithSignedDownloads } from './collections/MediaWithSignedDownloads.js'
 import { Users } from './collections/Users.js'
-import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js'
+import { mediaSlug, mediaWithPrefixSlug, mediaWithSignedDownloadsSlug, prefix } from './shared.js'
 const filename = fileURLToPath(import.meta.url)
 const dirname = path.dirname(filename)
 
@@ -25,7 +26,7 @@ export default buildConfigWithDefaults({
       baseDir: path.resolve(dirname),
     },
   },
-  collections: [Media, MediaWithPrefix, Users],
+  collections: [Media, MediaWithPrefix, MediaWithSignedDownloads, Users],
   onInit: async (payload) => {
     await payload.create({
       collection: 'users',
@@ -42,6 +43,9 @@ export default buildConfigWithDefaults({
         [mediaWithPrefixSlug]: {
           prefix,
         },
+        [mediaWithSignedDownloadsSlug]: {
+          signedDownloads: true,
+        },
       },
       bucket: process.env.S3_BUCKET,
       config: {

test/storage-s3/int.spec.ts

@@ -4,20 +4,24 @@ import * as AWS from '@aws-sdk/client-s3'
 import path from 'path'
 import { fileURLToPath } from 'url'
 
+import type { NextRESTClient } from '../helpers/NextRESTClient.js'
+
 import { initPayloadInt } from '../helpers/initPayloadInt.js'
-import { mediaSlug, mediaWithPrefixSlug, prefix } from './shared.js'
+import { mediaSlug, mediaWithPrefixSlug, mediaWithSignedDownloadsSlug, prefix } from './shared.js'
 
 const filename = fileURLToPath(import.meta.url)
 const dirname = path.dirname(filename)
 
+let restClient: NextRESTClient
+
 let payload: Payload
 
 describe('@payloadcms/storage-s3', () => {
   let TEST_BUCKET: string
   let client: AWS.S3Client
 
   beforeAll(async () => {
-    ;({ payload } = await initPayloadInt(dirname))
+    ;({ payload, restClient } = await initPayloadInt(dirname))
     TEST_BUCKET = process.env.S3_BUCKET
 
     client = new AWS.S3({
@@ -77,15 +81,38 @@ describe('@payloadcms/storage-s3', () => {
     expect(upload.url).toEqual(`/api/${mediaWithPrefixSlug}/file/${String(upload.filename)}`)
   })
 
+  it('can download with signed downloads', async () => {
+    await payload.create({
+      collection: mediaWithSignedDownloadsSlug,
+      data: {},
+      filePath: path.resolve(dirname, '../uploads/image.png'),
+    })
+
+    const response = await restClient.GET(`/${mediaWithSignedDownloadsSlug}/file/image.png`)
+    expect(response.status).toBe(302)
+    const url = response.headers.get('Location')
+    expect(url).toBeDefined()
+    expect(url!).toContain(`/${TEST_BUCKET}/image.png`)
+    expect(new URLSearchParams(url!).get('x-id')).toBe('GetObject')
+    const file = await fetch(url!)
+    expect(file.headers.get('Content-Type')).toBe('image/png')
+  })
+
   describe('R2', () => {
     it.todo('can upload')
   })
 
   async function createTestBucket() {
-    const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET }))
+    try {
+      const makeBucketRes = await client.send(new AWS.CreateBucketCommand({ Bucket: TEST_BUCKET }))
 
-    if (makeBucketRes.$metadata.httpStatusCode !== 200) {
-      throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`)
+      if (makeBucketRes.$metadata.httpStatusCode !== 200) {
+        throw new Error(`Failed to create bucket. ${makeBucketRes.$metadata.httpStatusCode}`)
+      }
+    } catch (e) {
+      if (e instanceof AWS.BucketAlreadyOwnedByYou) {
+        console.log('Bucket already exists')
+      }
     }
   }
 
@@ -96,7 +123,9 @@ describe('@payloadcms/storage-s3', () => {
       }),
     )
 
-    if (!listedObjects?.Contents?.length) return
+    if (!listedObjects?.Contents?.length) {
+      return
+    }
 
     const deleteParams = {
       Bucket: TEST_BUCKET,

test/storage-s3/payload-types.ts

@@ -69,6 +69,7 @@ export interface Config {
   collections: {
     media: Media;
     'media-with-prefix': MediaWithPrefix;
+    'media-with-signed-downloads': MediaWithSignedDownload;
     users: User;
     'payload-locked-documents': PayloadLockedDocument;
     'payload-preferences': PayloadPreference;
@@ -78,6 +79,7 @@ export interface Config {
   collectionsSelect: {
     media: MediaSelect<false> | MediaSelect<true>;
     'media-with-prefix': MediaWithPrefixSelect<false> | MediaWithPrefixSelect<true>;
+    'media-with-signed-downloads': MediaWithSignedDownloadsSelect<false> | MediaWithSignedDownloadsSelect<true>;
     users: UsersSelect<false> | UsersSelect<true>;
     'payload-locked-documents': PayloadLockedDocumentsSelect<false> | PayloadLockedDocumentsSelect<true>;
     'payload-preferences': PayloadPreferencesSelect<false> | PayloadPreferencesSelect<true>;
@@ -171,6 +173,24 @@ export interface MediaWithPrefix {
   focalX?: number | null;
   focalY?: number | null;
 }
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "media-with-signed-downloads".
+ */
+export interface MediaWithSignedDownload {
+  id: string;
+  updatedAt: string;
+  createdAt: string;
+  url?: string | null;
+  thumbnailURL?: string | null;
+  filename?: string | null;
+  mimeType?: string | null;
+  filesize?: number | null;
+  width?: number | null;
+  height?: number | null;
+  focalX?: number | null;
+  focalY?: number | null;
+}
 /**
  * This interface was referenced by `Config`'s JSON-Schema
  * via the `definition` "users".
@@ -203,6 +223,10 @@ export interface PayloadLockedDocument {
         relationTo: 'media-with-prefix';
         value: string | MediaWithPrefix;
       } | null)
+    | ({
+        relationTo: 'media-with-signed-downloads';
+        value: string | MediaWithSignedDownload;
+      } | null)
     | ({
         relationTo: 'users';
         value: string | User;
@@ -309,6 +333,23 @@ export interface MediaWithPrefixSelect<T extends boolean = true> {
   focalX?: T;
   focalY?: T;
 }
+/**
+ * This interface was referenced by `Config`'s JSON-Schema
+ * via the `definition` "media-with-signed-downloads_select".
+ */
+export interface MediaWithSignedDownloadsSelect<T extends boolean = true> {
+  updatedAt?: T;
+  createdAt?: T;
+  url?: T;
+  thumbnailURL?: T;
+  filename?: T;
+  mimeType?: T;
+  filesize?: T;
+  width?: T;
+  height?: T;
+  focalX?: T;
+  focalY?: T;
+}
 /**
  * This interface was referenced by `Config`'s JSON-Schema
  * via the `definition` "users_select".

test/storage-s3/shared.ts

@@ -1,3 +1,5 @@
 export const mediaSlug = 'media'
 export const mediaWithPrefixSlug = 'media-with-prefix'
 export const prefix = 'test-prefix'
+
+export const mediaWithSignedDownloadsSlug = 'media-with-signed-downloads'