diff --git a/packages/s3-store/IMPLEMENTATION_SUMMARY.md b/packages/s3-store/IMPLEMENTATION_SUMMARY.md new file mode 100644 index 00000000..ff3316fb --- /dev/null +++ b/packages/s3-store/IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,147 @@ +# S3 Object Prefix Implementation - Summary + +## Changes Made + +I've successfully implemented the S3 object prefix feature that allows you to organize uploaded files in folder structures within your S3 bucket. + +### Modified Files + +**`/packages/s3-store/src/index.ts`** + +1. **Added `objectPrefix` option to the `Options` type** (line ~49): + ```typescript + objectPrefix?: string + ``` + - Optional parameter + - Used to create pseudo-directory structures in S3 + - Example: `"uploads/"` or `"my-app/files/2024/"` + +2. **Added `objectPrefix` class property** (line ~109): + ```typescript + protected objectPrefix: string + ``` + +3. **Initialize `objectPrefix` in constructor** (line ~116): + ```typescript + this.objectPrefix = objectPrefix ?? '' + ``` + - Defaults to empty string if not provided (maintains backward compatibility) + +4. **Updated `infoKey()` method** (line ~224): + ```typescript + protected infoKey(id: string) { + return `${this.objectPrefix}${id}.info` + } + ``` + +5. **Updated `partKey()` method** (line ~228): + ```typescript + protected partKey(id: string, isIncomplete = false) { + if (isIncomplete) { + id += '.part' + } + return `${this.objectPrefix}${id}` + } + ``` + +6. **Updated all S3 operations to use the prefix**: + - `uploadPart()` - Uses `this.partKey(metadata.file.id)` + - `create()` - Uses `this.partKey(upload.id)` for Key + - `read()` - Uses `this.partKey(id)` + - `finishMultipartUpload()` - Uses `this.partKey(metadata.file.id)` + - `retrieveParts()` - Uses `this.partKey(id)` + - `remove()` - Uses `this.partKey(id)` and `this.infoKey(id)` + +### Created Files + +**`/packages/s3-store/OBJECT_PREFIX_EXAMPLE.md`** +- Complete usage documentation +- Before/after examples showing folder structures +- Multiple use cases (single folder, nested folders) + +## How It Works + +### Without objectPrefix (default behavior): +``` +my-bucket/ + ├── file-abc123 + ├── file-abc123.info + ├── file-xyz789 + └── file-xyz789.info +``` + +### With objectPrefix: 'uploads/': +``` +my-bucket/ + └── uploads/ + ├── file-abc123 + ├── file-abc123.info + ├── file-xyz789 + └── file-xyz789.info +``` + +### With objectPrefix: 'my-app/uploads/2024/': +``` +my-bucket/ + └── my-app/ + └── uploads/ + └── 2024/ + ├── file-abc123 + ├── file-abc123.info + ├── file-xyz789 + └── file-xyz789.info +``` + +## Usage Example + +```typescript +import {S3Store} from '@tus/s3-store'; + +const store = new S3Store({ + objectPrefix: 'uploads/', // Add this line! + + s3ClientConfig: { + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: 'YOUR_ACCESS_KEY', + secretAccessKey: 'YOUR_SECRET_KEY', + }, + }, +}); + +// Now all uploads will be stored in the 'uploads/' folder +``` + +## Backward Compatibility + +✅ **Fully backward compatible** - If `objectPrefix` is not provided, it defaults to an empty string, maintaining the current behavior of storing files at the bucket root. + +## Testing Recommendations + +1. Test with no prefix (default behavior) +2. Test with single folder: `'uploads/'` +3. Test with nested folders: `'my-app/uploads/2024/'` +4. Test with prefix without trailing slash (should still work) +5. Verify all operations work: + - Upload files + - Resume uploads + - Read files + - Delete files + - List expired files + +## Benefits + +- **Organization**: Keep uploads organized in logical folder structures +- **Multi-tenant**: Separate uploads by user/tenant: `'users/user-123/uploads/'` +- **Time-based**: Organize by date: `'uploads/2024/10/'` +- **Environment separation**: Different folders for dev/staging/prod +- **Easier lifecycle policies**: Apply S3 lifecycle rules to specific prefixes +- **Better billing analysis**: Track storage costs by prefix + +## Notes + +- The prefix applies to all S3 objects: main files, `.info` metadata files, and `.part` incomplete files +- Make sure to include trailing slash for folder-like structures +- S3 doesn't have real folders - prefixes create a "pseudo-directory" structure +- All existing code continues to work without any changes diff --git a/packages/s3-store/OBJECT_PREFIX_EXAMPLE.md b/packages/s3-store/OBJECT_PREFIX_EXAMPLE.md new file mode 100644 index 00000000..8d0c7239 --- /dev/null +++ b/packages/s3-store/OBJECT_PREFIX_EXAMPLE.md @@ -0,0 +1,66 @@ +# S3 Object Prefix Example + +The S3Store now supports an `objectPrefix` option that allows you to organize your uploads in a folder structure within your S3 bucket. + +## Usage + +```typescript +import {S3Store} from '@tus/s3-store'; + +const store = new S3Store({ + // Specify a folder path for your uploads + objectPrefix: 'uploads/', // Files will be stored as: uploads/ + + // Or create nested folders + // objectPrefix: 'my-app/uploads/2024/', // Files: my-app/uploads/2024/ + + s3ClientConfig: { + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: 'your-access-key', + secretAccessKey: 'your-secret-key', + }, + }, +}); +``` + +## Before and After + +### Without `objectPrefix` (default): +``` +my-bucket/ + ├── file-id-1 + ├── file-id-1.info + ├── file-id-2 + └── file-id-2.info +``` + +### With `objectPrefix: 'uploads/'`: +``` +my-bucket/ + └── uploads/ + ├── file-id-1 + ├── file-id-1.info + ├── file-id-2 + └── file-id-2.info +``` + +### With `objectPrefix: 'my-app/uploads/2024/'`: +``` +my-bucket/ + └── my-app/ + └── uploads/ + └── 2024/ + ├── file-id-1 + ├── file-id-1.info + ├── file-id-2 + └── file-id-2.info +``` + +## Notes + +- The prefix is optional - if not provided, files will be stored at the bucket root (current behavior) +- Make sure to include a trailing slash if you want a folder-like structure +- The prefix applies to both the upload file and its `.info` metadata file +- The prefix is also applied to incomplete parts (`.part` files) diff --git a/packages/s3-store/QUICK_REFERENCE.md b/packages/s3-store/QUICK_REFERENCE.md new file mode 100644 index 00000000..fc2dc65b --- /dev/null +++ b/packages/s3-store/QUICK_REFERENCE.md @@ -0,0 +1,130 @@ +# Quick Reference: S3 Object Prefix + +## TL;DR + +Add `objectPrefix` to your S3Store configuration to organize files in folders: + +```typescript +const store = new S3Store({ + objectPrefix: 'uploads/', // ← Add this! + s3ClientConfig: { /* ... */ } +}); +``` + +## Common Use Cases + +### 1. Simple folder organization +```typescript +objectPrefix: 'uploads/' +// Result: bucket/uploads/file-id +``` + +### 2. Multi-tenant application +```typescript +objectPrefix: `tenants/${tenantId}/uploads/` +// Result: bucket/tenants/abc-123/uploads/file-id +``` + +### 3. Date-based organization +```typescript +const date = new Date(); +objectPrefix: `uploads/${date.getFullYear()}/${date.getMonth() + 1}/` +// Result: bucket/uploads/2024/10/file-id +``` + +### 4. Environment separation +```typescript +const env = process.env.NODE_ENV; +objectPrefix: `${env}/uploads/` +// Result: bucket/production/uploads/file-id or bucket/dev/uploads/file-id +``` + +### 5. File type segregation +```typescript +objectPrefix: 'media/images/' +// Result: bucket/media/images/file-id +``` + +### 6. User-specific uploads +```typescript +objectPrefix: `users/${userId}/files/` +// Result: bucket/users/user-456/files/file-id +``` + +## What Gets Prefixed? + +✅ Main upload file: `bucket/prefix/file-id` +✅ Info metadata file: `bucket/prefix/file-id.info` +✅ Incomplete parts: `bucket/prefix/file-id.part` + +## Tips + +- Always use trailing slash for folder-like structure: `'uploads/'` ✅ not `'uploads'` +- Prefix is optional - without it, files go to bucket root (backward compatible) +- S3 doesn't have real folders - prefixes create visual organization +- Use prefixes to apply different lifecycle policies to different file types +- Combine with S3 bucket policies for fine-grained access control + +## S3 Console View + +With `objectPrefix: 'uploads/'`, your S3 console will show: + +``` +📁 my-bucket + └── 📁 uploads + ├── 📄 file-abc123 + ├── 📄 file-abc123.info + ├── 📄 file-xyz789 + └── 📄 file-xyz789.info +``` + +Without prefix: + +``` +📁 my-bucket + ├── 📄 file-abc123 + ├── 📄 file-abc123.info + ├── 📄 file-xyz789 + └── 📄 file-xyz789.info +``` + +## Complete Example + +```typescript +import {Server} from '@tus/server'; +import {S3Store} from '@tus/s3-store'; + +const tusServer = new Server({ + path: '/files', + datastore: new S3Store({ + objectPrefix: 'uploads/', + s3ClientConfig: { + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, + }, + }), +}); +``` + +## Migration from No Prefix + +If you're adding a prefix to an existing setup: + +1. **Option A**: Keep old files where they are, new files use prefix (recommended) +2. **Option B**: Use AWS CLI to move existing files: + ```bash + aws s3 mv s3://my-bucket/ s3://my-bucket/uploads/ --recursive + ``` + +## Benefits + +🎯 **Organization**: Logical folder structure +🔐 **Security**: Apply IAM policies per prefix +💰 **Cost**: Track storage costs by prefix +⏰ **Lifecycle**: Different retention policies per folder +🧹 **Cleanup**: Easy to delete all files in a prefix +📊 **Analytics**: Better insights with S3 analytics diff --git a/packages/s3-store/examples.ts b/packages/s3-store/examples.ts new file mode 100644 index 00000000..45b635cf --- /dev/null +++ b/packages/s3-store/examples.ts @@ -0,0 +1,183 @@ +import {Server} from '@tus/server'; +import {S3Store} from '@tus/s3-store'; +import express from 'express'; + +// Example 1: Basic usage with uploads folder +const basicStore = new S3Store({ + objectPrefix: 'uploads/', + s3ClientConfig: { + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, +}); + +// Example 2: Multi-tenant application +const createTenantStore = (tenantId: string) => { + return new S3Store({ + objectPrefix: `tenants/${tenantId}/uploads/`, + s3ClientConfig: { + bucket: 'my-multi-tenant-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, + }); +}; + +// Example 3: Environment-based organization +const envPrefix = process.env.NODE_ENV === 'production' ? 'prod/' : 'dev/'; +const envStore = new S3Store({ + objectPrefix: `${envPrefix}uploads/`, + s3ClientConfig: { + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, +}); + +// Example 4: Date-based organization +const today = new Date(); +const year = today.getFullYear(); +const month = String(today.getMonth() + 1).padStart(2, '0'); +const dateBasedStore = new S3Store({ + objectPrefix: `uploads/${year}/${month}/`, + s3ClientConfig: { + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, +}); + +// Example 5: Full Express server implementation +const app = express(); + +// Setup tus server with S3 store +const tusServer = new Server({ + path: '/files', + datastore: new S3Store({ + objectPrefix: 'uploads/', + partSize: 8 * 1024 * 1024, // 8MB + s3ClientConfig: { + bucket: process.env.S3_BUCKET!, + region: process.env.AWS_REGION || 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, + }), +}); + +app.all('/files/*', (req, res) => { + tusServer.handle(req, res); +}); + +// Example 6: Per-user uploads with middleware +app.all('/user-uploads/*', (req, res) => { + // Extract user ID from authentication (simplified example) + const userId = req.headers['x-user-id'] as string; + + if (!userId) { + res.status(401).json({error: 'Unauthorized'}); + return; + } + + const userTusServer = new Server({ + path: '/user-uploads', + datastore: new S3Store({ + objectPrefix: `users/${userId}/uploads/`, + s3ClientConfig: { + bucket: process.env.S3_BUCKET!, + region: process.env.AWS_REGION || 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, + }), + }); + + userTusServer.handle(req, res); +}); + +const PORT = process.env.PORT || 3000; +app.listen(PORT, () => { + console.log(`Server running on port ${PORT}`); + console.log(`Upload endpoint: http://localhost:${PORT}/files`); +}); + +// Example 7: Document type segregation +const createDocumentStore = (documentType: 'images' | 'videos' | 'documents') => { + return new S3Store({ + objectPrefix: `media/${documentType}/`, + s3ClientConfig: { + bucket: 'my-media-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, + }); +}; + +// Example 8: Advanced - Lifecycle policy friendly structure +// With this structure, you can set S3 lifecycle policies per prefix +// e.g., delete temp uploads after 7 days, archive old uploads after 90 days +const lifecycleFriendlyStore = new S3Store({ + // Temp uploads can have a lifecycle policy to delete after 7 days + objectPrefix: 'temp-uploads/', + expirationPeriodInMilliseconds: 7 * 24 * 60 * 60 * 1000, // 7 days + s3ClientConfig: { + bucket: 'my-bucket', + region: 'us-east-1', + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID!, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY!, + }, + }, +}); + +/* +Example S3 Lifecycle Policy for the above setup: + +{ + "Rules": [ + { + "Id": "DeleteTempUploads", + "Status": "Enabled", + "Filter": { + "Prefix": "temp-uploads/" + }, + "Expiration": { + "Days": 7 + } + }, + { + "Id": "ArchiveOldUploads", + "Status": "Enabled", + "Filter": { + "Prefix": "uploads/" + }, + "Transitions": [ + { + "Days": 90, + "StorageClass": "GLACIER" + } + ] + } + ] +} +*/ diff --git a/packages/s3-store/src/index.ts b/packages/s3-store/src/index.ts index e371f317..56e54bcb 100644 --- a/packages/s3-store/src/index.ts +++ b/packages/s3-store/src/index.ts @@ -45,6 +45,12 @@ export type Options = { maxConcurrentPartUploads?: number cache?: KvStore expirationPeriodInMilliseconds?: number + /** + * An optional prefix to prepend to all S3 object keys. + * This can be used to create a folder structure in the bucket. + * For example, "uploads/" will store files as "uploads/" + */ + objectPrefix?: string // Options to pass to the AWS S3 SDK. s3ClientConfig: S3ClientConfig & {bucket: string} } @@ -100,13 +106,14 @@ export class S3Store extends DataStore { protected expirationPeriodInMilliseconds = 0 protected useTags = true protected partUploadSemaphore: Semaphore + protected objectPrefix: string public maxMultipartParts = 10_000 public minPartSize = 5_242_880 // 5MiB public maxUploadSize = 5_497_558_138_880 as const // 5TiB constructor(options: Options) { super() - const {maxMultipartParts, partSize, minPartSize, s3ClientConfig} = options + const {maxMultipartParts, partSize, minPartSize, objectPrefix, s3ClientConfig} = options const {bucket, ...restS3ClientConfig} = s3ClientConfig this.extensions = [ 'creation', @@ -116,6 +123,7 @@ export class S3Store extends DataStore { 'expiration', ] this.bucket = bucket + this.objectPrefix = objectPrefix ?? '' this.preferredPartSize = partSize || 8 * 1024 * 1024 if (minPartSize) { this.minPartSize = minPartSize @@ -214,7 +222,7 @@ export class S3Store extends DataStore { } protected infoKey(id: string) { - return `${id}.info` + return `${this.objectPrefix}${id}.info` } protected partKey(id: string, isIncomplete = false) { @@ -222,11 +230,10 @@ export class S3Store extends DataStore { id += '.part' } - // TODO: introduce ObjectPrefixing for parts and incomplete parts. // ObjectPrefix is prepended to the name of each S3 object that is created // to store uploaded files. It can be used to create a pseudo-directory - // structure in the bucket, e.g. "path/to/my/uploads". - return id + // structure in the bucket, e.g. "path/to/my/uploads/". + return `${this.objectPrefix}${id}` } protected async uploadPart( @@ -236,7 +243,7 @@ export class S3Store extends DataStore { ): Promise { const data = await this.client.uploadPart({ Bucket: this.bucket, - Key: metadata.file.id, + Key: this.partKey(metadata.file.id), UploadId: metadata['upload-id'], PartNumber: partNumber, Body: readStream, @@ -459,7 +466,7 @@ export class S3Store extends DataStore { protected async finishMultipartUpload(metadata: MetadataValue, parts: Array) { const response = await this.client.completeMultipartUpload({ Bucket: this.bucket, - Key: metadata.file.id, + Key: this.partKey(metadata.file.id), UploadId: metadata['upload-id'], MultipartUpload: { Parts: parts.map((part) => { @@ -485,7 +492,7 @@ export class S3Store extends DataStore { const params: AWS.ListPartsCommandInput = { Bucket: this.bucket, - Key: id, + Key: this.partKey(id), UploadId: metadata['upload-id'], PartNumberMarker: partNumberMarker, } @@ -549,7 +556,7 @@ export class S3Store extends DataStore { log(`[${upload.id}] initializing multipart upload`) const request: AWS.CreateMultipartUploadCommandInput = { Bucket: this.bucket, - Key: upload.id, + Key: this.partKey(upload.id), Metadata: {'tus-version': TUS_RESUMABLE}, } @@ -578,7 +585,7 @@ export class S3Store extends DataStore { async read(id: string) { const data = await this.client.getObject({ Bucket: this.bucket, - Key: id, + Key: this.partKey(id), }) return data.Body as Readable } @@ -688,7 +695,7 @@ export class S3Store extends DataStore { if (uploadId) { await this.client.abortMultipartUpload({ Bucket: this.bucket, - Key: id, + Key: this.partKey(id), UploadId: uploadId, }) } @@ -703,7 +710,7 @@ export class S3Store extends DataStore { await this.client.deleteObjects({ Bucket: this.bucket, Delete: { - Objects: [{Key: id}, {Key: this.infoKey(id)}], + Objects: [{Key: this.partKey(id)}, {Key: this.infoKey(id)}], }, })