Merge pull request #8300 from dannyzaken/danny-nc-lifecycle

dannyzaken · web-flow · commit 37429af9ccc8 · 2024-08-22T15:45:18.000+03:00
Added lifecycle configuration support in bucketspace_fs
diff --git a/.jsbeautifyrc b/.jsbeautifyrc
@@ -1,10 +1,10 @@
 {
-  "indent_size": 4,
-  "indent_char": " ",
-  "indent_level": 0,
-  "indent_with_tabs": false,
-  "brace_style": "collapse,preserve-inline",
-  "preserve_newlines": true,
-  "max_preserve_newlines": 10,
-  "end_with_newline": true
+    "indent_size": 4,
+    "indent_char": " ",
+    "indent_level": 0,
+    "indent_with_tabs": false,
+    "brace_style": "collapse,preserve-inline",
+    "preserve_newlines": true,
+    "max_preserve_newlines": 10,
+    "end_with_newline": true
 }
diff --git a/docs/NooBaaNonContainerized/S3Ops.md b/docs/NooBaaNonContainerized/S3Ops.md
@@ -25,6 +25,10 @@ The following lists describe the bucket and object operations available in NooBa
 - S3 DeleteBucketPolicy
 - S3 GetBucketPolicy
 - S3 GetBucketPolicyStatus
+- S3 PutBucketLifecycleConfiguration
+- S3 GetBucketLifecycleConfiguration
+- S3 DeleteBucketLifecycle
+
 
 ### Supported S3 Object Operations
 
diff --git a/src/api/bucket_api.js b/src/api/bucket_api.js
@@ -599,12 +599,7 @@ module.exports = {
                 type: 'object',
                 properties: {
                     name: { $ref: 'common_api#/definitions/bucket_name' },
-                    rules: {
-                        type: 'array',
-                        items: {
-                            $ref: 'common_api#/definitions/bucket_lifecycle_rule'
-                        },
-                    }
+                    rules: { $ref: 'common_api#/definitions/bucket_lifecycle_configuration' }
                 }
             },
             auth: {
@@ -620,12 +615,7 @@ module.exports = {
                     name: { $ref: 'common_api#/definitions/bucket_name' },
                 }
             },
-            reply: {
-                type: 'array',
-                items: {
-                    $ref: 'common_api#/definitions/bucket_lifecycle_rule'
-                }
-            },
+            reply: { $ref: 'common_api#/definitions/bucket_lifecycle_configuration' },
             auth: {
                 system: 'admin'
             }
@@ -1385,7 +1375,7 @@ module.exports = {
                                 type: 'object',
                                 properties: {
                                     prefix: { type: 'string' },
-                                    // s3 support also tag or and operator of 2 tags/ tag and prefix 
+                                    // s3 support also tag or and operator of 2 tags/ tag and prefix
                                 }
                             },
                             sync_deletions: { type: 'boolean' },
diff --git a/src/api/common_api.js b/src/api/common_api.js
@@ -195,6 +195,12 @@ module.exports = {
                 }
             }
         },
+        bucket_lifecycle_configuration: {
+            type: 'array',
+            items: {
+                $ref: '#/definitions/bucket_lifecycle_rule'
+            }
+        },
         bucket_lifecycle_rule: {
             type: 'object',
             required: ['id', 'filter', 'expiration', 'status'],
@@ -259,27 +265,26 @@ module.exports = {
 
         bucket_policy_principal: {
             anyOf: [{
-                    wrapper: SensitiveString
-                }, {
-                    type: 'object',
-                    required: ['AWS'],
-                    properties: {
-                        AWS: {
-                            anyOf: [{
-                                    wrapper: SensitiveString
-                                }, {
-                                    type: 'array',
-                                    items: {
-                                        wrapper: SensitiveString
-                                    }
-                                }]
-                        }
+                wrapper: SensitiveString
+            }, {
+                type: 'object',
+                required: ['AWS'],
+                properties: {
+                    AWS: {
+                        anyOf: [{
+                            wrapper: SensitiveString
+                        }, {
+                            type: 'array',
+                            items: {
+                                wrapper: SensitiveString
+                            }
+                        }]
                     }
                 }
-            ]},
+            }]
+        },
         string_or_string_array: {
-            anyOf: [
-                {
+            anyOf: [{
                     type: 'string'
                 },
                 {
@@ -288,7 +293,7 @@ module.exports = {
                         type: 'string'
                     }
                 }
-              ]
+            ]
         },
 
         bucket_policy_string_condition: {
@@ -314,8 +319,7 @@ module.exports = {
                 Statement: {
                     type: 'array',
                     items: {
-                        allOf: [
-                            {
+                        allOf: [{
                                 type: 'object',
                                 required: ['Effect'],
                                 properties: {
@@ -378,8 +382,7 @@ module.exports = {
                             // the schema above. these schemas are only here to validate the mutually exclusive required fields
                             // they should be replaced once we support allOf for individual fields
                             {
-                                oneOf: [
-                                    {
+                                oneOf: [{
                                         type: 'object',
                                         required: ["Principal"],
                                         additionalProperties: true,
@@ -394,8 +397,7 @@ module.exports = {
                                 ],
                             },
                             {
-                                oneOf: [
-                                    {
+                                oneOf: [{
                                         type: 'object',
                                         required: ["Action"],
                                         additionalProperties: true,
@@ -407,11 +409,10 @@ module.exports = {
                                         additionalProperties: true,
                                         properties: {}
                                     }
-                                    ],
+                                ],
                             },
                             {
-                                oneOf: [
-                                    {
+                                oneOf: [{
                                         type: 'object',
                                         required: ["Resource"],
                                         additionalProperties: true,
@@ -1270,7 +1271,7 @@ module.exports = {
             }
         },
 
-        // nsfs 
+        // nsfs
 
         fs_backend: {
             type: 'string',
@@ -1301,7 +1302,7 @@ module.exports = {
                 }
             }, {
                 type: 'object',
-                required: [ 'distinguished_name', 'new_buckets_path', 'nsfs_only'],
+                required: ['distinguished_name', 'new_buckets_path', 'nsfs_only'],
                 properties: {
                     distinguished_name: { wrapper: SensitiveString },
                     new_buckets_path: { type: 'string' },
diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js
@@ -19,19 +19,25 @@ const { ConfigFS, JSON_SUFFIX, CONFIG_TYPES } = require('./config_fs');
 const mongo_utils = require('../util/mongo_utils');
 
 const KeysSemaphore = require('../util/keys_semaphore');
-const { get_umasked_mode, validate_bucket_creation, get_bucket_tmpdir_full_path, folder_delete,
-    entity_enum, translate_error_codes, get_process_fs_context} = require('../util/native_fs_utils');
+const {
+    get_umasked_mode,
+    validate_bucket_creation,
+    get_bucket_tmpdir_full_path,
+    folder_delete,
+    entity_enum,
+    translate_error_codes,
+    get_process_fs_context
+} = require('../util/native_fs_utils');
 const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
 const { anonymous_access_key } = require('./object_sdk');
 
 const dbg = require('../util/debug_module')(__filename);
 const bucket_semaphore = new KeysSemaphore(1);
 
 
-
 class BucketSpaceFS extends BucketSpaceSimpleFS {
-    constructor({config_root}, stats) {
-        super({ fs_root: ''});
+    constructor({ config_root }, stats) {
+        super({ fs_root: '' });
         this.config_root = config_root;
         this.stats = stats;
         this.fs_context = get_process_fs_context(
@@ -85,7 +91,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
                 throw new RpcError('NO_SUCH_ACCOUNT', `Account with access_key not found.`, err);
             }
             //account access failed
-            new NoobaaEvent(NoobaaEvent.ACCOUNT_NOT_FOUND).create_event(access_key, {access_key: access_key}, err);
+            new NoobaaEvent(NoobaaEvent.ACCOUNT_NOT_FOUND).create_event(access_key, { access_key: access_key }, err);
             throw new RpcError('NO_SUCH_ACCOUNT', err.message);
         }
     }
@@ -154,7 +160,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
         } catch (err) {
             const rpc_error = translate_error_codes(err, entity_enum.BUCKET);
             if (err.rpc_code === 'INVALID_SCHEMA') err.rpc_code = 'INVALID_BUCKET_STATE';
-            new NoobaaEvent(NoobaaEvent[rpc_error.rpc_code]).create_event(name, {bucket_name: name}, err);
+            new NoobaaEvent(NoobaaEvent[rpc_error.rpc_code]).create_event(name, { bucket_name: name }, err);
             throw rpc_error;
         }
     }
@@ -237,7 +243,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             }
             // currently we do not allow IAM account to create a bucket (temporary)
             if (sdk.requesting_account.owner !== undefined) {
-            dbg.warn('create_bucket: account is IAM account (currently not allowed to create buckets)');
+                dbg.warn('create_bucket: account is IAM account (currently not allowed to create buckets)');
                 throw new RpcError('UNAUTHORIZED', 'Not allowed to create new buckets');
             }
             if (!sdk.requesting_account.nsfs_account_config || !sdk.requesting_account.nsfs_account_config.new_buckets_path) {
@@ -271,17 +277,18 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
                 nsfs_schema_utils.validate_bucket_schema(bucket_to_validate);
                 await this.config_fs.create_bucket_config_file(name, bucket_config);
             } catch (err) {
-                new NoobaaEvent(NoobaaEvent.BUCKET_CREATION_FAILED).create_event(name, {bucket_name: name}, err);
+                new NoobaaEvent(NoobaaEvent.BUCKET_CREATION_FAILED).create_event(name, { bucket_name: name }, err);
                 throw translate_error_codes(err, entity_enum.BUCKET);
             }
 
             // create bucket's underlying storage directory
             try {
                 await nb_native().fs.mkdir(fs_context, bucket_storage_path, get_umasked_mode(config.BASE_MODE_DIR));
-                new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {bucket_name: name});
+                new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, { bucket_name: name });
             } catch (err) {
                 dbg.error('BucketSpaceFS: create_bucket could not create underlying directory - nsfs, deleting bucket', err);
-                new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED).create_event(name, {bucket: name, path: bucket_storage_path}, err);
+                new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
+                    .create_event(name, { bucket: name, path: bucket_storage_path }, err);
                 await nb_native().fs.unlink(this.fs_context, bucket_config_path);
                 throw translate_error_codes(err, entity_enum.BUCKET);
             }
@@ -358,8 +365,8 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
                 new NoobaaEvent(NoobaaEvent.BUCKET_DELETE).create_event(name, { bucket_name: name });
             } catch (err) {
                 dbg.error('BucketSpaceFS: delete_bucket: bucket name', name, 'error', err);
-                new NoobaaEvent(NoobaaEvent.BUCKET_DELETE_FAILED).create_event(name,
-                    { bucket_name: name, bucket_path: bucket_config_path }, err);
+                new NoobaaEvent(NoobaaEvent.BUCKET_DELETE_FAILED)
+                    .create_event(name, { bucket_name: name, bucket_path: bucket_config_path }, err);
                 throw translate_error_codes(err, entity_enum.BUCKET);
             }
         });
@@ -371,15 +378,41 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
     //////////////////////
 
     async get_bucket_lifecycle_configuration_rules(params) {
-        // TODO
+        try {
+            const { name } = params;
+            dbg.log0('BucketSpaceFS.get_bucket_lifecycle_configuration_rules: Bucket name', name);
+            const bucket = await this.config_fs.get_bucket_by_name(name);
+            return bucket.lifecycle_configuration_rules || [];
+        } catch (error) {
+            throw translate_error_codes(error, entity_enum.BUCKET);
+        }
     }
 
     async set_bucket_lifecycle_configuration_rules(params) {
-        // TODO
+        try {
+            const { name, rules } = params;
+            dbg.log0('BucketSpaceFS.set_bucket_lifecycle_configuration_rules: Bucket name, rules', name, rules);
+            const bucket = await this.config_fs.get_bucket_by_name(name);
+            bucket.lifecycle_configuration_rules = rules;
+            nsfs_schema_utils.validate_bucket_schema(bucket);
+            const update_bucket = JSON.stringify(bucket);
+            await this.config_fs.update_bucket_config_file(name, update_bucket);
+        } catch (err) {
+            throw translate_error_codes(err, entity_enum.BUCKET);
+        }
     }
 
     async delete_bucket_lifecycle(params) {
-        // TODO
+        try {
+            const { name } = params;
+            dbg.log0('BucketSpaceFS.delete_bucket_lifecycle: Bucket name', name);
+            const bucket = await this.config_fs.get_bucket_by_name(name);
+            delete bucket.lifecycle_configuration_rules;
+            nsfs_schema_utils.validate_bucket_schema(bucket);
+            await this.config_fs.update_bucket_config_file(name, JSON.stringify(bucket));
+        } catch (error) {
+            throw translate_error_codes(error, entity_enum.BUCKET);
+        }
     }
 
     ///////////////////////
@@ -517,7 +550,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             // in case it is algorithm: 'AES256', the property would be undefined
             const bucket_to_validate = _.omitBy(bucket, _.isUndefined);
             dbg.log2("put_bucket_encryption: bucket properties before validate_bucket_schema",
-            bucket_to_validate);
+                bucket_to_validate);
             nsfs_schema_utils.validate_bucket_schema(bucket_to_validate);
             const update_bucket = JSON.stringify(bucket);
             await this.config_fs.update_bucket_config_file(name, update_bucket);
@@ -565,7 +598,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             bucket.website = website;
             const bucket_to_validate = _.omitBy(bucket, _.isUndefined);
             dbg.log2("put_bucket_website: bucket properties before validate_bucket_schema",
-            bucket_to_validate);
+                bucket_to_validate);
             nsfs_schema_utils.validate_bucket_schema(bucket_to_validate);
             const update_bucket = JSON.stringify(bucket);
             await this.config_fs.update_bucket_config_file(name, update_bucket);
@@ -591,9 +624,9 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
     }
 
     /**
-    * @param {object} params
-    * @returns {Promise<object>}
-    */
+     * @param {object} params
+     * @returns {Promise<object>}
+     */
     async get_bucket_website(params) {
         try {
             const { name } = params;
@@ -620,7 +653,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             dbg.log2('put_bucket_policy: bucket properties before validate_bucket_schema', bucket_to_validate);
             nsfs_schema_utils.validate_bucket_schema(bucket_to_validate);
             await bucket_policy_utils.validate_s3_policy(bucket.s3_policy, bucket.name, async principal =>
-                 this.config_fs.get_account_by_name(principal, { silent_if_missing: true }));
+                this.config_fs.get_account_by_name(principal, { silent_if_missing: true }));
             const update_bucket = JSON.stringify(bucket);
             await this.config_fs.update_bucket_config_file(name, update_bucket);
         } catch (err) {
@@ -686,10 +719,10 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
         const bucket_policy = bucket.s3_policy;
 
         if (!bucket_policy) {
-        // in case we do not have bucket policy
-        // we allow IAM account to access a bucket that that is owned by their root account
-        const is_iam_and_same_root_account_owner = account.owner !== undefined &&
-            account.owner === bucket.owner_account.id;
+            // in case we do not have bucket policy
+            // we allow IAM account to access a bucket that that is owned by their root account
+            const is_iam_and_same_root_account_owner = account.owner !== undefined &&
+                account.owner === bucket.owner_account.id;
             return is_owner || is_iam_and_same_root_account_owner;
         }
         if (!action) {
diff --git a/src/server/system_services/schemas/bucket_schema.js b/src/server/system_services/schemas/bucket_schema.js
@@ -201,10 +201,7 @@ module.exports = {
 
         //lifecycle rules if exist
         lifecycle_configuration_rules: {
-            type: 'array',
-            items: {
-                $ref: 'common_api#/definitions/bucket_lifecycle_rule'
-            }
+            $ref: 'common_api#/definitions/bucket_lifecycle_configuration'
         },
         tagging: {
             $ref: 'common_api#/definitions/tagging',
diff --git a/src/server/system_services/schemas/nsfs_bucket_schema.js b/src/server/system_services/schemas/nsfs_bucket_schema.js
diff --git a/src/test/unit_tests/test_bucketspace_fs.js b/src/test/unit_tests/test_bucketspace_fs.js
