Merge pull request #9043 from tangledbytes/utkarsh/feat/cli/add-support-for-uls-creation

tangledbytes · web-flow · commit 05da04aafdea · 2025-05-28T19:11:45.000+05:30
[NSFS] Add support for uls deletion to NooBaa CLI
diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js
@@ -149,6 +149,11 @@ async function fetch_bucket_data(action, user_input) {
             data.s3_policy = user_input.bucket_policy;
         }
     }
+    if (action === ACTIONS.ADD || action === ACTIONS.UPDATE) {
+        if (user_input.should_create_underlying_storage !== undefined) {
+            data.should_create_underlying_storage = Boolean(user_input.should_create_underlying_storage);
+        }
+    }
     if (action === ACTIONS.UPDATE || action === ACTIONS.DELETE) {
         // @ts-ignore
         data = _.omitBy(data, _.isUndefined);
@@ -221,9 +226,20 @@ async function merge_new_and_existing_config_data(user_input_bucket_data) {
 async function add_bucket(data) {
     data._id = mongo_utils.mongoObjectId();
     const parsed_bucket_data = await config_fs.create_bucket_config_file(data);
-    await set_bucker_owner(parsed_bucket_data);
+
+    const account = await account_id_cache.get_with_cache({ _id: parsed_bucket_data.owner_account, config_fs });
+    await set_bucker_owner(parsed_bucket_data, account);
 
     const [reserved_tag_event_args] = BucketSpaceFS._generate_reserved_tag_event_args({}, data.tag);
+    if (parsed_bucket_data.should_create_underlying_storage) {
+        await BucketSpaceFS._create_uls(
+            config_fs.fs_context,
+            await native_fs_utils.get_fs_context(account.nsfs_account_config, parsed_bucket_data.fs_backend),
+            data.name,
+            data.path,
+            config_fs.get_bucket_path_by_name(data.name)
+        );
+    }
 
     return {
         code: ManageCLIResponse.BucketCreated,
@@ -295,6 +311,13 @@ async function delete_bucket(data, force) {
 
         await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
         await config_fs.delete_bucket_config_file(data.name);
+        if (data.should_create_underlying_storage) {
+            try {
+                await nb_native().fs.rmdir(config_fs.fs_context, data.path);
+            } catch (error) {
+                dbg.warn('failed to delete underlying storage directory:', error);
+            }
+        }
         return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
     } catch (err) {
         if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name);
@@ -784,11 +807,13 @@ function get_access_keys(action, user_input) {
 /**
  * set_bucker_owner gets bucket owner from cache by its id and sets bucket_owner name on the bucket data
  * @param {object} bucket_data 
+ * @param {*} [account_data]
  */
-async function set_bucker_owner(bucket_data) {
-    let account_data;
+async function set_bucker_owner(bucket_data, account_data) {
     try {
-        account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
+        if (!account_data) {
+            account_data = await account_id_cache.get_with_cache({ _id: bucket_data.owner_account, config_fs });
+        }
     } catch (err) {
         dbg.warn(`set_bucker_owner.couldn't find bucket owner data by id ${bucket_data.owner_account}`);
     }
diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js
@@ -61,8 +61,8 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = {
 };
 
 const VALID_OPTIONS_BUCKET = {
-    'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
-    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]),
+    'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', 'should_create_underlying_storage', FROM_FILE, ...CLI_MUTUAL_OPTIONS]),
+    'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', 'should_create_underlying_storage', ...CLI_MUTUAL_OPTIONS]),
     'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]),
     'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]),
     'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]),
@@ -143,6 +143,7 @@ const OPTION_TYPE = {
     force: 'boolean',
     anonymous: 'boolean',
     default_connection: 'string',
+    should_create_underlying_storage: 'boolean',
     // health options
     deployment_type: 'string',
     all_account_details: 'boolean',
diff --git a/src/manage_nsfs/manage_nsfs_help_utils.js b/src/manage_nsfs/manage_nsfs_help_utils.js
@@ -243,7 +243,7 @@ Flags:
     --fs_backend <none | GPFS | CEPH_FS | NFSv4>          (optional)        Set the filesystem type (default config.NSFS_NC_STORAGE_BACKEND)
     --force_md5_etag <true | false>                       (optional)        Set the bucket to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
     --from_file <string>                                  (optional)        Use details from the JSON file, there is no need to mention all the properties individually in the CLI
-
+    --should_create_underlying_storage <true | false>     (optional)        Force creation of bucket's underlying storage directory
 `;
 
 const BUCKET_FLAGS_UPDATE = `
@@ -264,6 +264,7 @@ Flags:
     --bucket_policy <string>                              (optional)        Update the bucket policy, type is a string of valid JSON policy (unset with '')
     --fs_backend <none | GPFS | CEPH_FS | NFSv4>          (optional)        Update the filesystem type (unset with '') (default config.NSFS_NC_STORAGE_BACKEND)
     --force_md5_etag <true | false>                       (optional)        Update the bucket to force md5 etag calculation (unset with '') (will override default config.NSFS_NC_STORAGE_BACKEND)
+    --should_create_underlying_storage <true | false>     (optional)        Update the bucket to manage the underlying storage
 
 `;
 
diff --git a/src/manage_nsfs/manage_nsfs_validations.js b/src/manage_nsfs/manage_nsfs_validations.js
@@ -455,7 +455,7 @@ async function validate_bucket_args(config_fs, data, action) {
         await check_new_name_exists(TYPES.BUCKET, config_fs, action, data);
         // in case we have the fs_backend it changes the fs_context that we use for the path
         const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
-        if (!config.NC_DISABLE_ACCESS_CHECK) {
+        if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
             const exists = await native_fs_utils.is_path_exists(fs_context_fs_backend, data.path);
             if (!exists) {
                 throw_cli_error(ManageCLIError.InvalidStoragePath, data.path);
@@ -467,7 +467,7 @@ async function validate_bucket_args(config_fs, data, action) {
 
         const account_fs_context = await native_fs_utils.get_fs_context(owner_account_data.nsfs_account_config,
             owner_account_data.nsfs_account_config.fs_backend);
-        if (!config.NC_DISABLE_ACCESS_CHECK) {
+        if (!data.should_create_underlying_storage && !config.NC_DISABLE_ACCESS_CHECK) {
             const accessible = await native_fs_utils.is_dir_accessible(account_fs_context, data.path);
             if (!accessible) {
                 throw_cli_error(ManageCLIError.InaccessibleStoragePath, data.path);
diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js
@@ -324,24 +324,16 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             }
 
             // create bucket's underlying storage directory
-            try {
-                await nb_native().fs.mkdir(fs_context, bucket_storage_path, get_umasked_mode(config.BASE_MODE_DIR));
-                const reserved_tag_event_args = Object.keys(config.NSFS_GLACIER_RESERVED_BUCKET_TAGS).reduce((curr, tag) => {
-                    const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag];
-                    if (tag_info.event) return Object.assign(curr, { [tag]: tag_info.default });
-                    return curr;
-                }, {});
-
-                new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {
-                    ...reserved_tag_event_args, bucket_name: name, account: sdk.requesting_account.name
-                });
-            } catch (err) {
-                dbg.error('BucketSpaceFS: create_bucket could not create underlying directory - nsfs, deleting bucket', err);
-                new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
-                    .create_event(name, { bucket: name, path: bucket_storage_path }, err);
-                await nb_native().fs.unlink(this.fs_context, bucket_config_path);
-                throw translate_error_codes(err, entity_enum.BUCKET);
-            }
+            await BucketSpaceFS._create_uls(this.fs_context, fs_context, name, bucket_storage_path, bucket_config_path);
+            const reserved_tag_event_args = Object.keys(config.NSFS_GLACIER_RESERVED_BUCKET_TAGS).reduce((curr, tag) => {
+                const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag];
+                if (tag_info.event) return Object.assign(curr, { [tag]: tag_info.default });
+                return curr;
+            }, {});
+
+            new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, {
+                ...reserved_tag_event_args, bucket_name: name, account: sdk.requesting_account.name
+            });
         });
     }
 
@@ -1094,6 +1086,26 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
     static _objectify_tagging_arr(tagging) {
         return (tagging || []).reduce((curr, tag) => Object.assign(curr, { [tag.key]: tag.value }), {});
     }
+
+    /**
+     * _create_uls creates underylying storage at the given storage_path
+     * @param {*} fs_context - root fs_context 
+     * @param {*} acc_fs_context - fs_context associated with the performing account
+     * @param {*} name - bucket name
+     * @param {*} storage_path - bucket's storage path
+     * @param {*} cfg_path - bucket's configuration path
+     */
+    static async _create_uls(fs_context, acc_fs_context, name, storage_path, cfg_path) {
+        try {
+            await nb_native().fs.mkdir(acc_fs_context, storage_path, get_umasked_mode(config.BASE_MODE_DIR));
+        } catch (error) {
+            dbg.error('BucketSpaceFS: _create_uls could not create underlying directory - nsfs, deleting bucket', error);
+            new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED)
+                .create_event(name, { bucket: name, path: storage_path }, error);
+            await nb_native().fs.unlink(fs_context, cfg_path);
+            throw translate_error_codes(error, entity_enum.BUCKET);
+        }
+    }
 }
 
 module.exports = BucketSpaceFS;
diff --git a/src/test/unit_tests/jest_tests/test_nc_bucket_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_bucket_cli.test.js
@@ -265,6 +265,26 @@ describe('manage nsfs cli bucket flow', () => {
             expect(JSON.parse(res_list).response.reply.map(item => item.name))
                 .toEqual(expect.arrayContaining([bucket_options.name]));
         });
+
+        it('cli create bucket - with ULS true', async () => {
+            await fs_utils.create_fresh_path(root_path);
+            await set_path_permissions_and_owner(root_path, account_defaults, 0o700);
+
+            const action = ACTIONS.ADD;
+            const bucket_options = { config_root, ...bucket_defaults, should_create_underlying_storage: true };
+            await exec_manage_cli(TYPES.BUCKET, action, bucket_options);
+            await fs_utils.file_must_exist(bucket_options.path);
+
+            const bucket = await config_fs.get_bucket_by_name(bucket_options.name);
+            expect(bucket).toBeDefined();
+            expect(bucket.name).toBe(bucket_options.name);
+            expect(bucket.should_create_underlying_storage).toBe(true);
+
+            const res = await exec_manage_cli(TYPES.BUCKET, ACTIONS.STATUS, { name: bucket_options.name, config_root });
+            const parsed_res = JSON.parse(res);
+            expect(parsed_res.response.reply.name).toBe(bucket_options.name);
+            expect(parsed_res.response.reply.should_create_underlying_storage).toBe(true);
+        });
     });
 
     describe('cli create bucket using from_file', () => {
@@ -805,6 +825,26 @@ describe('manage nsfs cli bucket flow', () => {
             const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name);
             expect(is_bucket_exists).toBe(false);
         });
+
+
+        it('cli delete bucket when should_create_underlying_storage is true', async () => {
+            let path_exists = await is_path_exists(DEFAULT_FS_CONFIG, bucket_defaults.path);
+            expect(path_exists).toBe(true);
+
+            const bucket_options = { config_root, name: 'bucket1' };
+
+            const bucket = await config_fs.get_bucket_by_name(bucket_options.name);
+            await config_fs.update_bucket_config_file({ ...bucket, should_create_underlying_storage: true });
+
+            const action = ACTIONS.DELETE;
+            const res = await exec_manage_cli(TYPES.BUCKET, action, bucket_options);
+            expect(JSON.parse(res.trim()).response.code).toBe(ManageCLIResponse.BucketDeleted.code);
+            const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name);
+            expect(is_bucket_exists).toBe(false);
+
+            path_exists = await is_path_exists(DEFAULT_FS_CONFIG, bucket_defaults.path);
+            expect(path_exists).toBe(false);
+        });
     });
 
     describe('cli status bucket ', () => {
