Merge pull request #8258 from romayalon/romy-refactor-config-dir

NC | ConfigFS Refactoring

Author: romayalon

src/cmd/manage_nsfs.js

@@ -15,13 +15,13 @@ const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').Ma
  * manage_diagnose_operations handles cli diagnose operations
  * @param {string} action 
  * @param {Object} user_input 
- * @param {Object} global_config 
+ * @param {import('../sdk/config_fs').ConfigFS} config_fs 
  * @returns {Promise<Void>}
  */
-async function manage_diagnose_operations(action, user_input, global_config) {
+async function manage_diagnose_operations(action, user_input, config_fs) {
     switch (action) {
         case DIAGNOSE_ACTIONS.HEALTH:
-            await health.get_health_status(user_input, global_config);
+            await health.get_health_status(user_input, config_fs);
             break;
         case DIAGNOSE_ACTIONS.GATHER_LOGS:
             await gather_logs();

src/manage_nsfs/diagnose.js

@@ -2,7 +2,6 @@
 'use strict';
 
 const dbg = require('../util/debug_module')(__filename);
-const path = require('path');
 const _ = require('lodash');
 const P = require('../util/promise');
 const config = require('../../config');
@@ -11,6 +10,7 @@ const nb_native = require('../util/nb_native');
 const native_fs_utils = require('../util/native_fs_utils');
 const { read_stream_join } = require('../util/buffer_utils');
 const { make_https_request } = require('../util/http_utils');
+const { JSON_SUFFIX } = require('../sdk/config_fs');
 const { TYPES } = require('./manage_nsfs_constants');
 const { get_boolean_or_string_value, throw_cli_error, write_stdout_response } = require('./manage_nsfs_cli_utils');
 const { ManageCLIResponse } = require('./manage_nsfs_cli_responses');
@@ -87,9 +87,9 @@ process.env.AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE = '1';
 class NSFSHealth {
     constructor(options) {
         this.https_port = options.https_port;
-        this.config_root = options.config_root;
         this.all_account_details = options.all_account_details;
         this.all_bucket_details = options.all_bucket_details;
+        this.config_fs = options.config_fs;
     }
 
     /**
@@ -116,8 +116,8 @@ class NSFSHealth {
         const service_health = service_status !== 'active' || pid === '0' || response_code !== 'RUNNING' ? 'NOTOK' : 'OK';
 
         const error_code = await this.get_error_code(service_status, pid, response_code);
-        if (this.all_bucket_details) bucket_details = await this.get_bucket_status(this.config_root);
-        if (this.all_account_details) account_details = await this.get_account_status(this.config_root);
+        if (this.all_bucket_details) bucket_details = await this.get_bucket_status();
+        if (this.all_account_details) account_details = await this.get_account_status();
         const health = {
             service_name: NOOBAA_SERVICE,
             status: service_health,
@@ -306,38 +306,48 @@ class NSFSHealth {
         };
     }
 
-    async get_bucket_status(config_root) {
-        const bucket_details = await this.get_storage_status(config_root, TYPES.BUCKET, this.all_bucket_details);
+    async get_bucket_status() {
+        const bucket_details = await this.get_storage_status(TYPES.BUCKET, this.all_bucket_details);
         return bucket_details;
     }
 
-    async get_account_status(config_root) {
-        const account_details = await this.get_storage_status(config_root, TYPES.ACCOUNT, this.all_account_details);
+    async get_account_status() {
+        const account_details = await this.get_storage_status(TYPES.ACCOUNT, this.all_account_details);
         return account_details;
     }
 
-    async get_storage_status(config_root, type, all_details) {
-        const fs_context = this.get_root_fs_context();
-        const config_root_type_path = this.get_config_path(config_root, type);
+    async get_storage_status(type, all_details) {
         const invalid_storages = [];
         const valid_storages = [];
         //check for account and buckets dir paths
-        try {
-            await nb_native().fs.stat(fs_context, config_root_type_path);
-        } catch (err) {
-            dbg.log1(`Config root path missing ${type} folder in ${config_root_type_path}`);
+        let config_root_type_exists;
+        let config_dir_path;
+        if (type === TYPES.BUCKET) {
+            config_dir_path = this.config_fs.buckets_dir_path;
+            config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path);
+        } else if (type === TYPES.ACCOUNT) {
+            // TODO - handle iam accounts when directory structure changes - read_account_by_id
+            config_dir_path = this.config_fs.accounts_dir_path;
+            config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path);
+        }
+        // TODO - this is not a good handling for that - we need to take it to an upper level
+        if (!config_root_type_exists) {
+            dbg.log1(`Config directory type - ${type} is missing, ${config_dir_path}`);
             return {
                 invalid_storages: invalid_storages,
                 valid_storages: valid_storages
             };
         }
-        const entries = await nb_native().fs.readdir(fs_context, config_root_type_path);
-        const config_files = entries.filter(entree => !native_fs_utils.isDirectory(entree) && entree.name.endsWith('.json'));
+
+        const entries = type === TYPES.BUCKET ?
+            await this.config_fs.list_buckets() :
+            await this.config_fs.list_root_accounts();
+
+        const config_files = entries.filter(entree => !native_fs_utils.isDirectory(entree) && entree.name.endsWith(JSON_SUFFIX));
         for (const config_file of config_files) {
             // config_file get data or push error
-            const config_file_path = path.join(config_root_type_path, config_file.name);
             const { config_data = undefined, err_obj = undefined } =
-            await get_config_file_data(fs_context, config_file_path, config_file.name);
+                await this.get_config_file_data_or_error_object(type, config_file.name);
             if (!config_data && err_obj) {
                 invalid_storages.push(err_obj.invalid_storage);
                 continue;
@@ -351,9 +361,10 @@ class NSFSHealth {
                 config_data.nsfs_account_config.new_buckets_path;
 
             if (type === TYPES.ACCOUNT) {
+                const config_file_path = this.config_fs.get_account_path_by_name(config_file.name);
                 res = await is_new_buckets_path_valid(config_file_path, config_data, storage_path);
             } else if (type === TYPES.BUCKET) {
-                res = await is_bucket_storage_path_exists(fs_context, config_data, storage_path);
+                res = await is_bucket_storage_path_exists(this.config_fs.fs_context, config_data, storage_path);
             }
             if (all_details && res.valid_storage) {
                 valid_storages.push(res.valid_storage);
@@ -367,21 +378,52 @@ class NSFSHealth {
         };
     }
 
-    get_config_path(config_root, type) {
-        return path.join(config_root, type === TYPES.BUCKET ? '/buckets' : '/accounts');
+    /**
+     * get_config_file_data_or_error_object return an object containing config_data or err_obj if error occurred
+     * @param {string} type
+     * @param {string} config_file_name
+     * @returns {Promise<object>}
+     */
+    async get_config_file_data_or_error_object(type, config_file_name) {
+        let config_data;
+        let err_obj;
+        try {
+            config_data = type === TYPES.BUCKET ?
+                await this.config_fs.get_bucket_by_name(config_file_name) :
+                // TODO - should be changed to id when moving to new structure for supporting iam accounts
+                await this.config_fs.get_account_by_name(config_file_name);
+        } catch (err) {
+            let err_code;
+            const config_file_path = type === TYPES.BUCKET ?
+                await this.config_fs.get_bucket_path_by_name(config_file_name) :
+                // TODO - should be changed to id when moving to new structure for supporting iam accounts
+                await this.config_fs.get_account_path_by_name(config_file_name);
+
+            if (err.code === 'ENOENT') {
+                dbg.log1(`Error: Config file path should be a valid path`, config_file_path, err);
+                err_code = health_errors.MISSING_CONFIG.error_code;
+            } else {
+                dbg.log1('Error: while accessing the config file: ', config_file_path, err);
+                err_code = health_errors.INVALID_CONFIG.error_code;
+            }
+            err_obj = get_invalid_object(config_file_name, config_file_path, undefined, err_code);
+        }
+        return {
+            config_data,
+            err_obj
+        };
     }
 }
 
-async function get_health_status(argv, global_config) {
+async function get_health_status(argv, config_fs) {
     try {
-        const config_root = global_config.config_root;
         const https_port = Number(argv.https_port) || config.ENDPOINT_SSL_PORT;
         const deployment_type = argv.deployment_type || 'nc';
         const all_account_details = get_boolean_or_string_value(argv.all_account_details);
         const all_bucket_details = get_boolean_or_string_value(argv.all_bucket_details);
 
         if (deployment_type === 'nc') {
-            const health = new NSFSHealth({ https_port, config_root, all_account_details, all_bucket_details });
+            const health = new NSFSHealth({ https_port, all_account_details, all_bucket_details, config_fs });
             const health_status = await health.nc_nsfs_health();
             write_stdout_response(ManageCLIResponse.HealthStatus, health_status);
         } else {
@@ -432,6 +474,7 @@ async function is_new_buckets_path_valid(config_file_path, config_data, new_buck
     }
 
     try {
+        await nb_native().fs.stat(account_fs_context, new_buckets_path);
         const accessible = await native_fs_utils.is_dir_rw_accessible(account_fs_context, new_buckets_path);
         if (!accessible) {
             const new_err = new Error('ACCESS DENIED');
@@ -452,36 +495,6 @@ async function is_new_buckets_path_valid(config_file_path, config_data, new_buck
     return res_obj;
 }
 
-/**
- * get_config_file_data return an object containing config_data or err_obj if error occurred
- * @param {nb.NativeFSContext} fs_context
- * @param {string} config_file_path
- * @param {string} config_file_name
- * @returns {Promise<object>}
- */
-async function get_config_file_data(fs_context, config_file_path, config_file_name) {
-    let config_data;
-    let err_obj;
-    try {
-        const { data } = await nb_native().fs.readFile(fs_context, config_file_path);
-        config_data = JSON.parse(data.toString());
-    } catch (err) {
-        let err_code;
-        if (err.code === 'ENOENT') {
-            dbg.log1(`Error: Config file path should be a valid path`, config_file_path, err);
-            err_code = health_errors.MISSING_CONFIG.error_code;
-        } else {
-            dbg.log1('Error: while accessing the config file: ', config_file_path, err);
-            err_code = health_errors.INVALID_CONFIG.error_code;
-        }
-        err_obj = get_invalid_object(config_file_name, config_file_path, undefined, err_code);
-    }
-    return {
-        config_data,
-        err_obj
-    };
-}
-
 /**
  * is_bucket_storage_path_exists checks if the underlying storage path of a bucket exists 
  * @param {nb.NativeFSContext} fs_context

src/manage_nsfs/health.js

@@ -3,8 +3,6 @@
 
 const dbg = require('../util/debug_module')(__filename);
 const _ = require('lodash');
-const path = require('path');
-const config = require('../../config');
 const nb_native = require('../util/nb_native');
 const native_fs_utils = require('../util/native_fs_utils');
 const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
@@ -15,36 +13,6 @@ const { BOOLEAN_STRING_VALUES } = require('../manage_nsfs/manage_nsfs_constants'
 const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
 const mongo_utils = require('../util/mongo_utils');
 
-/**
- * @param {object} global_config
- */
-async function check_and_create_config_dirs(global_config) {
-    const pre_req_dirs = [
-        global_config.config_root,
-        global_config.buckets_dir_path,
-        global_config.accounts_dir_path,
-        global_config.access_keys_dir_path,
-    ];
-
-    if (config.NSFS_GLACIER_LOGS_ENABLED) {
-        pre_req_dirs.push(config.NSFS_GLACIER_LOGS_DIR);
-    }
-
-    for (const dir_path of pre_req_dirs) {
-        try {
-            const fs_context = native_fs_utils.get_process_fs_context(global_config.config_root_backend);
-            const dir_exists = await native_fs_utils.is_path_exists(fs_context, dir_path);
-            if (dir_exists) {
-                dbg.log1('nsfs.check_and_create_config_dirs: config dir exists:', dir_path);
-            } else {
-                await native_fs_utils._create_path(dir_path, fs_context, config.BASE_MODE_CONFIG_DIR);
-                dbg.log1('nsfs.check_and_create_config_dirs: config dir was created:', dir_path);
-            }
-        } catch (err) {
-            dbg.log1('nsfs.check_and_create_config_dirs: could not create pre requisite path', dir_path);
-        }
-    }
-}
 
 function throw_cli_error(error, detail, event_arg) {
     const error_event = NSFS_CLI_ERROR_EVENT_MAP[error.code];
@@ -66,56 +34,15 @@ function write_stdout_response(response_code, detail, event_arg) {
     });
 }
 
-function get_config_file_path(config_type_path, file_name) {
-    return path.join(config_type_path, file_name + '.json');
-}
-
-function get_symlink_config_file_path(config_type_path, file_name) {
-    return path.join(config_type_path, file_name + '.symlink');
-}
-
-/**
- * get_config_data will read a config file and return its content 
- * while omitting secrets if show_secrets flag was not provided
- * @param {string} config_root_backend
- * @param {string} config_file_path
- * @param {boolean} [show_secrets]
- */
-async function get_config_data(config_root_backend, config_file_path, show_secrets = false) {
-    const fs_context = native_fs_utils.get_process_fs_context(config_root_backend);
-    const { data } = await nb_native().fs.readFile(fs_context, config_file_path);
-    const config_data = _.omit(JSON.parse(data.toString()), show_secrets ? [] : ['access_keys']);
-    return config_data;
-}
-
-/**
- * get_config_data_if_exists will read a config file and return its content 
- * while omitting secrets if show_secrets flag was not provided
- * if the config file was deleted (encounter ENOENT error) - continue (returns undefined)
- * @param {string} config_root_backend
- * @param {string} config_file_path
- * @param {boolean} [show_secrets]
- */
-async function get_config_data_if_exists(config_root_backend, config_file_path, show_secrets = false) {
-    try {
-        const config_data = await get_config_data(config_root_backend, config_file_path, show_secrets);
-        return config_data;
-    } catch (err) {
-        dbg.warn('get_config_data_if_exists: with config_file_path', config_file_path, 'got an error', err);
-        if (err.code !== 'ENOENT') throw err;
-    }
-}
-
 /**
  * get_bucket_owner_account will return the account of the bucket_owner
  * otherwise it would throw an error
- * @param {object} global_config
+ * @param {import('../sdk/config_fs').ConfigFS} config_fs
  * @param {string} bucket_owner
  */
-async function get_bucket_owner_account(global_config, bucket_owner) {
-    const account_config_path = get_config_file_path(global_config.accounts_dir_path, bucket_owner);
+async function get_bucket_owner_account(config_fs, bucket_owner) {
     try {
-        const account = await get_config_data(global_config.config_root_backend, account_config_path);
+        const account = await config_fs.get_account_by_name(bucket_owner);
         return account;
     } catch (err) {
         if (err.code === 'ENOENT') {
@@ -206,15 +133,10 @@ function check_root_account_owns_user(root_account, account) {
 // EXPORTS
 exports.throw_cli_error = throw_cli_error;
 exports.write_stdout_response = write_stdout_response;
-exports.get_config_file_path = get_config_file_path;
-exports.get_symlink_config_file_path = get_symlink_config_file_path;
 exports.get_boolean_or_string_value = get_boolean_or_string_value;
-exports.get_config_data = get_config_data;
 exports.get_bucket_owner_account = get_bucket_owner_account;
 exports.get_options_from_file = get_options_from_file;
 exports.has_access_keys = has_access_keys;
 exports.generate_id = generate_id;
 exports.set_debug_level = set_debug_level;
 exports.check_root_account_owns_user = check_root_account_owns_user;
-exports.get_config_data_if_exists = get_config_data_if_exists;
-exports.check_and_create_config_dirs = check_and_create_config_dirs;

src/manage_nsfs/manage_nsfs_cli_utils.js

@@ -30,11 +30,6 @@ const DIAGNOSE_ACTIONS = {
     METRICS: 'metrics'
 };
 
-const CONFIG_SUBDIRS = {
-    ACCOUNTS: 'accounts',
-    BUCKETS: 'buckets',
-    ACCESS_KEYS: 'access_keys'
-};
 
 const CONFIG_ROOT_FLAG = 'config_root';
 const CLI_MUTUAL_OPTIONS = new Set([CONFIG_ROOT_FLAG, 'config_root_backend', 'debug']);
@@ -139,7 +134,6 @@ exports.TYPES = TYPES;
 exports.ACTIONS = ACTIONS;
 exports.GLACIER_ACTIONS = GLACIER_ACTIONS;
 exports.DIAGNOSE_ACTIONS = DIAGNOSE_ACTIONS;
-exports.CONFIG_SUBDIRS = CONFIG_SUBDIRS;
 exports.VALID_OPTIONS = VALID_OPTIONS;
 exports.OPTION_TYPE = OPTION_TYPE;
 exports.FROM_FILE = FROM_FILE;