Merge pull request #8882 from romayalon/romy-backport-lifecycle-5.18

NC | Lifecycle | backports to 5.18

Author: nimrod-becker

config.js

@@ -998,6 +998,30 @@ config.NC_DISABLE_HEALTH_ACCESS_CHECK = false;
 config.NC_DISABLE_POSIX_MODE_ACCESS_CHECK = true;
 config.NC_DISABLE_SCHEMA_CHECK = false;
 
+////////// NC LIFECYLE  //////////
+
+config.NC_LIFECYCLE_LOGS_DIR = '/var/log/noobaa/lifecycle';
+config.NC_LIFECYCLE_CONFIG_DIR_NAME = 'lifecycle';
+config.NC_LIFECYCLE_TIMEOUT_MS = 8 * 60 * 60 * 1000;
+
+// NC_LIFECYCLE_RUN_TIME must be of the format hh:mm which specifies
+// when NooBaa should allow running nc lifecycle process
+// NOTE: This will also be in the same timezone as specified in
+// NC_LIFECYCLE_TZ
+config.NC_LIFECYCLE_RUN_TIME = '00:00';
+
+// NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS configures the delay
+// tolerance in minutes.
+//
+// eg. If the expiry run time is set to 00:00 and the tolerance is
+// set to be 2 mins then the expiry can trigger till 00:02 (unless
+// already triggered between 00:00 - 00:02
+config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = 2;
+
+/** @type {'UTC' | 'LOCAL'} */
+config.NC_LIFECYCLE_TZ = 'LOCAL';
+
+config.NC_LIFECYCLE_GPFS_ILM_ENABLED = false;
 ////////// GPFS //////////
 config.GPFS_DOWN_DELAY = 1000;
 

src/cmd/manage_nsfs.js

@@ -1,7 +1,18 @@
 /* Copyright (C) 2020 NooBaa */
 'use strict';
 
+// DO NOT PUT NEW REQUIREMENTS BEFORE SETTING process.env.NC_NSFS_NO_DB_ENV = 'true' 
+// NC nsfs deployments specifying process.env.LOCAL_MD_SERVER=true deployed together with a db
+// when a system_store object is initialized VaccumAnalyzer is being called once a day.
+// when NC nsfs deployed without db we would like to avoid running VaccumAnalyzer in any flow there is
+// because running it will cause a panic.
+if (process.env.LOCAL_MD_SERVER !== 'true') {
+    process.env.NC_NSFS_NO_DB_ENV = 'true';
+}
+
 const dbg = require('../util/debug_module')(__filename);
+if (!dbg.get_process_name()) dbg.set_process_name('noobaa-cli');
+
 const _ = require('lodash');
 const path = require('path');
 const minimist = require('minimist');
@@ -17,6 +28,7 @@ const { account_id_cache } = require('../sdk/accountspace_fs');
 const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
 const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse;
 const manage_nsfs_glacier = require('../manage_nsfs/manage_nsfs_glacier');
+const noobaa_cli_lifecycle = require('../manage_nsfs/nc_lifecycle');
 const manage_nsfs_logging = require('../manage_nsfs/manage_nsfs_logging');
 const noobaa_cli_diagnose = require('../manage_nsfs/diagnose');
 const noobaa_cli_upgrade = require('../manage_nsfs/upgrade');
@@ -76,6 +88,8 @@ async function main(argv = minimist(process.argv.slice(2))) {
             await notification_management();
         } else if (type === TYPES.CONNECTION) {
             await connection_management(action, user_input);
+        } else if (type === TYPES.LIFECYCLE) {
+            await lifecycle_management(argv);
         } else {
             throw_cli_error(ManageCLIError.InvalidType);
         }
@@ -252,7 +266,7 @@ async function delete_bucket(data, force) {
         }
         await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true);
         await config_fs.delete_bucket_config_file(data.name);
-        return { code: ManageCLIResponse.BucketDeleted, detail: '', event_arg: { bucket: data.name } };
+        return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } };
     } catch (err) {
         if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.NoSuchBucket, data.name);
         throw err;
@@ -461,7 +475,7 @@ async function update_account(data) {
  */
 async function delete_account(data) {
     await config_fs.delete_account_config_file(data);
-    return { code: ManageCLIResponse.AccountDeleted, detail: '', event_arg: { account: data.name } };
+    return { code: ManageCLIResponse.AccountDeleted, detail: { name: data.name }, event_arg: { account: data.name } };
 }
 
 /**
@@ -754,12 +768,12 @@ async function connection_management(action, user_input) {
             break;
         case ACTIONS.DELETE:
             await config_fs.delete_connection_config_file(user_input.name);
-            response = { code: ManageCLIResponse.ConnectionDeleted };
+            response = { code: ManageCLIResponse.ConnectionDeleted, detail: {name: user_input.name} };
             break;
         case ACTIONS.UPDATE:
             await notifications_util.update_connect_file(user_input.name, user_input.key,
                 user_input.value, user_input.remove_key, config_fs);
-            response = { code: ManageCLIResponse.ConnectionUpdated };
+            response = { code: ManageCLIResponse.ConnectionUpdated, detail: {name: user_input.name} };
             break;
         case ACTIONS.STATUS:
             data = await new notifications_util.Notificator({
@@ -780,5 +794,31 @@ async function connection_management(action, user_input) {
     write_stdout_response(response.code, response.detail, response.event_arg);
 }
 
+////////////////////
+///// LIFECYCLE ////
+////////////////////
+
+/**
+ * lifecycle_management runs the nc lifecycle management
+ * @returns {Promise<void>}
+ */
+async function lifecycle_management(args) {
+    const disable_service_validation = get_boolean_or_string_value(args.disable_service_validation);
+    const disable_runtime_validation = get_boolean_or_string_value(args.disable_runtime_validation);
+    const short_status = get_boolean_or_string_value(args.short_status);
+    try {
+        const options = { disable_service_validation, disable_runtime_validation, short_status };
+        const { should_run, lifecycle_run_status } = await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, options);
+        if (should_run) {
+            write_stdout_response(ManageCLIResponse.LifecycleSuccessful, lifecycle_run_status);
+        } else {
+            write_stdout_response(ManageCLIResponse.LifecycleWorkerNotRunning);
+        }
+    } catch (err) {
+        dbg.error('manage_nsfs.lifecycle_management: Error while running run_lifecycle_under_lock', config_fs.config_json_path, err);
+        throw_cli_error(err);
+    }
+}
+
 exports.main = main;
 if (require.main === module) main();

src/cmd/nsfs.js

@@ -32,16 +32,14 @@ if (process.env.LOCAL_MD_SERVER === 'true') {
 //const js_utils = require('../util/js_utils');
 const nb_native = require('../util/nb_native');
 //const schema_utils = require('../util/schema_utils');
-const RpcError = require('../rpc/rpc_error');
-const ObjectSDK = require('../sdk/object_sdk');
 const { cluster } = require('../util/fork_utils');
-const NamespaceFS = require('../sdk/namespace_fs');
 const BucketSpaceSimpleFS = require('../sdk/bucketspace_simple_fs');
 const BucketSpaceFS = require('../sdk/bucketspace_fs');
 const SensitiveString = require('../util/sensitive_string');
 const endpoint_stats_collector = require('../sdk/endpoint_stats_collector');
 //const { RPC_BUFFERS } = require('../rpc');
 const AccountSDK = require('../sdk/account_sdk');
+const NsfsObjectSDK = require('../sdk/nsfs_object_sdk');
 const AccountSpaceFS = require('../sdk/accountspace_fs');
 const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent;
 const { set_debug_level } = require('../manage_nsfs/manage_nsfs_cli_utils');
@@ -121,98 +119,6 @@ function print_usage() {
 
 let nsfs_config_root;
 
-class NsfsObjectSDK extends ObjectSDK {
-    constructor(fs_root, fs_config, account, versioning, config_root, nsfs_system) {
-        // const rpc_client_hooks = new_rpc_client_hooks();
-        // rpc_client_hooks.account.read_account_by_access_key = async ({ access_key }) => {
-        //     if (access_key) {
-        //         return { access_key };
-        //     }
-        // };
-        // rpc_client_hooks.bucket.read_bucket_sdk_info = async ({ name }) => {
-        //     if (name) {
-        //         return { name };
-        //     }
-        // };
-        let bucketspace;
-        if (config_root) {
-            bucketspace = new BucketSpaceFS({ config_root }, endpoint_stats_collector.instance());
-        } else {
-            bucketspace = new BucketSpaceSimpleFS({ fs_root });
-        }
-        super({
-            rpc_client: null,
-            internal_rpc_client: null,
-            object_io: null,
-            bucketspace,
-            stats: endpoint_stats_collector.instance(),
-        });
-        this.nsfs_config_root = nsfs_config_root;
-        this.nsfs_fs_root = fs_root;
-        this.nsfs_fs_config = fs_config;
-        this.nsfs_account = account;
-        this.nsfs_versioning = versioning;
-        this.nsfs_namespaces = {};
-        this.nsfs_system = nsfs_system;
-        if (!config_root) {
-            this._get_bucket_namespace = bucket_name => this._simple_get_single_bucket_namespace(bucket_name);
-            this.load_requesting_account = auth_req => this._simple_load_requesting_account(auth_req);
-            this.read_bucket_sdk_policy_info = bucket_name => this._simple_read_bucket_sdk_policy_info(bucket_name);
-            this.read_bucket_sdk_config_info = () => undefined;
-            this.read_bucket_usage_info = () => undefined;
-            this.read_bucket_sdk_website_info = () => undefined;
-            this.read_bucket_sdk_namespace_info = () => undefined;
-            this.read_bucket_sdk_caching_info = () => undefined;
-        }
-    }
-
-    async _simple_get_single_bucket_namespace(bucket_name) {
-        const existing_ns = this.nsfs_namespaces[bucket_name];
-        if (existing_ns) return existing_ns;
-        const ns_fs = new NamespaceFS({
-            fs_backend: this.nsfs_fs_config.backend,
-            bucket_path: this.nsfs_fs_root + '/' + bucket_name,
-            bucket_id: 'nsfs',
-            namespace_resource_id: undefined,
-            access_mode: undefined,
-            versioning: this.nsfs_versioning,
-            stats: endpoint_stats_collector.instance(),
-            force_md5_etag: false,
-        });
-        this.nsfs_namespaces[bucket_name] = ns_fs;
-        return ns_fs;
-    }
-
-    async _simple_load_requesting_account(auth_req) {
-        const access_key = this.nsfs_account.access_keys?.[0]?.access_key;
-        if (access_key) {
-            const token = this.get_auth_token();
-            if (!token) {
-                throw new RpcError('UNAUTHORIZED', `Anonymous access to bucket not allowed`);
-            }
-            if (token.access_key !== access_key.unwrap()) {
-                throw new RpcError('INVALID_ACCESS_KEY_ID', `Account with access_key not found`);
-            }
-        }
-        this.requesting_account = this.nsfs_account;
-    }
-
-    async _simple_read_bucket_sdk_policy_info(bucket_name) {
-        return {
-            s3_policy: {
-                Version: '2012-10-17',
-                Statement: [{
-                    Effect: 'Allow',
-                    Action: ['*'],
-                    Resource: ['*'],
-                    Principal: [new SensitiveString('*')],
-                }]
-            },
-            bucket_owner: new SensitiveString('nsfs'),
-            owner_account: new SensitiveString('nsfs-id'), // temp
-        };
-    }
-}
 
 // NsfsAccountSDK was based on NsfsObjectSDK
 // simple flow was not implemented

src/manage_nsfs/health.js

@@ -12,13 +12,13 @@ const native_fs_utils = require('../util/native_fs_utils');
 const { read_stream_join } = require('../util/buffer_utils');
 const { make_https_request } = require('../util/http_utils');
 const { TYPES } = require('./manage_nsfs_constants');
-const { get_boolean_or_string_value, throw_cli_error, write_stdout_response, get_bucket_owner_account_by_id } = require('./manage_nsfs_cli_utils');
+const { get_boolean_or_string_value, throw_cli_error, write_stdout_response,
+    get_bucket_owner_account_by_id, get_service_status, NOOBAA_SERVICE_NAME } = require('./manage_nsfs_cli_utils');
 const { ManageCLIResponse } = require('./manage_nsfs_cli_responses');
 const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError;
 
 
 const HOSTNAME = 'localhost';
-const NOOBAA_SERVICE = 'noobaa';
 
 const health_errors = {
     NOOBAA_SERVICE_FAILED: {
@@ -116,7 +116,7 @@ class NSFSHealth {
     async nc_nsfs_health() {
         let endpoint_state;
         let memory;
-        const noobaa_service_state = await this.get_service_state(NOOBAA_SERVICE);
+        const noobaa_service_state = await this.get_service_state(NOOBAA_SERVICE_NAME);
         const { service_status, pid } = noobaa_service_state;
         if (pid !== '0') {
             endpoint_state = await this.get_endpoint_response();
@@ -135,7 +135,7 @@ class NSFSHealth {
         if (this.all_bucket_details) bucket_details = await this.get_bucket_status();
         if (this.all_account_details) account_details = await this.get_account_status();
         const health = {
-            service_name: NOOBAA_SERVICE,
+            service_name: NOOBAA_SERVICE_NAME,
             status: service_health,
             memory: memory,
             error: error_code,
@@ -204,18 +204,8 @@ class NSFSHealth {
     }
 
     async get_service_state(service_name) {
-        let service_status;
         let pid;
-        try {
-            service_status = await os_util.exec('systemctl show -p ActiveState --value ' + service_name, {
-                ignore_rc: false,
-                return_stdout: true,
-                trim_stdout: true,
-            });
-        } catch (err) {
-            dbg.warn('could not receive service active state', service_name, err);
-            service_status = 'missing service status info';
-        }
+        const service_status = await get_service_status(service_name);
         try {
             pid = await os_util.exec('systemctl show --property MainPID --value ' + service_name, {
                 ignore_rc: false,
@@ -302,13 +292,13 @@ class NSFSHealth {
     async get_service_memory_usage() {
         let memory_status;
         try {
-            memory_status = await os_util.exec('systemctl status ' + NOOBAA_SERVICE + ' | grep Memory ', {
+            memory_status = await os_util.exec('systemctl status ' + NOOBAA_SERVICE_NAME + ' | grep Memory ', {
                 ignore_rc: false,
                 return_stdout: true,
                 trim_stdout: true,
             });
         } catch (err) {
-            dbg.warn('could not receive service active state', NOOBAA_SERVICE, err);
+            dbg.warn('could not receive service active state', NOOBAA_SERVICE_NAME, err);
             memory_status = 'Memory: missing memory info';
         }
         if (memory_status) {

src/manage_nsfs/manage_nsfs_cli_errors.js

@@ -500,6 +500,34 @@ ManageCLIError.MissingCliParam = Object.freeze({
     http_code: 400,
 });
 
+//////////////////////////////
+//     LIFECYCLE ERRORS     //
+//////////////////////////////
+
+ManageCLIError.SystemJsonIsMissing = Object.freeze({
+    code: 'SystemJsonIsMissing',
+    message: 'Lifecycle worker can not run when system.json is missing.',
+    http_code: 400,
+});
+
+ManageCLIError.NooBaaServiceIsNotActive = Object.freeze({
+    code: 'NooBaaServiceIsNotActive',
+    message: 'Lifecycle worker can not run when NooBaa service is not active.',
+    http_code: 400,
+});
+
+ManageCLIError.LifecycleFailed = Object.freeze({
+    code: 'LifecycleFailed',
+    message: 'Lifecycle worker run failed.',
+    http_code: 400,
+});
+
+ManageCLIError.LifecycleWorkerReachedTimeout = Object.freeze({
+    code: 'LifecycleWorkerReachedTimeout',
+    message: `Lifecycle worker reached timeout - configured timeout is ${config.NC_LIFECYCLE_TIMEOUT_MS} ms`,
+    http_code: 400,
+});
+
 ///////////////////////////////
 //       ERRORS MAPPING      //
 ///////////////////////////////
@@ -522,7 +550,8 @@ ManageCLIError.RPC_ERROR_TO_MANAGE = Object.freeze({
     NO_SUCH_USER: ManageCLIError.InvalidAccountDistinguishedName,
     INVALID_MASTER_KEY: ManageCLIError.InvalidMasterKey,
     INVALID_BUCKET_NAME: ManageCLIError.InvalidBucketName,
-    CONFIG_DIR_VERSION_MISMATCH: ManageCLIError.ConfigDirUpdateBlocked
+    CONFIG_DIR_VERSION_MISMATCH: ManageCLIError.ConfigDirUpdateBlocked,
+    LIFECYCLE_WORKER_TIMEOUT: ManageCLIError.LifecycleWorkerReachedTimeout
 });
 
 const NSFS_CLI_ERROR_EVENT_MAP = {
@@ -535,7 +564,9 @@ const NSFS_CLI_ERROR_EVENT_MAP = {
     BucketSetForbiddenBucketOwnerNotExists: NoobaaEvent.UNAUTHORIZED, // GAP - add event
     BucketSetForbiddenBucketOwnerIsIAMAccount: NoobaaEvent.UNAUTHORIZED, // // GAP - add event
     LoggingExportFailed: NoobaaEvent.LOGGING_FAILED,
-    UpgradeFailed: NoobaaEvent.CONFIG_DIR_UPGRADE_FAILED
+    UpgradeFailed: NoobaaEvent.CONFIG_DIR_UPGRADE_FAILED,
+    LifecycleFailed: NoobaaEvent.LIFECYCLE_FAILED,
+    LifecycleWorkerReachedTimeout: NoobaaEvent.LIFECYCLE_TIMEOUT
 };
 
 exports.ManageCLIError = ManageCLIError;