NC | Lifecycle | Small configuration adjustments/ fixes

Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com>
(cherry picked from commit 955f07e)

Author: romayalon

config.js

@@ -1001,20 +1001,21 @@ config.NC_DISABLE_SCHEMA_CHECK = false;
 ////////// NC LIFECYLE  //////////
 
 config.NC_LIFECYCLE_LOGS_DIR = '/var/log/noobaa/lifecycle';
-config.NC_LIFECYCLE_TIMEOUT_MS = 6 * 60 * 60 * 1000;
+config.NC_LIFECYCLE_CONFIG_DIR_NAME = 'lifecycle';
+config.NC_LIFECYCLE_TIMEOUT_MS = 8 * 60 * 60 * 1000;
 
 // NC_LIFECYCLE_RUN_TIME must be of the format hh:mm which specifies
 // when NooBaa should allow running nc lifecycle process
 // NOTE: This will also be in the same timezone as specified in
 // NC_LIFECYCLE_TZ
-config.NC_LIFECYCLE_RUN_TIME = '01:00';
+config.NC_LIFECYCLE_RUN_TIME = '00:00';
 
 // NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS configures the delay
 // tolerance in minutes.
 //
-// eg. If the expiry run time is set to 01:00 and the tolerance is
-// set to be 2 mins then the expiry can trigger till 01:02 (unless
-// already triggered between 01:00 - 01:02
+// eg. If the expiry run time is set to 00:00 and the tolerance is
+// set to be 2 mins then the expiry can trigger till 00:02 (unless
+// already triggered between 00:00 - 00:02
 config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = 2;
 
 /** @type {'UTC' | 'LOCAL'} */

src/cmd/manage_nsfs.js

@@ -805,8 +805,19 @@ async function connection_management(action, user_input) {
 async function lifecycle_management(args) {
     const disable_service_validation = get_boolean_or_string_value(args.disable_service_validation);
     const disable_runtime_validation = get_boolean_or_string_value(args.disable_runtime_validation);
-    const short = get_boolean_or_string_value(args.short);
-    await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, { disable_service_validation, disable_runtime_validation, short });
+    const short_status = get_boolean_or_string_value(args.short_status);
+    try {
+        const options = { disable_service_validation, disable_runtime_validation, short_status };
+        const { should_run, lifecycle_run_status } = await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, options);
+        if (should_run) {
+            write_stdout_response(ManageCLIResponse.LifecycleSuccessful, lifecycle_run_status);
+        } else {
+            write_stdout_response(ManageCLIResponse.LifecycleWorkerNotRunning);
+        }
+    } catch (err) {
+        dbg.error('manage_nsfs.lifecycle_management: Error while running run_lifecycle_under_lock', config_fs.config_json_path, err);
+        throw_cli_error(err);
+    }
 }
 
 exports.main = main;

src/manage_nsfs/manage_nsfs_constants.js

@@ -96,7 +96,7 @@ const VALID_OPTIONS_CONNECTION = {
     'status': new Set(['name', 'decrypt', ...CLI_MUTUAL_OPTIONS]),
 };
 
-const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short', ...CLI_MUTUAL_OPTIONS]);
+const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short_status', ...CLI_MUTUAL_OPTIONS]);
 
 const VALID_OPTIONS_WHITELIST = new Set(['ips', ...CLI_MUTUAL_OPTIONS]);
 
@@ -169,7 +169,7 @@ const OPTION_TYPE = {
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];
 const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force',
-    'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous', 'disable_service_validation', 'disable_runtime_validation', 'short']);
+    'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous', 'disable_service_validation', 'disable_runtime_validation', 'short_status']);
 
 //options that can be unset using ''
 const LIST_UNSETABLE_OPTIONS = ['fs_backend', 's3_policy', 'force_md5_etag'];

src/manage_nsfs/nc_lifecycle.js

@@ -13,8 +13,7 @@ const NsfsObjectSDK = require('../sdk/nsfs_object_sdk');
 const native_fs_utils = require('../util/native_fs_utils');
 const { NoobaaEvent } = require('./manage_nsfs_events_utils');
 const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError;
-const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse;
-const { write_stdout_response, throw_cli_error, get_service_status, NOOBAA_SERVICE_NAME,
+const { throw_cli_error, get_service_status, NOOBAA_SERVICE_NAME,
     is_desired_time, record_current_time } = require('./manage_nsfs_cli_utils');
 
 // TODO:
@@ -24,7 +23,7 @@ const { write_stdout_response, throw_cli_error, get_service_status, NOOBAA_SERVI
 // 3. GPFS ILM policy and apply for scanning and filtering optimization
 // TODO - we will filter during the scan except for get_candidates_by_expiration_rule on GPFS that does the filter on the file system
 
-const CLUSTER_LOCK = 'cluster.lock';
+const LIFECYCLE_CLUSTER_LOCK = 'lifecycle_cluster.lock';
 const LIFECYLE_TIMESTAMP_FILE = 'lifecycle.timestamp';
 const config_fs_options = { silent_if_missing: true };
 
@@ -33,7 +32,7 @@ const lifecycle_run_status = {
     total_stats: _get_default_stats(), buckets_statuses: {}
 };
 
-let short_status = false;
+let return_short_status = false;
 
 const TIMED_OPS = Object.freeze({
     RUN_LIFECYLE: 'run_lifecycle',
@@ -50,36 +49,40 @@ const TIMED_OPS = Object.freeze({
  * run_lifecycle_under_lock runs the lifecycle workflow under a file system lock
  * lifecycle workflow is being locked to prevent multiple instances from running the lifecycle workflow
  * @param {import('../sdk/config_fs').ConfigFS} config_fs
- * @param {{disable_service_validation?: boolean, disable_runtime_validation?: boolean, short?: boolean}} flags
+ * @param {{disable_service_validation?: boolean, disable_runtime_validation?: boolean, short_status?: boolean}} flags
+ * @returns {Promise<{should_run: Boolean, lifecycle_run_status: Object}>}
  */
 async function run_lifecycle_under_lock(config_fs, flags) {
-    const { disable_service_validation = false, disable_runtime_validation = false, short = false } = flags;
-    short_status = short;
+    const { disable_service_validation = false, disable_runtime_validation = false, short_status = false } = flags;
+    return_short_status = short_status;
     const fs_context = config_fs.fs_context;
     const lifecyle_logs_dir_path = config.NC_LIFECYCLE_LOGS_DIR;
-    const lock_path = path.join(lifecyle_logs_dir_path, CLUSTER_LOCK);
-    const lifecycle_timestamp_file_path = path.join(lifecyle_logs_dir_path, LIFECYLE_TIMESTAMP_FILE);
+    const lifecycle_config_dir_path = path.join(config_fs.config_root, config.NC_LIFECYCLE_CONFIG_DIR_NAME);
+    const lock_path = path.join(lifecycle_config_dir_path, LIFECYCLE_CLUSTER_LOCK);
+    const lifecycle_timestamp_file_path = path.join(lifecycle_config_dir_path, LIFECYLE_TIMESTAMP_FILE);
     await config_fs.create_dir_if_missing(lifecyle_logs_dir_path);
+    await config_fs.create_dir_if_missing(lifecycle_config_dir_path);
 
+    let should_run = true;
     await native_fs_utils.lock_and_run(fs_context, lock_path, async () => {
-        dbg.log0('run_lifecycle_under_lock acquired lock - start lifecycle');
-        const should_run = await _should_lifecycle_run(fs_context, lifecycle_timestamp_file_path, disable_runtime_validation);
-        if (!should_run) write_stdout_response(ManageCLIResponse.LifecycleWorkerNotRunning);
+        dbg.log0('run_lifecycle_under_lock acquired lock - verifying');
+        should_run = await _should_lifecycle_run(fs_context, lifecycle_timestamp_file_path, disable_runtime_validation);
+        if (!should_run) return;
 
         try {
             dbg.log0('run_lifecycle_under_lock acquired lock - start lifecycle');
             new NoobaaEvent(NoobaaEvent.LIFECYCLE_STARTED).create_event();
             await run_lifecycle_or_timeout(config_fs, disable_service_validation);
-            write_stdout_response(ManageCLIResponse.LifecycleSuccessful, lifecycle_run_status);
         } catch (err) {
             dbg.error('run_lifecycle_under_lock failed with error', err, err.code, err.message);
-            throw_cli_error(err);
+            throw err;
         } finally {
             await record_current_time(fs_context, lifecycle_timestamp_file_path);
             await write_lifecycle_log_file(config_fs.fs_context, lifecyle_logs_dir_path);
             dbg.log0('run_lifecycle_under_lock done lifecycle - released lock');
         }
     });
+    return { should_run, lifecycle_run_status };
 }
 
 /**
@@ -627,7 +630,7 @@ async function _call_op_and_update_status({ bucket_name = undefined, rule_id = u
     let error;
     let reply;
     try {
-        if (!short_status) update_status({ ...update_options, op_times: { start_time } });
+        if (!return_short_status) update_status({ ...update_options, op_times: { start_time } });
         reply = await op_func();
         return reply;
     } catch (e) {
@@ -636,7 +639,7 @@ async function _call_op_and_update_status({ bucket_name = undefined, rule_id = u
     } finally {
         end_time = Date.now();
         took_ms = end_time - start_time;
-        const op_times = short_status ? { took_ms } : { end_time, took_ms };
+        const op_times = return_short_status ? { took_ms } : { end_time, took_ms };
         update_status({ ...update_options, op_times, reply, error });
     }
 }

src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js

@@ -47,6 +47,7 @@ describe('noobaa cli - lifecycle - lock check', () => {
         config.NC_LIFECYCLE_RUN_TIME = original_lifecycle_run_time;
         config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = original_lifecycle_run_delay;
         await fs_utils.folder_delete(config.NC_LIFECYCLE_LOGS_DIR);
+        await fs_utils.folder_delete(config_root);
     });
 
     afterAll(async () => {
@@ -104,7 +105,7 @@ describe('noobaa cli - lifecycle - lock check', () => {
         expect(parsed_res.message).toBe(ManageCLIResponse.LifecycleSuccessful.message);
     });
 
-    it('lifecycle_cli - change run time to 1 minutes ago - should fail ', async () => {
+    it('lifecycle_cli - change run time to 1 minute in the future - should fail ', async () => {
         const lifecyle_run_date = new Date();
         lifecyle_run_date.setMinutes(lifecyle_run_date.getMinutes() + 1);
         await config_fs.create_config_json_file(JSON.stringify({