NC | Lifecycle | Add lock to lifecycle worker

Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com>
(cherry picked from commit 32926c4)

Author: romayalon

config.js

@@ -998,6 +998,27 @@ config.NC_DISABLE_HEALTH_ACCESS_CHECK = false;
 config.NC_DISABLE_POSIX_MODE_ACCESS_CHECK = true;
 config.NC_DISABLE_SCHEMA_CHECK = false;
 
+////////// NC LIFECYLE  //////////
+
+config.LIFECYCLE_LOGS_DIR = '/var/log/noobaa/lifecycle';
+
+// NC_LIFECYCLE_RUN_TIME must be of the format hh:mm which specifies
+// when NooBaa should allow running nc lifecycle process
+// NOTE: This will also be in the same timezone as specified in
+// NC_LIFECYCLE_TZ
+config.NC_LIFECYCLE_RUN_TIME = '01:00';
+
+// NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS configures the delay
+// tolerance in minutes.
+//
+// eg. If the expiry run time is set to 01:00 and the tolerance is
+// set to be 2 mins then the expiry can trigger till 01:02 (unless
+// already triggered between 01:00 - 01:02
+config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = 2;
+
+/** @type {'UTC' | 'LOCAL'} */
+config.NC_LIFECYCLE_TZ = 'LOCAL';
+
 ////////// GPFS //////////
 config.GPFS_DOWN_DELAY = 1000;
 

src/cmd/manage_nsfs.js

@@ -803,8 +803,9 @@ async function connection_management(action, user_input) {
  * @returns {Promise<void>}
  */
 async function lifecycle_management(args) {
-    const disable_service_validation = args.disable_service_validation === 'true';
-    await noobaa_cli_lifecycle.run_lifecycle(config_fs, disable_service_validation);
+    const disable_service_validation = get_boolean_or_string_value(args.disable_service_validation);
+    const disable_runtime_validation = get_boolean_or_string_value(args.disable_runtime_validation);
+    await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, disable_service_validation, disable_runtime_validation);
 }
 
 exports.main = main;

src/manage_nsfs/manage_nsfs_cli_utils.js

@@ -198,6 +198,91 @@ async function get_service_status(service_name) {
     return service_status;
 }
 
+/**
+ * is_desired_time returns true if the given time matches with
+ * the desired time or if 
+ * @param {nb.NativeFSContext} fs_context 
+ * @param {Date} current
+ * @param {string} desire time in format 'hh:mm'
+ * @param {number} delay_limit_mins
+ * @param {string} timestamp_file_path 
+ * @param {"UTC" | "LOCAL"} timezone
+ * @returns {Promise<boolean>}
+ */
+async function is_desired_time(fs_context, current, desire, delay_limit_mins, timestamp_file_path, timezone) {
+    const [desired_hour, desired_min] = desire.split(':').map(Number);
+    if (
+        isNaN(desired_hour) ||
+        isNaN(desired_min) ||
+        (desired_hour < 0 || desired_hour >= 24) ||
+        (desired_min < 0 || desired_min >= 60)
+    ) {
+        throw new Error('invalid desired_time - must be hh:mm');
+    }
+
+    const min_time = get_tz_date(desired_hour, desired_min, 0, timezone);
+    const max_time = get_tz_date(desired_hour, desired_min + delay_limit_mins, 0, timezone);
+
+    if (current >= min_time && current <= max_time) {
+        try {
+            const { data } = await nb_native().fs.readFile(fs_context, timestamp_file_path);
+            const lastrun = new Date(data.toString());
+
+            // Last run should NOT be in this window
+            if (lastrun >= min_time && lastrun <= max_time) return false;
+        } catch (error) {
+            if (error.code === 'ENOENT') return true;
+            console.error('failed to read last run timestamp:', error, 'timestamp_file_path:', timestamp_file_path);
+
+            throw error;
+        }
+
+        return true;
+    }
+
+    return false;
+}
+
+
+/**
+ * record_current_time stores the current timestamp in ISO format into
+ * the given timestamp file
+ * @param {nb.NativeFSContext} fs_context 
+ * @param {string} timestamp_file_path 
+ */
+async function record_current_time(fs_context, timestamp_file_path) {
+    await nb_native().fs.writeFile(
+        fs_context,
+        timestamp_file_path,
+        Buffer.from(new Date().toISOString()),
+    );
+}
+
+/**
+ * @param {number} hours
+ * @param {number} mins
+ * @param {number} secs
+ * @param {'UTC' | 'LOCAL'} tz
+ * @returns {Date}
+ */
+function get_tz_date(hours, mins, secs, tz) {
+    const date = new Date();
+
+    if (tz === 'UTC') {
+        date.setUTCHours(hours);
+        date.setUTCMinutes(hours);
+        date.setUTCSeconds(secs);
+        date.setUTCMilliseconds(0);
+    } else {
+        date.setHours(hours);
+        date.setMinutes(mins);
+        date.setSeconds(secs);
+        date.setMilliseconds(0);
+    }
+
+    return date;
+}
+
 // EXPORTS
 exports.throw_cli_error = throw_cli_error;
 exports.write_stdout_response = write_stdout_response;
@@ -212,3 +297,6 @@ exports.is_name_update = is_name_update;
 exports.is_access_key_update = is_access_key_update;
 exports.get_service_status = get_service_status;
 exports.NOOBAA_SERVICE_NAME = NOOBAA_SERVICE_NAME;
+exports.is_desired_time = is_desired_time;
+exports.record_current_time = record_current_time;
+exports.get_tz_date = get_tz_date;

src/manage_nsfs/manage_nsfs_constants.js

@@ -96,7 +96,7 @@ const VALID_OPTIONS_CONNECTION = {
     'status': new Set(['name', 'decrypt', ...CLI_MUTUAL_OPTIONS]),
 };
 
-const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', ...CLI_MUTUAL_OPTIONS]);
+const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', ...CLI_MUTUAL_OPTIONS]);
 
 const VALID_OPTIONS_WHITELIST = new Set(['ips', ...CLI_MUTUAL_OPTIONS]);
 
@@ -154,7 +154,8 @@ const OPTION_TYPE = {
     custom_upgrade_scripts_dir: 'string',
     skip_verification: 'boolean',
     // lifecycle options
-    disable_service_validation: 'string',
+    disable_service_validation: 'boolean',
+    disable_runtime_validation: 'boolean',
     //connection
     notification_protocol: 'string',
     agent_request_object: 'string',
@@ -167,7 +168,7 @@ const OPTION_TYPE = {
 
 const BOOLEAN_STRING_VALUES = ['true', 'false'];
 const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force',
-    'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous']);
+    'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous', 'disable_service_validation', 'disable_runtime_validation']);
 
 //options that can be unset using ''
 const LIST_UNSETABLE_OPTIONS = ['fs_backend', 's3_policy', 'force_md5_etag'];

src/manage_nsfs/manage_nsfs_glacier.js

@@ -7,14 +7,16 @@ const config = require('../../config');
 const nb_native = require('../util/nb_native');
 const { Glacier } = require('../sdk/glacier');
 const native_fs_utils = require('../util/native_fs_utils');
+const { is_desired_time, record_current_time } = require('./manage_nsfs_cli_utils');
 
 const CLUSTER_LOCK = 'cluster.lock';
 const SCAN_LOCK = 'scan.lock';
 
 async function process_migrations() {
     const fs_context = native_fs_utils.get_process_fs_context();
 
-    await lock_and_run(fs_context, CLUSTER_LOCK, async () => {
+    const lock_path = path.join(config.NSFS_GLACIER_LOGS_DIR, CLUSTER_LOCK);
+    await native_fs_utils.lock_and_run(fs_context, lock_path, async () => {
         const backend = Glacier.getBackend();
 
         if (
@@ -23,7 +25,8 @@ async function process_migrations() {
             await migrate_log_exceeds_threshold()
         ) {
             await run_glacier_migrations(fs_context, backend);
-            await record_current_time(fs_context, Glacier.MIGRATE_TIMESTAMP_FILE);
+            const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, Glacier.MIGRATE_TIMESTAMP_FILE);
+            await record_current_time(fs_context, timestamp_file_path);
         }
     });
 }
@@ -41,7 +44,8 @@ async function run_glacier_migrations(fs_context, backend) {
 async function process_restores() {
     const fs_context = native_fs_utils.get_process_fs_context();
 
-    await lock_and_run(fs_context, CLUSTER_LOCK, async () => {
+    const lock_path = path.join(config.NSFS_GLACIER_LOGS_DIR, CLUSTER_LOCK);
+    await native_fs_utils.lock_and_run(fs_context, lock_path, async () => {
         const backend = Glacier.getBackend();
 
         if (
@@ -51,7 +55,8 @@ async function process_restores() {
 
 
         await run_glacier_restore(fs_context, backend);
-        await record_current_time(fs_context, Glacier.RESTORE_TIMESTAMP_FILE);
+        const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, Glacier.RESTORE_TIMESTAMP_FILE);
+        await record_current_time(fs_context, timestamp_file_path);
     });
 }
 
@@ -67,68 +72,27 @@ async function run_glacier_restore(fs_context, backend) {
 
 async function process_expiry() {
     const fs_context = native_fs_utils.get_process_fs_context();
-
-    await lock_and_run(fs_context, SCAN_LOCK, async () => {
+    const lock_path = path.join(config.NSFS_GLACIER_LOGS_DIR, SCAN_LOCK);
+    await native_fs_utils.lock_and_run(fs_context, lock_path, async () => {
         const backend = Glacier.getBackend();
+        const timestamp_file_path = path.join(config.NSFS_GLACIER_LOGS_DIR, Glacier.EXPIRY_TIMESTAMP_FILE);
         if (
             await backend.low_free_space() ||
             await is_desired_time(
-                    fs_context,
-                    new Date(),
-                    config.NSFS_GLACIER_EXPIRY_RUN_TIME,
-                    config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS,
-                    Glacier.EXPIRY_TIMESTAMP_FILE,
+                fs_context,
+                new Date(),
+                config.NSFS_GLACIER_EXPIRY_RUN_TIME,
+                config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS,
+                timestamp_file_path,
+                config.NSFS_GLACIER_EXPIRY_TZ
             )
         ) {
             await backend.expiry(fs_context);
-            await record_current_time(fs_context, Glacier.EXPIRY_TIMESTAMP_FILE);
+            await record_current_time(fs_context, timestamp_file_path);
         }
     });
 }
 
-/**
- * is_desired_time returns true if the given time matches with
- * the desired time or if 
- * @param {nb.NativeFSContext} fs_context 
- * @param {Date} current
- * @param {string} desire time in format 'hh:mm'
- * @param {number} delay_limit_mins
- * @param {string} timestamp_file 
- * @returns {Promise<boolean>}
- */
-async function is_desired_time(fs_context, current, desire, delay_limit_mins, timestamp_file) {
-    const [desired_hour, desired_min] = desire.split(':').map(Number);
-    if (
-        isNaN(desired_hour) ||
-        isNaN(desired_min) ||
-        (desired_hour < 0 || desired_hour >= 24) ||
-        (desired_min < 0 || desired_min >= 60)
-    ) {
-        throw new Error('invalid desired_time - must be hh:mm');
-    }
-
-    const min_time = get_tz_date(desired_hour, desired_min, 0, config.NSFS_GLACIER_EXPIRY_TZ);
-    const max_time = get_tz_date(desired_hour, desired_min + delay_limit_mins, 0, config.NSFS_GLACIER_EXPIRY_TZ);
-
-    if (current >= min_time && current <= max_time) {
-        try {
-            const { data } = await nb_native().fs.readFile(fs_context, path.join(config.NSFS_GLACIER_LOGS_DIR, timestamp_file));
-            const lastrun = new Date(data.toString());
-
-            // Last run should NOT be in this window
-            if (lastrun >= min_time && lastrun <= max_time) return false;
-        } catch (error) {
-            if (error.code === 'ENOENT') return true;
-            console.error('failed to read last run timestamp:', error, 'timestamp_file:', timestamp_file);
-
-            throw error;
-        }
-
-        return true;
-    }
-
-    return false;
-}
 
 /**
  * time_exceeded returns true if the time between last run recorded in the given
@@ -154,20 +118,6 @@ async function time_exceeded(fs_context, interval, timestamp_file) {
     return false;
 }
 
-/**
- * record_current_time stores the current timestamp in ISO format into
- * the given timestamp file
- * @param {nb.NativeFSContext} fs_context 
- * @param {string} timestamp_file 
- */
-async function record_current_time(fs_context, timestamp_file) {
-    await nb_native().fs.writeFile(
-        fs_context,
-        path.join(config.NSFS_GLACIER_LOGS_DIR, timestamp_file),
-        Buffer.from(new Date().toISOString()),
-    );
-}
-
 /**
  * migrate_log_exceeds_threshold returns true if the underlying backend
  * decides that the migrate log size has exceeded the given size threshold.
@@ -210,49 +160,6 @@ async function run_glacier_operation(fs_context, log_namespace, cb) {
     }
 }
 
-/**
- * @param {number} hours
- * @param {number} mins
- * @param {number} secs
- * @param {'UTC' | 'LOCAL'} tz
- * @returns {Date}
- */
-function get_tz_date(hours, mins, secs, tz) {
-    const date = new Date();
-
-    if (tz === 'UTC') {
-        date.setUTCHours(hours);
-        date.setUTCMinutes(hours);
-        date.setUTCSeconds(secs);
-        date.setUTCMilliseconds(0);
-    } else {
-        date.setHours(hours);
-        date.setMinutes(mins);
-        date.setSeconds(secs);
-        date.setMilliseconds(0);
-    }
-
-    return date;
-}
-
-/**
- * lock_and_run acquires a flock and calls the given callback after
- * acquiring the lock
- * @param {nb.NativeFSContext} fs_context 
- * @param {string} lockfilename
- * @param {Function} cb 
- */
-async function lock_and_run(fs_context, lockfilename, cb) {
-    const lockfd = await nb_native().fs.open(fs_context, path.join(config.NSFS_GLACIER_LOGS_DIR, lockfilename), 'w');
-
-    try {
-        await lockfd.fcntllock(fs_context, 'EXCLUSIVE');
-        await cb();
-    } finally {
-        await lockfd.close(fs_context);
-    }
-}
-
 /**
  * prepare_galcier_fs_context returns a shallow copy of given
  * fs_context with backend set to 'GPFS'.