NC | Lifecycle | Health

romayalon · romayalon · commit 5d8f505ed32c · 2025-03-25T14:33:58.000+02:00
Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> (cherry picked from commit b331051)
diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js
@@ -3,6 +3,7 @@
 
 const dbg = require('../util/debug_module')(__filename);
 const _ = require('lodash');
+const path = require('path');
 const P = require('../util/promise');
 const config = require('../../config');
 const os_util = require('../util/os_utils');
@@ -104,6 +105,7 @@ class NSFSHealth {
         this.all_bucket_details = options.all_bucket_details;
         this.all_connection_details = options.all_connection_details;
         this.notif_storage_threshold = options.notif_storage_threshold;
+        this.lifecycle = options.lifecycle;
         this.config_fs = options.config_fs;
     }
 
@@ -133,6 +135,7 @@ class NSFSHealth {
         let account_details;
         let connection_details;
         let notif_storage_threshold_details;
+        let latest_lifecycle_run_status;
         const endpoint_response_code = (endpoint_state && endpoint_state.response?.response_code) || 'UNKNOWN_ERROR';
         const health_check_params = { service_status, pid, endpoint_response_code, config_directory_status };
         const service_health = this._calc_health_status(health_check_params);
@@ -141,6 +144,8 @@ class NSFSHealth {
         if (this.all_account_details) account_details = await this.get_account_status();
         if (this.all_connection_details) connection_details = await this.get_connection_status();
         if (this.notif_storage_threshold) notif_storage_threshold_details = this.get_notif_storage_threshold_status();
+        if (this.lifecycle) latest_lifecycle_run_status = await this.get_lifecycle_health_status();
+
         const health = {
             service_name: NOOBAA_SERVICE_NAME,
             status: service_health,
@@ -164,7 +169,8 @@ class NSFSHealth {
                     error_type: health_errors_tyes.PERSISTENT,
                 },
                 connections_status: connection_details,
-                notif_storage_threshold_details
+                notif_storage_threshold_details,
+                latest_lifecycle_run_status
             }
         };
         if (!this.all_account_details) delete health.checks.accounts_status;
@@ -333,10 +339,10 @@ class NSFSHealth {
         };
     }
 
-    async validate_config_dir_exists(path, type) {
-        const config_root_type_exists = await this.config_fs.validate_config_dir_exists(path);
+    async validate_config_dir_exists(config_dir_path, type) {
+        const config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path);
         if (!config_root_type_exists) {
-            dbg.log1(`Config directory type - ${type} is missing, ${path}`);
+            dbg.log1(`Config directory type - ${type} is missing, ${config_dir_path}`);
             return {
                 invalid_storages: [],
                 valid_storages: []
@@ -446,6 +452,48 @@ class NSFSHealth {
         return res;
     }
 
+    /////////////////////////////
+    // LIFECYCLE HEALTH STATUS //
+    /////////////////////////////
+
+    /**
+     * get_lifecycle_health_status returns the lifecycle rules status based on the status of the latest lifecycle wroker run
+     * on the same host
+     * @returns {Promise<object>}
+     */
+    async get_lifecycle_health_status() {
+        const latest_lifecycle_run_status = await this.get_latest_lifecycle_run_status({ silent_if_missing: true });
+        if (!latest_lifecycle_run_status) return {};
+        return {
+            total_stats: latest_lifecycle_run_status.total_stats,
+            lifecycle_run_times: latest_lifecycle_run_status.lifecycle_run_times,
+            errors: latest_lifecycle_run_status.errors
+        };
+    }
+
+
+     /**
+     * get_latest_lifecycle_run_status returns the latest lifecycle run status
+     * latest run can be found by maxing the lifecycle log entry names, log entry name is the lifecycle_run_{timestamp}.json of the run
+     * @params {{silent_if_missing: boolean}} options
+     * @returns {Promise<object | undefined >}
+     */
+    async get_latest_lifecycle_run_status(options) {
+        const { silent_if_missing = false } = options;
+        try {
+            const lifecycle_log_entries = await nb_native().fs.readdir(this.config_fs.fs_context, config.NC_LIFECYCLE_LOGS_DIR);
+            const latest_lifecycle_run = _.maxBy(lifecycle_log_entries, entry => entry.name);
+            const latest_lifecycle_run_status_path = path.join(config.NC_LIFECYCLE_LOGS_DIR, latest_lifecycle_run.name);
+            const latest_lifecycle_run_status = await this.config_fs.get_config_data(latest_lifecycle_run_status_path, options);
+            return latest_lifecycle_run_status;
+        } catch (err) {
+            if (err.code === 'ENOENT' && silent_if_missing) {
+                return;
+            }
+            throw err;
+        }
+    }
+
     /**
      * get_config_file_data_or_error_object return an object containing config_data or err_obj if error occurred
      * @param {string} type
@@ -613,10 +661,11 @@ async function get_health_status(argv, config_fs) {
         const all_bucket_details = get_boolean_or_string_value(argv.all_bucket_details);
         const all_connection_details = get_boolean_or_string_value(argv.all_connection_details);
         const notif_storage_threshold = get_boolean_or_string_value(argv.notif_storage_threshold);
+        const lifecycle = get_boolean_or_string_value(argv.lifecycle);
 
         if (deployment_type === 'nc') {
             const health = new NSFSHealth({ https_port,
-                all_account_details, all_bucket_details, all_connection_details, notif_storage_threshold, config_fs });
+                all_account_details, all_bucket_details, all_connection_details, notif_storage_threshold, lifecycle, config_fs });
             const health_status = await health.nc_nsfs_health();
             write_stdout_response(ManageCLIResponse.HealthStatus, health_status);
         } else {
diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js
@@ -75,7 +75,7 @@ const VALID_OPTIONS_GLACIER = {
 };
 
 const VALID_OPTIONS_DIAGNOSE = {
-    'health': new Set([ 'https_port', 'deployment_type', 'all_account_details', 'all_bucket_details', 'all_connection_details', 'notif_storage_threshold', ...CLI_MUTUAL_OPTIONS]),
+    'health': new Set([ 'https_port', 'deployment_type', 'all_account_details', 'all_bucket_details', 'all_connection_details', 'notif_storage_threshold', 'lifecycle', ...CLI_MUTUAL_OPTIONS]),
     'gather-logs': new Set([ CONFIG_ROOT_FLAG]),
     'metrics': new Set([CONFIG_ROOT_FLAG])
 };
@@ -150,6 +150,7 @@ const OPTION_TYPE = {
     notif_storage_threshold: 'boolean',
     https_port: 'number',
     debug: 'number',
+    lifecycle: 'boolean',
     // upgrade options
     expected_version: 'string',
     expected_hosts: 'string',
diff --git a/src/manage_nsfs/manage_nsfs_help_utils.js b/src/manage_nsfs/manage_nsfs_help_utils.js
@@ -371,6 +371,7 @@ Flags:
     --all_account_details    <boolean>       (optional)          Set a flag for returning all account details.
     --all_bucket_details     <boolean>       (optional)          Set a flag for returning all bucket details.
     --all_connection_details <boolean>       (optional)          Set a flag for returning all connection details.
+    --lifecycle              <boolean>       (optional)          Set a flag for returning lifecycle details on the current host.
     --debug                  <number>        (optional)          Use for increasing the log verbosity of health cli commands.
     --config_root            <string>        (optional)          Set Configuration files path for Noobaa standalon NSFS. (default config.NSFS_NC_DEFAULT_CONF_DIR)
 
diff --git a/src/test/unit_tests/test_nc_health.js b/src/test/unit_tests/test_nc_health.js
@@ -4,6 +4,7 @@
 'use strict';
 
 const path = require('path');
+const _ = require('lodash');
 const mocha = require('mocha');
 const sinon = require('sinon');
 const assert = require('assert');
@@ -19,7 +20,7 @@ const NSFSHealth = require('../../manage_nsfs/health').NSFSHealth;
 const { get_process_fs_context } = require('../../util/native_fs_utils');
 const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors');
 const { TYPES, DIAGNOSE_ACTIONS, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants');
-const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli } = require('../system_tests/test_utils');
+const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli, TEST_TIMEOUT } = require('../system_tests/test_utils');
 const { CONFIG_DIR_PHASES } = require('../../sdk/config_fs');
 
 const tmp_fs_path = path.join(TMP_PATH, 'test_nc_health');
@@ -56,12 +57,13 @@ const connection_from_file = {
   name: "http_notif",
 };
 
-mocha.describe('nsfs nc health', function() {
+const config_root = path.join(tmp_fs_path, 'config_root_nsfs_health');
+const config_fs = new ConfigFS(config_root);
+const root_path = path.join(tmp_fs_path, 'root_path_nsfs_health/');
+const config_root_invalid = path.join(tmp_fs_path, 'config_root_nsfs_health_invalid');
+const tmp_lifecycle_logs_dir_path = path.join(TMP_PATH, 'test_lifecycle_logs');
 
-    const config_root = path.join(tmp_fs_path, 'config_root_nsfs_health');
-    const config_fs = new ConfigFS(config_root);
-    const root_path = path.join(tmp_fs_path, 'root_path_nsfs_health/');
-    const config_root_invalid = path.join(tmp_fs_path, 'config_root_nsfs_health_invalid');
+mocha.describe('nsfs nc health', function() {
     let Health;
 
     mocha.before(async () => {
@@ -693,6 +695,85 @@ mocha.describe('nsfs nc health', function() {
     });
 });
 
+
+mocha.describe('health - lifecycle', function() {
+    const Health = new NSFSHealth({ config_root, config_fs, lifecycle: true });
+    const orig_lifecycle_logs_dir = config.NC_LIFECYCLE_LOGS_DIR;
+
+    mocha.before(async () => {
+        await fs_utils.create_fresh_path(config_root);
+        config.NC_LIFECYCLE_LOGS_DIR = tmp_lifecycle_logs_dir_path;
+    });
+
+    mocha.after(async () => {
+        fs_utils.folder_delete(config_root);
+        config.NC_LIFECYCLE_LOGS_DIR = orig_lifecycle_logs_dir;
+    });
+
+    mocha.beforeEach(async () => {
+        await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path }));
+    });
+
+    mocha.afterEach(async () => {
+        fs_utils.folder_delete(tmp_lifecycle_logs_dir_path);
+        await config_fs.delete_config_json_file();
+    });
+
+    mocha.it('Health lifecycle - lifecycle worker never ran on host', async function() {
+        try {
+            await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path);
+            assert.fail('tmp_lifecycle_logs_dir_path should not exist');
+        } catch (err) {
+            assert.equal(err.code, 'ENOENT');
+        }
+        const health_status = await Health.nc_nsfs_health();
+        const empty_lifecycle_health_status = {};
+        assert_lifecycle_status(health_status, empty_lifecycle_health_status, false);
+    });
+
+    mocha.it('Health lifecycle - after 1 run of the lifecycle worker', async function() {
+        await exec_manage_cli(TYPES.LIFECYCLE, '', { config_root, disable_service_validation: true, disable_runtime_validation: true }, true);
+        const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path);
+        assert.strictEqual(lifecycle_log_entries.length, 1);
+        const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name);
+        const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true });
+        const health_status = await Health.nc_nsfs_health();
+        assert_lifecycle_status(health_status, lifecycle_log_json, false);
+    });
+
+    mocha.it('Health lifecycle - should report on timeout error', async function() {
+        await config_fs.update_config_json_file(JSON.stringify({
+            NC_LIFECYCLE_TIMEOUT_MS: 1,
+            NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path
+        }));
+        await exec_manage_cli(TYPES.LIFECYCLE, '', { config_root, disable_service_validation: true, disable_runtime_validation: true }, true);
+        const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path);
+        assert.strictEqual(lifecycle_log_entries.length, 1);
+        const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name);
+        const lifecycle_log_json = await config_fs.get_config_data(log_file_path, {silent_if_missing: true});
+        const health_status = await Health.nc_nsfs_health();
+        assert_lifecycle_status(health_status, lifecycle_log_json, true);
+    });
+
+    mocha.it('Health lifecycle - run lifecycle 3 times - should report on latest run', async function() {
+        this.timeout(TEST_TIMEOUT);// eslint-disable-line no-invalid-this
+        let latest_lifecycle;
+        for (let i = 0; i < 3; i++) {
+            latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { config_root, disable_service_validation: true, disable_runtime_validation: true }, true);
+        }
+        const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle);
+        const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path);
+        assert.strictEqual(lifecycle_log_entries.length, 3);
+        const latest_log_file_path = _.maxBy(lifecycle_log_entries, entry => entry.name);
+        const is_latest = latest_log_file_path.name.endsWith(parsed_res_latest_lifecycle.response.reply.lifecycle_run_times.run_lifecycle_start_time + '.json');
+        assert.equal(is_latest, true);
+        const log_file_path = path.join(tmp_lifecycle_logs_dir_path, latest_log_file_path.name);
+        const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true });
+        const health_status = await Health.nc_nsfs_health();
+        assert_lifecycle_status(health_status, lifecycle_log_json, false);
+    });
+});
+
 /**
  * assert_config_dir_status asserts config directory status
  * @param {Object} health_status 
@@ -723,6 +804,21 @@ function assert_upgrade_status(actual_upgrade_status, expected_upgrade_status) {
     assert.deepStrictEqual(actual_upgrade_status_res, expected_upgrade_status_res);
 }
 
+/**
+ * assert_lifecycle_status asserts the lifecycle status
+ * @param {Object} health_status 
+ * @param {Object} lifecycle_log_json 
+ * @param {Boolean} is_error_expected
+ * @returns {Void}
+ */
+function assert_lifecycle_status(health_status, lifecycle_log_json, is_error_expected = false) {
+    assert.deepStrictEqual(health_status.checks.latest_lifecycle_run_status.total_stats, lifecycle_log_json.total_stats);
+    assert.deepStrictEqual(health_status.checks.latest_lifecycle_run_status.lifecycle_run_times,
+        lifecycle_log_json.lifecycle_run_times);
+    const err_value = is_error_expected ? lifecycle_log_json.errors : undefined;
+    assert.deepStrictEqual(health_status.checks.latest_lifecycle_run_status.errors, err_value);
+}
+
 /**
  * restore_health_if_needed restores health obj functions if needed
  * @param {*} health_obj 
@@ -740,7 +836,8 @@ function restore_health_if_needed(health_obj) {
  * the value is an array of responses by the order of their call
  * @param {Object} Health 
  * @param {{get_endpoint_response?: Object[], get_service_state?: Object[], 
- * get_system_config_file?: Object[], get_service_memory_usage?: Object[]}} mock_function_responses 
+ * get_system_config_file?: Object[], get_service_memory_usage?: Object[],
+ * get_lifecycle_health_status?: Object, get_latest_lifecycle_run_status?: Object}} mock_function_responses 
  */
 function set_mock_functions(Health, mock_function_responses) {
     for (const mock_function_name of Object.keys(mock_function_responses)) {
