bucket notifications - add notif storage test to health (#8850)

Signed-off-by: Amit Prinz Setter <alphaprinz@gmail.com>

Author: alphaprinz

config.js

@@ -723,7 +723,7 @@ config.NOTIFICATION_LOG_NS = 'notification_logging';
 config.NOTIFICATION_LOG_DIR = process.env.NOTIFICATION_LOG_DIR;
 config.NOTIFICATION_BATCH = process.env.BATCH || 10;
 config.NOTIFICATION_REQ_PER_SPACE_CHECK = process.env.NOTIFICATION_REQ_PER_SPACE_CHECK || 0;
-config.NOTIFICATION_SPACE_CHECK_THRESHOLD = parseInt(process.env.NOTIFICATION_SPACE_CHECK_THRESHOLD, 10) || 0.1;
+config.NOTIFICATION_SPACE_CHECK_THRESHOLD = parseFloat(process.env.NOTIFICATION_SPACE_CHECK_THRESHOLD) || 0.2;
 
 ///////////////////////////
 //      KEY ROTATOR      //

docs/NooBaaNonContainerized/Health.md

@@ -55,7 +55,7 @@ The `health` command is used to analyze NooBaa health with customizable options.
 
 ```sh
 noobaa-cli diagnose health [--deployment_type][--https_port]
-[--all_account_details][--all_bucket_details][--config_root][--debug]
+[--all_account_details][--all_bucket_details][--all_connection_details][--notif_storage_threshold][--config_root][--debug]
 ```
 ### Flags -
 
@@ -84,6 +84,11 @@ noobaa-cli diagnose health [--deployment_type][--https_port]
     - Default: false
     - Description: Indicates if health output should contain connection test result.
 
+- `notif_storage_threshold`
+    - Type: Boolean
+    - Default: false
+    - Description: Whether health ouput should check if notification storage FS is below threshold.
+
 - `config_root`
     - Type: String
     - Description: Indicates the config directory (default config.NSFS_NC_DEFAULT_CONF_DIR). config_root flag should be used only for dev/tests envs.
@@ -269,6 +274,11 @@ Output:
         }
       ]
     },
+    "notif_storage_threshold_details": {
+      "threshold": 0.2,
+      "ratio": 0.9,
+      "result": "above threshold"
+    }
     "config_directory": {
       "phase": "CONFIG_DIR_UNLOCKED",
       "config_dir_version": "1.0.0",

src/endpoint/s3/s3_bucket_logging.js

@@ -6,7 +6,7 @@ const http_utils = require('../../util/http_utils');
 const dgram = require('node:dgram');
 const { Buffer } = require('node:buffer');
 const config = require('../../../config');
-const {compose_notification_req, check_notif_relevant, check_free_space} = require('../../util/notifications_util');
+const {compose_notification_req, check_notif_relevant, check_free_space_if_needed} = require('../../util/notifications_util');
 
 async function send_bucket_op_logs(req, res) {
     if (req.params && req.params.bucket &&
@@ -44,7 +44,7 @@ async function send_bucket_op_logs(req, res) {
                         buffer: JSON.stringify(notif)
                     });
 
-                    check_free_space(req);
+                    check_free_space_if_needed(req);
                 }
             }
         }

src/manage_nsfs/health.js

@@ -103,6 +103,7 @@ class NSFSHealth {
         this.all_account_details = options.all_account_details;
         this.all_bucket_details = options.all_bucket_details;
         this.all_connection_details = options.all_connection_details;
+        this.notif_storage_threshold = options.notif_storage_threshold;
         this.config_fs = options.config_fs;
     }
 
@@ -131,13 +132,15 @@ class NSFSHealth {
         let bucket_details;
         let account_details;
         let connection_details;
+        let notif_storage_threshold_details;
         const endpoint_response_code = (endpoint_state && endpoint_state.response?.response_code) || 'UNKNOWN_ERROR';
         const health_check_params = { service_status, pid, endpoint_response_code, config_directory_status };
         const service_health = this._calc_health_status(health_check_params);
         const error_code = this.get_error_code(health_check_params);
         if (this.all_bucket_details) bucket_details = await this.get_bucket_status();
         if (this.all_account_details) account_details = await this.get_account_status();
         if (this.all_connection_details) connection_details = await this.get_connection_status();
+        if (this.notif_storage_threshold) notif_storage_threshold_details = this.get_notif_storage_threshold_status();
         const health = {
             service_name: NOOBAA_SERVICE_NAME,
             status: service_health,
@@ -160,7 +163,8 @@ class NSFSHealth {
                     valid_buckets: bucket_details === undefined ? undefined : bucket_details.valid_storages,
                     error_type: health_errors_tyes.PERSISTENT,
                 },
-                connections_status: connection_details
+                connections_status: connection_details,
+                notif_storage_threshold_details
             }
         };
         if (!this.all_account_details) delete health.checks.accounts_status;
@@ -428,6 +432,20 @@ class NSFSHealth {
         };
     }
 
+    get_notif_storage_threshold_status() {
+        const check = notifications_util.check_free_space();
+        const res = {
+            threshold: config.NOTIFICATION_SPACE_CHECK_THRESHOLD,
+            ratio: check.ratio
+        };
+        if (check.below) {
+            res.result = 'below threshold';
+        } else {
+            res.result = 'above threshold';
+        }
+        return res;
+    }
+
     /**
      * get_config_file_data_or_error_object return an object containing config_data or err_obj if error occurred
      * @param {string} type
@@ -594,9 +612,11 @@ async function get_health_status(argv, config_fs) {
         const all_account_details = get_boolean_or_string_value(argv.all_account_details);
         const all_bucket_details = get_boolean_or_string_value(argv.all_bucket_details);
         const all_connection_details = get_boolean_or_string_value(argv.all_connection_details);
+        const notif_storage_threshold = get_boolean_or_string_value(argv.notif_storage_threshold);
 
         if (deployment_type === 'nc') {
-            const health = new NSFSHealth({ https_port, all_account_details, all_bucket_details, all_connection_details, config_fs });
+            const health = new NSFSHealth({ https_port,
+                all_account_details, all_bucket_details, all_connection_details, notif_storage_threshold, config_fs });
             const health_status = await health.nc_nsfs_health();
             write_stdout_response(ManageCLIResponse.HealthStatus, health_status);
         } else {

src/manage_nsfs/manage_nsfs_constants.js

@@ -75,7 +75,7 @@ const VALID_OPTIONS_GLACIER = {
 };
 
 const VALID_OPTIONS_DIAGNOSE = {
-    'health': new Set([ 'https_port', 'deployment_type', 'all_account_details', 'all_bucket_details', 'all_connection_details', ...CLI_MUTUAL_OPTIONS]),
+    'health': new Set([ 'https_port', 'deployment_type', 'all_account_details', 'all_bucket_details', 'all_connection_details', 'notif_storage_threshold', ...CLI_MUTUAL_OPTIONS]),
     'gather-logs': new Set([ CONFIG_ROOT_FLAG]),
     'metrics': new Set([CONFIG_ROOT_FLAG])
 };
@@ -147,6 +147,7 @@ const OPTION_TYPE = {
     all_account_details: 'boolean',
     all_bucket_details: 'boolean',
     all_connection_details: 'boolean',
+    notif_storage_threshold: 'boolean',
     https_port: 'number',
     debug: 'number',
     // upgrade options

src/test/unit_tests/test_nc_health.js

@@ -233,6 +233,16 @@ mocha.describe('nsfs nc health', function() {
             assert.strictEqual(health_status.checks.connections_status.invalid_storages[0].name, connection_from_file.name);
         });
 
+        mocha.it('health should test notification storage', async function() {
+            Health.notif_storage_threshold = true;
+            config.NOTIFICATION_LOG_DIR = TMP_PATH;
+            const health_status = await Health.nc_nsfs_health();
+            Health.notif_storage_limit = false;
+
+            assert.strictEqual(health_status.checks.notif_storage_threshold_details.result, 'above threshold');
+            assert.strictEqual(health_status.checks.notif_storage_threshold_details.threshold, config.NOTIFICATION_SPACE_CHECK_THRESHOLD);
+        });
+
         mocha.it('NooBaa service is inactive', async function() {
             set_mock_functions(Health, {
                 get_service_state: [{ service_status: 'inactive', pid: 0 }],

src/util/notifications_util.js

@@ -564,7 +564,7 @@ function get_notification_logger(locking, namespace, poll_interval) {
 }
 
 //If space check is configures, create an event in case free space is below threshold.
-function check_free_space(req) {
+function check_free_space_if_needed(req) {
     if (!req.object_sdk.nsfs_config_root || !config.NOTIFICATION_REQ_PER_SPACE_CHECK) {
         //free space check is disabled. nothing to do.
         return;
@@ -577,13 +577,23 @@ function check_free_space(req) {
         req.notification_logger.writes_counter = 0;
         const fs_stat = fs.statfsSync(config.NOTIFICATION_LOG_DIR);
         //is the ratio of available blocks less than the configures threshold?
-        if (fs_stat.bavail / fs_stat.blocks < config.NOTIFICATION_SPACE_CHECK_THRESHOLD) {
+        if (check_free_space().below) {
             //yes. raise an event.
             new NoobaaEvent(NoobaaEvent.NOTIFICATION_LOW_SPACE).create_event(null, {fs_stat});
         }
     }
 }
 
+function check_free_space() {
+    const fs_stat = fs.statfsSync(config.NOTIFICATION_LOG_DIR);
+    const ratio = fs_stat.bavail / fs_stat.blocks;
+    //is the ratio of available blocks less than the configures threshold?
+    return {
+        below: ratio < config.NOTIFICATION_SPACE_CHECK_THRESHOLD,
+        ratio
+    };
+}
+
 /**
  * add_connect_file Creates a new connection file from the given content.
  * If content has an auth field in it's request_options_object, it is encrypted.
@@ -649,4 +659,5 @@ exports.get_notification_logger = get_notification_logger;
 exports.add_connect_file = add_connect_file;
 exports.update_connect_file = update_connect_file;
 exports.check_free_space = check_free_space;
+exports.check_free_space_if_needed = check_free_space_if_needed;
 exports.OP_TO_EVENT = OP_TO_EVENT;