Merge pull request #9095 from achouhan09/repl_met

Added metrices to show the replication status per bucket

Author: achouhan09

src/server/analytic_services/prometheus_reports/noobaa_core_report.js

@@ -382,6 +382,27 @@ const NOOBAA_CORE_METRICS = js_utils.deep_freeze([{
             help: 'Number of error objects replication_id in last replication cycle',
             labelNames: ['replication_id']
         }
+    }, {
+        type: 'Gauge',
+        name: 'bucket_last_cycle_total_objects_num',
+        configuration: {
+            help: 'Total number of objects scanned per bucket in last replication cycle',
+            labelNames: ['bucket_name']
+        }
+    }, {
+        type: 'Gauge',
+        name: 'bucket_last_cycle_replicated_objects_num',
+        configuration: {
+            help: 'Number of objects replicated per bucket in last replication cycle',
+            labelNames: ['bucket_name']
+        }
+    }, {
+        type: 'Gauge',
+        name: 'bucket_last_cycle_error_objects_num',
+        configuration: {
+            help: 'Number of objects failed to replicate per bucket in last replication cycle',
+            labelNames: ['bucket_name']
+        }
     }, {
         type: 'Gauge',
         name: 'bucket_used_bytes',
@@ -606,9 +627,12 @@ class NooBaaCoreReport extends BasePrometheusReport {
     set_replication_status(repl_info) {
         if (!this._metrics) return;
         const replication_id = repl_info.replication_id;
+        const bucket_name = repl_info.bucket_name;
         delete this._metrics.replication_status.hashMap[String(repl_info.replication_id)];
         this._metrics.replication_status.set(_.omit(repl_info, ['last_cycle_writes_size',
-            'last_cycle_writes_num', 'last_cycle_error_writes_size', 'last_cycle_error_writes_num'
+            'last_cycle_writes_num', 'last_cycle_error_writes_size', 'last_cycle_error_writes_num',
+            'bucket_last_cycle_total_objects_num', 'bucket_last_cycle_replicated_objects_num',
+            'bucket_last_cycle_error_objects_num'
         ]), Date.now());
 
         delete this._metrics.replication_last_cycle_writes_size.hashMap[String(repl_info.replication_id)];
@@ -622,6 +646,15 @@ class NooBaaCoreReport extends BasePrometheusReport {
 
         delete this._metrics.replication_last_cycle_error_writes_num.hashMap[String(repl_info.replication_id)];
         this._metrics.replication_last_cycle_error_writes_num.set({ replication_id }, repl_info.last_cycle_error_writes_num);
+
+        delete this._metrics.bucket_last_cycle_total_objects_num.hashMap[String(bucket_name)];
+        this._metrics.bucket_last_cycle_total_objects_num.set({ bucket_name }, repl_info.bucket_last_cycle_total_objects_num);
+
+        delete this._metrics.bucket_last_cycle_replicated_objects_num.hashMap[String(bucket_name)];
+        this._metrics.bucket_last_cycle_replicated_objects_num.set({ bucket_name }, repl_info.bucket_last_cycle_replicated_objects_num);
+
+        delete this._metrics.bucket_last_cycle_error_objects_num.hashMap[String(bucket_name)];
+        this._metrics.bucket_last_cycle_error_objects_num.set({ bucket_name }, repl_info.bucket_last_cycle_error_objects_num);
     }
 }
 

src/server/bg_services/replication_scanner.js

@@ -86,6 +86,7 @@ class ReplicationScanner {
                 for_replication: config.BUCKET_DIFF_FOR_REPLICATION
             });
             dbg.log1(`scan:: cur_src_cont_token: ${cur_src_cont_token},cur_dst_cont_token: ${cur_dst_cont_token}`);
+
             const {
                 keys_diff_map,
                 first_bucket_cont_token: src_cont_token,
@@ -126,9 +127,10 @@ class ReplicationScanner {
 
             // update the prometheus metrics only if we have diff
             if (Object.keys(keys_diff_map).length) {
-                const replication_status = replication_utils.get_rule_status(rule.rule_id, src_cont_token, keys_diff_map, copy_res);
+                const {rule_status, bucket_status} = replication_utils.get_rule_and_bucket_status(
+                    rule.rule_id, src_cont_token, keys_diff_map, copy_res);
 
-                replication_utils.update_replication_prom_report(src_bucket.name, replication_id, replication_status);
+                replication_utils.update_replication_prom_report(src_bucket.name, replication_id, rule_status, bucket_status);
             }
         }));
     }

src/server/utils/replication_utils.js

@@ -16,6 +16,9 @@ const PARTIAL_SINGLE_BUCKET_REPLICATION_DEFAULTS = {
     last_cycle_writes_size: 0,
     last_cycle_error_writes_num: 0,
     last_cycle_error_writes_size: 0,
+    bucket_last_cycle_total_objects_num: 0,
+    bucket_last_cycle_replicated_objects_num: 0,
+    bucket_last_cycle_error_objects_num: 0,
 };
 
 //TODO: this function is not being used anymore, commenting out and keeping it as reference 
@@ -39,7 +42,7 @@ const PARTIAL_SINGLE_BUCKET_REPLICATION_DEFAULTS = {
 //     return false;
 // }
 
-function get_rule_status(rule, src_cont_token, keys_diff_map, copy_res) {
+function get_rule_and_bucket_status(rule, src_cont_token, keys_diff_map, copy_res) {
     const { num_keys_to_copy, num_bytes_to_copy } = Object.entries(keys_diff_map).reduce(
         (acc, [key, value]) => {
             acc.num_keys_to_copy += value.length;
@@ -51,22 +54,31 @@ function get_rule_status(rule, src_cont_token, keys_diff_map, copy_res) {
     const num_keys_moved = copy_res.num_of_objects;
     const num_bytes_moved = copy_res.size_of_objects;
 
-    const status = {
+    const rule_status = {
         last_cycle_rule_id: rule,
         last_cycle_writes_num: num_keys_moved,
         last_cycle_writes_size: num_bytes_moved,
         last_cycle_error_writes_num: num_keys_to_copy - num_keys_moved,
         last_cycle_error_writes_size: num_bytes_to_copy - num_bytes_moved,
     };
-    if (src_cont_token) status.last_cycle_src_cont_token = src_cont_token;
-    dbg.log1('get_rule_status: ', status);
-    return status;
+    if (src_cont_token) rule_status.last_cycle_src_cont_token = src_cont_token;
+    dbg.log1('get_rule_and_bucket_status:: rule_status: ', rule_status);
+
+    const bucket_status = {
+        bucket_last_cycle_total_objects_num: num_keys_to_copy,
+        bucket_last_cycle_replicated_objects_num: num_keys_moved,
+        bucket_last_cycle_error_objects_num: num_keys_to_copy - num_keys_moved,
+    };
+    dbg.log1('get_rule_and_bucket_status:: bucket_status: ', bucket_status);
+
+    return {rule_status, bucket_status};
 }
 
-function update_replication_prom_report(bucket_name, replication_policy_id, replication_status) {
+function update_replication_prom_report(bucket_name, replication_policy_id, rule_status, bucket_status) {
     const core_report = prom_reporting.get_core_report();
     const last_cycle_status = _.defaults({
-        ...replication_status,
+        ...rule_status,
+        ...bucket_status,
         bucket_name: bucket_name.unwrap(),
         replication_id: replication_policy_id
     }, PARTIAL_SINGLE_BUCKET_REPLICATION_DEFAULTS);
@@ -185,7 +197,7 @@ async function delete_objects(scanner_semaphore, client, bucket_name, keys) {
 }
 
 // EXPORTS
-exports.get_rule_status = get_rule_status;
+exports.get_rule_and_bucket_status = get_rule_and_bucket_status;
 exports.update_replication_prom_report = update_replication_prom_report;
 exports.get_object_md = get_object_md;
 exports.find_src_and_dst_buckets = find_src_and_dst_buckets;