v1.14: adds metrics tracking deduper saturations (backport of #30779) (#30852)

adds metrics tracking deduper saturations (#30779)

(cherry picked from commit 4b595eb)

Co-authored-by: behzad nouri <behzadnouri@gmail.com>

Author: mergify[bot]

core/src/shred_fetch_stage.rs

@@ -63,7 +63,9 @@ impl ShredFetchStage {
         let mut stats = ShredFetchStats::default();
 
         for mut packet_batch in recvr {
-            deduper.maybe_reset(&mut rng, DEDUPER_FALSE_POSITIVE_RATE, DEDUPER_RESET_CYCLE);
+            if deduper.maybe_reset(&mut rng, DEDUPER_FALSE_POSITIVE_RATE, DEDUPER_RESET_CYCLE) {
+                stats.num_deduper_saturations += 1;
+            }
             if last_updated.elapsed().as_millis() as u64 > DEFAULT_MS_PER_SLOT {
                 last_updated = Instant::now();
                 {

core/src/sigverify_stage.rs

@@ -80,6 +80,7 @@ struct SigVerifierStats {
     dedup_packets_pp_us_hist: histogram::Histogram, // per-packet time to call verify_batch
     batches_hist: histogram::Histogram,         // number of packet batches per verify call
     packets_hist: histogram::Histogram,         // number of packets per verify call
+    num_deduper_saturations: usize,
     total_batches: usize,
     total_packets: usize,
     total_dedup: usize,
@@ -196,6 +197,7 @@ impl SigVerifierStats {
             ("packets_min", self.packets_hist.minimum().unwrap_or(0), i64),
             ("packets_max", self.packets_hist.maximum().unwrap_or(0), i64),
             ("packets_mean", self.packets_hist.mean().unwrap_or(0), i64),
+            ("num_deduper_saturations", self.num_deduper_saturations, i64),
             ("total_batches", self.total_batches, i64),
             ("total_packets", self.total_packets, i64),
             ("total_dedup", self.total_dedup, i64),
@@ -418,7 +420,9 @@ impl SigVerifyStage {
                 let mut rng = rand::thread_rng();
                 let mut deduper = Deduper::<2>::new(&mut rng, DEDUPER_NUM_BITS);
                 loop {
-                    deduper.maybe_reset(&mut rng, DEDUPER_FALSE_POSITIVE_RATE, MAX_DEDUPER_AGE);
+                    if deduper.maybe_reset(&mut rng, DEDUPER_FALSE_POSITIVE_RATE, MAX_DEDUPER_AGE) {
+                        stats.num_deduper_saturations += 1;
+                    }
                     if let Err(e) =
                         Self::verifier(&deduper, &packet_receiver, &mut verifier, &mut stats)
                     {

ledger/src/shred/stats.rs

@@ -32,6 +32,7 @@ pub struct ProcessShredsStats {
 pub struct ShredFetchStats {
     pub index_overrun: usize,
     pub shred_count: usize,
+    pub num_deduper_saturations: usize,
     pub(crate) num_shreds_merkle_code: usize,
     pub(crate) num_shreds_merkle_data: usize,
     pub ping_count: usize,
@@ -117,6 +118,7 @@ impl ShredFetchStats {
             name,
             ("index_overrun", self.index_overrun, i64),
             ("shred_count", self.shred_count, i64),
+            ("num_deduper_saturations", self.num_deduper_saturations, i64),
             ("num_shreds_merkle_code", self.num_shreds_merkle_code, i64),
             ("num_shreds_merkle_data", self.num_shreds_merkle_data, i64),
             ("ping_count", self.ping_count, i64),