feat: add monitoring for signer agreement state change reason, #5918

Author: fdefelici

stacks-signer/src/monitoring/mod.rs

@@ -20,6 +20,33 @@ mod prometheus;
 #[cfg(feature = "monitoring_prom")]
 mod server;
 
+/// Represent different state change reason on signer agreement protocol
+pub enum SignerAgreementStateChangeReason {
+    /// A new burn block has arrived
+    BurnBlockArrival,
+    /// A new stacks block has arrived
+    StacksBlockArrival,
+    /// A miner is inactive when it should be starting its tenure
+    InactiveMiner,
+    /// Signer agreement protocol version has been upgraded
+    ProtocalUpgrade,
+}
+
+impl std::fmt::Display for SignerAgreementStateChangeReason {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(
+            f,
+            "{}",
+            match self {
+                SignerAgreementStateChangeReason::BurnBlockArrival => "burn_block_arrival",
+                SignerAgreementStateChangeReason::StacksBlockArrival => "stacks_block_arrival",
+                SignerAgreementStateChangeReason::InactiveMiner => "inactive_miner",
+                SignerAgreementStateChangeReason::ProtocalUpgrade => "protocol_upgrade",
+            }
+        )
+    }
+}
+
 /// Actions for updating metrics
 #[cfg(feature = "monitoring_prom")]
 pub mod actions {
@@ -29,6 +56,7 @@ pub mod actions {
 
     use crate::config::GlobalConfig;
     use crate::monitoring::prometheus::*;
+    use crate::monitoring::SignerAgreementStateChangeReason;
     use crate::v0::signer_state::LocalStateMachine;
 
     /// Update stacks tip height gauge
@@ -108,6 +136,16 @@ pub mod actions {
             .replace(state);
     }
 
+    /// Increment signer agreement state change reason counter
+    pub fn increment_signer_agreement_state_change_reason(
+        reason: SignerAgreementStateChangeReason,
+    ) {
+        let label_value = reason.to_string();
+        SIGNER_AGREEMENT_STATE_CHANGE_REASONS
+            .with_label_values(&[&label_value])
+            .inc();
+    }
+
     /// Start serving monitoring metrics.
     /// This will only serve the metrics if the `monitoring_prom` feature is enabled.
     pub fn start_serving_monitoring_metrics(config: GlobalConfig) -> Result<(), String> {
@@ -131,6 +169,7 @@ pub mod actions {
     use blockstack_lib::chainstate::nakamoto::NakamotoBlock;
     use stacks_common::info;
 
+    use crate::monitoring::SignerAgreementStateChangeReason;
     use crate::v0::signer_state::LocalStateMachine;
     use crate::GlobalConfig;
 
@@ -179,6 +218,12 @@ pub mod actions {
     /// Record the current local state machine
     pub fn record_local_state(_state: LocalStateMachine) {}
 
+    /// Increment signer agreement state change reason counter
+    pub fn increment_signer_agreement_state_change_reason(
+        _reason: SignerAgreementStateChangeReason,
+    ) {
+    }
+
     /// Start serving monitoring metrics.
     /// This will only serve the metrics if the `monitoring_prom` feature is enabled.
     pub fn start_serving_monitoring_metrics(config: GlobalConfig) -> Result<(), String> {

stacks-signer/src/monitoring/prometheus.rs

@@ -79,6 +79,12 @@ lazy_static! {
         vec![0.005, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0]
     ), &[]).unwrap();
 
+    pub static ref SIGNER_AGREEMENT_STATE_CHANGE_REASONS: IntCounterVec = register_int_counter_vec!(
+        "stacks_signer_agreement_state_change_reasons",
+        "The number of state machine changes in signer agreement protocol. `reason` can be one of: 'burn_block_arrival', 'stacks_block_arrival', 'inactive_miner', 'protocol_upgrade'",
+        &["reason"]
+    ).unwrap();
+
     pub static ref SIGNER_LOCAL_STATE_MACHINE: Mutex<Option<LocalStateMachine>> = Mutex::new(None);
 }
 

stacks-signer/src/v0/signer_state.rs

@@ -271,6 +271,11 @@ impl LocalStateMachine {
                 "inactive_tenure_ch" => %inactive_tenure_ch,
                 "new_active_tenure_ch" => %new_active_tenure_ch
             );
+
+            crate::monitoring::actions::increment_signer_agreement_state_change_reason(
+                crate::monitoring::SignerAgreementStateChangeReason::InactiveMiner,
+            );
+
             Ok(())
         } else {
             warn!("Current miner timed out due to inactivity, but prior miner is not valid. Allowing current miner to continue");
@@ -393,6 +398,11 @@ impl LocalStateMachine {
         *parent_tenure_last_block = *block_id;
         *parent_tenure_last_block_height = height;
         *self = LocalStateMachine::Initialized(prior_state_machine);
+
+        crate::monitoring::actions::increment_signer_agreement_state_change_reason(
+            crate::monitoring::SignerAgreementStateChangeReason::StacksBlockArrival,
+        );
+
         Ok(())
     }
 
@@ -447,7 +457,7 @@ impl LocalStateMachine {
         // set self to uninitialized so that if this function errors,
         //  self is left as uninitialized.
         let prior_state = std::mem::replace(self, Self::Uninitialized);
-        let prior_state_machine = match prior_state {
+        let prior_state_machine = match prior_state.clone() {
             // if the local state machine was uninitialized, just initialize it
             LocalStateMachine::Uninitialized => Self::place_holder(),
             LocalStateMachine::Initialized(signer_state_machine) => signer_state_machine,
@@ -526,6 +536,12 @@ impl LocalStateMachine {
             active_signer_protocol_version: prior_state_machine.active_signer_protocol_version,
         });
 
+        if prior_state != *self {
+            crate::monitoring::actions::increment_signer_agreement_state_change_reason(
+                crate::monitoring::SignerAgreementStateChangeReason::BurnBlockArrival,
+            );
+        }
+
         Ok(())
     }
 }