Merge pull request #1157 from input-output-hk/chain-not-moving-up

Add warning if blockchain is not moving up

Author: NicolasDP

doc/quickstart/02_passive_node.md

@@ -84,6 +84,9 @@ Description of the fields:
       this node should maintain.
 - `explorer`: (optional) Explorer settings
     - `enabled`: True or false
+- `no_blockchain_updates_warning_interval`: (optional, seconds) if no new blocks
+  were received after this period of time, the node will start sending you
+  warnings in the logs.
 
 [multiaddr]: https://github.com/multiformats/multiaddr
 

jormungandr/src/blockchain_stuck_notifier.rs

@@ -0,0 +1,54 @@
+use crate::{blockchain, utils::task::TokioServiceInfo};
+use chain_time::{
+    era::{EpochPosition, EpochSlotOffset},
+    Epoch,
+};
+use futures::prelude::*;
+use std::time::{Duration, SystemTime};
+use tokio::timer::Interval;
+
+pub fn check_last_block_time(
+    service_info: TokioServiceInfo,
+    blockchain_tip: blockchain::Tip,
+    check_interval: Duration,
+) -> impl Future<Item = (), Error = ()> {
+    let logger = service_info.logger().clone();
+    let err_logger = logger.clone();
+
+    // those are different values, because check_interval can be big
+    // (30 minutes) and the notification may remain unseen
+    let check_period = check_interval;
+    let notification_period = Duration::from_secs(60);
+
+    Interval::new_interval(notification_period)
+        .map_err(move |e| error!(err_logger, "timer error: {}", e))
+        .and_then(move |_| blockchain_tip.get_ref())
+        .for_each(move |tip| {
+            let tip_date = tip.block_date();
+            let slot = tip
+                .epoch_leadership_schedule()
+                .era()
+                .from_era_to_slot(EpochPosition {
+                    epoch: Epoch(tip_date.epoch),
+                    slot: EpochSlotOffset(tip_date.slot_id),
+                });
+            let tip_time = tip.time_frame().slot_to_systemtime(slot).ok_or_else(|| {
+                error!(logger, "cannot convert the block tip date to system time");
+            })?;
+            let period_since_last_block =
+                SystemTime::now().duration_since(tip_time).map_err(|e| {
+                    error!(
+                        logger,
+                        "cannot compute the time passed since the last block: {}", e
+                    );
+                })?;
+            if period_since_last_block > check_period {
+                warn!(
+                    logger,
+                    "blockchain is not moving up, the last block was {} seconds ago",
+                    period_since_last_block.as_secs()
+                );
+            }
+            Ok(())
+        })
+}

jormungandr/src/main.rs

@@ -73,6 +73,7 @@ use std::time::Duration;
 
 pub mod blockcfg;
 pub mod blockchain;
+pub mod blockchain_stuck_notifier;
 pub mod client;
 pub mod explorer;
 pub mod fragment;
@@ -281,7 +282,7 @@ fn start_services(bootstrapped_node: BootstrappedNode) -> Result<(), start_up::E
         let full_context = rest::FullContext {
             stats_counter,
             blockchain,
-            blockchain_tip,
+            blockchain_tip: blockchain_tip.clone(),
             network_task: network_msgbox,
             transaction_task: fragment_msgbox,
             logs: pool_logs,
@@ -293,6 +294,22 @@ fn start_services(bootstrapped_node: BootstrappedNode) -> Result<(), start_up::E
         rest_context.set_node_state(NodeState::Running);
     };
 
+    {
+        let blockchain_tip = blockchain_tip.clone();
+        let no_blockchain_updates_warning_interval = bootstrapped_node
+            .settings
+            .no_blockchain_updates_warning_interval
+            .clone();
+
+        services.spawn_future("blockchain_stuck_notifier", move |info| {
+            blockchain_stuck_notifier::check_last_block_time(
+                info,
+                blockchain_tip,
+                no_blockchain_updates_warning_interval,
+            )
+        });
+    }
+
     services.wait_any_finished();
     info!(bootstrapped_node.logger, "Shutting down node");
     Ok(())

jormungandr/src/settings/start/config.rs

@@ -30,6 +30,10 @@ pub struct Config {
     pub p2p: P2pConfig,
 
     pub explorer: Option<Explorer>,
+
+    /// the time interval with no blockchain updates after which alerts are thrown
+    #[serde(default)]
+    pub no_blockchain_updates_warning_interval: Option<Duration>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]

jormungandr/src/settings/start/mod.rs

@@ -14,6 +14,7 @@ use std::{fs::File, path::PathBuf};
 const DEFAULT_FILTER_LEVEL: FilterLevel = FilterLevel::Info;
 const DEFAULT_LOG_FORMAT: LogFormat = LogFormat::Plain;
 const DEFAULT_LOG_OUTPUT: LogOutput = LogOutput::Stderr;
+const DEFAULT_NO_BLOCKCHAIN_UPDATES_WARNING_INTERVAL: u64 = 1800; // 30 min
 
 custom_error! {pub Error
    ConfigIo { source: std::io::Error } = "Cannot read the node configuration file: {source}",
@@ -34,6 +35,7 @@ pub struct Settings {
     pub mempool: Mempool,
     pub leadership: Leadership,
     pub explorer: bool,
+    pub no_blockchain_updates_warning_interval: std::time::Duration,
 }
 
 pub struct RawSettings {
@@ -173,6 +175,13 @@ impl RawSettings {
                 .as_ref()
                 .map_or(Leadership::default(), |cfg| cfg.leadership.clone()),
             explorer,
+            no_blockchain_updates_warning_interval: config
+                .as_ref()
+                .and_then(|config| config.no_blockchain_updates_warning_interval.clone())
+                .map(|d| d.into())
+                .unwrap_or(std::time::Duration::from_secs(
+                    DEFAULT_NO_BLOCKCHAIN_UPDATES_WARNING_INTERVAL,
+                )),
         })
     }
 }