Include most recent changes into subspace-customization branch (#41)

* Disable nodes in memory and hide list and map views

* Add the node count to the header

* Limit nodes in map view to 1300

* Sort nodes only when receive new best block (#11)

* Enable nodes list (#13)

* Show the stats view when nodes are disabled

* Clean up Header Tile styles

* Add node count to Header's shouldComponentUpdate (#19)

* Make space pledged visible (#20)

* add space pledged tile to Header

* update npm packages

update tsconfig

remove babelrc, babel presets as well as stable package

fix svg namespace tag syntax errors

fix reference error due to namespace and class component having same name

replace tslint with eslint

make eslint happier

update .nvmrc to 14

update node version to 14 in gh workflow

fix eslint warnings due to warnings treated as errors on CI (process.env.CI = true)

pretty fix

* format space pledged value

minor

minor

* only display space pledged for gemini II

* minor

* Update map (#21)

* Map: render node locations instead of individual nodes

* render city details

* Merge branch 'telemetry-frontend-stress-test' of github.com:subspace/substrate-telemetry into map-update

* Add graceful degradation screen for list view to redirect to polkadot apps (#25)

* add DISABLE_NODE_LIST env var

* add message on checking balance

* update message

* Remove unnecessary fields (#26)

* Remove some unnecessary fields and fix deserialization

* hide validator and implementation columns

minor

* Ignore broken test

Co-authored-by: i1i1 <vanyarybin1@live.ru>

* Deployment fixes (#27)

* Replace boolean value with number in docker-compose.yml

* bump docker to 14

* use createRoot instead of react-dom, remove service worker

* update browsers list in package.json

* Omit sending node data

* fix boolean to disable node list

* use window.process_env instead of process.env (#32)

* Adjust styles (#33)

* Omit more data related to node (#34)

* Move omiting data from env to cli

* Drain messages to locator if we skip node info

* Omit node data delta updates

* Add trait for feed message writer

* Skip serialization for node update

* Send updates every 10 seconds for top bar

* Supply period for updates from cli

* Update docker compose

* Show unique reward address count (#29)

* adjust styles for second row

* add tile for unique reward address count

* add uniq address count into state

* add tile for unique address count

* fetch uniq address from from subscan API

* setup minimal server

* fetch address count every 10s

* added cors related headers

* fetch uniq addr count from our backend instead of subscan

* hide uniq addr count component if value is not present

* adjust styles

* update header markup

* provide api url as env variable

* rename folder

* add service to docker-compose

* fetch metadata, remove polkadot.js api

* parse string values

* add dockerfile for metadata

update Dockerfile

update

debug logs

remove logs

* Reduce metadata queries (#37)

* Reduce metadata queries

* addressing PR comments

* retry if failed request

* Update frontend/src/App.tsx

Co-authored-by: Nazar Mokrynskyi <nazar@mokrynskyi.com>

Co-authored-by: Nazar Mokrynskyi <nazar@mokrynskyi.com>

* Add second local shard

* Use Math.log to calc dot size (#39)

* Use Math.log to calc dot size

* update comment

* pretty fix

Co-authored-by: dnieh <danielnieh@gmail.com>
Co-authored-by: i1i1 <vanyarybin1@live.ru>
Co-authored-by: Nazar Mokrynskyi <nazar@mokrynskyi.com>

Author: 4 people

.github/workflows/frontend.yml

@@ -25,7 +25,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [10.x, 12.x]
+        node-version: [14.x]
 
     steps:
     - uses: actions/checkout@v2

backend/common/src/node_message.rs

@@ -60,7 +60,6 @@ pub enum Payload {
     BlockImport(Block),
     NotifyFinalized(Finalized),
     AfgAuthoritySet(AfgAuthoritySet),
-    HwBench(NodeHwBench),
 }
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -94,14 +93,6 @@ pub struct AfgAuthoritySet {
     pub authority_set_id: Box<str>,
 }
 
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct NodeHwBench {
-    pub cpu_hashrate_score: u64,
-    pub memory_memcpy_score: u64,
-    pub disk_sequential_write_score: Option<u64>,
-    pub disk_random_write_score: Option<u64>,
-}
-
 impl Payload {
     pub fn best_block(&self) -> Option<&Block> {
         match self {
@@ -152,15 +143,10 @@ mod tests {
                 node: NodeDetails {
                     chain: "foo".into(),
                     name: "foo".into(),
-                    implementation: "foo".into(),
                     version: "foo".into(),
-                    target_arch: Some("x86_64".into()),
-                    target_os: Some("linux".into()),
-                    target_env: Some("env".into()),
                     validator: None,
                     network_id: ArrayString::new(),
                     startup_time: None,
-                    sysinfo: None,
                 },
             }),
         });

backend/common/src/node_types.rs

@@ -33,45 +33,10 @@ pub type NetworkId = ArrayString<64>;
 pub struct NodeDetails {
     pub chain: Box<str>,
     pub name: Box<str>,
-    pub implementation: Box<str>,
     pub version: Box<str>,
     pub validator: Option<Box<str>>,
     pub network_id: NetworkId,
     pub startup_time: Option<Box<str>>,
-    pub target_os: Option<Box<str>>,
-    pub target_arch: Option<Box<str>>,
-    pub target_env: Option<Box<str>>,
-    pub sysinfo: Option<NodeSysInfo>,
-}
-
-/// Hardware and software information for the node.
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct NodeSysInfo {
-    /// The exact CPU model.
-    pub cpu: Option<Box<str>>,
-    /// The total amount of memory, in bytes.
-    pub memory: Option<u64>,
-    /// The number of physical CPU cores.
-    pub core_count: Option<u32>,
-    /// The Linux kernel version.
-    pub linux_kernel: Option<Box<str>>,
-    /// The exact Linux distribution used.
-    pub linux_distro: Option<Box<str>>,
-    /// Whether the node's running under a virtual machine.
-    pub is_virtual_machine: Option<bool>,
-}
-
-/// Hardware benchmark results for the node.
-#[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct NodeHwBench {
-    /// The CPU speed, as measured in how many MB/s it can hash using the BLAKE2b-256 hash.
-    pub cpu_hashrate_score: u64,
-    /// Memory bandwidth in MB/s, calculated by measuring the throughput of `memcpy`.
-    pub memory_memcpy_score: u64,
-    /// Sequential disk write speed in MB/s.
-    pub disk_sequential_write_score: Option<u64>,
-    /// Random disk write speed in MB/s.
-    pub disk_random_write_score: Option<u64>,
 }
 
 /// A couple of node statistics.

backend/telemetry_core/src/aggregator/aggregator.rs

@@ -22,6 +22,7 @@ use futures::{future, Sink, SinkExt};
 use std::net::IpAddr;
 use std::sync::atomic::AtomicU64;
 use std::sync::Arc;
+use std::time::Duration;
 
 id_type! {
     /// A unique Id is assigned per websocket connection (or more accurately,
@@ -44,6 +45,8 @@ pub struct AggregatorOpts {
     /// How many nodes from third party chains are allowed to connect
     /// before we prevent connections from them.
     pub max_third_party_nodes: usize,
+    /// Send updates periodically
+    pub update_every: Option<Duration>,
 }
 
 struct AggregatorInternal {
@@ -61,25 +64,63 @@ struct AggregatorInternal {
 
 impl Aggregator {
     /// Spawn a new Aggregator. This connects to the telemetry backend
-    pub async fn spawn(opts: AggregatorOpts) -> anyhow::Result<Aggregator> {
+    pub async fn spawn(
+        AggregatorOpts {
+            denylist,
+            max_queue_len,
+            max_third_party_nodes,
+            update_every,
+        }: AggregatorOpts,
+    ) -> anyhow::Result<Aggregator> {
         let (tx_to_aggregator, rx_from_external) = flume::unbounded();
 
-        // Kick off a locator task to locate nodes, which hands back a channel to make location requests
-        let tx_to_locator =
-            find_location(tx_to_aggregator.clone().into_sink().with(|(node_id, msg)| {
-                future::ok::<_, flume::SendError<_>>(inner_loop::ToAggregator::FromFindLocation(
-                    node_id, msg,
-                ))
-            }));
-
-        // Handle any incoming messages in our handler loop:
-        tokio::spawn(Aggregator::handle_messages(
-            rx_from_external,
-            tx_to_locator,
-            opts.max_queue_len,
-            opts.denylist,
-            opts.max_third_party_nodes,
-        ));
+        match update_every {
+            None => {
+                // Kick off a locator task to locate nodes, which hands back a channel to make location requests
+                let tx_to_locator =
+                    find_location(tx_to_aggregator.clone().into_sink().with(|(node_id, msg)| {
+                        future::ok::<_, flume::SendError<_>>(
+                            inner_loop::ToAggregator::FromFindLocation(node_id, msg),
+                        )
+                    }));
+
+                // Handle any incoming messages in our handler loop:
+                tokio::spawn(Aggregator::handle_messages(
+                    rx_from_external,
+                    tx_to_locator.into_sink(),
+                    max_queue_len,
+                    denylist,
+                    max_third_party_nodes,
+                    true,
+                ));
+            }
+            Some(update_every) => {
+                tokio::task::spawn({
+                    let tx_to_aggregator = tx_to_aggregator.clone();
+                    let mut timer = tokio::time::interval(update_every);
+                    // First tick is instant
+                    timer.tick().await;
+
+                    async move {
+                        while let Ok(()) =
+                            tx_to_aggregator.send(inner_loop::ToAggregator::SendUpdates)
+                        {
+                            timer.tick().await;
+                        }
+                    }
+                });
+
+                // Handle any incoming messages in our handler loop:
+                tokio::spawn(Aggregator::handle_messages(
+                    rx_from_external,
+                    futures::sink::drain(),
+                    max_queue_len,
+                    denylist,
+                    max_third_party_nodes,
+                    false,
+                ));
+            }
+        }
 
         // Return a handle to our aggregator:
         Ok(Aggregator(Arc::new(AggregatorInternal {
@@ -92,20 +133,24 @@ impl Aggregator {
     /// This is spawned into a separate task and handles any messages coming
     /// in to the aggregator. If nobody is holding the tx side of the channel
     /// any more, this task will gracefully end.
-    async fn handle_messages(
+    async fn handle_messages<A>(
         rx_from_external: flume::Receiver<inner_loop::ToAggregator>,
-        tx_to_aggregator: flume::Sender<(NodeId, IpAddr)>,
+        tx_to_aggregator: A,
         max_queue_len: usize,
         denylist: Vec<String>,
         max_third_party_nodes: usize,
-    ) {
+        send_node_data: bool,
+    ) where
+        A: Sink<(NodeId, IpAddr)> + Send + Unpin + 'static,
+    {
         inner_loop::InnerLoop::new(
             tx_to_aggregator,
             denylist,
             max_queue_len,
             max_third_party_nodes,
+            send_node_data,
         )
-        .handle(rx_from_external)
+        .handle(rx_from_external.into_stream())
         .await;
     }