Merge pull request #58 from paritytech/node-template-archive

Bumps Version to 0.3.1, Introduce Node-Template-Archive

Author: insipx

Cargo.lock

@@ -2,4 +2,5 @@
 members = [
     "archive",
     "kusama-archive",
+    "node-template-archive",
 ]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "substrate-archive"
-version = "0.3.0"
+version = "0.3.1"
 authors = ["Andrew Plaza <andrew.plaza@parity.io>"]
 edition = "2018"
 
@@ -38,22 +38,22 @@ kvdb-rocksdb = "0.8"
 parity-util-mem = "0.6.1"
 codec = { package = "parity-scale-codec", version = "1.2.0", default-features = false, features = ["derive", "full"] }
 
+
 # Substrate
-sp-database = { git = "https://github.com/paritytech/substrate", branch = "master", package="sp-database" }
-sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sc-service" }
-sc-client-db = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sc-client-db" }
-sp-storage = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sp-storage" }
-sp-runtime = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sp-runtime" }
-sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sc-client-api" }
-sp-blockchain = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sp-blockchain" }
-substrate-rpc-primitives = {  git = "https://github.com/paritytech/substrate", branch = "master", package = "sp-rpc" }
-sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sp-core" }
-sp-api = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sp-api" }
-runtime-version = {  git = "https://github.com/paritytech/substrate", branch = "master", package = "sp-version" }
-sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sc-executor" }
-sc-transaction-graph = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sc-transaction-graph" }
-sc-chain-spec = { git = "https://github.com/paritytech/substrate", branch = "master", package = "sc-chain-spec" }
-frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", package = "frame-system" }
+sp-database = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master", version = "2.0.0-rc2", package = "sp-database" }
+sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", version = "0.8.0-rc2", package = "sc-service" }
+sc-client-db = { git = "https://github.com/paritytech/substrate", branch = "master", version = "0.8.0-rc2", package = "sc-client-db" }
+sp-storage = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master", version = "2.0.0-rc2", package = "sp-storage" }
+sp-runtime = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master", version = "2.0.0-rc2", package = "sp-runtime" }
+sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "master", version = "2.0.0-rc2", package = "sc-client-api" }
+sp-blockchain = { git = "https://github.com/paritytech/substrate", default-features = false, branch = "master", version = "2.0.0-rc2", package = "sp-blockchain" }
+
+substrate-rpc-primitives = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "2.0.0-rc2", package = "sp-rpc" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "2.0.0-rc2", package = "sp-core" }
+sp-api = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "2.0.0-rc2", package = "sp-api" }
+runtime-version = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "2.0.0-rc2", package = "sp-version" }
+sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "0.8.0-rc2", package = "sc-executor" }
+frame-system = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "2.0.0-rc2", package = "frame-system" }
 
 [dev-dependencies]
 # desub-extras = { path = "../desub/extras/", package="desub-extras", features = ["polkadot"] }
@@ -62,4 +62,14 @@ pretty_env_logger = "0.4.0"
 
 
 [features]
+default = ["std"]
+std = [
+    "sp-storage/std",
+    "sp-runtime/std",
+    "sp-core/std",
+    "sp-api/std",
+    "runtime-version/std",
+    "sc-executor/std",
+    "frame-system/std"
+]
 logging = ["chrono", "fern"]

archive/Cargo.toml

@@ -39,9 +39,10 @@ pub use self::actors::Archive;
 pub use sp_storage::StorageKey;
 pub use sp_core::twox_128;
 pub use sp_blockchain::Error as BlockchainError;
+pub use sp_runtime::MultiSignature;
 pub mod chain_traits  {
     //! Traits defining functions on the client needed for indexing
     pub use sc_client_api::{backend::StorageProvider, client::BlockBackend, UsageProvider};
     pub use sp_blockchain::{HeaderBackend, HeaderMetadata};
-    pub use sp_runtime::traits::{BlakeTwo256, Block};
+    pub use sp_runtime::traits::{BlakeTwo256, Block, Verify, IdentifyAccount};
 }

archive/src/lib.rs

@@ -17,7 +17,7 @@ pretty_env_logger = "0.4.0"
 anyhow = "1.0.31"
 serde = "1.0.110"
 sqlx = { version = "0.3.5", features = ["postgres"] }
-async-std = "1.6.0"
+async-std = "1.5.0"
 async-ctrlc = "1.1.0"
 indicatif = "0.14.0"
 

kusama-archive/Cargo.toml

@@ -36,8 +36,9 @@ impl CliOpts {
         };
         let file = matches.value_of("config").expect("Config is a required value");
 
+
         CliOpts {
             file: PathBuf::from(file), log_level
         }
     }
-}
+}

kusama-archive/src/cli_opts.rs

@@ -0,0 +1,43 @@
+[package]
+name = "node-template-archive"
+version = "0.1.0"
+authors = ["Andrew Plaza <aplaza@liquidthink.net>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+substrate-archive = { version = "0.3.1", path = "../archive", default-features = false, features = ["logging"] }
+node-template-runtime = { git = "https://github.com/paritytech/substrate", branch="master", default-features = false, package = "node-template-runtime" }
+clap = { version = "2.33.1", features = ["yaml", "suggestions", "color"] }
+toml = "0.5"
+futures = "0.3.5"
+log = "0.4.8"
+pretty_env_logger = "0.4.0"
+anyhow = "1.0.31"
+serde = "1.0.110"
+sqlx = { version = "0.3.5", features = ["postgres"] }
+async-std = "1.5.0"
+async-ctrlc = "1.1.0"
+indicatif = "0.14.0"
+
+sp-consensus-aura = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "0.8.0-rc2", package = "sp-consensus-aura" }
+sp-finality-grandpa = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "2.0.0-rc2", package = "sp-finality-grandpa" }
+# sc-finality-grandpa = { version = "0.8.0-rc2" }
+
+sc-service = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "0.8.0-rc2", package = "sc-service" }
+# sp-runtime = { version = "2.0.0-rc2", package = "sp-runtime" }
+sp-core = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "2.0.0-rc2", package = "sp-core" }
+sc-executor = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false, version = "0.8.0-rc2", package = "sc-executor" }
+
+
+[features]
+default = ["std"]
+std = [
+    "substrate-archive/std",
+    "node-template-runtime/std",
+    "sp-consensus-aura/std",
+    "sp-finality-grandpa/std",
+    "sp-core/std",
+    "sc-executor/std"
+]

node-template-archive/Cargo.toml

@@ -0,0 +1,20 @@
+# Must be an absolute path to db (no ~/)
+db_path = "/home/insipx/.local/share/node-template/chains/dev/db"
+
+# Can be set by exporting environment variable "DATABASE_URL" too
+psql_url = "postgres://archive:default@localhost/archive"
+
+[[modules]]
+name = "System"
+keys = [
+	"Events",
+	"Account"
+]
+
+[[modules]]
+name = "Democracy"
+keys = [
+	"PublicPropCount",
+	"PublicProps"
+]
+

node-template-archive/archive_conf.toml

@@ -0,0 +1,59 @@
+// Copyright 2018-2019 Parity Technologies (UK) Ltd.
+// This file is part of substrate-archive.
+
+// substrate-archive is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// substrate-archive is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with substrate-archive.  If not, see <http://www.gnu.org/licenses/>.
+
+use node_template_runtime::{self as runtime, opaque::Block};
+use sc_executor::native_executor_instance;
+use substrate_archive::{Archive, backend::{self, ChainAccess}, chain_traits::HeaderBackend as _};
+use std::sync::Arc;
+use anyhow::Result;
+
+native_executor_instance!(
+    pub Executor,
+    node_template_runtime::api::dispatch,
+    node_template_runtime::native_version
+);
+
+pub fn run_archive(config: super::config::Config) -> Result<(Arc<impl ChainAccess<Block>>, Archive)> 
+{
+    let db = backend::open_database::<Block>(config.db_path(), 8192).unwrap();
+
+    // let spec = polkadot_service::chain_spec::kusama_config().unwrap();
+    let spec = config.cli().chain_spec.clone();
+    let client =
+        backend::client::<Block, runtime::RuntimeApi, Executor, _>(
+            db, spec,
+        )
+        .unwrap();
+
+    let info = client.info();
+    println!("{:?}", info);
+
+   
+    // TODO: use a better error-handling (this-error) crate with substrate-archive
+    // (failure is deprecated)
+    // run until we want to exit (Ctrl-C)
+    let archive = Archive::init::<runtime::Runtime, _>(
+        client.clone(),
+        "ws://127.0.0.1:9944".to_string(),
+        config.keys(),
+        config.psql_url()
+    ).expect("Init Failed");     
+    
+
+    Ok((
+        client, archive
+    ))
+}

node-template-archive/src/archive.rs

@@ -0,0 +1,132 @@
+use sp_core::{Pair, Public, sr25519};
+use node_template_runtime::{
+    AuraConfig, BalancesConfig, GenesisConfig, GrandpaConfig,
+	SudoConfig, SystemConfig, WASM_BINARY
+};
+use sp_consensus_aura::sr25519::AuthorityId as AuraId;
+use sp_finality_grandpa::AuthorityId as GrandpaId;
+// use sp_runtime::traits::{Verify, IdentifyAccount};
+use substrate_archive::{MultiSignature, chain_traits::{Verify, IdentifyAccount}};
+use sc_service::ChainType;
+
+// Note this is the URL for the telemetry server
+//const STAGING_TELEMETRY_URL: &str = "wss://telemetry.polkadot.io/submit/";
+
+/// Specialized `ChainSpec`. This is a specialization of the general Substrate ChainSpec type.
+pub type ChainSpec = sc_service::GenericChainSpec<GenesisConfig>;
+
+/// Helper function to generate a crypto pair from seed
+pub fn get_from_seed<TPublic: Public>(seed: &str) -> <TPublic::Pair as Pair>::Public {
+	TPublic::Pair::from_string(&format!("//{}", seed), None)
+		.expect("static values are valid; qed")
+		.public()
+}
+
+/// Alias to 512-bit hash when used in the context of a transaction signature on the chain.
+pub type Signature = MultiSignature;
+
+/// Some way of identifying an account on the chain. We intentionally make it equivalent
+/// to the public key of our transaction signing scheme.
+pub type AccountId = <<Signature as Verify>::Signer as IdentifyAccount>::AccountId;
+
+type AccountPublic = <Signature as Verify>::Signer;
+
+/// Helper function to generate an account ID from seed
+pub fn get_account_id_from_seed<TPublic: Public>(seed: &str) -> AccountId where
+	AccountPublic: From<<TPublic::Pair as Pair>::Public>
+{
+	AccountPublic::from(get_from_seed::<TPublic>(seed)).into_account()
+}
+
+/// Helper function to generate an authority key for Aura
+pub fn authority_keys_from_seed(s: &str) -> (AuraId, GrandpaId) {
+	(
+		get_from_seed::<AuraId>(s),
+		get_from_seed::<GrandpaId>(s),
+	)
+}
+
+pub fn development_config() -> ChainSpec {
+	ChainSpec::from_genesis(
+		"Development",
+		"dev",
+		ChainType::Development,
+		|| testnet_genesis(
+			vec![
+				authority_keys_from_seed("Alice"),
+			],
+			get_account_id_from_seed::<sr25519::Public>("Alice"),
+			vec![
+				get_account_id_from_seed::<sr25519::Public>("Alice"),
+				get_account_id_from_seed::<sr25519::Public>("Bob"),
+				get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+				get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
+			],
+			true,
+		),
+		vec![],
+		None,
+		None,
+		None,
+		None,
+	)
+}
+
+pub fn local_testnet_config() -> ChainSpec {
+	ChainSpec::from_genesis(
+		"Local Testnet",
+		"local_testnet",
+		ChainType::Local,
+		|| testnet_genesis(
+			vec![
+				authority_keys_from_seed("Alice"),
+				authority_keys_from_seed("Bob"),
+			],
+			get_account_id_from_seed::<sr25519::Public>("Alice"),
+			vec![
+				get_account_id_from_seed::<sr25519::Public>("Alice"),
+				get_account_id_from_seed::<sr25519::Public>("Bob"),
+				get_account_id_from_seed::<sr25519::Public>("Charlie"),
+				get_account_id_from_seed::<sr25519::Public>("Dave"),
+				get_account_id_from_seed::<sr25519::Public>("Eve"),
+				get_account_id_from_seed::<sr25519::Public>("Ferdie"),
+				get_account_id_from_seed::<sr25519::Public>("Alice//stash"),
+				get_account_id_from_seed::<sr25519::Public>("Bob//stash"),
+				get_account_id_from_seed::<sr25519::Public>("Charlie//stash"),
+				get_account_id_from_seed::<sr25519::Public>("Dave//stash"),
+				get_account_id_from_seed::<sr25519::Public>("Eve//stash"),
+				get_account_id_from_seed::<sr25519::Public>("Ferdie//stash"),
+			],
+			true,
+		),
+		vec![],
+		None,
+		None,
+		None,
+		None,
+	)
+}
+
+fn testnet_genesis(initial_authorities: Vec<(AuraId, GrandpaId)>,
+	root_key: AccountId,
+	endowed_accounts: Vec<AccountId>,
+	_enable_println: bool) -> GenesisConfig {
+	GenesisConfig {
+		system: Some(SystemConfig {
+			code: WASM_BINARY.to_vec(),
+			changes_trie_config: Default::default(),
+		}),
+		balances: Some(BalancesConfig {
+			balances: endowed_accounts.iter().cloned().map(|k|(k, 1 << 60)).collect(),
+		}),
+		aura: Some(AuraConfig {
+			authorities: initial_authorities.iter().map(|x| (x.0.clone())).collect(),
+		}),
+		grandpa: Some(GrandpaConfig {
+			authorities: initial_authorities.iter().map(|x| (x.1.clone(), 1)).collect(),
+		}),
+		sudo: Some(SudoConfig {
+			key: root_key,
+		}),
+	}
+}