General Cargo docs, wip (#432)

* update substrate to latest, document backend portion

Signed-off-by: Andrew Plaza <aplaza@liquidthink.net>

* update schema

* document sa-work-queue

* bring up to date with latest substrate master

* more general docs for archive

* language

* fmt

* Update substrate-archive-backend/src/frontend/client.rs

Co-authored-by: David <dvdplm@gmail.com>

* fix tests

Co-authored-by: David <dvdplm@gmail.com>

Author: insipx

Cargo.lock

@@ -14,8 +14,8 @@
 // You should have received a copy of the GNU General Public License
 // along with substrate-archive.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Custom Read-Only Database Instance using RocksDB Secondary features
-//! Will try catching up with primary database on every `get()`
+//! Custom Read-Only Database Instance using RocksDB Secondary features.
+//! Will try catching up with primary database on every `get()`.
 
 use std::{collections::HashMap, fmt, io, path::PathBuf};
 
@@ -28,7 +28,9 @@ const NUM_COLUMNS: u32 = 11;
 
 pub type KeyValuePair = (Box<[u8]>, Box<[u8]>);
 
-// Archive specific K/V database reader implementation
+/// Archive specific K/V database reader implementation.
+/// Any backend database similiar to RocksDB (e.g ParityDb)
+/// should implement this trait in order to be used with archive.
 pub trait ReadOnlyDb: Send + Sync {
 	/// Read key/value pairs from the database
 	fn get(&self, col: u32, key: &[u8]) -> Option<Vec<u8>>;

SubstrateArchiveSchema.pdf

@@ -18,7 +18,7 @@ mod client;
 
 use serde::Deserialize;
 use std::{
-	collections::HashMap,
+	collections::{BTreeMap, HashMap},
 	convert::{TryFrom, TryInto},
 	path::PathBuf,
 	str::FromStr,
@@ -33,9 +33,7 @@ use sc_executor::{WasmExecutionMethod, WasmExecutor};
 use sc_service::{ChainSpec, ClientConfig, LocalCallExecutor, TransactionStorageMode};
 use sp_api::ConstructRuntimeApi;
 use sp_core::traits::SpawnNamed;
-use sp_runtime::traits::{BlakeTwo256, Block as BlockT};
-use sp_wasm_interface::Function;
-use sp_wasm_interface::HostFunctions;
+use sp_runtime::traits::{BlakeTwo256, Block as BlockT, NumberFor};
 
 pub use self::client::{Client, GetMetadata};
 use crate::{database::ReadOnlyDb, error::BackendError, read_only_backend::ReadOnlyBackend, RuntimeApiCollection};
@@ -44,7 +42,8 @@ use crate::{database::ReadOnlyDb, error::BackendError, read_only_backend::ReadOn
 pub type TArchiveClient<TBl, TRtApi, D> = Client<TFullCallExecutor<TBl, D>, TBl, TRtApi, D>;
 
 /// Full client call executor type.
-type TFullCallExecutor<TBl, D> = LocalCallExecutor<TBl, ReadOnlyBackend<TBl, D>, WasmExecutor>;
+type TFullCallExecutor<TBl, D> =
+	LocalCallExecutor<TBl, ReadOnlyBackend<TBl, D>, WasmExecutor<sp_io::SubstrateHostFunctions>>;
 
 #[derive(Copy, Clone, Debug, Deserialize)]
 pub enum ExecutionMethod {
@@ -67,6 +66,8 @@ impl From<ExecutionMethod> for WasmExecutionMethod {
 	}
 }
 
+/// Configuration controls how Archive executes blocks in `execute_block`
+/// (in tasks.rs in `substrate-archive/`).
 #[derive(Clone, Debug, Deserialize)]
 pub struct RuntimeConfig {
 	/// How to execute the runtime code: interpreted (default) or JIT compiled.
@@ -85,7 +86,7 @@ pub struct RuntimeConfig {
 	/// are included in the chain_spec and primarily for fixing problematic on-chain wasm.
 	/// If both are in use, the `wasm_runtime_overrides` takes precedence.
 	#[serde(skip)]
-	code_substitutes: HashMap<String, Vec<u8>>,
+	code_substitutes: BTreeMap<String, Vec<u8>>,
 	/// Method of storing and retrieving transactions(extrinsics).
 	#[serde(skip, default = "default_storage_mode")]
 	pub storage_mode: TransactionStorageMode,
@@ -123,36 +124,41 @@ const fn default_storage_mode() -> TransactionStorageMode {
 impl<B> TryFrom<RuntimeConfig> for ClientConfig<B>
 where
 	B: BlockT,
-	B::Hash: FromStr,
 {
 	type Error = BackendError;
 	fn try_from(config: RuntimeConfig) -> Result<ClientConfig<B>, BackendError> {
 		let wasm_runtime_substitutes = config
 			.code_substitutes
 			.into_iter()
-			.map(|(hash, code)| {
-				let hash = B::Hash::from_str(&hash).map_err(|_| {
-					BackendError::Msg(format!("Failed to parse `{}` as block hash for code substitute.", hash))
+			.map(|(n, code)| {
+				let number = NumberFor::<B>::from_str(&n).map_err(|_| {
+					BackendError::Msg(format!(
+						"Failed to parse `{}` as block number for code substitutes. \
+						 In an old version the key for code substitute was a block hash. \
+						 Please update the chain spec to a version that is compatible with your node.",
+						n
+					))
 				})?;
-				Ok((hash, code))
+				Ok((number, code))
 			})
-			.collect::<Result<HashMap<B::Hash, Vec<u8>>, BackendError>>()?;
+			.collect::<Result<HashMap<_, _>, BackendError>>()?;
 
 		Ok(ClientConfig {
 			offchain_worker_enabled: false,
 			offchain_indexing_api: false,
 			wasm_runtime_overrides: config.wasm_runtime_overrides,
-			wasm_runtime_substitutes,
 			// we do not support 'no_genesis', so this value is inconsiquential
 			no_genesis: false,
+			wasm_runtime_substitutes,
 		})
 	}
 }
 
+/// Main entry to initialize the substrate-archive backend client, used to
+/// call into the runtime of the network being indexed (e.g to execute blocks).
 pub fn runtime_api<Block, Runtime, D: ReadOnlyDb + 'static>(
 	config: RuntimeConfig,
 	backend: Arc<ReadOnlyBackend<Block, D>>,
-	host_functions: Option<Vec<&'static dyn Function>>,
 	task_executor: impl SpawnNamed + 'static,
 ) -> Result<TArchiveClient<Block, Runtime, D>, BackendError>
 where
@@ -165,11 +171,13 @@ where
 		+ 'static,
 	<Runtime::RuntimeApi as sp_api::ApiExt<Block>>::StateBackend: sp_api::StateBackend<BlakeTwo256>,
 {
-	let host_functions =
-		if let Some(funcs) = host_functions { funcs } else { sp_io::SubstrateHostFunctions::host_functions() };
-
-	let executor =
-		WasmExecutor::new(config.exec_method.into(), config.wasm_pages, host_functions, config.block_workers, None);
+	let executor = WasmExecutor::<sp_io::SubstrateHostFunctions>::new(
+		config.exec_method.into(),
+		config.wasm_pages,
+		config.block_workers,
+		None,
+		128,
+	);
 	let executor = LocalCallExecutor::new(backend.clone(), executor, Box::new(task_executor), config.try_into()?)?;
 	let client = Client::new(backend, executor, ExecutionExtensions::new(execution_strategies(), None, None))?;
 	Ok(client)

SubstrateArchiveSchema.png

@@ -14,12 +14,14 @@
 // You should have received a copy of the GNU General Public License
 // along with substrate-archive.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Custom implementation of a Client for Archival use
+//! Custom implementation of a Client for Archival use.
+//! This client resembles the client in `client/service/src/client` except that it uses a read-only rocksdb
+//! database implementation, and that the backend state trie aims at avoid caching as much as possible, unlike the normal substrate client.
+//! (Substrate Archive generally does not expect to access values multiple times during a run).
 //! Implements Runtime API's
 //! Read-Only Access
-//! The Archive Client and it's backend are decoupled
 //! It's recommended to use the backend (ReadOnlyBackend) for anything that requires getting blocks, querying
-//! storage, or similar operations. Client usage should be reserved for calling into the Runtime
+//! storage, or similar operations. Client usage should be reserved for calling into the Runtime.
 
 use std::{marker::PhantomData, panic::UnwindSafe, sync::Arc};
 

examples/simple.rs

@@ -15,6 +15,21 @@
 // along with substrate-archive.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Read Only Interface with Substrate Backend (kvdb-rocksdb)
+//! Structs and traits in this module generally follow those defined in substrate.
+//! They only differentiate insofar as the use of a read-only Secondary RocksDB Database.
+//! Therefore some methods (like block importing) are not useful to us and left as stubs (generally with a warn! log).
+//! We need to implement substrate database traits in order to satisfy the requirements for using a client.
+//! We need a client implementation in order to access functions in a FRAME runtime (like block execution)
+//! to allow for faster indexing.
+//! These traits are:
+//! [`Backend`](https://paritytech.github.io/substrate/master/sc_client_api/backend/trait.Backend.html)
+//! [`BlockBackend`](https://paritytech.github.io/substrate/master/sp_blockchain/trait.Backend.html)
+//! [`AuxStore`](https://paritytech.github.io/substrate/master/sc_client_api/backend/trait.AuxStore.html)
+//! [`OffchainStorage`](https://paritytech.github.io/substrate/master/sp_core/offchain/trait.OffchainStorage.html)
+//! [`BlockImportOperation`](https://paritytech.github.io/substrate/master/sc_client_db/struct.BlockImportOperation.html)
+//!
+//! the trait [`StateBackend`](https://paritytech.github.io/substrate/master/sc_client_api/backend/trait.StateBackend.html)
+//! is implemented on a trimmed-down [`TrieState`] type and used in the archive client.
 
 #![forbid(unsafe_code)]
 #![deny(dead_code)]

substrate-archive-backend/src/database.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with substrate-archive.  If not, see <http://www.gnu.org/licenses/>.
 
-//! Implements Blockchain Backend (and required associated traits) for ReadOnlyBackend type
+//! Implements Blockchain Backend (and required associated traits) for ReadOnlyBackend type.
 
 use codec::{Decode, Encode};
 

substrate-archive-backend/src/frontend.rs

@@ -26,7 +26,7 @@ use sp_blockchain::{well_known_cache_keys::Id, Error as BlockchainError};
 use sp_core::offchain::OffchainStorage;
 use sp_runtime::{generic::BlockId, traits::Block as BlockT, Justification, Justifications};
 use sp_state_machine::{ChildStorageCollection, IndexOperation, StorageCollection};
-use sp_storage::Storage;
+use sp_storage::{StateVersion, Storage};
 
 use crate::{database::ReadOnlyDb, read_only_backend::ReadOnlyBackend, util::columns};
 
@@ -67,12 +67,17 @@ impl<Block: BlockT, D: ReadOnlyDb> BlockImportOperation<Block> for RealBlockImpo
 		Ok(())
 	}
 
-	fn set_genesis_state(&mut self, _storage: Storage, _commit: bool) -> sp_blockchain::Result<Block::Hash> {
+	fn set_genesis_state(
+		&mut self,
+		_storage: Storage,
+		_commit: bool,
+		_: StateVersion,
+	) -> sp_blockchain::Result<Block::Hash> {
 		log::warn!("Cannot set state of a read only backend. Genesis not set");
 		Ok(Default::default())
 	}
 
-	fn reset_storage(&mut self, _reset: Storage) -> ChainResult<Block::Hash> {
+	fn reset_storage(&mut self, _reset: Storage, _: StateVersion) -> ChainResult<Block::Hash> {
 		log::warn!("Cannot modify storage of a read only backend. Storage not reset.");
 		Ok(Default::default())
 	}