Merge pull request #1 from ideal-lab5/cm/acuity-index-poc

POC implementation to index Ideal Network Events using acuity substrate

Author: carloskiron

Cargo.toml

@@ -0,0 +1,11 @@
+[workspace]
+resolver = "2"
+
+members = [
+    "polkadot-metadata",
+    "kusama-metadata",
+    "rococo-metadata",
+    "westend-metadata",
+    "ideal-network-local-metadata",
+    "indexer",
+]

Dockerfile

@@ -0,0 +1,9 @@
+FROM rust:slim
+
+WORKDIR /usr/src/ideal-network-indexer
+
+COPY . .
+
+RUN cargo build --release -j 1
+
+EXPOSE 8172

LICENSE

@@ -175,18 +175,7 @@
 
    END OF TERMS AND CONDITIONS
 
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2023-2024 Jonathan Brown
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -1,2 +1,68 @@
-# idn-event-indexer
-Event indexer for Ideal Network.
+# Ideal Network Indexer
+
+Event indexer for the Ideal Network.
+
+## Architecture
+
+![Hybrid Architecture](https://raw.githubusercontent.com/ethernomad/hybrid-diagram/main/hybrid.png)
+
+IDN Indexer uses the [Hybrid Indexer](https://github.com/hybrid-explorer/hybrid-indexer) Rust library. It can be accessed using [Hybrid Dapp](https://github.com/hybrid-explorer/hybrid-dapp).
+
+## Building
+
+Ideal Network Indexer can be built using `cargo build`, however it is necessary to use the nightly `rustc`.
+
+```sh
+rustup default nightly
+cargo build --release
+```
+
+Compiling `metadata` can take a very long time.
+
+## Running
+
+```
+Usage: ideal-indexer [OPTIONS]
+
+Options:
+  -c, --chain <CHAIN>                Chain to index [default: ideal] [possible values: ideal,polkadot, kusama, rococo, westend]
+  -d, --db-path <DB_PATH>            Database path
+  -u, --url <URL>                    URL of Substrate node to connect to
+  -b, --block-number <BLOCK_NUMBER>  Block number to start indexing from
+      --queue-depth <QUEUE_DEPTH>    Maximum number of concurrent requests to the chain [default: 64]
+  -p, --port <PORT>                  Port to open for WebSocket queries [default: 8172]
+  -v, --verbose...                   More output per occurrence
+  -q, --quiet...                     Less output per occurrence
+  -h, --help                         Print help
+  -V, --version                      Print version
+```
+
+## Docker
+
+First build the docker image:
+
+```sh
+docker build .
+```
+
+Run the docker image for each chain in a separate tab (replace `[image_hash]` with the hash of the docker image displayed at the end of the build):
+
+```sh
+docker run --rm -p 8172:8172 [image_hash] -c ideal -b 16730000 -p 8172  
+```
+
+```sh
+docker run --rm -p 8172:8172 [image_hash] -c polkadot -b 16730000 -p 8172
+```
+
+```sh
+docker run --rm -p 8172:8172 [image_hash] -c kusama -b 16730000 -p 8172
+```
+  
+```sh
+docker run --rm -p 8174:8174 [image_hash] -c rococo -b 6550000 -p 8174
+```
+  
+```sh
+docker run --rm -p 8175:8175 [image_hash] -c westend -b 16940000 -p 8175
+```

ideal-network-local-metadata/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "ideal-network-local-metadata"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+subxt = "0.35.3"

ideal-network-local-metadata/ideal.scale

@@ -0,0 +1,2 @@
+#[subxt::subxt(runtime_metadata_path = "ideal.scale")]
+pub mod ideal_network_local_metadata {}

ideal-network-local-metadata/src/lib.rs

@@ -0,0 +1,22 @@
+[package]
+name = "acuity-index-polkadot"
+version = "0.2.2"
+edition = "2021"
+
+[dependencies]
+polkadot-metadata = { path = "../polkadot-metadata" }
+kusama-metadata = { path = "../kusama-metadata" }
+rococo-metadata = { path = "../rococo-metadata" }
+westend-metadata = { path = "../westend-metadata" }
+ideal-network-local-metadata = { path = "../ideal-network-local-metadata" }
+tokio = { version = "1.28.2", features = ["macros", "rt", "rt-multi-thread"] }
+subxt = "0.35.3"
+clap = { version = "4.3.19", features = ["derive"] }
+hex-literal = "0.4.1"
+clap-verbosity-flag = "2.0.1"
+sled = { version = "0.34.7", default-features = false }
+byte-unit = "4.0.19"
+serde = { version = "1.0.162", features = ["derive"] }
+zerocopy = "0.7.8"
+tracing-log = "0.2.0"
+acuity-index-substrate = { path = "../../idn-acuity-index-substrate" }

indexer/Cargo.toml

@@ -0,0 +1,54 @@
+use ideal_network_local_metadata::ideal_network_local_metadata::{
+    runtime_types::{
+        frame_system::pallet::Event as SystemEvent,
+        pallet_balances::pallet::Event as BalancesEvent,
+        // Add other relevant event imports for your ideal network
+    },
+    Event,
+};
+
+use crate::*;
+use acuity_index_substrate::*;
+use hex_literal::hex;
+
+pub struct IdealIndexer;
+
+impl acuity_index_substrate::shared::RuntimeIndexer for IdealIndexer {
+    type RuntimeConfig = subxt::PolkadotConfig;  // You might need to adjust this based on your chain's configuration
+    type ChainKey = ChainKey;
+
+    fn get_name() -> &'static str {
+        "ideal"
+    }
+
+    fn get_genesis_hash() -> <Self::RuntimeConfig as subxt::Config>::Hash {
+        // Replace with your chain's genesis hash
+        hex!["af97825bf72091072a08b9dbff88d6664e2061bcb4e28a90f17bd85572d8f8ae"].into()  // Temporary placeholder
+    }
+
+    fn get_versions() -> &'static [u32] {
+        &[0]
+    }
+
+    fn get_default_url() -> &'static str {
+        "ws://127.0.0.1:1234"  // Replace with your actual endpoint
+    }
+
+    fn process_event(
+        indexer: &acuity_index_substrate::substrate::Indexer<Self>,
+        block_number: u32,
+        event_index: u16,
+        event: subxt::events::EventDetails<Self::RuntimeConfig>,
+    ) -> Result<u32, IndexError> {
+        Ok(match event.as_root_event::<Event>()? {
+            Event::System(event) => {
+                index_system_event![SystemEvent, event, indexer, block_number, event_index]
+            }
+            Event::Balances(event) => {
+                index_balances_event![BalancesEvent, event, indexer, block_number, event_index]
+            }
+            // Add other event handlers as needed
+            _ => 0,
+        })
+    }
+}

indexer/src/ideal.rs

@@ -0,0 +1,188 @@
+use kusama_metadata::kusama_metadata::{
+    runtime_types::{
+        frame_system::pallet::Event as SystemEvent,
+        pallet_bags_list::pallet::Event as BagsListEvent,
+        pallet_balances::pallet::Event as BalancesEvent,
+        pallet_bounties::pallet::Event as BountiesEvent,
+        pallet_child_bounties::pallet::Event as ChildBountiesEvent,
+        pallet_election_provider_multi_phase::pallet::Event as ElectionProviderMultiPhaseEvent,
+        pallet_fast_unstake::pallet::Event as FastUnstakeEvent,
+        pallet_identity::pallet::Event as IdentityEvent,
+        pallet_indices::pallet::Event as IndicesEvent,
+        pallet_multisig::pallet::Event as MultisigEvent,
+        pallet_nomination_pools::pallet::Event as NominationPoolsEvent,
+        pallet_preimage::pallet::Event as PreimageEvent,
+        pallet_proxy::pallet::Event as ProxyEvent,
+        pallet_session::pallet::Event as SessionEvent,
+        pallet_staking::pallet::pallet::Event as StakingEvent,
+        pallet_transaction_payment::pallet::Event as TransactionPaymentEvent,
+        pallet_treasury::pallet::Event as TreasuryEvent,
+        pallet_vesting::pallet::Event as VestingEvent,
+        polkadot_runtime_common::{
+            auctions::pallet::Event as AuctionsEvent, claims::pallet::Event as ClaimsEvent,
+            crowdloan::pallet::Event as CrowdloanEvent,
+            paras_registrar::pallet::Event as ParasRegistrarEvent,
+            slots::pallet::Event as SlotsEvent,
+        },
+        polkadot_runtime_parachains::{
+            disputes::pallet::Event as DisputesEvent, hrmp::pallet::Event as HrmpEvent,
+            paras::pallet::Event as ParasEvent,
+        },
+    },
+    Event,
+};
+
+use crate::*;
+use acuity_index_substrate::*;
+
+use hex_literal::hex;
+
+pub struct KusamaIndexer;
+
+impl acuity_index_substrate::shared::RuntimeIndexer for KusamaIndexer {
+    type RuntimeConfig = subxt::PolkadotConfig;
+    type ChainKey = ChainKey;
+
+    fn get_name() -> &'static str {
+        "kusama"
+    }
+
+    fn get_genesis_hash() -> <Self::RuntimeConfig as subxt::Config>::Hash {
+        hex!["b0a8d493285c2df73290dfb7e61f870f17b41801197a149ca93654499ea3dafe"].into()
+    }
+
+    fn get_versions() -> &'static [u32] {
+        &[0]
+    }
+
+    fn get_default_url() -> &'static str {
+        "wss://kusama-rpc.polkadot.io:443"
+    }
+
+    fn process_event(
+        indexer: &acuity_index_substrate::substrate::Indexer<Self>,
+        block_number: u32,
+        event_index: u16,
+        event: subxt::events::EventDetails<Self::RuntimeConfig>,
+    ) -> Result<u32, IndexError> {
+        Ok(match event.as_root_event::<Event>()? {
+            // Substrate pallets.
+            Event::System(event) => {
+                index_system_event![SystemEvent, event, indexer, block_number, event_index]
+            }
+            Event::Preimage(event) => {
+                index_preimage_event![PreimageEvent, event, indexer, block_number, event_index]
+            }
+            Event::Indices(event) => {
+                index_indices_event![IndicesEvent, event, indexer, block_number, event_index]
+            }
+            Event::Balances(event) => {
+                index_balances_event![BalancesEvent, event, indexer, block_number, event_index]
+            }
+            Event::TransactionPayment(event) => {
+                index_transaction_payment_event![
+                    TransactionPaymentEvent,
+                    event,
+                    indexer,
+                    block_number,
+                    event_index
+                ]
+            }
+            Event::Staking(event) => {
+                index_staking_event![StakingEvent, event, indexer, block_number, event_index]
+            }
+            Event::Session(event) => {
+                index_session_event![SessionEvent, event, indexer, block_number, event_index]
+            }
+            Event::Treasury(event) => {
+                index_treasury_event![TreasuryEvent, event, indexer, block_number, event_index]
+            }
+            Event::Vesting(event) => {
+                index_vesting_event![VestingEvent, event, indexer, block_number, event_index]
+            }
+            Event::Identity(event) => {
+                index_identity_event![IdentityEvent, event, indexer, block_number, event_index]
+            }
+            Event::Proxy(event) => {
+                index_proxy_event![ProxyEvent, event, indexer, block_number, event_index]
+            }
+            Event::Multisig(event) => {
+                index_multisig_event![MultisigEvent, event, indexer, block_number, event_index]
+            }
+            Event::Bounties(event) => {
+                index_bounties_event![BountiesEvent, event, indexer, block_number, event_index]
+            }
+            Event::ChildBounties(event) => {
+                index_child_bounties_event![
+                    ChildBountiesEvent,
+                    event,
+                    indexer,
+                    block_number,
+                    event_index
+                ]
+            }
+            Event::ElectionProviderMultiPhase(event) => {
+                index_election_provider_multi_phase_event![
+                    ElectionProviderMultiPhaseEvent,
+                    event,
+                    indexer,
+                    block_number,
+                    event_index
+                ]
+            }
+            Event::VoterList(event) => {
+                index_bags_list_event![BagsListEvent, event, indexer, block_number, event_index]
+            }
+            Event::NominationPools(event) => {
+                index_nomination_pools_event![
+                    NominationPoolsEvent,
+                    event,
+                    indexer,
+                    block_number,
+                    event_index
+                ]
+            }
+            Event::FastUnstake(event) => {
+                index_fast_unstake_event![
+                    FastUnstakeEvent,
+                    event,
+                    indexer,
+                    block_number,
+                    event_index
+                ]
+            }
+            // Polkadot pallets.
+            Event::Claims(event) => {
+                index_claims_event![ClaimsEvent, event, indexer, block_number, event_index]
+            }
+            Event::Paras(event) => {
+                index_paras_event![ParasEvent, event, indexer, block_number, event_index]
+            }
+            Event::Hrmp(event) => {
+                index_hrmp_event![HrmpEvent, event, indexer, block_number, event_index]
+            }
+            Event::ParasDisputes(event) => {
+                index_disputes_event![DisputesEvent, event, indexer, block_number, event_index]
+            }
+            Event::Registrar(event) => {
+                index_paras_registrar_event![
+                    ParasRegistrarEvent,
+                    event,
+                    indexer,
+                    block_number,
+                    event_index
+                ]
+            }
+            Event::Slots(event) => {
+                index_slots_event![SlotsEvent, event, indexer, block_number, event_index]
+            }
+            Event::Auctions(event) => {
+                index_auctions_event![AuctionsEvent, event, indexer, block_number, event_index]
+            }
+            Event::Crowdloan(event) => {
+                index_crowdloan_event![CrowdloanEvent, event, indexer, block_number, event_index]
+            }
+            _ => 0,
+        })
+    }
+}