Deepbook Indexer

Cargo.toml

@@ -99,6 +99,7 @@ members = [
     "crates/sui-cost",
     "crates/sui-data-ingestion",
     "crates/sui-data-ingestion-core",
+    "crates/sui-deepbook-indexer",
     "crates/sui-e2e-tests",
     "crates/sui-enum-compat-util",
     "crates/sui-faucet",

crates/sui-deepbook-indexer/Cargo.toml

@@ -0,0 +1,43 @@
+[package]
+name = "sui-deepbook-indexer"
+version = "0.1.0"
+authors = ["Mysten Labs <build@mystenlabs.com>"]
+license = "Apache-2.0"
+publish = false
+edition = "2021"
+
+[dependencies]
+serde.workspace = true
+tap.workspace = true
+diesel = { version = "2.1.4", features = ["postgres", "r2d2", "serde_json", "numeric"] }
+tokio = { workspace = true, features = ["full"] }
+anyhow.workspace = true
+futures.workspace = true
+async-trait.workspace = true
+bcs.workspace = true
+bin-version.workspace = true
+clap.workspace = true
+mysten-metrics.workspace = true
+prometheus.workspace = true
+serde_yaml.workspace = true
+sui-bridge.workspace = true
+sui-sdk.workspace = true
+sui-json-rpc-types.workspace = true
+sui-data-ingestion-core.workspace = true
+sui-types.workspace = true
+telemetry-subscribers.workspace = true
+tracing.workspace = true
+backoff.workspace = true
+sui-config.workspace = true
+sui-indexer-builder.workspace = true
+bigdecimal = "0.4.0"
+
+[dev-dependencies]
+sui-types = { workspace = true, features = ["test-utils"] }
+sui-test-transaction-builder.workspace = true
+test-cluster.workspace = true
+hex-literal = "0.3.4"
+
+[[bin]]
+name = "deepbook-indexer"
+path = "src/main.rs"

crates/sui-deepbook-indexer/config.yaml

@@ -0,0 +1,22 @@
+# config.yaml format:
+
+# URL of the remote store
+# remote_store_url: <url>
+# Database connection URL
+# db_url: <url>
+# Path to the checkpoints
+# checkpoints_path: <path>
+# DeepBook package ID
+# deepbook_package_id:
+# DeepBook genesis checkpoint
+# deepbook_genesis_checkpoint:
+# Number of concurrent operations
+# concurrency: 1
+# Client metric URL
+# metric_url: <url>
+# Client metric port
+# metric_port: <port>
+# checkpoint size of each backfill worker, use 432000 for 1 worker per day, assume 5 checkpoint per second
+# back_fill_lot_size: <backfill lot size>
+# Optional starting checkpoint for realtime ingestion task
+# resume_from_checkpoint: <Sui checkpoint>

crates/sui-deepbook-indexer/diesel.toml

@@ -0,0 +1,9 @@
+# For documentation on how to configure this file,
+# see https://diesel.rs/guides/configuring-diesel-cli
+
+[print_schema]
+file = "src/schema.rs"
+#custom_type_derives = ["diesel::query_builder::QueryId"]
+
+[migrations_directory]
+dir = "src/migrations"

crates/sui-deepbook-indexer/src/config.rs

@@ -0,0 +1,27 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize};
+use std::env;
+
+/// config as loaded from `config.yaml`.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct IndexerConfig {
+    pub remote_store_url: String,
+    #[serde(default = "default_db_url")]
+    pub db_url: String,
+    pub checkpoints_path: String,
+    pub sui_rpc_url: String,
+    pub deepbook_package_id: String,
+    pub deepbook_genesis_checkpoint: u64,
+    pub concurrency: u64,
+    pub metric_url: String,
+    pub metric_port: u16,
+    pub resume_from_checkpoint: Option<u64>,
+}
+
+impl sui_config::Config for IndexerConfig {}
+
+pub fn default_db_url() -> String {
+    env::var("DB_URL").expect("db_url must be set in config or via the $DB_URL env var")
+}

crates/sui-deepbook-indexer/src/events.rs

@@ -0,0 +1,131 @@
+// Copyright (c) Mysten Labs, Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+use serde::{Deserialize, Serialize};
+use sui_types::base_types::{ObjectID, SuiAddress};
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveOrderFilledEvent {
+    pub pool_id: ObjectID,
+    pub maker_order_id: u128,
+    pub taker_order_id: u128,
+    pub maker_client_order_id: u64,
+    pub taker_client_order_id: u64,
+    pub price: u64,
+    pub taker_is_bid: bool,
+    pub taker_fee: u64,
+    pub maker_fee: u64,
+    pub base_quantity: u64,
+    pub quote_quantity: u64,
+    pub maker_balance_manager_id: ObjectID,
+    pub taker_balance_manager_id: ObjectID,
+    pub timestamp: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveOrderCanceledEvent {
+    pub balance_manager_id: ObjectID,
+    pub pool_id: ObjectID,
+    pub order_id: u128,
+    pub client_order_id: u64,
+    pub trader: SuiAddress,
+    pub price: u64,
+    pub is_bid: bool,
+    pub original_quantity: u64,
+    pub base_asset_quantity_canceled: u64,
+    pub timestamp: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveOrderModifiedEvent {
+    pub balance_manager_id: ObjectID,
+    pub pool_id: ObjectID,
+    pub order_id: u128,
+    pub client_order_id: u64,
+    pub trader: SuiAddress,
+    pub price: u64,
+    pub is_bid: bool,
+    pub new_quantity: u64,
+    pub timestamp: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveOrderPlacedEvent {
+    pub balance_manager_id: ObjectID,
+    pub pool_id: ObjectID,
+    pub order_id: u128,
+    pub client_order_id: u64,
+    pub trader: SuiAddress,
+    pub price: u64,
+    pub is_bid: bool,
+    pub placed_quantity: u64,
+    pub expire_timestamp: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MovePriceAddedEvent {
+    pub conversion_rate: u64,
+    pub timestamp: u64,
+    pub is_base_conversion: bool,
+    pub reference_pool: ObjectID,
+    pub target_pool: ObjectID,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveFlashLoanBorrowedEvent {
+    pub pool_id: ObjectID,
+    pub borrow_quantity: u64,
+    pub type_name: String,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveBalanceEvent {
+    pub balance_manager_id: ObjectID,
+    pub asset: String,
+    pub amount: u64,
+    pub deposit: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveTradeParamsUpdateEvent {
+    pub taker_fee: u64,
+    pub maker_fee: u64,
+    pub stake_required: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveStakeEvent {
+    pub pool_id: ObjectID,
+    pub balance_manager_id: ObjectID,
+    pub epoch: u64,
+    pub amount: u64,
+    pub stake: bool,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveProposalEvent {
+    pub pool_id: ObjectID,
+    pub balance_manager_id: ObjectID,
+    pub epoch: u64,
+    pub taker_fee: u64,
+    pub maker_fee: u64,
+    pub stake_required: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveVoteEvent {
+    pub pool_id: ObjectID,
+    pub balance_manager_id: ObjectID,
+    pub epoch: u64,
+    pub from_proposal_id: Option<ObjectID>,
+    pub to_proposal_id: ObjectID,
+    pub stake: u64,
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct MoveRebateEvent {
+    pub pool_id: ObjectID,
+    pub balance_manager_id: ObjectID,
+    pub epoch: u64,
+    pub claim_amount: u64,
+}