refactor: prover client (#1773)

* refactor: indexer in light-client

refactor: light-client ProofRpcResultV2

refactor: add from v1 proof result

refactor: light-client unify get_validity_proof and v2

refactor: light client unify get_compressed_account_by_owner and change return type to Account

refactor: light-client get_compressed_token_accounts return type and unify v2

refactor: get compressed token balances

stopped compiling conversion.rs

refactor: light-client add response types to Indexer trait

stash

stash

fix: lint in light client

fixed forester-utils

program test compiles

fix: light-test-utils

fix: forester

system-cpi-test compiles

fix: system-test

fixed: light-compressed-token

fixed: program-tests

fix: forester test and lint

uncomment test-sbf feature

remove unused conversions.rs

fix: forester test and lint

fix: sdk test

fix: tests

fix: get_validity_proof

feat: extend Indexer trait

stash

simplify ItemsWithCursor

fixed forester utils

fixed: light-program-test, light-test-utils, forester-utils

fix: tests

chore: rename get_compressed_token_balances_by_owner -> get_compressed_token_balances_by_owner_v2

use filter in get_compressed_accounts

refactor: rename ValiditProofWithContext compressed_proof -> proof

chore: add cpi context pubkeys to QUEUE_TREE_MAPPING and TreeInfo

fix: compile error

feat: add sparse Merkle tree crate

refactor: light-prover-client structure

fix: tests

* fix: compilation

* fix: system cpit test v2

* fix: compilation

* fix: lint

* fix

Author: ananas-block

.github/workflows/rust.yml

@@ -62,6 +62,7 @@ jobs:
               cargo test -p light-sdk
               cargo test -p light-program-test
               cargo test -p light-client
+              cargo test -p light-sparse-merkle-tree
               cargo test -p light-batched-merkle-tree --features test-only -- --skip test_simulate_transactions --skip test_e2e
 
     name: Test ${{ matrix.group.name }}

Cargo.lock

@@ -43,6 +43,7 @@ members = [
     "program-tests/merkle-tree",
     "forester-utils",
     "forester",
+    "sparse-merkle-tree",
 ]
 
 resolver = "2"
@@ -67,6 +68,8 @@ solana-client = "2.2.1"
 solana-cli-output = "2.2.1"
 solana-transaction-status = "2.2.1"
 solana-account-decoder = "2.2.1"
+solana-account-decoder-client-types = "2.2.1"
+solana-transaction-status-client-types = "2.2.1"
 solana-rpc = "2.2.1"
 solana-rpc-client-api = "2.2.1"
 solana-transaction-context = "2.2.1"
@@ -142,6 +145,7 @@ ark-std = "0.5"
 light-hash-set = { version = "2.0.0", path = "program-libs/hash-set" }
 light-indexed-merkle-tree = { version = "2.0.0", path = "program-libs/indexed-merkle-tree" }
 light-concurrent-merkle-tree = { version = "2.0.1", path = "program-libs/concurrent-merkle-tree" }
+light-sparse-merkle-tree = { version = "0.1.0", path = "sparse-merkle-tree" }
 light-client = { path = "sdk-libs/client", version = "0.9.1" }
 light-hasher = { path = "program-libs/hasher", version = "3.0.0" }
 light-macros = { path = "program-libs/macros", version = "2.0.0" }

Cargo.toml

@@ -20,7 +20,7 @@ test-sbf = []
 [dependencies]
 aligned-sized = { workspace = true }
 anchor-lang = { workspace = true }
-account-compression = { workspace = true }
+account-compression = { workspace = true, features = ["cpi"] }
 light-compressed-account = { workspace = true }
 light-zero-copy = { workspace = true }
 zerocopy = { workspace = true }

anchor-programs/system/Cargo.toml

@@ -11,7 +11,7 @@ use light_program_test::{
     test_env::{setup_test_programs_with_accounts_v2, TestAccounts},
     program_test::LightProgramTest,
 };
-use light_prover_client::gnark::helpers::{spawn_prover, ProverConfig, ProverMode};
+use light_prover_client::prover::{spawn_prover, ProverConfig, ProverMode};
 use light_sdk::{
     account_meta::LightAccountMeta,
     address::derive_address,

examples/anchor/memo/tests/test.rs

@@ -13,7 +13,7 @@ use light_program_test::{
     test_env::{setup_test_programs_with_accounts_v2, TestAccounts},
     program_test::LightProgramTest,
 };
-use light_prover_client::gnark::helpers::{spawn_prover, ProverConfig};
+use light_prover_client::prover::{spawn_prover, ProverConfig};
 use light_sdk::{
     account_meta::LightAccountMeta,
     address::derive_address,

examples/anchor/name-service-without-macros/tests/test.rs

@@ -21,7 +21,7 @@ light-indexed-array = { workspace = true }
 light-compressed-account = { workspace = true }
 light-batched-merkle-tree = { workspace = true }
 light-merkle-tree-metadata = { workspace = true }
-light-merkle-tree-reference = { workspace = true }
+light-sparse-merkle-tree = { workspace = true }
 light-account-checks = { workspace = true }
 light-sdk = { workspace = true }
 

forester-utils/Cargo.toml

@@ -9,12 +9,14 @@ use light_batched_merkle_tree::{
 };
 use light_client::{indexer::Indexer, rpc::RpcConnection};
 use light_compressed_account::hash_chain::create_hash_chain_from_slice;
-use light_concurrent_merkle_tree::changelog::ChangelogEntry;
 use light_hasher::{bigint::bigint_to_be_bytes_array, Poseidon};
-use light_indexed_array::changelog::IndexedChangelogEntry;
-use light_merkle_tree_reference::sparse_merkle_tree::SparseMerkleTree;
 use light_prover_client::{
-    batch_address_append::get_batch_address_append_circuit_inputs, proof_client::ProofClient,
+    proof_client::ProofClient,
+    proof_types::batch_address_append::get_batch_address_append_circuit_inputs,
+};
+use light_sdk::verifier::CompressedProof;
+use light_sparse_merkle_tree::{
+    changelog::ChangelogEntry, indexed_changelog::IndexedChangelogEntry, SparseMerkleTree,
 };
 use tracing::{debug, error, info, warn};
 
@@ -263,7 +265,11 @@ where
                 debug!("Successfully generated proof for batch {}", i);
                 instruction_data_vec.push(InstructionDataAddressAppendInputs {
                     new_root,
-                    compressed_proof,
+                    compressed_proof: CompressedProof {
+                        a: compressed_proof.a,
+                        b: compressed_proof.b,
+                        c: compressed_proof.c,
+                    },
                 });
             }
             Err(e) => {

forester-utils/src/instructions/address_batch_update.rs

@@ -6,15 +6,15 @@ use light_batched_merkle_tree::{
 };
 use light_client::{indexer::Indexer, rpc::RpcConnection};
 use light_compressed_account::instruction_data::compressed_proof::CompressedProof;
-use light_concurrent_merkle_tree::changelog::ChangelogEntry;
 use light_hasher::bigint::bigint_to_be_bytes_array;
 use light_merkle_tree_metadata::QueueType;
 use light_prover_client::{
-    batch_append_with_proofs::{
+    proof_client::ProofClient,
+    proof_types::batch_append::{
         get_batch_append_with_proofs_inputs, BatchAppendWithProofsCircuitInputs,
     },
-    proof_client::ProofClient,
 };
+use light_sparse_merkle_tree::changelog::ChangelogEntry;
 use tracing::{error, trace};
 
 use crate::{error::ForesterUtilsError, utils::wait_for_indexer};
@@ -117,15 +117,15 @@ pub async fn create_append_batch_ix_data<R: RpcConnection, I: Indexer>(
         let adjusted_start_index =
             merkle_tree_next_index as u32 + (batch_idx * zkp_batch_size as usize) as u32;
 
-        let (circuit_inputs, batch_changelogs) = get_batch_append_with_proofs_inputs(
+        let (circuit_inputs, batch_changelogs) = get_batch_append_with_proofs_inputs::<32>(
             current_root,
             adjusted_start_index,
             leaves,
             *leaves_hash_chain,
             old_leaves,
             merkle_proofs,
             zkp_batch_size as u32,
-            &all_changelogs,
+            all_changelogs.as_slice(),
         )
         .map_err(|e| {
             error!("Failed to get circuit inputs: {:?}", e);
@@ -166,10 +166,18 @@ async fn generate_zkp_proof(
     circuit_inputs: BatchAppendWithProofsCircuitInputs,
 ) -> Result<(CompressedProof, [u8; 32]), ForesterUtilsError> {
     let proof_client = ProofClient::local();
-    proof_client
+    let (proof, new_root) = proof_client
         .generate_batch_append_proof(circuit_inputs)
         .await
-        .map_err(|e| ForesterUtilsError::Prover(e.to_string()))
+        .map_err(|e| ForesterUtilsError::Prover(e.to_string()))?;
+    Ok((
+        CompressedProof {
+            a: proof.a,
+            b: proof.b,
+            c: proof.c,
+        },
+        new_root,
+    ))
 }
 
 /// Get metadata from the Merkle tree account

forester-utils/src/instructions/state_batch_append.rs

@@ -8,8 +8,8 @@ use light_compressed_account::instruction_data::compressed_proof::CompressedProo
 use light_hasher::{bigint::bigint_to_be_bytes_array, Hasher, Poseidon};
 use light_merkle_tree_metadata::QueueType;
 use light_prover_client::{
-    batch_update::{get_batch_update_inputs, BatchUpdateCircuitInputs},
     proof_client::ProofClient,
+    proof_types::batch_update::{get_batch_update_inputs, BatchUpdateCircuitInputs},
 };
 use tracing::{error, trace};
 
@@ -247,8 +247,16 @@ async fn generate_nullify_zkp_proof(
     inputs: BatchUpdateCircuitInputs,
 ) -> Result<(CompressedProof, [u8; 32]), ForesterUtilsError> {
     let proof_client = ProofClient::local();
-    proof_client
+    let (proof, new_root) = proof_client
         .generate_batch_update_proof(inputs)
         .await
-        .map_err(|e| ForesterUtilsError::Prover(e.to_string()))
+        .map_err(|e| ForesterUtilsError::Prover(e.to_string()))?;
+    Ok((
+        CompressedProof {
+            a: proof.a,
+            b: proof.b,
+            c: proof.c,
+        },
+        new_root,
+    ))
 }