perf: indexed array remove next index (#1629)

* created separate indexed array crate, updated MockBatchedAddressForester, created indexed tree in test mt

* refactor: remove next_index from non-inclusion circuit

* chore: debugged tests: batched merkle tree, light-prover-client, added new reference indexed array and Merkle tree

* stash, empty_address_queue_test v1 wip

* more fixes, e2e tests work

* fix: e2e, registry tests

* fix: concurrent tree tests

* chore: clean up light-indexed-array

* cleanup and regenerate batched address tree account

* chore: updated proving keys download url

* Update GitHub Actions to use 8vCPU runner for prover tests

* remove warnings

* Update prover/server/prover/proving_keys_utils.go

* chore: add comma for ,input

Co-authored-by: Sergey Timoshin <sergeytimoshin@proton.me>

* chore: remove leftover

* remove unwrap

* remove legacy tests

---------

Co-authored-by: Sergey Timoshin <sergeytimoshin@proton.me>

Author: ananas-block

.github/workflows/prover-test.yml

@@ -24,7 +24,7 @@ on:
 jobs:
   build-and-test:
     if: github.event.pull_request.draft == false
-    runs-on: buildjet-4vcpu-ubuntu-2204
+    runs-on: buildjet-8vcpu-ubuntu-2204
     timeout-minutes: 120
     steps:
       - name: Checkout sources

.gitignore

@@ -83,3 +83,4 @@ output.txt
 .nx/cache
 .nx/workspace-data
 output1.txt
+.zed

Cargo.lock

@@ -12,6 +12,7 @@ members = [
     "program-libs/concurrent-merkle-tree",
     "program-libs/hash-set",
     "program-libs/indexed-merkle-tree",
+    "program-libs/indexed-array",
     "programs/account-compression",
     "programs/system",
     "programs/compressed-token",
@@ -145,6 +146,7 @@ light-bloom-filter = { path = "program-libs/bloom-filter", version = "0.1.0" }
 light-bounded-vec = { version = "1.1.0" }
 light-poseidon = { version = "0.3.0" }
 light-test-utils = { path = "program-tests/utils", version = "1.2.1" }
+light-indexed-array = { path = "program-libs/indexed-array", version = "0.1.0" }
 create-address-program-test = { path = "program-tests/create-address-test-program", version = "1.0.0" }
 groth16-solana = { version = "0.0.3" }
 # anchor 0.29 requires 1.19

Cargo.toml

@@ -40,6 +40,12 @@ pub enum ForesterUtilsError {
     RpcError(String),
     #[error("indexer error: {0:?}")]
     IndexerError(String),
+    #[error("Converted indexer error: {0}")]
+    ConvertedIndexerError(#[from] light_client::indexer::IndexerError),
+    #[error("Converted rpc error: {0}")]
+    ConvertedRpcError(#[from] light_client::rpc::RpcError),
+    #[error("Batched Merkle tree error: {0}")]
+    BatchedMerkleTreeError(#[from] light_batched_merkle_tree::errors::BatchedMerkleTreeError),
 }
 
 pub async fn create_batch_update_address_tree_instruction_data<R, I>(
@@ -65,8 +71,7 @@ where
         let merkle_tree = BatchedMerkleTreeAccount::address_from_bytes(
             merkle_tree_account.data.as_mut_slice(),
             &merkle_tree_pubkey.into(),
-        )
-        .unwrap();
+        )?;
 
         let full_batch_index = merkle_tree.queue_batches.pending_batch_index;
         let batch = &merkle_tree.queue_batches.batches[full_batch_index as usize];
@@ -85,7 +90,11 @@ where
         .iter()
         .find(|x| x.accounts.merkle_tree == merkle_tree_pubkey)
         .unwrap()
-        .merkle_tree
+        .get_v2_indexed_merkle_tree()
+        .ok_or(ForesterUtilsError::IndexerError(format!(
+            "Merkle tree {:?} is not a batched address Merkle tree",
+            merkle_tree_pubkey
+        )))?
         .merkle_tree
         .rightmost_index;
 
@@ -221,8 +230,7 @@ pub async fn create_append_batch_ix_data<R: RpcConnection, I: Indexer<R>>(
         let merkle_tree = BatchedMerkleTreeAccount::state_from_bytes(
             merkle_tree_account.data.as_mut_slice(),
             &merkle_tree_pubkey.into(),
-        )
-        .unwrap();
+        )?;
 
         (
             merkle_tree.next_index,
@@ -239,8 +247,7 @@ pub async fn create_append_batch_ix_data<R: RpcConnection, I: Indexer<R>>(
     let (zkp_batch_size, leaves_hash_chain) = {
         let mut output_queue_account = rpc.get_account(output_queue_pubkey).await.unwrap().unwrap();
         let output_queue =
-            BatchedQueueAccount::output_from_bytes(output_queue_account.data.as_mut_slice())
-                .unwrap();
+            BatchedQueueAccount::output_from_bytes(output_queue_account.data.as_mut_slice())?;
 
         let full_batch_index = output_queue.batch_metadata.pending_batch_index;
         let zkp_batch_size = output_queue.batch_metadata.zkp_batch_size;
@@ -366,8 +373,7 @@ pub async fn create_nullify_batch_ix_data<R: RpcConnection, I: Indexer<R>>(
         let merkle_tree = BatchedMerkleTreeAccount::state_from_bytes(
             account.data.as_mut_slice(),
             &merkle_tree_pubkey.into(),
-        )
-        .unwrap();
+        )?;
 
         log::debug!("queue_batches: {:?}", merkle_tree.queue_batches);
 

cli/accounts/batch_address_merkle_tree_EzKE84aVTkCUhDHLELqyJaq1Y7UVVmqxXqZjVHwHY3rK.json

@@ -19,8 +19,8 @@ pub const DEFAULT_ZKP_BATCH_SIZE: u64 = 500;
 pub const DEFAULT_CPI_CONTEXT_ACCOUNT_SIZE: u64 = 20 * 1024 + 8;
 
 pub const ADDRESS_TREE_INIT_ROOT_40: [u8; 32] = [
-    12, 181, 82, 66, 55, 110, 245, 173, 233, 250, 111, 145, 10, 171, 183, 79, 64, 245, 169, 160, 8,
-    34, 54, 170, 14, 74, 170, 75, 72, 147, 141, 17,
+    28, 65, 107, 255, 208, 234, 51, 3, 131, 95, 62, 130, 202, 177, 176, 26, 216, 81, 64, 184, 200,
+    25, 95, 124, 248, 129, 44, 109, 229, 146, 106, 76,
 ];
 
 pub const ACCOUNT_COMPRESSION_PROGRAM_ID: solana_program::pubkey::Pubkey =

forester-utils/src/instructions.rs

@@ -298,7 +298,7 @@ impl<'a> BatchedMerkleTreeAccount<'a> {
             // 2. element:
             // H(452312848583266388373324160190187140051835877600158453279131187530910662655, 0, 0)
             // ... other elements: 0
-            account_metadata.next_index = 2;
+            account_metadata.next_index = 1;
         }
         let next_index = account_metadata.next_index;
 

program-libs/batched-merkle-tree/src/constants.rs

@@ -104,7 +104,7 @@ impl BatchedMerkleTreeMetadata {
             rollover_fee,
         );
         // inited address tree contains two elements.
-        tree.next_index = 2;
+        tree.next_index = 1;
         tree
     }
 
@@ -154,7 +154,7 @@ impl BatchedMerkleTreeMetadata {
                 zkp_batch_size,
                 num_iters,
                 if tree_type == TreeType::BatchedAddress {
-                    2
+                    1
                 } else {
                     0
                 },

program-libs/batched-merkle-tree/src/merkle_tree.rs

@@ -1300,14 +1300,15 @@ pub async fn perform_address_update(
                 account.get_metadata().queue_batches.batch_size as u32,
                 account.get_metadata().queue_batches.zkp_batch_size as u32,
                 *leaves_hash_chain,
-                next_index as usize, // % batch.zkp_batch_size as usize
+                next_index as usize,
                 batch_start_index as usize,
                 *current_root,
             )
             .await
             .unwrap();
 
         mock_indexer.finalize_batch_address_update(10);
+        assert_eq!(mock_indexer.merkle_tree.root(), new_root);
         let instruction_data = InstructionDataBatchNullifyInputs {
             new_root,
             compressed_proof: CompressedProof {