chore: bump versions, add batched Merkle tree placeholder bytes (#1767)

ananas-block · web-flow · commit b0f456fbc987 · 2025-05-30T23:53:05.000+01:00
* chore: bump light-batched-merkle-tree -&gt; 0.2.0, light-merkle-tree-metadata -&gt; 0.2.0, light-concurrent-merkle-tree -&gt; 2.0.0, light-merkle-tree-reference 2.0.0

* chore: add batched-merkle-tree placeholder bytes

* regenerate accounts

* fix: make concurrent mt *mut fields private

* chore: bump light-hash-set version -&gt; 2.0.0

* bump: light-bloom-filter version -&gt; 0.2.0

* fix: light-test-utils usage of concurrent-merkle-tree

* chore: bump concurrent-merkle-tree version -&gt; 2.0.1

* chore: bump light-indexed-merkle-tree version -&gt; 2.0.0
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -139,13 +139,13 @@ ark-std = "0.5"
 
 
 # Light Protocol
-light-hash-set = { version = "1.2.0", path = "program-libs/hash-set" }
-light-indexed-merkle-tree = { version = "1.1.0", path = "program-libs/indexed-merkle-tree" }
-light-concurrent-merkle-tree = { version = "1.1.0", path = "program-libs/concurrent-merkle-tree" }
+light-hash-set = { version = "2.0.0", path = "program-libs/hash-set" }
+light-indexed-merkle-tree = { version = "2.0.0", path = "program-libs/indexed-merkle-tree" }
+light-concurrent-merkle-tree = { version = "2.0.1", path = "program-libs/concurrent-merkle-tree" }
 light-client = { path = "sdk-libs/client", version = "0.9.1" }
 light-hasher = { path = "program-libs/hasher", version = "3.0.0" }
 light-macros = { path = "program-libs/macros", version = "2.0.0" }
-light-merkle-tree-reference = { path = "program-tests/merkle-tree", version = "1.1.0" }
+light-merkle-tree-reference = { path = "program-tests/merkle-tree", version = "2.0.0" }
 light-heap = { path = "program-libs/heap", version = "1.1.0" }
 light-prover-client = { path = "prover/client", version = "1.3.0" }
 light-sdk = { path = "sdk-libs/sdk", version = "0.12.0" }
@@ -172,11 +172,11 @@ create-address-test-program = { path = "program-tests/create-address-test-progra
     "cpi",
 ] }
 light-program-test = { path = "sdk-libs/program-test", version = "0.1.0" }
-light-batched-merkle-tree = { path = "program-libs/batched-merkle-tree", version = "0.1.0" }
-light-merkle-tree-metadata = { path = "program-libs/merkle-tree-metadata", version = "0.1.0" }
+light-batched-merkle-tree = { path = "program-libs/batched-merkle-tree", version = "0.2.0" }
+light-merkle-tree-metadata = { path = "program-libs/merkle-tree-metadata", version = "0.2.0" }
 aligned-sized = { path = "program-libs/aligned-sized", version = "1.1.0" }
-light-bloom-filter = { path = "program-libs/bloom-filter", version = "0.1.0" }
-light-bounded-vec = { version = "1.1.0", git = "https://github.com/lightprotocol/program-libs", rev = "f5aa99e4617672bab00073377a2db92e42610a73" }
+light-bloom-filter = { path = "program-libs/bloom-filter", version = "0.2.0" }
+light-bounded-vec = { version = "2.0.0" }
 light-poseidon = { version = "0.3.0" }
 light-test-utils = { path = "program-tests/utils", version = "1.2.1" }
 light-indexed-array = { path = "program-libs/indexed-array", version = "0.1.0" }
diff --git a/cli/accounts/address_merkle_tree_amt1Ayt45jfbdw5YSo7iz6WZxUmnZsQTYXy82hVwyC2.json b/cli/accounts/address_merkle_tree_amt1Ayt45jfbdw5YSo7iz6WZxUmnZsQTYXy82hVwyC2.json
diff --git a/cli/accounts/batch_address_merkle_tree_EzKE84aVTkCUhDHLELqyJaq1Y7UVVmqxXqZjVHwHY3rK.json b/cli/accounts/batch_address_merkle_tree_EzKE84aVTkCUhDHLELqyJaq1Y7UVVmqxXqZjVHwHY3rK.json
diff --git a/cli/accounts/batch_state_merkle_tree_HLKs5NJ8FXkJg8BrzJt56adFYYuwg5etzDtBbQYTsixu.json b/cli/accounts/batch_state_merkle_tree_HLKs5NJ8FXkJg8BrzJt56adFYYuwg5etzDtBbQYTsixu.json
diff --git a/cli/accounts/group_pda_32tgQWfaASE7SfTQvMaSJDrqe728tJQ2SGzxtDDrpDwS.json b/cli/accounts/group_pda_32tgQWfaASE7SfTQvMaSJDrqe728tJQ2SGzxtDDrpDwS.json
@@ -0,0 +1 @@
+{"pubkey":"32tgQWfaASE7SfTQvMaSJDrqe728tJQ2SGzxtDDrpDwS","account":{"lamports":1392000,"data":["D88EoH8mjqIQpl591jkE+Dg60Dze4Hi5FNhzGIA7FTWAcNeS4Fz950vfSocYeLHitXp0BCgI+ALv3zEIJJy2OBeyZp+6b7UE","base64"],"owner":"compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq","executable":false,"rentEpoch":18446744073709551615,"space":72}}
diff --git a/program-libs/batched-merkle-tree/Cargo.toml b/program-libs/batched-merkle-tree/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "light-batched-merkle-tree"
-version = "0.1.0"
+version = "0.2.0"
 description = "Batch Merkle tree implementation."
 repository = "https://github.com/Lightprotocol/light-protocol"
 license = "Apache-2.0"
@@ -62,7 +62,7 @@ tokio = { workspace = true }
 serial_test = { workspace = true }
 light-account-checks = { workspace = true, features = ["test-only"] }
 light-compressed-account = { workspace = true, features = ["new-unique"] }
-light-test-utils = { workspace = true, features = [ "test-only"]}
+light-test-utils = { workspace = true, features = ["test-only"] }
 
 [lints.rust.unexpected_cfgs]
 level = "allow"
diff --git a/program-libs/batched-merkle-tree/src/merkle_tree.rs b/program-libs/batched-merkle-tree/src/merkle_tree.rs
@@ -1115,7 +1115,7 @@ mod test {
     #[test]
     fn test_zero_out() {
         let current_slot = 1;
-        let mut account_data = vec![0u8; 3184];
+        let mut account_data = vec![0u8; 3312];
         let batch_size = 4;
         let zkp_batch_size = 1;
         let num_zkp_updates = batch_size / zkp_batch_size;
@@ -1503,7 +1503,7 @@ mod test {
 
     #[test]
     fn test_check_queue_next_index_reached_tree_capacity() {
-        let mut account_data = vec![0u8; 15728];
+        let mut account_data = vec![0u8; 15856];
         let batch_size = 200;
         let zkp_batch_size = 1;
         let root_history_len = 10;
@@ -1557,7 +1557,7 @@ mod test {
 
     #[test]
     fn test_check_non_inclusion() {
-        let mut account_data = vec![0u8; 3248];
+        let mut account_data = vec![0u8; 3376];
         let batch_size = 5;
         let zkp_batch_size = 1;
         let root_history_len = 10;
@@ -1657,7 +1657,7 @@ mod test {
 
     #[test]
     fn test_tree_is_full() {
-        let mut account_data = vec![0u8; 3248];
+        let mut account_data = vec![0u8; 3376];
         let batch_size = 5;
         let zkp_batch_size = 1;
         let root_history_len = 10;
@@ -1692,7 +1692,7 @@ mod test {
     }
     #[test]
     fn test_increment_next_index() {
-        let mut account_data = vec![0u8; 3248];
+        let mut account_data = vec![0u8; 3376];
         let batch_size = 5;
         let zkp_batch_size = 1;
         let root_history_len = 10;
@@ -1730,7 +1730,7 @@ mod test {
 
     #[test]
     fn test_get_pubkey_and_associated_queue() {
-        let mut account_data = vec![0u8; 3248];
+        let mut account_data = vec![0u8; 3376];
         let batch_size = 5;
         let zkp_batch_size = 1;
         let root_history_len = 10;
diff --git a/program-libs/batched-merkle-tree/src/merkle_tree_metadata.rs b/program-libs/batched-merkle-tree/src/merkle_tree_metadata.rs
@@ -46,6 +46,7 @@ pub struct BatchedMerkleTreeMetadata {
     /// + 1 byte padding) Merkle tree pubkey.
     pub hashed_pubkey: [u8; 32],
     pub nullifier_next_index: u64,
+    pub placeholder_bytes: [u8; 128],
 }
 
 impl Default for BatchedMerkleTreeMetadata {
@@ -68,6 +69,7 @@ impl Default for BatchedMerkleTreeMetadata {
             },
             hashed_pubkey: [0u8; 32],
             nullifier_next_index: 0,
+            placeholder_bytes: [0u8; 128],
         }
     }
 }
@@ -158,6 +160,7 @@ impl BatchedMerkleTreeMetadata {
             capacity: 2u64.pow(height),
             hashed_pubkey: hash_to_bn254_field_size_be(&tree_pubkey.to_bytes()),
             nullifier_next_index: 0,
+            placeholder_bytes: [0u8; 128],
         }
     }
 }
diff --git a/program-libs/batched-merkle-tree/tests/rollover_address_tree.rs b/program-libs/batched-merkle-tree/tests/rollover_address_tree.rs
@@ -283,7 +283,7 @@ fn test_rnd_rollover() {
 /// 6. failing: rollower threshold not set
 #[test]
 fn test_batched_tree_is_ready_for_rollover() {
-    let mut account_data = vec![0u8; 6128];
+    let mut account_data = vec![0u8; 6256];
     let batch_size = 50;
     let zkp_batch_size = 1;
     let root_history_len = 10;
diff --git a/program-libs/bloom-filter/Cargo.toml b/program-libs/bloom-filter/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "light-bloom-filter"
-version = "0.1.0"
+version = "0.2.0"
 description = "Experimental bloom filter."
 repository = "https://github.com/Lightprotocol/light-protocol"
 license = "Apache-2.0"
diff --git a/program-libs/concurrent-merkle-tree/Cargo.toml b/program-libs/concurrent-merkle-tree/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "light-concurrent-merkle-tree"
-version = "1.1.0"
+version = "2.0.1"
 edition = "2021"
 description = "Concurrent Merkle tree implementation"
 repository = "https://github.com/Lightprotocol/light-protocol"
diff --git a/program-libs/concurrent-merkle-tree/src/lib.rs b/program-libs/concurrent-merkle-tree/src/lib.rs
@@ -51,9 +51,9 @@ where
     pub height: usize,
     pub canopy_depth: usize,
 
-    pub next_index: *mut usize,
-    pub sequence_number: *mut usize,
-    pub rightmost_leaf: *mut [u8; 32],
+    next_index: *mut usize,
+    sequence_number: *mut usize,
+    rightmost_leaf: *mut [u8; 32],
 
     /// Hashes of subtrees.
     pub filled_subtrees: BoundedVec<[u8; 32]>,
@@ -262,7 +262,7 @@ where
         unsafe { *self.next_index }
     }
 
-    fn inc_next_index(&mut self) -> Result<(), ConcurrentMerkleTreeError> {
+    pub fn inc_next_index(&mut self) -> Result<(), ConcurrentMerkleTreeError> {
         unsafe {
             *self.next_index = self
                 .next_index()
@@ -276,7 +276,7 @@ where
         unsafe { *self.sequence_number }
     }
 
-    fn inc_sequence_number(&mut self) -> Result<(), ConcurrentMerkleTreeError> {
+    pub fn inc_sequence_number(&mut self) -> Result<(), ConcurrentMerkleTreeError> {
         unsafe {
             *self.sequence_number = self
                 .sequence_number()
diff --git a/program-libs/hash-set/Cargo.toml b/program-libs/hash-set/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "light-hash-set"
-version = "1.2.0"
+version = "2.0.0"
 description = "Hash set which can be stored on a Solana account"
 repository = "https://github.com/Lightprotocol/light-protocol"
 license = "Apache-2.0"
@@ -16,9 +16,6 @@ solana-program-error = { workspace = true, optional = true }
 thiserror = { workspace = true }
 light-hasher = { workspace = true }
 
-[target.'cfg(target_os = "solana")'.dependencies]
-light-heap = { workspace = true }
-
 [dev-dependencies]
 ark-bn254 = { workspace = true }
 ark-ff = { workspace = true }
diff --git a/program-libs/indexed-merkle-tree/Cargo.toml b/program-libs/indexed-merkle-tree/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "light-indexed-merkle-tree"
-version = "1.1.0"
+version = "2.0.0"
 description = "Implementation of indexed (and concurrent) Merkle tree in Rust"
 repository = "https://github.com/Lightprotocol/light-protocol"
 license = "Apache-2.0"
diff --git a/program-libs/merkle-tree-metadata/Cargo.toml b/program-libs/merkle-tree-metadata/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 name = "light-merkle-tree-metadata"
-version = "0.1.0"
-description = "Merkle tree utils."
+version = "0.2.0"
+description = "Merkle tree metadata for light-concurrent-merkle-tree, light-indexed-merkle-tree, light-batched-merkle-tree."
 repository = "https://github.com/Lightprotocol/light-protocol"
 license = "Apache-2.0"
 edition = "2021"
diff --git a/program-tests/account-compression-test/tests/merkle_tree_tests.rs b/program-tests/account-compression-test/tests/merkle_tree_tests.rs
@@ -2186,8 +2186,8 @@ pub async fn set_state_merkle_tree_sequence<R: RpcConnection + TestRpc>(
                 &mut merkle_tree.data[8 + mem::size_of::<StateMerkleTreeAccount>()..],
             )
             .unwrap();
-        unsafe {
-            *merkle_tree_deserialized.sequence_number = sequence_number as usize;
+        while merkle_tree_deserialized.sequence_number() < sequence_number as usize {
+            merkle_tree_deserialized.inc_sequence_number().unwrap();
         }
     }
     let mut account_share_data = AccountSharedData::from(merkle_tree);
diff --git a/program-tests/merkle-tree/Cargo.toml b/program-tests/merkle-tree/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "light-merkle-tree-reference"
-version = "1.1.0"
+version = "2.0.0"
 description = "Non-sparse reference Merkle tree implementation"
 repository = "https://github.com/Lightprotocol/light-protocol"
 license = "Apache-2.0"
diff --git a/program-tests/system-test/tests/test.rs b/program-tests/system-test/tests/test.rs
@@ -1619,8 +1619,9 @@ async fn regenerate_accounts() {
         report_work_phase_length: 100,
         ..ProtocolConfig::default()
     };
-    let mut config = ProgramTestConfig::new(false, None);
+    let mut config = ProgramTestConfig::default_with_batched_trees(false);
     config.protocol_config = protocol_config;
+    config.skip_protocol_init = true;
     let mut rpc = LightProgramTest::new(config).await.unwrap();
     let keypairs = for_regenerate_accounts();
 
diff --git a/program-tests/utils/src/address_tree_rollover.rs b/program-tests/utils/src/address_tree_rollover.rs
@@ -47,8 +47,8 @@ pub async fn set_address_merkle_tree_next_index<R: RpcConnection + TestRpc + Ind
             &mut merkle_tree.data[8 + std::mem::size_of::<AddressMerkleTreeAccount>()..],
         )
         .unwrap();
-    unsafe {
-        *merkle_tree_deserialized.next_index = next_index as usize;
+    while merkle_tree_deserialized.next_index() < next_index as usize {
+        merkle_tree_deserialized.inc_next_index().unwrap();
     }
     let mut account_share_data = AccountSharedData::from(merkle_tree);
     account_share_data.set_lamports(lamports);
diff --git a/program-tests/utils/src/state_tree_rollover.rs b/program-tests/utils/src/state_tree_rollover.rs
@@ -135,8 +135,8 @@ pub async fn set_state_merkle_tree_next_index<R: RpcConnection + TestRpc + Index
                         &mut merkle_tree.data[8 + std::mem::size_of::<StateMerkleTreeAccount>()..],
                     )
                     .unwrap();
-                unsafe {
-                    *merkle_tree_deserialized.next_index = next_index as usize;
+                while merkle_tree_deserialized.next_index() < next_index as usize {
+                    merkle_tree_deserialized.inc_next_index().unwrap();
                 }
             }
 
diff --git a/sdk-libs/program-test/src/program_test/config.rs b/sdk-libs/program-test/src/program_test/config.rs
@@ -23,6 +23,7 @@ pub struct ProgramTestConfig {
     pub v1_address_queue_config: AddressQueueConfig,
     pub v2_state_tree_config: Option<InitStateTreeAccountsInstructionData>,
     pub v2_address_tree_config: Option<InitAddressTreeAccountsInstructionData>,
+    pub skip_protocol_init: bool,
 }
 
 impl ProgramTestConfig {
@@ -110,6 +111,7 @@ impl Default for ProgramTestConfig {
             v1_nullifier_queue_config: NullifierQueueConfig::default(),
             v2_state_tree_config: None,
             v2_address_tree_config: None,
+            skip_protocol_init: false,
         }
     }
 }
diff --git a/sdk-libs/program-test/src/program_test/light_program_test.rs b/sdk-libs/program-test/src/program_test/light_program_test.rs
@@ -57,12 +57,15 @@ impl LightProgramTest {
         )
         .await?;
         airdrop_lamports(&mut context, &keypairs.forester.pubkey(), 10_000_000_000).await?;
-        let test_accounts = initialize_accounts(&mut context, &config, keypairs).await?;
-        let batch_size = config
-            .v2_state_tree_config
-            .as_ref()
-            .map(|config| config.output_queue_batch_size as usize);
-        context.add_indexer(&test_accounts, batch_size).await?;
+
+        if !config.skip_protocol_init {
+            let test_accounts = initialize_accounts(&mut context, &config, keypairs).await?;
+            let batch_size = config
+                .v2_state_tree_config
+                .as_ref()
+                .map(|config| config.output_queue_batch_size as usize);
+            context.add_indexer(&test_accounts, batch_size).await?;
+        }
 
         // Will always start a prover server.
         #[cfg(feature = "devenv")]

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{"pubkey":"32tgQWfaASE7SfTQvMaSJDrqe728tJQ2SGzxtDDrpDwS","account":{"lamports":1392000,"data":["D88EoH8mjqIQpl591jkE+Dg60Dze4Hi5FNhzGIA7FTWAcNeS4Fz950vfSocYeLHitXp0BCgI+ALv3zEIJJy2OBeyZp+6b7UE","base64"],"owner":"compr6CUsB5m2jS4Y3831ztGSTnDpnKJTKS95d64XVq","executable":false,"rentEpoch":18446744073709551615,"space":72}}`
Original file line number	Diff line number	Diff line change
`@@ -46,6 +46,7 @@ pub struct BatchedMerkleTreeMetadata {`
`46`	`46`	`/// + 1 byte padding) Merkle tree pubkey.`
`47`	`47`	`pub hashed_pubkey: [u8; 32],`
`48`	`48`	`pub nullifier_next_index: u64,`
	`49`	`+ pub placeholder_bytes: [u8; 128],`
`49`	`50`	`}`
`50`	`51`
`51`	`52`	`impl Default for BatchedMerkleTreeMetadata {`
`@@ -68,6 +69,7 @@ impl Default for BatchedMerkleTreeMetadata {`
`68`	`69`	`},`
`69`	`70`	`hashed_pubkey: [0u8; 32],`
`70`	`71`	`nullifier_next_index: 0,`
	`72`	`+ placeholder_bytes: [0u8; 128],`
`71`	`73`	`}`
`72`	`74`	`}`
`73`	`75`	`}`
`@@ -158,6 +160,7 @@ impl BatchedMerkleTreeMetadata {`
`158`	`160`	`capacity: 2u64.pow(height),`
`159`	`161`	`hashed_pubkey: hash_to_bn254_field_size_be(&tree_pubkey.to_bytes()),`
`160`	`162`	`nullifier_next_index: 0,`
	`163`	`+ placeholder_bytes: [0u8; 128],`
`161`	`164`	`}`
`162`	`165`	`}`
`163`	`166`	`}`