feat: add event queue seq number (#1586)

* feat: add event queue seq number

* fix: tests

* chore: updated photon

* fix: address Merkle tree test & remove comment

Author: ananas-block

js/stateless.js/src/programs/layout.ts

@@ -331,12 +331,28 @@ export function deserializeAppendNullifyCreateAddressInputsIndexer(
         addresses.push(address);
         offset += InsertAddressInputLayout.span;
     }
-    const sequenceNumbersCount = buffer.readUInt8(offset);
+    const outputSequenceNumbersCount = buffer.readUInt8(offset);
     offset += 1;
-    const sequence_numbers = [];
-    for (let i = 0; i < sequenceNumbersCount; i++) {
+    const output_sequence_numbers = [];
+    for (let i = 0; i < outputSequenceNumbersCount; i++) {
         const seq = MerkleTreeSequenceNumberLayout.decode(buffer, offset);
-        sequence_numbers.push(seq);
+        output_sequence_numbers.push(seq);
+        offset += MerkleTreeSequenceNumberLayout.span;
+    }
+    const inputSequenceNumbersCount = buffer.readUInt8(offset);
+    offset += 1;
+    const inputSequence_numbers = [];
+    for (let i = 0; i < inputSequenceNumbersCount; i++) {
+        const seq = MerkleTreeSequenceNumberLayout.decode(buffer, offset);
+        inputSequence_numbers.push(seq);
+        offset += MerkleTreeSequenceNumberLayout.span;
+    }
+    const addressSequenceNumbersCount = buffer.readUInt8(offset);
+    offset += 1;
+    const addressSequence_numbers = [];
+    for (let i = 0; i < addressSequenceNumbersCount; i++) {
+        const seq = MerkleTreeSequenceNumberLayout.decode(buffer, offset);
+        addressSequence_numbers.push(seq);
         offset += MerkleTreeSequenceNumberLayout.span;
     }
     const outputLeafIndicesCount = buffer.readUInt8(offset);
@@ -352,7 +368,7 @@ export function deserializeAppendNullifyCreateAddressInputsIndexer(
         leaves,
         nullifiers,
         addresses,
-        sequence_numbers,
+        sequence_numbers: output_sequence_numbers,
         output_leaf_indices,
     };
 }

js/stateless.js/tests/unit/utils/conversion.test.ts

@@ -32,8 +32,8 @@ describe('deserialize apc cpi', () => {
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
         1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0,
-        0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0,
+        0, 0, 0,
     ];
 
     it('deserialize acp cpi', () => {

program-libs/batched-merkle-tree/src/merkle_tree.rs

@@ -668,7 +668,10 @@ impl<'a> BatchedMerkleTreeAccount<'a> {
             Some(compressed_account_hash),
             None,
             current_slot,
-        )
+        )?;
+        // Increment queue next index so that the indexer can use it like a sequence number.
+        self.increment_queue_next_index();
+        Ok(())
     }
 
     pub fn insert_address_into_queue(

program-libs/batched-merkle-tree/tests/merkle_tree.rs

@@ -33,7 +33,6 @@ use light_compressed_account::{
     pubkey::Pubkey,
 };
 use light_hasher::{Hasher, Poseidon};
-use light_merkle_tree_metadata::merkle_tree::TreeType;
 use light_merkle_tree_reference::MerkleTree;
 use light_prover_client::{
     gnark::helpers::{spawn_prover, ProofType, ProverConfig},
@@ -137,10 +136,8 @@ pub fn assert_input_queue_insert(
             .batches
             .get_mut(inserted_batch_index)
             .unwrap();
-        // Address queue insertions append state, input queue insertions only nullify existing state
-        if pre_account.tree_type == TreeType::BatchedAddress as u64 {
-            pre_account.queue_batches.next_index += 1;
-        }
+
+        pre_account.queue_batches.next_index += 1;
 
         println!(
             "assert input queue batch update: expected_batch: {:?}",

program-libs/compressed-account/src/event.rs

@@ -31,12 +31,20 @@ pub struct PublicTransactionEvent {
     pub pubkey_array: Vec<Pubkey>,
     pub message: Option<Vec<u8>>,
 }
-
+#[derive(Debug, Clone)]
 pub struct NewAddress {
     pub address: [u8; 32],
     pub mt_pubkey: Pubkey,
 }
 
+#[derive(Debug, Clone)]
+pub struct BatchPublicTransactionEvent {
+    pub event: PublicTransactionEvent,
+    pub new_addresses: Vec<NewAddress>,
+    pub input_sequence_numbers: Vec<MerkleTreeSequenceNumber>,
+    pub address_sequence_numbers: Vec<MerkleTreeSequenceNumber>,
+}
+
 // TODO: remove unwraps
 /// We piece the event together from 2 instructions:
 /// 1. light_system_program::{Invoke, InvokeCpi, InvokeCpiReadOnly} (one of the 3)
@@ -54,15 +62,15 @@ pub struct NewAddress {
 pub fn event_from_light_transaction(
     instructions: &[Vec<u8>],
     remaining_accounts: Vec<Vec<Pubkey>>,
-) -> Result<(Option<PublicTransactionEvent>, Option<Vec<NewAddress>>), ZeroCopyError> {
+) -> Result<Option<BatchPublicTransactionEvent>, ZeroCopyError> {
     let mut event = PublicTransactionEvent::default();
     let mut ix_set_cpi_context = false;
     let found_event = instructions.iter().any(|x| {
         match_system_program_instruction(x, false, &mut event, &mut ix_set_cpi_context).unwrap()
     });
     println!("found event {}", found_event);
     if !found_event {
-        return Ok((None, None));
+        return Ok(None);
     }
     println!("ix_set_cpi_context {}", ix_set_cpi_context);
     // If an instruction set the cpi context add the instructions that set the cpi context.
@@ -74,6 +82,8 @@ pub fn event_from_light_transaction(
     }
     // New addresses in batched trees.
     let mut new_addresses = Vec::new();
+    let mut input_sequence_numbers = Vec::new();
+    let mut address_sequence_numbers = Vec::new();
     let pos = instructions.iter().enumerate().position(|(i, x)| {
         if remaining_accounts[i].len() < 3 {
             return false;
@@ -82,6 +92,8 @@ pub fn event_from_light_transaction(
             x,
             &mut event,
             &mut new_addresses,
+            &mut input_sequence_numbers,
+            &mut address_sequence_numbers,
             &remaining_accounts[i][2..],
         )
         .unwrap()
@@ -92,21 +104,25 @@ pub fn event_from_light_transaction(
         println!("remaining accounts {:?}", remaining_accounts);
         event.pubkey_array = remaining_accounts[pos][2..].to_vec().clone();
         println!("event pubkey array {:?}", event.pubkey_array);
-        let new_addresses = if new_addresses.is_empty() {
-            None
-        } else {
-            Some(new_addresses)
-        };
-        Ok((Some(event), new_addresses))
+        println!("input_sequence_numbers {:?}", input_sequence_numbers);
+        println!("address_sequence_numbers {:?}", address_sequence_numbers);
+        Ok(Some(BatchPublicTransactionEvent {
+            event,
+            new_addresses,
+            input_sequence_numbers,
+            address_sequence_numbers,
+        }))
     } else {
-        Ok((None, None))
+        Ok(None)
     }
 }
 
 pub fn match_account_compression_program_instruction(
     instruction: &[u8],
     event: &mut PublicTransactionEvent,
     new_addresses: &mut Vec<NewAddress>,
+    input_sequence_numbers: &mut Vec<MerkleTreeSequenceNumber>,
+    address_sequence_numbers: &mut Vec<MerkleTreeSequenceNumber>,
     accounts: &[Pubkey],
 ) -> Result<bool, ZeroCopyError> {
     if instruction.len() < 8 {
@@ -122,7 +138,7 @@ pub fn match_account_compression_program_instruction(
                 data.nullifiers.iter().map(|x| x.account_hash).collect();
             event.output_compressed_account_hashes = data.leaves.iter().map(|x| x.leaf).collect();
             event.sequence_numbers = data
-                .sequence_numbers
+                .output_sequence_numbers
                 .iter()
                 .map(|x| MerkleTreeSequenceNumber {
                     pubkey: x.pubkey.into(),
@@ -151,6 +167,22 @@ pub fn match_account_compression_program_instruction(
                     });
                 }
             });
+            data.input_sequence_numbers.iter().for_each(|x| {
+                if x.pubkey != Pubkey::default().into() {
+                    input_sequence_numbers.push(MerkleTreeSequenceNumber {
+                        pubkey: x.pubkey.into(),
+                        seq: x.seq.into(),
+                    });
+                }
+            });
+            data.address_sequence_numbers.iter().for_each(|x| {
+                if x.pubkey != Pubkey::default().into() {
+                    address_sequence_numbers.push(MerkleTreeSequenceNumber {
+                        pubkey: x.pubkey.into(),
+                        seq: x.seq.into(),
+                    });
+                }
+            });
             Ok(true)
         }
         _ => Ok(false),

program-libs/compressed-account/src/instruction_data/insert_into_queues.rs

@@ -65,7 +65,9 @@ pub struct InsertIntoQueuesInstructionData<'a> {
     pub leaves: ZeroCopySlice<'a, u8, AppendLeavesInput, false>,
     pub nullifiers: ZeroCopySlice<'a, u8, InsertNullifierInput, false>,
     pub addresses: ZeroCopySlice<'a, u8, InsertAddressInput, false>,
-    pub sequence_numbers: ZeroCopySlice<'a, u8, MerkleTreeSequenceNumber, false>,
+    pub output_sequence_numbers: ZeroCopySlice<'a, u8, MerkleTreeSequenceNumber, false>,
+    pub input_sequence_numbers: ZeroCopySlice<'a, u8, MerkleTreeSequenceNumber, false>,
+    pub address_sequence_numbers: ZeroCopySlice<'a, u8, MerkleTreeSequenceNumber, false>,
     pub output_leaf_indices: ZeroCopySlice<'a, u8, U32, false>,
 }
 
@@ -95,9 +97,12 @@ impl<'a> Deserialize<'a> for InsertIntoQueuesInstructionData<'a> {
 
         let (addresses, bytes) =
             ZeroCopySlice::<u8, InsertAddressInput, false>::from_bytes_at(bytes)?;
-        let (sequence_numbers, bytes) =
+        let (output_sequence_numbers, bytes) =
+            ZeroCopySlice::<u8, MerkleTreeSequenceNumber, false>::from_bytes_at(bytes)?;
+        let (input_sequence_numbers, bytes) =
+            ZeroCopySlice::<u8, MerkleTreeSequenceNumber, false>::from_bytes_at(bytes)?;
+        let (address_sequence_numbers, bytes) =
             ZeroCopySlice::<u8, MerkleTreeSequenceNumber, false>::from_bytes_at(bytes)?;
-
         let output_leaf_indices =
             ZeroCopySlice::<u8, zerocopy::little_endian::U32, false>::from_bytes(bytes)?;
         Ok((
@@ -106,7 +111,9 @@ impl<'a> Deserialize<'a> for InsertIntoQueuesInstructionData<'a> {
                 leaves,
                 nullifiers,
                 addresses,
-                sequence_numbers,
+                output_sequence_numbers,
+                input_sequence_numbers,
+                address_sequence_numbers,
                 output_leaf_indices,
             },
             bytes,
@@ -134,7 +141,9 @@ pub struct InsertIntoQueuesInstructionDataMut<'a> {
     pub leaves: ZeroCopySliceMut<'a, u8, AppendLeavesInput, false>,
     pub nullifiers: ZeroCopySliceMut<'a, u8, InsertNullifierInput, false>,
     pub addresses: ZeroCopySliceMut<'a, u8, InsertAddressInput, false>,
-    pub sequence_numbers: ZeroCopySliceMut<'a, u8, MerkleTreeSequenceNumber, false>,
+    pub output_sequence_numbers: ZeroCopySliceMut<'a, u8, MerkleTreeSequenceNumber, false>,
+    pub input_sequence_numbers: ZeroCopySliceMut<'a, u8, MerkleTreeSequenceNumber, false>,
+    pub address_sequence_numbers: ZeroCopySliceMut<'a, u8, MerkleTreeSequenceNumber, false>,
     pub output_leaf_indices: ZeroCopySliceMut<'a, u8, U32, false>,
 }
 
@@ -147,11 +156,35 @@ impl<'a> InsertIntoQueuesInstructionDataMut<'a> {
         self.meta.is_invoked_by_program = value as u8;
     }
 
+    pub fn insert_input_sequence_number(&mut self, index: &mut usize, pubkey: &Pubkey, seq: u64) {
+        Self::insert_sequence_number(&mut self.input_sequence_numbers, index, pubkey, seq);
+    }
+
+    pub fn insert_address_sequence_number(&mut self, index: &mut usize, pubkey: &Pubkey, seq: u64) {
+        Self::insert_sequence_number(&mut self.address_sequence_numbers, index, pubkey, seq);
+    }
+
+    fn insert_sequence_number(
+        sequence_numbers: &mut ZeroCopySliceMut<'a, u8, MerkleTreeSequenceNumber, false>,
+        index: &mut usize,
+        pubkey: &Pubkey,
+        seq: u64,
+    ) {
+        let pos = sequence_numbers.iter().position(|x| x.pubkey == *pubkey);
+        if pos.is_none() {
+            sequence_numbers[*index].pubkey = *pubkey;
+            sequence_numbers[*index].seq = seq.into();
+            *index += 1;
+        }
+    }
+
     pub fn required_size_for_capacity(
         leaves_capacity: u8,
         nullifiers_capacity: u8,
         addresses_capacity: u8,
         num_output_trees: u8,
+        num_input_trees: u8,
+        num_address_trees: u8,
     ) -> usize {
         size_of::<InsertIntoQueuesInstructionDataMeta>()
             + ZeroCopySliceMut::<u8, AppendLeavesInput, false>::required_size_for_capacity(
@@ -166,6 +199,12 @@ impl<'a> InsertIntoQueuesInstructionDataMut<'a> {
             + ZeroCopySliceMut::<u8, MerkleTreeSequenceNumber, false>::required_size_for_capacity(
                 num_output_trees,
             )
+            + ZeroCopySliceMut::<u8, MerkleTreeSequenceNumber, false>::required_size_for_capacity(
+                num_input_trees,
+            )
+            + ZeroCopySliceMut::<u8, MerkleTreeSequenceNumber, false>::required_size_for_capacity(
+                num_address_trees,
+            )
             + ZeroCopySliceMut::<u8, U32, false>::required_size_for_capacity(leaves_capacity)
     }
 
@@ -175,6 +214,8 @@ impl<'a> InsertIntoQueuesInstructionDataMut<'a> {
         nullifiers_capacity: u8,
         addresses_capacity: u8,
         num_output_trees: u8,
+        num_input_trees: u8,
+        num_address_trees: u8,
     ) -> std::result::Result<Self, ZeroCopyError> {
         let (meta, bytes) =
             Ref::<&mut [u8], InsertIntoQueuesInstructionDataMeta>::from_prefix(bytes)?;
@@ -186,18 +227,30 @@ impl<'a> InsertIntoQueuesInstructionDataMut<'a> {
         )?;
         let (addresses, bytes) =
             ZeroCopySliceMut::<u8, InsertAddressInput, false>::new_at(addresses_capacity, bytes)?;
-        let (sequence_numbers, bytes) =
-            ZeroCopySliceMut::<u8, MerkleTreeSequenceNumber, false>::new_at(
-                num_output_trees,
-                bytes,
-            )?;
+        let (output_sequence_numbers, bytes) = ZeroCopySliceMut::<
+            u8,
+            MerkleTreeSequenceNumber,
+            false,
+        >::new_at(num_output_trees, bytes)?;
+        let (input_sequence_numbers, bytes) = ZeroCopySliceMut::<
+            u8,
+            MerkleTreeSequenceNumber,
+            false,
+        >::new_at(num_input_trees, bytes)?;
+        let (address_sequence_numbers, bytes) = ZeroCopySliceMut::<
+            u8,
+            MerkleTreeSequenceNumber,
+            false,
+        >::new_at(num_address_trees, bytes)?;
         let output_leaf_indices = ZeroCopySliceMut::<u8, U32, false>::new(leaves_capacity, bytes)?;
         Ok(InsertIntoQueuesInstructionDataMut {
             meta,
             leaves,
             nullifiers,
             addresses,
-            sequence_numbers,
+            output_sequence_numbers,
+            input_sequence_numbers,
+            address_sequence_numbers,
             output_leaf_indices,
         })
     }
@@ -230,11 +283,15 @@ fn test_rnd_insert_into_queues_ix_data() {
         let nullifiers_capacity: u8 = rng.gen();
         let addresses_capacity: u8 = rng.gen();
         let num_output_trees: u8 = rng.gen();
+        let num_input_trees: u8 = rng.gen();
+        let num_address_trees: u8 = rng.gen();
         let size = InsertIntoQueuesInstructionDataMut::required_size_for_capacity(
             leaves_capacity,
             nullifiers_capacity,
             addresses_capacity,
             num_output_trees,
+            num_input_trees,
+            num_address_trees,
         );
         let mut bytes = vec![0u8; size];
         let mut new_data = InsertIntoQueuesInstructionDataMut::new(
@@ -243,6 +300,8 @@ fn test_rnd_insert_into_queues_ix_data() {
             nullifiers_capacity,
             addresses_capacity,
             num_output_trees,
+            num_input_trees,
+            num_address_trees,
         )
         .unwrap();
         *new_data.meta = InsertIntoQueuesInstructionDataMeta {