refactor: batch ops (#1838)

* bump photon hash

* refactor forester v2 processing

* remove debug prints

* cleanup

* Set test environment variables for v1 and v2 test modes

* add docs for process_stream function

* fail-fast proof generation with recovery

* spawn forester executing test transactions in e2e test

* Add tmate debugging session to forester tests workflow

* use new keypair for TestMode::Local in forester e2e tests

* reduce rpc calls for v2 processing

* add tmate to forester ci

* remove tmate from forester tests ci

* format

* add get_registration_phase_start_slot utility function and update e2e test to use it

* remove unused imports

* update photon git commit reference in constants and install script

* update photon commit hash

* update photon commit hash

* refactor: optimize batch processing and instruction streaming

Author: sergeytimoshin

.github/workflows/forester-tests.yml

@@ -30,6 +30,11 @@ concurrency:
 env:
   RUST_BACKTRACE: "1"
   RUSTFLAGS: "--cfg tokio_unstable -D warnings"
+  TEST_MODE: "local"
+  TEST_V1_STATE: "true"
+  TEST_V2_STATE: "true"
+  TEST_V1_ADDRESS: "true"
+  TEST_V2_ADDRESS: "true"
 
 jobs:
   test:
@@ -38,41 +43,17 @@ jobs:
         test-name:
           [
             {
-              name: "e2e",
-              command: "test_state_indexer_async_batched",
+              name: "e2e (legacy)",
+              command: "test_e2e_v1",
               timeout: 60,
               needs-test-program: false,
             },
             {
-              name: "address-batched",
-              command: "test_address_batched",
+              name: "e2e",
+              command: "test_e2e_v2",
               timeout: 60,
               needs-test-program: true,
             },
-            {
-              name: "state-batched",
-              command: "test_state_batched",
-              timeout: 60,
-              needs-test-program: false,
-            },
-            {
-              name: "state-photon-batched",
-              command: "test_state_indexer_batched",
-              timeout: 60,
-              needs-test-program: false,
-            },
-            {
-              name: "2-foresters",
-              command: "test_epoch_monitor_with_2_foresters",
-              timeout: 60,
-              needs-test-program: false,
-            },
-            {
-              name: "double-registration",
-              command: "test_epoch_double_registration",
-              timeout: 60,
-              needs-test-program: false,
-            },
           ]
     name: test-${{ matrix.test-name.name }}
     runs-on: warp-ubuntu-latest-x64-4x
@@ -117,6 +98,7 @@ jobs:
         run: |
           source ./scripts/devenv.sh
           cargo test-sbf -p create-address-test-program
+
       - name: Run ${{ matrix.test-name.name }} tests
         run: |
           source ./scripts/devenv.sh

Cargo.lock

@@ -24,7 +24,7 @@ export const PHOTON_VERSION = "0.51.0";
 // Set these to override Photon requirements with a specific git commit:
 export const USE_PHOTON_FROM_GIT = true; // If true, will show git install command instead of crates.io.
 export const PHOTON_GIT_REPO = "https://github.com/lightprotocol/photon.git";
-export const PHOTON_GIT_COMMIT = "6ee3c027226ab9c90dc9d16691cdf76dd2f29dbf"; // If empty, will use main branch.
+export const PHOTON_GIT_COMMIT = "c938ee83ad1b34abc389943334627a899da72953"; // If empty, will use main branch.
 
 export const LIGHT_PROTOCOL_PROGRAMS_DIR_ENV = "LIGHT_PROTOCOL_PROGRAMS_DIR";
 export const BASE_PATH = "../../bin/";

cli/src/utils/constants.ts

@@ -34,6 +34,7 @@ account-compression = { workspace = true, features = ["cpi"] }
 
 tokio = { workspace = true }
 futures = { workspace = true }
+async-stream = "0.3"
 
 anchor-lang = { workspace = true }
 

forester-utils/Cargo.toml

@@ -1,5 +1,9 @@
+use light_batched_merkle_tree::errors::BatchedMerkleTreeError;
+use light_hasher::HasherError;
 use thiserror::Error;
 
+use crate::rpc_pool::PoolError;
+
 #[derive(Error, Debug)]
 pub enum ForesterUtilsError {
     #[error("parse error: {0:?}")]
@@ -12,4 +16,18 @@ pub enum ForesterUtilsError {
     Indexer(String),
     #[error("invalid slot number")]
     InvalidSlotNumber,
+    #[error("Hasher error: {0}")]
+    Hasher(#[from] HasherError),
+
+    #[error("Account zero-copy error: {0}")]
+    AccountZeroCopy(String),
+
+    #[error("light client error: {0}")]
+    LightClient(#[from] light_client::rpc::RpcError),
+
+    #[error("batched merkle tree error: {0}")]
+    BatchedMerkleTree(#[from] BatchedMerkleTreeError),
+
+    #[error("pool error: {0}")]
+    Pool(#[from] PoolError),
 }