Fix General e2e improvements (#1168)

Co-authored-by: Miraculous Owonubi <omiraculous@gmail.com>

Author: alenmestrov

.github/workflows/e2e-tests.yml

@@ -12,6 +12,7 @@ on:
       - '.github/workflows/e2e_tests.yml'
 
 env:
+  # Or you can use 'prerelease-{PR-NUMBER}'
   CALIMERO_CONTRACTS_VERSION: 'latest'
 
 jobs:
@@ -71,41 +72,6 @@ jobs:
         run: |
           chmod +x target/debug/e2e-tests target/debug/merod target/debug/meroctl
 
-      - name: Generate ports
-        id: generate_ports
-        run: |
-          get_random_port() {
-            python3 -c 'import socket; s = socket.socket(); s.bind(("", 0)); port = s.getsockname()[1]; s.close(); print(port)'
-          }
-          
-          SWARM_PORT=$(get_random_port)
-          SERVER_PORT=$(get_random_port)
-          ICP_PORT=$(get_random_port)
-          SWARM_HOST=$(ifconfig | grep 'inet ' | grep -v '127.0.0.1' | awk '{print $2}' | head -n 1)
-          
-          # Set outputs for other steps
-          echo "SWARM_PORT=$SWARM_PORT" >> $GITHUB_OUTPUT
-          echo "SERVER_PORT=$SERVER_PORT" >> $GITHUB_OUTPUT
-          echo "ICP_PORT=$ICP_PORT" >> $GITHUB_OUTPUT
-          echo "SWARM_HOST=$SWARM_HOST" >> $GITHUB_OUTPUT
-          
-          # Update config.json
-          jq --arg swarmPort "$SWARM_PORT" \
-            --arg serverPort "$SERVER_PORT" \
-            --arg swarmHost "$SWARM_HOST" \
-            '.network.swarmHost = ($swarmHost) |
-            .network.startSwarmPort = ($swarmPort | tonumber) |
-            .network.startServerPort = ($serverPort | tonumber)' \
-            e2e-tests/config/config.json > updated_config.json
-          
-          mv updated_config.json e2e-tests/config/config.json
-          
-          # Print ports for debugging
-          echo "Generated ports:"
-          echo "SWARM_PORT: $SWARM_PORT"
-          echo "SERVER_PORT: $SERVER_PORT"
-          echo "ICP_PORT: $ICP_PORT"
-
       - name: Download Contracts
         run: |
           echo "Downloading contracts, target version: $CALIMERO_CONTRACTS_VERSION"
@@ -121,7 +87,7 @@ jobs:
       - name: Deploy ICP Devnet
         if: matrix.protocol == 'icp'
         env:
-          ICP_PORT: ${{ steps.generate_ports.outputs.ICP_PORT }}
+          ICP_PORT: 4943
         run: |
           echo "ICP_PORT=$ICP_PORT"
           ./scripts/icp/deploy-devnet.sh
@@ -133,7 +99,7 @@ jobs:
       - name: Deploy Stellar Devnet
         if: matrix.protocol == 'stellar'
         env:
-          ICP_PORT: ${{ steps.generate_ports.outputs.ICP_PORT }}
+          ICP_PORT: 4943
         run: |
           echo "Installing stellar CLI..."
 
@@ -196,20 +162,16 @@ jobs:
         id: e2e-tests
         env:
           NO_COLOR: '1'
-          SWARM_PORT: ${{ steps.generate_ports.outputs.SWARM_PORT }}
-          SERVER_PORT: ${{ steps.generate_ports.outputs.SERVER_PORT }}
-          SWARM_HOST: ${{ steps.generate_ports.outputs.SWARM_HOST }}
         continue-on-error: true
         run: |
           echo "Running e2e tests for ${{ matrix.protocol }}"
-          echo "Using ports: SWARM_PORT=$SWARM_PORT, SERVER_PORT=$SERVER_PORT"
 
           ./target/debug/e2e-tests \
             --input-dir ./e2e-tests/config \
             --output-dir ./e2e-tests/corpus \
             --merod-binary ./target/debug/merod \
             --meroctl-binary ./target/debug/meroctl \
-            --scenario ${{ matrix.protocol }}
+            --scenarios ${{ matrix.protocol }}
 
       - name: Upload Test Artifacts
         uses: actions/upload-artifact@v4

Cargo.lock

@@ -1,8 +1,9 @@
 use std::io::Cursor;
 
+use alloy::dyn_abi::{DynSolType, DynSolValue};
 use alloy_sol_types::SolValue;
 use candid::{Decode, Encode};
-use eyre::OptionExt;
+use eyre::eyre;
 use serde::Serialize;
 use soroban_sdk::xdr::{Limited, Limits, ReadXdr, ScVal, ToXdr};
 use soroban_sdk::{Bytes, Env, TryIntoVal};
@@ -38,7 +39,7 @@ impl Method<Near> for ContextStorageEntriesRequest {
     fn decode(response: Vec<u8>) -> eyre::Result<Self::Returns> {
         // Decode the response as Vec of tuples with boxed slices
         let entries: Vec<(Box<[u8]>, Box<[u8]>)> = serde_json::from_slice(&response)
-            .map_err(|e| eyre::eyre!("Failed to decode response: {}", e))?;
+            .map_err(|e| eyre!("Failed to decode response: {}", e))?;
 
         // Convert to ContextStorageEntry
         Ok(entries
@@ -77,7 +78,7 @@ impl Method<Starknet> for ContextStorageEntriesRequest {
             .map(|chunk| {
                 let chunk_array: [u8; 32] = chunk
                     .try_into()
-                    .map_err(|e| eyre::eyre!("Failed to convert chunk to array: {}", e))?;
+                    .map_err(|e| eyre!("Failed to convert chunk to array: {}", e))?;
                 Ok(Felt::from_bytes_be(&chunk_array))
             })
             .collect::<eyre::Result<Vec<Felt>>>()?;
@@ -95,14 +96,13 @@ impl Method<Icp> for ContextStorageEntriesRequest {
 
     fn encode(self) -> eyre::Result<Vec<u8>> {
         // Encode offset and limit using Candid
-        Encode!(&self.offset, &self.limit)
-            .map_err(|e| eyre::eyre!("Failed to encode request: {}", e))
+        Encode!(&self.offset, &self.limit).map_err(|e| eyre!("Failed to encode request: {}", e))
     }
 
     fn decode(response: Vec<u8>) -> eyre::Result<Self::Returns> {
         // Decode the response as Vec of tuples
         let entries: Vec<(Vec<u8>, Vec<u8>)> = Decode!(&response, Vec<(Vec<u8>, Vec<u8>)>)
-            .map_err(|e| eyre::eyre!("Failed to decode response: {}", e))?;
+            .map_err(|e| eyre!("Failed to decode response: {}", e))?;
 
         // Convert to ContextStorageEntry
         Ok(entries
@@ -113,10 +113,10 @@ impl Method<Icp> for ContextStorageEntriesRequest {
 }
 
 impl Method<Stellar> for ContextStorageEntriesRequest {
-    type Returns = Vec<ContextStorageEntry>;
-
     const METHOD: &'static str = "context_storage_entries";
 
+    type Returns = Vec<ContextStorageEntry>;
+
     fn encode(self) -> eyre::Result<Vec<u8>> {
         let env = Env::default();
         let offset_val: u32 = self.offset as u32;
@@ -154,33 +154,51 @@ impl Method<Stellar> for ContextStorageEntriesRequest {
 }
 
 impl Method<Ethereum> for ContextStorageEntriesRequest {
-    type Returns = Vec<ContextStorageEntry>;
-
     const METHOD: &'static str = "contextStorageEntries(uint32,uint32)";
 
+    type Returns = Vec<ContextStorageEntry>;
+
     fn encode(self) -> eyre::Result<Vec<u8>> {
         let offset = u32::try_from(self.offset)
             .map_err(|e| eyre::eyre!("Offset too large for u32: {}", e))?;
         let limit =
             u32::try_from(self.limit).map_err(|e| eyre::eyre!("Limit too large for u32: {}", e))?;
-
         Ok((offset, limit).abi_encode())
     }
+
     fn decode(response: Vec<u8>) -> eyre::Result<Self::Returns> {
-        let decoded: Vec<alloy::primitives::Bytes> = SolValue::abi_decode(&response, false)?;
-        let mut decoded = decoded.into_iter();
-        let mut entries = Vec::with_capacity(decoded.len() >> 1);
-        while let Some(key) = decoded.next() {
-            let value = decoded
-                .next()
-                .ok_or_eyre("missing value for storage entry")?;
-
-            entries.push(ContextStorageEntry {
-                key: key.into(),
-                value: value.into(),
-            });
-        }
+        // Define the struct type as a tuple
+        let struct_type = "tuple(bytes,bytes)[]".parse::<DynSolType>()?;
+        // Decode using dynamic ABI decoder
+        let decoded = struct_type.abi_decode(&response)?;
+        // Convert the decoded value to our type
+        let DynSolValue::Array(entries) = decoded else {
+            return Err(eyre!("Expected array"));
+        };
 
-        Ok(entries)
+        Ok(entries
+            .into_iter()
+            .map(|entry| {
+                let DynSolValue::Tuple(fields) = entry else {
+                    return Err(eyre!("Expected tuple"));
+                };
+
+                let all_bytes = fields[1]
+                    .as_bytes()
+                    .ok_or_else(|| eyre!("Failed to get bytes from field"))?;
+
+                // Get key
+                let key_len = all_bytes[31] as usize;
+                let key = all_bytes[32..32 + key_len].to_vec();
+
+                // Get value
+                #[allow(clippy::integer_division, reason = "Need this for 32-byte alignment")]
+                let value_offset = 32 + ((key_len + 31) / 32) * 32;
+                let value_len = all_bytes[value_offset + 31] as usize;
+                let value = all_bytes[value_offset + 32..value_offset + 32 + value_len].to_vec();
+
+                Ok(ContextStorageEntry { key, value })
+            })
+            .collect::<Result<Vec<_>, _>>()?)
     }
 }

crates/context/config/src/client/env/proxy/query/context_storage_entries.rs

@@ -9,31 +9,33 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+async-compression = { version = "0.4.22", features = ["tokio", "gzip"] }
 camino = { workspace = true, features = ["serde1"] }
 clap = { workspace = true, features = ["env", "derive"] }
 const_format.workspace = true
 eyre.workspace = true
+futures-util.workspace = true
 near-workspaces.workspace = true
 nix = { version = "0.29.0", features = ["signal"] }
 rand.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 tokio = { workspace = true, features = ["fs", "io-util", "macros", "process", "rt", "rt-multi-thread", "time"] }
-url = { workspace = true }
-reqwest = { workspace = true }
+reqwest.workspace = true
+tokio-util.workspace = true
+url.workspace = true
 
 # Add ICP dependencies from workspace
-ic-agent = { workspace = true }
-candid = { workspace = true }
+ic-agent.workspace = true
+candid.workspace = true
 
 # Add Ethereum dependencies from workspace
-alloy = { workspace = true }
+alloy.workspace = true
 
 # Add Stellar dependencies from workspace
-soroban-client = { workspace = true }
-soroban-sdk = { workspace = true }
-base64 = { workspace = true }
-flate2 = "1.1.0"
+soroban-client.workspace = true
+soroban-sdk.workspace = true
+base64.workspace = true
 
 [lints]
 workspace = true

e2e-tests/Cargo.toml

@@ -0,0 +1,93 @@
+{
+  "protocol": "ethereum",
+  "steps": [
+    {
+      "applicationInstall": {
+        "application": { "localFile": "./apps/kv-store/res/kv_store.wasm" },
+        "target": "allMembers"
+      }
+    },
+    {
+      "contextCreate": null
+    },
+    {
+      "contextCreateAlias": {
+        "aliasName": "my_context"
+      }
+    },
+    {
+      "call": {
+        "methodName": "set",
+        "argsJson": { "key": "foo", "value": "bar" },
+        "expectedResultJson": null,
+        "target": "inviter"
+      }
+    },
+    {
+      "call": {
+        "methodName": "get",
+        "argsJson": { "key": "foo" },
+        "expectedResultJson": "bar",
+        "target": "inviter"
+      }
+    },
+    {
+      "contextInviteJoin": null
+    },
+    {
+      "wait": {
+        "for": "consensus",
+        "durationMs": 5000,
+        "description": [
+          "assuming it takes 5s to propagate we should",
+          "only need to wait 5 * ceil(log2(nodes)) seconds"
+        ]
+      }
+    },
+    {
+      "call": {
+        "methodName": "get",
+        "argsJson": { "key": "foo" },
+        "expectedResultJson": "bar",
+        "target": "allMembers",
+        "retries": 20,
+        "intervalMs": 5000,
+        "description": [
+          "if we don't reach consensus in the ideal case",
+          "wait 5 seconds for nodes that have not yet synced",
+          "in the worst case, wait 20 * 5s for nodes that uselessly",
+          "keep syncing with themselves without having the state"
+        ]
+      }
+    },
+    {
+      "call": {
+        "methodName": "set",
+        "argsJson": { "key": "foo", "value": "baz" },
+        "expectedResultJson": null,
+        "target": "inviter"
+      }
+    },
+    {
+      "wait": {
+        "for": "broadcast",
+        "durationMs": 5000,
+        "description": ["wait exactly 5s for the broadcast to propagate"]
+      }
+    },
+    {
+      "call": {
+        "methodName": "get",
+        "argsJson": { "key": "foo" },
+        "expectedResultJson": "baz",
+        "target": "allMembers",
+        "retries": 5,
+        "intervalMs": 5000,
+        "description": [
+          "if a node still hasn't received the broadcast",
+          "try 5 more times in 5 seconds, but no more"
+        ]
+      }
+    }
+  ]
+}