feat(tests): ckzg functions for peerdas tests (#1614)

* added kzg functions for peerdas tests

* fix: use proofs only instead of full cells

* tests passing except for third one, too much gas (nethermind) or too many blobs (7vs9, geth)

* implemented feedback

* function that loads trusted setup now has to return trusted setup otherwise it cant be used as input to function that requires trusted setup to be passed

* removed debug print

* added filelock for r/w blobs

* removed logging changes from this PR

* removed changes to conversions.py

* fixes

* fix

* fixes

---------

Co-authored-by: Mario Vega <marioevz@gmail.com>

Author: felix314159

docs/CHANGELOG.md

@@ -52,7 +52,8 @@ Users can select any of the artifacts depending on their testing needs for their
 
 #### `execute`
 
-- ✨ Add `blob_transaction_test` execute test spec, which allows tests that send blob transactions to a running client and verifying its `engine_getBlobsVX` endpoint behavior ([#1644](https://github.com/ethereum/execution-spec-tests/pull/1644)).
+- ✨ Added new `Blob` class which can use the ckzg library to generate valid blobs at runtime ([#1614](https://github.com/ethereum/execution-spec-tests/pull/1614)).
+- ✨ Added `blob_transaction_test` execute test spec, which allows tests that send blob transactions to a running client and verifying its `engine_getBlobsVX` endpoint behavior ([#1644](https://github.com/ethereum/execution-spec-tests/pull/1644)).
 
 ### 📋 Misc
 

pyproject.toml

@@ -50,6 +50,7 @@ dependencies = [
     "pytest-regex>=0.2.0,<0.3",
     "eth-abi>=5.2.0",
     "joblib>=1.4.2",
+    "ckzg>=2.1.1",
 ]
 
 [project.urls]

src/ethereum_test_execution/blob_transaction.py

@@ -1,8 +1,10 @@
 """Test execution format to get blobs from the execution client."""
 
+from hashlib import sha256
 from typing import ClassVar, Dict, List
 
 from ethereum_test_base_types import Hash
+from ethereum_test_base_types.base_types import Bytes
 from ethereum_test_forks import Fork
 from ethereum_test_rpc import BlobAndProofV1, BlobAndProofV2, EngineRPC, EthRPC
 from ethereum_test_types import NetworkWrappedTransaction, Transaction
@@ -18,18 +20,19 @@ def versioned_hashes_with_blobs_and_proofs(
     proofs.
     """
     versioned_hashes: Dict[Hash, BlobAndProofV1 | BlobAndProofV2] = {}
-    for blob in tx.blobs:
-        versioned_hash = blob.versioned_hash()
-        if blob.kzg_proof is not None:
-            versioned_hashes[versioned_hash] = BlobAndProofV1(blob=blob.data, proof=blob.kzg_proof)
-        elif blob.kzg_cell_proofs is not None:
-            versioned_hashes[versioned_hash] = BlobAndProofV2(
-                blob=blob.data, proofs=blob.kzg_cell_proofs
+    for blob in tx.blob_objects:
+        if isinstance(blob.proof, Bytes):
+            versioned_hashes[blob.versioned_hash] = BlobAndProofV1(
+                blob=blob.data, proof=blob.proof
+            )
+        elif isinstance(blob.proof, list):
+            versioned_hashes[blob.versioned_hash] = BlobAndProofV2(
+                blob=blob.data, proofs=blob.proof
             )
         else:
             raise ValueError(
-                f"Blob with versioned hash {versioned_hash.hex()} requires either kzg_proof "
-                "or kzg_cell_proofs, but both are None"
+                f"Blob with versioned hash {blob.versioned_hash.hex()} requires a proof "
+                "that is not None"
             )
 
     return versioned_hashes
@@ -96,7 +99,31 @@ def execute(self, fork: Fork, eth_rpc: EthRPC, engine_rpc: EngineRPC | None):
                 if expected_blob.blob != received_blob.blob:
                     raise ValueError("Blob mismatch.")
                 if expected_blob.proofs != received_blob.proofs:
-                    raise ValueError("Proofs mismatch.")
+                    error_message = "Proofs mismatch."
+                    error_message += f"len(expected_blob.proofs) = {len(expected_blob.proofs)}, "
+                    error_message += f"len(received_blob.proofs) = {len(received_blob.proofs)}\n"
+                    if len(expected_blob.proofs) == len(received_blob.proofs):
+                        index = 0
+
+                        for expected_proof, received_proof in zip(
+                            expected_blob.proofs, received_blob.proofs, strict=False
+                        ):
+                            if len(expected_proof) != len(received_proof):
+                                error_message += f"Proof length mismatch. index = {index},"
+                                error_message += f"expected_proof length = {len(expected_proof)}, "
+                                error_message += f"received_proof length = {len(received_proof)}\n"
+                                index += 1
+                                continue
+                            if expected_proof != received_proof:
+                                error_message += f"Proof mismatch. index = {index},"
+                                error_message += (
+                                    f"expected_proof hash = {sha256(expected_proof).hexdigest()}, "
+                                )
+                                error_message += (
+                                    f"received_proof hash = {sha256(received_proof).hexdigest()}\n"
+                                )
+                            index += 1
+                    raise ValueError(error_message)
             else:
                 raise ValueError(f"Unexpected blob type: {type(expected_blob)}")
 

src/ethereum_test_forks/base_fork.py

@@ -1,7 +1,20 @@
 """Abstract base class for Ethereum forks."""
 
 from abc import ABC, ABCMeta, abstractmethod
-from typing import Any, ClassVar, List, Mapping, Optional, Protocol, Sized, Tuple, Type
+from typing import (
+    Any,
+    ClassVar,
+    Dict,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Protocol,
+    Sized,
+    Tuple,
+    Type,
+    Union,
+)
 
 from semver import Version
 
@@ -155,6 +168,14 @@ class BaseFork(ABC, metaclass=BaseForkMeta):
     _solc_name: ClassVar[Optional[str]] = None
     _ignore: ClassVar[bool] = False
 
+    # make mypy happy
+    BLOB_CONSTANTS: ClassVar[Dict[str, Union[int, Literal["big"]]]] = {}
+
+    @classmethod
+    def get_blob_constant(cls, name: str) -> int | Literal["big"]:
+        """Return value of requested blob constant."""
+        raise NotImplementedError
+
     def __init_subclass__(
         cls,
         *,

src/ethereum_test_forks/forks/forks.py

@@ -4,7 +4,7 @@
 from hashlib import sha256
 from os.path import realpath
 from pathlib import Path
-from typing import List, Mapping, Optional, Sized, Tuple
+from typing import List, Literal, Mapping, Optional, Sized, Tuple
 
 from semver import Version
 
@@ -908,6 +908,27 @@ def valid_opcodes(
 class Cancun(Shanghai):
     """Cancun fork."""
 
+    BLOB_CONSTANTS = {  # every value is an int or a Literal
+        "FIELD_ELEMENTS_PER_BLOB": 4096,
+        "BYTES_PER_FIELD_ELEMENT": 32,
+        "CELL_LENGTH": 2048,
+        "BLS_MODULUS": 0x73EDA753299D7D483339D80809A1D80553BDA402FFFE5BFEFFFFFFFF00000001,  # EIP-2537: Main subgroup order = q, due to this BLS_MODULUS every blob byte (uint256) must be smaller than 116  # noqa: E501
+        # https://github.com/ethereum/consensus-specs/blob/cc6996c22692d70e41b7a453d925172ee4b719ad/specs/deneb/polynomial-commitments.md?plain=1#L78
+        "BYTES_PER_PROOF": 48,
+        "BYTES_PER_COMMITMENT": 48,
+        "KZG_ENDIANNESS": "big",
+        "AMOUNT_CELL_PROOFS": 0,
+    }
+
+    @classmethod
+    def get_blob_constant(cls, name: str) -> int | Literal["big"]:
+        """Return blob constant if it exists."""
+        retrieved_constant = cls.BLOB_CONSTANTS.get(name)
+        assert retrieved_constant is not None, (
+            f"You tried to retrieve the blob constant {name} but it does not exist!"
+        )
+        return retrieved_constant
+
     @classmethod
     def solc_min_version(cls) -> Version:
         """Return minimum version of solc that supports this fork."""
@@ -1092,6 +1113,16 @@ def valid_opcodes(
 class Prague(Cancun):
     """Prague fork."""
 
+    # update some blob constants
+    BLOB_CONSTANTS = {
+        **Cancun.BLOB_CONSTANTS,  # same base constants as cancun
+        "MAX_BLOBS_PER_BLOCK": 9,  # but overwrite or add these
+        "TARGET_BLOBS_PER_BLOCK": 6,
+        "MAX_BLOB_GAS_PER_BLOCK": 1179648,
+        "TARGET_BLOB_GAS_PER_BLOCK": 786432,
+        "BLOB_BASE_FEE_UPDATE_FRACTION": 5007716,
+    }
+
     @classmethod
     def is_deployed(cls) -> bool:
         """
@@ -1343,6 +1374,12 @@ def engine_forkchoice_updated_version(
 class Osaka(Prague, solc_name="cancun"):
     """Osaka fork."""
 
+    # update some blob constants
+    BLOB_CONSTANTS = {
+        **Prague.BLOB_CONSTANTS,  # same base constants as prague
+        "AMOUNT_CELL_PROOFS": 128,
+    }
+
     @classmethod
     def engine_get_payload_version(
         cls, block_number: int = 0, timestamp: int = 0

src/ethereum_test_rpc/types.py

@@ -152,14 +152,14 @@ def blob_versioned_hashes(self, versioned_hash_version: int = 1) -> List[Hash]:
 
 
 class BlobAndProofV1(CamelModel):
-    """Represents a blob and single-proof structure."""
+    """Represents a blob and single-proof structure (< Osaka)."""
 
     blob: Bytes
     proof: Bytes
 
 
 class BlobAndProofV2(CamelModel):
-    """Represents a blob and proof structure."""
+    """Represents a blob and cell proof structure (>= Osaka)."""
 
     blob: Bytes
     proofs: List[Bytes]

src/ethereum_test_types/__init__.py

@@ -1,6 +1,7 @@
 """Common definitions and types."""
 
 from .account_types import EOA, Alloc
+from .blob_types import Blob
 from .block_types import (
     Environment,
     EnvironmentDefaults,
@@ -23,7 +24,6 @@
 )
 from .transaction_types import (
     AuthorizationTuple,
-    Blob,
     NetworkWrappedTransaction,
     Transaction,
     TransactionDefaults,