feat(specs,tests): exception_test marker (#1436)

* feat(specs): Introduce/enforce `negative` marker

* feat(tests): Add marker to all invalid transaction/block tests

* changelog

* refactor(specs): Rename marker to `pytest.mark.exception_test`

* refactor(tests): Rename marker to `pytest.mark.exception_test`

* refactor(specs): Make `request` a `PrivateAttr`

* nit

* fix(specs): Review suggestions

Co-authored-by: spencer <spencer.taylor-brown@ethereum.org>

---------

Co-authored-by: spencer <spencer.taylor-brown@ethereum.org>

Author: marioevz

docs/CHANGELOG.md

@@ -13,6 +13,7 @@ Test fixtures for use by clients are available for each release on the [Github r
 #### `fill`
 
 - ✨ The `static_filler` plug-in now has support for static state tests (from [GeneralStateTests](https://github.com/ethereum/tests/tree/develop/src/GeneralStateTestsFiller)) ([#1362](https://github.com/ethereum/execution-spec-tests/pull/1362)).
+- ✨ Introduce `pytest.mark.exception_test` to mark tests that contain an invalid transaction or block ([#1436](https://github.com/ethereum/execution-spec-tests/pull/1436)).
 - 🐞 Fix `DeprecationWarning: Pickle, copy, and deepcopy support will be removed from itertools in Python 3.14.` by avoiding use `itertools` object in the spec `BaseTest` pydantic model ([#1414](https://github.com/ethereum/execution-spec-tests/pull/1414)).
 
 #### `consume`

src/cli/eofwrap.py

@@ -308,7 +308,6 @@ def _wrap_fixture(self, fixture: BlockchainFixture, traces: bool):
                 raise TypeError("not a FixtureBlock")
 
         result = test.generate(
-            request=None,  # type: ignore
             t8n=t8n,
             fork=Osaka,
             fixture_format=BlockchainFixture,

src/ethereum_clis/tests/test_transition_tools_support.py

@@ -236,7 +236,6 @@ def test_t8n_support(fork: Fork, installed_t8n: TransitionTool):
         blocks=[block_1, block_2],
     )
     test.generate(
-        request=None,  # type: ignore
         t8n=installed_t8n,
         fork=fork,
         fixture_format=BlockchainFixture,

src/ethereum_test_specs/base.py

@@ -4,10 +4,10 @@
 from functools import reduce
 from os import path
 from pathlib import Path
-from typing import Callable, ClassVar, Dict, Generator, List, Optional, Sequence
+from typing import Callable, ClassVar, Dict, Generator, List, Optional, Sequence, Type, TypeVar
 
 import pytest
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, PrivateAttr
 
 from ethereum_clis import Result, TransitionTool
 from ethereum_test_base_types import to_hex
@@ -41,11 +41,16 @@ def verify_result(result: Result, env: Environment):
         assert result.withdrawals_root == to_hex(Withdrawal.list_root(env.withdrawals))
 
 
+T = TypeVar("T", bound="BaseTest")
+
+
 class BaseTest(BaseModel):
     """Represents a base Ethereum test which must return a single test fixture."""
 
     tag: str = ""
 
+    _request: pytest.FixtureRequest | None = PrivateAttr(None)
+
     # Transition tool specific fields
     t8n_dump_dir: Path | None = Field(None, exclude=True)
     t8n_call_counter: int = Field(0, exclude=True)
@@ -65,6 +70,22 @@ def discard_fixture_format_by_marks(
         """Discard a fixture format from filling if the appropriate marker is used."""
         return False
 
+    @classmethod
+    def from_test(
+        cls: Type[T],
+        *,
+        base_test: "BaseTest",
+        **kwargs,
+    ) -> T:
+        """Create a test in a different format from a base test."""
+        new_instance = cls(
+            tag=base_test.tag,
+            t8n_dump_dir=base_test.t8n_dump_dir,
+            **kwargs,
+        )
+        new_instance._request = base_test._request
+        return new_instance
+
     @classmethod
     def discard_execute_format_by_marks(
         cls,
@@ -79,7 +100,6 @@ def discard_execute_format_by_marks(
     def generate(
         self,
         *,
-        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         fixture_format: FixtureFormat,
@@ -119,5 +139,51 @@ def get_next_transition_tool_output_path(self) -> str:
             str(current_value),
         )
 
+    def is_slow_test(self) -> bool:
+        """Check if the test is slow."""
+        if self._request is not None and hasattr(self._request, "node"):
+            return self._request.node.get_closest_marker("slow") is not None
+        return False
+
+    def is_exception_test(self) -> bool | None:
+        """
+        Check if the test is an exception test (invalid block, invalid transaction).
+
+        `None` is returned if it's not possible to determine if the test is negative or not.
+        This is the case when the test is not run in pytest.
+        """
+        if self._request is not None and hasattr(self._request, "node"):
+            return self._request.node.get_closest_marker("exception_test") is not None
+        return None
+
+    def node_id(self) -> str:
+        """Return the node ID of the test."""
+        if self._request is not None and hasattr(self._request, "node"):
+            return self._request.node.nodeid
+        return ""
+
+    def check_exception_test(
+        self,
+        *,
+        exception: bool,
+    ):
+        """Compare the test marker against the outcome of the test."""
+        negative_test_marker = self.is_exception_test()
+        if negative_test_marker is None:
+            return
+        if negative_test_marker != exception:
+            if exception:
+                raise Exception(
+                    "Test produced an invalid block or transaction but was not marked with the "
+                    "`exception_test` marker. Add the `@pytest.mark.exception_test` decorator "
+                    "to the test."
+                )
+            else:
+                raise Exception(
+                    "Test didn't produce an invalid block or transaction but was marked with the "
+                    "`exception_test` marker. Remove the `@pytest.mark.exception_test` decorator "
+                    "from the test."
+                )
+
 
 TestSpec = Callable[[Fork], Generator[BaseTest, None, None]]

src/ethereum_test_specs/blockchain.py

@@ -49,7 +49,7 @@
 
 from .base import BaseTest, verify_result
 from .debugging import print_traces
-from .helpers import is_slow_test, verify_block, verify_transactions
+from .helpers import verify_block, verify_transactions
 
 
 def environment_from_parent_header(parent: "FixtureHeader") -> "Environment":
@@ -562,7 +562,6 @@ def make_fixture(
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
-        slow: bool = False,
     ) -> BlockchainFixture:
         """Create a fixture from the blockchain test definition."""
         fixture_blocks: List[FixtureBlock | InvalidFixtureBlock] = []
@@ -572,7 +571,7 @@ def make_fixture(
         alloc = pre
         env = environment_from_parent_header(genesis.header)
         head = genesis.header.block_hash
-
+        invalid_blocks = 0
         for block in self.blocks:
             if block.rlp is None:
                 # This is the most common case, the RLP needs to be constructed
@@ -585,7 +584,7 @@ def make_fixture(
                     previous_env=env,
                     previous_alloc=alloc,
                     eips=eips,
-                    slow=slow,
+                    slow=self.is_slow_test(),
                 )
                 fixture_block = FixtureBlockBase(
                     header=header,
@@ -616,6 +615,7 @@ def make_fixture(
                             ),
                         ),
                     )
+                    invalid_blocks += 1
             else:
                 assert block.exception is not None, (
                     "test correctness: if the block's rlp is hard-coded, "
@@ -627,12 +627,13 @@ def make_fixture(
                         expect_exception=block.exception,
                     ),
                 )
+                invalid_blocks += 1
 
             if block.expected_post_state:
                 self.verify_post_state(
                     t8n, t8n_state=alloc, expected_state=block.expected_post_state
                 )
-
+        self.check_exception_test(exception=invalid_blocks > 0)
         self.verify_post_state(t8n, t8n_state=alloc)
         network_info = BlockchainTest.network_info(fork, eips)
         return BlockchainFixture(
@@ -655,7 +656,6 @@ def make_hive_fixture(
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
-        slow: bool = False,
     ) -> BlockchainEngineFixture:
         """Create a hive fixture from the blocktest definition."""
         fixture_payloads: List[FixtureEngineNewPayload] = []
@@ -664,7 +664,7 @@ def make_hive_fixture(
         alloc = pre
         env = environment_from_parent_header(genesis.header)
         head_hash = genesis.header.block_hash
-
+        invalid_blocks = 0
         for block in self.blocks:
             header, txs, requests, new_alloc, new_env = self.generate_block_data(
                 t8n=t8n,
@@ -673,7 +673,7 @@ def make_hive_fixture(
                 previous_env=env,
                 previous_alloc=alloc,
                 eips=eips,
-                slow=slow,
+                slow=self.is_slow_test(),
             )
             if block.rlp is None:
                 fixture_payloads.append(
@@ -691,12 +691,14 @@ def make_hive_fixture(
                     alloc = new_alloc
                     env = apply_new_parent(env, header)
                     head_hash = header.block_hash
+                else:
+                    invalid_blocks += 1
 
             if block.expected_post_state:
                 self.verify_post_state(
                     t8n, t8n_state=alloc, expected_state=block.expected_post_state
                 )
-
+        self.check_exception_test(exception=invalid_blocks > 0)
         fcu_version = fork.engine_forkchoice_updated_version(header.number, header.timestamp)
         assert fcu_version is not None, (
             "A hive fixture was requested but no forkchoice update is defined."
@@ -752,7 +754,6 @@ def make_hive_fixture(
 
     def generate(
         self,
-        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         fixture_format: FixtureFormat,
@@ -761,9 +762,9 @@ def generate(
         """Generate the BlockchainTest fixture."""
         t8n.reset_traces()
         if fixture_format == BlockchainEngineFixture:
-            return self.make_hive_fixture(t8n, fork, eips, slow=is_slow_test(request))
+            return self.make_hive_fixture(t8n, fork, eips)
         elif fixture_format == BlockchainFixture:
-            return self.make_fixture(t8n, fork, eips, slow=is_slow_test(request))
+            return self.make_fixture(t8n, fork, eips)
 
         raise Exception(f"Unknown fixture format: {fixture_format}")
 

src/ethereum_test_specs/eof.py

@@ -305,7 +305,6 @@ def model_post_init(self, __context):
     def make_eof_test_fixture(
         self,
         *,
-        request: pytest.FixtureRequest,
         fork: Fork,
         eips: Optional[List[int]],
     ) -> EOFFixture:
@@ -316,7 +315,7 @@ def make_eof_test_fixture(
                 f"Duplicate EOF test: {container_bytes}, "
                 f"existing test: {existing_tests[container_bytes]}"
             )
-        existing_tests[container_bytes] = request.node.nodeid
+        existing_tests[container_bytes] = self.node_id()
         vectors = [
             Vector(
                 code=container_bytes,
@@ -438,18 +437,17 @@ def generate_eof_contract_create_transaction(self) -> Transaction:
 
     def generate_state_test(self, fork: Fork) -> StateTest:
         """Generate the StateTest filler."""
-        return StateTest(
+        return StateTest.from_test(
+            base_test=self,
             pre=self.pre,
             tx=self.generate_eof_contract_create_transaction(),
             env=Environment(),
             post=self.post,
-            t8n_dump_dir=self.t8n_dump_dir,
         )
 
     def generate(
         self,
         *,
-        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
@@ -458,10 +456,10 @@ def generate(
     ) -> BaseFixture:
         """Generate the BlockchainTest fixture."""
         if fixture_format == EOFFixture:
-            return self.make_eof_test_fixture(request=request, fork=fork, eips=eips)
+            return self.make_eof_test_fixture(fork=fork, eips=eips)
         elif fixture_format in StateTest.supported_fixture_formats:
             return self.generate_state_test(fork).generate(
-                request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
+                t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
             )
         raise Exception(f"Unknown fixture format: {fixture_format}")
 
@@ -583,18 +581,17 @@ def generate_state_test(self, fork: Fork) -> StateTest:
         assert self.pre is not None, "pre must be set to generate a StateTest."
         assert self.post is not None, "post must be set to generate a StateTest."
 
-        return StateTest(
+        return StateTest.from_test(
+            base_test=self,
             pre=self.pre,
             tx=self,
             env=self.env,
             post=self.post,
-            t8n_dump_dir=self.t8n_dump_dir,
         )
 
     def generate(
         self,
         *,
-        request: pytest.FixtureRequest,
         t8n: TransitionTool,
         fork: Fork,
         eips: Optional[List[int]] = None,
@@ -606,11 +603,11 @@ def generate(
             if Bytes(self.container) in existing_tests:
                 # Gracefully skip duplicate tests because one EOFStateTest can generate multiple
                 # state fixtures with the same data.
-                pytest.skip(f"Duplicate EOF container on EOFStateTest: {request.node.nodeid}")
-            return self.make_eof_test_fixture(request=request, fork=fork, eips=eips)
+                pytest.skip(f"Duplicate EOF container on EOFStateTest: {self.node_id()}")
+            return self.make_eof_test_fixture(fork=fork, eips=eips)
         elif fixture_format in StateTest.supported_fixture_formats:
             return self.generate_state_test(fork).generate(
-                request=request, t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
+                t8n=t8n, fork=fork, fixture_format=fixture_format, eips=eips
             )
 
         raise Exception(f"Unknown fixture format: {fixture_format}")

src/ethereum_test_specs/helpers.py

@@ -4,8 +4,6 @@
 from enum import Enum
 from typing import Any, Dict, List
 
-import pytest
-
 from ethereum_clis import Result
 from ethereum_test_exceptions import (
     BlockException,
@@ -306,10 +304,3 @@ def verify_block(
         got_exception=result.block_exception,
     )
     info.verify(strict_match=transition_tool_exceptions_reliable)
-
-
-def is_slow_test(request: pytest.FixtureRequest) -> bool:
-    """Check if the test is slow."""
-    if hasattr(request, "node"):
-        return request.node.get_closest_marker("slow") is not None
-    return False