Merge branch 'main' into spencer-tb/move-prague-to-stable

Author: danceratopz

.github/configs/evm.yaml

@@ -11,7 +11,7 @@ static:
   repo: ethereum/evmone
   ref: master
   targets: ["evmone-t8n", "evmone-eofparse"]
-zkevm:
+benchmark:
   impl: evmone
   repo: ethereum/evmone
   ref: master

.github/configs/feature.yaml

@@ -11,24 +11,24 @@ static:
   evm-type: static
   fill-params: --until=Osaka --fill-static-tests ./tests/static
   solc: 0.8.21
-zkevm_36M:
-  evm-type: zkevm
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 36000000 -m zkevm ./tests
+benchmark_30M:
+  evm-type: benchmark
+  fill-params: --from=Cancun --until=Prague --block-gas-limit 30000000 -m benchmark ./tests
   solc: 0.8.21
   feature_only: true
-zkevm_60M:
-  evm-type: zkevm
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 60000000 -m zkevm ./tests
+benchmark_60M:
+  evm-type: benchmark
+  fill-params: --from=Cancun --until=Prague --block-gas-limit 60000000 -m benchmark ./tests
   solc: 0.8.21
   feature_only: true
-zkevm_90M:
-  evm-type: zkevm
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 90000000 -m zkevm ./tests
+benchmark_90M:
+  evm-type: benchmark
+  fill-params: --from=Cancun --until=Prague --block-gas-limit 90000000 -m benchmark ./tests
   solc: 0.8.21
   feature_only: true
-zkevm_120M:
-  evm-type: zkevm
-  fill-params: --from=Cancun --until=Prague --block-gas-limit 120000000 -m zkevm ./tests
+benchmark_120M:
+  evm-type: benchmark
+  fill-params: --from=Cancun --until=Prague --block-gas-limit 120000000 -m benchmark ./tests
   solc: 0.8.21
   feature_only: true
 eip7692:

.github/scripts/fill_prepatched_tests.sh

@@ -16,9 +16,10 @@ FILL_UNTIL="${5:-Cancun}"
 echo "--------------------"
 echo "converted-ethereum-tests.txt seem untouched, try to fill pre-patched version of .py files:"
 
+PATCH_COMMIT=$(git rev-parse HEAD)
 git checkout main
-PREV_COMMIT=$(git rev-parse HEAD)
-echo "Checkout head $PREV_COMMIT"
+BASE_COMMIT=$(git rev-parse HEAD)
+echo "Checkout head $BASE_COMMIT"
 
 echo "Select files that were changed and exist on the main branch:"
 echo "$MODIFIED_DELETED_FILES"
@@ -38,7 +39,19 @@ if grep -q "ERROR collecting test session" filloutput.log; then
     exit 1
 fi
 
-# TODO: Here we can inspect $BASE_TEST_PATH vs $PATCH_TEST_PATH and remove fixtures with the same hash in both directories, to only leave fixtures that have been modified or removed,
-#       and then set any_modified_fixtures=false if the fixture set before the PR is empty after this check.
-echo "any_modified_fixtures=true" >> "$GITHUB_OUTPUT"
+git checkout $PATCH_COMMIT
+echo "Checkout back to patch $PATCH_COMMIT"
+# abort-on-empty-patch is used to ensure that the patch folder is not empty after fixture removal.
+# If the patch folder would be empty, it means that fixtures were removed in the PR, in which case we still want to run the coverage check.
+uv run compare_fixtures --abort-on-empty-patch $BASE_TEST_PATH $PATCH_TEST_PATH
+
+if [ -d $BASE_TEST_PATH ]; then
+    # If the base folder is not empty, it means there's at least one fixture that was modified in the PR, continue with the coverage check.
+    echo "Base folder is not empty after fixture comparison, continuing with coverage check."
+    echo "any_modified_fixtures=true" >> "$GITHUB_OUTPUT"
+else
+    # If the base folder is empty, it means there were no fixtures that were modified in the PR, or fixtures were only added, so we can skip the coverage check.
+    echo "Base folder is empty after fixture comparison, skipping coverage check."
+    echo "any_modified_fixtures=false" >> "$GITHUB_OUTPUT"
+fi
 exit 0

.github/workflows/tox_verify.yaml

@@ -83,7 +83,7 @@ jobs:
     steps:
       - name: Checkout ethereum/execution-spec-tests
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-      - uses: DavidAnson/markdownlint-cli2-action@b4c9feab76d8025d1e83c653fa3990936df0e6c8
+      - uses: DavidAnson/markdownlint-cli2-action@05f32210e84442804257b2a6f20b273450ec8265
         with:
           globs: |
             README.md
@@ -182,8 +182,8 @@ jobs:
         env:
           EELS_RESOLUTIONS_FILE: ${{ github.workspace }}/.github/configs/eels_resolutions.json
         run: uvx --with=tox-uv tox -e tests-deployed
-  tests_deployed_zkevm:
-    name: Fill zkEVM test cases, deployed, ${{ matrix.os }}, ${{ matrix.python }}
+  tests_deployed_benchmark:
+    name: Fill benchmark test cases, deployed, ${{ matrix.os }}, ${{ matrix.python }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -209,4 +209,4 @@ jobs:
       - name: Run tox - fill tests for deployed forks
         env:
           EELS_RESOLUTIONS_FILE: ${{ github.workspace }}/.github/configs/eels_resolutions.json
-        run: uvx --with=tox-uv tox -e tests-deployed-zkevm
+        run: uvx --with=tox-uv tox -e tests-deployed-benchmark

.gitignore

@@ -3,7 +3,6 @@
 .LSOverride
 
 .python-version
-__pycache__/
 *.py[cod]
 *$py.class
 .venv/
@@ -58,23 +57,40 @@ verify_kzg_proof
 .vscode/launch.json
 
 # docs
-.cache
 _readthedocs
 site
 venv-docs/
 .pyspelling_en.dict
 
-# cached fixture downloads (consume)
+# cache-related
 cached_downloads/
+.cache
+__pycache__/
+.mypy_cache/
+.pytest_cache/
+.ruff_cache/
+
+
 # pytest report
 assets
 *.html
 
 # Environment
 env.yaml
+.env
+.env.*
 
 # Framework logs
 logs/
+*.log
+
+# node
+node_modules/
+package-lock.json
 
 # Running github actions locally with act
 .act_github_vars
+
+# temporary files
+*.tmp
+*.temp

.vscode/settings.json

@@ -18,7 +18,8 @@
         "editor.formatOnSave": true,
         "editor.defaultFormatter": "charliermarsh.ruff",
         "editor.codeActionsOnSave": {
-            "source.organizeImports": "explicit"
+            "source.organizeImports": "explicit",
+            "source.fixAll.ruff": "explicit",
         },
     },
     "python.analysis.autoFormatStrings": true,
@@ -57,9 +58,9 @@
     "ruff.logLevel": "error",
     "extensions.ignoreRecommendations": false,
     "search.exclude": {
-      "tests/legacy": true
+      "tests/static": true
     },
     "files.watcherExclude": {
-      "tests/legacy": true
+      "tests/static": true
     }
 }

README.md

@@ -6,6 +6,8 @@
 [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 [![License](https://img.shields.io/github/license/ethereum/execution-spec-tests)](https://github.com/ethereum/execution-spec-tests/blob/main/LICENSE)
 
+The full execution-spec-tests documentation can be found [here](https://eest.ethereum.org/main/).
+
 [ethereum/execution-spec-tests](https://github.com/ethereum/execution-spec-tests) is both a collection of test cases and a framework implemented in Python to generate tests for Ethereum execution clients.
 
 The framework collects and executes the test cases in order to generate _test fixtures_ (JSON) which can be consumed by any execution client to verify their implementation of [ethereum/execution-specs](https://github.com/ethereum/execution-specs). The fixtures, which define state transition and block tests, are generated by the framework using one of the `t8n` command-line tools that are provided by most execution clients, see below for an overview of the supported `t8n` tools.

docs/CHANGELOG.md

@@ -19,9 +19,11 @@ The output behavior of `fill` has changed ([#1608](https://github.com/ethereum/e
 - Before: `fill` wrote fixtures into the directory specified by the `--output` flag (default: `fixtures`). This could have many unintended consequences, including unexpected errors if old or invalid fixtures existed in the directory (for details see [#1030](https://github.com/ethereum/execution-spec-tests/issues/1030)).
 - Now: `fill` will exit without filling any tests if the specified directory exists and is not-empty. This may be overridden by adding the `--clean` flag, which will first remove the specified directory.
 
-#### Feature `zkevm` expanded to `zkevm_36M`, `zkevm_60M`, `zkevm_90M`, `zkevm_120M`
+#### Feature `zkevm` updated to `benchmark`
 
-Renames `zkevm` to `zkevm_36M` and further expands the zkEVM features to run using 60M, 90M and 120M block gas limits in `fixtures_zkevm_36M.tar.gz`, `fixtures_zkevm_60M.tar.gz`, `fixtures_zkevm_90M.tar.gz` and `fixtures_zkevm_120M.tar.gz` respectively.
+Due to the crossover between `zkevm` and `benchmark` tests, all instances of the former have been replaced with the latter nomenclature. Repository PR labels and titles are additionally updated to reflect this change.
+
+This update renames the `zkevm` feature release to `benchmark_30M` and further expands the latter for 60M, 90M, and 120M block gas limits in `fixtures_benchmark_30M.tar.gz`, `fixtures_benchmark_60M.tar.gz`, `fixtures_benchmark_90M.tar.gz`, and `fixtures_benchmark_120M.tar.gz` respectively.
 
 Users can select any of the artifacts depending on their testing needs for their provers.
 
@@ -40,22 +42,26 @@ Users can select any of the artifacts depending on their testing needs for their
 - 🔀 Refactor: Encapsulate `fill`'s fixture output options (`--output`, `--flat-output`, `--single-fixture-per-file`) into a `FixtureOutput` class ([#1471](https://github.com/ethereum/execution-spec-tests/pull/1471),[#1612](https://github.com/ethereum/execution-spec-tests/pull/1612)).
 - ✨ Don't warn about a "high Transaction gas_limit" for `zkevm` tests ([#1598](https://github.com/ethereum/execution-spec-tests/pull/1598)).
 - 🐞 `fill` no longer writes generated fixtures into an existing, non-empty output directory; it must now be empty or `--clean` must be used to delete it first ([#1608](https://github.com/ethereum/execution-spec-tests/pull/1608)).
-- 🐞 zkEVM marked tests have been removed from `tests-deployed` tox environment into its own separate workflow `tests-deployed-zkevm` and are filled by `evmone-t8n` ([#1617](https://github.com/ethereum/execution-spec-tests/pull/1617)).
+- 🐞 `zkevm` marked tests have been removed from `tests-deployed` tox environment into its own separate workflow `tests-deployed-zkevm` and are filled by `evmone-t8n` ([#1617](https://github.com/ethereum/execution-spec-tests/pull/1617)).
 - ✨ Field `postStateHash` is now added to all `blockchain_test` and `blockchain_test_engine` tests that use `exclude_full_post_state_in_output` in place of `postState`. Fixes `evmone-blockchaintest` test consumption and indirectly fixes coverage runs for these tests ([#1667](https://github.com/ethereum/execution-spec-tests/pull/1667)).
 - 🔀 Changed INVALID_DEPOSIT_EVENT_LAYOUT to a BlockException instead of a TransactionException ([#1773](https://github.com/ethereum/execution-spec-tests/pull/1773)).
 
 #### `consume`
 
+- 🐞 Fix `consume cache --cache-folder` parameter being ignored, now properly caches fixtures in the specified directory instead of always using the default system cache location.
 - 🔀 `consume` now automatically avoids GitHub API calls when using direct release URLs (better for CI environments), while release specifiers like `stable@latest` continue to use the API for version resolution ([#1788](https://github.com/ethereum/execution-spec-tests/pull/1788)).
 - 🔀 Refactor consume simulator architecture to use explicit pytest plugin structure with forward-looking architecture ([#1801](https://github.com/ethereum/execution-spec-tests/pull/1801)).
 - 🔀 Add exponential retry logic to initial fcu within consume engine ([#1815](https://github.com/ethereum/execution-spec-tests/pull/1815)).
 
 #### `execute`
 
-- ✨ Add `blob_transaction_test` execute test spec, which allows tests that send blob transactions to a running client and verifying its `engine_getBlobsVX` endpoint behavior ([#1644](https://github.com/ethereum/execution-spec-tests/pull/1644)).
+- ✨ Added new `Blob` class which can use the ckzg library to generate valid blobs at runtime ([#1614](https://github.com/ethereum/execution-spec-tests/pull/1614)).
+- ✨ Added `blob_transaction_test` execute test spec, which allows tests that send blob transactions to a running client and verifying its `engine_getBlobsVX` endpoint behavior ([#1644](https://github.com/ethereum/execution-spec-tests/pull/1644)).
 
 ### 📋 Misc
 
+- 🔀 Use only relative imports in `tests/` directory ([#1848](https://github.com/ethereum/execution-spec-tests/pull/1848)).
+- 🔀 Misc. doc updates, including a navigation footer ([#1846](https://github.com/ethereum/execution-spec-tests/pull/1846)).
 - 🔀 Remove Python 3.10 support ([#1808](https://github.com/ethereum/execution-spec-tests/pull/1808)).
 - 🔀 Modernize codebase with Python 3.11 language features ([#1812](https://github.com/ethereum/execution-spec-tests/pull/1812)).
 - ✨ Add changelog formatting validation to CI to ensure consistent punctuation in bullet points [#1691](https://github.com/ethereum/execution-spec-tests/pull/1691).
@@ -74,7 +80,9 @@ Users can select any of the artifacts depending on their testing needs for their
 - ✨ Added automatic checklist generation for every EIP inside of the `tests` folder. The checklist is appended to each EIP in the documentation in the "Test Case Reference" section ([#1679](https://github.com/ethereum/execution-spec-tests/pull/1679), [#1718](https://github.com/ethereum/execution-spec-tests/pull/1718)).
 - 🔀 Add macOS hive development mode workaround to the docs [#1786](https://github.com/ethereum/execution-spec-tests/pull/1786).
 - 🔀 Refactor and clean up of exceptions including EOF exceptions within client specific mappers [#1803](https://github.com/ethereum/execution-spec-tests/pull/1803).
-- 🔀 Move Prague to stable and Osaka to develop ([#1573](https://github.com/ethereum/execution-spec-tests/pull/1573)).
+- 🔀 Move Prague to the stable and Osaka to the develop fixture release ([#1573](https://github.com/ethereum/execution-spec-tests/pull/1573)).
+- 🔀 Rename `tests/zkevm/` to `tests/benchmark/` and replace the `zkevm` pytest mark with `benchmark` [#1804](https://github.com/ethereum/execution-spec-tests/pull/1804).
+- 🔀 Add fixture comparison check to optimize coverage workflow in CI ([#1833](https://github.com/ethereum/execution-spec-tests/pull/1833)).
 
 ### 🧪 Test Cases
 
@@ -85,7 +93,7 @@ Users can select any of the artifacts depending on their testing needs for their
 - ✨ EIP-7823, EIP-7883: Add test cases for ModExp precompile gas-cost updates and input limits on Osaka ([#1579](https://github.com/ethereum/execution-spec-tests/pull/1579), [#1729](https://github.com/ethereum/execution-spec-tests/pull/1729)).
 - ✨ [EIP-7825](https://eips.ethereum.org/EIPS/eip-7825): Add test cases for the transaction gas limit of 30M gas ([#1711](https://github.com/ethereum/execution-spec-tests/pull/1711)).
 - ✨ [EIP-7951](https://eips.ethereum.org/EIPS/eip-7951): add test cases for `P256VERIFY` precompile to support secp256r1 curve [#1670](https://github.com/ethereum/execution-spec-tests/pull/1670).
-- ✨ Introduce blockchain tests for ZKEVM to cover the scenario of pure ether transfers [#1742](https://github.com/ethereum/execution-spec-tests/pull/1742).
+- ✨ Introduce blockchain tests for benchmark to cover the scenario of pure ether transfers [#1742](https://github.com/ethereum/execution-spec-tests/pull/1742).
 - ✨ [EIP-7934](https://eips.ethereum.org/EIPS/eip-7934): Add test cases for the block RLP max limit of 10MiB ([#1730](https://github.com/ethereum/execution-spec-tests/pull/1730)).
 - ✨ [EIP-7939](https://eips.ethereum.org/EIPS/eip-7939) Add count leading zeros (CLZ) opcode tests for Osaka ([#1733](https://github.com/ethereum/execution-spec-tests/pull/1733)).
 - ✨ [EIP-7918](https://eips.ethereum.org/EIPS/eip-7918): Blob base fee bounded by execution cost test cases (initial), includes some adjustments to [EIP-4844](https://eips.ethereum.org/EIPS/eip-4844) tests ([#1685](https://github.com/ethereum/execution-spec-tests/pull/1685)).
@@ -112,12 +120,12 @@ A new fork `EOFv1` has additionally been created to fill and consume EOF related
 - ✨ Engine API updates for Osaka, add `get_blobs` rpc method ([#1510](https://github.com/ethereum/execution-spec-tests/pull/1510)).
 - ✨ The EIP Version checker has been moved from `fill` and `execute` to it's own command-line tool `check_eip_versions` that gets ran daily as a Github Action ([#1537](https://github.com/ethereum/execution-spec-tests/pull/1537)).
 - 🔀 Add new `tests/unscheduled` folder, move EOF from Osaka to unscheduled, add `EOFv1` fork name for EOF tests ([#1507](https://github.com/ethereum/execution-spec-tests/pull/1507)).
-- ✨ CI features now contain an optional field to skip them from EEST full releases, `zkevm` and EOF features are now feature only ([#1596](https://github.com/ethereum/execution-spec-tests/pull/1596)).
+- ✨ CI features now contain an optional field to skip them from EEST full releases, `benchmark` and EOF features are now feature only ([#1596](https://github.com/ethereum/execution-spec-tests/pull/1596)).
 - 🐞 Don't attempt to install `solc` via `solc-select` on ARM (official Linux ARM builds of `solc` are not available at the time of writing, cf [ethereum/solidity#11351](https://github.com/ethereum/solidity/issues/11351)) and add a version sanity check ([#1556](https://github.com/ethereum/execution-spec-tests/pull/1556)).
 
 ### 🧪 Test Cases
 
-- 🔀 Automatically apply the `zkevm` marker to all tests under `./tests/zkevm/` and `./tests/prague/eip2537_bls_12_381_precompiles/` via conftest configuration ([#1534](https://github.com/ethereum/execution-spec-tests/pull/1534)).
+- 🔀 Automatically apply the `benchmark` marker to all tests under `./tests/benchmark/` and `./tests/prague/eip2537_bls_12_381_precompiles/` via conftest configuration ([#1534](https://github.com/ethereum/execution-spec-tests/pull/1534)).
 - ✨ Port [calldataload](https://github.com/ethereum/tests/blob/ae4791077e8fcf716136e70fe8392f1a1f1495fb/src/GeneralStateTestsFiller/VMTests/vmTests/calldatacopyFiller.yml) and [calldatasize](https://github.com/ethereum/tests/blob/81862e4848585a438d64f911a19b3825f0f4cd95/src/GeneralStateTestsFiller/VMTests/vmTests/calldatasizeFiller.yml) tests ([#1236](https://github.com/ethereum/execution-spec-tests/pull/1236)).
 
 ## [v4.4.0](https://github.com/ethereum/execution-spec-tests/releases/tag/v4.4.0) - 2025-04-29
@@ -134,9 +142,9 @@ The tests have been filled using the new static test filler introduced in [#1336
 
 Users can expect that all tests currently living in [ethereum/tests](https://github.com/ethereum/tests/tree/develop/src) should eventually make its way into [`./tests/static`](https://github.com/ethereum/execution-spec-tests/tree/main/tests/static) and can rely that these tests, filled for new forks even, will be included in `fixtures_static.tar.gz`.
 
-#### `fixtures_zkevm`
+#### `fixtures_benchmark`
 
-Another new fixture tarball has been included in this release: `fixtures_zkevm.tar.gz`.
+Another new fixture tarball has been included in this release: `fixtures_benchmark.tar.gz`.
 
 Includes tests that are tailored specifically to test the execution layer proof generators.