fix(fill): fix --collect-only after introducing FixtureOutput class (#1612)

* test(fill): add a `CliRunner`-based test for `--collect-only`

* fix(fill): evalute output options before returning if `--collect-only`

* docs: update changelog

Author: danceratopz

docs/CHANGELOG.md

@@ -19,7 +19,7 @@ The output behavior of `fill` has changed ([#1608](https://github.com/ethereum/e
 
 #### `fill`
 
-- 🔀 Refactor: Encapsulate `fill`'s fixture output options (`--output`, `--flat-output`, `--single-fixture-per-file`) into a `FixtureOutput` class ([#1471](https://github.com/ethereum/execution-spec-tests/pull/1471)).
+- 🔀 Refactor: Encapsulate `fill`'s fixture output options (`--output`, `--flat-output`, `--single-fixture-per-file`) into a `FixtureOutput` class ([#1471](https://github.com/ethereum/execution-spec-tests/pull/1471),[#1612](https://github.com/ethereum/execution-spec-tests/pull/1612)).
 - ✨ Don't warn about a "high Transaction gas_limit" for `zkevm` tests ([#1598](https://github.com/ethereum/execution-spec-tests/pull/1598)).
 - 🐞 `fill` no longer writes generated fixtures into an existing, non-empty output directory; it must now be empty or `--clean` must be used to delete it first ([#1608](https://github.com/ethereum/execution-spec-tests/pull/1608)).
 

src/pytest_plugins/filler/filler.py

@@ -222,12 +222,13 @@ def pytest_configure(config):
     # Modify the block gas limit if specified.
     if config.getoption("block_gas_limit"):
         EnvironmentDefaults.gas_limit = config.getoption("block_gas_limit")
-    if config.option.collectonly:
-        return
 
     # Initialize fixture output configuration
     config.fixture_output = FixtureOutput.from_config(config)
 
+    if config.option.collectonly:
+        return
+
     try:
         # Check whether the directory exists and is not empty; if --clean is set, it will delete it
         config.fixture_output.create_directories(is_master=not hasattr(config, "workerinput"))
@@ -865,7 +866,6 @@ def pytest_sessionfinish(session: pytest.Session, exitstatus: int):
         return
 
     fixture_output = session.config.fixture_output  # type: ignore[attr-defined]
-    # When using --collect-only it should not matter whether fixtures folder exists or not
     if fixture_output.is_stdout or session.config.option.collectonly:
         return
 

src/pytest_plugins/filler/tests/test_collect_only.py

@@ -0,0 +1,55 @@
+"""Test the fill command's --collect-only pytest option."""
+
+import textwrap
+
+from click.testing import CliRunner
+
+from cli.pytest_commands.fill import fill
+
+test_module_dummy = textwrap.dedent(
+    """\
+    import pytest
+
+    from ethereum_test_tools import Environment
+
+    @pytest.mark.valid_at("Istanbul")
+    def test_dummy_collect_only_test(state_test):
+        state_test(env=Environment(), pre={}, post={}, tx=None)
+    """
+)
+
+
+def test_collect_only_output(testdir):
+    """Test that --collect-only option produces expected output."""
+    tests_dir = testdir.mkdir("tests")
+    istanbul_tests_dir = tests_dir.mkdir("istanbul")
+    dummy_dir = istanbul_tests_dir.mkdir("dummy_test_module")
+    test_module = dummy_dir.join("test_dummy_collect.py")
+    test_module.write(test_module_dummy)
+
+    testdir.copy_example(name="pytest.ini")
+
+    runner = CliRunner()
+    result = runner.invoke(
+        fill,
+        [
+            "--fork",
+            "Istanbul",
+            "tests/istanbul/dummy_test_module/",
+            "--collect-only",
+            "-q",
+        ],
+    )
+
+    assert result.exit_code == 0, f"Fill command failed:\n{result.output}"
+
+    assert (
+        "tests/istanbul/dummy_test_module/test_dummy_collect.py::test_dummy_collect_only_test[fork_Istanbul-state_test]"
+        in result.output
+    )
+    assert (
+        "tests/istanbul/dummy_test_module/test_dummy_collect.py::test_dummy_collect_only_test[fork_Istanbul-blockchain_test_from_state_test]"
+        in result.output
+    )
+    # fill generates 3 test variants: state_test, blockchain_test, and blockchain_test_engine
+    assert "3 tests collected" in result.output