Allow different test output for different Python versions (#10382)

Author: cdce8p

doc/development_guide/contributor_guide/tests/writing_test.rst

@@ -66,6 +66,32 @@ test runner. The following options are currently supported:
 - "except_implementations": List of python implementations on which the test should not run
 - "exclude_platforms": List of operating systems on which the test should not run
 
+**Different output for different Python versions**
+
+Sometimes the linting result can change between Python releases. In these cases errors can be marked as conditional.
+Supported operators are ``<``, ``<=``, ``>`` and ``>=``.
+
+.. code-block:: python
+
+    def some_func() -> X:  # <3.14:[undefined-variable]
+      ...
+
+    # It can also be combined with offsets
+    # +1:<3.14:[undefined-variable]
+    def some_other_func() -> X:
+      ...
+
+    class X: ...
+
+Since the output messages are different, it is necessary to add two separate files for it.
+First ``<test-file-name>.314.txt``, this will include the output messages for ``>=3.14``, i.e. should be empty here.
+Second ``<test-file-name>.txt``, this will be the default for all other Python versions.
+
+.. note::
+
+    This does only work if the code itself is parsable in all tested Python versions.
+    For new syntax, use ``min_pyver`` / ``max_pyver`` instead.
+
 **Functional test file locations**
 
 For existing checkers, new test cases should preferably be appended to the existing test file.

doc/whatsnew/fragments/10382.internal

@@ -0,0 +1,3 @@
+Modified test framework to allow for different test output for different Python versions.
+
+Refs #10382

pylint/testutils/functional/test_file.py

@@ -5,9 +5,13 @@
 from __future__ import annotations
 
 import configparser
+import sys
 from collections.abc import Callable
-from os.path import basename, exists, join
-from typing import TypedDict
+from os.path import basename, exists, join, split
+from pathlib import Path
+from typing import Final, TypedDict
+
+_CURRENT_VERSION: Final = sys.version_info[:2]
 
 
 def parse_python_version(ver_str: str) -> tuple[int, ...]:
@@ -99,7 +103,20 @@ def module(self) -> str:
 
     @property
     def expected_output(self) -> str:
-        return self._file_type(".txt", check_exists=False)
+        files = [
+            p.stem
+            for p in Path(self._directory).glob(f"{split(self.base)[-1]}.[0-9]*.txt")
+        ]
+        output_options = [
+            (int(version[0]), int(version[1:]))
+            for s in files
+            if (version := s.rpartition(".")[2]).isalnum()
+        ]
+        for opt in sorted(output_options, reverse=True):
+            if _CURRENT_VERSION >= opt:
+                str_opt = "".join([str(s) for s in opt])
+                return join(self._directory, f"{self.base}.{str_opt}.txt")
+        return join(self._directory, self.base + ".txt")
 
     @property
     def source(self) -> str:

tests/functional/ext/typing/unnecessary_default_type_args_py313.txt renamed to tests/functional/ext/typing/unnecessary_default_type_args.313.txt

@@ -3,10 +3,10 @@
 import typing as t
 
 a1: t.Generator[int, str, str]
-a2: t.Generator[int, None, None]
+a2: t.Generator[int, None, None]  # >=3.13:[unnecessary-default-type-args]
 a3: t.Generator[int]
 b1: t.AsyncGenerator[int, str]
-b2: t.AsyncGenerator[int, None]
+b2: t.AsyncGenerator[int, None]  # >=3.13:[unnecessary-default-type-args]
 b3: t.AsyncGenerator[int]
 
 c1: ca.Generator[int, str, str]

tests/functional/ext/typing/unnecessary_default_type_args.py

@@ -1,3 +1,2 @@
 [main]
-py-version=3.10
 load-plugins=pylint.extensions.typing

tests/functional/ext/typing/unnecessary_default_type_args.rc

@@ -4,8 +4,14 @@
 
 """Tests for the functional test framework."""
 
+import contextlib
+import os
+import os.path
+import shutil
+import tempfile
+from collections.abc import Iterator
 from pathlib import Path
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 import pytest
 from _pytest.outcomes import Skipped
@@ -20,6 +26,26 @@
 DATA_DIRECTORY = HERE / "data"
 
 
+@contextlib.contextmanager
+def tempdir() -> Iterator[str]:
+    """Create a temp directory and change the current location to it.
+
+    This is supposed to be used with a *with* statement.
+    """
+    tmp = tempfile.mkdtemp()
+
+    # Get real path of tempfile, otherwise test fail on mac os x
+    current_dir = os.getcwd()
+    os.chdir(tmp)
+    abs_tmp = os.path.abspath(".")
+
+    try:
+        yield abs_tmp
+    finally:
+        os.chdir(current_dir)
+        shutil.rmtree(abs_tmp)
+
+
 @pytest.fixture(name="pytest_config")
 def pytest_config_fixture() -> MagicMock:
     def _mock_getoption(option: str) -> bool:
@@ -69,6 +95,37 @@ def test_get_functional_test_files_from_crowded_directory() -> None:
     assert "max_overflow" not in str(exc_info.value)
 
 
+@pytest.mark.parametrize(
+    ["files", "output_file_name"],
+    [
+        ([], "file.txt"),
+        (["file.txt"], "file.txt"),
+        (["file.314.txt"], "file.txt"),  # don't match 3.14
+        (["file.42.txt"], "file.txt"),  # don't match 4.2
+        (["file.32.txt", "file.txt"], "file.32.txt"),
+        (["file.312.txt", "file.txt"], "file.312.txt"),
+        (["file.313.txt", "file.txt"], "file.313.txt"),
+        (["file.310.txt", "file.313.txt", "file.312.txt", "file.txt"], "file.313.txt"),
+        # don't match other test file names accidentally
+        ([".file.313.txt"], "file.txt"),
+        (["file_other.313.txt"], "file.txt"),
+        (["other_file.313.txt"], "file.txt"),
+    ],
+)
+def test_expected_output_file_matching(files: list[str], output_file_name: str) -> None:
+    """Test output file matching. Pin current Python version to 3.13."""
+    with tempdir():
+        for file in files:
+            with open(file, "w", encoding="utf-8"):
+                ...
+        test_file = FunctionalTestFile(".", "file.py")
+        with patch(
+            "pylint.testutils.functional.test_file._CURRENT_VERSION",
+            new=(3, 13),
+        ):
+            assert test_file.expected_output == f".{os.path.sep}{output_file_name}"
+
+
 def test_minimal_messages_config_enabled(pytest_config: MagicMock) -> None:
     """Test that all messages not targeted in the functional test are disabled
     when running with --minimal-messages-config.