add test run for 3.13t (#1626)

Author: davidhewitt

.github/workflows/ci.yml

@@ -68,11 +68,15 @@ jobs:
           - '3.11'
           - '3.12'
           - '3.13'
+          - '3.13t'
           - 'pypy3.9'
           - 'pypy3.10'
 
     runs-on: ubuntu-latest
 
+    # TODO: get test suite stable with free-threaded python
+    continue-on-error: ${{ endsWith(matrix.python-version, 't') }}
+
     steps:
       - uses: actions/checkout@v4
 
@@ -101,6 +105,11 @@ jobs:
       - run: uv run pytest
         env:
           HYPOTHESIS_PROFILE: slow
+          # TODO: remove -x when test suite is more stable; we use it so that first error (hopefully) gets
+          # reported without the interpreter crashing
+          PYTEST_ADDOPTS: ${{ endsWith(matrix.python-version, 't') && '--parallel-threads=2 -x' || '' }}
+          # TODO: add `gil_used = false` to the PyO3 `#[pymodule]` when test suite is ok
+          PYTHON_GIL: ${{ endsWith(matrix.python-version, 't') && '0' || '1' }}
 
   test-os:
     name: test on ${{ matrix.os }}

pyproject.toml

@@ -52,6 +52,7 @@ testing = [
     'pytest-speed',
     'pytest-mock',
     'pytest-pretty',
+    'pytest-run-parallel',
     'pytest-timeout',
     'python-dateutil',
     # numpy doesn't offer prebuilt wheels for all versions and platforms we test in CI e.g. aarch64 musllinux

src/argument_markers.rs

@@ -5,7 +5,12 @@ use pyo3::types::{PyDict, PyTuple};
 
 use crate::tools::safe_repr;
 
-#[pyclass(module = "pydantic_core._pydantic_core", get_all, frozen, freelist = 100)]
+// see https://github.com/PyO3/pyo3/issues/4894 - freelist is currently unsound with GIL disabled
+#[cfg_attr(
+    not(Py_GIL_DISABLED),
+    pyclass(module = "pydantic_core._pydantic_core", get_all, frozen, freelist = 100)
+)]
+#[cfg_attr(Py_GIL_DISABLED, pyclass(module = "pydantic_core._pydantic_core", get_all, frozen))]
 #[derive(Debug, Clone)]
 pub struct ArgsKwargs {
     pub(crate) args: Py<PyTuple>,

tests/conftest.py

@@ -1,13 +1,15 @@
 from __future__ import annotations as _annotations
 
 import functools
+import gc
 import importlib.util
 import json
 import os
 import re
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any, Literal
+from time import sleep, time
+from typing import Any, Callable, Literal
 
 import hypothesis
 import pytest
@@ -160,3 +162,20 @@ def infinite_generator():
     while True:
         yield i
         i += 1
+
+
+def assert_gc(test: Callable[[], bool], timeout: float = 10) -> None:
+    """Helper to retry garbage collection until the test passes or timeout is
+    reached.
+
+    This is useful on free-threading where the GC collect call finishes before
+    all cleanup is done.
+    """
+    start = now = time()
+    while now - start < timeout:
+        if test():
+            return
+        gc.collect()
+        sleep(0.1)
+        now = time()
+    raise AssertionError('Timeout waiting for GC')

tests/test_docstrings.py

@@ -14,6 +14,7 @@ def find_examples(*_directories):
 
 @pytest.mark.skipif(CodeExample is None or sys.platform not in {'linux', 'darwin'}, reason='Only on linux and macos')
 @pytest.mark.parametrize('example', find_examples('python/pydantic_core/core_schema.py'), ids=str)
+@pytest.mark.thread_unsafe  # TODO investigate why pytest_examples seems to be thread unsafe here
 def test_docstrings(example: CodeExample, eval_example: EvalExample):
     eval_example.set_config(quotes='single')
 
@@ -27,6 +28,7 @@ def test_docstrings(example: CodeExample, eval_example: EvalExample):
 
 @pytest.mark.skipif(CodeExample is None or sys.platform not in {'linux', 'darwin'}, reason='Only on linux and macos')
 @pytest.mark.parametrize('example', find_examples('README.md'), ids=str)
+@pytest.mark.thread_unsafe  # TODO investigate why pytest_examples seems to be thread unsafe here
 def test_readme(example: CodeExample, eval_example: EvalExample):
     eval_example.set_config(line_length=100, quotes='single')
     if eval_example.update_examples:

tests/test_garbage_collection.py

@@ -1,4 +1,3 @@
-import gc
 import platform
 from collections.abc import Iterable
 from typing import Any
@@ -8,6 +7,8 @@
 
 from pydantic_core import SchemaSerializer, SchemaValidator, core_schema
 
+from .conftest import assert_gc
+
 GC_TEST_SCHEMA_INNER = core_schema.definitions_schema(
     core_schema.definition_reference_schema(schema_ref='model'),
     [
@@ -43,11 +44,7 @@ class MyModel(BaseModel):
 
         del MyModel
 
-    gc.collect(0)
-    gc.collect(1)
-    gc.collect(2)
-
-    assert len(cache) == 0
+    assert_gc(lambda: len(cache) == 0)
 
 
 @pytest.mark.xfail(
@@ -75,11 +72,7 @@ class MyModel(BaseModel):
 
         del MyModel
 
-    gc.collect(0)
-    gc.collect(1)
-    gc.collect(2)
-
-    assert len(cache) == 0
+    assert_gc(lambda: len(cache) == 0)
 
 
 @pytest.mark.xfail(
@@ -114,8 +107,4 @@ def __next__(self):
         v.validate_python({'iter': iterable})
         del iterable
 
-    gc.collect(0)
-    gc.collect(1)
-    gc.collect(2)
-
-    assert len(cache) == 0
+    assert_gc(lambda: len(cache) == 0)

tests/test_hypothesis.py

@@ -19,12 +19,14 @@ def datetime_schema():
 
 
 @given(strategies.datetimes())
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_datetime_datetime(datetime_schema, data):
     assert datetime_schema.validate_python(data) == data
 
 
 @pytest.mark.skipif(sys.platform == 'win32', reason='Can fail on windows, I guess due to 64-bit issue')
 @given(strategies.integers(min_value=-11_676_096_000, max_value=253_402_300_799_000))
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_datetime_int(datetime_schema, data):
     try:
         if abs(data) > 20_000_000_000:
@@ -41,6 +43,7 @@ def test_datetime_int(datetime_schema, data):
 
 
 @given(strategies.binary())
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_datetime_binary(datetime_schema, data):
     try:
         datetime_schema.validate_python(data)
@@ -89,6 +92,7 @@ class BranchModel(TypedDict):
 
 @pytest.mark.skipif(sys.platform == 'emscripten', reason='Seems to fail sometimes on pyodide no idea why')
 @given(strategies.from_type(BranchModel))
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_recursive(definition_schema, data):
     assert definition_schema.validate_python(data) == data
 
@@ -108,6 +112,7 @@ def branch_models_with_cycles(draw, existing=None):
 
 
 @given(branch_models_with_cycles())
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_definition_cycles(definition_schema, data):
     try:
         assert definition_schema.validate_python(data) == data
@@ -130,6 +135,7 @@ def test_definition_broken(definition_schema):
 
 
 @given(strategies.timedeltas())
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_pytimedelta_as_timedelta(dt):
     v = SchemaValidator({'type': 'timedelta', 'gt': dt})
     # simplest way to check `pytimedelta_as_timedelta` is correct is to extract duration from repr of the validator
@@ -150,6 +156,7 @@ def url_validator():
 
 
 @given(strategies.text())
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_urls_text(url_validator, text):
     try:
         url_validator.validate_python(text)
@@ -166,6 +173,7 @@ def multi_host_url_validator():
 
 
 @given(strategies.text())
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_multi_host_urls_text(multi_host_url_validator, text):
     try:
         multi_host_url_validator.validate_python(text)
@@ -182,6 +190,7 @@ def str_serializer():
 
 
 @given(strategies.text())
+@pytest.mark.thread_unsafe  # https://github.com/Quansight-Labs/pytest-run-parallel/issues/20
 def test_serialize_string(str_serializer: SchemaSerializer, data):
     assert str_serializer.to_python(data) == data
     assert json.loads(str_serializer.to_json(data)) == data

tests/validators/test_dataclasses.py

@@ -1,5 +1,4 @@
 import dataclasses
-import gc
 import platform
 import re
 import sys
@@ -11,7 +10,7 @@
 
 from pydantic_core import ArgsKwargs, SchemaValidator, ValidationError, core_schema
 
-from ..conftest import Err, PyAndJson
+from ..conftest import Err, PyAndJson, assert_gc
 
 
 @pytest.mark.parametrize(
@@ -1586,12 +1585,8 @@ def _wrap_validator(cls, v, validator, info):
     assert ref() is not None
 
     del klass
-    gc.collect(0)
-    gc.collect(1)
-    gc.collect(2)
-    gc.collect()
 
-    assert ref() is None
+    assert_gc(lambda: ref() is None)
 
 
 init_test_cases = [

tests/validators/test_frozenset.py

@@ -82,6 +82,7 @@ def test_frozenset_no_validators_both(py_and_json: PyAndJson, input_value, expec
         ('abc', Err('Input should be a valid frozenset')),
     ],
 )
+@pytest.mark.thread_unsafe  # generators in parameters not compatible with pytest-run-parallel, https://github.com/Quansight-Labs/pytest-run-parallel/issues/14
 def test_frozenset_ints_python(input_value, expected):
     v = SchemaValidator({'type': 'frozenset', 'items_schema': {'type': 'int'}})
     if isinstance(expected, Err):
@@ -165,6 +166,7 @@ def generate_repeats():
         ),
     ],
 )
+@pytest.mark.thread_unsafe  # generators in parameters not compatible with pytest-run-parallel, https://github.com/Quansight-Labs/pytest-run-parallel/issues/14
 def test_frozenset_kwargs_python(kwargs: dict[str, Any], input_value, expected):
     v = SchemaValidator({'type': 'frozenset', **kwargs})
     if isinstance(expected, Err):

tests/validators/test_list.py

@@ -71,6 +71,7 @@ def gen_ints():
     ],
     ids=repr,
 )
+@pytest.mark.thread_unsafe  # generators in parameters not compatible with pytest-run-parallel, https://github.com/Quansight-Labs/pytest-run-parallel/issues/14
 def test_list_int(input_value, expected):
     v = SchemaValidator({'type': 'list', 'items_schema': {'type': 'int'}})
     if isinstance(expected, Err):
@@ -170,6 +171,7 @@ def test_list_error(input_value, index):
         ),
     ],
 )
+@pytest.mark.thread_unsafe  # generators in parameters not compatible with pytest-run-parallel, https://github.com/Quansight-Labs/pytest-run-parallel/issues/14
 def test_list_length_constraints(kwargs: dict[str, Any], input_value, expected):
     v = SchemaValidator({'type': 'list', **kwargs})
     if isinstance(expected, Err):
@@ -511,6 +513,7 @@ class ListInputTestCase:
     ],
     ids=repr,
 )
+@pytest.mark.thread_unsafe  # generators in parameters not compatible with pytest-run-parallel, https://github.com/Quansight-Labs/pytest-run-parallel/issues/14
 def test_list_allowed_inputs_python(testcase: ListInputTestCase):
     v = SchemaValidator(core_schema.list_schema(core_schema.int_schema(), strict=testcase.strict))
     if isinstance(testcase.output, Err):