Lots of test failures when using `-n` other than 4

[mgorny]

Describe the bug
It seems that tests are not parallel-safe and that they are passing with -n4 only by accident. When I'm using a job count higher than 6, the tests quickly start failing.

To Reproduce

1. Start redis servers.
2. tox -e py313-dj52-redislatest -- -n7 -x

Expected behavior
Tests reliably passing with any job count, or explicitly declaring that they are not compatible with xdist (and then tests/conftest.py not requiring pytest-xdist unconditionally).

Stack trace

.pkg: _optional_hooks> python /usr/lib/python3.14/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: get_requires_for_build_sdist> python /usr/lib/python3.14/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: get_requires_for_build_wheel> python /usr/lib/python3.14/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: prepare_metadata_for_build_wheel> python /usr/lib/python3.14/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
.pkg: build_sdist> python /usr/lib/python3.14/site-packages/pyproject_api/_backend.py True setuptools.build_meta __legacy__
py313-dj52-redislatest: install_package> python -I -m pip install --force-reinstall --no-deps /tmp/django-redis/.tox/.tmp/package/5/django_redis-6.0.0.tar.gz
py313-dj52-redislatest: commands[0]> .tox/py313-dj52-redislatest/bin/python -m pytest -n 4 -n7 -x
============================= test session starts ==============================
platform linux -- Python 3.13.5, pytest-6.2.5, py-1.11.0, pluggy-1.6.0
cachedir: .tox/py313-dj52-redislatest/.pytest_cache
rootdir: /tmp/django-redis, configfile: setup.cfg, testpaths: tests
plugins: cov-6.2.1, mock-3.14.1, pythonpath-0.7.4, xdist-3.5.0
created: 7/7 workers
7 workers [1647 items]

......FEFEFEFEFEFEFEF..........................................EFEFEFEFE [  3%]
............FEFEFE...................................................... [  7%]
.................................ssss..................s................. [ 12%]
........................................................................ [ 16%]
........................................................................ [ 20%]
.....FEFE.EFEF..E................FEFEF............E.EFEF................ [ 24%]
.....................................................................E.. [ 29%]
..............................................................s.s....... [ 33%]
...s.s.................................................................. [ 37%]
........................................................................ [ 42%]
........................................................................ [ 46%]
..........................ss...........s.....s............s...s......FEFE [ 50%]
FEFEFEFEFEFE.E.EFE.EFEFEFEFEFEFEFEFEFEFEFEFEFEFEFE.E.EFEFEFEFEFEFEFEFEFE [ 53%]
FEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEFEEEEEEEEE.E [ 55%]
.E.E.E.E.E.E.E.E.E.E.E.E.EFEFEFEFEFEFEFE.E.E.EFE.E.E.E.E.E.E.E.Es.EFE
==================================== ERRORS ====================================
____ ERROR at teardown of TestDjangoRedisCache.test_setnx[sqlite_sentinel] _____
[gw6] linux -- Python 3.13.5 /tmp/django-redis/.tox/py313-dj52-redislatest/bin/python

self = <django_redis.cache.RedisCache object at 0x7fafa9058ec0>, args = ()
kwargs = {}

    @functools.wraps(method)
    def _decorator(self, *args, **kwargs):
        try:
>           return method(self, *args, **kwargs)

django_redis/cache.py:29: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <django_redis.cache.RedisCache object at 0x7fafa9058ec0>

    @omit_exception
    def clear(self):
>       return self.client.clear()

django_redis/cache.py:118: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <django_redis.client.default.DefaultClient object at 0x7fafa9059010>
client = <redis.client.Redis(<redis.sentinel.SentinelConnectionPool(service=127.0.0.1(master))>)>

    def clear(self, client: Optional[Redis] = None) -> None:
        """
        Flush all cache keys.
        """
    
        if client is None:
            client = self.get_client(write=True)
    
        try:
            client.flushdb()
        except _main_exceptions as e:
>           raise ConnectionInterrupted(connection=client) from e
E           django_redis.exceptions.ConnectionInterrupted: Redis MasterNotFoundError: No master found for '127.0.0.1'

django_redis/client/default.py:493: ConnectionInterrupted

During handling of the above exception, another exception occurred:

cache_settings = 'sqlite_sentinel'

    @pytest.fixture()
    def cache(cache_settings: str) -> Iterable[BaseCache]:
        from django import setup
    
        environ["DJANGO_SETTINGS_MODULE"] = f"settings.{cache_settings}"
        setup()
    
        from django.core.cache import cache as default_cache
    
        yield default_cache
>       default_cache.clear()

tests/conftest.py:48: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
django_redis/cache.py:36: in _decorator
    raise e.__cause__  # noqa: B904
django_redis/client/default.py:491: in clear
    client.flushdb()
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/commands/core.py:948: in flushdb
    return self.execute_command("FLUSHDB", *args, **kwargs)
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/client.py:623: in execute_command
    return self._execute_command(*args, **options)
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/client.py:629: in _execute_command
    conn = self.connection or pool.get_connection()
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/utils.py:191: in wrapper
    return func(*args, **kwargs)
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/connection.py:1530: in get_connection
    connection.connect()
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/sentinel.py:58: in connect
    return self.retry.call_with_retry(self._connect_retry, lambda error: None)
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/retry.py:92: in call_with_retry
    raise error
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/retry.py:87: in call_with_retry
    return do()
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/sentinel.py:48: in _connect_retry
    self.connect_to(self.connection_pool.get_master_address())
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/sentinel.py:110: in get_master_address
    master_address = self.sentinel_manager.discover_master(self.service_name)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <redis.sentinel.Sentinel(sentinels=[127.0.0.1:26379])>
service_name = '127.0.0.1'

    def discover_master(self, service_name):
        """
        Asks sentinel servers for the Redis master's address corresponding
        to the service labeled ``service_name``.
    
        Returns a pair (address, port) or raises MasterNotFoundError if no
        master is found.
        """
        collected_errors = list()
        for sentinel_no, sentinel in enumerate(self.sentinels):
            try:
                masters = sentinel.sentinel_masters()
            except (ConnectionError, TimeoutError) as e:
                collected_errors.append(f"{sentinel} - {e!r}")
                continue
            state = masters.get(service_name)
            if state and self.check_master_state(state, service_name):
                # Put this sentinel at the top of the list
                self.sentinels[0], self.sentinels[sentinel_no] = (
                    sentinel,
                    self.sentinels[0],
                )
    
                ip = (
                    self._force_master_ip
                    if self._force_master_ip is not None
                    else state["ip"]
                )
                return ip, state["port"]
    
        error_info = ""
        if len(collected_errors) > 0:
            error_info = f" : {', '.join(collected_errors)}"
>       raise MasterNotFoundError(f"No master found for {service_name!r}{error_info}")
E       redis.sentinel.MasterNotFoundError: No master found for '127.0.0.1'

.tox/py313-dj52-redislatest/lib/python3.13/site-packages/redis/sentinel.py:320: MasterNotFoundError
[...]

Environment (please complete the following information):

• Python version: 3.13.5
• Django Redis Version: 4002301
• Django Version: 5.2.3
• Redis Version: 6.0.2
• redis-py Version: 6.2.0

Additional context
It seems that something is going wrong in the configs and it ends up expecting a master called 127.0.0.1 rather than default_service. If I add such an entry to sentinel config (in addition to default_service), I get fewer, different failures:

______________________ test_session_save_does_not_resurrect_session_logged_out_in_other_context[sqlite_sentinel] ______________________
[gw6] linux -- Python 3.13.5 /tmp/django-redis/.tox/py313-dj52-redislatest/bin/python

session = <django.contrib.sessions.backends.cache.SessionStore object at 0x7effebd85dd0>

    def test_session_save_does_not_resurrect_session_logged_out_in_other_context(session):
        """
        Sessions shouldn't be resurrected by a concurrent request.
        """
        from django.contrib.sessions.backends.base import UpdateError
    
        # Create new session.
        s1 = SessionStore()
        s1["test_data"] = "value1"
>       s1.save(must_create=True)

tests/test_session.py:373: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/django/contrib/sessions/backends/cache.py:83: in save
    return self.create()
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/django/contrib/sessions/backends/cache.py:55: in create
    self._session_key = self._get_new_session_key()
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/django/contrib/sessions/backends/base.py:196: in _get_new_session_key
    if not self.exists(session_key):
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/django/contrib/sessions/backends/cache.py:117: in exists
    bool(session_key) and (self.cache_key_prefix + session_key) in self._cache
.tox/py313-dj52-redislatest/lib/python3.13/site-packages/django/core/cache/backends/base.py:301: in __contains__
    return self.has_key(key)
django_redis/cache.py:29: in _decorator
    return method(self, *args, **kwargs)
django_redis/cache.py:138: in has_key
    return self.client.has_key(*args, **kwargs)
django_redis/cache.py:76: in client
    self._client = self._client_cls(self._server, self._params, self)
django_redis/client/default.py:78: in __init__
    self.connection_factory = pool.get_connection_factory(options=self._options)
django_redis/pool.py:200: in get_connection_factory
    return cls(options or {})
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <django_redis.pool.SentinelConnectionFactory object at 0x7effebd87150>
options = {'CLOSE_CONNECTION': True, 'CONNECTION_POOL_CLASS': 'redis.sentinel.SentinelConnectionPool'}

    def __init__(self, options):
        # allow overriding the default SentinelConnectionPool class
        options.setdefault(
            "CONNECTION_POOL_CLASS", "redis.sentinel.SentinelConnectionPool"
        )
        super().__init__(options)
    
        sentinels = options.get("SENTINELS")
        if not sentinels:
            error_message = "SENTINELS must be provided as a list of (host, port)."
>           raise ImproperlyConfigured(error_message)
E           django.core.exceptions.ImproperlyConfigured: SENTINELS must be provided as a list of (host, port).

django_redis/pool.py:142: ImproperlyConfigured

[WisdomPill]

hello @mgorny, are you talking about your personal project or django-redis?

A while back I've made this PR which uses different Redis databases for the sake of test suite speed. It might be related.

[mgorny]

django-redis and yes, I'm pretty sure it is related :-). After all, prior to that I had to run tests for every configuration separately.

[WisdomPill]

I am not yet sure what is the root cause. I see there is this issue in redis-py. It affects only sentinel, and I suspect it might have to do with too much concurrency/timeouts. I've tried locally to solve it by tweaking the configuration but without success. To be honest I do not see it as a big issue, xdist and parallelisation were added just to speed up the CI since the package has a huge matrix of configuration to go through on every push and that annoyed me when working on the project. If you want to tackle the issue, be my guest, I am probably not going to spend the little time I have on this project to solve this issue, I prefer finishing the support for sets and hash maps, maybe asyncio or even cluster support.

[mgorny]

I'm pretty sure the problem is in django-redis, particularly the config leaking between tests somehow. If I add add --forked magic via pytest-forked, these problems disappear. Instead, I hit another problem (that I also hit without xdist at all):

______________ ERROR at setup of TestDjangoRedisCache.test_delete_pattern_with_settings_default_scan_count[sqlite_usock] ______________
[gw9] linux -- Python 3.13.5 /tmp/portage/dev-python/django-redis-6.0.0/work/django_redis-6.0.0-python3_13/install/usr/bin/python3.13
@pytest.fixture
    def patch_itersize_setting() -> Iterable[None]:
        # destroy cache to force recreation with overriden settings
>       del caches["default"]
            ^^^^^^^^^^^^^^^^^


tests/test_backend.py:25: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/lib/python3.13/site-packages/django/utils/connection.py:70: in __delitem__
    delattr(self._connections, key)
        key        = 'default'
        self       = <django.core.cache.CacheHandler object at 0x7fae02f72cf0>
/usr/lib/python3.13/site-packages/asgiref/local.py:128: in __delattr__
    delattr(storage, key)
        key        = 'default'
        self       = <asgiref.local.Local object at 0x7fae033e7390>
        storage    = <asgiref.local._CVar object at 0x7fae033e74d0>
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <asgiref.local._CVar object at 0x7fae033e74d0>, key = 'default'

    def __delattr__(self, key: str) -> None:
        storage_object = self._data.get({})
        if key in storage_object:
            del storage_object[key]
            self._data.set(storage_object)
        else:
>           raise AttributeError(f"{self!r} object has no attribute {key!r}")
E           AttributeError: <asgiref.local._CVar object at 0x7fae033e74d0> object has no attribute 'default'

key        = 'default'
self       = <asgiref.local._CVar object at 0x7fae033e74d0>
storage_object = {}

/usr/lib/python3.13/site-packages/asgiref/local.py:37: AttributeError

(and for every variant of this test)