From a0beafc8af15869b98e961b2b909494df8c1a459 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 10 Jun 2025 10:32:02 -0600 Subject: [PATCH 1/4] Add an option to skip thread-unsafe tests --- README.md | 10 +++++++++- src/pytest_run_parallel/plugin.py | 31 ++++++++++++++++++++++++------- tests/test_run_parallel.py | 22 ++++++++++++++++++++++ 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4e2eb94..82b6b67 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,11 @@ those fixtures are shared between threads. ## Features -- Two global CLI flags: +- Three global CLI flags: - `--parallel-threads` to run a test suite in parallel - `--iterations` to run multiple times in each thread + - `--skip-thread-unsafe` to skip running tests marked as or + detected to be thread-unsafe. - Three corresponding markers: - `pytest.mark.parallel_threads(n)` to mark a single test to run @@ -239,6 +241,12 @@ def test_skip_if_parallel(num_parallel_threads): ... ``` +You can skip tests marked as or detected to be thread-unsafe by passing +`--skip-thread-unsafe` in your pytest invocation. This is useful when running +pytest-run-parallel under [Thread Sanitizer](). Setting +`--skip-thread-unsafe=True` will avoid unnecessarily running tests where thread +sanitizer cannot detect races because the test is not parallelized. + Finally, the `thread_comp` fixture allows for parallel test debugging, by providing an instance of `ThreadComparator`, whose `__call__` method allows to check if all the values produced by all threads during an diff --git a/src/pytest_run_parallel/plugin.py b/src/pytest_run_parallel/plugin.py index 206b5a9..1032577 100644 --- a/src/pytest_run_parallel/plugin.py +++ b/src/pytest_run_parallel/plugin.py @@ -32,6 +32,14 @@ def pytest_addoption(parser): type=int, help="Set the number of threads used to execute each test concurrently.", ) + parser.addoption( + "--skip-thread-unsafe", + action="store", + dest="skip_thread_unsafe", + help="Whether to skip running thread-unsafe tests", + type=bool, + default=False, + ) parser.addini( "thread_unsafe_fixtures", "list of thread-unsafe fixture names that cause a test to " @@ -145,6 +153,8 @@ def pytest_itemcollected(item): fixtures = getattr(item, "fixturenames", ()) n_iterations = item.config.option.iterations + skip_thread_unsafe = item.config.option.skip_thread_unsafe + m = item.get_closest_marker("iterations") if m is not None: n_iterations = int(m.args[0]) @@ -153,13 +163,14 @@ def pytest_itemcollected(item): if n_workers > 1 and m is not None: n_workers = 1 reason = m.kwargs.get("reason", None) - if reason is not None: - item.user_properties.append(("thread_unsafe_reason", reason)) + if reason is None: + reason = "uses thread_unsafe marker" + item.user_properties.append(("thread_unsafe_reason", reason)) + if skip_thread_unsafe: + item.add_marker(pytest.mark.skip( + reason=f"Thread unsafe: {reason}")) else: - item.user_properties.append( - ("thread_unsafe_reason", "uses thread_unsafe marker") - ) - item.add_marker(pytest.mark.parallel_threads(1)) + item.add_marker(pytest.mark.parallel_threads(1)) if not hasattr(item, "obj"): if hasattr(item, "_parallel_custom_item"): @@ -183,6 +194,8 @@ def pytest_itemcollected(item): ] skipped_functions = frozenset((".".join(x[:-1]), x[-1]) for x in skipped_functions) + skip_thread_unsafe = item.config.option.skip_thread_unsafe + if n_workers > 1: thread_unsafe, thread_unsafe_reason = identify_thread_unsafe_nodes( item.obj, skipped_functions @@ -190,7 +203,11 @@ def pytest_itemcollected(item): if thread_unsafe: n_workers = 1 item.user_properties.append(("thread_unsafe_reason", thread_unsafe_reason)) - item.add_marker(pytest.mark.parallel_threads(1)) + if skip_thread_unsafe: + item.add_marker(pytest.mark.skip( + reason=f"Thread unsafe: {thread_unsafe_reason}")) + else: + item.add_marker(pytest.mark.parallel_threads(1)) unsafe_fixtures = _thread_unsafe_fixtures | set( item.config.getini("thread_unsafe_fixtures") diff --git a/tests/test_run_parallel.py b/tests/test_run_parallel.py index d8c16cd..5dabf34 100644 --- a/tests/test_run_parallel.py +++ b/tests/test_run_parallel.py @@ -589,6 +589,16 @@ def test_should_run_single_2(num_parallel_threads): ] ) + # check that skipping works too + result = pytester.runpytest( + "--parallel-threads=10", "--skip-thread-unsafe=True", "-v") + + result.stdout.fnmatch_lines([ + "*::test_should_run_single SKIPPED*", + "*::test_should_run_single_2 SKIPPED*" + ] + ) + def test_pytest_warns_detection(pytester): # create a temporary pytest test module @@ -636,6 +646,18 @@ def test_single_thread_warns_4(num_parallel_threads): ] ) + # check that skipping works too + result = pytester.runpytest( + "--parallel-threads=10", "--skip-thread-unsafe=True", "-v") + + result.stdout.fnmatch_lines( + [ + "*::test_single_thread_warns_1 SKIPPED*", + "*::test_single_thread_warns_2 SKIPPED*", + "*::test_single_thread_warns_3 SKIPPED*", + "*::test_single_thread_warns_4 SKIPPED*", + ] + ) @pytest.mark.skipif(psutil is None, reason="psutil needs to be installed") def test_auto_detect_cpus_psutil_affinity( From 75b8369dfa70548bc2494e38464c3099cea37d75 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci-lite[bot]" <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 16:33:26 +0000 Subject: [PATCH 2/4] [pre-commit.ci lite] apply automatic fixes --- src/pytest_run_parallel/plugin.py | 8 ++++---- tests/test_run_parallel.py | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/pytest_run_parallel/plugin.py b/src/pytest_run_parallel/plugin.py index 1032577..6b582fa 100644 --- a/src/pytest_run_parallel/plugin.py +++ b/src/pytest_run_parallel/plugin.py @@ -167,8 +167,7 @@ def pytest_itemcollected(item): reason = "uses thread_unsafe marker" item.user_properties.append(("thread_unsafe_reason", reason)) if skip_thread_unsafe: - item.add_marker(pytest.mark.skip( - reason=f"Thread unsafe: {reason}")) + item.add_marker(pytest.mark.skip(reason=f"Thread unsafe: {reason}")) else: item.add_marker(pytest.mark.parallel_threads(1)) @@ -204,8 +203,9 @@ def pytest_itemcollected(item): n_workers = 1 item.user_properties.append(("thread_unsafe_reason", thread_unsafe_reason)) if skip_thread_unsafe: - item.add_marker(pytest.mark.skip( - reason=f"Thread unsafe: {thread_unsafe_reason}")) + item.add_marker( + pytest.mark.skip(reason=f"Thread unsafe: {thread_unsafe_reason}") + ) else: item.add_marker(pytest.mark.parallel_threads(1)) diff --git a/tests/test_run_parallel.py b/tests/test_run_parallel.py index 5dabf34..ea51386 100644 --- a/tests/test_run_parallel.py +++ b/tests/test_run_parallel.py @@ -591,12 +591,11 @@ def test_should_run_single_2(num_parallel_threads): # check that skipping works too result = pytester.runpytest( - "--parallel-threads=10", "--skip-thread-unsafe=True", "-v") + "--parallel-threads=10", "--skip-thread-unsafe=True", "-v" + ) - result.stdout.fnmatch_lines([ - "*::test_should_run_single SKIPPED*", - "*::test_should_run_single_2 SKIPPED*" - ] + result.stdout.fnmatch_lines( + ["*::test_should_run_single SKIPPED*", "*::test_should_run_single_2 SKIPPED*"] ) @@ -648,7 +647,8 @@ def test_single_thread_warns_4(num_parallel_threads): # check that skipping works too result = pytester.runpytest( - "--parallel-threads=10", "--skip-thread-unsafe=True", "-v") + "--parallel-threads=10", "--skip-thread-unsafe=True", "-v" + ) result.stdout.fnmatch_lines( [ @@ -659,6 +659,7 @@ def test_single_thread_warns_4(num_parallel_threads): ] ) + @pytest.mark.skipif(psutil is None, reason="psutil needs to be installed") def test_auto_detect_cpus_psutil_affinity( pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch From 5ba1ae0d3e3a521ba1e2033a15d294eaa28c880e Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 10 Jun 2025 10:47:15 -0600 Subject: [PATCH 3/4] respond to lysandros' review --- src/pytest_run_parallel/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pytest_run_parallel/plugin.py b/src/pytest_run_parallel/plugin.py index 6b582fa..35020bb 100644 --- a/src/pytest_run_parallel/plugin.py +++ b/src/pytest_run_parallel/plugin.py @@ -193,8 +193,6 @@ def pytest_itemcollected(item): ] skipped_functions = frozenset((".".join(x[:-1]), x[-1]) for x in skipped_functions) - skip_thread_unsafe = item.config.option.skip_thread_unsafe - if n_workers > 1: thread_unsafe, thread_unsafe_reason = identify_thread_unsafe_nodes( item.obj, skipped_functions From e38e28dbccb50e295648ee63309af63cc50abac4 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Tue, 10 Jun 2025 10:48:13 -0600 Subject: [PATCH 4/4] add link --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 82b6b67..cb1b3fc 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,8 @@ def test_skip_if_parallel(num_parallel_threads): You can skip tests marked as or detected to be thread-unsafe by passing `--skip-thread-unsafe` in your pytest invocation. This is useful when running -pytest-run-parallel under [Thread Sanitizer](). Setting +pytest-run-parallel under [Thread +Sanitizer](https://clang.llvm.org/docs/ThreadSanitizer.html). Setting `--skip-thread-unsafe=True` will avoid unnecessarily running tests where thread sanitizer cannot detect races because the test is not parallelized.