From a9a303f6888fbbd034613e35e75f80a47711d5b8 Mon Sep 17 00:00:00 2001 From: Alon Livne <2005alonlivne@gmail.com> Date: Sat, 11 Jan 2025 17:02:13 +0200 Subject: [PATCH 1/6] add proxy gateways with --px arg Proxy gateways do not run workers, and are meant to be passed with the `via` attribute to additional gateways. They are useful for running multiple workers on remote machines. Example usage: ``` pytest -sv --dist=load --px "id=my_proxy//socket=IP:PORT" --tx "5*popen//via=my_proxy" ``` Proxy gateways do not run workers, anda re meant to be passed --- src/xdist/plugin.py | 11 +++++++++++ src/xdist/scheduler/each.py | 4 ++-- src/xdist/scheduler/load.py | 4 ++-- src/xdist/scheduler/loadscope.py | 4 ++-- src/xdist/scheduler/worksteal.py | 4 ++-- src/xdist/workermanage.py | 18 ++++++++++++++---- testing/test_plugin.py | 4 ++-- 7 files changed, 35 insertions(+), 14 deletions(-) diff --git a/src/xdist/plugin.py b/src/xdist/plugin.py index f670d9de..0cf90f86 100644 --- a/src/xdist/plugin.py +++ b/src/xdist/plugin.py @@ -139,6 +139,17 @@ def pytest_addoption(parser: pytest.Parser) -> None: "--tx ssh=user@codespeak.net//chdir=testcache" ), ) + group.addoption( + "--px", + dest="px", + action="append", + default=[], + metavar="xspec", + help=( + "Add a proxy gateway to pass to test execution environments using `via`. Example:\n" + "--px id=my_proxy//socket=192.168.1.102:8888 --tx 5*popen//via=my_proxy" + ), + ) group._addoption( "-d", action="store_true", diff --git a/src/xdist/scheduler/each.py b/src/xdist/scheduler/each.py index 8552b53f..bf914519 100644 --- a/src/xdist/scheduler/each.py +++ b/src/xdist/scheduler/each.py @@ -6,7 +6,7 @@ from xdist.remote import Producer from xdist.report import report_collection_diff -from xdist.workermanage import parse_spec_config +from xdist.workermanage import parse_tx_spec_config from xdist.workermanage import WorkerController @@ -26,7 +26,7 @@ class EachScheduling: def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: self.config = config - self.numnodes = len(parse_spec_config(config)) + self.numnodes = len(parse_tx_spec_config(config)) self.node2collection: dict[WorkerController, list[str]] = {} self.node2pending: dict[WorkerController, list[int]] = {} self._started: list[WorkerController] = [] diff --git a/src/xdist/scheduler/load.py b/src/xdist/scheduler/load.py index ac011e57..56f6092f 100644 --- a/src/xdist/scheduler/load.py +++ b/src/xdist/scheduler/load.py @@ -7,7 +7,7 @@ from xdist.remote import Producer from xdist.report import report_collection_diff -from xdist.workermanage import parse_spec_config +from xdist.workermanage import parse_tx_spec_config from xdist.workermanage import WorkerController @@ -58,7 +58,7 @@ class LoadScheduling: """ def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: - self.numnodes = len(parse_spec_config(config)) + self.numnodes = len(parse_tx_spec_config(config)) self.node2collection: dict[WorkerController, list[str]] = {} self.node2pending: dict[WorkerController, list[int]] = {} self.pending: list[int] = [] diff --git a/src/xdist/scheduler/loadscope.py b/src/xdist/scheduler/loadscope.py index ee3f2fdf..114561b4 100644 --- a/src/xdist/scheduler/loadscope.py +++ b/src/xdist/scheduler/loadscope.py @@ -8,7 +8,7 @@ from xdist.remote import Producer from xdist.report import report_collection_diff -from xdist.workermanage import parse_spec_config +from xdist.workermanage import parse_tx_spec_config from xdist.workermanage import WorkerController @@ -91,7 +91,7 @@ class LoadScopeScheduling: """ def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: - self.numnodes = len(parse_spec_config(config)) + self.numnodes = len(parse_tx_spec_config(config)) self.collection: list[str] | None = None self.workqueue: OrderedDict[str, dict[str, bool]] = OrderedDict() diff --git a/src/xdist/scheduler/worksteal.py b/src/xdist/scheduler/worksteal.py index 28708fcc..550372ee 100644 --- a/src/xdist/scheduler/worksteal.py +++ b/src/xdist/scheduler/worksteal.py @@ -7,7 +7,7 @@ from xdist.remote import Producer from xdist.report import report_collection_diff -from xdist.workermanage import parse_spec_config +from xdist.workermanage import parse_tx_spec_config from xdist.workermanage import WorkerController @@ -65,7 +65,7 @@ class WorkStealingScheduling: """ def __init__(self, config: pytest.Config, log: Producer | None = None) -> None: - self.numnodes = len(parse_spec_config(config)) + self.numnodes = len(parse_tx_spec_config(config)) self.node2collection: dict[WorkerController, list[str]] = {} self.node2pending: dict[WorkerController, list[int]] = {} self.pending: list[int] = [] diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index 08ba243f..ca873818 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -23,7 +23,7 @@ from xdist.remote import WorkerInfo -def parse_spec_config(config: pytest.Config) -> list[str]: +def parse_tx_spec_config(config: pytest.Config) -> list[str]: xspeclist = [] tx: list[str] = config.getvalue("tx") for xspec in tx: @@ -57,8 +57,15 @@ def __init__( if self.testrunuid is None: self.testrunuid = uuid.uuid4().hex self.group = execnet.Group(execmodel="main_thread_only") + for proxy_spec in self._getpxspecs(): + # Proxy gateways do not run workers, and are meant to be passed with the `via` attribute + # to additional gateways. + # They are useful for running multiple workers on remote machines. + if getattr(proxy_spec, "id", None) is None: + raise pytest.UsageError(f"Proxy gateway {proxy_spec} must include an id") + self.group.makegateway(proxy_spec) if specs is None: - specs = self._getxspecs() + specs = self._gettxspecs() self.specs: list[execnet.XSpec] = [] for spec in specs: if not isinstance(spec, execnet.XSpec): @@ -107,8 +114,11 @@ def setup_node( def teardown_nodes(self) -> None: self.group.terminate(self.EXIT_TIMEOUT) - def _getxspecs(self) -> list[execnet.XSpec]: - return [execnet.XSpec(x) for x in parse_spec_config(self.config)] + def _gettxspecs(self) -> list[execnet.XSpec]: + return [execnet.XSpec(x) for x in parse_tx_spec_config(self.config)] + + def _getpxspecs(self) -> list[execnet.XSpec]: + return [execnet.XSpec(x) for x in self.config.getoption("px")] def _getrsyncdirs(self) -> list[Path]: for spec in self.specs: diff --git a/testing/test_plugin.py b/testing/test_plugin.py index f3526670..81d9e873 100644 --- a/testing/test_plugin.py +++ b/testing/test_plugin.py @@ -295,7 +295,7 @@ class TestDistOptions: def test_getxspecs(self, pytester: pytest.Pytester) -> None: config = pytester.parseconfigure("--tx=popen", "--tx", "ssh=xyz") nodemanager = NodeManager(config) - xspecs = nodemanager._getxspecs() + xspecs = nodemanager._gettxspecs() assert len(xspecs) == 2 print(xspecs) assert xspecs[0].popen @@ -303,7 +303,7 @@ def test_getxspecs(self, pytester: pytest.Pytester) -> None: def test_xspecs_multiplied(self, pytester: pytest.Pytester) -> None: config = pytester.parseconfigure("--tx=3*popen") - xspecs = NodeManager(config)._getxspecs() + xspecs = NodeManager(config)._gettxspecs() assert len(xspecs) == 3 assert xspecs[1].popen From 17ac3359e357aa949352b579d855567cc8ad5338 Mon Sep 17 00:00:00 2001 From: Alon Livne <2005alonlivne@gmail.com> Date: Sat, 11 Jan 2025 17:14:02 +0200 Subject: [PATCH 2/6] add proxy gateway docs Add docs for using proxy gateways to run multiple workers on remote machines --- docs/remote.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/remote.rst b/docs/remote.rst index f42fb812..6fe806f9 100644 --- a/docs/remote.rst +++ b/docs/remote.rst @@ -66,6 +66,20 @@ new socket host with something like this:: pytest -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg +Using proxies to run multiple workers on remote machines +--------------------------------------- + +In case you want to run multiple workers on a remote machine, +you can create a proxy gateway for the machine, and run multiple +workers using the `via` attribute.:: + + pytest -d --px id=my_proxy//socket=192.168.1.102:8888 --tx 5*popen//via=my_proxy + +Here we declare a proxy gateway using the `--px` arg, and +create 5 workers that run on the remote server using the proxy. +Note that the proxy gateway does not run a worker, thus only 5 +workers are created. + Running tests on many platforms at once --------------------------------------- From 333e2d34350e7220118b162255ed83efb46203f5 Mon Sep 17 00:00:00 2001 From: Alon Livne <2005alonlivne@gmail.com> Date: Sat, 11 Jan 2025 17:18:45 +0200 Subject: [PATCH 3/6] add proxy gateways changelog --- changelog/1170.feature | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog/1170.feature diff --git a/changelog/1170.feature b/changelog/1170.feature new file mode 100644 index 00000000..d67e5947 --- /dev/null +++ b/changelog/1170.feature @@ -0,0 +1,4 @@ +Add the `--px` arg to create proxy gateways. + +Proxy gateways are passed to additional gateways using the `via` keyword. +They can serve as a way to run multiple workers on remote machines. From 35db0e79c2335b41b0f157b8661cb92e704e2451 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 12 Jan 2025 18:38:19 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/xdist/workermanage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/xdist/workermanage.py b/src/xdist/workermanage.py index ca873818..201c8e71 100644 --- a/src/xdist/workermanage.py +++ b/src/xdist/workermanage.py @@ -62,7 +62,9 @@ def __init__( # to additional gateways. # They are useful for running multiple workers on remote machines. if getattr(proxy_spec, "id", None) is None: - raise pytest.UsageError(f"Proxy gateway {proxy_spec} must include an id") + raise pytest.UsageError( + f"Proxy gateway {proxy_spec} must include an id" + ) self.group.makegateway(proxy_spec) if specs is None: specs = self._gettxspecs() From 8632b38203fb9c1748720d260105f9d4e7943712 Mon Sep 17 00:00:00 2001 From: Alon Livne <2005alonlivne@gmail.com> Date: Sun, 12 Jan 2025 21:30:43 +0200 Subject: [PATCH 5/6] add tests for proxy gateways The test `test_proxy_gateway_setup_nodes` makes sure that 2 execnet gateways are created, but only one node is allocated. The test `test_proxy_gateway` tries to run a test on a proxy. --- testing/test_workermanage.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/testing/test_workermanage.py b/testing/test_workermanage.py index 491d0424..ee4c62ac 100644 --- a/testing/test_workermanage.py +++ b/testing/test_workermanage.py @@ -368,6 +368,35 @@ def test_one(): (rep,) = reprec.getreports("pytest_runtest_logreport") assert rep.passed + def test_proxy_gateway_setup_nodes(self, pytester: pytest.Pytester) -> None: + nodemanager = NodeManager( + pytester.parseconfig( + "--px", "popen//id=my_proxy", "--tx", "popen//via=my_proxy" + ) + ) + nodes = nodemanager.setup_nodes(None) # type: ignore[arg-type] + + # Proxy gateways are considered as an execnet gateway + assert len(nodemanager.group) == 2 + + # Proxy gateways do not run workers + assert len(nodes) == 1 + + def test_proxy_gateway(self, pytester: pytest.Pytester) -> None: + pytester.makepyfile( + __init__="", + test_x=""" + def test_one(): + pass + """, + ) + reprec = pytester.inline_run( + "-d", "--px", "popen//id=my_proxy", "--tx", "popen//via=my_proxy" + ) + rep = reprec.getreports("pytest_runtest_logreport") + assert rep[1].passed + + class MyWarning(UserWarning): pass From c11ca7a8fe0a675907b1aa4e65bf83f85226c956 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 12 Jan 2025 19:32:00 +0000 Subject: [PATCH 6/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- testing/test_workermanage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/test_workermanage.py b/testing/test_workermanage.py index ee4c62ac..6fb2795c 100644 --- a/testing/test_workermanage.py +++ b/testing/test_workermanage.py @@ -397,7 +397,6 @@ def test_one(): assert rep[1].passed - class MyWarning(UserWarning): pass