From 5141c2eb33e16d6cf571690890c97e644062a2a0 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 2 May 2025 14:27:04 -0500 Subject: [PATCH 1/6] fix Signed-off-by: Edward Oakes --- .../tests/test_runtime_env_conda_and_pip_4.py | 73 ++++++------------- 1 file changed, 24 insertions(+), 49 deletions(-) diff --git a/python/ray/tests/test_runtime_env_conda_and_pip_4.py b/python/ray/tests/test_runtime_env_conda_and_pip_4.py index 3eaaad030c2a..b10db2af4e48 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip_4.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip_4.py @@ -76,59 +76,34 @@ def f3(): assert all(ray.get([f.remote(), f2.remote(), f3.remote()])) -class TestGC: - @pytest.mark.skipif( - os.environ.get("CI") and sys.platform != "linux", - reason="Requires PR wheels built in CI, so only run on linux CI machines.", - ) - @pytest.mark.parametrize("field", ["pip"]) - def test_pip_ray_is_overwritten(self, start_cluster, field): - cluster, address = start_cluster - - # It should be OK to install packages with ray dependency. - ray.init(address, runtime_env={"pip": ["pip-install-test==0.5", "ray"]}) - - @ray.remote - def f(): - import pip_install_test # noqa: F401 - - return True - - # Ensure that the runtime env has been installed. - assert ray.get(f.remote()) - - ray.shutdown() - - # It should be OK if cluster ray meets the installing ray version. - ray.init(address, runtime_env={"pip": ["pip-install-test==0.5", "ray>=1.12.0"]}) - - @ray.remote - def f(): - import pip_install_test # noqa: F401 - - return True - - # Ensure that the runtime env has been installed. - assert ray.get(f.remote()) - - ray.shutdown() - - # It will raise exceptions if ray is overwritten. - with pytest.raises(Exception): - ray.init( - address, runtime_env={"pip": ["pip-install-test==0.5", "ray<=1.6.0"]} - ) +@pytest.mark.skipif( + os.environ.get("CI") and sys.platform != "linux", + reason="Requires PR wheels built in CI, so only run on linux CI machines.", +) +@pytest.mark.parametrize("field", ["pip"]) +def test_pip_ray_is_overwritten(start_cluster, field): + cluster, address = start_cluster + ray.init(address) - @ray.remote - def f(): - import pip_install_test # noqa: F401 + @ray.remote + def f(): + import pip_install_test # noqa: F401 - return True + # Test an unconstrained "ray" dependency (should work). + ray.get(f.options(runtime_env={"pip": ["pip-install-test==0.5", "ray"]}).remote()) - # Ensure that the runtime env has been installed. - assert ray.get(f.remote()) + # Test a constrained "ray" dependency that matches the env (should work). + ray.get( + f.options(runtime_env={"pip": ["pip-install-test==0.5", "ray>=2.0"]}).remote() + ) - ray.shutdown() + # Test a constrained "ray" dependency that doesn't match the env (shouldn't work). + with pytest.raises(Exception): + ray.get( + f.options( + runtime_env={"pip": ["pip-install-test==0.5", "ray<2.0"]} + ).remote() + ) # pytest-virtualenv doesn't support Python 3.12 as of now, see more details here: From 1a99714b9fd0dfeb20fed29276e51ee682252dc2 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 2 May 2025 14:39:46 -0500 Subject: [PATCH 2/6] WIP Signed-off-by: Edward Oakes --- .../tests/test_runtime_env_conda_and_pip_4.py | 65 +++++++------------ 1 file changed, 22 insertions(+), 43 deletions(-) diff --git a/python/ray/tests/test_runtime_env_conda_and_pip_4.py b/python/ray/tests/test_runtime_env_conda_and_pip_4.py index b10db2af4e48..b7b9e24d962d 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip_4.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip_4.py @@ -12,17 +12,13 @@ os.environ["RAY_RUNTIME_ENV_LOCAL_DEV_MODE"] = "1" -def test_in_virtualenv(start_cluster): +def test_in_virtualenv(ray_start_regular_shared): assert ( virtualenv_utils.is_in_virtualenv() is False and "IN_VIRTUALENV" not in os.environ ) or (virtualenv_utils.is_in_virtualenv() is True and "IN_VIRTUALENV" in os.environ) - cluster, address = start_cluster - runtime_env = {"pip": ["pip-install-test==0.5"]} - ray.init(address, runtime_env=runtime_env) - - @ray.remote + @ray.remote(runtime_env={"pip": ["pip-install-test==0.5"]}) def f(): import pip_install_test # noqa: F401 @@ -33,47 +29,33 @@ def f(): assert ray.get(f.remote()) -def test_multiple_pip_installs(start_cluster, monkeypatch): +def test_multiple_pip_installs(ray_start_regular_shared): """Test that multiple pip installs don't interfere with each other.""" - monkeypatch.setenv("RUNTIME_ENV_RETRY_TIMES", "0") - cluster, address = start_cluster - if sys.platform == "win32" and "ray" not in address: pytest.skip( "Failing on windows, as python.exe is in use during deletion attempt." ) - ray.init( - address, - runtime_env={ - "pip": ["pip-install-test"], - "env_vars": {"TEST_VAR_1": "test_1"}, - }, - ) - @ray.remote def f(): - return True + return os.environ["TEST_VAR"] - @ray.remote( - runtime_env={ - "pip": ["pip-install-test"], - "env_vars": {"TEST_VAR_2": "test_2"}, - } - ) - def f2(): - return True - - @ray.remote( - runtime_env={ - "pip": ["pip-install-test"], - "env_vars": {"TEST_VAR_3": "test_3"}, - } - ) - def f3(): - return True - - assert all(ray.get([f.remote(), f2.remote(), f3.remote()])) + assert ray.get( + [ + f.options( + runtime_env={ + "pip": ["pip-install-test"], + "env_vars": {"TEST_VAR": "1"}, + } + ).remote(), + f.options( + runtime_env={ + "pip": ["pip-install-test"], + "env_vars": {"TEST_VAR": "2"}, + } + ).remote(), + ] + ) == ["1", "2"] @pytest.mark.skipif( @@ -81,10 +63,7 @@ def f3(): reason="Requires PR wheels built in CI, so only run on linux CI machines.", ) @pytest.mark.parametrize("field", ["pip"]) -def test_pip_ray_is_overwritten(start_cluster, field): - cluster, address = start_cluster - ray.init(address) - +def test_pip_ray_is_overwritten(ray_start_regular_shared, field): @ray.remote def f(): import pip_install_test # noqa: F401 @@ -133,7 +112,7 @@ def test_run_in_virtualenv(cloned_virtualenv): @pytest.mark.skipif( "IN_VIRTUALENV" in os.environ, reason="Pip option not supported in virtual env." ) -def test_runtime_env_with_pip_config(start_cluster): +def test_runtime_env_with_pip_config(ray_start_regular_shared): @ray.remote( runtime_env={ "pip": {"packages": ["pip-install-test==0.5"], "pip_version": "==24.1.2"} From df33ef5e60f970a4ef3f5af544508ea0bc5e932a Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 2 May 2025 14:41:47 -0500 Subject: [PATCH 3/6] fix Signed-off-by: Edward Oakes --- python/ray/tests/test_runtime_env_conda_and_pip_4.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python/ray/tests/test_runtime_env_conda_and_pip_4.py b/python/ray/tests/test_runtime_env_conda_and_pip_4.py index b7b9e24d962d..1085f4b5fb24 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip_4.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip_4.py @@ -29,12 +29,11 @@ def f(): assert ray.get(f.remote()) +@pytest.mark.skipif( + sys.platform == "win32", reason="python.exe in use during deletion." +) def test_multiple_pip_installs(ray_start_regular_shared): """Test that multiple pip installs don't interfere with each other.""" - if sys.platform == "win32" and "ray" not in address: - pytest.skip( - "Failing on windows, as python.exe is in use during deletion attempt." - ) @ray.remote def f(): @@ -62,8 +61,7 @@ def f(): os.environ.get("CI") and sys.platform != "linux", reason="Requires PR wheels built in CI, so only run on linux CI machines.", ) -@pytest.mark.parametrize("field", ["pip"]) -def test_pip_ray_is_overwritten(ray_start_regular_shared, field): +def test_pip_ray_is_overwritten(ray_start_regular_shared): @ray.remote def f(): import pip_install_test # noqa: F401 From 0c1473bc98fa866037718ab504a296f51aa517d5 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 2 May 2025 15:12:56 -0500 Subject: [PATCH 4/6] fix Signed-off-by: Edward Oakes --- python/ray/tests/test_runtime_env_conda_and_pip.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/python/ray/tests/test_runtime_env_conda_and_pip.py b/python/ray/tests/test_runtime_env_conda_and_pip.py index df9ba385c938..8c47261a5d90 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip.py @@ -116,7 +116,7 @@ class TestGC: reason="Needs PR wheels built in CI, so only run on linux CI machines.", ) @pytest.mark.parametrize("field", ["conda", "pip"]) - @pytest.mark.parametrize("spec_format", ["file", "python_object"]) + @pytest.mark.parametrize("spec_format", ["python_object"]) def test_job_level_gc( self, runtime_env_disable_URI_cache, start_cluster, field, spec_format, tmp_path ): @@ -139,9 +139,6 @@ def f(): # Ensure that the runtime env has been installed. assert ray.get(f.remote()) - # Sleep some seconds before checking that we didn't GC. Otherwise this - # check may spuriously pass. - time.sleep(2) assert not check_local_files_gced(cluster) ray.shutdown() @@ -163,7 +160,7 @@ def f(): reason="Requires PR wheels built in CI, so only run on linux CI machines.", ) @pytest.mark.parametrize("field", ["conda", "pip"]) - @pytest.mark.parametrize("spec_format", ["file", "python_object"]) + @pytest.mark.parametrize("spec_format", ["python_object"]) def test_detached_actor_gc( self, runtime_env_disable_URI_cache, start_cluster, field, spec_format, tmp_path ): From fcc2ff16f1971be348092a94025ebcbb5649597b Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Fri, 2 May 2025 17:51:11 -0500 Subject: [PATCH 5/6] fix Signed-off-by: Edward Oakes --- python/ray/tests/test_runtime_env_conda_and_pip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ray/tests/test_runtime_env_conda_and_pip.py b/python/ray/tests/test_runtime_env_conda_and_pip.py index 8c47261a5d90..ac3b2ebdd742 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip.py @@ -2,7 +2,6 @@ import pytest import sys import platform -import time from ray._private.test_utils import ( wait_for_condition, chdir, From 09e9c57fc9f00c6e969ba2a8132f4ae7dc0bda64 Mon Sep 17 00:00:00 2001 From: Edward Oakes Date: Sat, 3 May 2025 07:24:37 -0500 Subject: [PATCH 6/6] fix Signed-off-by: Edward Oakes --- python/ray/tests/test_runtime_env_conda_and_pip.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/ray/tests/test_runtime_env_conda_and_pip.py b/python/ray/tests/test_runtime_env_conda_and_pip.py index ac3b2ebdd742..44293b0ae127 100644 --- a/python/ray/tests/test_runtime_env_conda_and_pip.py +++ b/python/ray/tests/test_runtime_env_conda_and_pip.py @@ -15,6 +15,7 @@ MAX_INTERNAL_PIP_FILENAME_TRIES, ) from ray.runtime_env import RuntimeEnv +from ray.util.state import list_tasks import yaml import tempfile @@ -138,7 +139,12 @@ def f(): # Ensure that the runtime env has been installed. assert ray.get(f.remote()) - assert not check_local_files_gced(cluster) + + # Check that after the task is finished, the runtime_env is not GC'd + # because the job is still alive. + wait_for_condition(lambda: list_tasks()[0].state == "FINISHED") + for _ in range(5): + assert not check_local_files_gced(cluster) ray.shutdown()