Skip tests for MLDataset when ray>=2.0 (#3319)

Use latest ray

Temporarily pin numpy

Author: vcfgv

.github/workflows/platform-ci.yml

@@ -103,10 +103,12 @@ jobs:
               rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
             fi
             if [ -n "$WITH_RAY" ] || [ -n "$WITH_RAY_DAG" ] || [ -n "$WITH_RAY_DEPLOY" ]; then
-              pip install "xgboost_ray==0.1.5" "xgboost<1.6.0" "protobuf<4"
+              pip install "xgboost_ray" "protobuf<4"
+              # Future PR will Unpin numpy.
+              pip install "numpy<1.24"
               # Use standard ray releases when ownership bug is fixed
-              pip uninstall -y ray
-              pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/c03d0432f3bb40f3c597b7fc450870ba5e34ad56/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
+              # pip uninstall -y ray
+              # pip install https://s3-us-west-2.amazonaws.com/ray-wheels/master/c03d0432f3bb40f3c597b7fc450870ba5e34ad56/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
               # Ray Datasets need pyarrow>=6.0.1
               pip install "pyarrow>=6.0.1"
             fi

mars/dataframe/contrib/raydataset/tests/test_mldataset.py

@@ -23,6 +23,7 @@
 from .....deploy.oscar.session import new_session
 from .....tests.core import require_ray
 from .....utils import lazy_import
+from ....utils import ray_deprecate_ml_dataset
 from ....contrib import raydataset as mdd
 
 ray = lazy_import("ray")
@@ -53,6 +54,10 @@ async def create_cluster(request):
 
 @require_ray
 @pytest.mark.asyncio
+@pytest.mark.skipif(
+    ray_deprecate_ml_dataset in (True, None),
+    reason="Ray (>=2.0) has deprecated MLDataset.",
+)
 async def test_dataset_related_classes(ray_start_regular_shared):
     from ..mldataset import ChunkRefBatch
 
@@ -74,6 +79,10 @@ async def test_dataset_related_classes(ray_start_regular_shared):
 @require_ray
 @pytest.mark.asyncio
 @pytest.mark.parametrize("chunk_size_and_num_shards", [[5, 5], [5, 4], [None, None]])
+@pytest.mark.skipif(
+    ray_deprecate_ml_dataset in (True, None),
+    reason="Ray (>=2.0) has deprecated MLDataset.",
+)
 async def test_convert_to_ray_mldataset(
     ray_start_regular_shared, create_cluster, chunk_size_and_num_shards
 ):
@@ -92,6 +101,10 @@ async def test_convert_to_ray_mldataset(
 @require_ray
 @pytest.mark.asyncio
 @pytest.mark.skipif(xgboost_ray is None, reason="xgboost_ray not installed")
+@pytest.mark.skipif(
+    ray_deprecate_ml_dataset in (True, None),
+    reason="Ray (>=2.0) has deprecated MLDataset.",
+)
 async def test_mars_with_xgboost(ray_start_regular_shared, create_cluster):
     from xgboost_ray import RayDMatrix, RayParams, train, predict
     from sklearn.datasets import load_breast_cancer

mars/dataframe/datasource/tests/test_datasource.py

@@ -46,6 +46,7 @@
     read_ray_mldataset,
     DataFrameReadMLDataset,
 )
+from ...utils import ray_deprecate_ml_dataset
 from ..series import from_pandas as from_pandas_series
 
 
@@ -587,6 +588,10 @@ def test_date_range():
 
 
 @require_ray
+@pytest.mark.skipif(
+    ray_deprecate_ml_dataset in (True, None),
+    reason="Ray (>=2.0) has deprecated MLDataset.",
+)
 def test_read_ray_mldataset(ray_start_regular):
     test_df1 = pd.DataFrame(
         {

mars/dataframe/datasource/tests/test_datasource_execution.py

@@ -43,13 +43,13 @@
 from ....config import option_context
 from ....tests.core import require_cudf, require_ray
 from ....utils import arrow_array_to_objects, lazy_import, pd_release_version
+from ...utils import ray_deprecate_ml_dataset
 from ..dataframe import from_pandas as from_pandas_df
 from ..series import from_pandas as from_pandas_series
 from ..index import from_pandas as from_pandas_index, from_tileable
 from ..from_tensor import dataframe_from_tensor, dataframe_from_1d_tileables
 from ..from_records import from_records
 
-
 ray = lazy_import("ray")
 _date_range_use_inclusive = pd_release_version[:2] >= (1, 4)
 
@@ -1214,7 +1214,10 @@ def test_read_raydataset(ray_start_regular, ray_create_mars_cluster):
 
 
 @require_ray
-@pytest.mark.skip_ray_dag  # mldataset is not compatible with Ray DAG
+@pytest.mark.skipif(
+    ray_deprecate_ml_dataset in (True, None),
+    reason="Ray (>=2.0) has deprecated MLDataset.",
+)
 def test_read_ray_mldataset(ray_start_regular, ray_create_mars_cluster):
     test_dfs = [
         pd.DataFrame(

mars/dataframe/utils.py

@@ -44,6 +44,7 @@
     is_full_slice,
     parse_readable_size,
     is_on_ray,
+    parse_version,
 )
 
 try:
@@ -53,6 +54,14 @@
 
 cudf = lazy_import("cudf", rename="cudf")
 vineyard = lazy_import("vineyard")
+try:
+    import ray
+
+    ray_release_version = parse_version(ray.__version__).release
+    ray_deprecate_ml_dataset = ray_release_version[:2] >= (2, 0)
+except ImportError:
+    ray_release_version = None
+    ray_deprecate_ml_dataset = None
 logger = logging.getLogger(__name__)