Merge branch 'master' into lkchen-ray_data_llm

Signed-off-by: Linkun Chen <github@lkchen.net>

Author: lk-chen

.buildkite/cicd.rayci.yml

@@ -13,6 +13,7 @@ steps:
     depends_on:
       - oss-ci-base_test
       - forge
+    tags: tools
   - label: ":coral: reef: privileged container tests"
     commands:
       - bazel run //ci/ray_ci:test_in_docker --
@@ -25,3 +26,4 @@ steps:
     depends_on:
       - oss-ci-base_test
       - forge
+    tags: tools

.buildkite/core.rayci.yml

@@ -338,6 +338,7 @@ steps:
     key: core_flaky_tests
     tags:
       - python
+      - flaky
       - skip-on-premerge
     instance_type: large
     soft_fail: true

.buildkite/data.rayci.yml

@@ -221,6 +221,7 @@ steps:
     tags:
       - python
       - data
+      - flaky
       - skip-on-premerge
     instance_type: medium
     soft_fail: true

.buildkite/lint.rayci.yml

@@ -2,6 +2,9 @@ group: lint
 steps:
   - label: ":lint-roller: lint: {{matrix}}"
     key: lint-small
+    tags:
+      - lint
+      - always
     depends_on:
       - forge
     commands:
@@ -25,6 +28,7 @@ steps:
     tags:
       - oss
       - lint
+      - always
     key: lint-medium
     instance_type: medium
     depends_on: docbuild

.buildkite/macos/macos.rayci.yml

@@ -111,6 +111,7 @@ steps:
       - python
       - macos_wheels
       - oss
+      - flaky
       - skip_on_premerge
     job_env: MACOS
     instance_type: macos

.buildkite/ml.rayci.yml

@@ -274,6 +274,7 @@ steps:
     key: ml_flaky_tests
     tags:
       - train
+      - flaky
       - skip-on-premerge
     instance_type: large
     commands:

.buildkite/others.rayci.yml

@@ -18,6 +18,8 @@ steps:
     soft_fail: true
     job_env: oss-ci-base_test-py3.11
     depends_on: oss-ci-base_test-multipy
+    tags:
+      - always
 
   - label: ":tapioca: build: uv pip compile LLM dependencies"
     key: uv_pip_compile_llm_dependencies

.buildkite/rllib.rayci.yml

@@ -194,6 +194,7 @@ steps:
     tags:
       - rllib_gpu
       - gpu
+      - flaky
       - skip-on-premerge
     instance_type: gpu
     commands:
@@ -209,6 +210,7 @@ steps:
     key: rllib_flaky_tests_01
     tags:
       - rllib
+      - flaky
       - skip-on-premerge
     instance_type: large
     commands:
@@ -238,6 +240,7 @@ steps:
     key: rllib_flaky_tests_02
     tags:
       - rllib
+      - flaky
       - skip-on-premerge
     instance_type: large
     commands:

.buildkite/serve.rayci.yml

@@ -183,8 +183,9 @@ steps:
     key: serve_flaky_tests
     tags:
       - serve
-      - skip-on-premerge
       - python
+      - flaky
+      - skip-on-premerge
     instance_type: medium
     soft_fail: true
     commands:

.buildkite/windows.rayci.yml

@@ -128,7 +128,9 @@ steps:
 
   - label: "flaky :windows: core tests"
     key: windows_core_flaky_tests
-    tags: skip-on-premerge
+    tags:
+      - flaky
+      - skip-on-premerge
     job_env: WINDOWS
     instance_type: windows
     commands:
@@ -146,7 +148,9 @@ steps:
 
   - label: "flaky :windows: serverless tests"
     key: windows_serverless_flaky_tests
-    tags: skip-on-premerge
+    tags:
+      - flaky
+      - skip-on-premerge
     job_env: WINDOWS
     instance_type: windows
     commands:
@@ -164,7 +168,9 @@ steps:
 
   - label: "flaky :windows: serve tests"
     key: windows_serve_flaky_tests
-    tags: skip-on-premerge
+    tags:
+      - flaky
+      - skip-on-premerge
     job_env: WINDOWS
     instance_type: windows
     commands:

.pre-commit-config.yaml

@@ -58,9 +58,9 @@ repos:
     rev: 8.0.1
     hooks:
       - id: buildifier
-        files: ^(src|cpp|python)(/[^/]+)*/BUILD$
+        files: ^(src|cpp|python|rllib)(/[^/]+)*/BUILD$
       - id: buildifier-lint
-        files: ^(src|cpp|python)(/[^/]+)*/BUILD$
+        files: ^(src|cpp|python|rllib)(/[^/]+)*/BUILD$
 
   - repo: https://github.com/psf/black
     rev: 22.10.0

.rayciversion

@@ -1 +1 @@
-0.10.0
+0.11.0

BUILD.bazel

@@ -712,17 +712,6 @@ ray_cc_library(
     ],
 )
 
-ray_cc_library(
-    name = "gcs_autoscaler_state_manager",
-    srcs = ["src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc"],
-    hdrs = ["src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h"],
-    deps = [
-        ":gcs_init_data",
-        ":gcs_kv_manager",
-        ":gcs_service_rpc",
-    ],
-)
-
 ray_cc_library(
     name = "gcs_worker_manager",
     srcs = ["src/ray/gcs/gcs_server/gcs_worker_manager.cc"],
@@ -890,9 +879,7 @@ ray_cc_library(
         "@com_google_absl//absl/container:flat_hash_map",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
-        "@com_google_googletest//:gtest",
-        "@io_opencensus_cpp//opencensus/exporters/stats/prometheus:prometheus_exporter",
-        "@io_opencensus_cpp//opencensus/exporters/stats/stdout:stdout_exporter",
+        "@com_google_googletest//:gtest_prod",
         "@io_opencensus_cpp//opencensus/stats",
         "@io_opencensus_cpp//opencensus/tags",
     ],
@@ -2010,6 +1997,7 @@ ray_cc_test(
     deps = [
         ":grpc_common_lib",
         ":test_service_cc_grpc",
+        "@com_google_googletest//:gtest_main",
     ],
 )
 

ci/k8s/run-kuberay-doc-tests.sh

@@ -10,6 +10,61 @@ pip install -c python/requirements_compiled.txt pytest nbval bash_kernel
 python -m bash_kernel.install
 pip install "ray[default]==2.41.0"
 
+echo "--- Run a deliberate failure test to ensure the test script fails on error"
+# The following Jupyter notebook only contains a single cell that runs the `date` command.
+# The test script should fail because the output of the `date` command is different everytime.
+cat <<EOF > test.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "43a8bb95-f6f2-45a8-ba48-b16856b2106d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Wed Mar 26 06:28:51 PM CST 2025\n"
+     ]
+    }
+   ],
+   "source": [
+    "date"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Bash",
+   "language": "bash",
+   "name": "bash"
+  },
+  "language_info": {
+   "codemirror_mode": "shell",
+   "file_extension": ".sh",
+   "mimetype": "text/x-sh",
+   "name": "bash"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
+EOF
+set +e
+if pytest --nbval test.ipynb --nbval-kernel-name bash; then
+  echo "The test script should have failed but it didn't."
+  exit 1
+fi
+set -e
+
 echo "--- Run doc tests"
 cd doc/source/cluster/kubernetes
-py.test --nbval getting-started/raycluster-quick-start.ipynb --nbval-kernel-name bash --sanitize-with doc_sanitize.cfg
+TESTS=(
+  "getting-started/raycluster-quick-start.ipynb"
+)
+for test in "${TESTS[@]}"; do
+  echo "Running test: ${test}"
+  pytest --nbval "${test}" --nbval-kernel-name bash --sanitize-with doc_sanitize.cfg
+done

doc/source/data/examples/data_juicer_distributed_data_processing.md

@@ -23,7 +23,7 @@ See the [Data-Juicer 2.0: Cloud-Scale Adaptive Data Processing for Foundation Mo
 
 ### Ray mode in Data-Juicer
 
-- For most implementations of Data-Juicer [operators](https://github.com/modelscope/data-juicer/blob/main/docs/Operators.md), the core processing functions are engine-agnostic. Operators manage interoperability is primarily in [RayDataset](https://github.com/modelscope/data-juicer/blob/main//data_juicer/core/ray_data.py) and [RayExecutor](https://github.com/modelscope/data-juicer/blob/main//data_juicer/core/ray_executor.py), which are subclasses of the base `DJDataset` and `BaseExecutor`, respectively, and support both Ray [Tasks](ray-remote-functions) and [Actors](actor-guide).
+- For most implementations of Data-Juicer [operators](https://github.com/modelscope/data-juicer/blob/main/docs/Operators.md), the core processing functions are engine-agnostic. Operators manage interoperability is primarily in [RayDataset](https://github.com/modelscope/data-juicer/blob/main/data_juicer/core/data/ray_dataset.py) and [RayExecutor](https://github.com/modelscope/data-juicer/blob/main/data_juicer/core/executor/ray_executor.py), which are subclasses of the base `DJDataset` and `BaseExecutor`, respectively, and support both Ray [Tasks](ray-remote-functions) and [Actors](actor-guide).
 - The exception is the deduplication operators, which are challenging to scale in standalone mode. The names of these operators follow the pattern of [`ray_xx_deduplicator`](https://github.com/modelscope/data-juicer/blob/main//data_juicer/ops/deduplicator/).
 
 ### Subset splitting
@@ -152,4 +152,4 @@ python tools/process_data.py --config demos/process_on_ray/configs/dedup.yaml
 dj-process --config demos/process_on_ray/configs/dedup.yaml
 ```
 
-Data-Juicer deduplicates the demo dataset with the demo config file and export the result datasets to the directory specified by the `export_path` argument in the config file.
+Data-Juicer deduplicates the demo dataset with the demo config file and export the result datasets to the directory specified by the `export_path` argument in the config file.