Skip to content

Commit 4c83a8f

Browse files
lk-chenlk-chen
committed
Merge branch 'master' into lkchen-ray_data_llm
Signed-off-by: Linkun Chen <github@lkchen.net>
2 parents 1e8188d + 3045482 commit 4c83a8f

File tree

77 files changed

+1089
-300
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+1089
-300
lines changed

‎.buildkite/core.rayci.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ steps:
3636
- "3.10"
3737
- "3.11"
3838
- "3.12"
39+
- "3.13"
3940
env:
4041
PYTHON_VERSION: "{{matrix}}"
4142
EXTRA_DEPENDENCY: core
@@ -287,6 +288,7 @@ steps:
287288
- "3.10"
288289
- "3.11"
289290
- "3.12"
291+
- "3.13"
290292

291293
# cpp tests
292294
- label: ":ray: core: cpp tests"

‎.buildkite/others.rayci.yml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ steps:
2323
key: uv_pip_compile_llm_dependencies
2424
instance_type: small
2525
command: ./ci/test_compile_llm_requirements.sh
26-
soft_fail: true
2726
job_env: oss-ci-base_test-py3.11
2827
depends_on: oss-ci-base_test-multipy
2928

‎.buildkite/release-automation/pre_release.rayci.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,19 @@ steps:
5151
RAYCI_RELEASE: 1
5252
RAYCI_SCHEDULE: "nightly"
5353

54+
- label: "Trigger :kubernetes: Kuberay CI Tests"
55+
if: build.env("RAYCI_WEEKLY_RELEASE_NIGHTLY") == "1"
56+
trigger: "ray-ecosystem-ci-kuberay-ci"
57+
key: trigger-kuberay
58+
depends_on: trigger-postmerge-nightly
59+
build:
60+
branch: "release-1.3"
61+
message: "Triggered by release-automation build #${BUILDKITE_BUILD_NUMBER}"
62+
env:
63+
# KubeRay CI will pull an image based on this commit and the current date
64+
RAY_NIGHTLY_COMMIT: "${BUILDKITE_COMMIT}"
65+
IS_FROM_RAY_RELEASE_AUTOMATION: 1
66+
5467
- label: "Trigger Postmerge MacOS test"
5568
key: trigger-postmerge-macos
5669
trigger: "postmerge-macos"

‎BUILD.bazel

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -250,8 +250,8 @@ ray_cc_library(
250250
deps = [
251251
":grpc_common_lib",
252252
":node_manager_cc_grpc",
253-
"//src/ray/common:id",
254253
"//src/ray/common:asio",
254+
"//src/ray/common:id",
255255
"//src/ray/common:status",
256256
"@com_github_grpc_grpc//:grpc++",
257257
],
@@ -338,9 +338,9 @@ ray_cc_library(
338338
deps = [
339339
":grpc_common_lib",
340340
":pubsub_lib",
341+
":worker_cc_grpc",
341342
"//src/ray/common:asio",
342343
"//src/ray/common:id",
343-
":worker_cc_grpc",
344344
"@com_github_grpc_grpc//:grpc++",
345345
],
346346
)
@@ -594,18 +594,18 @@ ray_cc_library(
594594

595595
ray_cc_library(
596596
name = "gcs_state_util",
597-
hdrs = ["src/ray/gcs/gcs_server/state_util.h"],
598597
srcs = ["src/ray/gcs/gcs_server/state_util.cc"],
598+
hdrs = ["src/ray/gcs/gcs_server/state_util.h"],
599599
deps = [
600600
"//src/ray/protobuf:gcs_cc_proto",
601-
"@com_google_absl//absl/container:flat_hash_map",
601+
"@com_google_absl//absl/container:flat_hash_map",
602602
],
603603
)
604604

605605
ray_cc_library(
606606
name = "gcs_table_storage",
607-
hdrs = ["src/ray/gcs/gcs_server/gcs_table_storage.h"],
608607
srcs = ["src/ray/gcs/gcs_server/gcs_table_storage.cc"],
608+
hdrs = ["src/ray/gcs/gcs_server/gcs_table_storage.h"],
609609
deps = [
610610
":gcs_callback",
611611
":gcs_in_memory_store_client",
@@ -620,8 +620,8 @@ ray_cc_library(
620620

621621
ray_cc_library(
622622
name = "gcs_init_data",
623-
hdrs = ["src/ray/gcs/gcs_server/gcs_init_data.h"],
624623
srcs = ["src/ray/gcs/gcs_server/gcs_init_data.cc"],
624+
hdrs = ["src/ray/gcs/gcs_server/gcs_init_data.h"],
625625
deps = [
626626
":gcs_callback",
627627
":gcs_table_storage",
@@ -634,8 +634,8 @@ ray_cc_library(
634634

635635
ray_cc_library(
636636
name = "gcs_kv_manager",
637-
hdrs = ["src/ray/gcs/gcs_server/gcs_kv_manager.h"],
638637
srcs = ["src/ray/gcs/gcs_server/gcs_kv_manager.cc"],
638+
hdrs = ["src/ray/gcs/gcs_server/gcs_kv_manager.h"],
639639
deps = [
640640
":gcs_service_rpc",
641641
"//src/ray/common:asio",
@@ -657,8 +657,8 @@ ray_cc_library(
657657

658658
ray_cc_library(
659659
name = "gcs_usage_stats_client",
660-
hdrs = ["src/ray/gcs/gcs_server/usage_stats_client.h"],
661660
srcs = ["src/ray/gcs/gcs_server/usage_stats_client.cc"],
661+
hdrs = ["src/ray/gcs/gcs_server/usage_stats_client.h"],
662662
deps = [
663663
":gcs_kv_manager",
664664
"//src/ray/common:asio",
@@ -668,8 +668,8 @@ ray_cc_library(
668668

669669
ray_cc_library(
670670
name = "gcs_store_client_kv",
671-
hdrs = ["src/ray/gcs/gcs_server/store_client_kv.h"],
672671
srcs = ["src/ray/gcs/gcs_server/store_client_kv.cc"],
672+
hdrs = ["src/ray/gcs/gcs_server/store_client_kv.h"],
673673
deps = [
674674
":gcs_kv_manager",
675675
":gcs_store_client",
@@ -678,8 +678,8 @@ ray_cc_library(
678678

679679
ray_cc_library(
680680
name = "gcs_pubsub_handler",
681-
hdrs = ["src/ray/gcs/gcs_server/pubsub_handler.h"],
682681
srcs = ["src/ray/gcs/gcs_server/pubsub_handler.cc"],
682+
hdrs = ["src/ray/gcs/gcs_server/pubsub_handler.h"],
683683
deps = [
684684
":gcs_pub_sub_lib",
685685
"//src/ray/protobuf:gcs_service_cc_proto",
@@ -690,8 +690,8 @@ ray_cc_library(
690690

691691
ray_cc_library(
692692
name = "gcs_runtime_env_handler",
693-
hdrs = ["src/ray/gcs/gcs_server/runtime_env_handler.h"],
694693
srcs = ["src/ray/gcs/gcs_server/runtime_env_handler.cc"],
694+
hdrs = ["src/ray/gcs/gcs_server/runtime_env_handler.h"],
695695
deps = [
696696
":gcs_service_rpc",
697697
":node_manager_rpc",
@@ -703,8 +703,8 @@ ray_cc_library(
703703

704704
ray_cc_library(
705705
name = "gcs_redis_failure_detector",
706-
hdrs = ["src/ray/gcs/gcs_server/gcs_redis_failure_detector.h"],
707706
srcs = ["src/ray/gcs/gcs_server/gcs_redis_failure_detector.cc"],
707+
hdrs = ["src/ray/gcs/gcs_server/gcs_redis_failure_detector.h"],
708708
deps = [
709709
":redis_client",
710710
"//src/ray/common:asio",
@@ -714,8 +714,8 @@ ray_cc_library(
714714

715715
ray_cc_library(
716716
name = "gcs_autoscaler_state_manager",
717-
hdrs = ["src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h"],
718717
srcs = ["src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.cc"],
718+
hdrs = ["src/ray/gcs/gcs_server/gcs_autoscaler_state_manager.h"],
719719
deps = [
720720
":gcs_init_data",
721721
":gcs_kv_manager",
@@ -725,15 +725,15 @@ ray_cc_library(
725725

726726
ray_cc_library(
727727
name = "gcs_worker_manager",
728-
hdrs = ["src/ray/gcs/gcs_server/gcs_worker_manager.h"],
729728
srcs = ["src/ray/gcs/gcs_server/gcs_worker_manager.cc"],
729+
hdrs = ["src/ray/gcs/gcs_server/gcs_worker_manager.h"],
730730
deps = [
731-
":stats_metric",
732731
":gcs_kv_manager",
733-
":gcs_table_storage",
734-
":gcs_usage_stats_client",
735732
":gcs_pub_sub_lib",
736733
":gcs_service_rpc",
734+
":gcs_table_storage",
735+
":gcs_usage_stats_client",
736+
":stats_metric",
737737
],
738738
)
739739

@@ -767,23 +767,23 @@ ray_cc_library(
767767
"src/ray/gcs/gcs_server/gcs_task_manager.h",
768768
],
769769
deps = [
770-
":gcs_runtime_env_handler",
771-
":gcs_redis_failure_detector",
772-
":gcs_state_util",
773-
":gcs_pubsub_handler",
774-
":gcs_store_client_kv",
775-
":gcs_usage_stats_client",
776-
":gcs_worker_manager",
777-
":gcs_function_manager",
778-
":gcs_kv_manager",
779-
":gcs_init_data",
780-
":gcs_table_storage",
781770
":autoscaler_rpc",
782771
":gcs",
772+
":gcs_function_manager",
773+
":gcs_init_data",
774+
":gcs_kv_manager",
783775
":gcs_pub_sub_lib",
776+
":gcs_pubsub_handler",
777+
":gcs_redis_failure_detector",
778+
":gcs_runtime_env_handler",
784779
":gcs_service_cc_grpc",
785780
":gcs_service_rpc",
781+
":gcs_state_util",
782+
":gcs_store_client_kv",
783+
":gcs_table_storage",
786784
":gcs_table_storage_lib",
785+
":gcs_usage_stats_client",
786+
":gcs_worker_manager",
787787
":node_manager_rpc",
788788
":observable_store_client",
789789
":pubsub_lib",
@@ -846,8 +846,8 @@ ray_cc_library(
846846
name = "pubsub_rpc",
847847
# TODO(core): Revisit this dependency after grpc_common_lib is broken down into smaller targets.
848848
deps = [
849-
":pubsub_cc_grpc",
850849
":grpc_common_lib", # This is a large dependency, should be refined in the future.
850+
":pubsub_cc_grpc",
851851
],
852852
)
853853

@@ -1125,7 +1125,8 @@ ray_cc_library(
11251125
"src/ray/core_worker/reference_count.cc",
11261126
"src/ray/core_worker/store_provider/memory_store/memory_store.cc",
11271127
"src/ray/core_worker/store_provider/plasma_store_provider.cc",
1128-
"src/ray/core_worker/task_event_buffer.cc", "src/ray/core_worker/task_manager.cc",
1128+
"src/ray/core_worker/task_event_buffer.cc",
1129+
"src/ray/core_worker/task_manager.cc",
11291130
"src/ray/core_worker/transport/actor_scheduling_queue.cc",
11301131
"src/ray/core_worker/transport/actor_task_submitter.cc",
11311132
"src/ray/core_worker/transport/concurrency_group_manager.cc",
@@ -2820,7 +2821,7 @@ ray_cc_library(
28202821
":gcs_callback",
28212822
":gcs_store_client",
28222823
"//src/ray/common:asio",
2823-
"//src/ray/util",
2824+
"//src/ray/util:concurrent_flat_map",
28242825
"@com_google_absl//absl/container:node_hash_map",
28252826
],
28262827
)
@@ -3053,7 +3054,8 @@ pyx_library(
30533054
# shared lib, see python/ray/__init__.py.
30543055
cc_kwargs = dict(
30553056
srcs = PYX_SRCS,
3056-
copts = COPTS + PYX_COPTS,
3057+
# cython code is auto-generated, which is out of our control.
3058+
copts = COPTS + PYX_COPTS + ["-Wno-shadow"],
30573059
# see https://github.com/tensorflow/tensorflow/blob/r2.1/tensorflow/lite/BUILD#L444
30583060
linkopts = select({
30593061
"@platforms//os:osx": [

‎ci/env/install-miniconda.sh

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ install_miniconda() {
8585
(
8686
set +x
8787
echo "Updating Anaconda Python ${python_version} to ${PYTHON}..."
88+
"${WORKSPACE_DIR}"/ci/suppress_output conda remove --force -y anaconda-anon-usage
8889
"${WORKSPACE_DIR}"/ci/suppress_output conda install -q -y python="${PYTHON}"
8990
)
9091
elif [ "${MINIMAL_INSTALL-}" = "1" ]; then # Reset environment
@@ -95,9 +96,9 @@ install_miniconda() {
9596
)
9697
fi
9798

98-
if [[ "${PYTHON-}" != "3.12" ]]; then
99+
if [[ "${PYTHON-}" != "3.12" && "${PYTHON-}" != "3.13" ]]; then
99100
# Install mpi4py as a test dependency for Python <3.12; currently mpi4py is not
100-
# available for Python 3.12
101+
# available for Python 3.12 or 3.13
101102
"${WORKSPACE_DIR}"/ci/suppress_output conda install -c anaconda mpi4py -y
102103
fi
103104

‎ci/pipeline/determine_tests_to_run.py

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -217,15 +217,14 @@ def _emit(line: str):
217217

218218
_emit("ml tune train data serve core_cpp cpp java python doc")
219219
_emit("linux_wheels macos_wheels dashboard tools release_tests")
220-
else:
221-
_emit("ml tune train rllib rllib_directly serve")
222-
_emit("cpp core_cpp java python doc linux_wheels macos_wheels docker")
223-
_emit("dashboard tools workflow data release_tests")
224220

225-
# Log the modified environment variables visible in console.
226-
output_string = " ".join(list(tags))
227-
for tag in tags:
228-
assert tag in _ALL_TAGS, f"Unknown tag {tag}"
221+
# Log the modified environment variables visible in console.
222+
output_string = " ".join(list(tags))
223+
for tag in tags:
224+
assert tag in _ALL_TAGS, f"Unknown tag {tag}"
229225

230-
print(output_string, file=sys.stderr) # Debug purpose
231-
print(output_string)
226+
print(output_string, file=sys.stderr) # Debug purpose
227+
print(output_string)
228+
else:
229+
print("Run all tags", file=sys.stderr)
230+
print("*")

‎ci/pipeline/test_conditional_testing.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@
6969
- linux_wheels macos_wheels dashboard release_tests accelerated_dag
7070
7171
.github/CODEOWNERS: lint
72+
README.rst: lint
7273
BUILD.bazel:
7374
- lint ml tune train data serve core_cpp cpp java
7475
- python doc linux_wheels macos_wheels dashboard tools

‎ci/pipeline/test_rules.txt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,5 +206,17 @@ src/
206206
;
207207

208208
.github/
209+
README.rst
210+
SECURITY.md
211+
CONTRIBUTING.rst
212+
LICENSE
213+
pylintrc
214+
semgrep.yml
215+
setup_hooks.sh
216+
.pre-commit-config.yaml
217+
.isort.cfg
218+
.prettierrc.toml
219+
build.sh
220+
build-docker.sh
209221
# pass
210222
;

‎cpp/src/ray/runtime/task/task_executor.cc

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -335,8 +335,8 @@ void TaskExecutor::Invoke(
335335
}
336336
} catch (std::exception &e) {
337337
auto result = PackError(e.what());
338-
auto data = std::make_shared<msgpack::sbuffer>(std::move(result));
339-
runtime->Put(std::move(data), task_spec.ReturnId(0));
338+
auto result_data = std::make_shared<msgpack::sbuffer>(std::move(result));
339+
runtime->Put(std::move(result_data), task_spec.ReturnId(0));
340340
}
341341
}
342342

‎doc/source/data/api/input_output.rst

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,16 @@ ClickHouse
233233

234234
read_clickhouse
235235

236+
Daft
237+
----
238+
239+
.. autosummary::
240+
:nosignatures:
241+
:toctree: doc/
242+
243+
from_daft
244+
Dataset.to_daft
245+
236246
Dask
237247
----
238248

‎doc/source/data/comparisons.rst

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,11 @@ How does Ray Data compare to other solutions for offline inference?
1919

2020
For a more detailed performance comparison between Ray Data and Sagemaker Batch Transform, see `Offline Batch Inference: Comparing Ray, Apache Spark, and SageMaker <https://www.anyscale.com/blog/offline-batch-inference-comparing-ray-apache-spark-and-sagemaker>`_.
2121

22-
.. dropdown:: Distributed Data Processing Frameworks: Apache Spark
22+
.. dropdown:: Distributed Data Processing Frameworks: Apache Spark and Daft
2323

24-
Ray Data handles many of the same batch processing workloads as `Apache Spark <https://spark.apache.org/>`_, but with a streaming paradigm that is better suited for GPU workloads for deep learning inference.
24+
Ray Data handles many of the same batch processing workloads as `Apache Spark <https://spark.apache.org/>`_ and `Daft <https://www.getdaft.io>`_, but with a streaming paradigm that is better suited for GPU workloads for deep learning inference.
2525

26-
Ray Data doesn't have a SQL interface and isn't meant as a replacement for generic ETL pipelines like Spark.
26+
However, Ray Data doesn't have a SQL interface and isn't meant as a replacement for generic ETL pipelines like Spark and Daft.
2727

2828
For a more detailed performance comparison between Ray Data and Apache Spark, see `Offline Batch Inference: Comparing Ray, Apache Spark, and SageMaker <https://www.anyscale.com/blog/offline-batch-inference-comparing-ray-apache-spark-and-sagemaker>`_.
2929

0 commit comments

Comments
 (0)