Skip to content

Commit 24f678c

Browse files
martinbomiomartinbomio
authored
Merge branch 'master' into streaming-train-test-split
2 parents e8798e0 + b217645 commit 24f678c

60 files changed

Lines changed: 1072 additions & 1039 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.buildkite/build.rayci.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,16 @@ steps:
2121
tags: release_wheels
2222
depends_on: manylinux
2323

24+
- name: ray-java-build
25+
label: "wanda: java build (x86_64)"
26+
wanda: ci/docker/ray-java.wanda.yaml
27+
tags:
28+
- release_wheels
29+
- oss
30+
env:
31+
ARCH_SUFFIX: ""
32+
depends_on: manylinux
33+
2434
- label: ":tapioca: build: wheel {{matrix}} (x86_64)"
2535
key: linux_wheels
2636
tags:

‎.buildkite/core.rayci.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ steps:
6060
- bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/_common/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
6161
--install-mask all-ray-libraries
6262
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
63-
--except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask
63+
--except-tags custom_setup
6464
--install-mask all-ray-libraries
6565

6666
- label: ":ray: core: python {{matrix.python}} tests ({{matrix.worker_id}})"
@@ -74,7 +74,7 @@ steps:
7474
--install-mask all-ray-libraries
7575
--workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
7676
--python-version {{matrix.python}}
77-
--except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask
77+
--except-tags custom_setup
7878
depends_on: corebuild-multipy
7979
matrix:
8080
setup:
@@ -103,7 +103,7 @@ steps:
103103
--install-mask all-ray-libraries
104104
--workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
105105
--test-env=TEST_EXTERNAL_REDIS=1
106-
--except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask
106+
--except-tags custom_setup
107107

108108
- label: ":ray: core: memory pressure tests"
109109
tags:

‎.buildkite/linux_aarch64.rayci.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,17 @@ steps:
1313
wanda: ci/docker/manylinux.aarch64.wanda.yaml
1414
instance_type: builder-arm64
1515

16+
- name: ray-java-build-aarch64
17+
label: "wanda: java build (aarch64)"
18+
wanda: ci/docker/ray-java.wanda.yaml
19+
tags:
20+
- release_wheels
21+
- oss
22+
env:
23+
ARCH_SUFFIX: "-aarch64"
24+
instance_type: builder-arm64
25+
depends_on: manylinux-aarch64
26+
1627
- name: raycpubase-aarch64
1728
label: "wanda: ray.py{{matrix}}.cpu.base (aarch64)"
1829
tags:

‎.buildkite/release/build.rayci.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@ steps:
55
wanda: docker/base-extra-testdeps/cpu.wanda.yaml
66
matrix:
77
- "3.9"
8+
- "3.10"
89
- "3.11"
910
- "3.12"
1011
env:
1112
PYTHON_VERSION: "{{matrix}}"
1213
IMAGE_TYPE: "ray"
13-
REQUIREMENTS_FILE: "requirements_byod_{{matrix}}.txt"
14+
REQUIREMENTS_FILE: "ray_base_extra_testdeps_py{{matrix}}.lock"
1415
depends_on:
1516
- raycpubaseextra
1617

@@ -21,6 +22,7 @@ steps:
2122
setup:
2223
python:
2324
- "3.9"
25+
- "3.10"
2426
- "3.11"
2527
- "3.12"
2628
cuda:
@@ -29,7 +31,7 @@ steps:
2931
PYTHON_VERSION: "{{matrix.python}}"
3032
CUDA_VERSION: "{{matrix.cuda}}"
3133
IMAGE_TYPE: "ray"
32-
REQUIREMENTS_FILE: "requirements_byod_{{matrix.python}}.txt"
34+
REQUIREMENTS_FILE: "ray_base_extra_testdeps_py{{matrix.python}}.lock"
3335
depends_on:
3436
- raycudabaseextra
3537

@@ -57,13 +59,14 @@ steps:
5759
setup:
5860
python:
5961
- "3.9"
62+
- "3.10"
6063
cuda:
6164
- "12.1.1-cudnn8"
6265
env:
6366
PYTHON_VERSION: "{{matrix.python}}"
6467
CUDA_VERSION: "{{matrix.cuda}}"
6568
IMAGE_TYPE: "ray-ml"
66-
REQUIREMENTS_FILE: "requirements_ml_byod_{{matrix.python}}.txt"
69+
REQUIREMENTS_FILE: "ray_ml_base_extra_testdeps_py{{matrix.python}}.lock"
6770
depends_on:
6871
- ray-mlcudabaseextra
6972

@@ -87,6 +90,7 @@ steps:
8790
# This list should be kept in sync with the list of supported Python in
8891
# release test suite. We don't have release tests for Python 3.10 yet.
8992
- "3.9"
93+
- "3.10"
9094
- "3.11"
9195
- "3.12"
9296
platform:
@@ -125,3 +129,4 @@ steps:
125129
# release test suite. We don't have ray-ml release tests for Python 3.10 and 3.11
126130
# yet.
127131
- "3.9"
132+
- "3.10"

‎.github/PULL_REQUEST_TEMPLATE.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
## Checks
1414

1515
- [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR.
16-
- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
16+
- [ ] I've run pre-commit jobs to lint the changes in this PR. ([pre-commit setup](https://docs.ray.io/en/latest/ray-contribute/getting-involved.html#lint-and-formatting))
1717
- [ ] I've included any doc changes needed for https://docs.ray.io/en/master/.
1818
- [ ] I've added any new APIs to the API Reference. For example, if I added a
1919
method in Tune, I've added it in `doc/source/tune/api/` under the

‎.rayciversion‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.19.0
1+
0.20.0

‎ci/docker/ray-java.Dockerfile‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# syntax=docker/dockerfile:1.3-labs
2+
ARG ARCH_SUFFIX
3+
FROM cr.ray.io/rayproject/manylinux$ARCH_SUFFIX AS builder
4+
5+
ARG BUILDKITE_BAZEL_CACHE_URL
6+
ARG BUILDKITE_CACHE_READONLY
7+
8+
WORKDIR /home/forge/ray
9+
10+
COPY . .
11+
12+
RUN <<EOF
13+
#!/bin/bash
14+
15+
set -euo pipefail
16+
17+
export RAY_BUILD_ENV="manylinux"
18+
19+
if [[ "${BUILDKITE_CACHE_READONLY:-}" == "true" ]]; then
20+
echo "build --remote_upload_local_results=false" >> "$HOME/.bazelrc"
21+
fi
22+
23+
bazelisk build --config=ci //java:ray_java_pkg
24+
25+
cp bazel-bin/java/ray_java_pkg.zip /home/forge/ray_java_pkg.zip
26+
27+
EOF
28+
29+
FROM scratch
30+
31+
COPY --from=builder /home/forge/ray_java_pkg.zip /

‎ci/docker/ray-java.wanda.yaml‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
name: ray-java-build$ARCH_SUFFIX
2+
froms: ["cr.ray.io/rayproject/manylinux$ARCH_SUFFIX"]
3+
dockerfile: ci/docker/ray-java.Dockerfile
4+
srcs:
5+
- .bazelversion
6+
- .bazelrc
7+
- WORKSPACE
8+
- BUILD.bazel
9+
- bazel/
10+
- thirdparty/
11+
- src/
12+
- java/
13+
- gen_ray_pkg.py
14+
- release/BUILD.bazel
15+
- release/requirements_buildkite.txt
16+
build_args:
17+
- ARCH_SUFFIX
18+
- BUILDKITE_BAZEL_CACHE_URL
19+
build_hint_args:
20+
- BUILDKITE_CACHE_READONLY

‎doc/BUILD.bazel‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,7 @@ py_test(
166166
tags = [
167167
"exclusive",
168168
"post_wheel_build",
169+
"custom_setup",
169170
"team:core",
170171
],
171172
)
@@ -178,6 +179,7 @@ py_test(
178179
tags = [
179180
"exclusive",
180181
"mem_pressure",
182+
"custom_setup",
181183
"team:core",
182184
],
183185
)
@@ -190,6 +192,7 @@ py_test(
190192
tags = [
191193
"exclusive",
192194
"multi_gpu",
195+
"custom_setup",
193196
"team:core",
194197
],
195198
)
@@ -202,6 +205,7 @@ py_test(
202205
tags = [
203206
"exclusive",
204207
"multi_gpu",
208+
"custom_setup",
205209
"team:core",
206210
],
207211
)
@@ -214,6 +218,7 @@ py_test(
214218
tags = [
215219
"exclusive",
216220
"multi_gpu",
221+
"custom_setup",
217222
"team:core",
218223
],
219224
)
@@ -237,6 +242,7 @@ py_test(
237242
tags = [
238243
"exclusive",
239244
"multi_gpu",
245+
"custom_setup",
240246
"team:core",
241247
],
242248
)
@@ -249,6 +255,7 @@ py_test(
249255
tags = [
250256
"exclusive",
251257
"multi_gpu",
258+
"custom_setup",
252259
"team:core",
253260
],
254261
)
@@ -447,6 +454,7 @@ py_test_run_all_subdirectory(
447454
extra_srcs = [],
448455
tags = [
449456
"dask",
457+
"custom_setup",
450458
"exclusive",
451459
"team:data",
452460
],

‎docker/base-extra-testdeps/cpu.wanda.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ name: "$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra-testdeps"
22
froms: ["cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra"]
33
dockerfile: release/ray_release/byod/byod.Dockerfile
44
srcs:
5-
- release/ray_release/byod/requirements_byod_$PYTHON_VERSION.txt
5+
- release/ray_release/byod/$REQUIREMENTS_FILE
66
build_args:
77
- BASE_IMAGE=cr.ray.io/rayproject/$IMAGE_TYPE-py$PYTHON_VERSION-cpu-base-extra
88
- PIP_REQUIREMENTS=release/ray_release/byod/$REQUIREMENTS_FILE

0 commit comments

Comments
 (0)