Merge branch 'master' into sync_upstream

.bazelrc

@@ -1,5 +1,13 @@
 # Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd
 build --enable_platform_specific_config
+
+# Provides users an option to turn on strict action env.
+# TODO(aslonnie): make this default; fix the python tests..
+build:strict --incompatible_strict_action_env
+
+# To distinguish different incompatible environments.
+build --action_env=RAY_BUILD_ENV
+
 ###############################################################################
 # On       Windows, provide: BAZEL_SH, and BAZEL_LLVM (if using clang-cl)
 # On all platforms, provide: PYTHON3_BIN_PATH=python
@@ -150,30 +158,32 @@ test:asan-buildkite --test_env=LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libasan.so.
 # CI configuration:
 aquery:ci --color=no
 aquery:ci --noshow_progress
+
+test:ci-base --test_output=errors
+test:ci-base --test_verbose_timeout_warnings
+test:ci-base --flaky_test_attempts=3
+
 build:ci --color=yes
 build:ci --curses=no
 build:ci --keep_going
 build:ci --progress_report_interval=100
 build:ci --show_progress_rate_limit=15
 build:ci --ui_actions_shown=1024
 build:ci --show_timestamps
-test:ci --flaky_test_attempts=3
 # Disable test result caching because py_test under Bazel can import from outside of sandbox, but Bazel only looks at
 # declared dependencies to determine if a result should be cached. More details at:
 # https://github.com/bazelbuild/bazel/issues/7091, https://github.com/bazelbuild/rules_python/issues/382
+test:ci --config=ci-base
 test:ci --nocache_test_results
 test:ci --spawn_strategy=local
-test:ci --test_output=errors
 test:ci --experimental_ui_max_stdouterr_bytes=-1
-test:ci --test_verbose_timeout_warnings
+
+test:ci-debug --config=ci-base
 test:ci-debug -c dbg
 test:ci-debug --copt="-g"
-test:ci-debug --flaky_test_attempts=3
 # Disable test result caching for the same reason above.
 test:ci-debug --nocache_test_results
 test:ci-debug --spawn_strategy=local
-test:ci-debug --test_output=errors
-test:ci-debug --test_verbose_timeout_warnings
 
 aquery:get-toolchain --include_commandline=false
 aquery:get-toolchain --noimplicit_deps

.buildkite/README.md

@@ -2,7 +2,7 @@
 
 This directory contains buildkite pipelines used to start CI tests.
 
-Each step contains a buildkite step that is parsed and executed according to the 
+Each step contains a buildkite step that is parsed and executed according to the
 [Buildkite pipeline specification](https://buildkite.com/docs/pipelines).
 
 ## Conditions

.buildkite/_forge.rayci.yml

@@ -25,6 +25,7 @@ steps:
           - "12.1.1-cudnn8"
           - "12.3.2-cudnn9"
           - "12.4.1-cudnn"
+          - "12.5.1-cudnn"
     env:
       PYTHON_VERSION: "{{matrix.python}}"
       CUDA_VERSION: "{{matrix.cuda}}"
@@ -47,6 +48,23 @@ steps:
     env:
       PYTHON_VERSION: "{{matrix}}"
 
+  - name: ray-llmbase
+    label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base"
+    tags:
+      - python_dependencies
+      - docker
+    wanda: ci/docker/ray-llm.base.wanda.yaml
+    depends_on: raycudabase
+    matrix:
+      setup:
+        python:
+          - "3.11"
+        cuda:
+          - "12.4.1-cudnn"
+    env:
+      PYTHON_VERSION: "{{matrix.python}}"
+      CUDA_VERSION: "{{matrix.cuda}}"
+
   - name: ray-mlcudabase
     label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base"
     tags:

.buildkite/base.rayci.yml

@@ -12,7 +12,7 @@ steps:
       - "3.12"
     env:
       PYTHON: "{{matrix}}"
-  
+
   - name: oss-ci-base_build
     wanda: ci/docker/base.build.py39.wanda.yaml
     depends_on: oss-ci-base_test
@@ -31,7 +31,7 @@ steps:
   - name: oss-ci-base_test-aarch64
     wanda: ci/docker/base.test.aarch64.wanda.yaml
     instance_type: builder-arm64
-  
+
   - name: oss-ci-base_build-aarch64
     wanda: ci/docker/base.build.aarch64.wanda.yaml
     depends_on: oss-ci-base_test-aarch64
@@ -45,6 +45,7 @@ steps:
     label: "wanda: oss-ci-base_ml-py{{matrix}}"
     wanda: ci/docker/base.ml.wanda.yaml
     matrix:
+      - "3.11"
       - "3.12"
     env:
       PYTHON: "{{matrix}}"
@@ -57,6 +58,7 @@ steps:
     label: "wanda: oss-ci-base_gpu-py{{matrix}}"
     wanda: ci/docker/base.gpu.wanda.yaml
     matrix:
+      - "3.11"
       - "3.12"
     env:
       PYTHON: "{{matrix}}"

.buildkite/bisect/bisect.rayci.yml

@@ -1,12 +1,12 @@
 group: bisect
-depends_on: 
+depends_on:
   - forge
 steps:
   - name: macos test
     if: build.env("RAYCI_TEST_TYPE") != null && build.env("RAYCI_TEST_TYPE") == "macos_test"
     commands:
       - if [[ "$(buildkite-agent meta-data get test-type)" != "macos_test" ]]; then exit 0; fi
-      - RAYCI_BISECT_RUN=1 ./ci/ray_ci/macos/macos_ci.sh bisect "$(buildkite-agent meta-data get test-name)" 
+      - RAYCI_BISECT_RUN=1 ./ci/ray_ci/macos/macos_ci.sh bisect "$(buildkite-agent meta-data get test-name)"
           "$(buildkite-agent meta-data get passing-commit)" "$(buildkite-agent meta-data get failing-commit)"
     mount_buildkite_agent: true
     job_env: MACOS
@@ -17,7 +17,7 @@ steps:
     if: build.env("RAYCI_TEST_TYPE") != null && (build.env("RAYCI_TEST_TYPE") == "linux_test" || build.env("RAYCI_TEST_TYPE") == "windows_test")
     commands:
       - if [[ "$(buildkite-agent meta-data get test-type)" != "linux_test" && "$(buildkite-agent meta-data get test-type)" != "windows_test" ]]; then exit 0; fi
-      - bazel run //ci/ray_ci/bisect:bisect_test "$(buildkite-agent meta-data get test-name)" 
+      - bazel run //ci/ray_ci/bisect:bisect_test "$(buildkite-agent meta-data get test-name)"
           "$(buildkite-agent meta-data get passing-commit)" "$(buildkite-agent meta-data get failing-commit)"
     mount_buildkite_agent: true
     priority: 10

.buildkite/build.rayci.yml

@@ -13,6 +13,7 @@ steps:
       - "3.10"
       - "3.11"
       - "3.12"
+      - "3.13"
     depends_on:
       - manylinux
       - forge
@@ -59,7 +60,8 @@ steps:
       - bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
         --platform cu11.7.1-cudnn8 --platform cu11.8.0-cudnn8
         --platform cu12.1.1-cudnn8 --platform cu12.3.2-cudnn9
-        --platform cu12.4.1-cudnn --platform cpu
+        --platform cu12.4.1-cudnn --platform cu12.5.1-cudnn
+        --platform cpu
         --image-type ray --upload
     depends_on:
       - manylinux
@@ -72,6 +74,22 @@ steps:
       - "3.11"
       - "3.12"
 
+  - label: ":tapioca: build: ray-llm py{{matrix}} docker (x86_64)"
+    tags:
+      - python_dependencies
+      - docker
+      - oss
+    instance_type: medium
+    commands:
+      - bazel run //ci/ray_ci:build_in_docker -- docker --python-version {{matrix}}
+        --platform cu12.4.1-cudnn --image-type ray-llm --upload
+    depends_on:
+      - manylinux
+      - forge
+      - ray-llmbase
+    matrix:
+      - "3.11"
+
   - label: ":tapioca: build: ray-ml py{{matrix}} docker (x86_64)"
     tags:
       - python_dependencies

.buildkite/cicd.rayci.yml

@@ -3,9 +3,10 @@ steps:
   - label: ":coral: reef: ci+release tooling tests"
     key: reef-tests
     commands:
-      - bazel run //ci/ray_ci:test_in_docker -- //ci/ray_ci/... //release/... ci
+      - bazel run //ci/ray_ci:test_in_docker --
+          //ci/ray_ci/... //release/... //ci/pipeline/... ci
           --only-tags=release_unit,ci_unit
-          --parallelism-per-worker 2
+          --cache-test-results --parallelism-per-worker 2
           --build-name oss-ci-base_test
           --build-type skip
     instance_type: small