Merge branch 'master' into fix-broken-links

.bazelrc

@@ -155,7 +155,6 @@ build:ci --curses=no
 build:ci --keep_going
 build:ci --progress_report_interval=100
 build:ci --show_progress_rate_limit=15
-build:ci --show_task_finish
 build:ci --ui_actions_shown=1024
 build:ci --show_timestamps
 build:ci-travis --disk_cache=~/ray-bazel-cache
@@ -168,6 +167,7 @@ test:ci --flaky_test_attempts=3
 test:ci --nocache_test_results
 test:ci --spawn_strategy=local
 test:ci --test_output=errors
+test:ci --experimental_ui_max_stdouterr_bytes=-1
 test:ci --test_verbose_timeout_warnings
 test:ci-debug -c dbg
 test:ci-debug --copt="-g"

.bazelversion

@@ -0,0 +1 @@
+5.4.1

.buildkite/build.rayci.yml

@@ -40,6 +40,7 @@ steps:
     job_env: manylinux
 
   - label: ":tapioca: build: doc"
+    key: doc_build
     instance_type: medium
     commands:
       - FAST=True make -C doc/ html

.buildkite/core.rayci.yml

@@ -53,7 +53,7 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --only-tags use_all_core --skip-ray-installation
@@ -66,7 +66,7 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --test-env=TEST_EXTERNAL_REDIS=1
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core
+        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,container,manual,use_all_core,multi_gpu
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/dag/... python/ray/autoscaler/v2/... core
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}"
         --only-tags use_all_core --skip-ray-installation
@@ -142,10 +142,10 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker --
         python/ray/util/dask/... python/ray/tests/modin/... core
-        --build-name data15build
+        --build-name data16build
         --parallelism-per-worker 2
     depends_on:
-      - data15build
+      - data16build
       - forge
 
   - label: ":ray: core: dashboard tests"

.buildkite/data.rayci.yml

@@ -7,8 +7,8 @@ steps:
   - name: data6build
     wanda: ci/docker/data6.build.wanda.yaml
 
-  - name: data15build
-    wanda: ci/docker/data15.build.wanda.yaml
+  - name: data16build
+    wanda: ci/docker/data16.build.wanda.yaml
 
   - name: databuild-multipy
     label: "wanda: databuild-py{{matrix}}"
@@ -42,7 +42,7 @@ steps:
         --except-tags data_integration,doctest
     depends_on: data6build
 
-  - label: ":database: data: arrow 15 tests"
+  - label: ":database: data: arrow 16 tests"
     tags: 
       - python
       - data
@@ -52,15 +52,15 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //python/ray/air/... data
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" 
         --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --build-name data15build
+        --build-name data16build
         --except-tags data_integration,doctest
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: arrow nightly tests"
+    if: pipeline.id == "0189e759-8c96-4302-b6b5-b4274406bf89" || pipeline.id == "018f4f1e-1b73-4906-9802-92422e3badaa"
     tags: 
       - python
       - data
-      - skip-on-premerge
     instance_type: medium
     parallelism: 2
     commands:
@@ -91,17 +91,17 @@ steps:
     commands:
       # doc tests
       - bazel run //ci/ray_ci:test_in_docker -- python/ray/... //doc/... data 
-        --build-name data15build
+        --build-name data16build
         --except-tags gpu
         --only-tags doctest
         --parallelism-per-worker 2
       # doc examples
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... data 
-        --build-name data15build
+        --build-name data16build
         --except-tags gpu,post_wheel_build,doctest
         --parallelism-per-worker 2
         --skip-ray-installation
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: doc gpu tests"
     tags: 
@@ -144,9 +144,9 @@ steps:
     instance_type: small
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- python/ray/dashboard/... data
-        --build-name data15build
+        --build-name data16build
         --parallelism-per-worker 3
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: flaky tests"
     tags: 
@@ -158,9 +158,9 @@ steps:
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests 
         --parallelism-per-worker 3
-        --build-name data15build
+        --build-name data16build
         --except-tags gpu_only,gpu
-    depends_on: data15build
+    depends_on: data16build
 
   - label: ":database: data: flaky gpu tests"
     tags: 

.buildkite/lint.rayci.yml

@@ -1,92 +1,37 @@
 group: lint
 steps:
-  - label: ":lint-roller: lint: clang format"
-    depends_on:
-      - forge
-    commands:
-      - pip install -c python/requirements_compiled.txt clang-format
-      - ./ci/lint/check-git-clang-format-output.sh
-
-  - label: ":lint-roller: lint: code format"
-    depends_on:
-      - forge
-    commands:
-      - pip install -c python/requirements_compiled.txt -r python/requirements/lint-requirements.txt
-      - FORMAT_SH_PRINT_DIFF=1 ./ci/lint/format.sh --all-scripts
-
-  - label: ":lint-roller: lint: untested code snippet"
-    depends_on:
-      - forge
-    commands:
-      - pip install -c python/requirements_compiled.txt semgrep
-      - semgrep ci --config semgrep.yml
-
-  - label: ":lint-roller: lint: banned words"
-    depends_on:
-      - forge
-    commands:
-      - ./ci/lint/check-banned-words.sh
-
-  - label: ":lint-roller: lint: doc readme"
-    depends_on:
-      - forge
-    commands:
-      - pip install -c python/requirements_compiled.txt docutils
-      - cd python && python setup.py check --restructuredtext --strict --metadata
-
-  - label: ":lint-roller: lint: dashboard format"
-    depends_on:
-      - forge
-    commands:
-      - ./ci/lint/check-dashboard-format.sh
-
-  - label: ":lint-roller: lint: copyright format"
-    depends_on:
-      - forge
-    commands:
-      - ./ci/lint/copyright-format.sh -c
-
-  - label: ":lint-roller: lint: bazel team"
-    depends_on:
-      - forge
-    commands:
-      - bazel query 'kind("cc_test", //...)' --output=xml | python ./ci/lint/check-bazel-team-owner.py
-      - bazel query 'kind("py_test", //...)' --output=xml | python ./ci/lint/check-bazel-team-owner.py
-
-  - label: ":lint-roller: lint: bazel buildifier"
-    depends_on:
-      - forge
-    commands:
-      - ./ci/lint/check-bazel-buildifier.sh
-
-  - label: ":lint-roller: lint: pytest format"
-    depends_on:
-      - forge
-    commands:
-      - pip install -c python/requirements_compiled.txt yq
-      - ./ci/lint/check-pytest-format.sh
-
-  - label: ":lint-roller: lint: test coverage"
-    depends_on:
-      - forge
-    commands:
-      - python ci/pipeline/check-test-run.py
-
-  - label: ":lint-roller: lint: api annotations"
-    depends_on:
-      - forge
+  - label: ":lint-roller: lint: {{matrix}}"
+    key: lint-small
+    depends_on:
+      - forge
+    commands:
+      - ./ci/lint/lint.sh {{matrix}}
+    matrix:
+      - clang_format
+      - code_format
+      - untested_code_snippet
+      - banned_words
+      - doc_readme
+      - dashboard_format
+      - copyright_format
+      - bazel_team
+      - bazel_buildifier
+      - pytest_format
+      - test_coverage
+      - documentation_style
+
+  - label: ":lint-roller: lint: {{matrix}}"
+    key: lint-medium
     instance_type: medium
-    commands:
-      - RAY_DISABLE_EXTRA_CPP=1 pip install -e python/[all]
-      - ./ci/lint/check_api_annotations.py
-
-  - label: ":lint-roller: lint: documentation style"
     depends_on:
       - forge
     commands:
-      - ./ci/lint/check-documentation-style.sh
+      - ./ci/lint/lint.sh {{matrix}}
+    matrix:
+      - api_annotations
+      - api_discrepancy
 
-  - label: ":lint-roller: lint: all linkcheck"
+  - label: ":lint-roller: lint: linkcheck"
     instance_type: medium
     commands:
       - make -C doc/ linkcheck_all

.buildkite/macos/macos.rayci.yml

@@ -1,9 +1,9 @@
 group: macos tests
 sort_key: "~macos"
 steps:
-  # block on premerge
+  # block on premerge and microcheck
   - block: "run macos tests"
-    if: build.env("BUILDKITE_PIPELINE_ID") == "0189942e-0876-4b8f-80a4-617f988ec59b"
+    if: build.env("BUILDKITE_PIPELINE_ID") == "0189942e-0876-4b8f-80a4-617f988ec59b" || build.env("BUILDKITE_PIPELINE_ID") == "018f4f1e-1b73-4906-9802-92422e3badaa"
 
   # build
   - label: ":tapioca: build: :mac: wheels and jars (x86_64)"

.buildkite/others.rayci.yml

@@ -8,6 +8,7 @@ steps:
     wanda: ci/docker/doctest.build.wanda.yaml
 
   - label: ":tapioca: build: pip-compile dependencies"
+    key: pip_compile_dependencies
     instance_type: small
     commands:
       # uncomment the following line to update the pinned versions of pip dependencies
@@ -67,7 +68,10 @@ steps:
     if: build.branch == "master"
     instance_type: small
     commands:
-      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --production
+      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix linux:__ --production
+      # we use master branch to determine microcheck tests for darwin and windows, because darwin and windows tests do not run on premerge branches
+      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix darwin:__ --consider-master-branch --production
+      - bazel run //ci/ray_ci/automation:determine_microcheck_tests -- {{matrix}} 100 --test-prefix windows:__ --consider-master-branch --production
     matrix:
       - "core"
       - "serverless"

.buildkite/release-automation/pre_release.rayci.yml

@@ -16,6 +16,14 @@ steps:
     allow_dependency_failure: true
     commands:
       - bash .buildkite/release-automation/check-commit-hash.sh
+
+  - label: "Build update version binary"
+    key: build-update-version-zip
+    instance_type: default
+    job_env: forge
+    commands:
+      - bazel build --build_python_zip --incompatible_use_python_toolchains=false --python_path=python //ci/ray_ci/automation:update_version
+      - cp bazel-bin/ci/ray_ci/automation/update_version.zip /artifact-mount/
 
   - label: "Trigger Postmerge test"
     if: build.env("RAYCI_WEEKLY_RELEASE_NIGHTLY") != "1"

.buildkite/release-automation/wheels.rayci.yml

@@ -7,6 +7,8 @@ steps:
 
   - label: "Upload wheels from S3 to TestPyPI"
     key: upload-wheels-testpypi
+    depends_on:
+      - block-upload-wheels-testpypi
     job_env: forge
     instance_type: small_branch
     commands:
@@ -23,6 +25,8 @@ steps:
   - label: "Linux x86_64 Python {{matrix}}"
     key: validate-linux-x86_64-wheels
     job_env: forge
+    depends_on:
+      - block-validate-linux-x86_64-wheels
     commands:
       - export PYTHON_VERSION={{matrix}}
       - export RAY_VERSION="$RAY_VERSION"
@@ -41,6 +45,8 @@ steps:
     key: validate-linux-arm64-wheels
     instance_type: medium-arm64
     job_env: forge-arm64
+    depends_on:
+      - block-validate-linux-arm64-wheels
     commands:
       - export PYTHON_VERSION={{matrix}}
       - export RAY_VERSION="$RAY_VERSION"
@@ -52,26 +58,34 @@ steps:
 
   - block: "Download & validate Ray wheels from TestPyPI Mac"
     key: block-validate-macos-wheels
+    depends_on: []
 
   - label: "MacOS x86_64"
     key: validate-macos-x86_64-wheels
+    depends_on:
+      - block-validate-macos-wheels
     job_env: MACOS
     instance_type: macos
     commands:
       - ./.buildkite/release-automation/verify-macos-wheels.sh x86_64
 
   - label: "MacOS arm64"
     key: validate-macos-arm64-wheels
+    depends_on:
+      - block-validate-macos-wheels
     job_env: MACOS
     instance_type: macos-arm64
     commands:
       - ./.buildkite/release-automation/verify-macos-wheels.sh arm64
 
   - block: "Upload wheels to PyPI"
     key: block-upload-wheels-pypi
+    depends_on: []
 
   - label: "Upload wheels to PyPI"
     key: upload-wheels-pypi
+    depends_on:
+      - block-upload-wheels-pypi
     job_env: forge
     instance_type: small_branch
     commands: