From a6bcc44b53af622732d2c6bc5534010215ef05d8 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 10:44:36 -0400 Subject: [PATCH 01/85] Prune workflows based on changed files Only run tests based on things that have actually changed. For example, if only Python files have changed, we don't need to run the C++ tests. Contributes to https://github.com/rapidsai/build-planning/issues/94 --- .github/workflows/pr.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 74bdc666c68..f657fc8ecef 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -10,6 +10,20 @@ concurrency: cancel-in-progress: true jobs: + changed-files: + runs-on: ubuntu-latest + name: "Check changed files" + steps: + - uses: action/checkout@v4 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files_yaml: | + cpp: + - cpp/** + python: + - python/** pr-builder: needs: - checks @@ -58,6 +72,7 @@ jobs: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' with: build_type: pull-request conda-python-build: @@ -70,6 +85,7 @@ jobs: needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -78,6 +94,7 @@ jobs: needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: build_type: pull-request script: "ci/test_python_other.sh" @@ -139,6 +156,7 @@ jobs: needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: build_type: pull-request script: ci/test_wheel_cudf.sh @@ -155,6 +173,7 @@ jobs: needs: wheel-build-cudf-polars secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -175,6 +194,7 @@ jobs: needs: wheel-build-dask-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) From 68891bf80acb7da4f91d4194837b32e372c07c94 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 10:49:10 -0400 Subject: [PATCH 02/85] Remove checkout step --- .github/workflows/pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f657fc8ecef..812b30fc342 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,7 +14,6 @@ jobs: runs-on: ubuntu-latest name: "Check changed files" steps: - - uses: action/checkout@v4 - name: Get changed files id: changed-files uses: tj-actions/changed-files@v45 From 4c7b312cf9f1cde59f9418c57c32e37f28878a9c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 10:50:38 -0400 Subject: [PATCH 03/85] Workflow syntax --- .github/workflows/pr.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 812b30fc342..a42259e24f5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -71,7 +71,7 @@ jobs: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' + if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' with: build_type: pull-request conda-python-build: @@ -84,7 +84,7 @@ jobs: needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -93,7 +93,7 @@ jobs: needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: build_type: pull-request script: "ci/test_python_other.sh" @@ -155,7 +155,7 @@ jobs: needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: build_type: pull-request script: ci/test_wheel_cudf.sh @@ -172,7 +172,7 @@ jobs: needs: wheel-build-cudf-polars secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -193,7 +193,7 @@ jobs: needs: wheel-build-dask-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: jobs.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) From d569bcfa5a88cc89e12eb5fc89ac853426f4030b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 11:44:55 -0400 Subject: [PATCH 04/85] Refactor --- .github/workflows/pr.yaml | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a42259e24f5..88ff1968b5b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -13,12 +13,17 @@ jobs: changed-files: runs-on: ubuntu-latest name: "Check changed files" + outputs: + test_cpp: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' }} + test_python: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' || steps.changed-files.outputs.python_any_changed == 'true' }} steps: - name: Get changed files id: changed-files uses: tj-actions/changed-files@v45 with: files_yaml: | + ci: + - ci/** cpp: - cpp/** python: @@ -68,10 +73,10 @@ jobs: build_type: pull-request enable_check_symbols: true conda-cpp-tests: - needs: conda-cpp-build + needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' + if: needs.changed-files.outputs.test_cpp with: build_type: pull-request conda-python-build: @@ -81,19 +86,19 @@ jobs: with: build_type: pull-request conda-python-cudf-tests: - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: needs.changed-files.outputs.test_python with: build_type: pull-request script: "ci/test_python_cudf.sh" conda-python-other-tests: # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: needs.changed-files.outputs.test_python with: build_type: pull-request script: "ci/test_python_other.sh" @@ -152,10 +157,10 @@ jobs: build_type: pull-request script: "ci/build_wheel_cudf.sh" wheel-tests-cudf: - needs: wheel-build-cudf + needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: needs.changed-files.outputs.test_python with: build_type: pull-request script: ci/test_wheel_cudf.sh @@ -169,10 +174,10 @@ jobs: build_type: pull-request script: "ci/build_wheel_cudf_polars.sh" wheel-tests-cudf-polars: - needs: wheel-build-cudf-polars + needs: [wheel-build-cudf-polars, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: needs.changed-files.outputs.test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -190,10 +195,10 @@ jobs: build_type: pull-request script: "ci/build_wheel_dask_cudf.sh" wheel-tests-dask-cudf: - needs: wheel-build-dask-cudf + needs: [wheel-build-dask-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: job.changed-files.steps.changed-files.outputs.cpp_any_changed == 'true' || jobs.changed-files.steps.changed-files.outputs.python_any_changed == 'true' + if: needs.changed-files.outputs.test_python with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) From bdbd6aa1c78b42365a7c303ab55130e77e1d086d Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 11:47:10 -0400 Subject: [PATCH 05/85] Checkout --- .github/workflows/pr.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 88ff1968b5b..2cc841b8df3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,11 @@ jobs: test_cpp: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' }} test_python: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' || steps.changed-files.outputs.python_any_changed == 'true' }} steps: + - name: Checkout code repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + persist-credentials: false - name: Get changed files id: changed-files uses: tj-actions/changed-files@v45 From a31341438de7fda18c063a9cd47f4196c68c522a Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 11:48:57 -0400 Subject: [PATCH 06/85] pr-builder dependency --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2cc841b8df3..554b3e28a29 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -57,6 +57,7 @@ jobs: - unit-tests-cudf-pandas - pandas-tests - pandas-tests-diff + - changed-files secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: From 9c0238ddad68182f9683cbb6489238d0d60e3e79 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 11:55:32 -0400 Subject: [PATCH 07/85] Add rapids_config.cmake and build.sh --- .github/workflows/pr.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 554b3e28a29..2db83614d57 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,8 +14,17 @@ jobs: runs-on: ubuntu-latest name: "Check changed files" outputs: - test_cpp: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' }} - test_python: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' || steps.changed-files.outputs.python_any_changed == 'true' }} + test_cpp: | + ${{ + steps.changed-files.outputs.ci_any_changed == 'true' || + steps.changed-files.outputs.cpp_any_changed == 'true' + }} + test_python: | + ${{ + steps.changed-files.outputs.ci_any_changed == 'true' || + steps.changed-files.outputs.cpp_any_changed == 'true' || + steps.changed-files.outputs.python_any_changed == 'true' + }} steps: - name: Checkout code repo uses: actions/checkout@v4 @@ -29,6 +38,8 @@ jobs: files_yaml: | ci: - ci/** + - rapids_config.cmake + - build.sh cpp: - cpp/** python: From 32486316d1803bd468aba29ecb2a4a01e79d680a Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:31:30 -0400 Subject: [PATCH 08/85] Print outputs --- .github/workflows/pr.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 2db83614d57..f8461092b2e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -44,6 +44,11 @@ jobs: - cpp/** python: - python/** + - name: Print outputs + run: | + echo "CI changed: ${{ steps.changd-files.outputs.ci_any_changed }}" + echo "C++ changed: ${{ steps.changd-files.outputs.cpp_any_changed }}" + echo "Python changed: ${{ steps.changd-files.outputs.python_any_changed }}" pr-builder: needs: - checks From 30f6482b968cc589dc06bd46030fda818ccda94e Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:32:10 -0400 Subject: [PATCH 09/85] Oops --- .github/workflows/pr.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f8461092b2e..79beed1c2ce 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -46,9 +46,9 @@ jobs: - python/** - name: Print outputs run: | - echo "CI changed: ${{ steps.changd-files.outputs.ci_any_changed }}" - echo "C++ changed: ${{ steps.changd-files.outputs.cpp_any_changed }}" - echo "Python changed: ${{ steps.changd-files.outputs.python_any_changed }}" + echo "CI changed: ${{ steps.changed-files.outputs.ci_any_changed }}" + echo "C++ changed: ${{ steps.changed-files.outputs.cpp_any_changed }}" + echo "Python changed: ${{ steps.changed-files.outputs.python_any_changed }}" pr-builder: needs: - checks From 9174e2def4f2cbc34060c3bf2c7b4e3cf3f93871 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:35:00 -0400 Subject: [PATCH 10/85] Add test job --- .github/workflows/pr.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 79beed1c2ce..139395073ab 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -49,6 +49,16 @@ jobs: echo "CI changed: ${{ steps.changed-files.outputs.ci_any_changed }}" echo "C++ changed: ${{ steps.changed-files.outputs.cpp_any_changed }}" echo "Python changed: ${{ steps.changed-files.outputs.python_any_changed }}" + test-job: + needs: + - changed-files + runs-on: ubuntu-latest + name: "Test job" + if: ${{ needs.changed-files.outputs.ci_any_changed }} + steps: + - name: Echo + run: | + echo "CI changed: ${{ needs.changed-files.outputs.ci_any_changed }}" pr-builder: needs: - checks From 97c02c91f8091226a5ec2a9c0b1af245ea14e4cc Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:35:30 -0400 Subject: [PATCH 11/85] dep --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 139395073ab..04d944083bb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -84,6 +84,7 @@ jobs: - pandas-tests - pandas-tests-diff - changed-files + - test-job secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: From 01b9764708b595ffd99c1073b5b15a42c9813033 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:38:09 -0400 Subject: [PATCH 12/85] test_cpp --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 04d944083bb..1cf0837b8d5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -54,11 +54,11 @@ jobs: - changed-files runs-on: ubuntu-latest name: "Test job" - if: ${{ needs.changed-files.outputs.ci_any_changed }} + if: ${{ needs.changed-files.outputs.test_cpp }} steps: - name: Echo run: | - echo "CI changed: ${{ needs.changed-files.outputs.ci_any_changed }}" + echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" pr-builder: needs: - checks From 9a39bfc3517f45fd5d3955384b871cb06d0fc95c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:39:20 -0400 Subject: [PATCH 13/85] ==true --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1cf0837b8d5..7758e518ec6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -54,7 +54,7 @@ jobs: - changed-files runs-on: ubuntu-latest name: "Test job" - if: ${{ needs.changed-files.outputs.test_cpp }} + if: ${{ needs.changed-files.outputs.test_cpp == 'true' }} steps: - name: Echo run: | From 15a1bb5fdbb5fec9fcdd1c3d6ab2945b91311ae9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:40:15 -0400 Subject: [PATCH 14/85] Dummy comment --- ci/build_cpp.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index e5fcef17a83..92d6b201d4f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,3 +20,5 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp + +# Dummy comment From 3716ab2079109a40c9893d9acae673691b073aaf Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:41:52 -0400 Subject: [PATCH 15/85] Debugging --- .github/workflows/pr.yaml | 2 +- ci/build_cpp.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7758e518ec6..aaa6d5a9a3c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -54,11 +54,11 @@ jobs: - changed-files runs-on: ubuntu-latest name: "Test job" - if: ${{ needs.changed-files.outputs.test_cpp == 'true' }} steps: - name: Echo run: | echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" + echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" pr-builder: needs: - checks diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 92d6b201d4f..e5fcef17a83 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,5 +20,3 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp - -# Dummy comment From 7553959747dddaa04a19c792beddb1e48bf04710 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:43:00 -0400 Subject: [PATCH 16/85] Re-run CI From 5ea7a8ce026f0081db174092c2100511a3c64f3b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:44:14 -0400 Subject: [PATCH 17/85] Test --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index aaa6d5a9a3c..a6709d8ff8c 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -54,6 +54,7 @@ jobs: - changed-files runs-on: ubuntu-latest name: "Test job" + if: ${{ needs.changed-files.outputs.test_cpp == 'true' }} steps: - name: Echo run: | From a7c2e7f42966bb2076f95214cf5f539d0f3305d9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:45:23 -0400 Subject: [PATCH 18/85] Dummy comment --- ci/build_cpp.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index e5fcef17a83..92d6b201d4f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,3 +20,5 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp + +# Dummy comment From 62e1ae9d9def593aba4f4d5e6ab279042099453d Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:46:25 -0400 Subject: [PATCH 19/85] Re-run CI From 61997ce9d3aac2e614c3d0fc9664335bf170b753 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:47:28 -0400 Subject: [PATCH 20/85] Remove dummy comment --- ci/build_cpp.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 92d6b201d4f..e5fcef17a83 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,5 +20,3 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp - -# Dummy comment From fa10f14436a8b30b3cbd41d9260ffdd612cff99c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:49:24 -0400 Subject: [PATCH 21/85] Debugging --- .github/workflows/pr.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index a6709d8ff8c..6a7a91211e5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -54,12 +54,11 @@ jobs: - changed-files runs-on: ubuntu-latest name: "Test job" - if: ${{ needs.changed-files.outputs.test_cpp == 'true' }} steps: - name: Echo run: | - echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" - echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" + echo "Test C++: ${{ needs.changed-files.outputs.test_cpp == 'true' }}" + echo "Test Python: ${{ needs.changed-files.outputs.test_python == 'true' }}" pr-builder: needs: - checks From 30c4ec0dd37fe806b972034cd9f2712c62cda985 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:51:41 -0400 Subject: [PATCH 22/85] Dummy comment --- ci/build_cpp.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index e5fcef17a83..92d6b201d4f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,3 +20,5 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp + +# Dummy comment From f187145edc0810fa48154daad7762fddc4f81998 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:52:50 -0400 Subject: [PATCH 23/85] More debugging --- .github/workflows/pr.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6a7a91211e5..809948b0c4e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -57,8 +57,10 @@ jobs: steps: - name: Echo run: | - echo "Test C++: ${{ needs.changed-files.outputs.test_cpp == 'true' }}" - echo "Test Python: ${{ needs.changed-files.outputs.test_python == 'true' }}" + echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" + echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" + echo "Test C++ == 'true': ${{ needs.changed-files.outputs.test_cpp == 'true' }}" + echo "Test Python == 'true': ${{ needs.changed-files.outputs.test_python == 'true' }}" pr-builder: needs: - checks From 5f97a6fcb6f1268e29a2ad681939d853a750c3cd Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:56:47 -0400 Subject: [PATCH 24/85] String style --- .github/workflows/pr.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 809948b0c4e..45a6fa931e5 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,13 +14,11 @@ jobs: runs-on: ubuntu-latest name: "Check changed files" outputs: - test_cpp: | - ${{ + test_cpp: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' }} - test_python: | - ${{ + test_python: ${{ steps.changed-files.outputs.ci_any_changed == 'true' || steps.changed-files.outputs.cpp_any_changed == 'true' || steps.changed-files.outputs.python_any_changed == 'true' From 5e50398b0930de43788622626f3955d4590ae420 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:58:03 -0400 Subject: [PATCH 25/85] Remove dummy comment --- ci/build_cpp.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 92d6b201d4f..e5fcef17a83 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,5 +20,3 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp - -# Dummy comment From fb5ab2568a6579b11004fb23c0741e8a86b011f5 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 12:59:26 -0400 Subject: [PATCH 26/85] conditional --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 45a6fa931e5..8b2177e2809 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -52,6 +52,7 @@ jobs: - changed-files runs-on: ubuntu-latest name: "Test job" + if: ${{ needs.changed-files.outputs.test_cpp }} steps: - name: Echo run: | From f8c6a09d47ff490db6a39fd8f362f4ccee4f8dbe Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:00:28 -0400 Subject: [PATCH 27/85] == true --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8b2177e2809..0bc10ef1d05 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -52,7 +52,7 @@ jobs: - changed-files runs-on: ubuntu-latest name: "Test job" - if: ${{ needs.changed-files.outputs.test_cpp }} + if: ${{ needs.changed-files.outputs.test_cpp == 'true' }} steps: - name: Echo run: | From 07fb6fa56754e50faf0846a915a176b697a969b6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:05:34 -0400 Subject: [PATCH 28/85] Dummy comment --- ci/build_cpp.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index e5fcef17a83..92d6b201d4f 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,3 +20,5 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp + +# Dummy comment From 62150b43de144d105eb6156a4acd6eccbfe10298 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:07:48 -0400 Subject: [PATCH 29/85] Remove test job, fix conditional --- .github/workflows/pr.yaml | 13 ++++++------- ci/build_cpp.sh | 2 -- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 0bc10ef1d05..e7994da6322 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -85,7 +85,6 @@ jobs: - pandas-tests - pandas-tests-diff - changed-files - - test-job secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: @@ -110,7 +109,7 @@ jobs: needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_cpp + if: needs.changed-files.outputs.test_cpp == 'true' with: build_type: pull-request conda-python-build: @@ -123,7 +122,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python + if: needs.changed-files.outputs.test_python == 'true' with: build_type: pull-request script: "ci/test_python_cudf.sh" @@ -132,7 +131,7 @@ jobs: needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python + if: needs.changed-files.outputs.test_python == 'true' with: build_type: pull-request script: "ci/test_python_other.sh" @@ -194,7 +193,7 @@ jobs: needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python + if: needs.changed-files.outputs.test_python == 'true' with: build_type: pull-request script: ci/test_wheel_cudf.sh @@ -211,7 +210,7 @@ jobs: needs: [wheel-build-cudf-polars, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -232,7 +231,7 @@ jobs: needs: [wheel-build-dask-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 92d6b201d4f..e5fcef17a83 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -20,5 +20,3 @@ RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcudf rapids-upload-conda-to-s3 cpp - -# Dummy comment From fe508a1c471f8b1a21473d41a329b791d00f7bf0 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:09:11 -0400 Subject: [PATCH 30/85] Remove test job --- .github/workflows/pr.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e7994da6322..01f5c93e2cb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -42,24 +42,6 @@ jobs: - cpp/** python: - python/** - - name: Print outputs - run: | - echo "CI changed: ${{ steps.changed-files.outputs.ci_any_changed }}" - echo "C++ changed: ${{ steps.changed-files.outputs.cpp_any_changed }}" - echo "Python changed: ${{ steps.changed-files.outputs.python_any_changed }}" - test-job: - needs: - - changed-files - runs-on: ubuntu-latest - name: "Test job" - if: ${{ needs.changed-files.outputs.test_cpp == 'true' }} - steps: - - name: Echo - run: | - echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" - echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" - echo "Test C++ == 'true': ${{ needs.changed-files.outputs.test_cpp == 'true' }}" - echo "Test Python == 'true': ${{ needs.changed-files.outputs.test_python == 'true' }}" pr-builder: needs: - checks From f1bc715d037a86a08eaa01ef9cae44947fafb3f7 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:52:01 -0400 Subject: [PATCH 31/85] Broaden file checks --- .github/workflows/pr.yaml | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 01f5c93e2cb..42734054132 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -14,15 +14,8 @@ jobs: runs-on: ubuntu-latest name: "Check changed files" outputs: - test_cpp: ${{ - steps.changed-files.outputs.ci_any_changed == 'true' || - steps.changed-files.outputs.cpp_any_changed == 'true' - }} - test_python: ${{ - steps.changed-files.outputs.ci_any_changed == 'true' || - steps.changed-files.outputs.cpp_any_changed == 'true' || - steps.changed-files.outputs.python_any_changed == 'true' - }} + test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} + test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} steps: - name: Checkout code repo uses: actions/checkout@v4 @@ -34,14 +27,16 @@ jobs: uses: tj-actions/changed-files@v45 with: files_yaml: | - ci: - - ci/** - - rapids_config.cmake - - build.sh cpp: - - cpp/** + - ** + - '!docs/**' + - '!java/**' + - '!notebooks/**' + - '!python/**' python: - - python/** + - ** + - '!docs/**' + - '!java/**' pr-builder: needs: - checks From 42de87fe6b24b83072f025d06ec6697718781da8 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:53:50 -0400 Subject: [PATCH 32/85] Add test job --- .github/workflows/pr.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 42734054132..dd2d48fd881 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -37,6 +37,15 @@ jobs: - ** - '!docs/**' - '!java/**' + test-job: + needs: changed-files + runs-on: ubuntu-latest + name: "Output changed files" + steps: + - name: "Output changed files" + run: | + echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" + echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" pr-builder: needs: - checks @@ -62,6 +71,7 @@ jobs: - pandas-tests - pandas-tests-diff - changed-files + - test-job secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: From c049d1feb28e7d41f7343fa1bb0680e482e2977d Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:55:38 -0400 Subject: [PATCH 33/85] Quote wildcard --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index dd2d48fd881..39ce095ba9e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -28,13 +28,13 @@ jobs: with: files_yaml: | cpp: - - ** + - '**' - '!docs/**' - '!java/**' - '!notebooks/**' - '!python/**' python: - - ** + - '**' - '!docs/**' - '!java/**' test-job: From ae50935c5d977665ec5ca26f030a8c01b7a0bddf Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:56:44 -0400 Subject: [PATCH 34/85] Re-run CI From 88344d1ad8ad82d51ee5f52a6d1956969e96a63e Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:58:15 -0400 Subject: [PATCH 35/85] Dummy comment --- python/cudf/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 9db52164eca..f0ef0ad874e 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -147,3 +147,5 @@ wheel.packages = ["cudf"] provider = "scikit_build_core.metadata.regex" input = "cudf/VERSION" regex = "(?P.*)" + +# Dummy comment From 9eb540c188d5a213d24244f18e7be6d55942eeb1 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 13:59:43 -0400 Subject: [PATCH 36/85] Debug github context --- .github/workflows/pr.yaml | 1 + python/cudf/pyproject.toml | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 39ce095ba9e..92712460e88 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -46,6 +46,7 @@ jobs: run: | echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" + echo ${{ json(github) }} pr-builder: needs: - checks diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index f0ef0ad874e..9db52164eca 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -147,5 +147,3 @@ wheel.packages = ["cudf"] provider = "scikit_build_core.metadata.regex" input = "cudf/VERSION" regex = "(?P.*)" - -# Dummy comment From ffca924ea048a2dd78aa762439a92586ec60c4b8 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:01:50 -0400 Subject: [PATCH 37/85] toJSON() --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 92712460e88..54fed31bc64 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -46,7 +46,7 @@ jobs: run: | echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" - echo ${{ json(github) }} + echo ${{ toJSON(github) }} pr-builder: needs: - checks From 75dfc3cbb7171c305a1ddf93083e3727012fff9b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:02:58 -0400 Subject: [PATCH 38/85] Quote --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 54fed31bc64..cf2407d085d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -46,7 +46,7 @@ jobs: run: | echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" - echo ${{ toJSON(github) }} + echo "${{ toJSON(github) }}" pr-builder: needs: - checks From 3ea33894d8e06b9664d0e0a07c0bcc2075d1612b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:09:10 -0400 Subject: [PATCH 39/85] Get PR info --- .github/workflows/pr.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index cf2407d085d..7f8a45a7993 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -42,11 +42,14 @@ jobs: runs-on: ubuntu-latest name: "Output changed files" steps: + - name: Get PR Info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main - name: "Output changed files" run: | echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" - echo "${{ toJSON(github) }}" + echo "${{ toJSON(steps.get-pr-info) }}" pr-builder: needs: - checks From b769771ff0b08483b6e49f785213ba6beae05346 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:11:17 -0400 Subject: [PATCH 40/85] pr-info --- .github/workflows/pr.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7f8a45a7993..576cf917da8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -49,7 +49,9 @@ jobs: run: | echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" - echo "${{ toJSON(steps.get-pr-info) }}" + jq . < Date: Thu, 22 Aug 2024 14:15:11 -0400 Subject: [PATCH 41/85] Check against PR base sha --- .github/workflows/pr.yaml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 576cf917da8..3bfcdd160d7 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,10 +22,14 @@ jobs: with: ref: ${{ inputs.sha }} persist-credentials: false + - name: Get PR info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main - name: Get changed files id: changed-files uses: tj-actions/changed-files@v45 with: + base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} files_yaml: | cpp: - '**' @@ -37,21 +41,6 @@ jobs: - '**' - '!docs/**' - '!java/**' - test-job: - needs: changed-files - runs-on: ubuntu-latest - name: "Output changed files" - steps: - - name: Get PR Info - id: get-pr-info - uses: rapidsai/shared-actions/get-pr-info@main - - name: "Output changed files" - run: | - echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" - echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" - jq . < Date: Thu, 22 Aug 2024 14:22:49 -0400 Subject: [PATCH 42/85] Increase fetch depth --- .github/workflows/pr.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3bfcdd160d7..286800cac0f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,14 +17,15 @@ jobs: test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} steps: + - name: Get PR info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main - name: Checkout code repo uses: actions/checkout@v4 with: ref: ${{ inputs.sha }} + fetch-depth: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).commits }} persist-credentials: false - - name: Get PR info - id: get-pr-info - uses: rapidsai/shared-actions/get-pr-info@main - name: Get changed files id: changed-files uses: tj-actions/changed-files@v45 From 452e76592daef72b338f38a9fd4fd3fabf3bf60c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:26:20 -0400 Subject: [PATCH 43/85] Debugging --- .github/workflows/pr.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 286800cac0f..7ff28c75609 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -42,6 +42,15 @@ jobs: - '**' - '!docs/**' - '!java/**' + debug-job: + needs: changed-files + runs-on: ubuntu-latest + name: "Debug job" + steps: + - name: Debug change-files output + run: | + echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" + echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" pr-builder: needs: - checks @@ -67,6 +76,7 @@ jobs: - pandas-tests - pandas-tests-diff - changed-files + - debug-job secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: From d7843a500127b5e581aebf040437ebeb798dd5b2 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:29:46 -0400 Subject: [PATCH 44/85] More debugging --- .github/workflows/pr.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7ff28c75609..1078f21b9eb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,6 +16,8 @@ jobs: outputs: test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + test_cpp_minus_github: ${{ steps.changed-files.outputs.cpp_minus_github_any_changed == 'true' }} + test_python_minus_github: ${{ steps.changed-files.outputs.python_minus_github_any_changed == 'true' }} steps: - name: Get PR info id: get-pr-info @@ -42,6 +44,18 @@ jobs: - '**' - '!docs/**' - '!java/**' + cpp_minus_github: + - '**' + - '!docs/**' + - '!java/**' + - '!notebooks/**' + - '!python/**' + - '!.github/**' + python_minus_github: + - '**' + - '!docs/**' + - '!java/**' + - '!.github/**' debug-job: needs: changed-files runs-on: ubuntu-latest @@ -51,6 +65,8 @@ jobs: run: | echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" + echo "Test C++ minus .github: ${{ needs.changed-files.outputs.test_cpp_minus_github }}" + echo "Test Python minus .github: ${{ needs.changed-files.outputs.test_python_minus_github }}" pr-builder: needs: - checks From 2796540ce3d34ac34a1e681c10add4daa7fc836c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:30:42 -0400 Subject: [PATCH 45/85] Dummy comment --- python/cudf/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 9db52164eca..f0ef0ad874e 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -147,3 +147,5 @@ wheel.packages = ["cudf"] provider = "scikit_build_core.metadata.regex" input = "cudf/VERSION" regex = "(?P.*)" + +# Dummy comment From 3c49cdf5117c3518339ac3498d2de604e7c74398 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:31:34 -0400 Subject: [PATCH 46/85] Revert "Dummy comment" This reverts commit 2796540ce3d34ac34a1e681c10add4daa7fc836c. --- python/cudf/pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index f0ef0ad874e..9db52164eca 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -147,5 +147,3 @@ wheel.packages = ["cudf"] provider = "scikit_build_core.metadata.regex" input = "cudf/VERSION" regex = "(?P.*)" - -# Dummy comment From fdfe3e0bc862fc23788599f7b09c53dc32c1d9ac Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:30:42 -0400 Subject: [PATCH 47/85] Dummy comment --- python/cudf/pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index 9db52164eca..f0ef0ad874e 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -147,3 +147,5 @@ wheel.packages = ["cudf"] provider = "scikit_build_core.metadata.regex" input = "cudf/VERSION" regex = "(?P.*)" + +# Dummy comment From 2b3771d9583fc7a4d93f1f0462ba943678e4cd15 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:35:31 -0400 Subject: [PATCH 48/85] Re-run CI From 547028a5640ff178a4a259994a3174775fb855cd Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 14:38:23 -0400 Subject: [PATCH 49/85] Remove debugging --- .github/workflows/pr.yaml | 26 -------------------------- python/cudf/pyproject.toml | 2 -- 2 files changed, 28 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1078f21b9eb..286800cac0f 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,8 +16,6 @@ jobs: outputs: test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} - test_cpp_minus_github: ${{ steps.changed-files.outputs.cpp_minus_github_any_changed == 'true' }} - test_python_minus_github: ${{ steps.changed-files.outputs.python_minus_github_any_changed == 'true' }} steps: - name: Get PR info id: get-pr-info @@ -44,29 +42,6 @@ jobs: - '**' - '!docs/**' - '!java/**' - cpp_minus_github: - - '**' - - '!docs/**' - - '!java/**' - - '!notebooks/**' - - '!python/**' - - '!.github/**' - python_minus_github: - - '**' - - '!docs/**' - - '!java/**' - - '!.github/**' - debug-job: - needs: changed-files - runs-on: ubuntu-latest - name: "Debug job" - steps: - - name: Debug change-files output - run: | - echo "Test C++: ${{ needs.changed-files.outputs.test_cpp }}" - echo "Test Python: ${{ needs.changed-files.outputs.test_python }}" - echo "Test C++ minus .github: ${{ needs.changed-files.outputs.test_cpp_minus_github }}" - echo "Test Python minus .github: ${{ needs.changed-files.outputs.test_python_minus_github }}" pr-builder: needs: - checks @@ -92,7 +67,6 @@ jobs: - pandas-tests - pandas-tests-diff - changed-files - - debug-job secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: diff --git a/python/cudf/pyproject.toml b/python/cudf/pyproject.toml index f0ef0ad874e..9db52164eca 100644 --- a/python/cudf/pyproject.toml +++ b/python/cudf/pyproject.toml @@ -147,5 +147,3 @@ wheel.packages = ["cudf"] provider = "scikit_build_core.metadata.regex" input = "cudf/VERSION" regex = "(?P.*)" - -# Dummy comment From a4e45c20114987560e74f34d7c20089d6794c9f4 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:06:55 -0400 Subject: [PATCH 50/85] Filter Java and more Python jobs --- .github/workflows/pr.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 286800cac0f..211a0c5cecb 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,6 +16,7 @@ jobs: outputs: test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} steps: - name: Get PR info id: get-pr-info @@ -42,6 +43,11 @@ jobs: - '**' - '!docs/**' - '!java/**' + java: + - '**' + - '!docs/**' + - '!notebooks/**' + - '!python/**' pr-builder: needs: - checks @@ -121,6 +127,7 @@ jobs: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + if: needs.changed-files.outputs.test_java == 'true' with: build_type: pull-request node_type: "gpu-v100-latest-1" @@ -233,6 +240,7 @@ jobs: needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) @@ -243,6 +251,7 @@ jobs: needs: wheel-build-cudf secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' with: # This selects "ARCH=amd64 + the latest supported Python + CUDA". matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) From a33d3128311199cb2c55a8e0274d6bd73fcc19c9 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:08:30 -0400 Subject: [PATCH 51/85] Sort workflow jobs --- .github/workflows/pr.yaml | 54 +++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 211a0c5cecb..af493f96203 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -10,6 +10,33 @@ concurrency: cancel-in-progress: true jobs: + pr-builder: + needs: + - changed-files + - checks + - conda-cpp-build + - conda-cpp-checks + - conda-cpp-tests + - conda-python-build + - conda-python-cudf-tests + - conda-python-other-tests + - conda-java-tests + - static-configure + - conda-notebook-tests + - docs-build + - wheel-build-pylibcudf + - wheel-build-cudf + - wheel-tests-cudf + - wheel-build-cudf-polars + - wheel-tests-cudf-polars + - wheel-build-dask-cudf + - wheel-tests-dask-cudf + - devcontainer + - unit-tests-cudf-pandas + - pandas-tests + - pandas-tests-diff + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 changed-files: runs-on: ubuntu-latest name: "Check changed files" @@ -48,33 +75,6 @@ jobs: - '!docs/**' - '!notebooks/**' - '!python/**' - pr-builder: - needs: - - checks - - conda-cpp-build - - conda-cpp-checks - - conda-cpp-tests - - conda-python-build - - conda-python-cudf-tests - - conda-python-other-tests - - conda-java-tests - - static-configure - - conda-notebook-tests - - docs-build - - wheel-build-pylibcudf - - wheel-build-cudf - - wheel-tests-cudf - - wheel-build-cudf-polars - - wheel-tests-cudf-polars - - wheel-build-dask-cudf - - wheel-tests-dask-cudf - - devcontainer - - unit-tests-cudf-pandas - - pandas-tests - - pandas-tests-diff - - changed-files - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 From aefcee3b9175d57257c753ab318d31afbd2ea41a Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:10:31 -0400 Subject: [PATCH 52/85] Test requiring a job that never runs --- .github/workflows/pr.yaml | 257 +------------------------------------- 1 file changed, 6 insertions(+), 251 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index af493f96203..13f7573d4f1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,258 +12,13 @@ concurrency: jobs: pr-builder: needs: - - changed-files - - checks - - conda-cpp-build - - conda-cpp-checks - - conda-cpp-tests - - conda-python-build - - conda-python-cudf-tests - - conda-python-other-tests - - conda-java-tests - - static-configure - - conda-notebook-tests - - docs-build - - wheel-build-pylibcudf - - wheel-build-cudf - - wheel-tests-cudf - - wheel-build-cudf-polars - - wheel-tests-cudf-polars - - wheel-build-dask-cudf - - wheel-tests-dask-cudf - - devcontainer - - unit-tests-cudf-pandas - - pandas-tests - - pandas-tests-diff + - never-run secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 - changed-files: + never-run: runs-on: ubuntu-latest - name: "Check changed files" - outputs: - test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} - test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} - test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} + if: false steps: - - name: Get PR info - id: get-pr-info - uses: rapidsai/shared-actions/get-pr-info@main - - name: Checkout code repo - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - fetch-depth: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).commits }} - persist-credentials: false - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v45 - with: - base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} - files_yaml: | - cpp: - - '**' - - '!docs/**' - - '!java/**' - - '!notebooks/**' - - '!python/**' - python: - - '**' - - '!docs/**' - - '!java/**' - java: - - '**' - - '!docs/**' - - '!notebooks/**' - - '!python/**' - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 - with: - enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 - with: - build_type: pull-request - enable_check_symbols: true - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_cpp == 'true' - with: - build_type: pull-request - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 - with: - build_type: pull-request - conda-python-cudf-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' - with: - build_type: pull-request - script: "ci/test_python_cudf.sh" - conda-python-other-tests: - # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' - with: - build_type: pull-request - script: "ci/test_python_other.sh" - conda-java-tests: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - if: needs.changed-files.outputs.test_java == 'true' - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_java.sh" - static-configure: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - # Use the wheel container so we can skip conda solves and since our - # primary static consumers (Spark) are not in conda anyway. - container_image: "rapidsai/ci-wheel:latest" - run_script: "ci/configure_cpp_static.sh" - conda-notebook-tests: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" - wheel-build-pylibcudf: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 - with: - build_type: pull-request - script: "ci/build_wheel_pylibcudf.sh" - wheel-build-cudf: - needs: wheel-build-pylibcudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 - with: - build_type: pull-request - script: "ci/build_wheel_cudf.sh" - wheel-tests-cudf: - needs: [wheel-build-cudf, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' - with: - build_type: pull-request - script: ci/test_wheel_cudf.sh - wheel-build-cudf-polars: - needs: wheel-build-pylibcudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request - script: "ci/build_wheel_cudf_polars.sh" - wheel-tests-cudf-polars: - needs: [wheel-build-cudf-polars, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request - # This always runs, but only fails if this PR touches code in - # pylibcudf or cudf_polars - script: "ci/test_wheel_cudf_polars.sh" - wheel-build-dask-cudf: - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request - script: "ci/build_wheel_dask_cudf.sh" - wheel-tests-dask-cudf: - needs: [wheel-build-dask-cudf, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request - script: ci/test_wheel_dask_cudf.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_BENCHMARKS=ON --verbose; - sccache -s; - unit-tests-cudf-pandas: - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request - script: ci/cudf_pandas_scripts/run_tests.sh - pandas-tests: - # run the Pandas unit tests using PR branch - needs: wheel-build-cudf - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 - if: needs.changed-files.outputs.test_python == 'true' - with: - # This selects "ARCH=amd64 + the latest supported Python + CUDA". - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: pull-request - script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr - # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. - test_summary_show: "none" - pandas-tests-diff: - # diff the results of running the Pandas unit tests and publish a job summary - needs: pandas-tests - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 - with: - node_type: cpu4 - build_type: pull-request - run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh" + - run: | + echo "This should never run!" + exit 1 From bb4caa3d476278a72392ff6ed030c110591672b0 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:11:51 -0400 Subject: [PATCH 53/85] Run the never-run job --- .github/workflows/pr.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 13f7573d4f1..ae8f25aa448 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,8 +17,6 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 never-run: runs-on: ubuntu-latest - if: false steps: - run: | echo "This should never run!" - exit 1 From 1dab6f077d00e5f695acef4af0f9b82455055489 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:13:59 -0400 Subject: [PATCH 54/85] Revert "Run the never-run job" This reverts commit bb4caa3d476278a72392ff6ed030c110591672b0. --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ae8f25aa448..13f7573d4f1 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,8 @@ jobs: uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 never-run: runs-on: ubuntu-latest + if: false steps: - run: | echo "This should never run!" + exit 1 From 747de4ca414f8c75ebb30150c53430d3e1797cfe Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:16:04 -0400 Subject: [PATCH 55/85] Try conditional --- .github/workflows/pr.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 13f7573d4f1..e794765d2df 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,12 +12,18 @@ concurrency: jobs: pr-builder: needs: + - never-run-test - never-run secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + never-run-test: + runs-on: ubuntu-latest + outputs: + run-never-run: true never-run: + needs: never-run-test runs-on: ubuntu-latest - if: false + if: ${{ needs.never-run-test.outputs.run-never-run == 'true' }} steps: - run: | echo "This should never run!" From 32e985a7380eeb72a51a8dfd050a22106347f15a Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:17:11 -0400 Subject: [PATCH 56/85] exit 0 --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e794765d2df..efb5f105ce6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,6 +20,8 @@ jobs: runs-on: ubuntu-latest outputs: run-never-run: true + steps: + - run: exit 0 never-run: needs: never-run-test runs-on: ubuntu-latest From 4b9e3666782165ef6f668b061f6df1d0e0586556 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:17:56 -0400 Subject: [PATCH 57/85] false --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index efb5f105ce6..356febfdc32 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,7 +19,7 @@ jobs: never-run-test: runs-on: ubuntu-latest outputs: - run-never-run: true + run-never-run: false steps: - run: exit 0 never-run: From 5ada0b918252493e319711ea6d6567bb1675e2d1 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:19:08 -0400 Subject: [PATCH 58/85] if: always() --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 356febfdc32..6e2762f057b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,6 +16,7 @@ jobs: - never-run secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + if: always() never-run-test: runs-on: ubuntu-latest outputs: From 4a37ddbc3a21a1471e1a5ef6e52d05a36ce130cb Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:20:43 -0400 Subject: [PATCH 59/85] Run the never-run test --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6e2762f057b..e533275c917 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -20,7 +20,7 @@ jobs: never-run-test: runs-on: ubuntu-latest outputs: - run-never-run: false + run-never-run: true steps: - run: exit 0 never-run: From 1b26d708806ea0f14f782ed8e373ea717ec44e2e Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:38:21 -0400 Subject: [PATCH 60/85] Initial commit of fake-pr-builder --- .github/workflows/fake-pr-builder.yaml | 9 +++++++++ .github/workflows/pr.yaml | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/fake-pr-builder.yaml diff --git a/.github/workflows/fake-pr-builder.yaml b/.github/workflows/fake-pr-builder.yaml new file mode 100644 index 00000000000..4ac490fb195 --- /dev/null +++ b/.github/workflows/fake-pr-builder.yaml @@ -0,0 +1,9 @@ +on: + workflow_call: + +jobs: + run: + runs-on: ubuntu-latest + steps: + # This reusable workflow should depend on all other jobs in the calling workflow. + - run: exit 0 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index e533275c917..1b9ed811a9a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - never-run-test - never-run secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 + uses: KyleFromNVIDIA/cudf/.github/workflows/fake-pr-builder.yaml@test-pruning if: always() never-run-test: runs-on: ubuntu-latest From feed025aba1147df292c64fe82244c463135d73b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:41:12 -0400 Subject: [PATCH 61/85] Start using pr-builder-needs-check --- .github/workflows/fake-pr-builder.yaml | 9 --------- .github/workflows/pr.yaml | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) delete mode 100644 .github/workflows/fake-pr-builder.yaml diff --git a/.github/workflows/fake-pr-builder.yaml b/.github/workflows/fake-pr-builder.yaml deleted file mode 100644 index 4ac490fb195..00000000000 --- a/.github/workflows/fake-pr-builder.yaml +++ /dev/null @@ -1,9 +0,0 @@ -on: - workflow_call: - -jobs: - run: - runs-on: ubuntu-latest - steps: - # This reusable workflow should depend on all other jobs in the calling workflow. - - run: exit 0 diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 1b9ed811a9a..55df1b5cea8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -15,7 +15,7 @@ jobs: - never-run-test - never-run secrets: inherit - uses: KyleFromNVIDIA/cudf/.github/workflows/fake-pr-builder.yaml@test-pruning + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@pr-builder-needs-check if: always() never-run-test: runs-on: ubuntu-latest From 140f24cab6835042a0dc934fa10a3b50a59f2ab0 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:42:49 -0400 Subject: [PATCH 62/85] Re-run CI From 173591a2a48d32aad1445280d94a223b4b464440 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:42:49 -0400 Subject: [PATCH 63/85] Re-run CI From 9bbe5cc1e43b9fd58451594985cb493faae9e40c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:49:41 -0400 Subject: [PATCH 64/85] Re-run CI From a9f783a0da084ecbfd1afb08f5a997dd77c6493c Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:56:15 -0400 Subject: [PATCH 65/85] Pass needs input --- .github/workflows/pr.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 55df1b5cea8..ba5dbd93a38 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -17,6 +17,8 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@pr-builder-needs-check if: always() + with: + needs: ${{ toJSON(needs) }} never-run-test: runs-on: ubuntu-latest outputs: From c2dda976978b06ee6d84a9fdce819b3e89747ed1 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:57:53 -0400 Subject: [PATCH 66/85] Remove if: always() --- .github/workflows/pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ba5dbd93a38..ac705adeb27 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,7 +16,6 @@ jobs: - never-run secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@pr-builder-needs-check - if: always() with: needs: ${{ toJSON(needs) }} never-run-test: From 1ee9faaa80e3566b37646c70de57f1eaee181328 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 15:58:52 -0400 Subject: [PATCH 67/85] Revert "Remove if: always()" This reverts commit c2dda976978b06ee6d84a9fdce819b3e89747ed1. --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ac705adeb27..ba5dbd93a38 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -16,6 +16,7 @@ jobs: - never-run secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@pr-builder-needs-check + if: always() with: needs: ${{ toJSON(needs) }} never-run-test: From 09c86b7835f670b03df741898a7ea9798c1b7f40 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 16:01:35 -0400 Subject: [PATCH 68/85] Re-run CI From 023ec73c93c087d8a5a43f39867eb3ef5addb9b6 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 16:17:54 -0400 Subject: [PATCH 69/85] Re-run CI From b7b578fd72d0214f980eb3a764ccd593bb9a3374 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 16:21:16 -0400 Subject: [PATCH 70/85] Re-run CI From bbcd80f7b2dcb093d83dbdfaf58de5e994725b0b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 16:22:59 -0400 Subject: [PATCH 71/85] Re-run CI From e792537139a3176d4ed1c7a27f50398bf2ad9c4b Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 16:31:13 -0400 Subject: [PATCH 72/85] Re-run CI From 79cca7e0e3134e7aec82e977ca64a41a19cc108f Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 16:32:30 -0400 Subject: [PATCH 73/85] Skip never-run --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ba5dbd93a38..3f21f8f859b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,7 +22,7 @@ jobs: never-run-test: runs-on: ubuntu-latest outputs: - run-never-run: true + run-never-run: false steps: - run: exit 0 never-run: From 1866baf810a2c09a967796dd23fb466c41428d5a Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 17:00:47 -0400 Subject: [PATCH 74/85] Run never-run --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3f21f8f859b..ba5dbd93a38 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,7 +22,7 @@ jobs: never-run-test: runs-on: ubuntu-latest outputs: - run-never-run: false + run-never-run: true steps: - run: exit 0 never-run: From 39ba71a376e697706039f205dddd59d1a0fffc8e Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Thu, 22 Aug 2024 17:01:30 -0400 Subject: [PATCH 75/85] Revert "Run never-run" This reverts commit 1866baf810a2c09a967796dd23fb466c41428d5a. --- .github/workflows/pr.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ba5dbd93a38..3f21f8f859b 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,7 +22,7 @@ jobs: never-run-test: runs-on: ubuntu-latest outputs: - run-never-run: true + run-never-run: false steps: - run: exit 0 never-run: From fd06a97cbb02c9a28a3faf24645671a493b71827 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 10:18:47 -0400 Subject: [PATCH 76/85] Restore workflow and add if and needs to pr-builder --- .github/workflows/pr.yaml | 265 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 251 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 3f21f8f859b..347af4afd6a 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,24 +12,261 @@ concurrency: jobs: pr-builder: needs: - - never-run-test - - never-run + - changed-files + - checks + - conda-cpp-build + - conda-cpp-checks + - conda-cpp-tests + - conda-python-build + - conda-python-cudf-tests + - conda-python-other-tests + - conda-java-tests + - static-configure + - conda-notebook-tests + - docs-build + - wheel-build-pylibcudf + - wheel-build-cudf + - wheel-tests-cudf + - wheel-build-cudf-polars + - wheel-tests-cudf-polars + - wheel-build-dask-cudf + - wheel-tests-dask-cudf + - devcontainer + - unit-tests-cudf-pandas + - pandas-tests + - pandas-tests-diff secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@pr-builder-needs-check + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.10 if: always() with: needs: ${{ toJSON(needs) }} - never-run-test: + changed-files: runs-on: ubuntu-latest + name: "Check changed files" outputs: - run-never-run: false + test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} + test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} + test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} steps: - - run: exit 0 - never-run: - needs: never-run-test - runs-on: ubuntu-latest - if: ${{ needs.never-run-test.outputs.run-never-run == 'true' }} - steps: - - run: | - echo "This should never run!" - exit 1 + - name: Get PR info + id: get-pr-info + uses: rapidsai/shared-actions/get-pr-info@main + - name: Checkout code repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.sha }} + fetch-depth: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).commits }} + persist-credentials: false + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + files_yaml: | + cpp: + - '**' + - '!docs/**' + - '!java/**' + - '!notebooks/**' + - '!python/**' + python: + - '**' + - '!docs/**' + - '!java/**' + java: + - '**' + - '!docs/**' + - '!notebooks/**' + - '!python/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 + with: + enable_check_generated_files: false + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.10 + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.10 + with: + build_type: pull-request + enable_check_symbols: true + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_cpp == 'true' + with: + build_type: pull-request + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.10 + with: + build_type: pull-request + conda-python-cudf-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + build_type: pull-request + script: "ci/test_python_cudf.sh" + conda-python-other-tests: + # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + build_type: pull-request + script: "ci/test_python_other.sh" + conda-java-tests: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + if: needs.changed-files.outputs.test_java == 'true' + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_java.sh" + static-configure: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + # Use the wheel container so we can skip conda solves and since our + # primary static consumers (Spark) are not in conda anyway. + container_image: "rapidsai/ci-wheel:latest" + run_script: "ci/configure_cpp_static.sh" + conda-notebook-tests: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" + wheel-build-pylibcudf: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/build_wheel_pylibcudf.sh" + wheel-build-cudf: + needs: wheel-build-pylibcudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + build_type: pull-request + script: "ci/build_wheel_cudf.sh" + wheel-tests-cudf: + needs: [wheel-build-cudf, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + build_type: pull-request + script: ci/test_wheel_cudf.sh + wheel-build-cudf-polars: + needs: wheel-build-pylibcudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: "ci/build_wheel_cudf_polars.sh" + wheel-tests-cudf-polars: + needs: [wheel-build-cudf-polars, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + # This always runs, but only fails if this PR touches code in + # pylibcudf or cudf_polars + script: "ci/test_wheel_cudf_polars.sh" + wheel-build-dask-cudf: + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.10 + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: "ci/build_wheel_dask_cudf.sh" + wheel-tests-dask-cudf: + needs: [wheel-build-dask-cudf, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: ci/test_wheel_dask_cudf.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.10 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + build_command: | + sccache -z; + build-all -DBUILD_BENCHMARKS=ON --verbose; + sccache -s; + unit-tests-cudf-pandas: + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: ci/cudf_pandas_scripts/run_tests.sh + pandas-tests: + # run the Pandas unit tests using PR branch + needs: wheel-build-cudf + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 + if: needs.changed-files.outputs.test_python == 'true' + with: + # This selects "ARCH=amd64 + the latest supported Python + CUDA". + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: pull-request + script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr + # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. + test_summary_show: "none" + pandas-tests-diff: + # diff the results of running the Pandas unit tests and publish a job summary + needs: pandas-tests + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + with: + node_type: cpu4 + build_type: pull-request + run_script: "ci/cudf_pandas_scripts/pandas-tests/diff.sh" From 875628ca7542c5979188a6fffd4cdf8897790348 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 10:27:00 -0400 Subject: [PATCH 77/85] Temporarily ignore .github changes for testing --- .github/workflows/pr.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 347af4afd6a..740b7f09baa 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -69,15 +69,21 @@ jobs: - '!java/**' - '!notebooks/**' - '!python/**' + # TODO: Remove this before merging + - '!.github/**' python: - '**' - '!docs/**' - '!java/**' + # TODO: Remove this before merging + - '!.github/**' java: - '**' - '!docs/**' - '!notebooks/**' - '!python/**' + # TODO: Remove this before merging + - '!.github/**' checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 From 0761f0457d390a39f8b7a3f505798723733546d0 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 10:35:07 -0400 Subject: [PATCH 78/85] Filter notebook tests --- .github/workflows/pr.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 740b7f09baa..355f80e2824 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -45,8 +45,9 @@ jobs: name: "Check changed files" outputs: test_cpp: ${{ steps.changed-files.outputs.cpp_any_changed == 'true' }} - test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} test_java: ${{ steps.changed-files.outputs.java_any_changed == 'true' }} + test_notebooks: ${{ steps.changed-files.outputs.notebooks_any_changed == 'true' }} + test_python: ${{ steps.changed-files.outputs.python_any_changed == 'true' }} steps: - name: Get PR info id: get-pr-info @@ -71,17 +72,24 @@ jobs: - '!python/**' # TODO: Remove this before merging - '!.github/**' - python: + java: + - '**' + - '!docs/**' + - '!notebooks/**' + - '!python/**' + # TODO: Remove this before merging + - '!.github/**' + notebooks: - '**' - '!docs/**' - '!java/**' # TODO: Remove this before merging - '!.github/**' - java: + python: - '**' - '!docs/**' + - '!java/**' - '!notebooks/**' - - '!python/**' # TODO: Remove this before merging - '!.github/**' checks: @@ -157,6 +165,7 @@ jobs: needs: conda-python-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 + if: needs.changed-files.outputs.test_notebooks == 'true' with: build_type: pull-request node_type: "gpu-v100-latest-1" From acf267227f167fa4dd86ab94f9cc0aadd9a2aee3 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 10:36:49 -0400 Subject: [PATCH 79/85] Add needs changed-files --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 355f80e2824..f898b2fcf62 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -141,7 +141,7 @@ jobs: build_type: pull-request script: "ci/test_python_other.sh" conda-java-tests: - needs: conda-cpp-build + needs: [conda-cpp-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 if: needs.changed-files.outputs.test_java == 'true' @@ -162,7 +162,7 @@ jobs: container_image: "rapidsai/ci-wheel:latest" run_script: "ci/configure_cpp_static.sh" conda-notebook-tests: - needs: conda-python-build + needs: [conda-python-build, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.10 if: needs.changed-files.outputs.test_notebooks == 'true' From 46d07064cb076feeecc4348da745a6d39d1c7a88 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 10:37:39 -0400 Subject: [PATCH 80/85] Add needs changed-files --- .github/workflows/pr.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index f898b2fcf62..75c111cb5f3 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -255,7 +255,7 @@ jobs: build-all -DBUILD_BENCHMARKS=ON --verbose; sccache -s; unit-tests-cudf-pandas: - needs: wheel-build-cudf + needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 if: needs.changed-files.outputs.test_python == 'true' @@ -266,7 +266,7 @@ jobs: script: ci/cudf_pandas_scripts/run_tests.sh pandas-tests: # run the Pandas unit tests using PR branch - needs: wheel-build-cudf + needs: [wheel-build-cudf, changed-files] secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.10 if: needs.changed-files.outputs.test_python == 'true' From 0314fd62a79c21109ea14cfb76db91cd96c882e8 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 11:11:23 -0400 Subject: [PATCH 81/85] Don't ignore docs for notebooks --- .github/workflows/pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 75c111cb5f3..edd44c32568 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -81,7 +81,6 @@ jobs: - '!.github/**' notebooks: - '**' - - '!docs/**' - '!java/**' # TODO: Remove this before merging - '!.github/**' From fa16682ff6c61c60cfb3ecdbec92c423cf701dd2 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 11:12:08 -0400 Subject: [PATCH 82/85] Test a notebook failure --- docs/cudf/source/user_guide/10min.ipynb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/cudf/source/user_guide/10min.ipynb b/docs/cudf/source/user_guide/10min.ipynb index c3da2558db8..6c5efcd3829 100644 --- a/docs/cudf/source/user_guide/10min.ipynb +++ b/docs/cudf/source/user_guide/10min.ipynb @@ -38,6 +38,10 @@ "import cudf\n", "import dask_cudf\n", "\n", + "# Test a notebook failure\n", + "# TODO: Remove this before merging\n", + "exit(1)\n", + "\n", "cp.random.seed(12)\n", "\n", "#### Portions of this were borrowed and adapted from the\n", From 6f86576f1604ffa5a7df900314098f8568f71179 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Fri, 23 Aug 2024 13:04:44 -0400 Subject: [PATCH 83/85] Remove temporary testing modifications --- .github/workflows/pr.yaml | 8 -------- docs/cudf/source/user_guide/10min.ipynb | 4 ---- 2 files changed, 12 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 7fa0304cb37..6adcc538df8 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -71,27 +71,19 @@ jobs: - '!java/**' - '!notebooks/**' - '!python/**' - # TODO: Remove this before merging - - '!.github/**' java: - '**' - '!docs/**' - '!notebooks/**' - '!python/**' - # TODO: Remove this before merging - - '!.github/**' notebooks: - '**' - '!java/**' - # TODO: Remove this before merging - - '!.github/**' python: - '**' - '!docs/**' - '!java/**' - '!notebooks/**' - # TODO: Remove this before merging - - '!.github/**' checks: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.10 diff --git a/docs/cudf/source/user_guide/10min.ipynb b/docs/cudf/source/user_guide/10min.ipynb index 6c5efcd3829..c3da2558db8 100644 --- a/docs/cudf/source/user_guide/10min.ipynb +++ b/docs/cudf/source/user_guide/10min.ipynb @@ -38,10 +38,6 @@ "import cudf\n", "import dask_cudf\n", "\n", - "# Test a notebook failure\n", - "# TODO: Remove this before merging\n", - "exit(1)\n", - "\n", "cp.random.seed(12)\n", "\n", "#### Portions of this were borrowed and adapted from the\n", From c4c68f58959d85c225e3e93159e7162bec5a04f5 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 26 Aug 2024 15:56:52 -0400 Subject: [PATCH 84/85] Exclude README, CONTRIBUTING, and imgs --- .github/workflows/pr.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6adcc538df8..26eb728f51d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -67,21 +67,33 @@ jobs: files_yaml: | cpp: - '**' + - '!CONTRIBUTING.md' + - '!README.md' - '!docs/**' + - '!img/**' - '!java/**' - '!notebooks/**' - '!python/**' java: - '**' + - '!CONTRIBUTING.md' + - '!README.md' - '!docs/**' + - '!img/**' - '!notebooks/**' - '!python/**' notebooks: - '**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!img/**' - '!java/**' python: - '**' + - '!CONTRIBUTING.md' + - '!README.md' - '!docs/**' + - '!img/**' - '!java/**' - '!notebooks/**' checks: From 1f49df5c178d1d9f68e540eedefe1f1e1d440cfb Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Mon, 26 Aug 2024 15:57:16 -0400 Subject: [PATCH 85/85] Don't exclude imgs from notebooks --- .github/workflows/pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 26eb728f51d..35c7e3d95b6 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -86,7 +86,6 @@ jobs: - '**' - '!CONTRIBUTING.md' - '!README.md' - - '!img/**' - '!java/**' python: - '**'