From 7d6ca6eaf3e28eac69af6c5e459fb650e5d30418 Mon Sep 17 00:00:00 2001 From: Paul Taylor <178183+trxcllnt@users.noreply.github.com> Date: Fri, 24 May 2024 13:32:54 -0700 Subject: [PATCH] Add workflow to automatically release when PRs are merged (#311) * add release-all.yml workflow to release on push to branch-* and release tags * support only building a subset of images when pushing, and optionally when building manually * rename release.yaml to release-linux.yml --- .github/actions/feature-matrix/action.yml | 27 +++--- .github/actions/image-matrix/action.sh | 4 +- .github/actions/image-matrix/action.yml | 32 +++---- .github/workflows/build-all-rapids-repos.yml | 8 +- .github/workflows/build-and-test-feature.yml | 7 +- .../build-test-and-push-linux-image.yml | 6 +- .../build-test-and-push-windows-image.yml | 4 +- .github/workflows/docs.yml | 14 +-- .github/workflows/release-features.yml | 6 +- .github/workflows/release-linux.yml | 69 +++++++++++++++ .github/workflows/release-windows.yml | 9 +- .github/workflows/release.yml | 63 ++++++++++++-- .github/workflows/test.yml | 85 ++++++++++++++----- .github/workflows/validate-features-json.yml | 23 ----- 14 files changed, 247 insertions(+), 110 deletions(-) create mode 100644 .github/workflows/release-linux.yml delete mode 100644 .github/workflows/validate-features-json.yml diff --git a/.github/actions/feature-matrix/action.yml b/.github/actions/feature-matrix/action.yml index 0f864747..923a762b 100644 --- a/.github/actions/feature-matrix/action.yml +++ b/.github/actions/feature-matrix/action.yml @@ -3,9 +3,15 @@ name: feature-matrix description: Determine the feature matrix inputs: + base_sha: + type: string + default: "" full_matrix: type: string - default: 'false' + default: false + file_patterns: + type: string + default: "" outputs: features: @@ -16,25 +22,16 @@ outputs: runs: using: composite steps: - - id: get-pr-info - if: inputs.full_matrix != 'true' - name: Get PR target branch info - uses: ./.github/actions/get-pr-info - - - name: Check if files changed + - id: changes + name: Check if files changed if: inputs.full_matrix != 'true' - id: changes uses: tj-actions/changed-files@v44.0.1 with: - base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + base_sha: "${{ inputs.base_sha }}" + since_last_remote_commit: "${{ inputs.base_sha == '' && 'true' || 'false' }}" files: | features/src/** - features/test/** - .github/workflows/test.yml - .github/actions/feature-matrix/action.sh - .github/actions/feature-matrix/action.yml - .github/workflows/build-and-test-feature.yml - .github/actions/build-and-test-feature/action.yml + ${{ inputs.file_patterns }} - name: Report changes if: inputs.full_matrix != 'true' diff --git a/.github/actions/image-matrix/action.sh b/.github/actions/image-matrix/action.sh index 8df58bf2..7432cade 100755 --- a/.github/actions/image-matrix/action.sh +++ b/.github/actions/image-matrix/action.sh @@ -71,7 +71,7 @@ linux_images="$(\ | (.name | split("/")[-1] | split(":")[0]) + (.version // "" | tostring)) ) | (. + [$os]) - | join(" "), + | join("-"), } ) ) @@ -101,7 +101,7 @@ if `# Include all images if full_matrix is true` \ | (.name | split("/")[-1] | split(":")[0]) + (.version // "" | tostring)) ) | (. + [$os]) - | join(" "), + | join("-"), } ) ) diff --git a/.github/actions/image-matrix/action.yml b/.github/actions/image-matrix/action.yml index 856ccb5e..b5811aab 100644 --- a/.github/actions/image-matrix/action.yml +++ b/.github/actions/image-matrix/action.yml @@ -3,6 +3,9 @@ name: image-matrix description: Determine the image matrix inputs: + base_sha: + type: string + default: "" features: type: string required: false @@ -11,7 +14,10 @@ inputs: required: false full_matrix: type: string - default: 'true' + default: true + file_patterns: + type: string + default: "" outputs: linux: @@ -22,28 +28,18 @@ outputs: runs: using: composite steps: - - id: get-pr-info - if: inputs.full_matrix != 'true' - name: Get PR target branch info - uses: ./.github/actions/get-pr-info - - - name: Check if files changed + - id: changes + name: Check if files changed if: inputs.full_matrix != 'true' - id: changes uses: tj-actions/changed-files@v44.0.1 with: - base_sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }} + base_sha: "${{ inputs.base_sha }}" + since_last_remote_commit: "${{ inputs.base_sha == '' && 'true' || 'false' }}" files: | - image/* - windows/** matrix.yml - .github/actions/build-linux-image/* - .github/actions/build-windows-image/* - .github/actions/image-matrix/* - .github/actions/test-windows-image/* - .github/workflows/test.yml - .github/workflows/build-test-and-push-linux-image.yml - .github/workflows/build-test-and-push-windows-image.yml + windows/** + image/.devcontainer/* + ${{ inputs.file_patterns }} - name: Report changes if: inputs.full_matrix != 'true' diff --git a/.github/workflows/build-all-rapids-repos.yml b/.github/workflows/build-all-rapids-repos.yml index 06b3ac06..42646176 100644 --- a/.github/workflows/build-all-rapids-repos.yml +++ b/.github/workflows/build-all-rapids-repos.yml @@ -23,11 +23,17 @@ jobs: || echo "ok=false" | tee -a $GITHUB_OUTPUT; build-all-rapids-repos: - name: Build all RAPIDS repos + name: RAPIDS if: needs.check-event.outputs.ok == 'true' needs: check-event secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.06 + permissions: + actions: read + packages: read + id-token: write + contents: read + pull-requests: read with: arch: '["amd64"]' cuda: '["12.0", "12.2"]' diff --git a/.github/workflows/build-and-test-feature.yml b/.github/workflows/build-and-test-feature.yml index 6fd8fda1..51dc8ba0 100644 --- a/.github/workflows/build-and-test-feature.yml +++ b/.github/workflows/build-and-test-feature.yml @@ -5,7 +5,8 @@ on: inputs: name: type: string - required: true + required: false + default: "test" args: type: string required: true @@ -13,7 +14,7 @@ on: jobs: test: - name: ${{ inputs.name }} (${{ matrix.arch }}) + name: ${{ format(inputs.name, matrix.arch) }} runs-on: ${{ fromJSON(github.repository != 'rapidsai/devcontainers' && '"ubuntu-latest"' || format('"linux-{0}-cpu4"', matrix.arch)) }} strategy: fail-fast: false @@ -31,7 +32,7 @@ jobs: username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }} password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }} - - name: ${{ inputs.name }} + - name: Build and test feature uses: ./.github/actions/build-and-test-feature with: args: "${{ inputs.args }}" diff --git a/.github/workflows/build-test-and-push-linux-image.yml b/.github/workflows/build-test-and-push-linux-image.yml index a2951954..727ec03b 100644 --- a/.github/workflows/build-test-and-push-linux-image.yml +++ b/.github/workflows/build-test-and-push-linux-image.yml @@ -14,7 +14,7 @@ on: push: type: string required: false - default: 'false' + default: false description: Whether to push the image features: type: string @@ -28,7 +28,7 @@ on: jobs: build-test-and-upload: - name: Build (${{ matrix.arch }}) + name: ${{ matrix.arch }} runs-on: ${{ fromJSON(github.repository != 'rapidsai/devcontainers' && '"ubuntu-latest"' || format('"linux-{0}-cpu4"', matrix.arch)) }} strategy: fail-fast: false @@ -75,7 +75,7 @@ jobs: merge-artifacts: if: inputs.push == 'true' - name: Merge image artifacts + name: Download needs: build-test-and-upload runs-on: ubuntu-latest steps: diff --git a/.github/workflows/build-test-and-push-windows-image.yml b/.github/workflows/build-test-and-push-windows-image.yml index 20729835..115c69b2 100644 --- a/.github/workflows/build-test-and-push-windows-image.yml +++ b/.github/workflows/build-test-and-push-windows-image.yml @@ -14,7 +14,7 @@ on: push: type: string required: false - default: 'false' + default: false description: Whether to push the image features: type: string @@ -28,7 +28,7 @@ on: jobs: build-test-and-push: - name: Build (${{ matrix.edition }}) + name: ${{ matrix.edition }} runs-on: ${{ matrix.runner }} strategy: fail-fast: false diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 3d06e6c2..374d7be4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,13 +1,15 @@ name: Generate feature docs concurrency: - group: generate-feature-docs-on-${{ github.event_name }}-from-${{ github.ref_name }} + group: generate-feature-docs-from-${{ github.ref_name }} cancel-in-progress: true on: push: branches: - - branch-* + - "branch-[0-9][0-9].[0-9][0-9]" + tags: + - "v[0-9][0-9].[0-9][0-9].[0-9][0-9]" jobs: docs: @@ -29,9 +31,9 @@ jobs: - name: Generate feature docs uses: devcontainers/action@v1 with: - generate-docs: "true" - publish-features: "false" - disable-repo-tagging: "true" + generate-docs: true + publish-features: false + disable-repo-tagging: true base-path-to-features: "./features/src" features-namespace: "${{ github.repository }}/features" @@ -54,7 +56,7 @@ jobs: git add */**/README.md; git commit -m 'Automated documentation update [skip ci]' || export NO_UPDATES=true; # Push updates - if [ "$NO_UPDATES" != "true" ] ; then + if [ "$NO_UPDATES" != 'true' ] ; then git push origin "$branch"; gh pr create --base "$base" --title "$message" --body "$message"; fi diff --git a/.github/workflows/release-features.yml b/.github/workflows/release-features.yml index d79bcf05..53b2e788 100644 --- a/.github/workflows/release-features.yml +++ b/.github/workflows/release-features.yml @@ -24,9 +24,9 @@ jobs: - name: Release features uses: devcontainers/action@v1 with: - generate-docs: "true" - publish-features: "true" - disable-repo-tagging: "true" + generate-docs: true + publish-features: true + disable-repo-tagging: true base-path-to-features: "./features/src" features-namespace: "${{ github.repository }}/features" env: diff --git a/.github/workflows/release-linux.yml b/.github/workflows/release-linux.yml new file mode 100644 index 00000000..a8daee82 --- /dev/null +++ b/.github/workflows/release-linux.yml @@ -0,0 +1,69 @@ +name: Release Linux + +concurrency: + group: release-linux-from-${{ github.ref_name }} + cancel-in-progress: true + +on: + workflow_dispatch: + +jobs: + + features-matrix: + name: Determine features matrix + runs-on: ubuntu-latest + outputs: + features: "${{ steps.matrix.outputs.features }}" + scenarios: "${{ steps.matrix.outputs.scenarios }}" + steps: + - name: "Checkout ${{ github.repository }}" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Copy common scripts into features + uses: ./.github/actions/copy-common-scripts + + - id: matrix + name: Determine features matrix + uses: ./.github/actions/feature-matrix + with: + full_matrix: true + + image-matrix: + name: Determine image matrix + needs: features-matrix + runs-on: ubuntu-latest + outputs: + linux: ${{ steps.matrix.outputs.linux }} + steps: + - name: Checkout ${{ github.repository }} + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - id: matrix + name: Determine image matrix + uses: ./.github/actions/image-matrix + with: + features: "${{ needs.features-matrix.outputs.features }}" + scenarios: "${{ needs.features-matrix.outputs.scenarios }}" + full_matrix: true + + release-linux: + if: needs.image-matrix.outputs.linux != '{"include":[]}' + name: ${{ matrix.name }} + needs: image-matrix + secrets: inherit + uses: ./.github/workflows/build-test-and-push-linux-image.yml + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.image-matrix.outputs.linux) }} + with: + push: true + os: "${{ matrix.os }}" + features: "${{ toJSON(matrix.features) }}" + container_env: "${{ toJSON(matrix.env) }}" + repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" diff --git a/.github/workflows/release-windows.yml b/.github/workflows/release-windows.yml index d9e3de06..3c29f756 100644 --- a/.github/workflows/release-windows.yml +++ b/.github/workflows/release-windows.yml @@ -1,7 +1,7 @@ name: Release Windows concurrency: - group: release-windows-on-${{ github.event_name }}-from-${{ github.ref_name }} + group: release-windows-from-${{ github.ref_name }} cancel-in-progress: true on: @@ -25,9 +25,10 @@ jobs: name: Determine image matrix uses: ./.github/actions/image-matrix with: - full_matrix: 'true' + full_matrix: true - build-and-push-windows-images: + release-windows: + if: needs.image-matrix.outputs.windows != '{"include":[]}' name: ${{ matrix.name }} needs: image-matrix secrets: inherit @@ -36,7 +37,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.image-matrix.outputs.windows) }} with: - push: 'true' + push: true os: "${{ matrix.os }}" features: "${{ toJSON(matrix.features) }}" container_env: "${{ toJSON(matrix.env) }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 546bab65..61495ae5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,19 +1,48 @@ name: Release concurrency: - group: release-on-${{ github.event_name }}-from-${{ github.ref_name }} + group: release-from-${{ github.ref_name }} cancel-in-progress: true on: workflow_dispatch: + push: + branches: + - "branch-[0-9][0-9].[0-9][0-9]" + tags: + - "v[0-9][0-9].[0-9][0-9].[0-9][0-9]" jobs: + features-matrix: + name: Determine features matrix + runs-on: ubuntu-latest + outputs: + features: "${{ steps.matrix.outputs.features }}" + scenarios: "${{ steps.matrix.outputs.scenarios }}" + steps: + - name: "Checkout ${{ github.repository }}" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Copy common scripts into features + uses: ./.github/actions/copy-common-scripts + + - id: matrix + name: Determine features matrix + uses: ./.github/actions/feature-matrix + with: + full_matrix: "${{ github.event_name == 'workflow_dispatch' && 'true' || 'false' }}" + image-matrix: name: Determine image matrix + needs: features-matrix runs-on: ubuntu-latest outputs: linux: ${{ steps.matrix.outputs.linux }} + windows: ${{ steps.matrix.outputs.windows }} steps: - name: Checkout ${{ github.repository }} uses: actions/checkout@v4 @@ -25,10 +54,13 @@ jobs: name: Determine image matrix uses: ./.github/actions/image-matrix with: - full_matrix: 'true' + features: "${{ needs.features-matrix.outputs.features }}" + scenarios: "${{ needs.features-matrix.outputs.scenarios }}" + full_matrix: "${{ github.event_name == 'workflow_dispatch' && 'true' || 'false' }}" - build-and-push-linux-images: - name: ${{ matrix.name }} + release-linux: + if: needs.image-matrix.outputs.linux != '{"include":[]}' + name: ${{ matrix.name || 'Linux' }} needs: image-matrix secrets: inherit uses: ./.github/workflows/build-test-and-push-linux-image.yml @@ -36,14 +68,31 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.image-matrix.outputs.linux) }} with: - push: 'true' + push: true + os: "${{ matrix.os }}" + features: "${{ toJSON(matrix.features) }}" + container_env: "${{ toJSON(matrix.env) }}" + repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" + + release-windows: + if: needs.image-matrix.outputs.windows != '{"include":[]}' + name: ${{ matrix.name || 'Windows' }} + needs: image-matrix + secrets: inherit + uses: ./.github/workflows/build-test-and-push-windows-image.yml + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.image-matrix.outputs.windows) }} + with: + push: true os: "${{ matrix.os }}" features: "${{ toJSON(matrix.features) }}" container_env: "${{ toJSON(matrix.env) }}" repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" release-features: - needs: build-and-push-linux-images - name: Release Features + if: needs.features-matrix.outputs.features != '[]' || needs.features-matrix.outputs.scenarios != '[]' + name: Features + needs: features-matrix secrets: inherit uses: ./.github/workflows/release-features.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 644b1685..c0a5fce8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,7 +1,7 @@ name: Test concurrency: - group: test-on-${{ github.event_name }}-from-${{ github.ref_name }} + group: test-from-${{ github.ref_name }} cancel-in-progress: true on: @@ -26,12 +26,32 @@ jobs: && echo "ok=true" | tee -a $GITHUB_OUTPUT \ || echo "ok=false" | tee -a $GITHUB_OUTPUT; + build-all-rapids-repos: + if: needs.check-event.outputs.ok == 'true' && github.repository_owner == 'rapidsai' + name: Build + needs: check-event + secrets: inherit + uses: ./.github/workflows/build-all-rapids-repos.yml + validate-features-json: if: needs.check-event.outputs.ok == 'true' - name: Validate features + name: Validate Features needs: check-event - secrets: inherit - uses: ./.github/workflows/validate-features-json.yml + runs-on: ubuntu-latest + steps: + - name: Checkout ${{ github.repository }} + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install devcontainers CLI + uses: ./.github/actions/install-devcontainers-cli + + - name: Validate devcontainer-feature.json files + uses: devcontainers/action@v1 + with: + validate-only: true + base-path-to-features: "./features/src" features-matrix: name: Determine features matrix @@ -47,26 +67,39 @@ jobs: fetch-depth: 0 persist-credentials: false + - id: get-pr-info + name: Get PR target branch info + uses: ./.github/actions/get-pr-info + - name: Copy common scripts into features uses: ./.github/actions/copy-common-scripts - id: matrix name: Determine features matrix uses: ./.github/actions/feature-matrix + with: + full_matrix: false + base_sha: "${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}" + file_patterns: | + features/test/** + .github/workflows/test.yml + .github/actions/feature-matrix/* + .github/actions/build-and-test-feature/* + .github/workflows/build-and-test-feature.yml run-feature-integration-tests: if: needs.features-matrix.outputs.features != '[]' || needs.features-matrix.outputs.scenarios != '[]' - name: Integration + name: Feature integration needs: features-matrix secrets: inherit uses: ./.github/workflows/build-and-test-feature.yml with: - name: "Test global scenarios" + name: "{0}" args: "--global-scenarios-only" run-feature-scenario-tests: if: needs.features-matrix.outputs.scenarios != '[]' - name: Test scenarios + name: Test ${{ matrix.feature }} scenarios needs: features-matrix secrets: inherit uses: ./.github/workflows/build-and-test-feature.yml @@ -75,12 +108,12 @@ jobs: matrix: feature: ${{ fromJSON(needs.features-matrix.outputs.scenarios) }} with: - name: "Run scenario tests" + name: "{0}" args: "-f ${{ matrix.feature }} --skip-autogenerated" run-generated-feature-tests: if: needs.features-matrix.outputs.features != '[]' - name: Test feature + name: Test ${{ matrix.feature }} feature needs: features-matrix secrets: inherit uses: ./.github/workflows/build-and-test-feature.yml @@ -89,8 +122,8 @@ jobs: matrix: feature: ${{ fromJSON(needs.features-matrix.outputs.features) }} with: - name: "Run generated tests" - args: "--skip-scenarios -f ${{ matrix.feature }} -i ubuntu:22.04" + name: "{0}" + args: "-f ${{ matrix.feature }} -i ubuntu:22.04 --skip-scenarios" image-matrix: name: Determine image matrix @@ -106,17 +139,30 @@ jobs: fetch-depth: 0 persist-credentials: false + - id: get-pr-info + name: Get PR target branch info + uses: ./.github/actions/get-pr-info + - id: matrix name: Determine image matrix uses: ./.github/actions/image-matrix with: - full_matrix: 'false' + full_matrix: false features: "${{ needs.features-matrix.outputs.features }}" scenarios: "${{ needs.features-matrix.outputs.scenarios }}" + base_sha: "${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}" + file_patterns: | + .github/actions/build-linux-image/* + .github/actions/build-windows-image/* + .github/actions/image-matrix/* + .github/actions/test-windows-image/* + .github/workflows/test.yml + .github/workflows/build-test-and-push-linux-image.yml + .github/workflows/build-test-and-push-windows-image.yml build-and-test-linux-images: if: needs.image-matrix.outputs.linux != '{"include":[]}' - name: ${{ matrix.name }} + name: ${{ matrix.name || 'Linux' }} needs: image-matrix secrets: inherit uses: ./.github/workflows/build-test-and-push-linux-image.yml @@ -124,7 +170,7 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.image-matrix.outputs.linux) }} with: - push: 'false' + push: false os: "${{ matrix.os }}" features: "${{ toJSON(matrix.features) }}" container_env: "${{ toJSON(matrix.env) }}" @@ -132,7 +178,7 @@ jobs: build-and-test-windows-images: if: needs.image-matrix.outputs.windows != '{"include":[]}' - name: ${{ matrix.name }} + name: ${{ matrix.name || 'Windows' }} needs: image-matrix secrets: inherit uses: ./.github/workflows/build-test-and-push-windows-image.yml @@ -140,15 +186,8 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.image-matrix.outputs.windows) }} with: - push: 'false' + push: false os: "${{ matrix.os }}" features: "${{ toJSON(matrix.features) }}" container_env: "${{ toJSON(matrix.env) }}" repo: "${{ vars.DOCKERHUB_REPOSITORY || github.repository }}" - - build-all-rapids-repos: - name: Build all RAPIDS repos - if: needs.check-event.outputs.ok == 'true' - needs: check-event - secrets: inherit - uses: ./.github/workflows/build-all-rapids-repos.yml diff --git a/.github/workflows/validate-features-json.yml b/.github/workflows/validate-features-json.yml deleted file mode 100644 index d74b34ea..00000000 --- a/.github/workflows/validate-features-json.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Validate features - -on: - workflow_call: - -jobs: - validate: - name: Validate features - runs-on: ubuntu-latest - steps: - - name: Checkout ${{ github.repository }} - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Install devcontainers CLI - uses: ./.github/actions/install-devcontainers-cli - - - name: Validate devcontainer-feature.json files - uses: devcontainers/action@v1 - with: - validate-only: "true" - base-path-to-features: "./features/src"