From 282af6d80effe2b9bd665cd3b66327acc01bc607 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Fri, 13 Dec 2024 13:39:01 +0000 Subject: [PATCH 01/14] feat: add release-please workflow --- .github/workflows/release-please.yaml | 222 ++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100644 .github/workflows/release-please.yaml diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml new file mode 100644 index 0000000..8f820f4 --- /dev/null +++ b/.github/workflows/release-please.yaml @@ -0,0 +1,222 @@ +name: Release-please + +on: + workflow_call: + secrets: + gh_token: + description: 'GitHub token' + required: true + cargo_registry_token: + description: 'Cargo registry token' + required: true + slack_webhook: + description: 'Slack webhook for notifications' + required: true + inputs: + manifest: + type: string + description: 'Path to release-please manifest file' + required: true + config: + type: string + description: 'Path to release-please configuration file' + required: true + run_build: + type: boolean + description: 'Build the workspace before release' + required: false + default: true + run_tests: + type: boolean + description: 'Run tests before release' + required: false + default: false + concurrency-suffix: + type: string + description: 'Suffix to use for release-pr job concurrency.' + required: false + org-owner: + type: string + description: 'Organization to add as owner of the crates.' + required: false + default: 'github:matter-labs:crates-io' + git-user-name: + type: string + description: 'Name of the user to use for git operations.' + required: false + default: 'zksync-era-bot' + git-user-email: + type: string + description: 'Email of the user to use for git operations.' + required: false + default: 'zksync-era-bot@users.noreply.github.com' + +jobs: + + release-please: + runs-on: ubuntu-latest + outputs: + prs_created: ${{ steps.release.outputs.prs_created }} + releases_created: ${{ steps.release.outputs.releases_created }} + releases: ${{ steps.release.outputs.releases }} + prs: ${{ steps.release.outputs.prs }} + paths_released: ${{ steps.release.outputs.paths_released }} + steps: + + - name: Run release-please + id: release + uses: googleapis/release-please-action@v4 + with: + token: ${{ secrets.gh_token }} + config-file: ${{ inputs.config }} + manifest-file: ${{ inputs.manifest }} + + - name: Send Slack notification if release failed + if: failure() + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook: ${{ secrets.slack_webhook }} + webhook-type: incoming-webhook + payload: | + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "*🚨 GitHub Release Failed!*" + - type: "section" + fields: + - type: "mrkdwn" + text: "*Repository:*\n`${{ github.repository }}`" + - type: "mrkdwn" + text: "*Workflow:*\n`${{ github.workflow }}`" + - type: "mrkdwn" + text: "*Branch:*\n`${{ github.ref_name }}`" + - type: "mrkdwn" + text: "*Triggered By:*\n`${{ github.actor }}`" + - type: "section" + text: + type: "mrkdwn" + text: "You can view the detailed logs and troubleshoot the issue by visiting the link below:" + - type: "actions" + elements: + - type: "button" + text: + type: "plain_text" + text: "View Workflow Logs" + emoji: true + url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + style: "danger" + + + update-cargo-lock: + name: ${{ matrix.pr_branch }} + runs-on: ubuntu-latest + needs: release-please + if: ${{ needs.release-please.outputs.prs_created == 'true' }} + strategy: + matrix: + pr_branch: ${{ fromJson(needs.release-please.outputs.prs).*.headBranchName }} + fail-fast: false + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ matrix.pr_branch }} + token: ${{ secrets.gh_token }} + submodules: "recursive" + + # It will automatically check for rust-toolchain file in the repository + # and take care of the proper caching to speed up CI. + - name: Install Rust toolchain + uses: moonrepo/setup-rust@v1 + + - name: Run cargo check + shell: 'bash -ex {0}' + id: cargo-check + run: | + BRANCH="${{ matrix.pr_branch }}" + COMPONENT="${BRANCH##*components--}" + COMPONENT_PATH=$(cat ${{ inputs.config }} | jq -r --arg name "${COMPONENT}" '.packages[$name].path') + ( cd "${COMPONENT_PATH}" && cargo update --workspace ) + if ! git diff --exit-code --quiet; then + echo "needs_update=true" >> "${GITHUB_OUTPUT}" + fi + + - name: Update Cargo.lock + shell: 'bash -ex {0}' + if: steps.cargo-check.outputs.needs_update == 'true' + run: | + git config user.name "${{ inputs.git-user-name }}" + git config user.email "${{ inputs.git-user-email }}" + git add . + git commit -m "chore: update Cargo.lock" + git push --set-upstream origin ${{ matrix.pr_branch }} + + + release-crates: + needs: release-please + name: Crates Release + runs-on: ubuntu-latest + if: ${{ needs.release-please.outputs.releases_created == 'true' }} + strategy: + matrix: + path: ${{ fromJson(needs.release-please.outputs.paths_released) }} + fail-fast: false + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.gh_token }} + submodules: "recursive" + + # It will automatically check for rust-toolchain file in the repository + # and take care of the proper caching to speed up CI. + - name: Install Rust toolchain + uses: moonrepo/setup-rust@v1 + with: + bins: 'cargo-workspaces' + + - name: Build the workspace before release + shell: 'bash -ex {0}' + if: ${{ inputs.run_build == true || inputs.run_build == 'true' }} + working-directory: ${{ matrix.path }} + run: cargo build + + - name: Run tests before release + shell: 'bash -ex {0}' + if: ${{ inputs.run_tests == true || inputs.run_tests == 'true' }} + working-directory: ${{ matrix.path }} + run: cargo test + + # Here should go the step to release packages to crates.io + - name: Release packages to crates.io + shell: 'bash -ex {0}' + working-directory: ${{ matrix.path }} + run: | + echo "${{ matrix.path }}" + cargo login ${{ secrets.cargo_registry_token }} + cargo workspaces publish --from-git + + # - name: Update ownership + # shell: 'bash -ex {0}' + # if: success() && inputs.org-owner != '' + # working-directory: ${{ matrix.path }} + # run: | + # ORG_OWNER=${{ inputs.org-owner }} + # cargo login ${{ secrets.cargo_registry_token }} + # for PKG in $(echo '${{ steps.release-plz.outputs.releases }}' | jq -r '.[].package_name'); do + # if cargo owner --list --quiet "${PKG}" 2>/dev/null | grep -q "${ORG_OWNER}"; then + # echo "Owner ${ORG_OWNER} already exists for package ${PKG}." + # elif cargo owner --list --quiet "${PKG}" 2>/dev/null; then + # echo "Adding owner ${ORG_OWNER} to package ${PKG}." + # cargo owner --add "${ORG_OWNER}" "${PKG}" + # else + # echo "Package ${PKG} does not exist on crates.io. Skipping." + # fi + # done + + # TODO: create job to update other workspaces + # after successful release of the packages + # update-other-workspaces: From e4cf3c2cb9f245163ffe29d29926a7f3195d103d Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Mon, 16 Dec 2024 17:44:46 +0000 Subject: [PATCH 02/14] update with slack notifications --- .../actions/slack-notify-release/action.yaml | 47 ++++++++ .github/workflows/release-please.yaml | 103 ++++++------------ 2 files changed, 82 insertions(+), 68 deletions(-) create mode 100644 .github/actions/slack-notify-release/action.yaml diff --git a/.github/actions/slack-notify-release/action.yaml b/.github/actions/slack-notify-release/action.yaml new file mode 100644 index 0000000..fedd45f --- /dev/null +++ b/.github/actions/slack-notify-release/action.yaml @@ -0,0 +1,47 @@ +name: 'Slack notification for failed workflow' + +description: 'Send a Slack notification for failed workflow.' + +inputs: + webhook: + description: 'Slack Incoming Webhook URL' + required: true + +runs: + using: composite + steps: + - name: Slack failure notification + if: failure() + uses: slackapi/slack-github-action@v2.0.0 + with: + webhook: ${{ inputs.webhook }} + webhook-type: incoming-webhook + payload: | + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "*🚨 GitHub Workflow Failed!*" + - type: "section" + fields: + - type: "mrkdwn" + text: "*Repository:*\n`${{ github.repository }}`" + - type: "mrkdwn" + text: "*Workflow:*\n`${{ github.workflow }}`" + - type: "mrkdwn" + text: "*Branch:*\n`${{ github.ref_name }}`" + - type: "mrkdwn" + text: "*Triggered By:*\n`${{ github.actor }}`" + - type: "section" + text: + type: "mrkdwn" + text: "You can view the detailed logs and troubleshoot the issue by visiting the link below:" + - type: "actions" + elements: + - type: "button" + text: + type: "plain_text" + text: "View Workflow Logs" + emoji: true + url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + style: "danger" diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 8f820f4..ffe4293 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -31,10 +31,16 @@ on: description: 'Run tests before release' required: false default: false - concurrency-suffix: - type: string - description: 'Suffix to use for release-pr job concurrency.' + publish-to-crates-io: + type: boolean + description: 'Whether to publish to crates.io' + required: false + default: false + update-cargo-lock: + type: boolean + description: 'Whether to update Cargo.lock' required: false + default: true org-owner: type: string description: 'Organization to add as owner of the crates.' @@ -71,48 +77,12 @@ jobs: config-file: ${{ inputs.config }} manifest-file: ${{ inputs.manifest }} - - name: Send Slack notification if release failed - if: failure() - uses: slackapi/slack-github-action@v2.0.0 - with: - webhook: ${{ secrets.slack_webhook }} - webhook-type: incoming-webhook - payload: | - blocks: - - type: "section" - text: - type: "mrkdwn" - text: "*🚨 GitHub Release Failed!*" - - type: "section" - fields: - - type: "mrkdwn" - text: "*Repository:*\n`${{ github.repository }}`" - - type: "mrkdwn" - text: "*Workflow:*\n`${{ github.workflow }}`" - - type: "mrkdwn" - text: "*Branch:*\n`${{ github.ref_name }}`" - - type: "mrkdwn" - text: "*Triggered By:*\n`${{ github.actor }}`" - - type: "section" - text: - type: "mrkdwn" - text: "You can view the detailed logs and troubleshoot the issue by visiting the link below:" - - type: "actions" - elements: - - type: "button" - text: - type: "plain_text" - text: "View Workflow Logs" - emoji: true - url: "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - style: "danger" - update-cargo-lock: - name: ${{ matrix.pr_branch }} + name: Update Cargo.lock runs-on: ubuntu-latest needs: release-please - if: ${{ needs.release-please.outputs.prs_created == 'true' }} + if: ${{ needs.release-please.outputs.prs_created == 'true' && inputs.update-cargo-lock }} strategy: matrix: pr_branch: ${{ fromJson(needs.release-please.outputs.prs).*.headBranchName }} @@ -154,11 +124,11 @@ jobs: git push --set-upstream origin ${{ matrix.pr_branch }} - release-crates: + publish-crates: needs: release-please - name: Crates Release + name: Publish to crates.io runs-on: ubuntu-latest - if: ${{ needs.release-please.outputs.releases_created == 'true' }} + if: ${{ needs.release-please.outputs.releases_created == 'true' && inputs.publish-to-crates-io }} strategy: matrix: path: ${{ fromJson(needs.release-please.outputs.paths_released) }} @@ -167,7 +137,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - fetch-depth: 0 token: ${{ secrets.gh_token }} submodules: "recursive" @@ -190,33 +159,31 @@ jobs: working-directory: ${{ matrix.path }} run: cargo test - # Here should go the step to release packages to crates.io - name: Release packages to crates.io shell: 'bash -ex {0}' working-directory: ${{ matrix.path }} run: | - echo "${{ matrix.path }}" cargo login ${{ secrets.cargo_registry_token }} cargo workspaces publish --from-git - # - name: Update ownership - # shell: 'bash -ex {0}' - # if: success() && inputs.org-owner != '' - # working-directory: ${{ matrix.path }} - # run: | - # ORG_OWNER=${{ inputs.org-owner }} - # cargo login ${{ secrets.cargo_registry_token }} - # for PKG in $(echo '${{ steps.release-plz.outputs.releases }}' | jq -r '.[].package_name'); do - # if cargo owner --list --quiet "${PKG}" 2>/dev/null | grep -q "${ORG_OWNER}"; then - # echo "Owner ${ORG_OWNER} already exists for package ${PKG}." - # elif cargo owner --list --quiet "${PKG}" 2>/dev/null; then - # echo "Adding owner ${ORG_OWNER} to package ${PKG}." - # cargo owner --add "${ORG_OWNER}" "${PKG}" - # else - # echo "Package ${PKG} does not exist on crates.io. Skipping." - # fi - # done - - # TODO: create job to update other workspaces - # after successful release of the packages - # update-other-workspaces: + - name: Update ownership + shell: 'bash -ex {0}' + if: success() && inputs.org-owner != '' + working-directory: ${{ matrix.path }} + run: | + ORG_OWNER=${{ inputs.org-owner }} + cargo login ${{ secrets.cargo_registry_token }} + for PKG in $(cargo ws list); do + cargo owner --list --quiet ${PKG} | grep ${ORG_OWNER} || cargo owner --add ${ORG_OWNER} ${PKG} + done + + notify-slack-if-fail: + needs: [release-please, update-cargo-lock, publish-crates] + name: Notify Slack + runs-on: ubuntu-latest + if: failure() + steps: + - name: Send Slack notification if release failed + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + with: + webhook: ${{ secrets.slack_webhook }} From dd7d6a7a15e83a9ca2577e0b685e4f95ad9d7c5a Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Mon, 16 Dec 2024 18:00:50 +0000 Subject: [PATCH 03/14] test --- .../actions/slack-notify-release/action.yaml | 1 - .github/workflows/release-please.yaml | 72 +++++++++++-------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/.github/actions/slack-notify-release/action.yaml b/.github/actions/slack-notify-release/action.yaml index fedd45f..2321fa1 100644 --- a/.github/actions/slack-notify-release/action.yaml +++ b/.github/actions/slack-notify-release/action.yaml @@ -11,7 +11,6 @@ runs: using: composite steps: - name: Slack failure notification - if: failure() uses: slackapi/slack-github-action@v2.0.0 with: webhook: ${{ inputs.webhook }} diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index ffe4293..9e8adc4 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -31,16 +31,21 @@ on: description: 'Run tests before release' required: false default: false + workspace-dirs: + type: string + description: 'Path to the workspace directories. To be used in configurations of release-please with merged PR for different components.' + required: false + default: '.' publish-to-crates-io: - type: boolean + type: string description: 'Whether to publish to crates.io' required: false - default: false + default: 'false' update-cargo-lock: - type: boolean + type: string description: 'Whether to update Cargo.lock' required: false - default: true + default: 'true' org-owner: type: string description: 'Organization to add as owner of the crates.' @@ -56,6 +61,11 @@ on: description: 'Email of the user to use for git operations.' required: false default: 'zksync-era-bot@users.noreply.github.com' + cargo-lock-commit-message: + type: string + description: 'Commit message for Cargo.lock update.' + required: false + default: 'chore: update Cargo.lock' jobs: @@ -77,12 +87,18 @@ jobs: config-file: ${{ inputs.config }} manifest-file: ${{ inputs.manifest }} + - name: Slack notification + if: failure() + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + with: + webhook: ${{ secrets.slack_webhook }} + update-cargo-lock: name: Update Cargo.lock runs-on: ubuntu-latest needs: release-please - if: ${{ needs.release-please.outputs.prs_created == 'true' && inputs.update-cargo-lock }} + if: ${{ needs.release-please.outputs.prs_created == 'true' && inputs.update-cargo-lock == 'true' }} strategy: matrix: pr_branch: ${{ fromJson(needs.release-please.outputs.prs).*.headBranchName }} @@ -101,34 +117,39 @@ jobs: - name: Install Rust toolchain uses: moonrepo/setup-rust@v1 - - name: Run cargo check + - name: Update Cargo.lock shell: 'bash -ex {0}' - id: cargo-check run: | BRANCH="${{ matrix.pr_branch }}" - COMPONENT="${BRANCH##*components--}" - COMPONENT_PATH=$(cat ${{ inputs.config }} | jq -r --arg name "${COMPONENT}" '.packages[$name].path') - ( cd "${COMPONENT_PATH}" && cargo update --workspace ) + if [[ ${BRANCH} == *components* ]]; then + for COMPONENT in $(jq -r '.packages | keys[]' ${{ inputs.config }} ); do + ( cd "${COMPONENT}" && cargo update --workspace ) + done + else + for WORKSPACE in ${{ inputs.workspace-dirs }} ; do + ( cd "${WORKSPACE}" && cargo update --workspace ) + done + fi if ! git diff --exit-code --quiet; then - echo "needs_update=true" >> "${GITHUB_OUTPUT}" + git config user.name "${{ inputs.git-user-name }}" + git config user.email "${{ inputs.git-user-email }}" + git add . + git commit -m "${{ inputs.cargo-lock-commit-message }}" + git push --set-upstream origin ${{ matrix.pr_branch }} fi - - name: Update Cargo.lock - shell: 'bash -ex {0}' - if: steps.cargo-check.outputs.needs_update == 'true' - run: | - git config user.name "${{ inputs.git-user-name }}" - git config user.email "${{ inputs.git-user-email }}" - git add . - git commit -m "chore: update Cargo.lock" - git push --set-upstream origin ${{ matrix.pr_branch }} + - name: Slack notification + if: failure() + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + with: + webhook: ${{ secrets.slack_webhook }} publish-crates: needs: release-please name: Publish to crates.io runs-on: ubuntu-latest - if: ${{ needs.release-please.outputs.releases_created == 'true' && inputs.publish-to-crates-io }} + if: ${{ needs.release-please.outputs.releases_created == 'true' && inputs.publish-to-crates-io == 'true' }} strategy: matrix: path: ${{ fromJson(needs.release-please.outputs.paths_released) }} @@ -177,13 +198,8 @@ jobs: cargo owner --list --quiet ${PKG} | grep ${ORG_OWNER} || cargo owner --add ${ORG_OWNER} ${PKG} done - notify-slack-if-fail: - needs: [release-please, update-cargo-lock, publish-crates] - name: Notify Slack - runs-on: ubuntu-latest - if: failure() - steps: - - name: Send Slack notification if release failed + - name: Slack notification + if: failure() uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} From 85c24bf78944b5c7193b525804cb2abe3131128f Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Tue, 17 Dec 2024 15:32:57 +0000 Subject: [PATCH 04/14] feat: add support of dependencies upgrade job --- .github/workflows/release-please.yaml | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 9e8adc4..00075a3 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -46,6 +46,11 @@ on: description: 'Whether to update Cargo.lock' required: false default: 'true' + upgrade-dependencies: + type: string + description: 'Whether to upgrade dependencies' + required: false + default: 'false' org-owner: type: string description: 'Organization to add as owner of the crates.' @@ -103,6 +108,7 @@ jobs: matrix: pr_branch: ${{ fromJson(needs.release-please.outputs.prs).*.headBranchName }} fail-fast: false + max-parallel: 1 steps: - name: Checkout repository @@ -154,6 +160,7 @@ jobs: matrix: path: ${{ fromJson(needs.release-please.outputs.paths_released) }} fail-fast: false + max-parallel: 1 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -203,3 +210,56 @@ jobs: uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} + + + upgrade-published-dependencies: + needs: [release-please, publish-crates] + name: Upgrade dependencies + runs-on: ubuntu-latest + if: ${{ needs.release-please.outputs.releases_created == 'true' && inputs.upgrade-dependencies == 'true' }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{ github.repository.default_branch }} + token: ${{ secrets.gh_token }} + submodules: "recursive" + + # It will automatically check for rust-toolchain file in the repository + # and take care of the proper caching to speed up CI. + - name: Install Rust toolchain + uses: moonrepo/setup-rust@v1 + with: + bins: 'cargo-workspaces,cargo-edit' + + - name: Upgrade packages + shell: 'bash -ex {0}' + run: | + for PATH_RELEASED in $(echo ${{ needs.release-please.outputs.paths_released }} | jq -r '.[]'); do + for COMPONENT in $(jq -r '.packages | keys[]' ${{ inputs.config }} ); do + ( + cd "${COMPONENT}" + cargo metadata --manifest-path ${PATH_RELEASED}/Cargo.toml \ + --no-deps --format-version 1 | jq -r '.packages[] | "\(.name) \(.version)"' | while read -r PACKAGE VERSION; do + cargo upgrade --pinned -p ${PACKAGE}@${VERSION} + done + ) + done + done + + - name: Commit changes + shell: 'bash -ex {0}' + run: | + if ! git diff --exit-code --quiet; then + BRANCH="upgrade-crates-io-dependencies-${{ github.run_number }}" + git config user.name "${{ inputs.git-user-name }}" + git config user.email "${{ inputs.git-user-email }}" + git checkout -b ${BRANCH} + git add . + git commit -m "${{ inputs.cargo-lock-commit-message }}" + git push --set-upstream origin ${BRANCH} + fi + gh pr create --title "chore: upgrade crates.io dependencies" \ + --body "Upgrade workspace dependencies to the latest versions" \ + --base ${{ github.repository.default_branch }} \ + --head ${BRANCH} From cf8d95dffe2ec4cf50aeda1224eaefc04aff4fa5 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Tue, 17 Dec 2024 15:44:39 +0000 Subject: [PATCH 05/14] fix issues with token --- .github/workflows/release-please.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 00075a3..7c12f79 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -235,13 +235,14 @@ jobs: - name: Upgrade packages shell: 'bash -ex {0}' run: | - for PATH_RELEASED in $(echo ${{ needs.release-please.outputs.paths_released }} | jq -r '.[]'); do + set -o pipefail + for PATH_RELEASED in $(echo '${{ needs.release-please.outputs.paths_released }}' | jq -r '.[]'); do for COMPONENT in $(jq -r '.packages | keys[]' ${{ inputs.config }} ); do ( cd "${COMPONENT}" cargo metadata --manifest-path ${PATH_RELEASED}/Cargo.toml \ --no-deps --format-version 1 | jq -r '.packages[] | "\(.name) \(.version)"' | while read -r PACKAGE VERSION; do - cargo upgrade --pinned -p ${PACKAGE}@${VERSION} + cargo upgrade --pinned -p ${PACKAGE}@${VERSION} || true done ) done @@ -249,6 +250,8 @@ jobs: - name: Commit changes shell: 'bash -ex {0}' + env: + GH_TOKEN: ${{ secrets.gh_token }} run: | if ! git diff --exit-code --quiet; then BRANCH="upgrade-crates-io-dependencies-${{ github.run_number }}" @@ -263,3 +266,9 @@ jobs: --body "Upgrade workspace dependencies to the latest versions" \ --base ${{ github.repository.default_branch }} \ --head ${BRANCH} + + - name: Slack notification + if: failure() + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + with: + webhook: ${{ secrets.slack_webhook }} From b5647a36c1ac36280b36ebb6154f99e0a4cceae9 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Tue, 17 Dec 2024 16:02:12 +0000 Subject: [PATCH 06/14] try to fix quotes --- .github/actions/slack-notify-release/action.yaml | 6 +++++- .github/workflows/release-please.yaml | 12 ++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/actions/slack-notify-release/action.yaml b/.github/actions/slack-notify-release/action.yaml index 2321fa1..81291fc 100644 --- a/.github/actions/slack-notify-release/action.yaml +++ b/.github/actions/slack-notify-release/action.yaml @@ -6,6 +6,10 @@ inputs: webhook: description: 'Slack Incoming Webhook URL' required: true + context: + description: 'The context of the workflow run' + required: false + default: '' runs: using: composite @@ -20,7 +24,7 @@ runs: - type: "section" text: type: "mrkdwn" - text: "*🚨 GitHub Workflow Failed!*" + text: "*🚨 GitHub Workflow Failed: ${{ inputs.context }}!*" - type: "section" fields: - type: "mrkdwn" diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 7c12f79..92f3b95 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -97,6 +97,7 @@ jobs: uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} + context: 'Unable execute release-please' update-cargo-lock: @@ -149,6 +150,7 @@ jobs: uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} + context: 'Unable to update Cargo.lock' publish-crates: @@ -210,6 +212,7 @@ jobs: uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} + context: 'Unable to publish to crates.io' upgrade-published-dependencies: @@ -253,22 +256,23 @@ jobs: env: GH_TOKEN: ${{ secrets.gh_token }} run: | + BRANCH="upgrade-crates-io-dependencies-${{ github.run_number }}" if ! git diff --exit-code --quiet; then - BRANCH="upgrade-crates-io-dependencies-${{ github.run_number }}" git config user.name "${{ inputs.git-user-name }}" git config user.email "${{ inputs.git-user-email }}" git checkout -b ${BRANCH} git add . - git commit -m "${{ inputs.cargo-lock-commit-message }}" + git commit -m "chore: upgrade crates.io dependencies" git push --set-upstream origin ${BRANCH} fi gh pr create --title "chore: upgrade crates.io dependencies" \ --body "Upgrade workspace dependencies to the latest versions" \ - --base ${{ github.repository.default_branch }} \ - --head ${BRANCH} + --base "${{ github.repository.default_branch }}" \ + --head "${BRANCH}" - name: Slack notification if: failure() uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} + context: 'Unable to upgrade cargo dependencies' From af03cddb22e29e544eca0dc3d96d2ac236a90ba0 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Wed, 18 Dec 2024 11:14:01 +0000 Subject: [PATCH 07/14] fix: multiple components update --- .github/workflows/release-please.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 92f3b95..6550c66 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -129,9 +129,8 @@ jobs: run: | BRANCH="${{ matrix.pr_branch }}" if [[ ${BRANCH} == *components* ]]; then - for COMPONENT in $(jq -r '.packages | keys[]' ${{ inputs.config }} ); do - ( cd "${COMPONENT}" && cargo update --workspace ) - done + COMPONENT="${BRANCH##*--}" + ( cd "${COMPONENT}" && cargo update --workspace ) else for WORKSPACE in ${{ inputs.workspace-dirs }} ; do ( cd "${WORKSPACE}" && cargo update --workspace ) From 63808707e5bc887ed539dda5724c22a08ef056f1 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Wed, 18 Dec 2024 11:44:20 +0000 Subject: [PATCH 08/14] fix: use realpath for manifest --- .github/workflows/release-please.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 6550c66..9e4566c 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -241,8 +241,9 @@ jobs: for PATH_RELEASED in $(echo '${{ needs.release-please.outputs.paths_released }}' | jq -r '.[]'); do for COMPONENT in $(jq -r '.packages | keys[]' ${{ inputs.config }} ); do ( + MANIFEST=$(realpath "${PATH_RELEASED}/Cargo.toml") cd "${COMPONENT}" - cargo metadata --manifest-path ${PATH_RELEASED}/Cargo.toml \ + cargo metadata --manifest-path ${MANIFEST} \ --no-deps --format-version 1 | jq -r '.packages[] | "\(.name) \(.version)"' | while read -r PACKAGE VERSION; do cargo upgrade --pinned -p ${PACKAGE}@${VERSION} || true done From d1be81fa7b3563c4ceece7c7b56b5fcdb1acbbdc Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Wed, 18 Dec 2024 12:06:52 +0000 Subject: [PATCH 09/14] fix: create pr only if there are changes --- .github/workflows/release-please.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 9e4566c..9e895c5 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -264,11 +264,11 @@ jobs: git add . git commit -m "chore: upgrade crates.io dependencies" git push --set-upstream origin ${BRANCH} + gh pr create --title "chore: upgrade crates.io dependencies" \ + --body "Upgrade workspace dependencies to the latest versions" \ + --base "${{ github.repository.default_branch }}" \ + --head "${BRANCH}" fi - gh pr create --title "chore: upgrade crates.io dependencies" \ - --body "Upgrade workspace dependencies to the latest versions" \ - --base "${{ github.repository.default_branch }}" \ - --head "${BRANCH}" - name: Slack notification if: failure() From 76b6ab3e7c4c9163d1eccd68fa673912e8372d3f Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Wed, 18 Dec 2024 12:08:48 +0000 Subject: [PATCH 10/14] fix: default branch --- .github/workflows/release-please.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 9e895c5..508e217 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -223,7 +223,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 with: - ref: ${{ github.repository.default_branch }} + ref: ${{ github.event.repository.default_branch }} token: ${{ secrets.gh_token }} submodules: "recursive" @@ -256,7 +256,7 @@ jobs: env: GH_TOKEN: ${{ secrets.gh_token }} run: | - BRANCH="upgrade-crates-io-dependencies-${{ github.run_number }}" + BRANCH="upgrade-crates-io-dependencies-$(date +%s)" if ! git diff --exit-code --quiet; then git config user.name "${{ inputs.git-user-name }}" git config user.email "${{ inputs.git-user-email }}" @@ -266,7 +266,7 @@ jobs: git push --set-upstream origin ${BRANCH} gh pr create --title "chore: upgrade crates.io dependencies" \ --body "Upgrade workspace dependencies to the latest versions" \ - --base "${{ github.repository.default_branch }}" \ + --base "${{ github.event.repository.default_branch }}" \ --head "${BRANCH}" fi From c3af6bc9d8ccb57df6c6c7eef22110386e2ab423 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Wed, 18 Dec 2024 12:25:29 +0000 Subject: [PATCH 11/14] chore: make secrets not required --- .github/workflows/release-please.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 508e217..942314e 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -8,10 +8,10 @@ on: required: true cargo_registry_token: description: 'Cargo registry token' - required: true + required: false slack_webhook: description: 'Slack webhook for notifications' - required: true + required: false inputs: manifest: type: string From d9e993f5af97b5d067c5d7c6702933f07b9ec86e Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Wed, 18 Dec 2024 13:17:23 +0000 Subject: [PATCH 12/14] fix: properly identify component path --- .github/workflows/release-please.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 942314e..63c308a 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -1,5 +1,13 @@ name: Release-please +# This workflow automates Rust release management with release-please, including: +# - Versioning and changelog generation based on manifest and config files. +# - Optional Cargo workspace build and test steps before release. +# - Optional automatic updates to Cargo.lock for single and multi-component workspaces. +# - Publishing packages to crates.io with support for custom ownership settings. +# - Dependency upgrades with automatic PR creation for updated components across workspaces. +# - Slack notifications for release events and failure handling. + on: workflow_call: secrets: @@ -33,7 +41,7 @@ on: default: false workspace-dirs: type: string - description: 'Path to the workspace directories. To be used in configurations of release-please with merged PR for different components.' + description: 'Space-separated list of paths to the workspace directories. To be used in configurations of release-please with merged PR for different components.' required: false default: '.' publish-to-crates-io: @@ -130,7 +138,8 @@ jobs: BRANCH="${{ matrix.pr_branch }}" if [[ ${BRANCH} == *components* ]]; then COMPONENT="${BRANCH##*--}" - ( cd "${COMPONENT}" && cargo update --workspace ) + COMPONENT_PATH=$(jq -r --arg component ${COMPONENT} '.packages | to_entries[] | select(.value.component == $component) | .key' ${{ inputs.config }}) + ( cd "${COMPONENT_PATH}" && cargo update --workspace ) else for WORKSPACE in ${{ inputs.workspace-dirs }} ; do ( cd "${WORKSPACE}" && cargo update --workspace ) @@ -193,7 +202,7 @@ jobs: working-directory: ${{ matrix.path }} run: | cargo login ${{ secrets.cargo_registry_token }} - cargo workspaces publish --from-git + cargo workspaces publish --publish-as-is - name: Update ownership shell: 'bash -ex {0}' From 4498ea3ae2b8e8ba1b5f6cb17667f52587cddd13 Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Tue, 7 Jan 2025 11:15:07 +0000 Subject: [PATCH 13/14] chore: common publish-crates action and more comments --- .github/actions/publish-crates/action.yaml | 94 ++++++++++++++++++++++ .github/workflows/release-please.yaml | 87 ++++++++------------ 2 files changed, 129 insertions(+), 52 deletions(-) create mode 100644 .github/actions/publish-crates/action.yaml diff --git a/.github/actions/publish-crates/action.yaml b/.github/actions/publish-crates/action.yaml new file mode 100644 index 0000000..d4c062d --- /dev/null +++ b/.github/actions/publish-crates/action.yaml @@ -0,0 +1,94 @@ +name: 'Publish to crates.io' + +description: 'Publishes Rust workspace to crates.io' + +inputs: + workspace_path: + type: string + description: 'Path to the workspace to publish.' + default: '.' + required: false + org_owner: + type: string + description: 'Organization to add as owner of the crates.' + required: false + default: 'github:matter-labs:crates-io' + gh_token: + type: string + description: 'GitHub token to use for checking out the repository.' + required: true + run_build: + type: string + description: 'Whether to run build before release.' + required: false + default: 'true' + run_tests: + type: string + description: 'Whether to run tests before release.' + required: false + default: 'false' + cargo_registry_token: + type: string + description: 'Token to use for publishing to crates.io.' + required: true + slack_webhook: + type: string + description: 'Slack webhook to use for notifications.' + required: true + + +runs: + using: composite + steps: + + - name: Checkout repository + uses: actions/checkout@v4 + with: + token: ${{ inputs.gh_token }} + submodules: "recursive" + + - name: Install Rust toolchain + uses: moonrepo/setup-rust@v1 + with: + bins: 'cargo-workspaces' + + - name: Build the workspace before release + shell: 'bash -ex {0}' + if: ${{ inputs.run_build == true || inputs.run_build == 'true' }} + working-directory: ${{ inputs.workspace_path }} + run: cargo build + + - name: Run tests before release + shell: 'bash -ex {0}' + if: ${{ inputs.run_tests == true || inputs.run_tests == 'true' }} + working-directory: ${{ inputs.workspace_path }} + run: cargo test + + - name: Login to registry + shell: 'bash -ex {0}' + working-directory: ${{ inputs.workspace_path }} + run: cargo login ${{ inputs.cargo_registry_token }} + + - name: Release packages to crates.io + shell: 'bash -ex {0}' + working-directory: ${{ inputs.workspace_path }} + run: cargo workspaces publish --publish-as-is + + - name: Update ownership + shell: 'bash -ex {0}' + if: success() && inputs.org-owner != '' + working-directory: ${{ inputs.workspace_path }} + run: | + # Fail on error from pipe commands + set -o pipefail + ORG_OWNER=${{ inputs.org-owner }} + for PKG in $(cargo ws list); do + cargo owner --list --quiet ${PKG} | grep ${ORG_OWNER} || cargo owner --add ${ORG_OWNER} ${PKG} + done + + - name: Slack notification + if: failure() + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + with: + webhook: ${{ inputs.slack_webhook }} + context: 'Unable to publish to crates.io' diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 63c308a..f5d5f93 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -82,6 +82,10 @@ on: jobs: + # This job runs release-please to generate the release PR(s) and/or GitHub release(s) + # accordingly to config and manifest files. + # If the release or PRs are created, it will trigger the next jobs in the workflow. + # If the job fails, it will notify the Slack channel. release-please: runs-on: ubuntu-latest outputs: @@ -105,9 +109,14 @@ jobs: uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} - context: 'Unable execute release-please' + context: 'Failed to execute release-please' + # This job updates the Cargo.lock file for the workspace after a version update. + # It will trigger only if the release-please job created a new release PR for a component. + # Multiple PRs can be created depending on the release-please config and manifest files. + # If the update is successful, a new commit with the updated Cargo.lock will be pushed to the release PR. + # If the job fails, it will notify the Slack channel. update-cargo-lock: name: Update Cargo.lock runs-on: ubuntu-latest @@ -117,7 +126,6 @@ jobs: matrix: pr_branch: ${{ fromJson(needs.release-please.outputs.prs).*.headBranchName }} fail-fast: false - max-parallel: 1 steps: - name: Checkout repository @@ -136,15 +144,20 @@ jobs: shell: 'bash -ex {0}' run: | BRANCH="${{ matrix.pr_branch }}" + # Check if the PR branch is a component branch + # and update the Cargo.lock for the corresponding component. if [[ ${BRANCH} == *components* ]]; then COMPONENT="${BRANCH##*--}" COMPONENT_PATH=$(jq -r --arg component ${COMPONENT} '.packages | to_entries[] | select(.value.component == $component) | .key' ${{ inputs.config }}) ( cd "${COMPONENT_PATH}" && cargo update --workspace ) else + # In case of a one PR for multiple components + # update Cargo.lock for all components using the workspace-dirs input. for WORKSPACE in ${{ inputs.workspace-dirs }} ; do ( cd "${WORKSPACE}" && cargo update --workspace ) done fi + # Commit changes to Cargo.lock if any if ! git diff --exit-code --quiet; then git config user.name "${{ inputs.git-user-name }}" git config user.email "${{ inputs.git-user-email }}" @@ -158,9 +171,13 @@ jobs: uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support with: webhook: ${{ secrets.slack_webhook }} - context: 'Unable to update Cargo.lock' + context: 'Failed to update Cargo.lock' + # This job publishes the workspace packages to crates.io after a successful release. + # It will trigger only if the release-please job created a new release for a component and publishing is enabled. + # If the publishing fails, it will notify the Slack channel with the error. + # For more details about publishing, see .github/actions/publish-crates/action.yaml file. publish-crates: needs: release-please name: Publish to crates.io @@ -170,59 +187,25 @@ jobs: matrix: path: ${{ fromJson(needs.release-please.outputs.paths_released) }} fail-fast: false - max-parallel: 1 + max-parallel: 1 # Publish workspaces one by one to prevent possible rate limiting issues steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - token: ${{ secrets.gh_token }} - submodules: "recursive" - - # It will automatically check for rust-toolchain file in the repository - # and take care of the proper caching to speed up CI. - - name: Install Rust toolchain - uses: moonrepo/setup-rust@v1 + - name: Publish crates + uses: matter-labs/zksync-ci-common/.github/actions/publish-crates@aba-release-please-support with: - bins: 'cargo-workspaces' - - - name: Build the workspace before release - shell: 'bash -ex {0}' - if: ${{ inputs.run_build == true || inputs.run_build == 'true' }} - working-directory: ${{ matrix.path }} - run: cargo build - - - name: Run tests before release - shell: 'bash -ex {0}' - if: ${{ inputs.run_tests == true || inputs.run_tests == 'true' }} - working-directory: ${{ matrix.path }} - run: cargo test - - - name: Release packages to crates.io - shell: 'bash -ex {0}' - working-directory: ${{ matrix.path }} - run: | - cargo login ${{ secrets.cargo_registry_token }} - cargo workspaces publish --publish-as-is - - - name: Update ownership - shell: 'bash -ex {0}' - if: success() && inputs.org-owner != '' - working-directory: ${{ matrix.path }} - run: | - ORG_OWNER=${{ inputs.org-owner }} - cargo login ${{ secrets.cargo_registry_token }} - for PKG in $(cargo ws list); do - cargo owner --list --quiet ${PKG} | grep ${ORG_OWNER} || cargo owner --add ${ORG_OWNER} ${PKG} - done - - - name: Slack notification - if: failure() - uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support - with: - webhook: ${{ secrets.slack_webhook }} - context: 'Unable to publish to crates.io' + workspace_path: ${{ matrix.path }} + org_owner: ${{ inputs.org-owner }} + gh_token: ${{ secrets.gh_token }} + cargo_registry_token: ${{ secrets.cargo_registry_token }} + slack_webhook: ${{ secrets.slack_webhook }} + run_build: ${{ inputs.run_build }} + run_tests: ${{ inputs.run_tests }} + # This job upgrades the workspace dependencies across different workspaces of one repository. + # It will trigger only if the release-please job created a new release and packages were released to Cargo registry. + # Dependencies upgrade can be disabled by setting the upgrade-dependencies input to false. + # If the upgrade is successful, a new PR with the updated dependencies will be created that should be tested by CI and merged. + # If the job fails, it will notify the Slack channel. upgrade-published-dependencies: needs: [release-please, publish-crates] name: Upgrade dependencies From 0067d117fcf88eb50adf1ed5c63cc31eabb53b3d Mon Sep 17 00:00:00 2001 From: Anton Baliasnikov Date: Tue, 7 Jan 2025 11:24:50 +0000 Subject: [PATCH 14/14] chore: update to stable v1 tag for releasing --- .github/actions/publish-crates/action.yaml | 2 +- .github/workflows/release-please.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/publish-crates/action.yaml b/.github/actions/publish-crates/action.yaml index d4c062d..bc5c366 100644 --- a/.github/actions/publish-crates/action.yaml +++ b/.github/actions/publish-crates/action.yaml @@ -88,7 +88,7 @@ runs: - name: Slack notification if: failure() - uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@v1 with: webhook: ${{ inputs.slack_webhook }} context: 'Unable to publish to crates.io' diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index f5d5f93..813752a 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -106,7 +106,7 @@ jobs: - name: Slack notification if: failure() - uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@v1 with: webhook: ${{ secrets.slack_webhook }} context: 'Failed to execute release-please' @@ -168,7 +168,7 @@ jobs: - name: Slack notification if: failure() - uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@v1 with: webhook: ${{ secrets.slack_webhook }} context: 'Failed to update Cargo.lock' @@ -190,7 +190,7 @@ jobs: max-parallel: 1 # Publish workspaces one by one to prevent possible rate limiting issues steps: - name: Publish crates - uses: matter-labs/zksync-ci-common/.github/actions/publish-crates@aba-release-please-support + uses: matter-labs/zksync-ci-common/.github/actions/publish-crates@v1 with: workspace_path: ${{ matrix.path }} org_owner: ${{ inputs.org-owner }} @@ -264,7 +264,7 @@ jobs: - name: Slack notification if: failure() - uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@aba-release-please-support + uses: matter-labs/zksync-ci-common/.github/actions/slack-notify-release@v1 with: webhook: ${{ secrets.slack_webhook }} context: 'Unable to upgrade cargo dependencies'