From cd15c00e46da7a4fe4e1ffd34c6d6a4819f7a264 Mon Sep 17 00:00:00 2001 From: aeweda Date: Fri, 13 Sep 2024 18:40:28 +0400 Subject: [PATCH] ci: update & refactor all actions --- .github/actions/build-helper/action.yml | 2 +- .github/actions/build-linux/action.yml | 14 +- .github/actions/download-releases/action.yml | 36 ++--- .github/actions/versions-match/action.yml | 14 +- .github/workflows/bridge.yml | 34 ++-- .github/workflows/master.yml | 161 +++++++++++-------- .github/workflows/midnight.yml | 15 +- .github/workflows/push.yml | 56 +++---- 8 files changed, 188 insertions(+), 144 deletions(-) diff --git a/.github/actions/build-helper/action.yml b/.github/actions/build-helper/action.yml index 2dde18f23..c3951c5a7 100644 --- a/.github/actions/build-helper/action.yml +++ b/.github/actions/build-helper/action.yml @@ -14,6 +14,6 @@ runs: run: docker build -t witnet-rust/${{ inputs.imagename }}:latest -f ./docker/cross-compilation/${{ inputs.imagename }}/Dockerfile ./docker/cross-compilation/ - name: Upload image - uses: ishworkh/docker-image-artifact-upload@v1 + uses: ishworkh/container-image-artifact-upload@v2.0.0 with: image: "witnet-rust/${{ inputs.imagename }}:latest" diff --git a/.github/actions/build-linux/action.yml b/.github/actions/build-linux/action.yml index 035ba3e29..0146060ac 100644 --- a/.github/actions/build-linux/action.yml +++ b/.github/actions/build-linux/action.yml @@ -11,20 +11,20 @@ runs: steps: # Download Helper Image built in the previous jobs - name: Downloading helper Image - uses: ishworkh/docker-image-artifact-download@v1 + uses: ishworkh/container-image-artifact-download@v2.0.0 with: image: "witnet-rust/${{ inputs.target }}:latest" - # Export Vars - - shell: bash + - name: Export Vars + shell: bash run: | echo PWD=`pwd` >> $GITHUB_ENV echo TARGET=${{ inputs.target }} >> $GITHUB_ENV - # Run Binary Build - - shell: bash + - name: Build Binary + shell: bash run: docker run -v `pwd`:/project:ro -v `pwd`/target:/target -v ~/.cargo:/root/.cargo -w /project -i witnet-rust/$TARGET bash -c "cargo build --release --target=$TARGET --target-dir=/target -p witnet -p witnet_toolkit" - # Show Directory listing - - shell: bash + - name: List Directory + shell: bash run: ls -lsa ./target/${{ inputs.target }}/release diff --git a/.github/actions/download-releases/action.yml b/.github/actions/download-releases/action.yml index e95faf148..0b6ca2760 100644 --- a/.github/actions/download-releases/action.yml +++ b/.github/actions/download-releases/action.yml @@ -4,20 +4,20 @@ description: "Download All releases artifacted from the previous jobs & Signs" runs: using: "composite" steps: - # Make Directory - - shell: bash + - name: Create Release Directory + shell: bash run: mkdir release - # Download Genesis Block - - shell: bash + - name: Download Gensis Block + shell: bash run: | curl "https://raw.githubusercontent.com/witnet/genesis_block/master/latest/genesis_block.json" -o genesis_block.json cp genesis_block.json release/ cp witnet.toml release/ # Download & Hash Releases - # Macos - - uses: actions/download-artifact@v3 + - name: Download MacOS Build + uses: actions/download-artifact@v4 with: name: macos-release path: all-releases/macos/ @@ -28,8 +28,8 @@ runs: cp witnet.toml ./all-releases/macos/ mv all-releases/macos/witnet_toolkit release/witnet_toolkit-x86_64-apple-darwin - # Windows - - uses: actions/download-artifact@v3 + - name: Download Windows Build + uses: actions/download-artifact@v4 with: name: windows-release path: all-releases/windows/ @@ -40,8 +40,8 @@ runs: cp witnet.toml ./all-releases/windows/ mv all-releases/windows/witnet_toolkit.exe release/witnet_toolkit-x86_64-pc-windows-msvc.exe - # x86_64 - - uses: actions/download-artifact@v3 + - name: Download x86_64 Build + uses: actions/download-artifact@v4 with: name: x86_64-release path: all-releases/x86_64/ @@ -52,8 +52,8 @@ runs: cp witnet.toml ./all-releases/x86_64/ mv all-releases/x86_64/witnet_toolkit release/witnet_toolkit-x86_64-unknown-linux-gnu - # armv7 - - uses: actions/download-artifact@v3 + - name: Download armv7 Build + uses: actions/download-artifact@v4 with: name: armv7-release path: all-releases/armv7/ @@ -64,8 +64,8 @@ runs: cp witnet.toml ./all-releases/armv7/ mv all-releases/armv7/witnet_toolkit release/witnet_toolkit-armv7-unknown-linux-gnueabihf - # aarch64 - - uses: actions/download-artifact@v3 + - name: Download aarch64 Build + uses: actions/download-artifact@v4 with: name: aarch64-release path: all-releases/aarch64/ @@ -76,8 +76,8 @@ runs: cp witnet.toml ./all-releases/aarch64/ mv all-releases/aarch64/witnet_toolkit release/witnet_toolkit-aarch64-unknown-linux-gnu - # tar Releases - - shell: bash + - name: Tar Releases + shell: bash run: | tar -czvf release/witnet-${{github.ref_name}}-x86_64-apple-darwin.tar.gz -C all-releases/macos/ . tar -czvf release/witnet-${{github.ref_name}}-x86_64-pc-windows-msvc.tar.gz -C all-releases/windows/ . @@ -85,8 +85,8 @@ runs: tar -czvf release/witnet-${{github.ref_name}}-armv7-unknown-linux-gnueabihf.tar.gz -C all-releases/armv7/ . tar -czvf release/witnet-${{github.ref_name}}-aarch64-unknown-linux-gnu.tar.gz -C all-releases/aarch64/ . - # Sign Tars - - shell: bash + - name: Sign Tars + shell: bash run: | cd release sha256sum genesis_block.json >>SHA256SUMS diff --git a/.github/actions/versions-match/action.yml b/.github/actions/versions-match/action.yml index 335748d24..a9206a634 100644 --- a/.github/actions/versions-match/action.yml +++ b/.github/actions/versions-match/action.yml @@ -11,13 +11,13 @@ runs: using: "composite" steps: # Perform the version matching - - shell: bash + - name: Verify Matching Versions + shell: bash run: | echo "Checking version of subcrate \"${{ inputs.subcrate }}\"" - MAIN_VERSION=$(sed -nE 's/version\s?=\s?"([0-9]+\.[0-9]\.[0-9])"/\1/p' ./Cargo.toml | head -1) - SUBCRATE_VERSION=$(sed -nE 's/version\s?=\s?"([0-9]+\.[0-9]\.[0-9])"/\1/p' ./${{ inputs.subcrate }}/Cargo.toml | head -1) - if [[ "$SUBCRATE_VERSION" != "$MAIN_VERSION" ]]; \ - then \ - echo "Version of subcrate \"${{ inputs.subcrate }}\" ($SUBCRATE_VERSION) does not match that of main crate ($MAIN_VERSION)"; \ - exit 10; \ + MAIN_VERSION=$(sed -nE 's/version\s?=\s?"([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?)"/\1/p' ./Cargo.toml | head -1) + SUBCRATE_VERSION=$(sed -nE 's/version\s?=\s?"([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9.]+)?)"/\1/p' ./${{ inputs.subcrate }}/Cargo.toml | head -1) + if [[ "$SUBCRATE_VERSION" != "$MAIN_VERSION" ]]; then + echo "Version of subcrate \"${{ inputs.subcrate }}\" ($SUBCRATE_VERSION) does not match that of main crate ($MAIN_VERSION)" + exit 10 fi diff --git a/.github/workflows/bridge.yml b/.github/workflows/bridge.yml index 9276cb4b1..bcf1a9f4e 100644 --- a/.github/workflows/bridge.yml +++ b/.github/workflows/bridge.yml @@ -12,13 +12,17 @@ on: required: false default: true +env: + CARGO_TERM_COLOR: always + jobs: Bridge: runs-on: ubuntu-latest environment: tags if: ${{ github.event.workflow_run.conclusion == 'success' || inputs.force }} steps: - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - name: Install Protobuf run: | @@ -33,26 +37,36 @@ jobs: run: | cp target/release/witnet-centralized-ethereum-bridge ./ - - name: Set Version + - name: Check Pre-release run: | - echo VERNUM=$(sed -nE 's/version\s?=\s?"([0-9]+\.[0-9]\.[0-9])"/\1/p' ./bridges/centralized-ethereum/Cargo.toml | head -1) >> $GITHUB_ENV + TAG=${{ github.ref_name }} + if [[ "$TAG" =~ - ]]; then + echo "prerelease=true" >> $GITHUB_ENV + else + echo "prerelease=false" >> $GITHUB_ENV + fi - - name: Build docker Image + - name: Set Version run: | - docker build -t witnet/witnet-centralized-ethereum-bridge:"$VERNUM" -f ./docker/bridge/Dockerfile . - docker build -t witnet/witnet-centralized-ethereum-bridge:latest -f ./docker/bridge/Dockerfile . - docker image ls - + echo VERNUM=$(sed -nE 's/version\s?=\s?"([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9\.]+)?)"/\1/p' ./bridges/centralized-ethereum/Cargo.toml | head -1) >> $GITHUB_ENV + - name: Login to Docker hub Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Push Image + - name: Build & Push Tag Image run: | + docker build -t witnet/witnet-centralized-ethereum-bridge:"$VERNUM" -f ./docker/bridge/Dockerfile . docker push witnet/witnet-centralized-ethereum-bridge:"$VERNUM" + + - name: Build & Push Latest if not Pre-Release + run: | + docker build -t witnet/witnet-centralized-ethereum-bridge:latest -f ./docker/bridge/Dockerfile . docker push witnet/witnet-centralized-ethereum-bridge:latest + if: env.prerelease == 'false' + Failure: runs-on: ubuntu-latest diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index d402a0e54..151399fcf 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -5,6 +5,9 @@ on: tags: - '*' +env: + CARGO_TERM_COLOR: always + jobs: #?####################################################################################################?# #? ?# @@ -14,27 +17,33 @@ jobs: aarch64: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/build-helper - name: Building Docker Image + - name: Checkout + uses: actions/checkout@v4 + + - name: Building Docker Image + uses: ./.github/actions/build-helper with: imagename: aarch64-unknown-linux-gnu armv7: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/build-helper - name: Building Docker Image + - name: Checkout + uses: actions/checkout@v4 + + - name: Building Docker Image + uses: ./.github/actions/build-helper with: imagename: armv7-unknown-linux-gnueabihf x86_64: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/build-helper - name: Building Docker Image + - name: Checkout + uses: actions/checkout@v4 + + - name: Building Docker Image + uses: ./.github/actions/build-helper with: imagename: x86_64-unknown-linux-gnu @@ -58,17 +67,16 @@ jobs: needs: [aarch64] runs-on: ubuntu-latest steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - # Download Images & Build Binary - - uses: ./.github/actions/build-linux - name: Building aarch64 Binary + - name: Building aarch64 Binary + uses: ./.github/actions/build-linux with: target: aarch64-unknown-linux-gnu - # Upload Build Releases - - uses: actions/upload-artifact@v3 + - name: Upload Build + uses: actions/upload-artifact@v4 with: name: aarch64-release path: | @@ -79,17 +87,16 @@ jobs: needs: [armv7] runs-on: ubuntu-latest steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - # Download Images & Build Binary - - uses: ./.github/actions/build-linux - name: Building armv7 Binary + - name: Building armv7 Binary + uses: ./.github/actions/build-linux with: target: armv7-unknown-linux-gnueabihf - # Upload Build Releases - - uses: actions/upload-artifact@v3 + - name: Upload Build + uses: actions/upload-artifact@v4 with: name: armv7-release path: | @@ -100,17 +107,16 @@ jobs: needs: [x86_64] runs-on: ubuntu-latest steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - # Download Images & Build Binary - - uses: ./.github/actions/build-linux - name: Building x86_64 Binary + - name: Building x86_64 Binary + uses: ./.github/actions/build-linux with: target: x86_64-unknown-linux-gnu - # Upload Build Releases - - uses: actions/upload-artifact@v3 + - name: Upload Build + uses: actions/upload-artifact@v4 with: name: x86_64-release path: | @@ -122,19 +128,17 @@ jobs: needs: [macos_helper] runs-on: macos-latest steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - # Install Protobuf - name: Install Protobuf run: brew install protobuf - # MacOS Build - name: Building Macos Binary run: MACOSX_DEPLOYMENT_TARGET=10.14 OPENSSL_STATIC=1 OPENSSL_DIR="/usr/local/opt/openssl" cargo build --release -p witnet -p witnet_toolkit - # Upload Build Releases - - uses: actions/upload-artifact@v3 + - name: Upload Build + uses: actions/upload-artifact@v4 with: name: macos-release path: | @@ -146,10 +150,9 @@ jobs: needs: [windows_helper] runs-on: windows-latest steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - # Install Protobuf - name: Install Protobuf run: | Invoke-WebRequest -Uri https://github.com/protocolbuffers/protobuf/releases/download/v21.1/protoc-21.1-win64.zip -OutFile "protoc.zip" @@ -157,25 +160,22 @@ jobs: echo "C:\protoc\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append # TODO: Needs to be optimized takes wayyyy too long - # Install openssl static - name: Install openssl static run: | vcpkg.exe install openssl:x64-windows-static vcpkg.exe integrate install - # Windows Build - name: Build Windows run: | Set-Variable -Name "OPENSSL_STATIC" -Value "1" Set-Variable -Name "OPENSSL_DIR" -Value "C:\Program Files\vcpkg\installed\x64-windows-static" cargo build --release -p witnet -p witnet_toolkit - # Confirm openssl compiled statically - name: Confirm openssl compiled statically run: LDD.exe target\release\witnet.exe | Select-String -Pattern "ssl" - # Upload Build Releases - - uses: actions/upload-artifact@v3 + - name: Upload Build + uses: actions/upload-artifact@v4 with: name: windows-release path: | @@ -193,14 +193,12 @@ jobs: runs-on: ubuntu-latest environment: tags steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - # Download Releases - - uses: ./.github/actions/download-releases - name: Downloading Artifacts + - name: Downloading Artifacts + uses: ./.github/actions/download-releases - # Import GPG Key - name: Import GPG key id: import_gpg uses: crazy-max/ghaction-import-gpg@v5 @@ -208,7 +206,6 @@ jobs: gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} passphrase: ${{ secrets.GPG_PASSPHRASE }} - # Trust GPG key - name: Trust GPG key run: | gpg --no-tty --command-fd 0 --edit-key info@witnet.foundation << EOTRUST @@ -217,13 +214,14 @@ jobs: y quit EOTRUST + - name: Sign Hashes run: | cd release gpg --output SHA256SUMS.asc --default-key info@witnet.foundation --detach-sig --clearsign SHA256SUMS && rm SHA256SUMS - # Artifact Final Release Files - - uses: actions/upload-artifact@v3 + - name: Upload Build + uses: actions/upload-artifact@v4 with: name: final-release path: release/ @@ -234,48 +232,71 @@ jobs: permissions: contents: write steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - # Download Release Files - - uses: actions/download-artifact@v3 + - name: Download release artifacts + uses: actions/download-artifact@v4 with: name: final-release path: release/ - # Replace tag in RELEASE.md - - run: | - TAG=${{github.ref_name}} + - name: Inject Tag + run: | + TAG=${{ github.ref_name }} sed -i "s/LATEST_VERSION/$TAG/g" RELEASE.md - # Publish new github release - - uses: ncipollo/release-action@v1 + - name: Check Pre-Release + run: | + TAG=${{ github.ref_name }} + if [[ "$TAG" =~ - ]]; then + echo "prerelease=true" >> $GITHUB_ENV + else + echo "prerelease=false" >> $GITHUB_ENV + fi + + - name: Publish Github Release + uses: ncipollo/release-action@v1 with: - name: "Witnet-rust ${{github.ref_name}}" + name: "Witnet-rust ${{ github.ref_name }}" artifacts: "./release/*" bodyFile: "RELEASE.md" artifactErrorsFailBuild: true + prerelease: ${{ env.prerelease }} Publish: needs: [Release] runs-on: ubuntu-latest environment: tags steps: - # Checkout Repo - - uses: actions/checkout@v3 + - name: Checkout + uses: actions/checkout@v4 - name: Login to Docker Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - # Configure Buildx - - run: | + - name: Build Base + run: | docker run --rm --privileged multiarch/qemu-user-static --reset -p yes docker buildx create --name multiarch --driver docker-container --use docker buildx inspect --bootstrap # Build Docker images (Latest & TAG) - - run: docker buildx build -f docker/witnet-rust/Dockerfile --progress=plain --build-arg WITNET_VERSION=${{github.ref_name}} --platform linux/amd64,linux/arm64,linux/arm/v7 --tag witnet/witnet-rust:${{github.ref_name}} docker/witnet-rust --push --no-cache - - run: docker buildx build -f docker/witnet-rust/Dockerfile --progress=plain --build-arg WITNET_VERSION=latest --platform linux/amd64,linux/arm64,linux/arm/v7 --tag witnet/witnet-rust:latest docker/witnet-rust --push --no-cache + - name: Build TAG + run: docker buildx build -f docker/witnet-rust/Dockerfile --progress=plain --build-arg WITNET_VERSION=${{github.ref_name}} --platform linux/amd64,linux/arm64,linux/arm/v7 --tag witnet/witnet-rust:${{github.ref_name}} docker/witnet-rust --push --no-cache + + - name: Check Pre-release + run: | + TAG=${{ github.ref_name }} + if [[ "$TAG" =~ - ]]; then + echo "prerelease=true" >> $GITHUB_ENV + else + echo "prerelease=false" >> $GITHUB_ENV + fi + + - name: Build Latest if not Pre-Release + run: docker buildx build -f docker/witnet-rust/Dockerfile --progress=plain --build-arg WITNET_VERSION=latest --platform linux/amd64,linux/arm64,linux/arm/v7 --tag witnet/witnet-rust:latest docker/witnet-rust --push --no-cache + if: env.prerelease == 'false' diff --git a/.github/workflows/midnight.yml b/.github/workflows/midnight.yml index 9964b0fcf..7a0057104 100644 --- a/.github/workflows/midnight.yml +++ b/.github/workflows/midnight.yml @@ -4,16 +4,20 @@ on: schedule: - cron: '0 0 * * *' +env: + CARGO_TERM_COLOR: always + jobs: deps_audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - name: Checkout + uses: actions/checkout@v4 - name: Cargo dependencies security audit - uses: actions-rs/audit-check@v1 + uses: rustsec/audit-check@v1.4.1 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -22,13 +26,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Just command runner + uses: taiki-e/install-action@just - name: Install environment dependencies run: | sudo apt-get update -y -qq sudo apt-get install -y g++-9 cmake libcurl4-openssl-dev libelf-dev libdw-dev cmake gcc binutils-dev protobuf-compiler librocksdb-dev - just || curl -LSfs https://japaric.github.io/trust/install.sh | sh -s -- --git casey/just --target x86_64-unknown-linux-musl --to ~/.cargo/bin - name: Load persistent storage run: | diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index dc88d69c9..9dab828ff 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,9 +1,12 @@ -name: Check on every push +name: Check on every pull_request -on: [push, pull_request] +on: [pull_request] + +env: + CARGO_TERM_COLOR: always jobs: - build_ubuntu: + Build: runs-on: ubuntu-latest @@ -24,35 +27,40 @@ jobs: LD_LIBRARY_PATH: /usr/local/lib steps: - - uses: actions/checkout@v1 + - name: Checkout + uses: actions/checkout@v4 # Ensure some subcrate versions match that of the main crate - - uses: ./.github/actions/versions-match - name: Checking version of node + - name: Checking version of node + uses: ./.github/actions/versions-match with: subcrate: node - - uses: ./.github/actions/versions-match - name: Checking version of wallet + + - name: Checking version of wallet + uses: ./.github/actions/versions-match with: subcrate: wallet - - uses: ./.github/actions/versions-match - name: Checking version of toolkit + + - name: Checking version of toolkit + uses: ./.github/actions/versions-match with: subcrate: toolkit - - uses: ./.github/actions/versions-match - name: Checking version of data_structures + + - name: Checking version of data_structures + uses: ./.github/actions/versions-match with: subcrate: data_structures - - uses: ./.github/actions/versions-match - name: Checking version of bridges/centralized-ethereum + + - name: Checking version of bridges/centralized-ethereum + uses: ./.github/actions/versions-match with: subcrate: bridges/centralized-ethereum - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true - components: rustfmt, clippy + - name: Show Rust toolchain + run: rustup show + + - name: Install Just command runner + uses: taiki-e/install-action@just - name: Formatter uses: actions-rs/cargo@v1 @@ -74,16 +82,10 @@ jobs: sudo make install-shared - name: Clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all --all-targets --all-features -- ${{ env.CLIPPY_LINTS }} -A clippy::many-single-char-names + run: just clippy - name: Test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all --verbose + run: cargo test --all --verbose # This is used to ensure that Cargo.lock is up to date - name: Check for unstaged files