Major CI Improvements (#901)

* Add ccache to github actions builds * incrase cache size * Add ccache to buildInputs * Try to remove nix to build & run tests * run on all branches on push * run on all branches on push * fix image name * use other credentials ? * fix cli * Use ninja * Use sys ssl * add postgres service * temporarily change url for tests * build postgres url from env * Add macos build * Try to fix macos build * Upload cmake artifacts * add PGsql include dir * Rollback to pg 13 * Rollback to pg 13 on macos It seems to be related to update on brew package 14.5: Homebrew/homebrew-core@8060df9 * Rollback to pg 13 on macos It seems to be related to update on brew package 14.5: Homebrew/homebrew-core@8060df9 * Enable unixes * Cancel previous macos runs * Remove ccache from nix_release_builds.yml * don't use sys openssl on macos * Revert "don't use sys openssl on macos" This reverts commit 6cc934f. * fix db connection on macos * fix incldue path * Keep circleci only on main * remove appveyor.yml * run clang-format-check only on push & cancel previsou runs * Add more parameters to release workflow * fix workflow condition syntax * don't run nix_release_builds.yml on branches except manually asked for * Rename build * Add jar package workflow * Pass secrets * executable scripts * Publish to WD * update ccache keys * fix secret for publish & call publish workflow on Release * fix need dependency * fix checkout * fix checkout * reorder checkout to avoid clean * fetch ? * fix checkout * add some debug command to understand what is going on here * do not update submodule for now * add missing env var * update token * Update version numbers * print more * print more * fix empty var * update sbt version * Add script to publish jar locally * Don't use unique version number for snapshots * try to build in release instead of debug * some cleanings & version set * Revert "Don't use unique version number for snapshots" This reverts commit 2408538. * remove job to create WD PR * run linting only if updating code * print for debug * fix clang tidy condition * clang format on pull requests only * Revert "remove job to create WD PR" This reverts commit 637ad3b. * Create PR with bump of jar version * Revert "Create PR with bump of jar version" This reverts commit 1432199. * fix sigpipe error * Publish to WD on manual release * Rollback unused changes * Add documentation * Run package only manually & on call * Update build env * clean unused changes * Update after review
LedgerHQ · Sep 9, 2022 · 1842b11 · 1842b11
1 parent c949765
commit 1842b11
Show file tree

Hide file tree

Showing 26 changed files with 390 additions and 174 deletions.
diff --git a/.circleci/build_lib.sh b/.circleci/build_lib.sh
@@ -10,7 +10,7 @@ unamestr=`uname`
 export ARCH=$2
 export CMAKE=cmake
 export PATH=$PATH:~/cmake_folder/bin
-export PG_INCLUDE_DIR=`[[ "$unamestr" = "Darwin" ]] && echo -n "/usr/local/opt/postgresql/include" || echo -n "/usr/include/postgresql"`
+export PG_INCLUDE_DIR=`[[ "$unamestr" = "Darwin" ]] && echo -n "/usr/local/opt/postgresql@13/include" || echo -n "/usr/include/postgresql"`
 
 echo "Use $PG_INCLUDE_DIR for PGSQL"
 

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -242,6 +242,8 @@ workflows:
       - build_linux_release:
           <<: *default_context
           filters:
+            branches:
+              only: main
             tags:
               only: /.*/
       - build_linux_jni_release:
@@ -250,27 +252,37 @@ workflows:
             tags:
               only: /.*/
             branches:
-              only: /^(?!pull\/).*$/
+              only: main
       - build_macos_release:
           <<: *default_context
           filters:
             tags:
               only: /.*/
+            branches:
+              only: main
       - build_macos_jni_release:
           <<: *default_context
           filters:
             tags:
               only: /.*/
             branches:
-              only: /^(?!pull\/).*$/
+              only: main
       - build_linux_debug:
           <<: *default_context
+          filters:
+            branches:
+              only: main
       - build_macos_debug:
           <<: *default_context
+          filters:
+            branches:
+              only: main
       - publish_jar:
           requires:
             - build_linux_jni_release
             - build_macos_jni_release
           filters:
             tags:
-              only: /.*/
+              only: /.*/
+            branches:
+              only: main
diff --git a/.circleci/setup_macos.sh b/.circleci/setup_macos.sh
@@ -33,8 +33,8 @@ brew install --build-from-source cmake
 
 echo "========> Install PostgreSQL"
 # Install with verbose otherwise the setup may timeout the CI because it doesn't log.
-brew install --verbose postgresql
-export CPLUS_INCLUDE_PATH="/usr/local/Cellar/postgresql/12.3_4/include:$CPLUS_INCLUDE_PATH"
+brew install --verbose postgresql@13
+brew link --overwrite postgresql@13
 
 
 echo "========> Install Sqlite"

diff --git a/.github/scripts/build.sh b/.github/scripts/build.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir _build_tests
+cd _build_tests
+cmake .. -DTARGET_JNI=${BUILD_JNI-OFF} -DBUILD_TESTS=ON -DNIX_BUILD=OFF -DCCACHE=ON -DSYS_OPENSSL=ON -DOPENSSL_USE_STATIC_LIBS=TRUE -GNinja
+echo "========= Building libcore"
+cmake --build . --parallel --target  ledger-core-static
+cmake --build . --parallel --target  ledger-core
+cmake --build . --parallel
diff --git a/.github/scripts/publish_to_wd.sh b/.github/scripts/publish_to_wd.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd $GITHUB_WORKSPACE/ledger-wallet-daemon
+
+# Bump libcore version
+sed -i -E 's/(val libcore *= *\")(.*)\"/\1'$VERSION'\"/' build.sbt
+
+# Commit & push changes
+git checkout -b bump_libcore_$VERSION
+git add lib # libcore
+git config --global user.name "${GITHUB_ACTOR}"
+git config --global user.email "${GITHUB_ACTOR}@users.noreply.github.com"
+git commit -m "Bump libcore to $VERSION"
+git push --set-upstream origin bump_libcore_$VERSION
+gh pr create --title "Bump libcore to $VERSION" \
+   --body "Automatic update from libcore release" \
+   --base main \
+   --head bump_libcore_$VERSION
diff --git a/.github/scripts/test.sh b/.github/scripts/test.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd _build_tests
+ctest --output-on-failure
diff --git a/.github/workflows/clang-format-check.yml b/.github/workflows/clang-format-check.yml
@@ -1,10 +1,19 @@
 name: clang-format Check
-on: [push, pull_request]
+on:
+  pull_request:
+    paths:
+      - core/**
+      - .clang-format
+      - .github/workflows/clang-format-check.yml
 jobs:
   formatting-check:
     name: Formatting Check
     runs-on: ubuntu-latest
     steps:
+      - name: Cancel Previous Runs
+        uses: styfle/[email protected]
+        with:
+          access_token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@v2
       - name: Run clang-format style check for C/C++/Protobuf programs.
         uses: jidicula/[email protected]

diff --git a/.github/workflows/clang-tidty-review.yml b/.github/workflows/clang-tidty-review.yml
@@ -1,5 +1,10 @@
 name: "Clang-tidy Code Review"
-on: [pull_request]
+on:
+  pull_request:
+    paths:
+      - core/**
+      - .clang-tidy
+      - .github/workflows/clang-tidy-review.yml
 
 jobs:
   build:

diff --git a/.github/workflows/nix_release_builds.yml b/.github/workflows/nix_release_builds.yml
@@ -1,30 +1,28 @@
-name: "Nix Release builds"
+name: "Package libcore"
 on:
-  pull_request:
-    branches:
-      - main
-      - release/4.0.0
-      - release/4.1.0
-      - release/4.2.0
-      - develop
-  push:
-    branches:
-      - main
-      - release/4.0.0
-      - release/4.1.0
-      - release/4.2.0
-      - develop
-    tags:
-      - '*'
-  workflow_run:
-    workflows: ["Release libcore"]
-    types:
-      - completed
+  workflow_dispatch:
+    inputs:
+      release:
+        description: Create Release JAR package (instead of SNAPSHOT). Version number will be taken from CMakeLists.txt
+        required: true
+        type: boolean
+        default: false
+      version:
+        description: Custom version of jar package (applies only if `release` is true)
+        required: false
+        type: string
+  workflow_call:
+    inputs:
+      release:
+        required: false
+        type: boolean
+      version:
+        required: false
+        type: string
 jobs:
   libcore_version:
     name: Compute libcore version
     runs-on: ubuntu-latest
-    if: ${{ github.event_name != "workflow_run" || github.event.workflow_run.conclusion == 'success' }}
     outputs:
       lib_version: ${{ steps.lib_version.outputs.lib_version }}
       deploy_dynlibs: ${{ steps.lib_version.outputs.deploy_dynlibs }}
@@ -38,53 +36,25 @@ jobs:
                            # https://github.com/actions/checkout/blob/v2.3.4/README.md
       - name: Set version slug and push_to_S3 flags
         id: lib_version
-        run: bash nix/scripts/export_libcore_version.sh ${{ github.event.pull_request.head.ref }}
-  Ubuntu_jni:
-    name: Linux (with JNI)
-    needs: libcore_version
-    runs-on: ubuntu-latest
-    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/[email protected]
-      with:
-        access_token: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/[email protected]
-      with:
-        submodules: 'recursive'
-    - uses: cachix/[email protected]
-      with:
-        nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/5f746317f10f7206f1dbb8dfcfc2257b04507eee.tar.gz
-    - run: nix-build
-    - uses: actions/upload-artifact@v2
-      with:
-        name: ${{ needs.libcore_version.outputs.lib_version }}-linux-libledgercore
-        path: result/lib/libledger-core.so
-        retention-days: 30
-  MacOS_jni:
-    name: MacOS (with JNI)
-    needs: libcore_version
-    runs-on: macos-latest
-    steps:
-    - name: Cancel Previous Runs
-      uses: styfle/[email protected]
-      with:
-        access_token: ${{ secrets.GITHUB_TOKEN }}
-    - uses: actions/[email protected]
-      with:
-        submodules: 'recursive'
-    - uses: cachix/[email protected]
-      with:
-        nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/5f746317f10f7206f1dbb8dfcfc2257b04507eee.tar.gz
-    - run: nix-build
-    - uses: actions/upload-artifact@v2
-      with:
-        name: ${{ needs.libcore_version.outputs.lib_version }}-macos-libledgercore
-        path: result/lib/libledger-core.dylib
-        retention-days: 30
+        run: bash nix/scripts/export_libcore_version.sh
+        env:
+          RELEASE: ${{ inputs.release || 'false' }}
+          VERSION: ${{ inputs.version }}
+  Build_Release_JNI:
+    name: Build Release with JNI
+    uses: ./.github/workflows/tests.yml
+    secrets: inherit # pass all secrets
+    with:
+      buildJni: 'ON'
   JAR:
     name: Jar build (MacOS + Linux)
-    needs: [Ubuntu_jni, MacOS_jni, libcore_version]
+    needs: [libcore_version, Build_Release_JNI]
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/ledgerhq/lib-ledger-core-build-env/lib-ledger-core-build-env:1.0.3
+      credentials:
+        username: ${{ secrets.CI_BOT_USERNAME }}
+        password: ${{ secrets.CI_BOT_TOKEN }}
     steps:
     - name: Cancel Previous Runs
       uses: styfle/[email protected]
@@ -93,23 +63,19 @@ jobs:
     - uses: actions/[email protected]
       with:
         submodules: 'recursive'
-    - uses: cachix/[email protected]
-      with:
-        nix_path: nixpkgs=https://github.com/NixOS/nixpkgs/archive/5f746317f10f7206f1dbb8dfcfc2257b04507eee.tar.gz
     - run: mkdir -p jar_build/src/main/resources/resources/djinni_native_libs
     - name: Fetch Linux so
       uses: actions/download-artifact@v2
       with:
-        name: ${{ needs.libcore_version.outputs.lib_version }}-linux-libledgercore
+        name: linux-ubuntu-22.04-libledgercore
         path: jar_build/src/main/resources/resources/djinni_native_libs
     - name: Fetch MacOS dylib
       uses: actions/download-artifact@v2
       with:
-        name: ${{ needs.libcore_version.outputs.lib_version }}-macos-libledgercore
+        name: macos-libledgercore
         path: jar_build/src/main/resources/resources/djinni_native_libs
-    - run: nix-shell --run "bash nix/scripts/build_jar.sh" nix/libcore-jar.nix
+    - run: bash nix/scripts/build_jar.sh
       env:
-        LIB_VERSION: ${{ needs.libcore_version.outputs.lib_version }}
         GITHUB_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
         JAR_VERSION: ${{ needs.libcore_version.outputs.jar_version }}
     - uses: actions/upload-artifact@v2

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -3,20 +3,32 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: "Release version"
+        description: "Release version (tag)"
         required: true
         type: string
       releaseNote:
-        description: "Should create a Release"
+        description: "Create a Github Release note"
         required: true
         type: boolean
         default: true
+      preRelease:
+        description: "This is a pre-release"
+        required: true
+        type: boolean
+        default: false
+      draft:
+        description: "Save Release as draft"
+        required: true
+        type: boolean
+        default: false
 jobs:
   Release:
     runs-on: ubuntu-22.04
     env:
       VERSION: ${{ github.event.inputs.version }}
       RELEASE_NOTE: ${{ github.event.inputs.releaseNote }}
+      PRE_RELEASE: ${{ github.event.inputs.preRelease }}
+      DRAFT: ${{ github.event.inputs.draft }}
     steps:
     - uses: actions/[email protected]
     - name: Set version number
@@ -37,14 +49,42 @@ jobs:
         script: |
           try {
             await github.rest.repos.createRelease({
-              draft: false,
+              draft: process.env.DRAFT,
               generate_release_notes: true,
               name: process.env.RELEASE_TAG,
               owner: context.repo.owner,
-              prerelease: false,
+              prerelease: process.env.PRE_RELEASE,
               repo: context.repo.repo,
               tag_name: process.env.VERSION,
             });
           } catch (error) {
             core.setFailed(error.message);
-          }
+          }
+  Build:
+    name: Build Release with JNI
+    uses: ./.github/workflows/nix_release_builds.yml
+    secrets: inherit # pass all secrets
+    needs: [ Release ]
+    with:
+      release: true
+      version: ${{ github.event.inputs.version }}
+  Publish:
+    name: Publish to WD
+    needs: [Build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        name: Checkout libcore
+      - uses: actions/checkout@v3
+        name: Checkout wallet-daemont
+        with:
+          token: ${{ secrets.CI_BOT_TOKEN }}
+          repository: LedgerHQ/ledger-wallet-daemon
+          submodules: true
+          path: ledger-wallet-daemon
+      - name: Publish to WD
+        env:
+          GH_TOKEN: ${{ secrets.CI_BOT_TOKEN }}
+          VERSION: ${{ github.event.inputs.version }}
+        run: .github/scripts/publish_to_wd.sh
+