feat CI: consistent names and cache system for all OS, use gold for g++

------------------------ Note: by creating a PR or an issue you automatically agree to the CLA. See [CONTRIBUTING.md](https://github.com/userver-framework/userver/blob/develop/CONTRIBUTING.md). Feel free to remove this note, the agreement holds. Tests: протестировано CI Pull Request resolved: #820 Co-authored-by: fdr400 <[email protected]> commit_hash:50991c262fd479cefad3b2a5f7ee88bdb0fd45c6
userver-framework · Jan 6, 2025 · 65806bf · 65806bf
1 parent 19324d9
commit 65806bf
Show file tree

Hide file tree

Showing 7 changed files with 215 additions and 67 deletions.
diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml
@@ -1,4 +1,4 @@
-name: Alpine Linux
+name: Alpine
 
 'on':
     pull_request:
@@ -13,29 +13,53 @@ env:
     ASAN_OPTIONS: detect_odr_violation=2
     CXX: clang++-18
     CC: clang-18
+    CCACHE_DIR: /home/runner/.cache/ccache
+    CCACHE_NOHASHDIR: true
+    CPM_SOURCE_CACHE: /home/runner/.cache/CPM
 
 jobs:
     posix:
         strategy:
             fail-fast: false
 
-        name: Build only
+        name: ubuntu-24.04 (build-only)
         runs-on: ubuntu-24.04
 
         steps:
           - uses: actions/checkout@v4
             with:
                 fetch-depth: 0
 
+          - name: Restore cached directories
+            id: restore-cache
+            uses: actions/cache/restore@v4
+            with:
+                path: |
+                    ${{env.CCACHE_DIR}}
+                    ${{env.CPM_SOURCE_CACHE}}
+                key: 'alpine-cache-dir ${{github.ref}}'
+                restore-keys: |
+                    alpine-cache-dir ${{github.ref}}
+                    alpine-cache-dir
+
+          - name: Setup host cache dirs
+            run: |
+                mkdir -p ${{env.CCACHE_DIR}}
+                mkdir -p ${{env.CPM_SOURCE_CACHE}}
+
           - name: Setup latest Alpine Linux
             uses: jirutka/setup-alpine@v1
+            with:
+                volumes: |
+                    ${{env.CCACHE_DIR}}:${{env.CCACHE_DIR}}
+                    ${{env.CPM_SOURCE_CACHE}}:${{env.CPM_SOURCE_CACHE}}
 
           - name: Install dependencies
             shell: alpine.sh --root {0}
             run: |
                 pwd
                 cat /etc/alpine-release
-                apk add lld
+                apk add lld ccache git
                 apk add $(cat scripts/docs/en/deps/alpine.md)
 
           - name: Install test dependencies
@@ -45,16 +69,29 @@ jobs:
                         redis \
                         rabbitmq-server \
 
+          - name: Setup caches
+            shell: alpine.sh --root {0}
+            run: |
+                echo "Cached CPM packages:"
+                du -h -d 1 ${{env.CPM_SOURCE_CACHE}}
+                for f in $(find ${{env.CPM_SOURCE_CACHE}} -name "cmake.lock");
+                do
+                  repo=$(ls -d $(dirname $f)/*/);
+                  echo "Repository: $repo";
+                  git config --global --add safe.directory $repo;
+                done
+
+                ccache -M 2.0GB
+                ccache -s -v
 
           - name: Run cmake
-            shell: alpine.sh {0}
+            shell: alpine.sh --root {0}
             run: |
                 pwd
-                mkdir build_debug
-                cd build_debug
                 CMAKE_PROGRAM_PATH=/usr/lib/llvm17/bin/ \
-                cmake \
+                cmake -S . -B build_debug \
                     -DCMAKE_BUILD_TYPE=Debug \
+                    -DUSERVER_USE_LD=lld \
                     -DUSERVER_NO_WERROR=OFF \
                     -DUSERVER_BUILD_ALL_COMPONENTS=1 \
                     -DUSERVER_BUILD_SAMPLES=1 \
@@ -69,11 +106,23 @@ jobs:
                     -DUSERVER_FORCE_DOWNLOAD_ABSEIL=1 \
                     -DUSERVER_FORCE_DOWNLOAD_PROTOBUF=1 \
                     -DUSERVER_FORCE_DOWNLOAD_GRPC=1 \
-                    -DUSERVER_USE_LD=lld \
-                    ..
 
           - name: Compile
+            shell: alpine.sh --root {0}
+            run: |
+                cmake --build build_debug -j$(nproc)
+
+          - name: Save cached directories
+            uses: actions/cache/save@v4
+            with:
+                path: |
+                    ${{env.CCACHE_DIR}}
+                    ${{env.CPM_SOURCE_CACHE}}
+                key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+
+          - name: Show cache stats
             shell: alpine.sh {0}
             run: |
-                cd build_debug
-                cmake --build . -j$(nproc)
+                du -h -d 1 ${{env.CCACHE_DIR}}
+                du -h -d 1 ${{env.CPM_SOURCE_CACHE}}
+                ccache -s -v
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: CI
+name: Ubuntu
 
 'on':
     pull_request:
@@ -11,6 +11,9 @@ name: CI
 env:
     UBSAN_OPTIONS: print_stacktrace=1
     ASAN_OPTIONS: detect_odr_violation=2
+    CCACHE_DIR: /home/runner/.cache/ccache
+    CCACHE_NOHASHDIR: true
+    CPM_SOURCE_CACHE: /home/runner/.cache/CPM
 
 jobs:
     posix:
@@ -34,8 +37,8 @@ jobs:
                         -DUSERVER_NAMESPACE_END="}}"
                     os: ubuntu-24.04
                     info: clang-18 + debug + sanitize addr+ub
+                    id: ubuntu24_clang18_debug_sanitize-addr-ub
                     tests-flags: ''
-                    ccache-dir: ~/.cache/ccache
 
                   - cmake-flags: >-
                         -GNinja
@@ -53,12 +56,13 @@ jobs:
                         -DUSERVER_NAMESPACE_END="}}"
                     os: ubuntu-22.04
                     info: clang-14 + debug + sanitize addr+ub
+                    id: ubuntu22_clang14_debug_sanitize-addr-ub
                     tests-flags: ''
-                    ccache-dir: ~/.cache/ccache
 
                   - cmake-flags: >-
                         -GNinja
                         -DCMAKE_BUILD_TYPE=Release
+                        -DUSERVER_USE_LD=gold
                         -DUSERVER_BUILD_ALL_COMPONENTS=1
                         -DUSERVER_BUILD_SAMPLES=1
                         -DUSERVER_BUILD_TESTS=1
@@ -69,15 +73,16 @@ jobs:
                         -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0
                     os: ubuntu-20.04
                     info: g++-9 + release
+                    id: ubuntu20_gcc9_release
                     tests-flags: '--gtest_filter=-HttpClient.RedirectHeaders:HttpClient.TestUseIPv4v6'
-                    ccache-dir: ~/.ccache
 
                   # GCC-8 - testing minimal required compiler
                   - cmake-flags: >-
                         -GNinja
                         -DCMAKE_CXX_COMPILER=g++-8
                         -DCMAKE_C_COMPILER=gcc-8
                         -DCMAKE_BUILD_TYPE=Debug
+                        -DUSERVER_USE_LD=gold
                         -DUSERVER_BUILD_ALL_COMPONENTS=1
                         -DUSERVER_BUILD_SAMPLES=1
                         -DUSERVER_BUILD_TESTS=1
@@ -88,23 +93,26 @@ jobs:
                         -DUSERVER_DOWNLOAD_PACKAGE_ABSEIL=0
                     os: ubuntu-20.04
                     info: g++-8 + debug
+                    id: ubuntu20_gcc8_debug
                     tests-flags: '--gtest_filter=-HttpClient.RedirectHeaders:HttpClient.TestUseIPv4v6'
-                    ccache-dir: ~/.ccache
 
         name: '${{matrix.os}}: ${{matrix.info}}'
         runs-on: ${{matrix.os}}
 
         steps:
           - uses: actions/checkout@v4
 
-          - name: Reuse ccache directory
-            uses: actions/cache@v4
+          - name: Restore cached directories
+            id: restore-cache
+            uses: actions/cache/restore@v4
             with:
-                path: '${{matrix.ccache-dir}}'
-                key: '${{matrix.os}} ${{matrix.info}} ccache-dir ${{github.ref}} run-${{github.run_number}}'
+                path: |
+                    ${{env.CCACHE_DIR}}
+                    ${{env.CPM_SOURCE_CACHE}}
+                key: 'ubuntu-cache-dir ${{matrix.id}} ${{github.ref}}'
                 restore-keys: |
-                    ${{matrix.os}} ${{matrix.info}} ccache-dir ${{github.ref}} run-'
-                    ${{matrix.os}} ${{matrix.info}} ccache-
+                    ubuntu-cache-dir ${{matrix.id}} ${{github.ref}}
+                    ubuntu-cache-dir ${{matrix.id}}
 
           - name: Setup ramdrive for testsuites
             run: |
@@ -146,18 +154,10 @@ jobs:
                 wget https://apt.llvm.org/llvm.sh
                 chmod +x llvm.sh
                 sudo ./llvm.sh 14
-                sudo apt install clang-format-14
+                sudo apt install -y clang-format-14
                 sudo ln -sf /usr/bin/clang-format-14 /usr/bin/clang-format
                 clang-format --version
 
-          - name: Install grpc, protobuf and abseil
-            if: matrix.os == 'ubuntu-20.04'
-            run: |
-                git clone --recursive -b v1.45.0 https://github.com/grpc/grpc new_grpc
-                cmake -S./new_grpc -B./new_grpc/build -DABSL_PROPAGATE_CXX_STD=ON -DABSL_ENABLE_INSTALL=ON
-                cmake --build new_grpc/build -- -j$(nproc)
-                sudo cmake --install new_grpc/build
-
           - name: Add missing repositories
             run: |
                 sudo apt update
@@ -192,9 +192,37 @@ jobs:
           - name: Install common deps
             run: |
                 sudo apt update
-                sudo apt purge libpq5 libpq-dev postgresql-*
+                sudo apt purge -y libpq5 libpq-dev postgresql-*
                 sudo apt install --allow-downgrades -y clickhouse-common-static redis-server postgresql $(cat scripts/docs/en/deps/${{matrix.os}}.md | tr '\n' ' ')
 
+          - name: Install ccache
+            if: matrix.os == 'ubuntu-20.04'
+            run: |
+                sudo apt remove --purge -y ccache
+                git clone --depth 1 -b v4.10.2 https://github.com/ccache/ccache.git
+                cd ccache
+                cmake -S . -B build -GNinja -DENABLE_TESTING=OFF -DSTATIC_LINK=ON -DCMAKE_BUILD_TYPE=Release
+                cmake --build build -j$(nproc)
+                sudo cmake --install build
+
+          - name: Setup ccache
+            run: |
+                ccache -M 2.0GB
+                ccache -s -v
+
+          - name: Install grpc, protobuf and abseil
+            if: matrix.os == 'ubuntu-20.04'
+            run: |
+                sudo apt install -y ninja-build
+                git clone --recursive --depth 1 -b v1.45.0 https://github.com/grpc/grpc new_grpc
+                GRPC_BUILD_ENABLE_CCACHE=true \
+                  cmake -GNinja -S./new_grpc -B./new_grpc/build \
+                  -DABSL_PROPAGATE_CXX_STD=ON -DABSL_ENABLE_INSTALL=ON
+                cmake --build new_grpc/build -j$(nproc)
+                sudo cmake --install new_grpc/build
+                rm -rf new_grpc
+                ccache -s -v
+
           - name: Install g++-8
             if: matrix.info == 'g++-8 + debug'
             run: |
@@ -208,16 +236,9 @@ jobs:
             run: |
                 sudo apt install -y gdb python3-pytest
 
-          - name: Setup ccache
-            run: |
-                ccache -M 2.0GB
-                ccache -s
-
           - name: Run cmake
             run: |
-                mkdir build_debug
-                cd build_debug
-                cmake ${{matrix.cmake-flags}} ..
+                cmake -S . -B build_debug ${{matrix.cmake-flags}}
 
           - name: Run clang-tidy
             if: matrix.info == 'clang-14 + debug + sanitize addr+ub'
@@ -242,6 +263,21 @@ jobs:
                 cd build_debug
                 cmake --build . -j $(nproc)
 
+          - name: Save cached directories
+            uses: actions/cache/save@v4
+            with:
+                path: |
+                    ${{env.CCACHE_DIR}}
+                    ${{env.CPM_SOURCE_CACHE}}
+                key: ${{ steps.restore-cache.outputs.cache-primary-key }}
+
+          - name: Show cache stats
+            run: |
+                du -h -d 1 ${{env.CCACHE_DIR}}
+                du -h -d 1 ${{env.CPM_SOURCE_CACHE}}
+                ccache -s -v
+
+
           - name: Run tests (universal)
             run: |
                 echo "UBSAN_OPTIONS=${UBSAN_OPTIONS} ASAN_OPTIONS=${ASAN_OPTIONS}"
@@ -290,7 +326,7 @@ jobs:
             # libbson unnecessarily overaligns structures and GCC-9 does not support overaligned new
             if: matrix.info != 'g++-9 + release'
             run: |
-                sudo apt install mongodb-mongosh
+                sudo apt install -y mongodb-mongosh
                 cd build_debug
                 mkdir mongo || :
                 cd mongo
@@ -311,7 +347,7 @@ jobs:
             # MySQL benchmarks do not work on new Ubuntu
             if: matrix.os != 'ubuntu-24.04'
             run: |
-                sudo apt install mariadb-server
+                sudo apt install -y mariadb-server
                 cd build_debug
                 mkdir mysql || :
                 cd mysql