Backport github: enable arm CI with ubuntu-24.04-arm host (#787) (#789)

Backport #787 for LTS branch. In the previous versions, LTS package for arm was built on bare metal arm machine. It might be better to build it with GitHub Actions to integrate them for LTS release process. --------- Signed-off-by: Kentaro Hayashi <[email protected]>
fluent · Feb 6, 2025 · 3e53dcd · 3e53dcd
1 parent 9321f18
commit 3e53dcd
Show file tree

Hide file tree

Showing 7 changed files with 120 additions and 24 deletions.
diff --git a/.github/workflows/apt-arm.yml b/.github/workflows/apt-arm.yml
@@ -1,6 +1,10 @@
 name: Apt based Linux (AArch64)
 on:
-  workflow_dispatch:
+  push:
+    branches:
+      - master
+      - fluent-package-v5
+  pull_request:
 concurrency:
   group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
@@ -35,7 +39,7 @@ jobs:
             rake-job: ubuntu-noble
             rake-options: LINTIAN=no
             test-docker-image: arm64v8/ubuntu:noble
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@master
       - name: Install dependencies
@@ -44,9 +48,15 @@ jobs:
           sudo apt -V install ruby ruby-bundler ruby-serverspec
           sudo gem install bundler:2.2.9 --no-document
           sudo gem uninstall fileutils
+      - name: cache deb
+        uses: actions/cache@v4
+        id: cache-deb
+        with:
+          path: |
+            fluent-package/apt/repositories
+          key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', 'fluent-package/templates/**', 'fluent-package/debian/**', 'fluent-package/apt/**/Dockerfile') }}
       - name: Build deb with Docker
         run: |
-          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
           rake apt:build APT_TARGETS=${{ matrix.rake-job }}-arm64 ${{ matrix.rake-options }}
       - name: Upload fluent-package deb
         uses: actions/upload-artifact@master
@@ -62,6 +72,58 @@ jobs:
           docker run \
           --rm \
           --tty \
+          --env CI=true \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/apt/binstubs-test.sh
+      - name: Installation Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --rm \
+          --tty \
+          --env CI=true \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/apt/install-test.sh
+      - name: Piuparts (Install/Remove/Upgrade) Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --privileged \
+          --rm \
+          --tty \
+          --env CI=true \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/apt/piuparts-test.sh
+      - name: Serverspec Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --rm \
+          --tty \
+          --env CI=true \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/apt/serverspec-test.sh
+      - name: Confluent Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --rm \
+          --tty \
+          --env CI=true \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/apt/confluent-test.sh
+      - name: Binstubs Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --rm \
+          --tty \
+          --env CI=true \
           --volume ${PWD}:/fluentd:ro \
           ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/apt/binstubs-test.sh
diff --git a/.github/workflows/apt.yml b/.github/workflows/apt.yml
@@ -176,7 +176,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-apt-source-${{ matrix.rake-job }}
-      - uses: canonical/[email protected].1
+      - uses: canonical/[email protected].2
       - name: Run diagnostic
         run: |
           uname -a

diff --git a/.github/workflows/yum-arm.yml b/.github/workflows/yum-arm.yml
@@ -1,6 +1,10 @@
 name: Yum based Linux (AArch64)
 on:
-  workflow_dispatch:
+  push:
+    branches:
+      - master
+      - fluent-package-v5
+  pull_request:
 concurrency:
   group: ${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
@@ -14,34 +18,38 @@ jobs:
           - RockyLinux 8 aarch64
           - AlmaLinux 9 aarch64
           - Amazon Linux 2 aarch64
-          # It takes too long time on GitHub Actions.
-          #- Amazon Linux 2023 aarch64
+          - Amazon Linux 2023 aarch64
         include:
           - label: RockyLinux 8 aarch64
             rake-job: rockylinux-8
-            test-docker-image: rockylinux:8
+            test-docker-image: arm64v8/rockylinux:8
           - label: AlmaLinux 9 aarch64
             rake-job: almalinux-9
-            test-docker-image: almalinux:9
+            test-docker-image: arm64v8/almalinux:9
           - label: Amazon Linux 2 aarch64
             rake-job: amazonlinux-2
-            test-docker-image: amazonlinux:2
-          #- label: Amazon Linux 2023 aarch64
-          #  rake-job: amazonlinux-2023
-          #  test-docker-image: amazonlinux:2023
-    runs-on: ubuntu-latest
+            test-docker-image: arm64v8/amazonlinux:2
+          - label: Amazon Linux 2023 aarch64
+            rake-job: amazonlinux-2023
+            test-docker-image: arm64v8/amazonlinux:2023
+    runs-on: ubuntu-24.04-arm
     steps:
       - uses: actions/checkout@master
       - name: Install dependencies
         run: |
           sudo apt update
           sudo apt -V install ruby ruby-bundler ruby-serverspec
-          sudo apt -V install qemu-user-static
           sudo gem install bundler:2.2.9 --no-document
           sudo gem uninstall fileutils
+      - name: cache rpm
+        uses: actions/cache@v4
+        id: cache-rpm
+        with:
+          path: |
+            fluent-package/yum/repositories
+          key: ${{ runner.os }}-cache-${{ matrix.rake-job }}-${{ hashFiles('**/config.rb', '**/Rakefile', '**/Gemfile*', '**/*.spec.in', 'fluent-package/templates/**', 'fluent-package/yum/**/Dockerfile') }}
       - name: Build rpm with Docker
         run: |
-          cp /usr/bin/qemu-aarch64-static fluent-package/yum/${{ matrix.rake-job }}/
           rake yum:build YUM_TARGETS=${{ matrix.rake-job }}-aarch64
       - name: Upload fluent-package rpm
         uses: actions/upload-artifact@master
@@ -51,13 +59,33 @@ jobs:
       - name: Check Package Size
         run: |
           fluent-package/yum/pkgsize-test.sh ${{ matrix.rake-job }} aarch64
+      - name: Installation Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --rm \
+          --tty \
+          --env CI=true \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/yum/install-test.sh
+      - name: Serverspec Test
+        run: |
+          mkdir -p .bundle
+          docker run \
+          --rm \
+          --tty \
+          --env CI=true \
+          --volume ${PWD}:/fluentd:ro \
+          ${{ matrix.test-docker-image }} \
+          /fluentd/fluent-package/yum/serverspec-test.sh
       - name: Binstubs Test
         run: |
           mkdir -p .bundle
           docker run \
           --rm \
           --tty \
+          --env CI=true \
           --volume ${PWD}:/fluentd:ro \
-          --env CENTOS_STREAM=false \
-          arm64v8/${{ matrix.test-docker-image }} \
+          ${{ matrix.test-docker-image }} \
           /fluentd/fluent-package/yum/binstubs-test.sh
diff --git a/.github/workflows/yum.yml b/.github/workflows/yum.yml
@@ -125,7 +125,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
-      - uses: canonical/[email protected].1
+      - uses: canonical/[email protected].2
       - name: Run diagnostic
         run: |
           uname -a
@@ -177,6 +177,6 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           name: packages-${{ matrix.rake-job }}
-      - uses: canonical/[email protected].1
+      - uses: canonical/[email protected].2
       - name: Run Test ${{ matrix.test-file }} on ${{ matrix.test-lxc-image }}
         run: fluent-package/yum/systemd-test/test.sh ${{ matrix.test-lxc-image }} ${{ matrix.test-file }}
diff --git a/fluent-package/apt/commonvar.sh b/fluent-package/apt/commonvar.sh
@@ -14,6 +14,10 @@ case ${code_name} in
     distribution=ubuntu
     channel=universe
     mirror=http://archive.ubuntu.com/ubuntu/
+    if [ "$architecture" = "arm64" ]; then
+        echo "For ${code_name} (arm64), use ubuntu-ports"
+        mirror=http://ports.ubuntu.com/ubuntu-ports
+    fi
     ;;
   buster|bullseye)
     distribution=debian

diff --git a/fluent-package/yum/install-test.sh b/fluent-package/yum/install-test.sh
@@ -59,8 +59,9 @@ esac
 
 echo "INSTALL TEST"
 repositories_dir=/fluentd/fluent-package/yum/repositories
+ARCH=$(rpm --eval "%{_arch}")
 ${DNF} install -y \
-  ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/*.rpm
+  ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/${ARCH}/Packages/*.rpm
 
 fluentd --version
 test -e /etc/logrotate.d/fluentd
@@ -115,7 +116,7 @@ EOF
     # equivalent to tmpfiles.d
     mkdir -p /tmp/fluent
     ${DNF} install -y \
-           ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/*.rpm
+           ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/${ARCH}/Packages/*.rpm
 
     getent passwd td-agent >/dev/null
     getent group td-agent >/dev/null

diff --git a/fluent-package/yum/serverspec-test.sh b/fluent-package/yum/serverspec-test.sh
@@ -70,14 +70,15 @@ esac
 
 echo "INSTALL TEST"
 repositories_dir=/fluentd/fluent-package/yum/repositories
+ARCH=$(rpm --eval "%{_arch}")
 ${DNF} install -y \
-  ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/*.rpm
+  ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/${ARCH}/Packages/*.rpm
 
 fluentd --version
 
 if [ $ENABLE_SERVERSPEC_TEST -eq 1 ]; then
     curl -V > /dev/null 2>&1 || ${DNF} install -y curl
-    ${DNF} install -y which ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/x86_64/Packages/*.rpm
+    ${DNF} install -y which ${repositories_dir}/${distribution}/${DISTRIBUTION_VERSION}/${ARCH}/Packages/*.rpm
 
     /usr/sbin/fluent-gem install --no-document serverspec
     if [ $ENABLE_KAFKA_TEST -eq 1 ]; then