Merge pull request #4823 from kinke/gha_aarch64

CI: Migrate Linux AArch64 job from Cirrus CI to GitHub Actions main workflow

.cirrus.yml

@@ -34,31 +34,14 @@ common_steps_template: &COMMON_STEPS_TEMPLATE
       ctest --output-on-failure -R ldc2-unittest
     run_lit_testsuite_script: |
       cd $CIRRUS_WORKING_DIR/../build/tests
-      if [[ "$CI_OS-$CI_ARCH" == "linux-aarch64" ]]; then
-        # FIXME: the following tests fail
-        rm $CIRRUS_WORKING_DIR/tests/debuginfo/nested_gdb.d
-        rm $CIRRUS_WORKING_DIR/tests/sanitizers/fuzz_basic.d
-      fi
       python3 runlit.py -v -j $PARALLELISM .
     run_dmd_testsuite_script: |
       cd $CIRRUS_WORKING_DIR/../build
       DMD_TESTSUITE_MAKE_ARGS=-j$PARALLELISM ctest -V -R dmd-testsuite
     run_defaultlib_tests_script: |
       # Run defaultlib unittests & druntime integration tests
       cd $CIRRUS_WORKING_DIR/../build
-      excludes="dmd-testsuite|ldc2-unittest|lit-tests"
-      if [[ "$CI_OS-$CI_ARCH" == "linux-aarch64" ]]; then
-        # FIXME: failing unittest(s)
-        excludes+='|^std.internal.math.gammafunction'
-        # FIXME: failing unittest(s) with enabled optimizations
-        excludes+='|^std.math.exponential(-shared)?$'
-        # FIXME: subtest rt_trap_exceptions fails
-        excludes+='|^druntime-test-exceptions-debug$'
-        # std.path unittests apparently need HOME, which happens not to be set
-        export HOME=~
-        echo "Setting HOME to '$HOME'"
-      fi
-      ctest -j$PARALLELISM --output-on-failure -E "$excludes"
+      ctest -j$PARALLELISM --output-on-failure -E "dmd-testsuite|ldc2-unittest|lit-tests"
 
 # Performs the extra packaging steps for jobs producing a prebuilt package.
 # Requires env variables CI_ARCH, CI_OS, GITHUB_TOKEN and PARALLELISM (and CC for FreeBSD).
@@ -161,7 +144,7 @@ packaging_steps_template: &PACKAGING_STEPS_TEMPLATE
     fi
 
 # Installs Ubuntu 20.04+ prerequisites.
-# Requires env variables CI_ARCH, HOST_LDC_VERSION, EXTRA_APT_PACKAGES and EXTRA_CMAKE_FLAGS.
+# Requires env variables CI_ARCH, EXTRA_APT_PACKAGES and EXTRA_CMAKE_FLAGS.
 install_ubuntu_prerequisites_template: &INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
   install_prerequisites_script: |
     cd $CIRRUS_WORKING_DIR/..
@@ -177,32 +160,21 @@ install_ubuntu_prerequisites_template: &INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
     fi
     apt-get -q update
     apt-get -yq install \
-      git-core ninja-build $gcc_pkg \
+      git-core cmake ninja-build $gcc_pkg \
       zlib1g-dev $libcurl_pkg curl gdb python3 python3-pip tzdata unzip zip \
       $EXTRA_APT_PACKAGES
     python3 --version
-    # Download & extract CMake
-    curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.27.1/cmake-3.27.1-linux-$CI_ARCH.tar.gz
-    mkdir cmake
-    tar -xf cmake.tar.gz --strip 1 -C cmake
-    rm cmake.tar.gz
-    # Download & extract custom Ninja on x86_64
+    # Install custom Ninja on x86_64
     if [[ "$CI_ARCH" == "x86_64" ]]; then
       curl -fL --retry 3 --max-time 60 -O https://github.com/symmetryinvestments/ninja/releases/download/v1.11.1-sym1/ninja-linux.zip
       mkdir ninja
       unzip ninja-linux.zip -d ninja
       rm ninja-linux.zip
+      ln -sf $PWD/ninja/ninja /usr/bin/ninja
     fi
-    # Download & extract host LDC
-    curl -fL --retry 3 --max-time 300 -o ldc2.tar.xz https://github.com/ldc-developers/ldc/releases/download/v$HOST_LDC_VERSION/ldc2-$HOST_LDC_VERSION-linux-$CI_ARCH.tar.xz
-    mkdir host-ldc
-    tar -xf ldc2.tar.xz --strip 1 -C host-ldc
-    rm ldc2.tar.xz
 
 environment:
   CIRRUS_CLONE_DEPTH: 50
-  HOST_LDC_VERSION: 1.31.0
-  LLVM_VERSION: 19.1.7
   GITHUB_TOKEN: ENCRYPTED[0955bd48c8d4e5391446fc0149d0719ad0b63df27ec9e6c180a5730a5b10dc7f28f09d1383423db158d21380ee2b022a]
 
 task:
@@ -218,7 +190,6 @@ task:
     EXTRA_APT_PACKAGES: "gdmd llvm-dev libclang-common-19-dev lld"
     EXTRA_CMAKE_FLAGS: "-DBUILD_SHARED_LIBS=ON -DBUILD_LTO_LIBS=ON -DD_COMPILER=gdmd -DLDC_LINK_MANUALLY=ON"
     PARALLELISM: 8
-    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${CIRRUS_WORKING_DIR}/../ninja:${PATH}
     # for gdmd:
     LANG: C.UTF-8
   << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
@@ -234,82 +205,6 @@ task:
     rm tests/dmd/fail_compilation/needspkgmod.d
   << : *COMMON_STEPS_TEMPLATE
 
-task:
-  name: Ubuntu 20.04 aarch64
-  compute_engine_instance:
-    image_project: ubuntu-os-cloud
-    image: family/ubuntu-2004-lts-arm64
-    platform: linux
-    architecture: arm64
-    cpu: 4
-    memory: 8G
-    disk: 20
-  timeout_in: 60m
-  environment:
-    CI_ARCH: aarch64
-    CI_OS: linux
-    EXTRA_APT_PACKAGES: libxml2-dev libzstd-dev p7zip-full
-    EXTRA_CMAKE_FLAGS: >-
-      -DBUILD_LTO_LIBS=ON
-      -DD_COMPILER_FLAGS="-O -flto=full -defaultlib=phobos2-ldc-lto,druntime-ldc-lto"
-      -DEXTRA_CXXFLAGS=-flto=full
-      -DCMAKE_EXE_LINKER_FLAGS=-static-libstdc++
-      -DJITRT_EXTRA_LDFLAGS=-static-libstdc++
-      -DCOMPILER_RT_LIBDIR_OS=aarch64-unknown-linux-gnu
-      -DLLVM_ROOT_DIR=$CIRRUS_WORKING_DIR/../llvm
-      -DD_COMPILER=$CIRRUS_WORKING_DIR/../bootstrap-ldc/bin/ldmd2
-    PARALLELISM: 4
-    PATH: ${CIRRUS_WORKING_DIR}/../cmake/bin:${PATH}
-    CLANG_VERSION: '15.0.3' # 15.0.6 requires a more recent libstdc++.so.6 than shipped with Ubuntu 20
-    CC: $CIRRUS_WORKING_DIR/../clang/bin/clang
-    CXX: $CIRRUS_WORKING_DIR/../clang/bin/clang++
-  << : *INSTALL_UBUNTU_PREREQUISITES_TEMPLATE
-  # Download & extract LDC-flavoured LLVM
-  download_prebuilt_llvm_script: |
-    cd $CIRRUS_WORKING_DIR/..
-    assertsSuffix=""
-    if [[ "${CIRRUS_TAG:-}" == "" ]]; then
-      echo 'Using LLVM with enabled assertions'
-      assertsSuffix="-withAsserts"
-    fi
-    llvm_tag="CI"
-    if [[ "$LLVM_VERSION" = *.* ]]; then llvm_tag="ldc-v$LLVM_VERSION"; fi
-    llvm_url="https://github.com/ldc-developers/llvm-project/releases/download/$llvm_tag/llvm-$LLVM_VERSION-linux-$CI_ARCH$assertsSuffix.tar.xz"
-    curl -fL --retry 3 --max-time 300 -o llvm.tar.xz $llvm_url
-    mkdir -p llvm
-    tar -xf llvm.tar.xz --strip 1 -C llvm
-    rm llvm.tar.xz
-    # Make lld the default linker (likely with enabled assertions unfortunately)
-    ln -sf "$PWD/llvm/bin/ld.lld" /usr/bin/ld
-    ld --version
-    # Make sure to link libzstd statically
-    rm /usr/lib/aarch64-linux-gnu/libzstd.so
-  # Download & extract clang
-  download_prebuilt_clang_script: |
-    cd $CIRRUS_WORKING_DIR/..
-    curl -fL --retry 3 --max-time 300 -o clang.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-$CLANG_VERSION/clang+llvm-$CLANG_VERSION-$CI_ARCH-linux-gnu.tar.xz
-    mkdir clang
-    tar -xf clang.tar.xz --strip 1 -C clang
-    rm clang.tar.xz
-  clone_submodules_early_script: |
-    cd $CIRRUS_WORKING_DIR
-    git submodule update --init --depth $CIRRUS_CLONE_DEPTH
-  # Build a first LDC as host compiler for the actual build
-  build_bootstrap_ldc_script: |
-    cd $CIRRUS_WORKING_DIR/..
-    mkdir bootstrap-ldc
-    cd bootstrap-ldc
-    cmake -G Ninja $CIRRUS_WORKING_DIR \
-      -DCMAKE_BUILD_TYPE=Release \
-      -DLLVM_ROOT_DIR=$PWD/../llvm \
-      -DD_COMPILER=$PWD/../host-ldc/bin/ldmd2 \
-      -DBUILD_SHARED_LIBS=OFF \
-      -DBUILD_LTO_LIBS=ON
-    ninja -j$PARALLELISM obj/ldc2.o all
-    bin/ldc2 -version
-  << : *COMMON_STEPS_TEMPLATE
-  << : *PACKAGING_STEPS_TEMPLATE
-
 task:
   name: FreeBSD 13.3 x64
   freebsd_instance:

.github/actions/1-setup/action.yml

@@ -15,25 +15,46 @@ runs:
       shell: bash
       run: |
         set -eux
+        cd ..
         export DEBIAN_FRONTEND=noninteractive
-        sudo dpkg --add-architecture i386
+        arch='${{ inputs.arch }}'
+        if [[ $arch == x86_64 ]]; then
+          sudo dpkg --add-architecture i386
+        fi
         sudo apt-get -q update
-        # Don't use latest gdb v10+ from Ubuntu toolchain PPA with regressions, use official v9
-        sudo apt-get -yq install \
-          git-core cmake g++-multilib \
-          libcurl4 libcurl4:i386 libzstd-dev \
-          curl gdb=9.1-0ubuntu1 p7zip-full tzdata unzip zip python3-pip
+        packages=( \
+          git-core cmake g++ \
+          libcurl4 libxml2-dev libzstd-dev \
+          curl gdb p7zip-full tzdata unzip zip python3-pip \
+        )
+        if [[ $arch == x86_64 ]]; then
+          packages+=(g++-multilib libcurl4:i386)
+        fi
+        sudo -E apt-get -yq install ${packages[@]}
         # Make sure to link libzstd statically
-        sudo rm /usr/lib/x86_64-linux-gnu/libzstd.so
+        sudo rm /usr/lib/$arch-linux-gnu/libzstd.so
+        # Install a more recent CMake version in the AArch64 container (for mimalloc...)
+        if [[ $arch == aarch64 ]]; then
+          curl -fL --retry 3 --max-time 300 -o cmake.tar.gz https://github.com/Kitware/CMake/releases/download/v3.31.4/cmake-3.31.4-linux-$arch.tar.gz
+          mkdir cmake
+          tar -xf cmake.tar.gz --strip 1 -C cmake
+          rm cmake.tar.gz
+          sudo ln -sf $PWD/cmake/bin/cmake /usr/bin/cmake
+        fi
 
     - name: 'Linux: Download & extract clang' # into ../clang
       if: runner.os == 'Linux'
       shell: bash
       run: |
         set -eux
         cd ..
+        arch='${{ inputs.arch }}'
+        extraSuffix=''
+        if [[ $arch == x86_64 ]]; then
+          extraSuffix='-ubuntu-18.04'
+        fi
         curl -fL --retry 3 --max-time 300 -o clang.tar.xz \
-          https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ inputs.clang_version }}/clang+llvm-${{ inputs.clang_version }}-x86_64-linux-gnu-ubuntu-18.04.tar.xz
+          https://github.com/llvm/llvm-project/releases/download/llvmorg-${{ inputs.clang_version }}/clang+llvm-${{ inputs.clang_version }}-$arch-linux-gnu$extraSuffix.tar.xz
         mkdir clang
         tar -xf clang.tar.xz --strip 1 -C clang
         rm clang.tar.xz
@@ -105,7 +126,19 @@ runs:
         ld --version
 
     - name: Install ninja
+      if: inputs.arch != 'aarch64'
       uses: symmetryinvestments/gha-setup-ninja@v2
+    - name: Install ninja (Linux AArch64)
+      if: inputs.arch == 'aarch64'
+      shell: bash
+      run: |
+        set -eux
+        cd ..
+        curl -fL --retry 3 --max-time 60 -O https://github.com/ninja-build/ninja/releases/download/v1.12.1/ninja-linux-aarch64.zip
+        mkdir ninja
+        unzip ninja-linux-aarch64.zip -d ninja
+        rm ninja-linux-aarch64.zip
+        sudo ln -sf $PWD/ninja/ninja /usr/bin/ninja
 
     - name: Install D host compiler
       uses: dlang-community/setup-dlang@v1

.github/actions/4b-test-lit/action.yml

@@ -1,16 +1,27 @@
 name: Run LIT testsuite
 inputs:
   arch:
-    required: false # Windows only
+    required: true
 runs:
   using: composite
   steps:
 
     - name: 'Posix: Run LIT testsuite'
       if: runner.os != 'Windows'
       shell: bash
-      # temporarily add LLVM bin dir to PATH, so that e.g. wasm-ld is found
-      run: cd ../build && PATH="$PWD/../llvm/bin:$PATH" ctest -V -R "lit-tests"
+      run: |
+        set -eux
+
+        if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-aarch64 ]]; then
+          # FIXME: the following tests fail
+          rm tests/debuginfo/nested_gdb.d
+          rm tests/debuginfo/print_gdb.d # works for Cirrus CI though
+          rm tests/sanitizers/fuzz_basic.d
+        fi
+
+        cd ../build
+        # temporarily add LLVM bin dir to PATH, so that e.g. wasm-ld is found
+        PATH="$PWD/../llvm/bin:$PATH" ctest -V -R "lit-tests"
 
     - name: 'Windows: Run LIT testsuite'
       if: runner.os == 'Windows'

.github/actions/4d-test-libs/action.yml

@@ -1,7 +1,7 @@
 name: Run defaultlib unittests & druntime integration tests
 inputs:
   arch:
-    required: false # Windows only
+    required: true
 runs:
   using: composite
   steps:
@@ -17,7 +17,18 @@ runs:
         else
           N=$(nproc)
         fi
-        ctest -j$N --output-on-failure -E "dmd-testsuite|lit-tests|ldc2-unittest"
+
+        excludes="dmd-testsuite|lit-tests|ldc2-unittest"
+        if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-aarch64 ]]; then
+          # FIXME: failing unittest(s)
+          excludes+='|^std.internal.math.gammafunction'
+          # FIXME: failing unittest(s) with enabled optimizations
+          excludes+='|^std.math.exponential(-shared)?$'
+          # FIXME: subtest rt_trap_exceptions fails
+          excludes+='|^druntime-test-exceptions-debug$'
+        fi
+
+        ctest -j$N --output-on-failure -E "$excludes"
 
     - name: 'Windows: Run defaultlib unittests & druntime integration tests'
       if: runner.os == 'Windows'

.github/actions/6-integration-test/action.yml

@@ -1,4 +1,7 @@
 name: Run a few integration tests against the installed compiler
+inputs:
+  arch:
+    required: true
 runs:
   using: composite
   steps:
@@ -18,7 +21,7 @@ runs:
         fi
         installed/bin/ldc2 hello.d -link-defaultlib-shared
         ./hello
-        if [[ '${{ runner.os }}' == Linux ]]; then
+        if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-x86_64 ]]; then
           installed/bin/ldc2 hello.d -m32 -link-defaultlib-shared
           ./hello
         fi
@@ -31,7 +34,7 @@ runs:
         for mode in thin full; do
           installed/bin/ldc2 hello.d -of=hello_$mode -flto=$mode -defaultlib=phobos2-ldc-lto,druntime-ldc-lto
           ./hello_$mode
-          if [[ '${{ runner.os }}' == Linux ]]; then
+          if [[ '${{ runner.os }}-${{ inputs.arch }}' == Linux-x86_64 ]]; then
             installed/bin/ldc2 hello.d -m32 -of=hello_$mode-32 -flto=$mode -defaultlib=phobos2-ldc-lto,druntime-ldc-lto
             ./hello_$mode-32
           fi

.github/actions/7-package/action.yml

@@ -130,8 +130,8 @@ runs:
         echo "ARTIFACT_ID=$artifactID" >> $GITHUB_ENV
         echo "ARTIFACT_NAME=$os-${{ inputs.arch }}" >> $GITHUB_ENV
 
-    - name: 'Linux: Pack source dir'
-      if: runner.os == 'Linux' && inputs.os == ''
+    - name: 'Linux x86_64: Pack source dir'
+      if: runner.os == 'Linux' && inputs.os == '' && inputs.arch == 'x86_64'
       shell: bash
       run: |
         set -eux