Update Ubuntu CI runner from old AWS c6i ParallelCluster to c6a single-node instance

.github/workflows/ubuntu-ci-c6a-x86_64.yaml

@@ -0,0 +1,210 @@
+name: ubuntu-ci-c6a-x86_64-build
+on:
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - '**.md'
+      - '.github/ISSUE_TEMPLATE/*'
+      - '.gitignore'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  ubuntu-ci-c6a-x86_64-build:
+    runs-on: [ubuntu-ci-c6a-x86_64]
+
+    steps:
+      - name: cleanup
+        run: |
+          pwd
+          ls -lart
+          rm -fr *
+
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          submodules: true
+
+      - name: create-env
+        run: |
+          source ./setup.sh
+
+          # DH* 20230302 - to avoid using padded_length for build caches,
+          # always build in the same environment so that the length of the
+          # path doesn't change - see also other 'DH* 20230302' changes below
+          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.ubuntu-ci-x86_64
+          export ENVNAME=ci-env.ubuntu-ci-c6a-x86_64
+          # *DH
+          export ENVDIR=$PWD/envs/${ENVNAME}
+          spack stack create env --site linux.default --template ${{ inputs.template || 'unified-dev' }} --name ${ENVNAME}
+          spack env activate ${ENVDIR}
+          spack add ${{ inputs.specs || '' }}
+          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
+
+          # Find external packages
+          spack external find --scope system \
+              --exclude bison --exclude openssl \
+              --exclude curl --exclude python
+          spack external find --scope system sed
+          spack external find --scope system perl
+          spack external find --scope system wget
+          PATH="/usr/local/opt/qt5/bin:$PATH" \
+              spack external find --scope system qt
+          spack external find --scope system texlive
+          spack external find --scope system mysql
+
+          # For GNU
+          # TODO - anything needed?
+
+          # For Intel
+          # Need to find external Intel MPI and annotate with the
+          # correct compiler, no way to do that with spack commands.
+          # echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "  intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "    buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "    - spec: [email protected]%[email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "      prefix: /opt/intel" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "      modules:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "      - libfabric-aws/1.16.0~amzn4.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          # echo "      - intelmpi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
+          # For GNU
+          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "compilers:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    spec: gcc@=11.4.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      cc: /usr/bin/gcc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      cxx: /usr/bin/g++" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      f77: /usr/bin/gfortran" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      fc: /usr/bin/gfortran" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+
+          # For Intel
+          # echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "compilers:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    spec: [email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "      cc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "      cxx: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/icpc" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "      f77: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "      fc: /opt/intel/oneapi/compiler/2022.1.0/linux/bin/intel64/ifort" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    operating_system: ubuntu20.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    environment:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "      prepend_path:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "        LD_LIBRARY_PATH: '/opt/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "      set:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "        I_MPI_PMI_LIBRARY: '/opt/slurm/lib/libpmi.so'" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          # echo "    extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+
+          export -n SPACK_SYSTEM_CONFIG_PATH
+
+          # DH* 20230302 - Don't use yet, too many problems
+          ## For buildcaches
+          #spack config add config:install_tree:padded_length:true
+          # *DH
+
+          # Set compiler and MPI - for Intel
+          # spack config add "packages:all:providers:mpi:[[email protected]]"
+          # spack config add "packages:all:compiler:[[email protected]]"
+          # sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%intel'\]/g" $ENVDIR/spack.yaml
+
+          # Set compiler and MPI - for GNU
+          spack config add "packages:all:providers:mpi:[[email protected]]"
+          spack config add "packages:all:compiler:[[email protected]]"
+          sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%gcc'\]/g" $ENVDIR/spack.yaml
+
+          # Add additional variants for MET packages, different from config/common/packages.yaml
+          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
+
+          # Concretize and check for duplicates
+          spack concretize 2>&1 | tee log.concretize.gnu-11.4.0
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl
+
+          # Add and update source cache
+          spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
+          spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/
+
+          # Add binary cache and reindex it
+          spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
+          spack buildcache update-index local-binary
+          echo "Packages in combined spack build caches:"
+          spack buildcache list
+
+          # Break installation up in pieces and create build caches in between
+          # This allows us to "spin up" builds that altogether take longer than
+          # six hours, and/or fail later in the build process.
+
+          # base-env
+          echo "base-env ..."
+          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0.base-env
+          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
+
+          # the rest
+          echo "${{ inputs.template || 'unified-dev' }} ..."
+          spack install --fail-fast --source --no-check-signature 2>&1 | tee [email protected].${{ inputs.template || 'unified-dev' }}
+          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+
+          # Next steps: synchronize source and build cache to a central/combined mirror?
+          echo "Next steps ..."
+          ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+          spack clean -a
+          spack module tcl refresh -y
+          spack stack setup-meta-modules
+          spack env deactivate
+
+          # Test environment chaining
+          echo "Test environment chaining"
+          spack stack create env --name chaintest --template empty --site linux.default --upstream $(realpath envs/ci-env.ubuntu-ci-c6a-x86_64/install)
+          # Retain config from upstream so we don't have to rebuild:
+          cp -r $PWD/envs/ci-env.ubuntu-ci-c6a-x86_64/{site,common} $PWD/envs/chaintest/.
+          spack env activate ${PWD}/envs/chaintest
+          spack add [email protected]%gcc
+          spack concretize | tee envs/chaintest/log.concretize
+          unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv '[email protected]' || true ) | wc -l)
+          if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
+          spack env deactivate
+
+      - name: test-env
+        run: |
+          source /etc/profile.d/z00_lmod.sh
+          module use /usr/share/modules/modulefiles
+          module use /opt/intel/mpi/2021.6.0/modulefiles
+          module use /home/ubuntu/jedi/modulefiles
+
+          # DH* 20230302 - to avoid using padded_length for build caches,
+          # always build in the same environment so that the length of the
+          # path doesn't change - see also other 'DH* 20230302' changes below
+          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.ubuntu-ci-x86_64
+          export ENVNAME=ci-env.ubuntu-ci-c6a-x86_64
+          # *DH
+          export ENVDIR=$PWD/envs/${ENVNAME}
+          ls -l ${ENVDIR}/install/modulefiles/Core
+
+          module use ${ENVDIR}/install/modulefiles/Core
+          module load stack-gcc/11.4.0
+          module load stack-openmpi/5.0.1
+          module load stack-python/3.10.13
+          module available
+
+          module load jedi-ufs-env/1.0.0
+          module load ewok-env/1.0.0
+          module load soca-env/1.0.0
+          module list