Always build Python with spack in CI tests / update submodule pointer…

… for spack (moving packages to builtin) (#427)

* Update submodule pointer for spack
* Always build Python with spack in CI tests
* Set CI compiler as hard requirement for all packages
* Configure mvapich2 for Linux
* Separate bootstrap step from spack env install to be able to use other compiler
* Move gcc@10 build from macOS to Linux

.github/actions/setup-externals/action.yaml

@@ -2,13 +2,11 @@ name: 'setup-externals'
 description: 'Setup externals such as the Intel compiler, MPI libraries.'
 inputs:
   compiler:
-    description: 'Compiler (available options gcc@9, gcc@10, apple-clang, intel)'
+    description: 'Compiler (available options gcc, apple-clang, intel)'
     required: true
-    default: 'gcc@9'
   mpi:
-    description: 'Which MPI flavor (openmpi, mpich, intel-oneapi-mpi)'
-    required: false
-    default: 'mpich@4'
+    description: 'Which MPI flavor (openmpi, mpich, mvapich2, intel-oneapi-mpi)'
+    required: true
 
 runs:
   using: "composite"
@@ -91,13 +89,13 @@ runs:
         export CXX=icpc
       fi
 
-      export MPICH_VERSION="4.0.2"
-      export OPENMPI_VERSION="4.1.4"
+      MPI_NAME=`echo "${{ inputs.mpi }}" | cut -d "@" -f 1`
+      MPI_VERSION=`echo "${{ inputs.mpi }}" | cut -d "@" -f 2`
 
-      if [[ "${{ inputs.mpi }}" == "openmpi"* ]]; then
-        wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OPENMPI_VERSION}.tar.gz
-        tar -xzf openmpi-${OPENMPI_VERSION}.tar.gz
-        cd openmpi-${OPENMPI_VERSION}
+      if [[ "${MPI_NAME}" == "openmpi" ]]; then
+        wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${MPI_VERSION}.tar.gz
+        tar -xzf openmpi-${MPI_VERSION}.tar.gz
+        cd openmpi-${MPI_VERSION}
         # --with-hwloc=internal --with-libevent=internal : https://www.open-mpi.org/faq/?category=building#libevent-or-hwloc-errors-when-linking-fortran
         if [[ "$RUNNER_OS" == "macOS" ]]; then
           ./configure --prefix=$HOME/mpi \
@@ -112,10 +110,10 @@ runs:
         fi
         make -j4
         make install
-      elif [[ "${{ inputs.mpi }}" == "mpich"* ]]; then
-        wget http://www.mpich.org/static/downloads/${MPICH_VERSION}/mpich-${MPICH_VERSION}.tar.gz
-        tar -xzf mpich-${MPICH_VERSION}.tar.gz
-        cd mpich-${MPICH_VERSION}
+      elif [[ "${MPI_NAME}" == "mpich" ]]; then
+        wget http://www.mpich.org/static/downloads/${MPI_VERSION}/mpich-${MPI_VERSION}.tar.gz
+        tar -xzf mpich-${MPI_VERSION}.tar.gz
+        cd mpich-${MPI_VERSION}
         if [[ "$RUNNER_OS" == "macOS" ]]; then
           ./configure --prefix=$HOME/mpi \
               --enable-fortran --enable-cxx \
@@ -129,6 +127,27 @@ runs:
         fi
         make -j4
         make install
-      else
+      elif [[ "${MPI_NAME}" == "mvapich2" ]]; then
+        # Note that 2.3.7 apparently had a bug fix release, adding a "-1" to the tarball name
+        wget https://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-${MPI_VERSION}-1.tar.gz
+        tar -xzf mvapich2-${MPI_VERSION}-1.tar.gz
+        cd mvapich2-${MPI_VERSION}-1
+        if [[ "$RUNNER_OS" == "macOS" ]]; then
+          ./configure --prefix=$HOME/mpi \
+              --enable-fortran=yes --enable-cxx \
+              --enable-two-level-namespace \
+              --with-device=ch3:sock \
+              LIBS="-Wl,-commons,use_dylibs"
+        else
+          ./configure --prefix=$HOME/mpi \
+              --enable-fortran=yes --enable-cxx \
+              --with-device=ch3:sock
+        fi
+        make -j4
+        make install
+      elif [[ "${MPI_NAME}" == "intel-oneapi-mpi" ]]; then
         mkdir -p $HOME/mpi
+      else
+        echo "${MPI_NAME} not configured!"
+        exit 1
       fi

.github/actions/setup-os/action.yaml

@@ -29,43 +29,19 @@ runs:
         sudo apt-get install qt5-default qttools5-dev-tools libqt5svg5-dev
 
       elif [[ "$RUNNER_OS" == "macOS" ]]; then
-        # These are already installed
-        #brew install curl
-        #brew install git
-        #brew install git-lfs
+        brew install curl
+        brew install git
+        brew install git-lfs
         # For now we need gcc-10/gfortran-10
         brew install gcc@10
+        # Also install the llvm clang-12 compilers for later use
+        brew install llvm@12
         brew install lmod
         brew install qt@5
         brew install readline
         brew install wget
-        ls -l /usr/local/Cellar
-
-        # Remove macOS native Python 3.11 from /usr/local/bin
-        cd /usr/local/bin
-        rm 2to3-3.11
-        rm idle3.11
-        rm pip3.11
-        rm pydoc3.11
-        rm python3.11
-        rm python3.11-config
-        rm python3.11-intel64
-        ln -sf ../Cellar/[email protected]/3.10.8/bin/2to3-3.10 2to3 
-        ln -sf ../Cellar/[email protected]/3.10.8/bin/idle3.10 idle3
-        ln -sf ../Cellar/[email protected]/3.10.8/bin/pip3.10 pip3
-        ln -sf ../Cellar/[email protected]/3.10.8/bin/pydoc3.10 pydoc3
-        ln -sf ../Cellar/[email protected]/3.10.8/bin/python3.10 python3
-        ln -sf ../Cellar/[email protected]/3.10.8/bin/python3.10-config python3-config
-        #ln -sf ... python3-intel64 # doesn't exist
-        ls -l /usr/local/bin
 
         # Print version of xcode
         pkgutil --pkg-info=com.apple.pkg.CLTools_Executables
 
-        # Install Python poetry to avoid install errors in spack
-        python3 -m pip install poetry
-
       fi
-
-      ## Install Python poetry to avoid install errors in spack
-      #python3 -m pip install poetry

.github/actions/setup-spack-stack/action.yaml

@@ -10,13 +10,11 @@ inputs:
     required: false
     default: 'empty'
   compiler:
-    description: 'Compiler (available options gcc@9, gcc@10, apple-clang, intel)'
+    description: 'Compiler (available options gcc, apple-clang, intel)'
     required: true
-    default: 'gcc@9'
   mpi:
     description: 'Which MPI flavor (openmpi, mpich, intel-oneapi-mpi)'
-    required: false
-    default: 'mpich@4'
+    required: true
   path:
     description: 'Use custom checkout of Spack Stack. Useful in the Spack repo CI.'
     required: false
@@ -89,6 +87,13 @@ runs:
       echo "Cached ~/spack-mirror not found!"
       exit 1
 
+  - name: bootstrap-spack
+    shell: bash
+    run: |
+      cd ${{ inputs.path }}
+      source setup.sh
+      spack bootstrap now
+
   - name: setup-spack-env
     shell: bash
     run: |
@@ -109,46 +114,43 @@ runs:
         spack compiler find
       fi
 
-      # No external find for intel-oneapi-mpi, 
-      # And no way to add object entry to list using "spack config add"
-      # Add this first so "spack config add packages:" will append to this entry
-      if [[ "${{ inputs.mpi }}" == "intel-oneapi-mpi" ]]; then
-        impi_ver=$(find /opt/intel/oneapi/mpi -maxdepth 1 -mindepth 1 -type d | xargs basename)
-        echo "" >> ${SPACK_ENV}/spack.yaml
-        echo "  packages:" >> ${SPACK_ENV}/spack.yaml
-        echo "    intel-oneapi-mpi:" >> ${SPACK_ENV}/spack.yaml
-        echo "      buildable: False" >> ${SPACK_ENV}/spack.yaml
-        echo "      externals:" >> ${SPACK_ENV}/spack.yaml
-        echo "      - spec: intel-oneapi-mpi@${impi_ver}" >> ${SPACK_ENV}/spack.yaml
-        echo "        prefix: /opt/intel/oneapi" >> ${SPACK_ENV}/spack.yaml
+      # Need to find external packages and annotate with the
+      # correct compiler, no way to do that with spack commands.
+      MPI_NAME=`echo "${{ inputs.mpi }}" | cut -d "@" -f 1`
+      MPI_VERSION=`echo "${{ inputs.mpi }}" | cut -d "@" -f 2`
+      if [[ "${MPI_NAME}" == "intel-oneapi-mpi" ]]; then
+        MPI_PREFIX="/opt/intel/oneapi"
+      elif [[ "${MPI_NAME}" == "openmpi" ]]; then
+        MPI_PREFIX="$HOME/mpi"
+      elif [[ "${MPI_NAME}" == "mpich" ]]; then
+        MPI_PREFIX="$HOME/mpi"
+      elif [[ "${MPI_NAME}" == "mvapich2" ]]; then
+        MPI_PREFIX="$HOME/mpi"
       else
-        # So Spack can find external MPI
-        export "PATH=$HOME/mpi/bin:${PATH}"
+        echo "Malformed MPI provider, can only use 'intel-oneapi-mpi', 'openmpi', 'mpich', 'mvapich2'."
+        exit 1
       fi
+      echo "" >> ${SPACK_ENV}/spack.yaml
+      echo "  packages:" >> ${SPACK_ENV}/spack.yaml
+      echo "    ${MPI_NAME}:" >> ${SPACK_ENV}/spack.yaml
+      echo "      buildable: False" >> ${SPACK_ENV}/spack.yaml
+      echo "      externals:" >> ${SPACK_ENV}/spack.yaml
+      echo "      - spec: ${MPI_NAME}@${MPI_VERSION}%${{ inputs.compiler }}" >> ${SPACK_ENV}/spack.yaml
+      echo "        prefix: ${MPI_PREFIX}" >> ${SPACK_ENV}/spack.yaml
 
       # Spack external find is by default only looking for build-tools, Either
       # need to search for additional packages explicitly.
       spack external find
-      spack external find mpich
-      spack external find openmpi
       spack external find perl
-      #if [[ "$RUNNER_OS" == "Linux" ]]; then
-      #  spack external find --not-buildable python
-      #elif [[ "$RUNNER_OS" == "macOS" ]]; then
-      if [[ "$RUNNER_OS" == "macOS" ]]; then
-        # Jail spack external find to /usr/local to ignore Framework python
-        spack external find --not-buildable --path /usr/local python
-      fi
       spack external find wget
 
-      # Make homebrew qt@5 detectable on macOS
-      PATH="/usr/local/opt/qt@5/bin:${PATH}" spack external find qt
+      # Find homebrew qt@5 on macOS
+      spack external find --path /usr/local/opt/qt@5 qt
 
       if [[ "$RUNNER_OS" == "Linux" ]]; then
         spack external find curl
       elif [[ "$RUNNER_OS" == "macOS" ]]; then
-        # Make homebrew curl detectable on macOS
-        PATH="/usr/local/opt/curl/bin:${PATH}" spack external find curl
+        spack external find --path /usr/local/opt/curl curl
       fi
 
   - name: configure-options
@@ -166,8 +168,8 @@ runs:
       spack config add "packages:mpi:buildable:False"
       spack config add "packages:all:providers:mpi:[${{ inputs.mpi }}]"
       spack config add "packages:all:compiler:[${{ inputs.compiler }}]"
-      # This is not working as expected
-      #spack config add "packages:all:require:'%${{ inputs.compiler }}'"
+      # Set compiler as hard requirement for all packages
+      spack config add "packages:all:require:one_of:['%${{ inputs.compiler }}']"
 
       if [[ "$RUNNER_OS" == "macOS" ]]; then
         # Turn of SSL for ecflow CI builds for now
@@ -247,21 +249,17 @@ runs:
     run: |
       if [[ ! "${{ inputs.templates }}" == "empty" ]]; then
         cd ${{ inputs.path }}
-        if [[ "$RUNNER_OS" == "Linux" ]]; then
-          # Not yet: need lmod installed, loaded and module use command
-          echo ""
-        elif [[ "$RUNNER_OS" == "macOS" ]]; then
-          source /usr/local/opt/lmod/init/profile
-          module use $PWD/envs/${{ inputs.name }}/install/modulefiles/Core
-          # This needs more work: need to replace "${{ inputs.compiler }}" = "gcc@9"
-          # with "gcc/9", otherwise the module load command will fail.
-          # module load stack-${{ inputs.compiler }}
-          # module load stack-python
-          # # jedi-tools doesn't have any mpi dependencies
-          # if [[ ! "${{ inputs.specs }}" == "jedi-tools" ]]; then
-          #   module load stack-${{ inputs.mpi }}
-          # fi
-        fi
+        #if [[ "$RUNNER_OS" == "Linux" ]]; then
+        #  # Not yet: need lmod installed, loaded and module use command
+        #  echo ""
+        #elif [[ "$RUNNER_OS" == "macOS" ]]; then
+        #  # This needs more work
+        #  #source /usr/local/opt/lmod/init/profile
+        #  #module use $PWD/envs/${{ inputs.name }}/install/modulefiles/Core
+        #  #module load stack-${{ inputs.compiler }}
+        #  #module load stack-python
+        #  #module load stack-${{ inputs.mpi }}
+        #fi
       fi
 
   - name: Upload logs

.github/workflows/macos.yaml

@@ -41,7 +41,7 @@ jobs:
   cache-externals:
     strategy:
       matrix:
-        compiler-mpi: [ {compiler: apple-clang, mpi: openmpi@4}, {compiler: gcc@10, mpi: mpich@4} ]
+        compiler-mpi: [ {compiler: apple-clang@13.0.0, mpi: openmpi@4.1.4} ]
     runs-on: macos-11
     steps:
 
@@ -60,7 +60,7 @@ jobs:
     needs: [cache-externals, cache-spack-mirror]
     strategy:
       matrix:
-        compiler-mpi: [ {compiler: apple-clang, mpi: openmpi@4}, {compiler: gcc@10, mpi: mpich@4} ]
+        compiler-mpi: [ {compiler: apple-clang@13.0.0, mpi: openmpi@4.1.4} ]
         template: [skylab-dev, ufs-weather-model, ufs-srw-dev]
     runs-on: macos-11
     steps:

.github/workflows/ubuntu.yaml

@@ -41,7 +41,7 @@ jobs:
   cache-externals:
     strategy:
       matrix:
-        compiler-mpi: [ {compiler: intel, mpi: intel-oneapi-mpi}, {compiler: gcc@9, mpi: mpich@4} ]
+        compiler-mpi: [ {compiler: intel@2021.8.0, mpi: intel-oneapi-mpi@2021.8.0}, {compiler: gcc@9.4.0, mpi: [email protected]}, {compiler: [email protected], mpi: mpich@4.0.2} ]
     runs-on: ubuntu-20.04
     steps:
 
@@ -60,7 +60,7 @@ jobs:
     needs: [cache-externals, cache-spack-mirror]
     strategy:
       matrix:
-        compiler-mpi: [ {compiler: intel, mpi: intel-oneapi-mpi}, {compiler: gcc@9, mpi: mpich@4} ]
+        compiler-mpi: [ {compiler: intel@2021.8.0, mpi: intel-oneapi-mpi@2021.8.0}, {compiler: gcc@9.4.0, mpi: [email protected]}, {compiler: [email protected], mpi: mpich@4.0.2} ]
         template: [skylab-dev, ufs-weather-model, ufs-srw-dev]
     runs-on: ubuntu-20.04
     steps: