Add new template cylc-dev (and test building in GitHub actions)

.github/workflows/macos-ci-aarch64.yaml

@@ -111,7 +111,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -51,12 +51,14 @@ jobs:
           # Set up spack-stack
           source ./setup.sh
 
-          declare -a TEMPLATES=("unified-dev" "skylab-dev")
+          declare -a TEMPLATES=("unified-dev" "skylab-dev" "cylc-dev")
           for TEMPLATE in "${TEMPLATES[@]}"; do
             if [[ "${TEMPLATE}" == *"unified-dev"* ]]; then
               export ENVNAME=ue-gcc-11.4.0-buildcache
             elif [[ "${TEMPLATE}" == *"skylab-dev"* ]]; then
               export ENVNAME=se-gcc-11.4.0-buildcache
+            elif [[ "${TEMPLATE}" == *"cylc-dev"* ]]; then
+              export ENVNAME=ce-gcc-11.4.0-buildcache
             fi
             echo "Creating environment ${ENVNAME} from template ${TEMPLATE}"
 
@@ -68,7 +70,8 @@ jobs:
             # Find external packages
             spack external find --scope system \
                 --exclude bison --exclude openssl \
-                --exclude curl --exclude python
+                --exclude curl --exclude python \
+                --exclude meson
             spack external find --scope system grep
             spack external find --scope system sed
             spack external find --scope system perl
@@ -104,7 +107,7 @@ jobs:
               set +e
               spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
               spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-              set +e
+              set -e
               echo "Packages in spack binary cache:"
               spack buildcache list
             fi
@@ -113,20 +116,22 @@ jobs:
             # This allows us to "spin up" builds that altogether take longer than
             # six hours, and/or fail later in the build process.
 
-            # base-env
-            echo "base-env ..."
-            spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
-            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
-
-            # jedi-base-env
-            echo "jedi-base-env ..."
-            spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
-            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
-
-            # jedi-ufs-env
-            echo "jedi-ufs-env ..."
-            spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
-            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+            if [[ "${TEMPLATE}" == *"unified-dev"* || "${TEMPLATE}" == *"skylab-dev"* ]]; then
+              # base-env
+              echo "base-env ..."
+              spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
+              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
+
+              # jedi-base-env
+              echo "jedi-base-env ..."
+              spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
+              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+
+              # jedi-ufs-env
+              echo "jedi-ufs-env ..."
+              spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
+              spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+            fi
 
             # the rest
             echo "${TEMPLATE} ..."

.github/workflows/ubuntu-ci-x86_64-intel.yaml

@@ -59,7 +59,8 @@ jobs:
           # Find external packages
           spack external find --scope system \
               --exclude bison --exclude openssl \
-              --exclude curl --exclude python
+              --exclude curl --exclude python \
+              --exclude meson
           spack external find --scope system grep
           spack external find --scope system sed
           spack external find --scope system perl
@@ -144,7 +145,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml

@@ -60,7 +60,7 @@ jobs:
           spack external find --scope system \
               --exclude bison --exclude openssl \
               --exclude curl --exclude python \
-              --exclude gmake
+              --exclude meson --exclude gmake
           spack external find --scope system grep
           spack external find --scope system sed
           spack external find --scope system perl
@@ -150,7 +150,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.github/workflows/ubuntu-ci-x86_64-oneapi.yaml

@@ -60,7 +60,7 @@ jobs:
           spack external find --scope system \
               --exclude bison --exclude openssl \
               --exclude curl --exclude python \
-              --exclude gmake
+              --exclude meson --exclude gmake
           spack external find --scope system grep
           spack external find --scope system sed
           spack external find --scope system perl
@@ -150,7 +150,7 @@ jobs:
             set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
             spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
-            set +e
+            set -e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.gitmodules

@@ -1,7 +1,9 @@
 [submodule "spack"]
   path = spack
-  url = https://github.com/jcsda/spack
-  branch = spack-stack-dev
+  #url = https://github.com/jcsda/spack
+  #branch = spack-stack-dev
+  url = https://github.com/climbfuji/spack
+  branch = bugfix/freetype_zlibapi_depbuildenv
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules

configs/common/packages.yaml

@@ -215,10 +215,9 @@ packages:
   # Avoid full rust dependency: version 43+ requires py-maturin
   py-cryptography:
     require: '@:42 +rust_bootstrap'
-  # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython
-  # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
+  # Pin py-cython to avoid duplicate packages
   py-cython:
-    require: '@0.29.36'
+    require: '@3'
   # As long as we need the "delayed" variant for py-dask,
   # we are restricted to version 2021.03.0 (last one that has it)
   # and newer than 2021.01.0 because of these issues:
@@ -233,13 +232,15 @@ packages:
     require: '~mpi'
   # https://github.com/JCSDA/spack-stack/issues/1276
   py-matplotlib:
-     require: '@3.7.4'
+    require: '@3.7.4'
   # Variant ~mpi not working for latest py-netcdf4
   # https://github.com/spack/spack/issues/47652
   py-netcdf4:
     require: '@1.7.1 +mpi'
-  py-pandas:
-    require: '+excel'
+  # To avoid duplicate packages
+  py-numpy:
+    require:
+    - '@1.26'
   # To avoid duplicate packages
   py-ruamel-yaml:
     require: '@0.17.16'

configs/common/packages_intel.yaml

@@ -30,6 +30,9 @@ packages:
   boost:
     require:
     - '%gcc'
+  qt:
+    require:
+    - '%gcc'
   ecflow:
     require:
     - '%gcc'

configs/common/packages_oneapi.yaml

@@ -21,3 +21,6 @@ packages:
     require: '%gcc'
   libbsd:
     require: '%gcc'
+  qt:
+    require:
+    - '%gcc'

configs/templates/cylc-dev/spack.yaml

@@ -0,0 +1,32 @@
+# Template for CYLC (flow, rose, GUI). Supported with GCC only.
+# Creates a view that allows users to define (manually for now)
+# simple environments with just PATH and CYLC_PYTHONPATH set:
+#
+# #!/bin/bash
+# export VIEWDIR=/path/to/env/view-pys
+# export PATH=$VIEWDIR/bin:$PATH
+# export CYLC_PYTHONPATH=$VIEWDIR/lib/python3.11/site-packages
+#
+spack:
+  concretizer:
+    unify: true
+
+  view:
+    cylc:
+      root: view
+      select: [^python]
+      link: run
+      link_type: symlink
+  include: []
+
+  definitions:
+  - compilers: ['%gcc']
+  - packages:
+    - [email protected]
+    - [email protected]
+    - [email protected]
+
+  specs:
+  - matrix:
+    - [$packages]
+    - [$compilers]

spack-ext/repos/spack-stack/packages/jedi-tools-env/package.py

@@ -25,7 +25,7 @@ class JediToolsEnv(BundlePackage):
     # depends_on("aws-parallelcluster", type="run")
     depends_on("py-click", type="run")
     depends_on("py-openpyxl", type="run")
-    depends_on("py-pandas +excel", type="run")
+    depends_on("py-pandas", type="run")
     depends_on("py-pygithub", type="run")
     depends_on("py-scipy", type="run")
     depends_on("py-sphinx", type="run")