November 2023 main Spack merge + updates

.github/workflows/macos-ci-aarch64.yaml

@@ -105,7 +105,7 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
+          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
 
           # the rest
           echo "${{ inputs.template || 'unified-dev' }} ..."

.github/workflows/macos-ci-x86_64.yaml

@@ -98,7 +98,7 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
+          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
 
           # the rest
           echo "${{ inputs.template || 'unified-dev' }} ..."

.github/workflows/ubuntu-ci-x86_64.yaml

@@ -136,7 +136,7 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2022.1.0.base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
 
           # the rest
           echo "${{ inputs.template || 'unified-dev' }} ..."

.gitmodules

@@ -2,8 +2,8 @@
   path = spack
   #url = https://github.com/spack/spack
   #branch = develop
-  url = https://github.com/jcsda/spack
-  branch = jcsda_emc_spack_stack
+  url = https://github.com/AlexanderRichert-NOAA/spack
+  branch = nov2023_spackmerge
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules

configs/common/packages.yaml

@@ -10,6 +10,7 @@
         jpeg: [libjpeg-turbo]
         lapack: [openblas]
         yacc: [bison]
+        zlib-api: [zlib]
     #
     # This version of awscli goes with [email protected]
     awscli:
@@ -52,7 +53,7 @@
       variants: linalg=eigen,lapack compression=lz4,bzip2
     ecmwf-atlas:
       version: ['0.35.0']
-      variants: +fckit +ectrans +tesselation +fftw
+      variants: +fckit +trans +tesselation +fftw
     ectrans:
       version: ['1.2.0']
       variants: ~mkl +fftw
@@ -82,7 +83,7 @@
       #version: ['2023.01']
       #variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release
       version: ['2023.02.01']
-      variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +use_fmsio
+      variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io
     fontconfig:
       variants: +pic
     freetype:
@@ -93,6 +94,8 @@
       version: ['1.6.4']
     g2tmpl:
       version: ['1.10.2']
+    gettext:
+      version: ['0.21.1']
     gfsio:
       version: ['1.4.1']
     gftl-shared:
@@ -119,6 +122,7 @@
       variants: +hl +fortran +mpi ~threadsafe +szip
     ip:
       version: ['4.3.0']
+      variants: precision=4,d,8
     ip2:
       version: ['1.1.2']
     jasper:
@@ -152,6 +156,8 @@
       version: ['5.1.0']
     mpich:
       variants: ~hwloc +two_level_namespace
+    mysql:
+      variants: +download_boost
     nco:
       version: ['5.0.6']
       variants: ~doc
@@ -189,7 +195,6 @@
       variants: +internal-hwloc +two_level_namespace
     # Pin openssl to avoid duplicate packages being built
     openssl:
-      version: ['1.1.1u']
       variants: +shared
     p4est:
       version: ['2.8']
@@ -204,7 +209,7 @@
     pkgconf:
       buildable: False
     prod-util:
-      version: ['1.2.2']
+      version: ['2.1.0']
     proj:
       version: ['8.1.0']
       variants: ~tiff
@@ -219,9 +224,8 @@
       variants: +plotting
     py-click:
       version: ['8.0.3']
-    # Pin py-cryptography to a version earlier than 3.4 to avoid the rust dependency
     py-cryptography:
-      version: ['3.3.2']
+      variants: +rust_bootstrap
     py-eccodes:
       version: ['1.4.2']
     py-h5py:
@@ -237,8 +241,7 @@
       version: ['1.5.8']
       variants: ~mpi
     py-numpy:
-      version: ['1.22.3']
-      variants: +blas +lapack
+      require: ['@1.22.3']
     py-pandas:
       variants: +excel
     # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer
@@ -266,10 +269,9 @@
       version: ['1.9.3']
     # Pin the py-setuptools version to avoid duplicate Python packages
     py-setuptools:
-      version: ['59.4.0']
-    # Pin the py-setuptools-scm version to avoid duplicate Python packages
-    py-setuptools-scm:
-      version: ['7.0.5']
+      require: ['@63.4.3']
+    py-setuptools-rust:
+      variants: +rust_bootstrap
     py-shapely:
       version: ['1.8.0']
     qt:
@@ -284,13 +286,15 @@
     sigio:
       version: ['2.3.2']
     sp:
-      version: ['2.3.3']
+      version: ['2.5.0']
+      variants: precision=4,d,8
     udunits:
       version: ['2.2.28']
     upp:
       version: ['10.0.10']
     w3emc:
       version: ['2.10.0']
+      variants: precision=4,d,8
     w3nco:
       version: ['2.4.1']
     wget:

configs/containers/specs/jedi-ci.yaml

@@ -1,15 +1,15 @@
   ### spack-stack-1.5.1 / skylab-6.1.0 containers for fv3-jedi and mpas-jedi (but not for ufs-jedi)
   specs: [[email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected], [email protected], [email protected], ecflow@5,
-    [email protected], [email protected] +fckit +ectrans +tesselation +fftw, [email protected], [email protected] +fftw,
+    [email protected], [email protected] +fckit +trans +tesselation +fftw, [email protected], [email protected] +fftw,
     [email protected], [email protected], fms@release-jcsda, [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected],
     [email protected], py-pip, [email protected], [email protected], [email protected], [email protected],
-    sp@2.3.3, [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
+    sp@2.5.0, [email protected], [email protected], [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected], shumlib@macos_clang_linux_intel_port,
     [email protected], [email protected]]
     # Notes:

configs/sites/derecho/packages.yaml

@@ -29,6 +29,9 @@ packages:
   # Need to force spack to build python with ssl variant when using older openssl from OS
   python:
     require: "+ssl"
+  # Newer versions of patchelf (tested 0.18.0) don't build with Intel on Derecho
+  patchelf:
+    version:: ['0.17.2']
 
 ### All other external packages listed alphabetically
   autoconf:

configs/sites/emc-rhel/packages.yaml

@@ -21,10 +21,6 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  bison:
-    externals:
-    - spec: [email protected]
-      prefix: /usr
   bzip2:
     externals:
     - spec: [email protected]
@@ -128,22 +124,6 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  llvm:
-    externals:
-    - spec: [email protected]+clang~lld~lldb
-      prefix: /usr
-      extra_attributes:
-        compilers:
-          c: /usr/bin/clang
-          cxx: /usr/bin/clang++
-  llvm-doe:
-    externals:
-    - spec: [email protected]+clang~lld~lldb
-      prefix: /usr
-      extra_attributes:
-        compilers:
-          c: /usr/bin/clang
-          cxx: /usr/bin/clang++
   m4:
     externals:
     - spec: [email protected]
@@ -156,10 +136,6 @@ packages:
     externals:
     - spec: [email protected]
       prefix: /usr
-  python:
-    externals:
-    - spec: [email protected]+bz2+crypt+ctypes+dbm+lzma+nis+pyexpat~pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib
-      prefix: /usr
   rsync:
     externals:
     - spec: [email protected]

configs/sites/nautilus/packages.yaml

@@ -10,7 +10,7 @@ packages:
       fftw-api:: [intel-oneapi-mkl]
       lapack:: [intel-oneapi-mkl]
   ectrans:
-    variants:: +mkl
+    variants:: +mkl ~fftw
   gsibec:
     variants:: +mkl
 

configs/sites/s4/packages.yaml

@@ -144,10 +144,6 @@ packages:
       prefix: /data/prod/jedi/spack-stack/mysql-8.0.31
       modules:
       - mysql/8.0.31
-  perl:
-    externals:
-    - spec: [email protected]~cpanm+shared+threads
-      prefix: /usr
   pkg-config:
     externals:
     - spec: [email protected]

doc/source/KnownIssues.rst

@@ -31,6 +31,10 @@ General
 
    When using a role account to install spack-stack, it is sometimes necessary to run graphical applications such as the ``qt`` online installer. The following website describes in detail how this can be done: https://www.thegeekdiary.com/how-to-set-x11-forwarding-export-remote-display-for-users-who-switch-accounts-using-sudo/
 
+7. ``==> Error: the key "core_compilers" must be set in modules.yaml`` during ``spack module [lmod|tcl] refresh``
+
+   This error usually indicates that the wrong module type is used in the ``spack module ... refresh`` command. For example, the system is configured for ``lmod``, but the command used is ``spack module tcl refresh``.
+
 ==============================
 MSU Hercules
 ==============================
@@ -45,6 +49,10 @@ NASA Discover
 
    Discover's connection to the outside world can be very slow and spack sometimes aborts with fetch timeouts. Try again until it works, sometimes have to wait for a bit.
 
+2. ``configure: error: cannot guess build type; you must specify one`` when building ``freetype`` or other packages that use configure scripts
+
+   This can happen if a spack install is started in a ``screen`` session, because Discover puts the temporary data in directories like ``/gpfsm/dnb33/tdirs/login/discover13.29716.dheinzel``, which get wiped out after some time. Without ``screen``, this problem doesn't occur.
+
 ==============================
 NOAA Parallel Works
 ==============================

doc/source/NewSiteConfigs.rst

@@ -450,7 +450,6 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
        --exclude bison --exclude cmake \
        --exclude curl --exclude openssl \
        --exclude openssh
-   spack external find --scope system perl
    spack external find --scope system wget
    spack external find --scope system mysql
    spack external find --scope system texlive