From 0e3fae2b3b91b37edcb9d0b93e8485e2800edcc0 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 15 Mar 2024 13:47:42 -0600
Subject: [PATCH 01/11] Don't use spack user cache logic in setup.sh on macOS

---
 setup.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup.sh b/setup.sh
index 3f033d620..4b1b15e53 100755
--- a/setup.sh
+++ b/setup.sh
@@ -7,7 +7,10 @@ echo "Setting environment variable SPACK_STACK_DIR to ${SPACK_STACK_DIR}"
 
 source ${SPACK_STACK_DIR:?}/spack/share/spack/setup-env.sh
 echo "Sourcing spack environment ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh"
-export SPACK_USER_CACHE_PATH=$SPACK_ROOT/user_cache
+if [ "$(uname)" != "Darwin" ]; then
+    echo "DO IT"
+  export SPACK_USER_CACHE_PATH=$SPACK_ROOT/user_cache
+fi
 
 # Get the current hash of the spack-stack code
 export SPACK_STACK_HASH=`git rev-parse --short HEAD`

From 3ea61df2e496e2918015f065d31838039996031d Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Fri, 15 Mar 2024 13:48:12 -0600
Subject: [PATCH 02/11] Use ucx at runtime instead of ofi on Narwhal, also
 comment out node.js module in compiler config

---
 configs/sites/narwhal/compilers.yaml |  2 --
 doc/source/PreConfiguredSites.rst    | 10 +++++++++-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml
index d279a5213..4102aa0cd 100644
--- a/configs/sites/narwhal/compilers.yaml
+++ b/configs/sites/narwhal/compilers.yaml
@@ -11,7 +11,6 @@ compilers::
       modules:
       - PrgEnv-intel/8.3.2
       - intel-classic/2021.4.0
-      - node.js/20.10.0
       environment:
         prepend_path:
           PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin'
@@ -32,7 +31,6 @@ compilers::
       modules:
       - PrgEnv-gnu/8.3.2
       - gcc/10.3.0
-      - node.js/20.10.0
       environment:
         prepend_path:
           LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib'
diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index ebe307ce8..65df66f49 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -259,6 +259,9 @@ For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules
 
 .. code-block:: console
 
+   # These extra steps are required for performance reason, ofi is about 30% slower than ucx
+   module unload craype-network-ofi
+   module load craype-network-ucx
    module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core
    module load stack-intel/2021.4.0
    module load stack-cray-mpich/8.1.14
@@ -279,6 +282,8 @@ With GNU, the following is required for building new spack environments and for
    module load cray-python/3.9.7.1
    module unload cray-libsci
    module load cray-libsci/22.08.1.1
+   module unload craype-network-ofi
+   module load craype-network-ucx
 
    module use /p/app/projects/NEPTUNE/spack-stack/modulefiles
    module load ecflow/5.8.4
@@ -287,6 +292,9 @@ For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules:
 
 .. code-block:: console
 
+   # These extra steps are required for performance reason, ofi is about 30% slower than ucx
+   module unload craype-network-ofi
+   module load craype-network-ucx
    module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core
    module load stack-gcc/10.3.0
    module load stack-cray-mpich/8.1.14
@@ -393,7 +401,7 @@ The following is required for building new spack environments and for using spac
    module use /glade/work/epicufsrt/contrib/spack-stack/derecho/modulefiles
    module load ecflow/5.8.4
 
-For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules::
+For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules:
 
 .. code-block:: console
 

From 0f23137e71557ca7ff57e4b45ef07e716574da1b Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 18 Mar 2024 15:21:58 -0600
Subject: [PATCH 03/11] Temporarily disable github actions on macOS Intel and
 JCSDA R&D cluster

---
 .github/workflows/macos-ci-x86_64.yaml   | 15 ++++++++-------
 .github/workflows/ubuntu-rnd-x86_64.yaml |  7 ++++---
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml
index a5a216a29..b1afda59e 100644
--- a/.github/workflows/macos-ci-x86_64.yaml
+++ b/.github/workflows/macos-ci-x86_64.yaml
@@ -1,12 +1,13 @@
 name: macos-ci-x86_64-build
 on:
-  pull_request:
-    paths-ignore:
-      - 'doc/**'
-      - '**.md'
-      - '.github/ISSUE_TEMPLATE/*'
-      - '.gitignore'
-  workflow_dispatch:
+  # Temporarily disabled 2024/03/18
+  #pull_request:
+  #  paths-ignore:
+  #    - 'doc/**'
+  #    - '**.md'
+  #    - '.github/ISSUE_TEMPLATE/*'
+  #    - '.gitignore'
+  #workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml
index 0b32d4f24..d9db83df8 100644
--- a/.github/workflows/ubuntu-rnd-x86_64.yaml
+++ b/.github/workflows/ubuntu-rnd-x86_64.yaml
@@ -1,8 +1,9 @@
 name: ubuntu-rnd-x86_64-build
 on:
-  schedule:
-    - cron: '0 0 * * *'
-  workflow_dispatch:
+  # Temporarily disabled 2024/03/18
+  #schedule:
+  #  - cron: '0 0 * * *'
+  #workflow_dispatch:
 
 defaults:
   run:

From 045687ff9e8911344aed64124f60ca7ebf5774a0 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 18 Mar 2024 15:22:48 -0600
Subject: [PATCH 04/11] Update submodule pointer for spack

---
 spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spack b/spack
index b40323ef1..dbc8d59d7 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit b40323ef10659795b66e8e165a6b16c7ca1c0066
+Subproject commit dbc8d59d7c24d7d9014dc801cbfd4b1f5d9187f0

From 0fe0996f5f232e00a935cbdfd28569afd6ee32ba Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Mon, 18 Mar 2024 15:27:49 -0600
Subject: [PATCH 05/11] Bug fix for disabling github workflows

---
 .github/workflows/macos-ci-x86_64.yaml   | 284 +++++++-------
 .github/workflows/ubuntu-rnd-x86_64.yaml | 472 +++++++++++------------
 2 files changed, 378 insertions(+), 378 deletions(-)

diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml
index b1afda59e..762ae30fe 100644
--- a/.github/workflows/macos-ci-x86_64.yaml
+++ b/.github/workflows/macos-ci-x86_64.yaml
@@ -1,142 +1,142 @@
-name: macos-ci-x86_64-build
-on:
-  # Temporarily disabled 2024/03/18
-  #pull_request:
-  #  paths-ignore:
-  #    - 'doc/**'
-  #    - '**.md'
-  #    - '.github/ISSUE_TEMPLATE/*'
-  #    - '.gitignore'
-  #workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: true
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  macos-ci-x86_64-build:
-    runs-on: [macos-ci-x86_64]
-
-    steps:
-      - name: cleanup
-        run: |
-          pwd
-          ls -lart
-          rm -fr *
-
-      - name: checkout
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-
-      - name: create-env
-        run: |
-          source ./setup.sh
-
-          # DH* 20230302 - to avoid using padded_length for build caches,
-          # always build in the same environment so that the length of the
-          # path doesn't change - see also other 'DH* 20230302' changes below
-          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64
-          export ENVNAME=ci-env.macos-ci-x86_64
-          # *DH
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          spack stack create env --site macos.default --template ${{ inputs.template || 'unified-dev' }} --name ${ENVNAME}
-          spack env activate ${ENVDIR}
-          spack add ${{ inputs.specs || '' }}
-          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
-
-          # Find external packages
-          spack external find --scope system \
-              --exclude bison --exclude openssl  \
-              --exclude python --exclude gettext
-          # Need to find gettext outside of default (presumed to be a system) path for krb5
-          spack external find --path=/usr/local/Cellar/gettext/0.21.1 gettext
-          spack external find --scope system perl
-          spack external find --scope system wget
-          PATH="/usr/local/opt/curl/bin:$PATH" \
-              spack external find --scope system curl
-          PATH="/usr/local/opt/qt5/bin:$PATH" \
-              spack external find --scope system qt
-          spack external find --scope system texlive
-          spack external find --scope system mysql
-
-          # Find compilers
-          spack compiler find --scope system
-
-          export -n SPACK_SYSTEM_CONFIG_PATH
-
-          # DH* 20230302 - Don't use yet, too many problems
-          ## For buildcaches
-          #spack config add config:install_tree:padded_length:true
-          # *DH
-
-          # Set compiler and MPI
-          spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
-          spack config add "packages:all:compiler:[apple-clang@14.0.0]"
-          sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
-
-          # Add additional variants for MET packages, different from config/common/packages.yaml
-          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-
-          # Concretize and check for duplicates
-          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm -i esmf -i mapl
-
-          # Add and update source cache
-          spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
-          spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/
-
-          # Add binary cache and reindex it
-          spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
-          spack buildcache update-index local-binary
-          echo "Packages in combined spack build caches:"
-          spack buildcache list
-
-          # Break installation up in pieces and create build caches in between
-          # This allows us to "spin up" builds that altogether take longer than
-          # six hours, and/or fail later in the build process.
-
-          # base-env
-          echo "base-env ..."
-          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
-
-          # the rest
-          echo "${{ inputs.template || 'unified-dev' }} ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'unified-dev' }}
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
-
-          # Next steps: synchronize source and build cache to a central/combined mirror?
-          echo "Next steps ..."
-          spack clean -a
-          spack module lmod refresh -y
-          spack stack setup-meta-modules
-          spack env deactivate
-
-      - name: test-env
-        run: |
-          source /usr/local/opt/lmod/init/profile
-
-          # DH* 20230302 - to avoid using padded_length for build caches,
-          # always build in the same environment so that the length of the
-          # path doesn't change - see also other 'DH* 20230302' changes below
-          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64
-          export ENVNAME=ci-env.macos-ci-x86_64
-          # *DH
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          ls -l ${ENVDIR}/install/modulefiles/Core
-
-          module use ${ENVDIR}/install/modulefiles/Core
-          module load stack-apple-clang/14.0.0
-          module load stack-openmpi/5.0.1
-          module load stack-python/3.10.13
-          module available
-
-          module load jedi-ufs-env/1.0.0
-          module load ewok-env/1.0.0
-          module load soca-env/1.0.0
-          module list
+# Temporarily disabled 2024/03/18
+#name: macos-ci-x86_64-build
+#on:
+#  pull_request:
+#    paths-ignore:
+#      - 'doc/**'
+#      - '**.md'
+#      - '.github/ISSUE_TEMPLATE/*'
+#      - '.gitignore'
+#  workflow_dispatch:
+#
+#concurrency:
+#  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+#  cancel-in-progress: true
+#
+#defaults:
+#  run:
+#    shell: bash
+#
+#jobs:
+#  macos-ci-x86_64-build:
+#    runs-on: [macos-ci-x86_64]
+#
+#    steps:
+#      - name: cleanup
+#        run: |
+#          pwd
+#          ls -lart
+#          rm -fr *
+#
+#      - name: checkout
+#        uses: actions/checkout@v2
+#        with:
+#          submodules: true
+#
+#      - name: create-env
+#        run: |
+#          source ./setup.sh
+#
+#          # DH* 20230302 - to avoid using padded_length for build caches,
+#          # always build in the same environment so that the length of the
+#          # path doesn't change - see also other 'DH* 20230302' changes below
+#          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64
+#          export ENVNAME=ci-env.macos-ci-x86_64
+#          # *DH
+#          export ENVDIR=$PWD/envs/${ENVNAME}
+#          spack stack create env --site macos.default --template ${{ inputs.template || 'unified-dev' }} --name ${ENVNAME}
+#          spack env activate ${ENVDIR}
+#          spack add ${{ inputs.specs || '' }}
+#          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
+#
+#          # Find external packages
+#          spack external find --scope system \
+#              --exclude bison --exclude openssl  \
+#              --exclude python --exclude gettext
+#          # Need to find gettext outside of default (presumed to be a system) path for krb5
+#          spack external find --path=/usr/local/Cellar/gettext/0.21.1 gettext
+#          spack external find --scope system perl
+#          spack external find --scope system wget
+#          PATH="/usr/local/opt/curl/bin:$PATH" \
+#              spack external find --scope system curl
+#          PATH="/usr/local/opt/qt5/bin:$PATH" \
+#              spack external find --scope system qt
+#          spack external find --scope system texlive
+#          spack external find --scope system mysql
+#
+#          # Find compilers
+#          spack compiler find --scope system
+#
+#          export -n SPACK_SYSTEM_CONFIG_PATH
+#
+#          # DH* 20230302 - Don't use yet, too many problems
+#          ## For buildcaches
+#          #spack config add config:install_tree:padded_length:true
+#          # *DH
+#
+#          # Set compiler and MPI
+#          spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
+#          spack config add "packages:all:compiler:[apple-clang@14.0.0]"
+#          sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
+#
+#          # Add additional variants for MET packages, different from config/common/packages.yaml
+#          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
+#
+#          # Concretize and check for duplicates
+#          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0
+#          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm -i esmf -i mapl
+#
+#          # Add and update source cache
+#          spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
+#          spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/
+#
+#          # Add binary cache and reindex it
+#          spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
+#          spack buildcache update-index local-binary
+#          echo "Packages in combined spack build caches:"
+#          spack buildcache list
+#
+#          # Break installation up in pieces and create build caches in between
+#          # This allows us to "spin up" builds that altogether take longer than
+#          # six hours, and/or fail later in the build process.
+#
+#          # base-env
+#          echo "base-env ..."
+#          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env
+#          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
+#
+#          # the rest
+#          echo "${{ inputs.template || 'unified-dev' }} ..."
+#          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'unified-dev' }}
+#          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
+#
+#          # Next steps: synchronize source and build cache to a central/combined mirror?
+#          echo "Next steps ..."
+#          spack clean -a
+#          spack module lmod refresh -y
+#          spack stack setup-meta-modules
+#          spack env deactivate
+#
+#      - name: test-env
+#        run: |
+#          source /usr/local/opt/lmod/init/profile
+#
+#          # DH* 20230302 - to avoid using padded_length for build caches,
+#          # always build in the same environment so that the length of the
+#          # path doesn't change - see also other 'DH* 20230302' changes below
+#          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64
+#          export ENVNAME=ci-env.macos-ci-x86_64
+#          # *DH
+#          export ENVDIR=$PWD/envs/${ENVNAME}
+#          ls -l ${ENVDIR}/install/modulefiles/Core
+#
+#          module use ${ENVDIR}/install/modulefiles/Core
+#          module load stack-apple-clang/14.0.0
+#          module load stack-openmpi/5.0.1
+#          module load stack-python/3.10.13
+#          module available
+#
+#          module load jedi-ufs-env/1.0.0
+#          module load ewok-env/1.0.0
+#          module load soca-env/1.0.0
+#          module list
diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml
index d9db83df8..ebd8806b1 100644
--- a/.github/workflows/ubuntu-rnd-x86_64.yaml
+++ b/.github/workflows/ubuntu-rnd-x86_64.yaml
@@ -1,236 +1,236 @@
-name: ubuntu-rnd-x86_64-build
-on:
-  # Temporarily disabled 2024/03/18
-  #schedule:
-  #  - cron: '0 0 * * *'
-  #workflow_dispatch:
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  ubuntu-rnd-x86_64-build:
-    runs-on: [ubuntu-rnd-x86_64]
-    timeout-minutes: 720 # 12 hours
-
-    steps:
-      - name: cleanup
-        run: |
-          pwd
-          ls -lart
-          rm -fr *
-          rm -fr /home/ubuntu/spack-stack/CI/tmp
-          mkdir /home/ubuntu/spack-stack/CI/tmp
-
-          TODAY=$(date +%Y%m%d)
-          LAST_TO_KEEP=$(date --date='-1 days' +%Y%m%d)
-          echo "TODAY is ${TODAY}"
-          echo "LAST_TO_KEEP is ${LAST_TO_KEEP}"
-
-          cd /home/ubuntu/spack-stack/CI/unified-env
-          for dir in `ls -1`; do echo ${dir}; if [[ ${dir} < ${LAST_TO_KEEP} ]]; then echo "Deleting ${dir}"; rm -fr ${dir}; fi; done
-
-      - name: checkout
-        uses: actions/checkout@v3
-        with:
-          submodules: true
-
-      - name: create-buildcache
-        run: |
-          source ./setup.sh
-
-          # Get day of week to set default container for scheduled builds
-          DOW=$(date +%u)
-          # Monday is 1 ... Sunday is 7
-          if [[ $DOW == 7 ]]; then
-            export USE_BINARY_CACHE=false
-            echo "Ignore existing binary cache for creating buildcache environment"
-          else
-            export USE_BINARY_CACHE=true
-            echo "Use existing binary cache for creating buildcache environment"
-          fi
-
-          # Make environment for creating/updating binary cache
-          export TEMPLATE=${{ inputs.template || 'unified-dev' }}
-          export ENVNAME=buildcache-${TEMPLATE}.ubuntu-rnd-x86_64
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache
-          spack env activate ${ENVDIR}
-          spack add ${{ inputs.specs || '' }}
-
-          # For buildcaches
-          spack config add config:install_tree:padded_length:200
-
-          # Add additional variants for MET packages, different from config/common/packages.yaml
-          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-
-          # Overrides for spack build and staging areas to speed up builds,
-          # separate from the default site config locations to avoid conflicts
-          spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"
-          spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage"
-          spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache"
-          spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache"
-
-          # Loop over compilers
-          declare -a compilers=("gcc" "intel")
-          cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original
-
-          for compiler in "${compilers[@]}"
-          do
-            # Set compiler and MPI
-            cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml
-            sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml
-            cat ${ENVDIR}/spack.yaml
-
-            # Concretize and check for duplicates
-            spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler}
-            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl
-
-            # Update spack source cache
-            spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache
-
-            # Add binary cache if requested
-            if [ "$USE_BINARY_CACHE" = true ] ; then
-              spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/
-              spack buildcache update-index local-binary
-              echo "Packages in spack binary cache:"
-              spack buildcache list
-            fi
-
-            # Create/update binary cache
-            echo "Create/update build cache for environment ${TEMPLATE} and compiler ${compiler}..."
-            spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler}
-            spack buildcache create -a -u /mnt/experiments-efs/spack-stack/build-cache/
-
-            # Next steps: synchronize source and build cache to a central/combined mirror?
-            #echo "Next steps ..."
-
-            # Remove binary cache for next round of concretization
-            if [ "$USE_BINARY_CACHE" = true ] ; then
-              spack mirror rm local-binary
-            fi
-          done
-
-          spack env deactivate
-          rm -fr /home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache
-
-      - name: create-env
-        run: |
-          source ./setup.sh
-          TODAY=$(date +%Y%m%d)
-
-          # Create actual environment from binary cache
-          export TEMPLATE=${{ inputs.template || 'unified-dev' }}
-          export ENVNAME=${TEMPLATE}.ubuntu-rnd-x86_64
-          export ENVDIR=$PWD/envs/${ENVNAME}
-          spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/unified-env/${TODAY}
-          spack env activate ${ENVDIR}
-          spack add ${{ inputs.specs || '' }}
-
-          # Add additional variants for MET packages, different from config/common/packages.yaml
-          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-
-          # Overrides for spack build and staging areas to speed up builds,
-          # separate from the default site config locations to avoid conflicts
-          spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"
-          spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage"
-          spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache"
-          spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache"
-
-          # Loop over compilers
-          declare -a compilers=("gcc" "intel")
-          cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original
-
-          for compiler in "${compilers[@]}"
-          do
-            # Set compiler and MPI
-            cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml
-            sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml
-            cat ${ENVDIR}/spack.yaml
-
-            # Concretize and check for duplicates
-            spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler}
-            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl
-
-            # Update spack source cache
-            spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache
-
-            # Add binary cache
-            spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/
-            spack buildcache update-index local-binary
-            echo "Packages in spack binary cache:"
-            spack buildcache list
-
-            # Install from binary cache
-            echo "Install environment ${TEMPLATE} for compiler ${compiler} from binary cache ..."
-            spack install --fail-fast --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler}
-            spack clean -a
-
-            # Create modules
-            spack module lmod refresh -y
-            spack stack setup-meta-modules
-
-            # Remove binary cache for next round of concretization
-            spack mirror rm local-binary
-          done
-
-          spack env deactivate
-
-      - name: test-env
-        run: |
-          source /etc/profile.d/z00_lmod.sh
-          module use /usr/share/modules/modulefiles
-          module use /opt/intel/mpi/2021.6.0/modulefiles
-          module use /home/ubuntu/jedi/modulefiles
-
-          TODAY=$(date +%Y%m%d)
-          echo "TODAY is ${TODAY}"
-          ls -l /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core
-
-          module use /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core
-          module load stack-intel/2021.6.0
-          module load stack-intel-oneapi-mpi/2021.6.0
-          module load stack-python/3.10.13
-          module available
-
-          module load jedi-ufs-env/1.0.0
-          module load ewok-env/1.0.0
-          module load soca-env/1.0.0
-          module list
-
-          module purge
-          module load stack-gcc/9.4.0
-          module load stack-openmpi/4.1.4
-          module load stack-python/3.10.13
-          module available
-
-          module load jedi-ufs-env/1.0.0
-          module load ewok-env/1.0.0
-          module load soca-env/1.0.0
-          module list
-
-          # Create empty file to signal that this environment is ready for testing
-          touch /home/ubuntu/spack-stack/CI/unified-env/${TODAY}.ready
-
-      # Report status to JCSDA CI slack channel for nightly runs only
-      - name: Report Status
-        if: always()
-        uses: ravsamhq/notify-slack-action@v1
-        env:
-          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
-        with:
-          # https://www.ravsam.in/blog/send-slack-notification-when-github-actions-fails/#getting-a-webhook-url
-          # https://www.ravsam.in/blog/collect-form-responses-using-google-apps-script
-          # https://github.com/marketplace/actions/notify-slack-action
-          status: ${{ job.status }}
-          notify_when: 'success,failure,warnings'
-          notification_title: '{workflow} has {status_message}'
-          message_format: '{emoji} *{workflow}* {status_message} for branch {branch} in <{repo_url}|{repo}>'
-          footer: ${{ github.event.pull_request.number || github.event_name || 'workflow dispatched manually' }}
-          # For testing: only notify user Dom
-          #mention_users: 'U02NLGXF5HV'
-          #mention_users_when: 'failure,warnings'
-          # Default: notify channel
-          mention_groups: '!channel'
-          mention_groups_when: 'failure,warnings'
+# Temporarily disabled 2024/03/18
+#name: ubuntu-rnd-x86_64-build
+#on:
+#  schedule:
+#    - cron: '0 0 * * *'
+#  workflow_dispatch:
+#
+#defaults:
+#  run:
+#    shell: bash
+#
+#jobs:
+#  ubuntu-rnd-x86_64-build:
+#    runs-on: [ubuntu-rnd-x86_64]
+#    timeout-minutes: 720 # 12 hours
+#
+#    steps:
+#      - name: cleanup
+#        run: |
+#          pwd
+#          ls -lart
+#          rm -fr *
+#          rm -fr /home/ubuntu/spack-stack/CI/tmp
+#          mkdir /home/ubuntu/spack-stack/CI/tmp
+#
+#          TODAY=$(date +%Y%m%d)
+#          LAST_TO_KEEP=$(date --date='-1 days' +%Y%m%d)
+#          echo "TODAY is ${TODAY}"
+#          echo "LAST_TO_KEEP is ${LAST_TO_KEEP}"
+#
+#          cd /home/ubuntu/spack-stack/CI/unified-env
+#          for dir in `ls -1`; do echo ${dir}; if [[ ${dir} < ${LAST_TO_KEEP} ]]; then echo "Deleting ${dir}"; rm -fr ${dir}; fi; done
+#
+#      - name: checkout
+#        uses: actions/checkout@v3
+#        with:
+#          submodules: true
+#
+#      - name: create-buildcache
+#        run: |
+#          source ./setup.sh
+#
+#          # Get day of week to set default container for scheduled builds
+#          DOW=$(date +%u)
+#          # Monday is 1 ... Sunday is 7
+#          if [[ $DOW == 7 ]]; then
+#            export USE_BINARY_CACHE=false
+#            echo "Ignore existing binary cache for creating buildcache environment"
+#          else
+#            export USE_BINARY_CACHE=true
+#            echo "Use existing binary cache for creating buildcache environment"
+#          fi
+#
+#          # Make environment for creating/updating binary cache
+#          export TEMPLATE=${{ inputs.template || 'unified-dev' }}
+#          export ENVNAME=buildcache-${TEMPLATE}.ubuntu-rnd-x86_64
+#          export ENVDIR=$PWD/envs/${ENVNAME}
+#          spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache
+#          spack env activate ${ENVDIR}
+#          spack add ${{ inputs.specs || '' }}
+#
+#          # For buildcaches
+#          spack config add config:install_tree:padded_length:200
+#
+#          # Add additional variants for MET packages, different from config/common/packages.yaml
+#          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
+#
+#          # Overrides for spack build and staging areas to speed up builds,
+#          # separate from the default site config locations to avoid conflicts
+#          spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"
+#          spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage"
+#          spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache"
+#          spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache"
+#
+#          # Loop over compilers
+#          declare -a compilers=("gcc" "intel")
+#          cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original
+#
+#          for compiler in "${compilers[@]}"
+#          do
+#            # Set compiler and MPI
+#            cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml
+#            sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml
+#            cat ${ENVDIR}/spack.yaml
+#
+#            # Concretize and check for duplicates
+#            spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler}
+#            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl
+#
+#            # Update spack source cache
+#            spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache
+#
+#            # Add binary cache if requested
+#            if [ "$USE_BINARY_CACHE" = true ] ; then
+#              spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/
+#              spack buildcache update-index local-binary
+#              echo "Packages in spack binary cache:"
+#              spack buildcache list
+#            fi
+#
+#            # Create/update binary cache
+#            echo "Create/update build cache for environment ${TEMPLATE} and compiler ${compiler}..."
+#            spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler}
+#            spack buildcache create -a -u /mnt/experiments-efs/spack-stack/build-cache/
+#
+#            # Next steps: synchronize source and build cache to a central/combined mirror?
+#            #echo "Next steps ..."
+#
+#            # Remove binary cache for next round of concretization
+#            if [ "$USE_BINARY_CACHE" = true ] ; then
+#              spack mirror rm local-binary
+#            fi
+#          done
+#
+#          spack env deactivate
+#          rm -fr /home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache
+#
+#      - name: create-env
+#        run: |
+#          source ./setup.sh
+#          TODAY=$(date +%Y%m%d)
+#
+#          # Create actual environment from binary cache
+#          export TEMPLATE=${{ inputs.template || 'unified-dev' }}
+#          export ENVNAME=${TEMPLATE}.ubuntu-rnd-x86_64
+#          export ENVDIR=$PWD/envs/${ENVNAME}
+#          spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/unified-env/${TODAY}
+#          spack env activate ${ENVDIR}
+#          spack add ${{ inputs.specs || '' }}
+#
+#          # Add additional variants for MET packages, different from config/common/packages.yaml
+#          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
+#
+#          # Overrides for spack build and staging areas to speed up builds,
+#          # separate from the default site config locations to avoid conflicts
+#          spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"
+#          spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage"
+#          spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache"
+#          spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache"
+#
+#          # Loop over compilers
+#          declare -a compilers=("gcc" "intel")
+#          cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original
+#
+#          for compiler in "${compilers[@]}"
+#          do
+#            # Set compiler and MPI
+#            cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml
+#            sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml
+#            cat ${ENVDIR}/spack.yaml
+#
+#            # Concretize and check for duplicates
+#            spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler}
+#            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl
+#
+#            # Update spack source cache
+#            spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache
+#
+#            # Add binary cache
+#            spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/
+#            spack buildcache update-index local-binary
+#            echo "Packages in spack binary cache:"
+#            spack buildcache list
+#
+#            # Install from binary cache
+#            echo "Install environment ${TEMPLATE} for compiler ${compiler} from binary cache ..."
+#            spack install --fail-fast --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler}
+#            spack clean -a
+#
+#            # Create modules
+#            spack module lmod refresh -y
+#            spack stack setup-meta-modules
+#
+#            # Remove binary cache for next round of concretization
+#            spack mirror rm local-binary
+#          done
+#
+#          spack env deactivate
+#
+#      - name: test-env
+#        run: |
+#          source /etc/profile.d/z00_lmod.sh
+#          module use /usr/share/modules/modulefiles
+#          module use /opt/intel/mpi/2021.6.0/modulefiles
+#          module use /home/ubuntu/jedi/modulefiles
+#
+#          TODAY=$(date +%Y%m%d)
+#          echo "TODAY is ${TODAY}"
+#          ls -l /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core
+#
+#          module use /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core
+#          module load stack-intel/2021.6.0
+#          module load stack-intel-oneapi-mpi/2021.6.0
+#          module load stack-python/3.10.13
+#          module available
+#
+#          module load jedi-ufs-env/1.0.0
+#          module load ewok-env/1.0.0
+#          module load soca-env/1.0.0
+#          module list
+#
+#          module purge
+#          module load stack-gcc/9.4.0
+#          module load stack-openmpi/4.1.4
+#          module load stack-python/3.10.13
+#          module available
+#
+#          module load jedi-ufs-env/1.0.0
+#          module load ewok-env/1.0.0
+#          module load soca-env/1.0.0
+#          module list
+#
+#          # Create empty file to signal that this environment is ready for testing
+#          touch /home/ubuntu/spack-stack/CI/unified-env/${TODAY}.ready
+#
+#      # Report status to JCSDA CI slack channel for nightly runs only
+#      - name: Report Status
+#        if: always()
+#        uses: ravsamhq/notify-slack-action@v1
+#        env:
+#          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
+#        with:
+#          # https://www.ravsam.in/blog/send-slack-notification-when-github-actions-fails/#getting-a-webhook-url
+#          # https://www.ravsam.in/blog/collect-form-responses-using-google-apps-script
+#          # https://github.com/marketplace/actions/notify-slack-action
+#          status: ${{ job.status }}
+#          notify_when: 'success,failure,warnings'
+#          notification_title: '{workflow} has {status_message}'
+#          message_format: '{emoji} *{workflow}* {status_message} for branch {branch} in <{repo_url}|{repo}>'
+#          footer: ${{ github.event.pull_request.number || github.event_name || 'workflow dispatched manually' }}
+#          # For testing: only notify user Dom
+#          #mention_users: 'U02NLGXF5HV'
+#          #mention_users_when: 'failure,warnings'
+#          # Default: notify channel
+#          mention_groups: '!channel'
+#          mention_groups_when: 'failure,warnings'

From e951bac9d9e15ab4daceccf5c28d50aba987a153 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 19 Mar 2024 08:34:51 -0600
Subject: [PATCH 06/11] Remove .github/workflows/ubuntu-rnd-x86_64.yaml and
 .github/workflows/macos-ci-x86_64.yaml

---
 .github/workflows/macos-ci-x86_64.yaml   | 142 --------------
 .github/workflows/ubuntu-rnd-x86_64.yaml | 236 -----------------------
 2 files changed, 378 deletions(-)
 delete mode 100644 .github/workflows/macos-ci-x86_64.yaml
 delete mode 100644 .github/workflows/ubuntu-rnd-x86_64.yaml

diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml
deleted file mode 100644
index 762ae30fe..000000000
--- a/.github/workflows/macos-ci-x86_64.yaml
+++ /dev/null
@@ -1,142 +0,0 @@
-# Temporarily disabled 2024/03/18
-#name: macos-ci-x86_64-build
-#on:
-#  pull_request:
-#    paths-ignore:
-#      - 'doc/**'
-#      - '**.md'
-#      - '.github/ISSUE_TEMPLATE/*'
-#      - '.gitignore'
-#  workflow_dispatch:
-#
-#concurrency:
-#  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-#  cancel-in-progress: true
-#
-#defaults:
-#  run:
-#    shell: bash
-#
-#jobs:
-#  macos-ci-x86_64-build:
-#    runs-on: [macos-ci-x86_64]
-#
-#    steps:
-#      - name: cleanup
-#        run: |
-#          pwd
-#          ls -lart
-#          rm -fr *
-#
-#      - name: checkout
-#        uses: actions/checkout@v2
-#        with:
-#          submodules: true
-#
-#      - name: create-env
-#        run: |
-#          source ./setup.sh
-#
-#          # DH* 20230302 - to avoid using padded_length for build caches,
-#          # always build in the same environment so that the length of the
-#          # path doesn't change - see also other 'DH* 20230302' changes below
-#          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64
-#          export ENVNAME=ci-env.macos-ci-x86_64
-#          # *DH
-#          export ENVDIR=$PWD/envs/${ENVNAME}
-#          spack stack create env --site macos.default --template ${{ inputs.template || 'unified-dev' }} --name ${ENVNAME}
-#          spack env activate ${ENVDIR}
-#          spack add ${{ inputs.specs || '' }}
-#          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
-#
-#          # Find external packages
-#          spack external find --scope system \
-#              --exclude bison --exclude openssl  \
-#              --exclude python --exclude gettext
-#          # Need to find gettext outside of default (presumed to be a system) path for krb5
-#          spack external find --path=/usr/local/Cellar/gettext/0.21.1 gettext
-#          spack external find --scope system perl
-#          spack external find --scope system wget
-#          PATH="/usr/local/opt/curl/bin:$PATH" \
-#              spack external find --scope system curl
-#          PATH="/usr/local/opt/qt5/bin:$PATH" \
-#              spack external find --scope system qt
-#          spack external find --scope system texlive
-#          spack external find --scope system mysql
-#
-#          # Find compilers
-#          spack compiler find --scope system
-#
-#          export -n SPACK_SYSTEM_CONFIG_PATH
-#
-#          # DH* 20230302 - Don't use yet, too many problems
-#          ## For buildcaches
-#          #spack config add config:install_tree:padded_length:true
-#          # *DH
-#
-#          # Set compiler and MPI
-#          spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
-#          spack config add "packages:all:compiler:[apple-clang@14.0.0]"
-#          sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
-#
-#          # Add additional variants for MET packages, different from config/common/packages.yaml
-#          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-#
-#          # Concretize and check for duplicates
-#          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0
-#          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm -i esmf -i mapl
-#
-#          # Add and update source cache
-#          spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
-#          spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/
-#
-#          # Add binary cache and reindex it
-#          spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
-#          spack buildcache update-index local-binary
-#          echo "Packages in combined spack build caches:"
-#          spack buildcache list
-#
-#          # Break installation up in pieces and create build caches in between
-#          # This allows us to "spin up" builds that altogether take longer than
-#          # six hours, and/or fail later in the build process.
-#
-#          # base-env
-#          echo "base-env ..."
-#          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env
-#          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
-#
-#          # the rest
-#          echo "${{ inputs.template || 'unified-dev' }} ..."
-#          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'unified-dev' }}
-#          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
-#
-#          # Next steps: synchronize source and build cache to a central/combined mirror?
-#          echo "Next steps ..."
-#          spack clean -a
-#          spack module lmod refresh -y
-#          spack stack setup-meta-modules
-#          spack env deactivate
-#
-#      - name: test-env
-#        run: |
-#          source /usr/local/opt/lmod/init/profile
-#
-#          # DH* 20230302 - to avoid using padded_length for build caches,
-#          # always build in the same environment so that the length of the
-#          # path doesn't change - see also other 'DH* 20230302' changes below
-#          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-x86_64
-#          export ENVNAME=ci-env.macos-ci-x86_64
-#          # *DH
-#          export ENVDIR=$PWD/envs/${ENVNAME}
-#          ls -l ${ENVDIR}/install/modulefiles/Core
-#
-#          module use ${ENVDIR}/install/modulefiles/Core
-#          module load stack-apple-clang/14.0.0
-#          module load stack-openmpi/5.0.1
-#          module load stack-python/3.10.13
-#          module available
-#
-#          module load jedi-ufs-env/1.0.0
-#          module load ewok-env/1.0.0
-#          module load soca-env/1.0.0
-#          module list
diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml
deleted file mode 100644
index ebd8806b1..000000000
--- a/.github/workflows/ubuntu-rnd-x86_64.yaml
+++ /dev/null
@@ -1,236 +0,0 @@
-# Temporarily disabled 2024/03/18
-#name: ubuntu-rnd-x86_64-build
-#on:
-#  schedule:
-#    - cron: '0 0 * * *'
-#  workflow_dispatch:
-#
-#defaults:
-#  run:
-#    shell: bash
-#
-#jobs:
-#  ubuntu-rnd-x86_64-build:
-#    runs-on: [ubuntu-rnd-x86_64]
-#    timeout-minutes: 720 # 12 hours
-#
-#    steps:
-#      - name: cleanup
-#        run: |
-#          pwd
-#          ls -lart
-#          rm -fr *
-#          rm -fr /home/ubuntu/spack-stack/CI/tmp
-#          mkdir /home/ubuntu/spack-stack/CI/tmp
-#
-#          TODAY=$(date +%Y%m%d)
-#          LAST_TO_KEEP=$(date --date='-1 days' +%Y%m%d)
-#          echo "TODAY is ${TODAY}"
-#          echo "LAST_TO_KEEP is ${LAST_TO_KEEP}"
-#
-#          cd /home/ubuntu/spack-stack/CI/unified-env
-#          for dir in `ls -1`; do echo ${dir}; if [[ ${dir} < ${LAST_TO_KEEP} ]]; then echo "Deleting ${dir}"; rm -fr ${dir}; fi; done
-#
-#      - name: checkout
-#        uses: actions/checkout@v3
-#        with:
-#          submodules: true
-#
-#      - name: create-buildcache
-#        run: |
-#          source ./setup.sh
-#
-#          # Get day of week to set default container for scheduled builds
-#          DOW=$(date +%u)
-#          # Monday is 1 ... Sunday is 7
-#          if [[ $DOW == 7 ]]; then
-#            export USE_BINARY_CACHE=false
-#            echo "Ignore existing binary cache for creating buildcache environment"
-#          else
-#            export USE_BINARY_CACHE=true
-#            echo "Use existing binary cache for creating buildcache environment"
-#          fi
-#
-#          # Make environment for creating/updating binary cache
-#          export TEMPLATE=${{ inputs.template || 'unified-dev' }}
-#          export ENVNAME=buildcache-${TEMPLATE}.ubuntu-rnd-x86_64
-#          export ENVDIR=$PWD/envs/${ENVNAME}
-#          spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache
-#          spack env activate ${ENVDIR}
-#          spack add ${{ inputs.specs || '' }}
-#
-#          # For buildcaches
-#          spack config add config:install_tree:padded_length:200
-#
-#          # Add additional variants for MET packages, different from config/common/packages.yaml
-#          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-#
-#          # Overrides for spack build and staging areas to speed up builds,
-#          # separate from the default site config locations to avoid conflicts
-#          spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"
-#          spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage"
-#          spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache"
-#          spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache"
-#
-#          # Loop over compilers
-#          declare -a compilers=("gcc" "intel")
-#          cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original
-#
-#          for compiler in "${compilers[@]}"
-#          do
-#            # Set compiler and MPI
-#            cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml
-#            sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml
-#            cat ${ENVDIR}/spack.yaml
-#
-#            # Concretize and check for duplicates
-#            spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler}
-#            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl
-#
-#            # Update spack source cache
-#            spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache
-#
-#            # Add binary cache if requested
-#            if [ "$USE_BINARY_CACHE" = true ] ; then
-#              spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/
-#              spack buildcache update-index local-binary
-#              echo "Packages in spack binary cache:"
-#              spack buildcache list
-#            fi
-#
-#            # Create/update binary cache
-#            echo "Create/update build cache for environment ${TEMPLATE} and compiler ${compiler}..."
-#            spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler}
-#            spack buildcache create -a -u /mnt/experiments-efs/spack-stack/build-cache/
-#
-#            # Next steps: synchronize source and build cache to a central/combined mirror?
-#            #echo "Next steps ..."
-#
-#            # Remove binary cache for next round of concretization
-#            if [ "$USE_BINARY_CACHE" = true ] ; then
-#              spack mirror rm local-binary
-#            fi
-#          done
-#
-#          spack env deactivate
-#          rm -fr /home/ubuntu/spack-stack/CI/tmp/spack-stack-dev-buildcache
-#
-#      - name: create-env
-#        run: |
-#          source ./setup.sh
-#          TODAY=$(date +%Y%m%d)
-#
-#          # Create actual environment from binary cache
-#          export TEMPLATE=${{ inputs.template || 'unified-dev' }}
-#          export ENVNAME=${TEMPLATE}.ubuntu-rnd-x86_64
-#          export ENVDIR=$PWD/envs/${ENVNAME}
-#          spack stack create env --site aws-pcluster --template ${TEMPLATE} --name ${ENVNAME} --prefix=/home/ubuntu/spack-stack/CI/unified-env/${TODAY}
-#          spack env activate ${ENVDIR}
-#          spack add ${{ inputs.specs || '' }}
-#
-#          # Add additional variants for MET packages, different from config/common/packages.yaml
-#          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
-#
-#          # Overrides for spack build and staging areas to speed up builds,
-#          # separate from the default site config locations to avoid conflicts
-#          spack config add "config:build_stage:/home/ubuntu/spack-stack/CI/tmp/build_stage"
-#          spack config add "config:test_stage:/home/ubuntu/spack-stack/CI/tmp/test_stage"
-#          spack config add "config:source_cache:/home/ubuntu/spack-stack/CI/tmp/source_cache"
-#          spack config add "config:misc_cache:/home/ubuntu/spack-stack/CI/tmp/misc_cache"
-#
-#          # Loop over compilers
-#          declare -a compilers=("gcc" "intel")
-#          cp ${ENVDIR}/spack.yaml ${ENVDIR}/spack.yaml.original
-#
-#          for compiler in "${compilers[@]}"
-#          do
-#            # Set compiler and MPI
-#            cp ${ENVDIR}/spack.yaml.original ${ENVDIR}/spack.yaml
-#            sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%${compiler}'\]/g" ${ENVDIR}/spack.yaml
-#            cat ${ENVDIR}/spack.yaml
-#
-#            # Concretize and check for duplicates
-#            spack concretize --force --fresh 2>&1 | tee log.concretize.${ENVNAME}.${compiler}
-#            ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.${ENVNAME}.${compiler} -i fms -i crtm -i esmf -i mapl
-#
-#            # Update spack source cache
-#            spack mirror create -a -d /mnt/experiments-efs/spack-stack/source-cache
-#
-#            # Add binary cache
-#            spack mirror add local-binary file:///mnt/experiments-efs/spack-stack/build-cache/
-#            spack buildcache update-index local-binary
-#            echo "Packages in spack binary cache:"
-#            spack buildcache list
-#
-#            # Install from binary cache
-#            echo "Install environment ${TEMPLATE} for compiler ${compiler} from binary cache ..."
-#            spack install --fail-fast --no-check-signature 2>&1 | tee log.install.${TEMPLATE}.${compiler}
-#            spack clean -a
-#
-#            # Create modules
-#            spack module lmod refresh -y
-#            spack stack setup-meta-modules
-#
-#            # Remove binary cache for next round of concretization
-#            spack mirror rm local-binary
-#          done
-#
-#          spack env deactivate
-#
-#      - name: test-env
-#        run: |
-#          source /etc/profile.d/z00_lmod.sh
-#          module use /usr/share/modules/modulefiles
-#          module use /opt/intel/mpi/2021.6.0/modulefiles
-#          module use /home/ubuntu/jedi/modulefiles
-#
-#          TODAY=$(date +%Y%m%d)
-#          echo "TODAY is ${TODAY}"
-#          ls -l /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core
-#
-#          module use /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core
-#          module load stack-intel/2021.6.0
-#          module load stack-intel-oneapi-mpi/2021.6.0
-#          module load stack-python/3.10.13
-#          module available
-#
-#          module load jedi-ufs-env/1.0.0
-#          module load ewok-env/1.0.0
-#          module load soca-env/1.0.0
-#          module list
-#
-#          module purge
-#          module load stack-gcc/9.4.0
-#          module load stack-openmpi/4.1.4
-#          module load stack-python/3.10.13
-#          module available
-#
-#          module load jedi-ufs-env/1.0.0
-#          module load ewok-env/1.0.0
-#          module load soca-env/1.0.0
-#          module list
-#
-#          # Create empty file to signal that this environment is ready for testing
-#          touch /home/ubuntu/spack-stack/CI/unified-env/${TODAY}.ready
-#
-#      # Report status to JCSDA CI slack channel for nightly runs only
-#      - name: Report Status
-#        if: always()
-#        uses: ravsamhq/notify-slack-action@v1
-#        env:
-#          SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }}
-#        with:
-#          # https://www.ravsam.in/blog/send-slack-notification-when-github-actions-fails/#getting-a-webhook-url
-#          # https://www.ravsam.in/blog/collect-form-responses-using-google-apps-script
-#          # https://github.com/marketplace/actions/notify-slack-action
-#          status: ${{ job.status }}
-#          notify_when: 'success,failure,warnings'
-#          notification_title: '{workflow} has {status_message}'
-#          message_format: '{emoji} *{workflow}* {status_message} for branch {branch} in <{repo_url}|{repo}>'
-#          footer: ${{ github.event.pull_request.number || github.event_name || 'workflow dispatched manually' }}
-#          # For testing: only notify user Dom
-#          #mention_users: 'U02NLGXF5HV'
-#          #mention_users_when: 'failure,warnings'
-#          # Default: notify channel
-#          mention_groups: '!channel'
-#          mention_groups_when: 'failure,warnings'

From 0aacaa234c47d9547f9195debdb994f21cb10108 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 19 Mar 2024 17:36:03 -0600
Subject: [PATCH 07/11] Update .gitmodules and submodule pointer for spack

---
 .gitmodules                       | 10 ++++++----
 doc/source/PreConfiguredSites.rst |  4 ++--
 spack                             |  2 +-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index b1f475b64..a1ccdc045 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,9 +1,11 @@
 [submodule "spack"]
   path = spack
-  #url = https://github.com/spack/spack
-  #branch = develop
-  url = https://github.com/jcsda/spack
-  branch = jcsda_emc_spack_stack
+  ##url = https://github.com/spack/spack
+  ##branch = develop
+  #url = https://github.com/jcsda/spack
+  #branch = jcsda_emc_spack_stack
+  url = https://github.com/climbfuji/spack
+  branch = bugfix/py_jupyter_server_npm_node
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules
diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst
index 65df66f49..ffb5e788e 100644
--- a/doc/source/PreConfiguredSites.rst
+++ b/doc/source/PreConfiguredSites.rst
@@ -260,6 +260,7 @@ For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules
 .. code-block:: console
 
    # These extra steps are required for performance reason, ofi is about 30% slower than ucx
+   # Note we can't load craype-network-ucx for building spack-stack environments, must do here
    module unload craype-network-ofi
    module load craype-network-ucx
    module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core
@@ -282,8 +283,6 @@ With GNU, the following is required for building new spack environments and for
    module load cray-python/3.9.7.1
    module unload cray-libsci
    module load cray-libsci/22.08.1.1
-   module unload craype-network-ofi
-   module load craype-network-ucx
 
    module use /p/app/projects/NEPTUNE/spack-stack/modulefiles
    module load ecflow/5.8.4
@@ -293,6 +292,7 @@ For ``spack-stack-1.6.0`` with GNU, proceed with loading the following modules:
 .. code-block:: console
 
    # These extra steps are required for performance reason, ofi is about 30% slower than ucx
+   # Note we can't load craype-network-ucx for building spack-stack environments, must do here
    module unload craype-network-ofi
    module load craype-network-ucx
    module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core
diff --git a/spack b/spack
index dbc8d59d7..ca02e1331 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit dbc8d59d7c24d7d9014dc801cbfd4b1f5d9187f0
+Subproject commit ca02e133146edf08f41c6c3229594cc3485fae2c

From 255185b812559d185261d32bb4bd8fbecece815f Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Tue, 19 Mar 2024 17:40:39 -0600
Subject: [PATCH 08/11] Update submodule pointer for spack

---
 spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spack b/spack
index ca02e1331..92292b7ab 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit ca02e133146edf08f41c6c3229594cc3485fae2c
+Subproject commit 92292b7ab7fc4367aaf785d58239acc97904f919

From c09894224e5fbcb2f1f87defcd6e4b148bf64daf Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 20 Mar 2024 05:46:52 -0600
Subject: [PATCH 09/11] Bug fix in configs/common/modules_*.yaml: need protobuf
 and py-protobuf modules for cylc-jupyter server to work

---
 configs/common/modules_lmod.yaml | 2 --
 configs/common/modules_tcl.yaml  | 2 --
 setup.sh                         | 1 -
 spack                            | 2 +-
 4 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml
index 383071612..6b2b9af75 100644
--- a/configs/common/modules_lmod.yaml
+++ b/configs/common/modules_lmod.yaml
@@ -71,8 +71,6 @@ modules:
       - openssl
       - perl
       - pkgconf
-      - protobuf
-      - py-protobuf
       - qt
       - randrproto
       - readline
diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml
index 58fd733c7..8011b215f 100644
--- a/configs/common/modules_tcl.yaml
+++ b/configs/common/modules_tcl.yaml
@@ -73,8 +73,6 @@ modules:
       - openssl
       - perl
       - pkgconf
-      - protobuf
-      - py-protobuf
       - qt
       - randrproto
       - readline
diff --git a/setup.sh b/setup.sh
index 4b1b15e53..23c3b7ed1 100755
--- a/setup.sh
+++ b/setup.sh
@@ -8,7 +8,6 @@ echo "Setting environment variable SPACK_STACK_DIR to ${SPACK_STACK_DIR}"
 source ${SPACK_STACK_DIR:?}/spack/share/spack/setup-env.sh
 echo "Sourcing spack environment ${SPACK_STACK_DIR}/spack/share/spack/setup-env.sh"
 if [ "$(uname)" != "Darwin" ]; then
-    echo "DO IT"
   export SPACK_USER_CACHE_PATH=$SPACK_ROOT/user_cache
 fi
 
diff --git a/spack b/spack
index 92292b7ab..dde26846c 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit 92292b7ab7fc4367aaf785d58239acc97904f919
+Subproject commit dde26846c3cd2f687a0a25e0e244e5f074041ef0

From 9445c7c5dcd71f792e9534c7b29aaa402dac1a01 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 20 Mar 2024 07:30:15 -0600
Subject: [PATCH 10/11] Update submodule pointer for spack

---
 spack | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spack b/spack
index dde26846c..a8569fd31 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit dde26846c3cd2f687a0a25e0e244e5f074041ef0
+Subproject commit a8569fd31a0adc7fb92e42b0ef4ab265c002c92b

From ab616f9db9061631565e3a2aed4b1aa9e884c576 Mon Sep 17 00:00:00 2001
From: Dom Heinzeller <dom.heinzeller@icloud.com>
Date: Wed, 20 Mar 2024 09:15:15 -0600
Subject: [PATCH 11/11] Revert .gitmodules and update submodule pointer for
 spack

---
 .gitmodules | 10 ++++------
 spack       |  2 +-
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index a1ccdc045..b1f475b64 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,11 +1,9 @@
 [submodule "spack"]
   path = spack
-  ##url = https://github.com/spack/spack
-  ##branch = develop
-  #url = https://github.com/jcsda/spack
-  #branch = jcsda_emc_spack_stack
-  url = https://github.com/climbfuji/spack
-  branch = bugfix/py_jupyter_server_npm_node
+  #url = https://github.com/spack/spack
+  #branch = develop
+  url = https://github.com/jcsda/spack
+  branch = jcsda_emc_spack_stack
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules
diff --git a/spack b/spack
index a8569fd31..4fafe3ead 160000
--- a/spack
+++ b/spack
@@ -1 +1 @@
-Subproject commit a8569fd31a0adc7fb92e42b0ef4ab265c002c92b
+Subproject commit 4fafe3eadbe330e64423b8ee16c88f8309f48d5b