From c08f8cc6bd37b0c9f6825994a82f9391e7f1dc9f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 5 Oct 2023 09:33:05 -0600 Subject: [PATCH 01/12] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..f794b476c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack + branch = feature/enable_awscli_v2_update_swell [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 0148496ae..df0d10a96 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 0148496ae6cdf991338747934c85f3b609f0c7bf +Subproject commit df0d10a964c04c1745b4db0e662349c469961dc8 From 7c41e4c4530de4103a59fd8f83e8c55a55907eb7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 5 Oct 2023 09:33:40 -0600 Subject: [PATCH 02/12] Replace awscli with awscli-v2 in configs/containers/specs/jedi-ci.yaml --- configs/containers/specs/jedi-ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/containers/specs/jedi-ci.yaml b/configs/containers/specs/jedi-ci.yaml index dced36307..220eecc9d 100644 --- a/configs/containers/specs/jedi-ci.yaml +++ b/configs/containers/specs/jedi-ci.yaml @@ -11,7 +11,7 @@ py-pandas@1.5.3, py-pip, py-pyyaml@5.4.1, py-scipy@1.9.3, py-shapely@1.8.0, py-xarray@2022.3.0, sp@2.3.3, udunits@2.2.28, w3nco@2.4.1, w3emc@2.10.0, nco@5.0.6, esmf@8.4.2, mapl@2.35.2, yafyaml@0.5.1, zlib@1.2.13, zstd@1.5.2, odc@1.4.6, shumlib@macos_clang_linux_intel_port, - awscli@1.27.84, py-globus-cli@3.16.0] + awscli-v2@2.13.22, py-globus-cli@3.16.0] # Notes: # 1. Remove mapl@2.35.2 from clang/mpich container, because mapl doesn't work with mpich@4 # 2. Don't build CRTM by default so that it gets built in the JEDI bundles From c12165d325f52c9bcccae7148932f4a93e9ff76b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 5 Oct 2023 09:33:53 -0600 Subject: [PATCH 03/12] Add gmao-swell-env to configs/templates/unified-dev/spack.yaml --- configs/templates/unified-dev/spack.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 2effbc12c..3733c94b9 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -9,6 +9,7 @@ spack: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel'] - packages: - global-workflow-env + - gmao-swell-env - gsi-env - ewok-env - jedi-fv3-env From 6dc89068ddc2c21d0e5b1ae88d00b9d9e6f5f799 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 5 Oct 2023 13:42:15 -0600 Subject: [PATCH 04/12] Remove versions for py-pycodestyle and py-pyyaml from configs/common/packages.yaml to avoid duplicate packages being built --- configs/common/packages.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 835744771..92e67fe89 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -248,8 +248,8 @@ version: ['1.0.8'] py-pybind11: version: ['2.8.1'] - py-pycodestyle: - version: ['2.8.0'] + #py-pycodestyle: + # version: ['2.8.0'] py-pygithub: version: ['1.55'] py-pygrib: @@ -264,9 +264,9 @@ # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9, # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers version: ['0.12.2'] - # This version of py-pyyaml goes with awscli@1.27.84 - py-pyyaml: - version: ['5.4.1'] + ## This version of py-pyyaml goes with awscli@1.27.84 + #py-pyyaml: + # version: ['5.4.1'] py-scipy: version: ['1.9.3'] # Pin the py-setuptools version to avoid duplicate Python packages From 41e4796d520dcb7e4e0fcde30662b6503d0ff726 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 6 Oct 2023 10:48:10 -0600 Subject: [PATCH 05/12] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index df0d10a96..b493e78a6 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit df0d10a964c04c1745b4db0e662349c469961dc8 +Subproject commit b493e78a6ef25238ab650f95350cbdcf4163ad7f From a84ed41b2a2cc0aa24fd07e9b9c88b5d13f54352 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 6 Oct 2023 10:58:01 -0600 Subject: [PATCH 06/12] Clean up old comments in configs/common/packages.yaml --- configs/common/packages.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 92e67fe89..0cda25783 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -238,9 +238,6 @@ variants: +blas +lapack py-openpyxl: version: ['3.0.3'] - # DH* 20230719 try without version - #py-pandas: - # version: ['1.4.0'] # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer # versions of py-poetry-core when using external/homebrew Python as # we do at the moment in spack-stack. @@ -248,8 +245,6 @@ version: ['1.0.8'] py-pybind11: version: ['2.8.1'] - #py-pycodestyle: - # version: ['2.8.0'] py-pygithub: version: ['1.55'] py-pygrib: @@ -264,9 +259,6 @@ # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9, # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers version: ['0.12.2'] - ## This version of py-pyyaml goes with awscli@1.27.84 - #py-pyyaml: - # version: ['5.4.1'] py-scipy: version: ['1.9.3'] # Pin the py-setuptools version to avoid duplicate Python packages @@ -290,14 +282,6 @@ version: ['2.3.2'] sp: version: ['2.3.3'] - #texlive: - # Assume texlive is provided, hard to install - # because of its dependencies. Both Linux and - # macOS provide easy-to-install packages that - # can be added as external packages. - # Note: Uncommenting this entry will break - # the container builds. - #version: ['2.11.0'] udunits: version: ['2.2.28'] upp: From 91293ffe541aa01b8d86d12320d48b53c02e7e40 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 6 Oct 2023 19:14:05 -0600 Subject: [PATCH 07/12] Clean up .github/workflows/ubuntu-ci-containers-x86_64.yaml --- .github/workflows/ubuntu-ci-containers-x86_64.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ubuntu-ci-containers-x86_64.yaml b/.github/workflows/ubuntu-ci-containers-x86_64.yaml index 7de03c236..6be1cf822 100644 --- a/.github/workflows/ubuntu-ci-containers-x86_64.yaml +++ b/.github/workflows/ubuntu-ci-containers-x86_64.yaml @@ -82,7 +82,6 @@ jobs: # Report status to JCSDA CI slack channel for nightly runs only - name: Report Status if: always() - #'!cancelled()' && ${{ github.event_name == 'schedule' }} uses: ravsamhq/notify-slack-action@v1 env: SLACK_WEBHOOK_URL: ${{ secrets.ACTION_MONITORING_SLACK }} From 2b926fc2ed735a89b9ff3469f46a444f2765c0a1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 9 Oct 2023 07:17:03 -0600 Subject: [PATCH 08/12] Workaround for spack-stack-1.5.0 on Hera: use new noavx512 install, add target 'haswell' --- configs/sites/hera/packages.yaml | 2 ++ configs/sites/jet/packages.yaml | 1 + doc/source/PreConfiguredSites.rst | 8 +++----- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configs/sites/hera/packages.yaml b/configs/sites/hera/packages.yaml index 5b1a7d79a..8a0c351da 100644 --- a/configs/sites/hera/packages.yaml +++ b/configs/sites/hera/packages.yaml @@ -5,6 +5,8 @@ packages: providers: mpi:: [intel-oneapi-mpi@2021.5.1, openmpi@4.1.5] #mpi:: [intel-mpi@2018.0.4] + # To support hecflow01 + target: [haswell] mpi: buildable: False intel-mpi: diff --git a/configs/sites/jet/packages.yaml b/configs/sites/jet/packages.yaml index 677fe4df9..abbfe47f3 100644 --- a/configs/sites/jet/packages.yaml +++ b/configs/sites/jet/packages.yaml @@ -5,6 +5,7 @@ packages: providers: mpi:: [intel-oneapi-mpi@2021.5.1, openmpi@3.1.4] #mpi:: [intel-mpi@2018.4.274] + # To support all generations of jet target: [core2] mpi: buildable: False diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index db3a87a48..cd7b04df2 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -36,7 +36,7 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Gaea C5 | Intel | ``/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-1.5.0/envs/unified-env`` | Dom Heinzeller / ??? | | NOAA (RDHPCS) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hera^** | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | +| | Hera^** | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512`` | Mark Potts / Dom Heinzeller | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Jet^** | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -539,7 +539,7 @@ For ``spack-stack-1.5.0`` with Intel, load the following modules after loading m .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.10.8 @@ -549,7 +549,7 @@ For ``spack-stack-1.5.0`` with GNU, load the following modules after loading min .. code-block:: console - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core module load stack-gcc/9.2.0 module load stack-openmpi/4.1.5 module load stack-python/3.10.8 @@ -557,8 +557,6 @@ For ``spack-stack-1.5.0`` with GNU, load the following modules after loading min Note that on Hera, a dedicated node exists for ``ecflow`` server jobs (``hecflow01``). Users starting ``ecflow_server`` on the regular login nodes will see their servers being killed every few minutes, and may be barred from accessing the system. -Further, note that the ``spack-stack-1.5.0`` unified environment on Hera has an additional package ``yafyaml`` installed that does not exist in the default 1.5.0 installation. - .. _Preconfigured_Sites_Jet: ------------------------------ From 543d4f6ebc337c758e563e3af17bf985828ba7a3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 9 Oct 2023 07:24:35 -0600 Subject: [PATCH 09/12] Update .github/workflows/macos-ci-aarch64.yaml --- .github/workflows/macos-ci-aarch64.yaml | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index f183bfa6b..034743db8 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -89,15 +89,15 @@ jobs: # Set compiler and MPI spack config add "packages:all:providers:mpi:[openmpi]" - spack config add "packages:all:compiler:[apple-clang@14.0.0]" + spack config add "packages:all:compiler:[apple-clang@14.0.3]" sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.0 -i fms -i crtm + spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm # Add and update source cache spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ @@ -111,12 +111,6 @@ jobs: echo "Packages in combined spack build caches:" spack buildcache list - # Workaround for limited disk space on macOS arm instance - spack config add "config:build_stage:/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/build_stage" - spack config add "config:test_stage:/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/test_stage" - spack config add "config:source_cache:/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/source_cache" - spack config add "config:misc_cache:/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/misc_cache" - # Break installation up in pieces and create build caches in between # This allows us to "spin up" builds that altogether take longer than # six hours, and/or fail later in the build process. @@ -124,14 +118,14 @@ jobs: # base-env echo "base-env ..." # DH* 20230721 - todo remove --no-checksum - spack install --fail-fast --source --no-check-signature --no-checksum base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env + spack install --fail-fast --source --no-check-signature --no-checksum base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env # DH* 20230721 - todo remove || true (this was here all the time, but should not be needed if spack creates buildcaches correctly) spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ || true # the rest echo "${{ inputs.template || 'unified-dev' }} ..." # DH* 20230721 - todo remove --no-checksum - spack install --fail-fast --source --no-check-signature --no-checksum 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'unified-dev' }} + spack install --fail-fast --source --no-check-signature --no-checksum 2>&1 | tee log.install.apple-clang-14.0.3.${{ inputs.template || 'unified-dev' }} # DH* 20230721 - todo remove || true (this was here all the time, but should not be needed if spack creates buildcaches correctly) spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ || true @@ -158,7 +152,7 @@ jobs: ls -l ${ENVDIR}/install/modulefiles/Core module use ${ENVDIR}/install/modulefiles/Core - module load stack-apple-clang/14.0.0 + module load stack-apple-clang/14.0.3 module load stack-openmpi/4.1.5 module load stack-python/3.10.8 module available From 2db7a7fb9bdef168d57654ca0dcc1b36a3179b4d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 9 Oct 2023 07:34:12 -0600 Subject: [PATCH 10/12] Re-trigger CI From 577b082d75d93e77e30366762f0e631fdf50b3f1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 9 Oct 2023 13:07:12 -0600 Subject: [PATCH 11/12] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index b493e78a6..9b7c34e30 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit b493e78a6ef25238ab650f95350cbdcf4163ad7f +Subproject commit 9b7c34e30a5a7f6510172740747f1856f1b89e2f From 4f4562b8896093ffcf8af42b861445c24611bba6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 9 Oct 2023 13:08:33 -0600 Subject: [PATCH 12/12] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index f794b476c..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = jcsda_emc_spack_stack - url = https://github.com/climbfuji/spack - branch = feature/enable_awscli_v2_update_swell + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 9b7c34e30..35c3a6131 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 9b7c34e30a5a7f6510172740747f1856f1b89e2f +Subproject commit 35c3a613104ebc2db7b482df594c35182175d033