From fab1fc5245bc91e6fcee5dcebcf83c9cd51a9032 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 20 Jan 2023 07:35:30 -0700 Subject: [PATCH] Add gaea-c5 config, bugfix for Narwhal config (supersedes #439) (#445) * Add gaea-c5 config. Tested with ufs-weather-model-static template. * Bug fix in Narwhal compiler config, and update of Gaea C5 config * Progress on documentation update in doc/source/Platforms.rst * Final touches to gaea-c5 site config (packages) Co-authored-by: Alexander Richert --- configs/sites/gaea-c5/compilers.yaml | 27 +++ configs/sites/gaea-c5/config.yaml | 2 + configs/sites/gaea-c5/mirrors.yaml | 18 ++ configs/sites/gaea-c5/modules.yaml | 4 + configs/sites/gaea-c5/packages.yaml | 241 +++++++++++++++++++++++++++ configs/sites/narwhal/compilers.yaml | 2 +- doc/source/Platforms.rst | 31 +++- 7 files changed, 315 insertions(+), 10 deletions(-) create mode 100644 configs/sites/gaea-c5/compilers.yaml create mode 100644 configs/sites/gaea-c5/config.yaml create mode 100644 configs/sites/gaea-c5/mirrors.yaml create mode 100644 configs/sites/gaea-c5/modules.yaml create mode 100644 configs/sites/gaea-c5/packages.yaml diff --git a/configs/sites/gaea-c5/compilers.yaml b/configs/sites/gaea-c5/compilers.yaml new file mode 100644 index 000000000..5fc3ee871 --- /dev/null +++ b/configs/sites/gaea-c5/compilers.yaml @@ -0,0 +1,27 @@ +compilers: +- compiler: + spec: intel@2022.0.2 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + target: any + modules: + - PrgEnv-intel/8.3.3 + - intel/2022.0.2 + - craype/2.7.15 + - libfabric/1.11.0.4.128 + environment: + prepend_path: + PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' + LD_LIBRARY_PATH: '/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + set: + # OpenSUSE on Gaea C5 sets CONFIG_SITE so + # Automake-based builds are installed in lib64 + # which confuses some packages. + CONFIG_SITE: '' + extra_rpaths: [] diff --git a/configs/sites/gaea-c5/config.yaml b/configs/sites/gaea-c5/config.yaml new file mode 100644 index 000000000..7e559f33f --- /dev/null +++ b/configs/sites/gaea-c5/config.yaml @@ -0,0 +1,2 @@ +config: + build_jobs: 6 diff --git a/configs/sites/gaea-c5/mirrors.yaml b/configs/sites/gaea-c5/mirrors.yaml new file mode 100644 index 000000000..372fe7bd3 --- /dev/null +++ b/configs/sites/gaea-c5/mirrors.yaml @@ -0,0 +1,18 @@ +mirrors: + local-source: + fetch: + url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null + push: + url: file:///lustre/f2/pdata/esrl/gsd/spack-stack/source-cache + access_pair: + - null + - null + access_token: null + profile: null + endpoint_url: null diff --git a/configs/sites/gaea-c5/modules.yaml b/configs/sites/gaea-c5/modules.yaml new file mode 100644 index 000000000..4e9771f38 --- /dev/null +++ b/configs/sites/gaea-c5/modules.yaml @@ -0,0 +1,4 @@ +modules: + default: + enable:: + - lmod diff --git a/configs/sites/gaea-c5/packages.yaml b/configs/sites/gaea-c5/packages.yaml new file mode 100644 index 000000000..0f9cf4e99 --- /dev/null +++ b/configs/sites/gaea-c5/packages.yaml @@ -0,0 +1,241 @@ + packages: + all: + compiler:: [intel@2022.0.2] + providers: + mpi:: [cray-mpich@8.1.16] + cray-mpich: + externals: + - spec: cray-mpich@8.1.16%intel@2022.0.2~wrappers + modules: + - libfabric + - craype-network-ofi + - cray-mpich/8.1.16 + esmf: + version: [8.3.0b09] + variants: ~xerces ~pnetcdf +pio esmf_os=Linux esmf_comm=mpich3 + python: + buildable: false + externals: + - spec: python@3.9.12 + modules: [python/3.9.12] + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + automake: + externals: + - spec: automake@1.15.1 + prefix: /usr + bash: + externals: + - spec: bash@4.4.23 + prefix: /usr + binutils: + externals: + - spec: binutils@2.37.20211103 + prefix: /usr + bison: + externals: + - spec: bison@3.0.4 + prefix: /usr + # Don't use, it's missing the headers + #bzip2: + # externals: + # - spec: bzip2@1.0.6 + # prefix: /usr + cmake: + buildable: false + externals: + - spec: cmake@3.23.1 + modules: [cmake/3.23.1] + coreutils: + externals: + - spec: coreutils@8.32 + prefix: /usr + cpio: + externals: + - spec: cpio@2.12 + prefix: /usr + curl: + externals: + - spec: curl@7.66.0+gssapi+ldap+nghttp2 + prefix: /usr + diffutils: + externals: + - spec: diffutils@3.6 + prefix: /usr + dos2unix: + externals: + - spec: dos2unix@7.4.0 + prefix: /usr + ecflow: + buildable: False + externals: + - spec: ecflow@5.8.4+ui+static_boost + prefix: /lustre/f2/scratch/Dom.Heinzeller/spack-stack-c5/ecflow-5.8.4 + modules: [ecflow/5.8.4] + file: + externals: + - spec: file@5.32 + prefix: /usr + findutils: + externals: + - spec: findutils@4.8.0 + prefix: /usr + flex: + externals: + - spec: flex@2.6.4+lex + prefix: /usr + gawk: + externals: + - spec: gawk@4.2.1 + prefix: /usr + gettext: + externals: + - spec: gettext@0.20.2 + prefix: /usr + ghostscript: + externals: + - spec: ghostscript@9.52 + prefix: /usr + git: + buildable: false + externals: + - spec: git@2.35.2 + modules: [git/2.35.2] + git-lfs: + buildable: false + externals: + - spec: git-lfs@2.11.0 + modules: [git-lfs/2.11.0] + gmake: + externals: + - spec: gmake@4.2.1 + prefix: /usr + groff: + externals: + - spec: groff@1.22.3 + prefix: /usr + hwloc: + externals: + - spec: hwloc@2.6.0a1 + prefix: /usr + # This package is currently incomplete (no headers), but still works + krb5: + externals: + - spec: krb5@1.16.3 + #prefix: /usr/lib/mit + prefix: /usr + libfuse: + externals: + - spec: libfuse@2.9.7 + prefix: /usr + - spec: libfuse@3.6.1 + prefix: /usr + libtirpc: + variants: ~gssapi + libtool: + externals: + - spec: libtool@2.4.6 + prefix: /usr + # This package is currently incomplete (no headers), but still works + libxaw: + externals: + - spec: libxaw@1.10.13 + prefix: /usr + libxml2: + externals: + - spec: libxml2@2.9.7 + prefix: /usr + # This package is currently incomplete (no headers) and doesn't work + # for us. But it's only needed to build libxaw, for which we can use + # the existing (incomplete) installation in /usr, see above + #libxpm: + # externals: + # - spec: libxpm@4.11.0 + # prefix: /usr + lustre: + externals: + - spec: lustre@2.15.0.2_rc2_cray_113_g62287d0 + prefix: /usr + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + ncurses: + externals: + - spec: ncurses@6.1.20180317+termlib abi=6 + prefix: /usr + openjdk: + externals: + - spec: openjdk@11.0.16_8-suse-150000.3.83.1-x8664 + prefix: /usr + openssh: + externals: + - spec: openssh@8.4p1 + prefix: /usr + openssl: + externals: + - spec: openssl@1.1.1d + prefix: /usr + perl: + externals: + - spec: perl@5.26.1~cpanm+shared+threads + prefix: /usr + pkg-config: + buildable: false + externals: + - spec: pkg-config@0.29.2 + prefix: /usr + qt: + externals: + - spec: qt@5.15.2 + prefix: /lustre/f2/scratch/Dom.Heinzeller/spack-stack-c5/qt-5.15.2/5.15.2/gcc_64 + rdma-core: + externals: + - spec: rdma-core@37.0 + prefix: /usr + rsync: + externals: + - spec: rsync@3.1.3 + prefix: /usr + ruby: + externals: + - spec: ruby@2.5.9 + prefix: /usr + sed: + externals: + - spec: sed@4.4 + prefix: /usr + slurm: + externals: + - spec: slurm@21.08.8 + prefix: /usr + subversion: + externals: + - spec: subversion@1.10.6 + prefix: /usr + tar: + externals: + - spec: tar@1.34 + prefix: /usr + texinfo: + externals: + - spec: texinfo@6.5 + prefix: /usr + wget: + externals: + - spec: wget@1.20.3 + prefix: /usr + which: + externals: + - spec: which@2.21 + prefix: /usr + xz: + externals: + - spec: xz@5.2.3 + prefix: /usr + zip: + externals: + - spec: zip@3.0 + prefix: /usr diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 3c0485d0f..183f65bb1 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -11,7 +11,7 @@ compilers:: target: any modules: - PrgEnv-intel/8.3.2 - - intel/2021.3.0 + - intel-classic/2021.3.0 environment: prepend_path: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' diff --git a/doc/source/Platforms.rst b/doc/source/Platforms.rst index 8fd3fe627..40ca4739e 100644 --- a/doc/source/Platforms.rst +++ b/doc/source/Platforms.rst @@ -31,7 +31,7 @@ spack-stack-v1 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NASA Discover GNU | Dom Heinzeller | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NAVY HPCMP Narwhal | Dom Heinzeller | currently broken - will be repaired soon | +| NAVY HPCMP Narwhal | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NCAR-Wyoming Casper | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217-casper/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ @@ -41,7 +41,9 @@ spack-stack-v1 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NOAA Parallel Works (AWS, Azure, Gcloud) | | not yet supported - coming soon | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NOAA RDHPCS Gaea | Dom Heinzeller | ``/lustre/f2/pdata/esrl/gsd/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.3.0/install`` | +| NOAA RDHPCS Gaea (C3/C4) | Dom Heinzeller | ``/lustre/f2/pdata/esrl/gsd/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.3.0/install`` | ++----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ +| NOAA RDHPCS Gaea (C5) | Alex Richert / Dom Heinzeller | not yet supported - coming soon | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NOAA RDHPCS Hera Intel | Hang Lei / Dom Heinzeller | ``/scratch1/NCEPDEV/global/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.5.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ @@ -142,9 +144,6 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules NAVY HPCMP Narwhal ------------------------------ -.. note:: - ``spack-stack-1.2.0``/``skylab-3.0.0`` (and previous ``spack-stack-1.1.0``/``skylab-2.0.0``) is broken and will be fixed soon. - The following is required for building new spack environments and for using spack to build and run software. .. code-block:: console @@ -152,7 +151,7 @@ The following is required for building new spack environments and for using spac module unload PrgEnv-cray module load PrgEnv-intel/8.3.2 module unload intel - module load intel/2021.3.0 + module load intel-classic/2021.4.0 module unload cray-mpich module load cray-mpich/8.1.14 module unload cray-python @@ -167,8 +166,8 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following module .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.3.0/install/modulefiles/Core - module load stack-intel/2021.3.0 + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install/modulefiles/Core + module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 @@ -272,7 +271,7 @@ The following is required for building new spack environments and for using spac .. _Platforms_Gaea: ------------------------------ -NOAA RDHPCS Gaea +NOAA RDHPCS Gaea (C3/C4) ------------------------------ The following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Gaea! @@ -304,6 +303,20 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following module cmake -DCMAKE_CROSSCOMPILING_EMULATOR="/usr/bin/srun;-n;1" -DMPIEXEC_EXECUTABLE="/usr/bin/srun" -DMPIEXEC_NUMPROC_FLAG="-n" PATH_TO_SOURCE +------------------------------ +NOAA RDHPCS Gaea (C5) +------------------------------ + +The following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Gaea! + +.. code-block:: console + + module load PrgEnv-intel/8.3.3 + module load intel/2022.0.2 + module load cray-mpich/8.1.16 + module load python/3.9.12 + + .. _Platforms_Hera: ------------------------------