From c23b5100d5a9a304a5336fe57a4f4f909c117fff Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Jan 2024 17:54:28 +0000 Subject: [PATCH 01/13] Initial upgrade to spack-stack v1.6.0. #674 --- modulefiles/gsi_cheyenne.gnu.lua | 2 +- modulefiles/gsi_cheyenne.intel.lua | 2 +- modulefiles/gsi_common.lua | 6 +++--- modulefiles/gsi_gaea.lua | 2 +- modulefiles/gsi_hera.gnu.lua | 6 +++--- modulefiles/gsi_hera.intel.lua | 6 +++--- modulefiles/gsi_hercules.lua | 6 +++--- modulefiles/gsi_jet.lua | 6 +++--- modulefiles/gsi_orion.lua | 6 +++--- modulefiles/gsi_s4.lua | 6 +++--- modulefiles/gsi_wcoss2.lua | 2 +- 11 files changed, 25 insertions(+), 25 deletions(-) diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua index 1d903082a8..c3195821c3 100644 --- a/modulefiles/gsi_cheyenne.gnu.lua +++ b/modulefiles/gsi_cheyenne.gnu.lua @@ -20,7 +20,7 @@ load(pathJoin("stack-gcc", stack_gnu_ver)) load(pathJoin("stack-openmpi", stack_openmpi_ver)) load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) -load(pathJoin("prod_util", os.getenv("prod_util_ver") or "1.2.2")) +load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) load("gsi_common") diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua index 8c328e2b34..481afb49b3 100644 --- a/modulefiles/gsi_cheyenne.intel.lua +++ b/modulefiles/gsi_cheyenne.intel.lua @@ -21,7 +21,7 @@ load(pathJoin("stack-python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) load("gsi_common") -load(pathJoin("prod-util", os.getenv("prod_util_ver") or "1.2.2")) +load(pathJoin("prod-util", os.getenv("prod_util_ver") or "2.1.1")) pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230911") pushenv("CFLAGS", "-xHOST") diff --git a/modulefiles/gsi_common.lua b/modulefiles/gsi_common.lua index d3365a98dc..cb49a43878 100644 --- a/modulefiles/gsi_common.lua +++ b/modulefiles/gsi_common.lua @@ -3,19 +3,19 @@ Load common modules to build GSI on all machines ]]) local netcdf_c_ver=os.getenv("netcdf_c_ver") or "4.9.2" -local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.0" +local netcdf_fortran_ver=os.getenv("netcdf_fortran_ver") or "4.6.1" local bufr_ver=os.getenv("bufr_ver") or "11.7.0" local bacio_ver=os.getenv("bacio_ver") or "2.4.1" local w3emc_ver=os.getenv("w3emc_ver") or "2.10.0" -local sp_ver=os.getenv("sp_ver") or "2.3.3" +local sp_ver=os.getenv("sp_ver") or "2.5.0" local ip_ver=os.getenv("ip_ver") or "4.3.0" local sigio_ver=os.getenv("sigio_ver") or "2.3.2" local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" -local crtm_ver=os.getenv("crtm_ver") or "2.4.0" +local crtm_ver=os.getenv("crtm_ver") or "2.4.0.1" local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.2" load(pathJoin("netcdf-c", netcdf_c_ver)) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index ef6b9ddba7..622e075b6a 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -21,7 +21,7 @@ load(pathJoin("cmake", cmake_ver)) load("gsi_common") -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("prod-util", prod_util_ver)) -- Needed at runtime: diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index 550b01ee7b..c7bed47639 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -1,15 +1,15 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") --Needed for openmpi build prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0" local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" local openblas_ver=os.getenv("openblas_ver") or "0.3.19" load(pathJoin("stack-gcc", stack_gnu_ver)) diff --git a/modulefiles/gsi_hera.intel.lua b/modulefiles/gsi_hera.intel.lua index abdc6e5623..b967c32f74 100644 --- a/modulefiles/gsi_hera.intel.lua +++ b/modulefiles/gsi_hera.intel.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_hercules.lua b/modulefiles/gsi_hercules.lua index bf29bc21db..6bb7cff2ce 100644 --- a/modulefiles/gsi_hercules.lua +++ b/modulefiles/gsi_hercules.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_jet.lua b/modulefiles/gsi_jet.lua index 20b80ff61a..b2b98be7ff 100644 --- a/modulefiles/gsi_jet.lua +++ b/modulefiles/gsi_jet.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index 80ec342c93..9fdd2f61fa 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local stack_python_ver=os.getenv("python_ver") or "3.10.8" +local stack_python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua index a60ea3c16e..35ebc3785d 100644 --- a/modulefiles/gsi_s4.lua +++ b/modulefiles/gsi_s4.lua @@ -1,13 +1,13 @@ help([[ ]]) -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) diff --git a/modulefiles/gsi_wcoss2.lua b/modulefiles/gsi_wcoss2.lua index 8dde986e58..28b2f0d09d 100644 --- a/modulefiles/gsi_wcoss2.lua +++ b/modulefiles/gsi_wcoss2.lua @@ -20,7 +20,7 @@ local sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" local nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" local wrf_io_ver=os.getenv("wrf_io_ver") or "1.2.0" local ncio_ver=os.getenv("ncio_ver") or "1.1.2" -local crtm_ver=os.getenv("crtm_ver") or "2.4.0" +local crtm_ver=os.getenv("crtm_ver") or "2.4.0.1" local ncdiag_ver=os.getenv("ncdiag_ver") or "1.1.1" load(pathJoin("PrgEnv-intel", PrgEnv_intel_ver)) From 21368825301c2db1d0638fe3458480e6a48cf15f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 17 Jan 2024 21:00:19 +0000 Subject: [PATCH 02/13] Update CI library versions. --- ci/spack.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index deacdff0b5..6063badc5b 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -7,11 +7,11 @@ spack: - gcc@10:10 specs: - netcdf-c@4.9.2 - - netcdf-fortran@4.6.0 + - netcdf-fortran@4.6.1 - bufr@11.7.0 - bacio@2.4.1 - - w3emc@2.9.2 - - sp@2.3.3 + - w3emc@2.10.0 + - sp@2.5.0 - ip@3.3.3 - sigio@2.3.2 - sfcio@1.4.1 @@ -19,7 +19,7 @@ spack: - wrf-io@1.2.0 - ncio@1.1.2 - crtm@2.4.0 - - gsi-ncdiag@1.1.1 + - gsi-ncdiag@1.1.2 view: true concretizer: unify: true From 89d8bf8fac2777677b23645265ff8f7c8b9f3367 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 18 Jan 2024 12:34:24 +0000 Subject: [PATCH 03/13] Update CI library versions to SS versions. --- ci/spack.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/spack.yaml b/ci/spack.yaml index 6063badc5b..647904108e 100644 --- a/ci/spack.yaml +++ b/ci/spack.yaml @@ -12,13 +12,13 @@ spack: - bacio@2.4.1 - w3emc@2.10.0 - sp@2.5.0 - - ip@3.3.3 + - ip@4.3.0 - sigio@2.3.2 - sfcio@1.4.1 - nemsio@2.5.4 - wrf-io@1.2.0 - ncio@1.1.2 - - crtm@2.4.0 + - crtm@2.4.0.1 - gsi-ncdiag@1.1.2 view: true concretizer: From 955eedf1f70d77017dc8331ef7c54b482b42fbc4 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Jan 2024 16:38:00 +0000 Subject: [PATCH 04/13] Remove Cheyenne. --- modulefiles/gsi_cheyenne.gnu.lua | 32 ------------------------------ modulefiles/gsi_cheyenne.intel.lua | 32 ------------------------------ 2 files changed, 64 deletions(-) delete mode 100644 modulefiles/gsi_cheyenne.gnu.lua delete mode 100644 modulefiles/gsi_cheyenne.intel.lua diff --git a/modulefiles/gsi_cheyenne.gnu.lua b/modulefiles/gsi_cheyenne.gnu.lua deleted file mode 100644 index c3195821c3..0000000000 --- a/modulefiles/gsi_cheyenne.gnu.lua +++ /dev/null @@ -1,32 +0,0 @@ -help([[ -]]) - - -unload("ncarenv/1.3") -unload("intel/19.1.1") -unload("ncarcompilers/0.5.0") -unload("mpt/2.25") -unload("netcdf/4.8.1") - -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") - -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -local stack_gnu_ver=os.getenv("stack_gnu_ver") or "10.1.0" -local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.1" -local cmake_ver=os.getenv("cmake_ver") or "3.22.0" - -load(pathJoin("stack-gcc", stack_gnu_ver)) -load(pathJoin("stack-openmpi", stack_openmpi_ver)) -load(pathJoin("stack-python", stack_python_ver)) -load(pathJoin("cmake", cmake_ver)) -load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.1.1")) -load(pathJoin("openblas", os.getenv("openblas_ver") or "0.3.23")) - -load("gsi_common") - -pushenv("CFLAGS", "-xHOST") -pushenv("FFLAGS", "-xHOST") -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") - -whatis("Description: GSI environment on Cheyenne with GNU Compilers") diff --git a/modulefiles/gsi_cheyenne.intel.lua b/modulefiles/gsi_cheyenne.intel.lua deleted file mode 100644 index 481afb49b3..0000000000 --- a/modulefiles/gsi_cheyenne.intel.lua +++ /dev/null @@ -1,32 +0,0 @@ -help([[ -]]) - -unload("ncarenv/1.3") -unload("intel/19.1.1") -unload("ncarcompilers/0.5.0") -unload("mpt/2.25") -unload("netcdf/4.8.1") - -prepend_path("MODULEPATH", "/glade/work/epicufsrt/contrib/spack-stack/cheyenne/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/glade/work/jedipara/cheyenne/spack-stack/modulefiles/misc") - -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -local stack_intel_ver=os.getenv("stack_intel_ver") or "19.1.1.217" -local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2019.7.217" -local cmake_ver=os.getenv("cmake_ver") or "3.22.0" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-intel-mpi", stack_mpi_ver)) -load(pathJoin("stack-python", stack_python_ver)) -load(pathJoin("cmake", cmake_ver)) - -load("gsi_common") -load(pathJoin("prod-util", os.getenv("prod_util_ver") or "2.1.1")) -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230911") - -pushenv("CFLAGS", "-xHOST") -pushenv("FFLAGS", "-xHOST") - -pushenv("GSI_BINARY_SOURCE_DIR", "/glade/work/epicufsrt/contrib/GSI_data/fix/20230601") - -whatis("Description: GSI environment on Cheyenne with Intel Compilers") From 7f3969a475e782c7a0fa6ed91369e97b51ae4dcb Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 19 Jan 2024 20:22:57 +0000 Subject: [PATCH 05/13] Remove Cheyenne submit script. --- ush/sub_cheyenne | 169 ----------------------------------------------- 1 file changed, 169 deletions(-) delete mode 100644 ush/sub_cheyenne diff --git a/ush/sub_cheyenne b/ush/sub_cheyenne deleted file mode 100644 index 7389bfeb24..0000000000 --- a/ush/sub_cheyenne +++ /dev/null @@ -1,169 +0,0 @@ -#!/bin/sh --login -set -x -echo "starting sub_cheyenne" -usage="\ -Usage: $0 [options] executable [args] - where the options are: - -a account account (default: none) - -b binding run smt binding or not (default:NO) - -d dirin initial directory (default: cwd) - -e envars copy comma-separated environment variables - -g group group name - -i append standard input to command file - -j jobname specify jobname (default: executable basename) - -m machine machine on which to run (default: current) - -n write command file to stdout rather than submitting it - -o output specify output file (default: jobname.out) - -p procs[/nodes[/ppreq] - number of MPI tasks and optional nodes or Bblocking and - ppreq option (N or S) (defaults: serial, Bunlimited, S) - -q queue[/qpreq] queue name and optional requirement, e.g. dev/P - (defaults: 1 if serial or dev if parallel and none) - (queue 3 or 4 is dev or prod with twice tasks over ip) - (options: P=parallel, B=bigmem, b=batch) - -r rmem[/rcpu] resources memory and cpus/task (default: '1024 mb', 1) - -t timew wall time limit in [[hh:]mm:]ss format (default: 900) - -u userid userid to run under (default: self) - -v verbose mode - -w when when to run, in yyyymmddhh[mm], +hh[mm], thh[mm], or - Thh[mm] (full, incremental, today or tomorrow) format - (default: now) -Function: This command submits a job to the batch queue." -subcmd="$*" -stdin=NO -nosub=NO -account="" -binding="NO" -dirin="" -envars="" -group="" -jobname="" -machine="" -output="" -procs=0 -nodes="" -ppreq="" -queue="" -qpreq="" -rmem="1024" -rcpu="1" -timew="900" -userid="" -verbose=NO -when="" -while getopts a:b:d:e:g:ij:m:no:p:q:r:t:u:vw: opt;do - case $opt in - a) account="$OPTARG";; - b) binding="$OPTARG";; - d) dirin="$OPTARG";; - e) envars="$OPTARG";; - g) group="$OPTARG";; - i) stdin=YES;; - j) jobname=$OPTARG;; - m) machine="$OPTARG";; - n) nosub=YES;; - o) output=$OPTARG;; - p) procs=$(echo $OPTARG/|cut -d/ -f1);nodes=$(echo $OPTARG/|cut -d/ -f2);ppreq=$(echo $OPTARG/|cut -d/ -f3);; - q) queue=$(echo $OPTARG/|cut -d/ -f1);qpreq=$(echo $OPTARG/|cut -d/ -f2);; - r) rmem=$(echo $OPTARG/|cut -d/ -f1);rcpu=$(echo $OPTARG/|cut -d/ -f2);; - t) timew=$OPTARG;; - u) userid=$OPTARG;; - v) verbose=YES;; - w) when=$OPTARG;; - \?) echo $0: invalid option >&2;echo "$usage" >&2;exit 1;; - esac -done -shift $(($OPTIND-1)) -if [[ $# -eq 0 ]];then - echo $0: missing executable name >&2;echo "$usage" >&2;exit 1 -fi -exec=$1 -if [[ ! -s $exec ]]&&which $exec >/dev/null 2>&1;then - exec=$(which $exec) -fi -shift -args="$*" -bn=$(basename $exec) -export jobname=${jobname:-$bn} -output=${output:-$jobname.out} -myuser=$LOGNAME -myhost=$(hostname) - -DATA=/glade/scratch/$LOGNAME/tmp -mkdir -p $DATA - -timew=${timew:-01:20:00} -task_node=${task_node:-$procs} -size=$((nodes*task_node)) -envars=$envars -threads=${rcpu:-1} - -export TZ=GMT -cfile=$DATA/sub$$ -> $cfile -echo "#!/bin/sh --login" >> $cfile -echo "" >> $cfile -echo "#PBS -o $output" >> $cfile -echo "#PBS -N $jobname" >> $cfile -echo "#PBS -q $queue" >> $cfile -echo "#PBS -l walltime=$timew" >> $cfile -echo "#PBS -l select=$nodes:ncpus=$procs:mpiprocs=$procs" >> $cfile -echo "#PBS -j oe" >> $cfile -echo "#PBS -A $accnt" >> $cfile -echo "#PBS -V" >> $cfile - -echo "" >>$cfile -echo "export ntasks=$(( $nodes * $procs ))" >> $cfile -echo "export ppn=$procs" >> $cfile -echo "export threads=$threads" >> $cfile -echo "export OMP_NUM_THREADS=$threads" >> $cfile -echo "ulimit -s unlimited" >> $cfile -echo "" >>$cfile -echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile -echo "" >>$cfile - -echo "cfile = $cfile" -echo "source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh >> $cfile" -echo "module purge" >> $cfile -echo "module use $modulefiles" >> $cfile -echo "module load gsi_cheyenne.intel" >> $cfile -echo "module list" >> $cfile - -cat $exec >> $cfile - -if [[ $nosub = YES ]];then - cat $cfile - exit -elif [[ $verbose = YES ]];then - set -x - cat $cfile -fi - - -if [[ $stdin = YES ]];then - cat -fi >>$cfile -if [[ $nosub = YES ]];then - cat $cfile - exit -elif [[ $verbose = YES ]];then - set -x - cat $cfile -fi -qsub=${qsub:-qsub} - -ofile=$DATA/subout$$ ->$ofile -chmod 777 $ofile -$qsub $cfile >$ofile -rc=$? -cat $ofile -if [[ -w $SUBLOG ]];then - jobn=$(grep -i submitted $ofile|head -n1|cut -d\" -f2) - date -u +"%Y%m%d%H%M%S : $subcmd : $jobn" >>$SUBLOG -fi -rm $cfile $ofile -[[ $MKDATA = YES ]] && rmdir $DATA -echo "ending sub_cheyenne" -exit $rc - From 339bf16c6344500b44deccc338273ad233d5212b Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 22 Jan 2024 14:21:25 -0600 Subject: [PATCH 06/13] Fix gsi-addon paths for hercules, orion, and S4. --- modulefiles/gsi_hercules.lua | 2 +- modulefiles/gsi_orion.lua | 2 +- modulefiles/gsi_s4.lua | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modulefiles/gsi_hercules.lua b/modulefiles/gsi_hercules.lua index 6bb7cff2ce..c72323cbad 100644 --- a/modulefiles/gsi_hercules.lua +++ b/modulefiles/gsi_hercules.lua @@ -1,7 +1,7 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" diff --git a/modulefiles/gsi_orion.lua b/modulefiles/gsi_orion.lua index 9fdd2f61fa..114e5f8ad1 100644 --- a/modulefiles/gsi_orion.lua +++ b/modulefiles/gsi_orion.lua @@ -1,7 +1,7 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") local stack_python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" diff --git a/modulefiles/gsi_s4.lua b/modulefiles/gsi_s4.lua index 35ebc3785d..08e3d38516 100644 --- a/modulefiles/gsi_s4.lua +++ b/modulefiles/gsi_s4.lua @@ -1,7 +1,7 @@ help([[ ]]) -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") local python_ver=os.getenv("python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" From 04a737dd1f84dc29069499de4aa1990e7aa24425 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 31 Jan 2024 12:53:59 -0500 Subject: [PATCH 07/13] Update Gaea-C5 support. --- modulefiles/gsi_gaea.lua | 25 ++++++++----------------- regression/regression_param.sh | 34 +++++++++++++++++----------------- regression/regression_var.sh | 8 ++++---- ush/detect_machine.sh | 5 +---- ush/module-setup.sh | 2 +- ush/sub_gaea | 14 ++++---------- 6 files changed, 35 insertions(+), 53 deletions(-) diff --git a/modulefiles/gsi_gaea.lua b/modulefiles/gsi_gaea.lua index 622e075b6a..531be47548 100644 --- a/modulefiles/gsi_gaea.lua +++ b/modulefiles/gsi_gaea.lua @@ -1,17 +1,16 @@ help([[ ]]) -unload("intel") +unload("intel-classic") unload("cray-mpich") unload("cray-python") -unload("darshan") +unload("darshan-runtime") -prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" -local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20" +local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("stack-intel", stack_intel_ver)) @@ -22,16 +21,9 @@ load(pathJoin("cmake", cmake_ver)) load("gsi_common") local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" -load(pathJoin("prod-util", prod_util_ver)) +load(pathJoin("prod_util", prod_util_ver)) --- Needed at runtime: -load("alps") - -local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" -prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) -pushenv("MKLROOT", MKLROOT) - -pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230911") +pushenv("GSI_BINARY_SOURCE_DIR", "/gpfs/f5/epic/world-shared/GSI_data/fix/20230911") setenv("CC","cc") setenv("FC","ftn") @@ -39,4 +31,3 @@ setenv("CXX","CC") pushenv("CRAYPE_LINK_TYPE","dynamic") whatis("Description: GSI environment on Gaea with Intel Compilers") - diff --git a/regression/regression_param.sh b/regression/regression_param.sh index 46d2647ac0..edd7b01a43 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -26,8 +26,8 @@ case $machine in ;; Gaea) sub_cmd="sub_gaea" - memnode=64 - numcore=36 + memnode=251 + numcore=128 ;; wcoss2) sub_cmd="sub_wcoss2" @@ -77,8 +77,8 @@ case $regtest in topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" @@ -110,8 +110,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -143,8 +143,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -175,8 +175,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -208,8 +208,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" @@ -241,8 +241,8 @@ case $regtest in topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2" @@ -274,8 +274,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" @@ -341,7 +341,7 @@ elif [[ "$machine" = "Gaea" ]]; then export MPI_BUFS_PER_PROC=256 export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 - export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks" + export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" elif [[ "$machine" = "Cheyenne" ]]; then export OMP_STACKSIZE=1024M export MPI_BUFS_PER_PROC=256 diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 02ffb24b12..89dcf6692a 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -54,13 +54,13 @@ echo "Running Regression Tests on '$machine'"; case $machine in Gaea) export queue="normal" - export noscrub="/lustre/f2/scratch/$LOGNAME/gsi_tmp/noscrub" - export ptmp="/lustre/f2/scratch/$LOGNAME/gsi_tmp/ptmp" - export casesdir="/lustre/f2/dev/role.epic/contrib/GSI_data/CASES/regtest" + export noscrub="/gpfs/f5/scratch/$LOGNAME/gsi_tmp/noscrub" + export ptmp="/gpfs/f5/scratch/$LOGNAME/gsi_tmp/ptmp" + export casesdir="/gpfs/f5/epic/world-shared/GSI_data/CASES/regtest" export group="global" if [[ "$cmaketest" = "false" ]]; then - export basedir="/lustre/f2/dev/$LOGNAME/sandbox/GSI" + export basedir="/gpfs/f5/dev/$LOGNAME/sandbox/GSI" fi export check_resource="no" diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index ac6c7f58d1..9fff213727 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -9,10 +9,7 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea9) MACHINE_ID=gaea ;; ### gaea9 - gaea1[0-6]) MACHINE_ID=gaea ;; ### gaea10-16 - gaea9.ncrc.gov) MACHINE_ID=gaea ;; ### gaea9 - gaea1[0-6].ncrc.gov) MACHINE_ID=gaea ;; ### gaea10-16 + gaea5[0-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea50-58 (C5) hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-9 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 diff --git a/ush/module-setup.sh b/ush/module-setup.sh index d13da1efa3..b62b68d466 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -66,7 +66,7 @@ elif [[ $MACHINE_ID = gaea* ]] ; then __ms_source_etc_profile=yes fi - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh + module reset elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse diff --git a/ush/sub_gaea b/ush/sub_gaea index 6fed1b3c10..e229f99759 100755 --- a/ush/sub_gaea +++ b/ush/sub_gaea @@ -88,8 +88,8 @@ output=${output:-$jobname.out} myuser=$LOGNAME myhost=$(hostname) -if [ -d /lustre/f2/scratch/$LOGNAME ]; then - DATA=/lustre/f2/scratch/$LOGNAME/tmp +if [ -d /gpfs/f5/scratch/$LOGNAME ]; then + DATA=/gpfs/f5/scratch/$LOGNAME/tmp fi DATA=${DATA:-$ptmp/tmp} @@ -110,24 +110,18 @@ echo "" echo "#SBATCH --output=$output" >> $cfile echo "#SBATCH --job-name=$jobname" >> $cfile echo "#SBATCH --qos=$queue" >> $cfile -echo "#SBATCH --clusters=c4" >> $cfile echo "#SBATCH --time=$timew" >> $cfile echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile echo "#SBATCH --account=$accnt" >> $cfile echo "#SBATCH --mem=0" >> $cfile echo "" >>$cfile -echo "export ntasks=$(( $nodes * $procs ))" >> $cfile -echo "export ppn=$procs" >> $cfile -echo "export threads=$threads" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile -echo "ulimit -s unlimited" >> $cfile - echo "" >>$cfile echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile echo "" >>$cfile -echo "source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh" >> $cfile +echo "module purge" >> $cfile echo "module use $modulefiles" >> $cfile echo "module load gsi_gaea" >> $cfile echo "module list" >> $cfile @@ -158,7 +152,7 @@ sbatch=${sbatch:-sbatch} ofile=$DATA/subout$$ >$ofile chmod 777 $ofile -$sbatch --export=ALL $cfile >$ofile +$sbatch $cfile >$ofile rc=$? cat $ofile if [[ -w $SUBLOG ]];then From bfb46e29ea9eb7a61e2dfc661672eb234bbaa954 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 31 Jan 2024 12:59:05 -0500 Subject: [PATCH 08/13] Remove references to Cheyenne. --- regression/regression_param.sh | 32 -------------------------------- regression/regression_var.sh | 18 +----------------- ush/detect_machine.sh | 7 +------ ush/module-setup.sh | 7 ------- 4 files changed, 2 insertions(+), 62 deletions(-) diff --git a/regression/regression_param.sh b/regression/regression_param.sh index edd7b01a43..bbe6cad596 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -36,11 +36,6 @@ case $machine in ;; Discover) sub_cmd="sub_discover" - ;; - Cheyenne) - sub_cmd="sub_cheyenne" - memnode=128 - numcore=36 ;; *) # EXIT out for unresolved machine echo "unknown $machine" @@ -73,9 +68,6 @@ case $regtest in elif [[ "$machine" = "Discover" ]]; then topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" @@ -106,9 +98,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -139,9 +128,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -171,9 +157,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -204,9 +187,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" @@ -237,9 +217,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1" @@ -270,9 +247,6 @@ case $regtest in elif [[ "$machine" = "Jet" ]]; then topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" - elif [[ "$machine" = "Cheyenne" ]]; then - topts[1]="0:15:00" ; popts[1]="12/3/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" @@ -342,12 +316,6 @@ elif [[ "$machine" = "Gaea" ]]; then export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" -elif [[ "$machine" = "Cheyenne" ]]; then - export OMP_STACKSIZE=1024M - export MPI_BUFS_PER_PROC=256 - export MPI_BUFS_PER_HOST=256 - export MPI_GROUP_MAX=256 - export APRUN="mpirun -v -np \$ntasks" elif [[ "$machine" = "wcoss2" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=2G diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 89dcf6692a..61324233d4 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -30,9 +30,7 @@ else fi # Determine the machine -if [[ -d /glade ]]; then # Cheyenne - export machine="Cheyenne" -elif [[ -d /scratch1 ]]; then # Hera +if [[ -d /scratch1 ]]; then # Hera export machine="Hera" elif [[ -d /mnt/lfs4 || -d /jetmon || -d /mnt/lfs1 ]]; then # Jet export machine="Jet" @@ -66,20 +64,6 @@ case $machine in export check_resource="no" export accnt="nggps_emc" ;; - Cheyenne) - export queue="regular" - export noscrub="/glade/scratch/$LOGNAME/noscrub" - export group="global" - if [[ "$cmaketest" = "false" ]]; then - export basedir="/glade/scratch/$LOGNAME" - fi - export ptmp="/glade/scratch/$LOGNAME/$ptmpName" - - export casesdir="/glade/work/epicufsrt/contrib/GSI_data/CASES/regtest" - - export check_resource="no" - export accnt="NRAL0032" - ;; wcoss2) export local_or_default="${local_or_default:-/lfs/h2/emc/da/noscrub/$LOGNAME}" if [ -d $local_or_default ]; then diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index 9fff213727..3666e4edc1 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -24,11 +24,6 @@ case $(hostname -f) in Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 - cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 - cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 - chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 - chadmin[1-6].ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 - login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4 login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2 @@ -40,6 +35,6 @@ esac MACHINE_ID=${MACHINE:-${MACHINE_ID}} # Append compiler (only on machines that have multiple compilers) -if [ $MACHINE_ID = hera ] || [ $MACHINE_ID = cheyenne ]; then +if [ $MACHINE_ID = hera ]; then MACHINE_ID=${MACHINE_ID}.${COMPILER} fi diff --git a/ush/module-setup.sh b/ush/module-setup.sh index b62b68d466..ff3eed4747 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -40,13 +40,6 @@ elif [[ $MACHINE_ID = wcoss2 ]]; then # We are on WCOSS2 module reset -elif [[ $MACHINE_ID = cheyenne* ]] ; then - # We are on NCAR Cheyenne - if ( ! eval module help > /dev/null 2>&1 ) ; then - source /glade/u/apps/ch/modulefiles/default/localinit/localinit.sh - fi - module purge - elif [[ $MACHINE_ID = stampede* ]] ; then # We are on TACC Stampede if ( ! eval module help > /dev/null 2>&1 ) ; then From aac701003f113ea0b1c0ab7611cc8d325ebe406a Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 1 Feb 2024 13:16:54 -0600 Subject: [PATCH 09/13] Explicitly enable lustre parallel i/o. --- ush/sub_hercules | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ush/sub_hercules b/ush/sub_hercules index e854d5727b..03003307e9 100755 --- a/ush/sub_hercules +++ b/ush/sub_hercules @@ -118,6 +118,9 @@ echo "" >>$cfile echo "export ntasks=$(( $nodes * $procs ))" >> $cfile echo "export ppn=$procs" >> $cfile echo "export threads=$threads" >> $cfile +# With newer IMPI versions, its sometimes necessary to tell MPI how to use parallel I/O +echo "export I_MPI_EXTRA_FILESYSTEM=1" >> $cfile +echo "export I_MPI_EXTRA_FILESYSTEM_FORCE=lustre" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile ##echo "export OMP_STACKSIZE=2048M" >> $cfile echo "ulimit -s unlimited" >> $cfile From a906115ac3842cabbcb4cecccce10cbbc9ab0e08 Mon Sep 17 00:00:00 2001 From: David Huber Date: Fri, 9 Feb 2024 08:25:00 -0600 Subject: [PATCH 10/13] Revert Hercules modules to SS/1.5.1. --- modulefiles/gsi_hercules.intel.lua | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modulefiles/gsi_hercules.intel.lua b/modulefiles/gsi_hercules.intel.lua index c72323cbad..a646e9d3bf 100644 --- a/modulefiles/gsi_hercules.intel.lua +++ b/modulefiles/gsi_hercules.intel.lua @@ -1,19 +1,24 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" +local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) load(pathJoin("python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) +-- netcdf-fortran/4.6.1 does not work on Hercules yet +pushenv("netcdf_fortran_ver", "4.6.0") +pushenv("sp_ver", "2.3.3") +pushenv("crtm_ver", "2.4.0") + load("gsi_common") load(pathJoin("prod_util", prod_util_ver)) load("intel-oneapi-mkl/2022.2.1") From 4c923ab6b48f7ef4bf7bc5c593e0305d74ac94ad Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 12 Feb 2024 10:54:29 -0500 Subject: [PATCH 11/13] Revert Gaea support. --- modulefiles/gsi_gaea.intel.lua | 27 ++++++++++++++++++--------- regression/regression_param.sh | 34 +++++++++++++++++----------------- regression/regression_var.sh | 8 ++++---- ush/module-setup.sh | 2 +- ush/sub_gaea | 14 ++++++++++---- 5 files changed, 50 insertions(+), 35 deletions(-) diff --git a/modulefiles/gsi_gaea.intel.lua b/modulefiles/gsi_gaea.intel.lua index 531be47548..ef6b9ddba7 100644 --- a/modulefiles/gsi_gaea.intel.lua +++ b/modulefiles/gsi_gaea.intel.lua @@ -1,16 +1,17 @@ help([[ ]]) -unload("intel-classic") +unload("intel") unload("cray-mpich") unload("cray-python") -unload("darshan-runtime") +unload("darshan") -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev/install/modulefiles/Core") +prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles") -local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" -local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" -local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("stack-intel", stack_intel_ver)) @@ -20,10 +21,17 @@ load(pathJoin("cmake", cmake_ver)) load("gsi_common") -local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" -load(pathJoin("prod_util", prod_util_ver)) +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +load(pathJoin("prod-util", prod_util_ver)) -pushenv("GSI_BINARY_SOURCE_DIR", "/gpfs/f5/epic/world-shared/GSI_data/fix/20230911") +-- Needed at runtime: +load("alps") + +local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/" +prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64")) +pushenv("MKLROOT", MKLROOT) + +pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230911") setenv("CC","cc") setenv("FC","ftn") @@ -31,3 +39,4 @@ setenv("CXX","CC") pushenv("CRAYPE_LINK_TYPE","dynamic") whatis("Description: GSI environment on Gaea with Intel Compilers") + diff --git a/regression/regression_param.sh b/regression/regression_param.sh index bbe6cad596..2ac615fc4a 100755 --- a/regression/regression_param.sh +++ b/regression/regression_param.sh @@ -26,8 +26,8 @@ case $machine in ;; Gaea) sub_cmd="sub_gaea" - memnode=251 - numcore=128 + memnode=64 + numcore=36 ;; wcoss2) sub_cmd="sub_wcoss2" @@ -69,8 +69,8 @@ case $regtest in topts[1]="0:30:00" ; popts[1]="48/2" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="60/3" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="18/8/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="18/10/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:10:00" ; popts[1]="12/8/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/10/" ; ropts[2]="/2" @@ -99,8 +99,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -129,8 +129,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -158,8 +158,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="20/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="20/2/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="18/1/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="18/2/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="64/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="128/2/" ; ropts[2]="/1" @@ -188,8 +188,8 @@ case $regtest in topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" - topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" + topts[1]="0:15:00" ; popts[1]="4/4/" ; ropts[1]="/1" + topts[2]="0:15:00" ; popts[2]="6/6/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:15:00" ; popts[1]="28/1/" ; ropts[1]="/1" topts[2]="0:15:00" ; popts[2]="28/2/" ; ropts[2]="/1" @@ -218,8 +218,8 @@ case $regtest in topts[1]="0:30:00" ; popts[1]="6/12/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="8/12/" ; ropts[2]="/1" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" - topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/1" + topts[1]="0:30:00" ; popts[1]="8/6/" ; ropts[1]="/1" + topts[2]="0:30:00" ; popts[2]="8/8/" ; ropts[2]="/1" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:30:00" ; popts[1]="14/8/" ; ropts[1]="/1" topts[2]="0:30:00" ; popts[2]="14/14/" ; ropts[2]="/2" @@ -248,8 +248,8 @@ case $regtest in topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "Gaea" ]]; then - topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" - topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" + topts[1]="0:10:00" ; popts[1]="12/3/" ; ropts[1]="/1" + topts[2]="0:10:00" ; popts[2]="12/5/" ; ropts[2]="/2" elif [[ "$machine" = "wcoss2" ]]; then topts[1]="0:10:00" ; popts[1]="16/2/" ; ropts[1]="/1" topts[2]="0:10:00" ; popts[2]="16/4/" ; ropts[2]="/2" @@ -315,7 +315,7 @@ elif [[ "$machine" = "Gaea" ]]; then export MPI_BUFS_PER_PROC=256 export MPI_BUFS_PER_HOST=256 export MPI_GROUP_MAX=256 - export APRUN="srun -n \$ntasks --cpus-per-task=\$threads" + export APRUN="srun --export=ALL --mpi=pmi2 -n \$ntasks" elif [[ "$machine" = "wcoss2" ]]; then export OMP_PLACES=cores export OMP_STACKSIZE=2G diff --git a/regression/regression_var.sh b/regression/regression_var.sh index 61324233d4..9d91b2c41f 100755 --- a/regression/regression_var.sh +++ b/regression/regression_var.sh @@ -52,13 +52,13 @@ echo "Running Regression Tests on '$machine'"; case $machine in Gaea) export queue="normal" - export noscrub="/gpfs/f5/scratch/$LOGNAME/gsi_tmp/noscrub" - export ptmp="/gpfs/f5/scratch/$LOGNAME/gsi_tmp/ptmp" - export casesdir="/gpfs/f5/epic/world-shared/GSI_data/CASES/regtest" + export noscrub="/lustre/f2/scratch/$LOGNAME/gsi_tmp/noscrub" + export ptmp="/lustre/f2/scratch/$LOGNAME/gsi_tmp/ptmp" + export casesdir="/lustre/f2/dev/role.epic/contrib/GSI_data/CASES/regtest" export group="global" if [[ "$cmaketest" = "false" ]]; then - export basedir="/gpfs/f5/dev/$LOGNAME/sandbox/GSI" + export basedir="/lustre/f2/dev/$LOGNAME/sandbox/GSI" fi export check_resource="no" diff --git a/ush/module-setup.sh b/ush/module-setup.sh index ff3eed4747..c1893ab4ee 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -59,7 +59,7 @@ elif [[ $MACHINE_ID = gaea* ]] ; then __ms_source_etc_profile=yes fi - module reset + source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh elif [[ $MACHINE_ID = expanse* ]]; then # We are on SDSC Expanse diff --git a/ush/sub_gaea b/ush/sub_gaea index 3e39f88568..afad6aa7ab 100755 --- a/ush/sub_gaea +++ b/ush/sub_gaea @@ -88,8 +88,8 @@ output=${output:-$jobname.out} myuser=$LOGNAME myhost=$(hostname) -if [ -d /gpfs/f5/scratch/$LOGNAME ]; then - DATA=/gpfs/f5/scratch/$LOGNAME/tmp +if [ -d /lustre/f2/scratch/$LOGNAME ]; then + DATA=/lustre/f2/scratch/$LOGNAME/tmp fi DATA=${DATA:-$ptmp/tmp} @@ -110,18 +110,24 @@ echo "" echo "#SBATCH --output=$output" >> $cfile echo "#SBATCH --job-name=$jobname" >> $cfile echo "#SBATCH --qos=$queue" >> $cfile +echo "#SBATCH --clusters=c4" >> $cfile echo "#SBATCH --time=$timew" >> $cfile echo "#SBATCH --nodes=$nodes --ntasks-per-node=$procs --cpus-per-task=$threads" >> $cfile echo "#SBATCH --account=$accnt" >> $cfile echo "#SBATCH --mem=0" >> $cfile echo "" >>$cfile +echo "export ntasks=$(( $nodes * $procs ))" >> $cfile +echo "export ppn=$procs" >> $cfile +echo "export threads=$threads" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile +echo "ulimit -s unlimited" >> $cfile + echo "" >>$cfile echo ". "$(awk '{ print $1, $2, $3, $4, $5, $6, $7, $8, $9 }' $regdir/regression_var.out) >>$cfile echo "" >>$cfile -echo "module purge" >> $cfile +echo "source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh" >> $cfile echo "module use $modulefiles" >> $cfile echo "module load gsi_gaea.intel" >> $cfile echo "module list" >> $cfile @@ -152,7 +158,7 @@ sbatch=${sbatch:-sbatch} ofile=$DATA/subout$$ >$ofile chmod 777 $ofile -$sbatch $cfile >$ofile +$sbatch --export=ALL $cfile >$ofile rc=$? cat $ofile if [[ -w $SUBLOG ]];then From e56921ceb212b70785d4180ca06829300385a711 Mon Sep 17 00:00:00 2001 From: David Huber Date: Tue, 13 Feb 2024 10:24:43 -0600 Subject: [PATCH 12/13] Re-update Hercules to SS/1.6.0. --- modulefiles/gsi_hercules.intel.lua | 11 +++-------- ush/sub_hercules | 7 ++----- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/modulefiles/gsi_hercules.intel.lua b/modulefiles/gsi_hercules.intel.lua index a646e9d3bf..c72323cbad 100644 --- a/modulefiles/gsi_hercules.intel.lua +++ b/modulefiles/gsi_hercules.intel.lua @@ -1,24 +1,19 @@ help([[ ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +local stack_python_ver=os.getenv("stack_python_ver") or "3.11.6" local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) load(pathJoin("python", stack_python_ver)) load(pathJoin("cmake", cmake_ver)) --- netcdf-fortran/4.6.1 does not work on Hercules yet -pushenv("netcdf_fortran_ver", "4.6.0") -pushenv("sp_ver", "2.3.3") -pushenv("crtm_ver", "2.4.0") - load("gsi_common") load(pathJoin("prod_util", prod_util_ver)) load("intel-oneapi-mkl/2022.2.1") diff --git a/ush/sub_hercules b/ush/sub_hercules index 03003307e9..459b480559 100755 --- a/ush/sub_hercules +++ b/ush/sub_hercules @@ -118,11 +118,7 @@ echo "" >>$cfile echo "export ntasks=$(( $nodes * $procs ))" >> $cfile echo "export ppn=$procs" >> $cfile echo "export threads=$threads" >> $cfile -# With newer IMPI versions, its sometimes necessary to tell MPI how to use parallel I/O -echo "export I_MPI_EXTRA_FILESYSTEM=1" >> $cfile -echo "export I_MPI_EXTRA_FILESYSTEM_FORCE=lustre" >> $cfile echo "export OMP_NUM_THREADS=$threads" >> $cfile -##echo "export OMP_STACKSIZE=2048M" >> $cfile echo "ulimit -s unlimited" >> $cfile echo "" >>$cfile @@ -133,7 +129,8 @@ echo ". /apps/other/lmod/lmod/init/sh" >> $cfile echo "module purge" >> $cfile echo "module use $modulefiles" >> $cfile echo "module load gsi_hercules.intel" >> $cfile -echo "module list" >> $cfile +#TODO reenable I_MPI_EXTRA_FILESYSTEM once regional ctests can properly handle parallel I/O on Hercules +echo "unset I_MPI_EXTRA_FILESYSTEM" >> $cfile echo "" >> $cfile cat $exec >> $cfile From 11a4ffdac805278f96632252da4a283b6e20d196 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Wed, 14 Feb 2024 13:28:14 +0000 Subject: [PATCH 13/13] Update openblas version for GNU/SS v1.6.0. --- modulefiles/gsi_hera.gnu.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/gsi_hera.gnu.lua b/modulefiles/gsi_hera.gnu.lua index c7bed47639..d85e79b005 100644 --- a/modulefiles/gsi_hera.gnu.lua +++ b/modulefiles/gsi_hera.gnu.lua @@ -10,7 +10,7 @@ local stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0" local stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5" local cmake_ver=os.getenv("cmake_ver") or "3.23.1" local prod_util_ver=os.getenv("prod_util_ver") or "2.1.1" -local openblas_ver=os.getenv("openblas_ver") or "0.3.19" +local openblas_ver=os.getenv("openblas_ver") or "0.3.24" load(pathJoin("stack-gcc", stack_gnu_ver)) load(pathJoin("stack-openmpi", stack_openmpi_ver))