Clone, build, and run C48_ATM and C48_S2SW on Gaea C5 and C6 (#3106)

What:
Correct build/run for C48_ATM and C48_S2SW on Gaea C5. Add build and run
capability for C48_ATM, C48_S2SW, and C96_atm3DVar on Gaea C6.
Why:
After the C5 OS upgrade, submodules no longer built in the
global-workflow. This PR correct that and adds build/run capability to
C6.

Resolves #3011

env/GAEA.env → env/GAEAC5.env

@@ -26,19 +26,19 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     NTHREADS1=${threads_per_task:-1}
     [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
-    # This may be useful when Gaea is fully ported, so ignore SC warning
+    # This may be useful when GaeaC5 is fully ported, so ignore SC warning
     # shellcheck disable=SC2034
     APRUN_default="${launcher} -n ${ntasks}"
 else
-    echo "ERROR config.resources must be sourced before sourcing GAEA.env"
+    echo "ERROR config.resources must be sourced before sourcing GAEAC5.env"
     exit 2
 fi
 
 if [[ "${step}" = "prep" ]]; then
 
     export POE="NO"
     export BACK="NO"
-    export sys_tp="GAEA"
+    export sys_tp="GAEAC5"
     export launcher_PREP="srun"
 
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
@@ -86,7 +86,7 @@ elif [[ "${step}" = "upp" ]]; then
 
 elif [[ "${step}" = "atmos_products" ]]; then
 
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on Gaea
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on GaeaC5
 
 elif [[ "${step}" = "oceanice_products" ]]; then
 
@@ -98,4 +98,11 @@ elif [[ "${step}" = "fit2obs" ]]; then
     export NTHREADS_FIT2OBS=${NTHREADS1}
     export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
 
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 fi

env/GAEAC6.env

@@ -0,0 +1,108 @@
+#! /usr/bin/env bash
+
+if [[ $# -ne 1 ]]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    exit 1
+
+fi
+
+step=$1
+
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
+
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+
+ulimit -s unlimited
+ulimit -a
+
+# Calculate common variables
+# Check first if the dependent variables are set
+if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
+    max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
+    NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
+    NTHREADS1=${threads_per_task:-1}
+    [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
+    [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
+    # This may be useful when GaeaC6 is fully ported, so ignore SC warning
+    # shellcheck disable=SC2034
+    APRUN_default="${launcher} -n ${ntasks}"
+else
+    echo "ERROR config.resources must be sourced before sourcing GAEAC6.env"
+    exit 2
+fi
+
+if [[ "${step}" = "prep" ]]; then
+
+    export POE="NO"
+    export BACK="NO"
+    export sys_tp="GAEAC6"
+    export launcher_PREP="srun"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_GSI=${NTHREADSmax}
+    export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    ntasks_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    ntasks_gausfcanl=${ntasks_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+
+elif [[ "${step}" = "sfcanl" ]]; then
+
+    export NTHREADS_CYCLE=${threads_per_task:-14}
+    export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+elif [[ "${step}" = "fcst" ]]; then
+
+    (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
+    (( ufs_ntasks = nnodes*tasks_per_node ))
+    # With ESMF threading, the model wants to use the full node
+    export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
+    unset nnodes ufs_ntasks
+
+elif [[ "${step}" = "upp" ]]; then
+
+    export NTHREADS_UPP=${NTHREADS1}
+    export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}"
+
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on GaeaC6
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN_default} --cpus-per-task=${NTHREADS_FIT2OBS}"
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
+fi

modulefiles/module_base.gaea.lua → modulefiles/module_base.gaeac5.lua

@@ -1,5 +1,5 @@
 help([[
-Load environment to run GFS on Gaea
+Load environment to run GFS on Gaea C5
 ]])
 
 local spack_mod_path=(os.getenv("spack_mod_path") or "None")

modulefiles/module_base.gaeac6.lua

@@ -0,0 +1,47 @@
+help([[
+Load environment to run GFS on Gaea C6
+]])
+
+local spack_mod_path=(os.getenv("spack_mod_path") or "None")
+prepend_path("MODULEPATH", spack_mod_path)
+
+load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
+load(pathJoin("stack-cray-mpich", (os.getenv("stack_cray_mpich_ver") or "None")))
+load(pathJoin("python", (os.getenv("python_ver") or "None")))
+
+load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
+load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))
+load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
+load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
+load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
+load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None")))
+
+load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
+load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
+load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None")))
+load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None")))
+load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None")))
+load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
+load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
+load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
+load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
+load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
+load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
+load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None")))
+load(pathJoin("met", (os.getenv("met_ver") or "None")))
+load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
+
+setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
+
+prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
+
+prepend_path("MODULEPATH", pathJoin("/gpfs/f6/bil-fire8/world-shared/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
+load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
+
+whatis("Description: GFS run setup environment")

modulefiles/module_gwsetup.gaea.lua → modulefiles/module_gwsetup.gaeac5.lua

@@ -1,13 +1,13 @@
 help([[
-Load environment to run GFS workflow setup scripts on Gaea
+Load environment to run GFS workflow setup scripts on Gaea C5
 ]])
 
 prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles")
 load(pathJoin("rocoto"))
 
 prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
 
-local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0"
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
 local python_ver=os.getenv("python_ver") or "3.10.13"
 
 load(pathJoin("stack-intel", stack_intel_ver))

modulefiles/module_gwsetup.gaeac6.lua

@@ -0,0 +1,20 @@
+help([[
+Load environment to run GFS workflow setup scripts on Gaea C6
+]])
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/rocoto/modulefiles")
+load(pathJoin("rocoto"))
+
+prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core")
+
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0"
+local python_ver=os.getenv("python_ver") or "3.10.13"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
+load("git-lfs")
+
+whatis("Description: GFS run setup environment")

parm/config/gfs/config.resources

@@ -47,11 +47,16 @@ case ${machine} in
               # shellcheck disable=SC2034
               mem_node_max="96GB"
     ;;
-  "GAEA")
+  "GAEAC5")
               max_tasks_per_node=128
               # shellcheck disable=SC2034
               mem_node_max="251GB"
     ;;
+  "GAEAC6")
+              max_tasks_per_node=192
+              # shellcheck disable=SC2034
+              mem_node_max="384GB"
+    ;;
   "ORION")
               max_tasks_per_node=40
               # shellcheck disable=SC2034

parm/config/gfs/config.resources.GAEA → parm/config/gfs/config.resources.GAEAC5

@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-# Gaea-specific job resources
+# GaeaC5-specific job resources
 
 case ${step} in
   "prep")
@@ -26,6 +26,8 @@ case ${step} in
 
 esac
 
+export FI_VERBS_PREFER_XRC=0
+
 unset memory
 # shellcheck disable=SC2312
 for mem_var in $(env | grep '^memory_' | cut -d= -f1); do

parm/config/gfs/config.resources.GAEAC6

@@ -0,0 +1,9 @@
+#! /usr/bin/env bash
+
+# GaeaC6-specific job resources
+
+unset memory
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done

sorc/link_workflow.sh

@@ -76,7 +76,8 @@ case "${machine}" in
 "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;;
 "jet") FIX_DIR="/lfs5/HFIP/hfv3gfs/glopara/FIX/fix" ;;
 "s4") FIX_DIR="/data/prod/glopara/fix" ;;
-"gaea") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
+"gaeac5") FIX_DIR="/gpfs/f5/ufs-ard/world-shared/global/glopara/data/fix" ;;
+"gaeac6") FIX_DIR="/gpfs/f6/bil-fire8/world-shared/global/glopara/fix" ;;
 "noaacloud") FIX_DIR="/contrib/global-workflow-shared-data/fix" ;;
 *)
   echo "FATAL: Unknown target machine ${machine}, couldn't set FIX_DIR"

ush/detect_machine.sh

@@ -21,8 +21,11 @@ case $(hostname -f) in
   dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9
   dlogin10.dogwood.wcoss2.ncep.noaa.gov)     MACHINE_ID=wcoss2 ;; ### dogwood10
 
-  gaea5[1-8])          MACHINE_ID=gaea ;; ### gaea51-58
-  gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58
+  gaea5[1-8])          MACHINE_ID=gaeac5 ;; ### gaea51-58
+  gaea5[1-8].ncrc.gov) MACHINE_ID=gaeac5 ;; ### gaea51-58
+
+  gaea6[1-8])          MACHINE_ID=gaeac6 ;; ### gaea61-68
+  gaea6[1-8].ncrc.gov) MACHINE_ID=gaeac6 ;; ### gaea61-68
 
   hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09
   hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12
@@ -81,9 +84,12 @@ elif [[ -d /work ]]; then
   else
     MACHINE_ID=orion
   fi
-elif [[ -d /gpfs && -d /ncrc ]]; then
-  # We are on GAEA.
-  MACHINE_ID=gaea
+elif [[ -d /gpfs/f5 ]]; then
+  # We are on GAEAC5.
+  MACHINE_ID=gaeac5
+elif [[ -d /gpfs/f6 ]]; then
+  # We are on GAEAC6.
+  MACHINE_ID=gaeac6
 elif [[ -d /data/prod ]]; then
   # We are on SSEC's S4
   MACHINE_ID=s4

ush/load_fv3gfs_modules.sh

@@ -20,7 +20,7 @@ source "${HOMEgfs}/versions/run.ver"
 module use "${HOMEgfs}/modulefiles"
 
 case "${MACHINE_ID}" in
-  "wcoss2" | "hera" | "orion" | "hercules" | "gaea" | "jet" | "s4" | "noaacloud")
+  "wcoss2" | "hera" | "orion" | "hercules" | "gaeac5" | "gaeac6" | "jet" | "s4" | "noaacloud")
     module load "module_base.${MACHINE_ID}"
     ;;
   *)

ush/load_ufsda_modules.sh

@@ -34,7 +34,7 @@ source "${HOMEgfs}/ush/module-setup.sh"
 module use "${HOMEgfs}/sorc/gdas.cd/modulefiles"
 
 case "${MACHINE_ID}" in
-  ("hera" | "orion" | "hercules" | "wcoss2")
+  ("hera" | "orion" | "hercules" | "wcoss2" | "gaeac5" | "gaeac6")
     #TODO: Remove LMOD_TMOD_FIND_FIRST line when spack-stack on WCOSS2
     if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
       export LMOD_TMOD_FIND_FIRST=yes
@@ -48,7 +48,7 @@ case "${MACHINE_ID}" in
     NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 )
     export NETCDF
     ;;
-  ("jet" | "gaea" | "s4" | "acorn")
+  ("jet" | "s4" | "acorn")
     echo WARNING: UFSDA NOT SUPPORTED ON THIS PLATFORM
     ;;  
   *)

versions/run.gaea.ver → versions/build.gaeac6.ver

@@ -1,8 +1,5 @@
 export stack_intel_ver=2023.1.0
 export stack_cray_mpich_ver=8.1.25
 export spack_env=gsi-addon-dev
-
-export perl_ver=5.38.2
-
 source "${HOMEgfs:-}/versions/spack.ver"
 export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

versions/run.gaeac5.ver

@@ -0,0 +1,8 @@
+export stack_intel_ver=2023.2.0
+export stack_cray_mpich_ver=8.1.28
+export spack_env=gsi-addon-dev
+
+export perl_ver=5.38.2
+
+source "${HOMEgfs:-}/versions/spack.ver"
+export spack_mod_path="/ncrc/proj/epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"

versions/run.gaeac6.ver

@@ -0,0 +1,8 @@
+export stack_intel_ver=2023.2.0
+export stack_cray_mpich_ver=8.1.29
+export spack_env=gsi-addon
+
+export perl_ver=5.38.2
+
+source "${HOMEgfs:-}/versions/spack.ver"
+export spack_mod_path="/ncrc/proj/epic/spack-stack/c6/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"