Initial cycled support on Hercules. NOAA-EMC#1588

DavidHuber-NOAA · Jan 3, 2024 · 219b7e5 · 219b7e5
1 parent 11cb070
commit 219b7e5
Show file tree

Hide file tree

Showing 9 changed files with 36 additions and 13 deletions.
diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -12,7 +12,7 @@ fi
 
 step=$1
 
-export npe_node_max=40
+export npe_node_max=80
 export launcher="srun -l --export=ALL"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
@@ -26,11 +26,12 @@ export KMP_AFFINITY=scatter
 export OMP_STACKSIZE=2048000
 export NTHSTACK=1024000000
 #export LD_BIND_NOW=1
+export I_MPI_EXTRA_FILESYSTEM=1
+export I_MPI_EXTRA_FILESYSTEM_LIST=lustre
 
 ulimit -s unlimited
 ulimit -a
 
-<<<<<<< HEAD
 case ${step} in
  "prep" | "prepbufr")
 

diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua
@@ -8,7 +8,9 @@ prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack
 
 load(pathJoin("stack-intel", os.getenv("stack_intel_ver")))
 load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver")))
+load(pathJoin("intel-oneapi-mkl", os.getenv("intel_mkl_ver")))
 load(pathJoin("python", os.getenv("python_ver")))
+load(pathJoin("perl", os.getenv("perl_ver")))
 
 -- TODO load NCL once the SAs remove the 'depends_on' statements within it
 --      NCL is a static installation and does not depend on any libraries

diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn
@@ -61,9 +61,15 @@ export DO_NPOESS="NO"       # NPOESS products
 export DO_TRACKER="YES"    # Hurricane track verification
 export DO_GENESIS="YES"    # Cyclone genesis verification
 export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU)
-export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
-export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
-export DO_VMINMON="YES"    # GSI minimization monitoring
+if [[ "${machine}" == "HERCULES" ]]; then
+   export DO_VERFOZN="NO"     # Ozone data assimilation monitoring
+   export DO_VERFRAD="NO"     # Radiance data assimilation monitoring
+   export DO_VMINMON="NO"     # GSI minimization monitoring
+else
+   export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
+   export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
+   export DO_VMINMON="YES"    # GSI minimization monitoring
+fi
 export DO_MOS="NO"         # GFS Model Output Statistics - Only supported on WCOSS2
 
 # NO for retrospective parallel; YES for real-time parallel

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -56,7 +56,7 @@ elif [[ "${machine}" = "AWSPW" ]]; then
 elif [[ ${machine} = "ORION" ]]; then
    export npe_node_max=40
 elif [[ ${machine} = "HERCULES" ]]; then
-   export npe_node_max=40
+   export npe_node_max=80
 fi
 
 if [[ ${step} = "prep" ]]; then
@@ -907,10 +907,16 @@ elif [[ ${step} = "eobs" || ${step} = "eomg" ]]; then
     export npe_node_eobs
     export npe_node_eomg=${npe_node_eobs}
     export is_exclusive=True
-    #The number of tasks and cores used must be the same for eobs
-    #For S4, this is accomplished by running 10 tasks/node
+    # The number of tasks and cores used must be the same for eobs
+    # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
+    # For S4, this is accomplished by running 10 tasks/node
     if [[ ${machine} = "S4" ]]; then
        export npe_node_eobs=10
+    elif [[ ${machine} = "HERCULES" ]]; then
+       #For Hercules, this is only an issue at C384; use 20 tasks/node
+       if [[ ${CASE} = "C384" ]]; then
+          export npe_node_eobs=20
+       fi
     fi
 
 elif [[ ${step} = "ediag" ]]; then

diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
@@ -72,9 +72,12 @@ case "${machine}" in
   "WCOSS2")
     npe_node_max=128
     ;;
-  "HERA" | "ORION" | "HERCULES")
+  "HERA" | "ORION" )
     npe_node_max=40
     ;;
+  "HERCULES" )
+    npe_node_max=80
+    ;;
   "JET")
     case "${PARTITION_BATCH}" in
       "xjet")

diff --git a/scripts/exglobal_archive.sh b/scripts/exglobal_archive.sh
@@ -33,7 +33,9 @@ source "${HOMEgfs}/ush/file_utils.sh"
 
 [[ ! -d ${ARCDIR} ]] && mkdir -p "${ARCDIR}"
 nb_copy "${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat" "${ARCDIR}/gsistat.${RUN}.${PDY}${cyc}"
-nb_copy "${COM_CHEM_ANALYSIS}/${APREFIX}aerostat" "${ARCDIR}/aerostat.${RUN}.${PDY}${cyc}"
+if [[ ${DO_AERO} = "YES" ]]; then
+   nb_copy "${COM_CHEM_ANALYSIS}/${APREFIX}aerostat" "${ARCDIR}/aerostat.${RUN}.${PDY}${cyc}"
+fi
 nb_copy "${COM_ATMOS_GRIB_1p00}/${APREFIX}pgrb2.1p00.anl" "${ARCDIR}/pgbanl.${RUN}.${PDY}${cyc}.grib2"
 
 # Archive 1 degree forecast GRIB2 files for verification

diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd
diff --git a/ush/hpssarch_gen.sh b/ush/hpssarch_gen.sh
@@ -355,8 +355,10 @@ if [[ ${type} == "gdas" ]]; then
     if [[ -s "${COM_ATMOS_ANALYSIS}/${head}oznstat" ]]; then
        echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}oznstat"
     fi
-    if [[ -s "${COM_CHEM_ANALYSIS}/${head}aerostat" ]]; then
-       echo "${COM_CHEM_ANALYSIS/${ROTDIR}\//}/${head}aerostat"
+    if [[ ${DO_AERO} = "YES" ]]; then
+       if [[ -s "${COM_CHEM_ANALYSIS}/${head}aerostat" ]]; then
+          echo "${COM_CHEM_ANALYSIS/${ROTDIR}\//}/${head}aerostat"
+       fi
     fi
     if [[ -s "${COM_ATMOS_ANALYSIS}/${head}radstat" ]]; then
        echo "${COM_ATMOS_ANALYSIS/${ROTDIR}\//}/${head}radstat"

diff --git a/versions/run.hercules.ver b/versions/run.hercules.ver
@@ -1,5 +1,6 @@
 export stack_intel_ver=2021.9.0
 export stack_impi_ver=2021.9.0
+export intel_mkl_ver=2023.1.0
 
 export ncl_ver=6.6.2
+2 −2		ci/spack.yaml
+26 −0		modulefiles/gsiutils_hercules.lua
+2 −0		ush/detect_machine.sh
+8 −1		ush/module-setup.sh