Port rrfs-workflow to Hercules (#172)

* add hercules support * add mkl, add wflow_hercules and update wflow_orion * more upddates for hercules * add 'Hercules' to .github/PULL_REQUEST_TEMPLATE * load python in app_build.sh if a default one is not available, such as Hercules * I_MPI_EXTRA_FILESYSTEM=ON and load wget after intel * surpress 'python not found' error message --------- Co-authored-by: Guoqing Ge <[email protected]> Co-authored-by: ShawnMurdzek-NOAA <[email protected]>
NOAA-EMC · Dec 21, 2023 · a7f60e8 · a7f60e8
1 parent 1dd0688
commit a7f60e8
Show file tree

Hide file tree

Showing 44 changed files with 400 additions and 51 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
@@ -13,6 +13,7 @@
 - [ ] WCOSS2
 - [ ] Hera
 - [ ] Orion
+- [ ] Hercules
 - [ ] Jet
 
 - Test cases: 

diff --git a/README.md b/README.md
@@ -25,7 +25,7 @@ The above command is equal to:
 ./manage_externals/checkout_externals
 ./app_build.sh -p=[machine]
 ```
-where `[machine]` is `wcoss2`, `hera`, `jet`, or `orion`.
+where `[machine]` is `wcoss2`, `hera`, `jet`, `orion`, or `hercules`.
 
 4. Move to the home directory (rrfs-workflow):
 ```

diff --git a/doc/UsersGuide/source/Quickstart.rst b/doc/UsersGuide/source/Quickstart.rst
@@ -37,7 +37,7 @@ Building RRFS workflow
             
          ./app_build.sh --platform=<machine>
 
-      where ``<machine>`` is ``wcoss2``, ``hera``, ``jet``, or ``orion``.
+      where ``<machine>`` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``.
 
    #. Move to the home directory (rrfs-workflow):
 
@@ -58,15 +58,15 @@ Engineering Test: Non-DA
          module use modulefiles
          module load wflow_wcoss2
 
-      * On Hera | Jet | Orion:
+      * On Hera | Jet | Orion | Hercules:
 
       .. code-block:: console
          
          module use modulefiles
          module load wflow_<machine>
          conda activate workflow_tools
 
-      where ``<machine>`` is ``hera``, ``jet``, or ``orion``. 
+      where ``<machine>`` is ``hera``, ``jet``, ``orion``, or ``hercules``.
 
    #. Copy the pre-defined configuration file: 
 
@@ -75,7 +75,7 @@ Engineering Test: Non-DA
          cd ush
          cp sample_configs/non-DA_eng/config.nonDA.<format>.<machine>.sh config.sh
       
-      where ``<format>`` is ``grib2`` or ``netcdf``, and ``<machine>`` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``.
+      where ``<format>`` is ``grib2`` or ``netcdf``, and ``<machine>`` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``.
 
    #. Generate the experiment workflow:
 
@@ -112,15 +112,15 @@ Engineering Test: DA
          module use modulefiles
          module load wflow_wcoss2
 
-      * On Hera | Jet | Orion:
+      * On Hera | Jet | Orion | Hercules :
 
       .. code-block:: console
          
          module use modulefiles
          module load wflow_<machine>
          conda activate workflow_tools
 
-      where ``<machine>`` is ``hera``, ``jet``, or ``orion``. 
+      where ``<machine>`` is ``hera``, ``jet``, ``orion``, or ``hercules``. 
 
    #. Copy the pre-defined configuration file: 
 

diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua
@@ -0,0 +1,65 @@
+help([[
+This module loads libraries for building the RRFS workflow on
+the MSU machine Hercules using Intel-2021.9.0
+]])
+
+whatis([===[Loads libraries needed for building the RRFS worfklow on Hercules ]===])
+
+load("contrib")
+load("noaatools")
+
+prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0"))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0"))
+load("intel-oneapi-mkl/2022.2.1")
+load(pathJoin("cmake", os.getenv("cmake_ver") or "3.23.1"))
+
+load(pathJoin("jasper", os.getenv("jasper_ver") or "2.0.32"))
+load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
+load(pathJoin("parallelio", os.getenv("pio_ver") or "2.5.10"))
+--loading parallelio will load netcdf_c, netcdf_fortran, hdf5, zlib, etc
+load(pathJoin("esmf", os.getenv("esmf_ver") or "8.4.2"))
+load(pathJoin("fms", os.getenv("fms_ver") or "2023.01"))
+
+load(pathJoin("bacio", os.getenv("bacio_ver") or "2.4.1"))
+load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0"))
+load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5"))
+load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver") or "1.10.2"))
+load(pathJoin("ip", os.getenv("ip_ver") or "4.3.0"))
+load(pathJoin("sp", os.getenv("sp_ver") or "2.3.3"))
+
+load(pathJoin("gftl-shared", os.getenv("gftl-shared_ver") or "1.5.0"))
+--load(pathJoin("yafyaml", os.getenv("yafyaml_ver") or "0.5.1"))
+load(pathJoin("mapl", os.getenv("mapl_ver") or "2.35.2-esmf-8.4.2"))
+load(pathJoin("scotch", os.getenv("scotch_ver") or "7.0.4"))
+
+--load(pathJoin("bufr", os.getenv("bufr_ver") or "12.0.0"))
+load(pathJoin("gfsio", os.getenv("gfsio_ver") or "1.4.1"))
+load(pathJoin("landsfcutil", os.getenv("landsfcutil_ver") or "2.4.1"))
+load(pathJoin("nemsiogfs", os.getenv("nemsiogfs_ver") or "2.5.3"))
+load(pathJoin("sigio", os.getenv("sigio_ver") or "2.3.2"))
+load(pathJoin("sfcio", os.getenv("sfcio_ver") or "1.4.1"))
+load(pathJoin("nemsio", os.getenv("nemsio_ver") or "2.5.4"))
+--load(pathJoin("wrf_io", os.getenv("wrf_io_ver") or "1.2.0"))
+load(pathJoin("ncio", os.getenv("ncio_ver") or "1.1.2"))
+load(pathJoin("gsi-ncdiag", os.getenv("gsi-ncdiag_ver") or "1.1.1"))
+load(pathJoin("w3emc", os.getenv("w3emc_ver") or "2.10.0"))
+load(pathJoin("w3nco", os.getenv("w3nco_ver") or "2.4.1"))
+
+load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6"))
+--load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.0.14"))
+load(pathJoin("wgrib2", os.getenv("wgrib2_ver") or "3.1.1"))
+
+prepend_path("MODULEPATH", "/work/noaa/rtrr/gge/hercules/lua")
+load("prod_util/2.0.15")
+load("wrf_io/1.2.0a")
+load("bufr/11.7.0a")
+load("gsi-ncdiag/1.1.2a")
+
+unload("python/3.10.8")
+
+setenv("CMAKE_C_COMPILER","mpiicc")
+setenv("CMAKE_CXX_COMPILER","mpiicpc")
+setenv("CMAKE_Fortran_COMPILER","mpiifort")
+setenv("CMAKE_Platform","hercules.intel")
+setenv("I_MPI_EXTRA_FILESYSTEM","ON")
diff --git a/modulefiles/tasks/hercules/python_srw.lua b/modulefiles/tasks/hercules/python_srw.lua
@@ -0,0 +1,5 @@
+unload("python")
+append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
+load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+
+setenv("SRW_ENV", "workflow_tools")
diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua
@@ -0,0 +1,27 @@
+help([[
+This module loads python environement for running RRFS workflow on
+the MSU machine Hercules
+]])
+
+whatis([===[Loads libraries needed for running RRFS workflow on Hercules ]===])
+
+load("contrib")
+load("rocoto")
+
+prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0"))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0"))
+load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0"))
+
+load("wget")
+
+unload("python")
+append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles")
+load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+
+if mode() == "load" then
+   LmodMsgRaw([===[Please do the following to activate conda:
+       > conda activate workflow_tools
+]===])
+end
+
diff --git a/modulefiles/wflow_orion.lua b/modulefiles/wflow_orion.lua
@@ -9,10 +9,9 @@ load("contrib")
 load("rocoto")
 load("wget")
 
-prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack")
-load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
-load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2"))
-load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2"))
+prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2022.0.2"))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.5.1"))
 load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0"))
 
 unload("python")

diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml
@@ -745,7 +745,7 @@ MODULES_RUN_TASK_FP script.
       {%- elif machine in ["JET"] %}
         <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/@y@[email protected]@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc</cyclestr></datadep>
         <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/@y@[email protected]@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc</cyclestr></datadep>
-      {%- elif machine in ["ORION"] %}
+      {%- elif machine in ["ORION","HERCULES"] %}
         <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/@y@[email protected]@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc</cyclestr></datadep>
         <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/@y@[email protected]@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc</cyclestr></datadep>
       {%- endif %}
@@ -757,7 +757,7 @@ MODULES_RUN_TASK_FP script.
       <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }}</cyclestr></datadep>
       {%- elif machine in ["HERA"] %}
       <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }}</cyclestr></datadep>
-      {%- elif machine in ["ORION"] %}
+      {%- elif machine in ["ORION","HERCULES"] %}
       <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_ics_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }}</cyclestr></datadep>
       {%- endif %}
       {%- endif %}
@@ -866,7 +866,7 @@ MODULES_RUN_TASK_FP script.
          {%- elif machine in ["JET"] %}
           <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@[email protected]@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc</cyclestr></datadep>
           <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@[email protected]@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc</cyclestr></datadep>
-         {%- elif machine in ["ORION"] %}
+         {%- elif machine in ["ORION","HERCULES"] %}
           <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@[email protected]@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc</cyclestr></datadep>
           <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@[email protected]@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc</cyclestr></datadep>
          {%- endif %}
@@ -875,7 +875,7 @@ MODULES_RUN_TASK_FP script.
          <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }}</cyclestr></datadep>
          {%- elif machine in ["HERA"] %}
          <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }}</cyclestr></datadep>
-         {%- elif machine in ["ORION"] %}
+         {%- elif machine in ["ORION","HERCULES"] %}
          <datadep age="00:00:05:00"><cyclestr offset="-{{ extrn_mdl_lbcs_offset_hrs }}:00:00">{{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }}</cyclestr></datadep>
          {%- endif %}
          {%- endif %}
@@ -919,7 +919,7 @@ MODULES_RUN_TASK_FP script.
          {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_long_len_hrs+extrn_mdl_lbcs_offset_hrs+1, bc_update_interval) %}
          {%- if machine in ["WCOSS2"] %}
          <datadep age="00:00:05:00"><cyclestr>{{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/[email protected]{{ "%03d" % h }}</cyclestr></datadep>
-         {%- elif machine in ["ORION"]  %}
+         {%- elif machine in ["ORION","HERCULES"]  %}
          <datadep age="00:00:05:00"><cyclestr>{{ extrn_mdl_sysbasedir_lbcs }}/gdas.@Y@m@d/@H/atmos/[email protected]{{ "%03d" % h }}.nc</cyclestr></datadep>
          {%- else %}
          {%- if extrn_mdl_name_lbcs in ["GEFS"] %}

diff --git a/parm/FV3LAM_wflow_nonDA.xml b/parm/FV3LAM_wflow_nonDA.xml
@@ -22,7 +22,7 @@ Parameters needed by the job scheduler.
 <!ENTITY QUEUE_FCST      "{{ queue_fcst }}">
 <!ENTITY QUEUE_POST      "{{ queue_post }}">
 <!ENTITY RRFS_RESERVE  {% if reservation %}"--reservation={{ reservation }}"{% else %}""{% endif %}>
-<!ENTITY SCHED_NATIVE_CMD  {% if machine in ["HERA", "JET", "ORION"] %}"--export=NONE"{% else %}""{% endif %}>
+<!ENTITY SCHED_NATIVE_CMD  {% if machine in ["HERA", "JET", "ORION", "HERCULES"] %}"--export=NONE"{% else %}""{% endif %}>
 
 <!--
 Workflow task names.

diff --git a/scripts/exrrfs_bufrsnd.sh b/scripts/exrrfs_bufrsnd.sh
@@ -101,6 +101,11 @@ case $MACHINE in
     APRUNS="time"
     ;;
 
+  "HERCULES")
+    APRUNC="srun --export=ALL"
+    APRUNS="time"
+    ;;
+
   "JET")
     APRUNC="srun --export=ALL"
     APRUNS="time"

diff --git a/scripts/exrrfs_calc_ensmean.sh b/scripts/exrrfs_calc_ensmean.sh
@@ -78,6 +78,12 @@ case $MACHINE in
   APRUN="srun --export=ALL"
   ;;
 #
+"HERCULES")
+  export OMP_NUM_THREADS=1
+  export OMP_STACKSIZE=1024M
+  APRUN="srun --export=ALL"
+  ;;
+#
 "JET")
   APRUN="srun --export=ALL"
   ;;

diff --git a/scripts/exrrfs_make_ics.sh b/scripts/exrrfs_make_ics.sh
@@ -76,6 +76,10 @@ case "$MACHINE" in
     APRUN="srun --export=ALL"
     ;;
 
+  "HERCULES")
+    APRUN="srun --export=ALL"
+    ;;
+
   "JET")
     APRUN="srun --export=ALL"
     ;;

diff --git a/scripts/exrrfs_make_lbcs.sh b/scripts/exrrfs_make_lbcs.sh
@@ -97,6 +97,10 @@ case "$MACHINE" in
     APRUN="srun --export=ALL"
     ;;
 
+  "HERCULES")
+    APRUN="srun --export=ALL"
+    ;;
+
   "JET")
     APRUN="srun --export=ALL"
     ;;

diff --git a/scripts/exrrfs_make_sfc_climo.sh b/scripts/exrrfs_make_sfc_climo.sh
@@ -128,6 +128,10 @@ case $MACHINE in
     APRUN="srun --export=ALL"
     ;;
 
+  "HERCULES")
+    APRUN="srun --export=ALL"
+    ;;
+
   "JET")
     APRUN="srun --export=ALL"
     ;;

diff --git a/scripts/exrrfs_nonvarcldanl.sh b/scripts/exrrfs_nonvarcldanl.sh
@@ -96,6 +96,10 @@ case $MACHINE in
   APRUN="srun --export=ALL"
   ;;
 #
+"HERCULES")
+  APRUN="srun --export=ALL"
+  ;;
+#
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/scripts/exrrfs_process_bufr.sh b/scripts/exrrfs_process_bufr.sh
@@ -95,6 +95,10 @@ case $MACHINE in
   APRUN="srun --export=ALL"
   ;;
 #
+"HERCULES")
+  APRUN="srun --export=ALL"
+  ;;
+#
 esac
 #
 #-----------------------------------------------------------------------
@@ -173,7 +177,7 @@ else
     fi
 
     ;;
-  "JET" | "HERA" | "ORION")
+  "JET" | "HERA" | "ORION" | "HERCULES")
 
     obsfileprefix=${YYYYMMDDHH}.${obs_source}
     obspath_tmp=${OBSPATH}

diff --git a/scripts/exrrfs_process_lightning.sh b/scripts/exrrfs_process_lightning.sh
@@ -95,6 +95,10 @@ case $MACHINE in
   APRUN="srun --export=ALL"
   ;;
 #
+"HERCULES")
+  APRUN="srun --export=ALL"
+  ;;
+#
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/scripts/exrrfs_process_pm.sh b/scripts/exrrfs_process_pm.sh
@@ -95,6 +95,10 @@ case $MACHINE in
   APRUN="srun"
   ;;
 #
+"HERCULES")
+  APRUN="srun"
+  ;;
+#
 esac
 #
 #-----------------------------------------------------------------------

diff --git a/scripts/exrrfs_process_radarref.sh b/scripts/exrrfs_process_radarref.sh
@@ -96,6 +96,10 @@ case $MACHINE in
   APRUN="srun --export=ALL"
   ;;
 #
+"HERCULES")
+  APRUN="srun --export=ALL"
+  ;;
+#
 esac
 
 #
@@ -198,7 +202,7 @@ for bigmin in ${RADARREFL_TIMELEVEL[@]}; do
   "WCOSS2")
     obs_appendix=grib2.gz
     ;;
-  "JET" | "HERA" | "ORION")
+  "JET" | "HERA" | "ORION" | "HERCULES")
     obs_appendix=grib2
   esac
 

diff --git a/scripts/exrrfs_refl2tten.sh b/scripts/exrrfs_refl2tten.sh
@@ -96,6 +96,10 @@ case $MACHINE in
   APRUN="srun --export=ALL"
   ;;
 #
+"HERCULES")
+  APRUN="srun --export=ALL"
+  ;;
+#
 esac
 #
 #-----------------------------------------------------------------------