From a7f60e8f4f96d6c197f0066859add144968afc22 Mon Sep 17 00:00:00 2001 From: Guoqing Ge Date: Thu, 21 Dec 2023 12:02:41 -0800 Subject: [PATCH] Port rrfs-workflow to Hercules (#172) * add hercules support * add mkl, add wflow_hercules and update wflow_orion * more upddates for hercules * add 'Hercules' to .github/PULL_REQUEST_TEMPLATE * load python in app_build.sh if a default one is not available, such as Hercules * I_MPI_EXTRA_FILESYSTEM=ON and load wget after intel * surpress 'python not found' error message --------- Co-authored-by: Guoqing Ge Co-authored-by: ShawnMurdzek-NOAA <113145197+ShawnMurdzek-NOAA@users.noreply.github.com> --- .github/PULL_REQUEST_TEMPLATE | 1 + README.md | 2 +- doc/UsersGuide/source/Quickstart.rst | 12 ++-- modulefiles/build_hercules_intel.lua | 65 +++++++++++++++++++ modulefiles/tasks/hercules/python_srw.lua | 5 ++ modulefiles/wflow_hercules.lua | 27 ++++++++ modulefiles/wflow_orion.lua | 7 +- parm/FV3LAM_wflow.xml | 10 +-- parm/FV3LAM_wflow_nonDA.xml | 2 +- scripts/exrrfs_bufrsnd.sh | 5 ++ scripts/exrrfs_calc_ensmean.sh | 6 ++ scripts/exrrfs_make_ics.sh | 4 ++ scripts/exrrfs_make_lbcs.sh | 4 ++ scripts/exrrfs_make_sfc_climo.sh | 4 ++ scripts/exrrfs_nonvarcldanl.sh | 4 ++ scripts/exrrfs_process_bufr.sh | 6 +- scripts/exrrfs_process_lightning.sh | 4 ++ scripts/exrrfs_process_pm.sh | 4 ++ scripts/exrrfs_process_radarref.sh | 6 +- scripts/exrrfs_refl2tten.sh | 4 ++ scripts/exrrfs_run_analysis.sh | 10 ++- scripts/exrrfs_run_enkf.sh | 6 ++ scripts/exrrfs_run_fcst.sh | 4 ++ scripts/exrrfs_run_gsidiag.sh | 6 ++ scripts/exrrfs_run_post.sh | 6 ++ scripts/exrrfs_run_postanal.sh | 6 ++ scripts/exrrfs_run_prdgen.sh | 6 ++ scripts/exrrfs_run_prepstart.sh | 4 ++ scripts/exrrfs_run_prepstart_fromext.sh | 4 ++ scripts/exrrfs_run_recenter.sh | 6 ++ sorc/app_build.sh | 10 ++- ush/etc/lmod-setup.csh | 2 +- ush/etc/lmod-setup.sh | 2 +- ush/fix_rrfs_locations.sh | 7 +- ush/get_extrn_mdl_file_dir_info.sh | 18 ++--- ush/load_modules_wflow.sh | 2 +- .../non-DA_eng/config.nonDA.grib2.hercules.sh | 55 ++++++++++++++++ .../config.nonDA.netcdf.hercules.sh | 55 ++++++++++++++++ .../config.sh.RRFS-SD_PM25DA_CONUS_3km | 4 +- ush/set_extrn_mdl_params.sh | 16 ++--- ush/set_rrfs_config.sh | 2 +- ush/set_rrfs_config_general.sh | 23 ++++++- ush/setup.sh | 13 +++- ush/valid_param_vals.sh | 2 +- 44 files changed, 400 insertions(+), 51 deletions(-) create mode 100644 modulefiles/build_hercules_intel.lua create mode 100644 modulefiles/tasks/hercules/python_srw.lua create mode 100644 modulefiles/wflow_hercules.lua create mode 100644 ush/sample_configs/non-DA_eng/config.nonDA.grib2.hercules.sh create mode 100644 ush/sample_configs/non-DA_eng/config.nonDA.netcdf.hercules.sh diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index 1ebf80966..32d71f122 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -13,6 +13,7 @@ - [ ] WCOSS2 - [ ] Hera - [ ] Orion +- [ ] Hercules - [ ] Jet - Test cases: diff --git a/README.md b/README.md index 42ce15529..7066769c1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The above command is equal to: ./manage_externals/checkout_externals ./app_build.sh -p=[machine] ``` -where `[machine]` is `wcoss2`, `hera`, `jet`, or `orion`. +where `[machine]` is `wcoss2`, `hera`, `jet`, `orion`, or `hercules`. 4. Move to the home directory (rrfs-workflow): ``` diff --git a/doc/UsersGuide/source/Quickstart.rst b/doc/UsersGuide/source/Quickstart.rst index 4d9642687..108bb5d2f 100644 --- a/doc/UsersGuide/source/Quickstart.rst +++ b/doc/UsersGuide/source/Quickstart.rst @@ -37,7 +37,7 @@ Building RRFS workflow ./app_build.sh --platform= - where ```` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. + where ```` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``. #. Move to the home directory (rrfs-workflow): @@ -58,7 +58,7 @@ Engineering Test: Non-DA module use modulefiles module load wflow_wcoss2 - * On Hera | Jet | Orion: + * On Hera | Jet | Orion | Hercules: .. code-block:: console @@ -66,7 +66,7 @@ Engineering Test: Non-DA module load wflow_ conda activate workflow_tools - where ```` is ``hera``, ``jet``, or ``orion``. + where ```` is ``hera``, ``jet``, ``orion``, or ``hercules``. #. Copy the pre-defined configuration file: @@ -75,7 +75,7 @@ Engineering Test: Non-DA cd ush cp sample_configs/non-DA_eng/config.nonDA...sh config.sh - where ```` is ``grib2`` or ``netcdf``, and ```` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``. + where ```` is ``grib2`` or ``netcdf``, and ```` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``. #. Generate the experiment workflow: @@ -112,7 +112,7 @@ Engineering Test: DA module use modulefiles module load wflow_wcoss2 - * On Hera | Jet | Orion: + * On Hera | Jet | Orion | Hercules : .. code-block:: console @@ -120,7 +120,7 @@ Engineering Test: DA module load wflow_ conda activate workflow_tools - where ```` is ``hera``, ``jet``, or ``orion``. + where ```` is ``hera``, ``jet``, ``orion``, or ``hercules``. #. Copy the pre-defined configuration file: diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua new file mode 100644 index 000000000..8482cfdae --- /dev/null +++ b/modulefiles/build_hercules_intel.lua @@ -0,0 +1,65 @@ +help([[ +This module loads libraries for building the RRFS workflow on +the MSU machine Hercules using Intel-2021.9.0 +]]) + +whatis([===[Loads libraries needed for building the RRFS worfklow on Hercules ]===]) + +load("contrib") +load("noaatools") + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0")) +load("intel-oneapi-mkl/2022.2.1") +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.23.1")) + +load(pathJoin("jasper", os.getenv("jasper_ver") or "2.0.32")) +load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) +load(pathJoin("parallelio", os.getenv("pio_ver") or "2.5.10")) +--loading parallelio will load netcdf_c, netcdf_fortran, hdf5, zlib, etc +load(pathJoin("esmf", os.getenv("esmf_ver") or "8.4.2")) +load(pathJoin("fms", os.getenv("fms_ver") or "2023.01")) + +load(pathJoin("bacio", os.getenv("bacio_ver") or "2.4.1")) +load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) +load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5")) +load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver") or "1.10.2")) +load(pathJoin("ip", os.getenv("ip_ver") or "4.3.0")) +load(pathJoin("sp", os.getenv("sp_ver") or "2.3.3")) + +load(pathJoin("gftl-shared", os.getenv("gftl-shared_ver") or "1.5.0")) +--load(pathJoin("yafyaml", os.getenv("yafyaml_ver") or "0.5.1")) +load(pathJoin("mapl", os.getenv("mapl_ver") or "2.35.2-esmf-8.4.2")) +load(pathJoin("scotch", os.getenv("scotch_ver") or "7.0.4")) + +--load(pathJoin("bufr", os.getenv("bufr_ver") or "12.0.0")) +load(pathJoin("gfsio", os.getenv("gfsio_ver") or "1.4.1")) +load(pathJoin("landsfcutil", os.getenv("landsfcutil_ver") or "2.4.1")) +load(pathJoin("nemsiogfs", os.getenv("nemsiogfs_ver") or "2.5.3")) +load(pathJoin("sigio", os.getenv("sigio_ver") or "2.3.2")) +load(pathJoin("sfcio", os.getenv("sfcio_ver") or "1.4.1")) +load(pathJoin("nemsio", os.getenv("nemsio_ver") or "2.5.4")) +--load(pathJoin("wrf_io", os.getenv("wrf_io_ver") or "1.2.0")) +load(pathJoin("ncio", os.getenv("ncio_ver") or "1.1.2")) +load(pathJoin("gsi-ncdiag", os.getenv("gsi-ncdiag_ver") or "1.1.1")) +load(pathJoin("w3emc", os.getenv("w3emc_ver") or "2.10.0")) +load(pathJoin("w3nco", os.getenv("w3nco_ver") or "2.4.1")) + +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) +--load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.0.14")) +load(pathJoin("wgrib2", os.getenv("wgrib2_ver") or "3.1.1")) + +prepend_path("MODULEPATH", "/work/noaa/rtrr/gge/hercules/lua") +load("prod_util/2.0.15") +load("wrf_io/1.2.0a") +load("bufr/11.7.0a") +load("gsi-ncdiag/1.1.2a") + +unload("python/3.10.8") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","hercules.intel") +setenv("I_MPI_EXTRA_FILESYSTEM","ON") diff --git a/modulefiles/tasks/hercules/python_srw.lua b/modulefiles/tasks/hercules/python_srw.lua new file mode 100644 index 000000000..157e6884e --- /dev/null +++ b/modulefiles/tasks/hercules/python_srw.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua new file mode 100644 index 000000000..1be5fcb35 --- /dev/null +++ b/modulefiles/wflow_hercules.lua @@ -0,0 +1,27 @@ +help([[ +This module loads python environement for running RRFS workflow on +the MSU machine Hercules +]]) + +whatis([===[Loads libraries needed for running RRFS workflow on Hercules ]===]) + +load("contrib") +load("rocoto") + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0")) +load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) + +load("wget") + +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate workflow_tools +]===]) +end + diff --git a/modulefiles/wflow_orion.lua b/modulefiles/wflow_orion.lua index 0d2a8593b..b08e03f2c 100644 --- a/modulefiles/wflow_orion.lua +++ b/modulefiles/wflow_orion.lua @@ -9,10 +9,9 @@ load("contrib") load("rocoto") load("wget") -prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack") -load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2")) -load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2")) +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2022.0.2")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.5.1")) load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) unload("python") diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index a22fb1f63..cf0ced671 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -745,7 +745,7 @@ MODULES_RUN_TASK_FP script. {%- elif machine in ["JET"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {%- endif %} @@ -757,7 +757,7 @@ MODULES_RUN_TASK_FP script. {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} {%- elif machine in ["HERA"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} {%- endif %} {%- endif %} @@ -866,7 +866,7 @@ MODULES_RUN_TASK_FP script. {%- elif machine in ["JET"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc {%- endif %} @@ -875,7 +875,7 @@ MODULES_RUN_TASK_FP script. {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} {%- elif machine in ["HERA"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} {%- endif %} {%- endif %} @@ -919,7 +919,7 @@ MODULES_RUN_TASK_FP script. {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_long_len_hrs+extrn_mdl_lbcs_offset_hrs+1, bc_update_interval) %} {%- if machine in ["WCOSS2"] %} {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f{{ "%03d" % h }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf{{ "%03d" % h }}.nc {%- else %} {%- if extrn_mdl_name_lbcs in ["GEFS"] %} diff --git a/parm/FV3LAM_wflow_nonDA.xml b/parm/FV3LAM_wflow_nonDA.xml index 0551ee712..26ed8ec3f 100644 --- a/parm/FV3LAM_wflow_nonDA.xml +++ b/parm/FV3LAM_wflow_nonDA.xml @@ -22,7 +22,7 @@ Parameters needed by the job scheduler. - +