From 2bd4f83880a380565507adda1b1a0b736f7d5248 Mon Sep 17 00:00:00 2001 From: Guoqing Ge Date: Tue, 19 Dec 2023 20:51:57 -0700 Subject: [PATCH 1/7] add hercules support --- modulefiles/build_hercules_intel.lua | 63 +++++++++++++++++++ ush/fix_rrfs_locations.sh | 7 ++- ush/get_extrn_mdl_file_dir_info.sh | 18 +++--- ush/load_modules_wflow.sh | 2 +- .../non-DA_eng/config.nonDA.grib2.hercules.sh | 55 ++++++++++++++++ .../config.nonDA.netcdf.hercules.sh | 55 ++++++++++++++++ ush/set_extrn_mdl_params.sh | 16 ++--- ush/set_rrfs_config.sh | 2 +- ush/set_rrfs_config_general.sh | 23 ++++++- ush/setup.sh | 13 +++- ush/valid_param_vals.sh | 2 +- 11 files changed, 233 insertions(+), 23 deletions(-) create mode 100644 modulefiles/build_hercules_intel.lua create mode 100644 ush/sample_configs/non-DA_eng/config.nonDA.grib2.hercules.sh create mode 100644 ush/sample_configs/non-DA_eng/config.nonDA.netcdf.hercules.sh diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua new file mode 100644 index 000000000..e5715d8d5 --- /dev/null +++ b/modulefiles/build_hercules_intel.lua @@ -0,0 +1,63 @@ +help([[ +This module loads libraries for building the RRFS workflow on +the MSU machine Orion using Intel-2022.1.2 +]]) + +whatis([===[Loads libraries needed for building the RRFS worfklow on Orion ]===]) + +load("contrib") +load("noaatools") + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0")) +load(pathJoin("cmake", os.getenv("cmake_ver") or "3.23.1")) + +load(pathJoin("jasper", os.getenv("jasper_ver") or "2.0.32")) +load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) +load(pathJoin("parallelio", os.getenv("pio_ver") or "2.5.10")) +--loading parallelio will load netcdf_c, netcdf_fortran, hdf5, zlib, etc +load(pathJoin("esmf", os.getenv("esmf_ver") or "8.4.2")) +load(pathJoin("fms", os.getenv("fms_ver") or "2023.01")) + +load(pathJoin("bacio", os.getenv("bacio_ver") or "2.4.1")) +load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) +load(pathJoin("g2", os.getenv("g2_ver") or "3.4.5")) +load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver") or "1.10.2")) +load(pathJoin("ip", os.getenv("ip_ver") or "4.3.0")) +load(pathJoin("sp", os.getenv("sp_ver") or "2.3.3")) + +load(pathJoin("gftl-shared", os.getenv("gftl-shared_ver") or "1.5.0")) +--load(pathJoin("yafyaml", os.getenv("yafyaml_ver") or "0.5.1")) +load(pathJoin("mapl", os.getenv("mapl_ver") or "2.35.2-esmf-8.4.2")) +load(pathJoin("scotch", os.getenv("scotch_ver") or "7.0.4")) + +--load(pathJoin("bufr", os.getenv("bufr_ver") or "12.0.0")) +load(pathJoin("gfsio", os.getenv("gfsio_ver") or "1.4.1")) +load(pathJoin("landsfcutil", os.getenv("landsfcutil_ver") or "2.4.1")) +load(pathJoin("nemsiogfs", os.getenv("nemsiogfs_ver") or "2.5.3")) +load(pathJoin("sigio", os.getenv("sigio_ver") or "2.3.2")) +load(pathJoin("sfcio", os.getenv("sfcio_ver") or "1.4.1")) +load(pathJoin("nemsio", os.getenv("nemsio_ver") or "2.5.4")) +--load(pathJoin("wrf_io", os.getenv("wrf_io_ver") or "1.2.0")) +load(pathJoin("ncio", os.getenv("ncio_ver") or "1.1.2")) +load(pathJoin("gsi-ncdiag", os.getenv("gsi-ncdiag_ver") or "1.1.1")) +load(pathJoin("w3emc", os.getenv("w3emc_ver") or "2.10.0")) +load(pathJoin("w3nco", os.getenv("w3nco_ver") or "2.4.1")) + +load(pathJoin("nco", os.getenv("nco_ver") or "5.0.6")) +--load(pathJoin("prod_util", os.getenv("prod_util_ver") or "2.0.14")) +load(pathJoin("wgrib2", os.getenv("wgrib2_ver") or "3.1.1")) + +prepend_path("MODULEPATH", "/work/noaa/rtrr/gge/hercules/lua") +load("prod_util/2.0.15") +load("wrf_io/1.2.0a") +load("bufr/11.7.0a") +load("gsi-ncdiag/1.1.2a") + +unload("python/3.10.8") + +setenv("CMAKE_C_COMPILER","mpiicc") +setenv("CMAKE_CXX_COMPILER","mpiicpc") +setenv("CMAKE_Fortran_COMPILER","mpiifort") +setenv("CMAKE_Platform","orion.intel") diff --git a/ush/fix_rrfs_locations.sh b/ush/fix_rrfs_locations.sh index 25fc74b37..c038aa9fe 100755 --- a/ush/fix_rrfs_locations.sh +++ b/ush/fix_rrfs_locations.sh @@ -12,8 +12,13 @@ elif [[ -d /jetmon ]] ; then PLATFORM=jet FIX_RRFS_LOCATION="/lfs4/BMC/nrtrr/FIX_RRFS" elif [[ -d /work ]]; then - PLATFORM=orion FIX_RRFS_LOCATION="/work/noaa/rtrr/FIX_RRFS" + hoststr=$(hostname) + if [[ "$hoststr" == "hercules"* ]]; then + PLATFORM=hercules + else + PLATFORM=orion + fi else PLATFORM=unknown FIX_RRFS_LOCATION="/this/is/an/unknown/platform" diff --git a/ush/get_extrn_mdl_file_dir_info.sh b/ush/get_extrn_mdl_file_dir_info.sh index 49d037f04..3eb43ea90 100755 --- a/ush/get_extrn_mdl_file_dir_info.sh +++ b/ush/get_extrn_mdl_file_dir_info.sh @@ -419,7 +419,7 @@ case "${anl_or_fcst}" in elif [ "${MACHINE}" = "JET" ] ; then fns_on_disk=( "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.atmf0${fcst_hh}.${GDAS_MEM_NAME}.nc" "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.sfcf0${fcst_hh}.${GDAS_MEM_NAME}.nc") # use netcdf fns_in_arcv=( "gdas.t${hh}z.atmf0${fcst_hh}.nc" "gdas.t${hh}z.sfcf0${fcst_hh}.nc") # use netcdf - elif [ "${MACHINE}" = "ORION" ] ; then + elif [[ "${MACHINE}" = "ORION" ]] || [[ "${MACHINE}" = "HERCULES" ]]; then fns_on_disk=( "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.atmf0${fcst_hh}.${GDAS_MEM_NAME}.nc" "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.sfcf0${fcst_hh}.${GDAS_MEM_NAME}.nc") # use netcdf fns_in_arcv=( "gdas.t${hh}z.atmf0${fcst_hh}.nc" "gdas.t${hh}z.sfcf0${fcst_hh}.nc") # use netcdf fi @@ -548,7 +548,7 @@ and analysis or forecast (anl_or_fcst): elif [ "${MACHINE}" = "JET" ]; then fns_on_disk=( "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.atmf0${fcst_hh}.${GDAS_MEM_NAME}.nc" "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.sfcf0${fcst_hh}.${GDAS_MEM_NAME}.nc") # use netcdf fns_in_arcv=( "gdas.t${hh}z.atmf${fcst_hhh}.nc" "gdas.t${hh}z.sfcf${fcst_hhh}.nc" ) # for now. - elif [ "${MACHINE}" = "ORION" ]; then + elif [[ "${MACHINE}" = "ORION" ]] || [[ "${MACHINE}" = "HERCULES" ]]; then fns_on_disk=( "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.atmf0${fcst_hh}.${GDAS_MEM_NAME}.nc" "${yy}${ddd}${hh}${mn}.gdas.t${hh}z.sfcf0${fcst_hh}.${GDAS_MEM_NAME}.nc") # use netcdf fns_in_arcv=( "gdas.t${hh}z.atmf${fcst_hhh}.nc" "gdas.t${hh}z.sfcf${fcst_hhh}.nc" ) # for now. fi @@ -664,7 +664,7 @@ bination of external model (extrn_mdl_name) and analysis or forecast "HERA") sysdir="" ;; - "ORION") + "ORION"|"HERCULES") sysdir="$sysbasedir" ;; "JET") @@ -694,7 +694,7 @@ has not been specified for this external model and machine combination: sysdir="$sysbasedir" #sysdir="$sysbasedir/gfs.${yyyymmdd}/${hh}/atmos" ;; - "ORION") + "ORION"|"HERCULES") sysdir="$sysbasedir" #sysdir="$sysbasedir/gdas.${yyyymmdd}/${hh}/atmos" ;; @@ -722,7 +722,7 @@ has not been specified for this external model and machine combination: "JET") sysdir="$sysbasedir" ;; - "ORION") + "ORION"|"HERCULES") sysdir="$sysbasedir" ;; *) @@ -787,7 +787,7 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir" ;; - "ORION") + "ORION"|"HERCULES") sysdir="$sysbasedir" ;; "JET") @@ -812,7 +812,7 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir" ;; - "ORION") + "ORION"|"HERCULES") sysdir="$sysbasedir" ;; "JET") @@ -836,7 +836,7 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir" ;; - "ORION") + "ORION"|"HERCULES") sysdir="$sysbasedir" ;; "JET") @@ -860,7 +860,7 @@ has not been specified for this external model and machine combination: "HERA") sysdir="$sysbasedir" ;; - "ORION") + "ORION"|"HERCULES") sysdir="$sysbasedir" ;; "JET") diff --git a/ush/load_modules_wflow.sh b/ush/load_modules_wflow.sh index 11dfe4876..ecba3677b 100755 --- a/ush/load_modules_wflow.sh +++ b/ush/load_modules_wflow.sh @@ -16,7 +16,7 @@ Usage: source $0 PLATFORM OPTIONS: PLATFORM - name of machine you are on - (e.g. wcoss2 | hera | jet | orion ) + (e.g. wcoss2 | hera | jet | orion | hercules ) EOF_USAGE } diff --git a/ush/sample_configs/non-DA_eng/config.nonDA.grib2.hercules.sh b/ush/sample_configs/non-DA_eng/config.nonDA.grib2.hercules.sh new file mode 100644 index 000000000..9257c206e --- /dev/null +++ b/ush/sample_configs/non-DA_eng/config.nonDA.grib2.hercules.sh @@ -0,0 +1,55 @@ +MACHINE="hercules" +ACCOUNT="fv3-cam" +HPSS_ACCOUNT="fv3-cam" +EXPT_SUBDIR="test_nonDA_grib2" + +envir="test" +NET="test" +TAG="c0v00" +MODEL="test" +RUN="test" + +STMP="/work/noaa/fv3-cam/${USER}/test_nonDA_grib2" +PTMP="/work/noaa/fv3-cam/${USER}/test_nonDA_grib2" + +VERBOSE="TRUE" +PRINT_ESMF="TRUE" + +USE_CRON_TO_RELAUNCH="TRUE" +CRON_RELAUNCH_INTVL_MNTS="03" + +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" + +DATE_FIRST_CYCL="20190615" +DATE_LAST_CYCL="20190615" +CYCL_HRS=( "00" ) + +RUN_TASK_MAKE_GRID="TRUE" +RUN_TASK_MAKE_OROG="TRUE" +RUN_TASK_MAKE_SFC_CLIMO="TRUE" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" + +FV3GFS_FILE_FMT_ICS="grib2" +FV3GFS_FILE_FMT_LBCS="grib2" + +WTIME_RUN_FCST="01:00:00" +PPN_RUN_FCST="12" + +DO_NON_DA_RUN="TRUE" +DO_RETRO="TRUE" +VCOORD_FILE="global_hyblev.l65.txt" +WFLOW_XML_TMPL_FN="FV3LAM_wflow_nonDA.xml" +FV3_NML_YAML_CONFIG_FN="FV3.input.nonDA.yml" + +USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_SOURCE_BASEDIR_ICS="/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/2019061500" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/2019061500" diff --git a/ush/sample_configs/non-DA_eng/config.nonDA.netcdf.hercules.sh b/ush/sample_configs/non-DA_eng/config.nonDA.netcdf.hercules.sh new file mode 100644 index 000000000..6c90dbaa6 --- /dev/null +++ b/ush/sample_configs/non-DA_eng/config.nonDA.netcdf.hercules.sh @@ -0,0 +1,55 @@ +MACHINE="hercules" +ACCOUNT="fv3-cam" +HPSS_ACCOUNT="fv3-cam" +EXPT_SUBDIR="test_nonDA_netcdf" + +envir="test" +NET="test" +TAG="c0v00" +MODEL="test" +RUN="test" + +STMP="/work/noaa/fv3-cam/${USER}/test_nonDA_netcdf" +PTMP="/work/noaa/fv3-cam/${USER}/test_nonDA_netcdf" + +VERBOSE="TRUE" +PRINT_ESMF="TRUE" + +USE_CRON_TO_RELAUNCH="TRUE" +CRON_RELAUNCH_INTVL_MNTS="03" + +PREEXISTING_DIR_METHOD="rename" + +PREDEF_GRID_NAME="RRFS_CONUS_25km" +QUILTING="TRUE" + +CCPP_PHYS_SUITE="FV3_HRRR" +FCST_LEN_HRS="6" +LBC_SPEC_INTVL_HRS="6" + +DATE_FIRST_CYCL="20230217" +DATE_LAST_CYCL="20230217" +CYCL_HRS=( "06" ) + +RUN_TASK_MAKE_GRID="TRUE" +RUN_TASK_MAKE_OROG="TRUE" +RUN_TASK_MAKE_SFC_CLIMO="TRUE" + +EXTRN_MDL_NAME_ICS="FV3GFS" +EXTRN_MDL_NAME_LBCS="FV3GFS" + +FV3GFS_FILE_FMT_ICS="netcdf" +FV3GFS_FILE_FMT_LBCS="netcdf" + +WTIME_RUN_FCST="01:00:00" +PPN_RUN_FCST="12" + +DO_NON_DA_RUN="TRUE" +DO_RETRO="TRUE" +VCOORD_FILE="global_hyblev.l65.txt" +WFLOW_XML_TMPL_FN="FV3LAM_wflow_nonDA.xml" +FV3_NML_YAML_CONFIG_FN="FV3.input.nonDA.yml" + +USE_USER_STAGED_EXTRN_FILES="TRUE" +EXTRN_MDL_SOURCE_BASEDIR_ICS="/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/2023021706" +EXTRN_MDL_SOURCE_BASEDIR_LBCS="/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/netcdf/2023021706" diff --git a/ush/set_extrn_mdl_params.sh b/ush/set_extrn_mdl_params.sh index ca4e04041..392390134 100644 --- a/ush/set_extrn_mdl_params.sh +++ b/ush/set_extrn_mdl_params.sh @@ -59,7 +59,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_ICS="" ;; "JET") @@ -76,7 +76,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_ICS="" ;; "JET") @@ -121,7 +121,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_ICS="" ;; "JET") @@ -135,7 +135,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_ICS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_ICS="" ;; "JET") @@ -250,7 +250,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "JET") @@ -267,7 +267,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch1/NCEPDEV/rstprod/com/gfs/prod" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "JET") @@ -306,7 +306,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/rap/full/wrfnat" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "JET") @@ -320,7 +320,7 @@ else "HERA") EXTRN_MDL_SYSBASEDIR_LBCS="/scratch2/BMC/public/data/gsd/hrrr/conus/wrfnat" ;; - "ORION") + "ORION"|"HERCULES") EXTRN_MDL_SYSBASEDIR_LBCS="" ;; "JET") diff --git a/ush/set_rrfs_config.sh b/ush/set_rrfs_config.sh index 258d90445..4c23ff6a7 100644 --- a/ush/set_rrfs_config.sh +++ b/ush/set_rrfs_config.sh @@ -108,7 +108,7 @@ if [[ $DO_RETRO == "TRUE" ]] ; then RAPHRR_SOIL_ROOT=${RETRODATAPATH}/rap_hrrr_soil FIRE_RAVE_DIR=${RETRODATAPATH}/RAVE_RAW fi - if [[ $MACHINE == "orion" ]] ; then + if [[ $MACHINE == "orion" ]] || [[ $MACHINE == "hercules" ]] ; then if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then if [[ ${EXTRN_MDL_NAME_ICS} == "GDASENKF" ]]; then EXTRN_MDL_SOURCE_BASEDIR_ICS="/work/noaa/wrfruc/mhu/rrfs/data/enkf/atm" diff --git a/ush/set_rrfs_config_general.sh b/ush/set_rrfs_config_general.sh index 8ee1bf568..05e92e670 100644 --- a/ush/set_rrfs_config_general.sh +++ b/ush/set_rrfs_config_general.sh @@ -36,7 +36,7 @@ elif [[ $MACHINE == "hera" ]] ; then QUEUE_GRAPHICS="batch" elif [[ $MACHINE == "orion" ]] ; then - ACCOUNT="fv3-cam" + ACCOUNT="${ACCOUNT:-fv3-cam}" PARTITION_ANALYSIS=orion QUEUE_ANALYSIS="batch" QUEUE_PRDGEN="batch" @@ -44,6 +44,15 @@ elif [[ $MACHINE == "orion" ]] ; then QUEUE_POST="batch" NCORES_PER_NODE=24 +elif [[ $MACHINE == "hercules" ]] ; then + ACCOUNT="${ACCOUNT:-rtrr}" + PARTITION_ANALYSIS=hercules + QUEUE_ANALYSIS="batch" + QUEUE_PRDGEN="batch" + QUEUE_GRAPHICS="batch" + QUEUE_POST="batch" + NCORES_PER_NODE=24 + elif [[ $MACHINE == "wcoss2" ]] ; then ACCOUNT=RRFS-DEV HPSS_ACCOUNT="RRFS-DEV" @@ -275,6 +284,12 @@ if [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_3km" ]] ; then LAYOUT_Y="40" PPN_RUN_FCST="40" WTIME_RUN_PRDGEN="00:20:00" + + elif [[ $MACHINE == "hercules" ]] ; then + LAYOUT_X="30" + LAYOUT_Y="40" + PPN_RUN_FCST="40" + WTIME_RUN_PRDGEN="00:20:00" fi elif [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_13km" ]] ; then @@ -350,6 +365,12 @@ elif [[ ${PREDEF_GRID_NAME} == "RRFS_CONUS_13km" ]] ; then PPN_RUN_FCST="40" WTIME_RUN_PRDGEN="00:20:00" + elif [[ $MACHINE == "hercules" ]] ; then + LAYOUT_X="30" + LAYOUT_Y="40" + PPN_RUN_FCST="40" + WTIME_RUN_PRDGEN="00:20:00" + elif [[ $MACHINE == "hera" ]] ; then NNODES_MAKE_ICS="1" NNODES_MAKE_LBCS="1" diff --git a/ush/setup.sh b/ush/setup.sh index b60a99e7f..5a19b47d3 100755 --- a/ush/setup.sh +++ b/ush/setup.sh @@ -561,6 +561,17 @@ case $MACHINE in QUEUE_FCST=${QUEUE_FCST:-"batch"} ;; + "HERCULES") + NCORES_PER_NODE=40 + SCHED="${SCHED:-slurm}" + PARTITION_DEFAULT=${PARTITION_DEFAULT:-"hercules"} + QUEUE_DEFAULT=${QUEUE_DEFAULT:-"batch"} + PARTITION_HPSS=${PARTITION_HPSS:-"service"} + QUEUE_HPSS=${QUEUE_HPSS:-"batch"} + PARTITION_FCST=${PARTITION_FCST:-"hercules"} + QUEUE_FCST=${QUEUE_FCST:-"batch"} + ;; + "JET") NCORES_PER_NODE=${NCORES_PER_NODE} SCHED="${SCHED:-slurm}" @@ -817,7 +828,7 @@ case $MACHINE in FIXLAM_NCO_BASEDIR=${FIXLAM_NCO_BASEDIR:-"/scratch1/NCEPDEV/nems/role.epic/UFS_SRW_data/develop/FV3LAM_pregen"} ;; - "ORION") + "ORION"|"HERCULES") FIXgsm=${FIXgsm:-"/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_am"} TOPO_DIR=${TOPO_DIR:-"/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_orog"} SFC_CLIMO_INPUT_DIR=${SFC_CLIMO_INPUT_DIR:-"/work/noaa/epic/role-epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo"} diff --git a/ush/valid_param_vals.sh b/ush/valid_param_vals.sh index 0e86c7084..28c1f4651 100644 --- a/ush/valid_param_vals.sh +++ b/ush/valid_param_vals.sh @@ -1,6 +1,6 @@ valid_vals_VERBOSE=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") valid_vals_SAVE_CYCLE_LOG=("TRUE" "true" "YES" "yes" "FALSE" "false" "NO" "no") -valid_vals_MACHINE=("WCOSS2" "HERA" "ORION" "JET") +valid_vals_MACHINE=("WCOSS2" "HERA" "ORION" "JET" "HERCULES") valid_vals_SCHED=("slurm" "pbspro" "lsf" "lsfcray" "none") valid_vals_PREDEF_GRID_NAME=( \ "RRFS_CONUS_25km" \ From 9ce29848524cc1da2c21e543ead6133b8cbb5f41 Mon Sep 17 00:00:00 2001 From: "guoqing.ge" Date: Tue, 19 Dec 2023 22:19:43 -0700 Subject: [PATCH 2/7] add mkl, add wflow_hercules and update wflow_orion --- modulefiles/build_hercules_intel.lua | 1 + modulefiles/wflow_hercules.lua | 26 ++++++++++++++++++++++++++ modulefiles/wflow_orion.lua | 7 +++---- 3 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 modulefiles/wflow_hercules.lua diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index e5715d8d5..550a4f9da 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -11,6 +11,7 @@ load("noaatools") prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0")) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0")) +load("intel-oneapi-mkl/2022.2.1") load(pathJoin("cmake", os.getenv("cmake_ver") or "3.23.1")) load(pathJoin("jasper", os.getenv("jasper_ver") or "2.0.32")) diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua new file mode 100644 index 000000000..400b20139 --- /dev/null +++ b/modulefiles/wflow_hercules.lua @@ -0,0 +1,26 @@ +help([[ +This module loads python environement for running RRFS workflow on +the MSU machine Orion +]]) + +whatis([===[Loads libraries needed for running RRFS workflow on Orion ]===]) + +load("contrib") +load("rocoto") +load("wget") + +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2021.9.0")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0")) +load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) + +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +if mode() == "load" then + LmodMsgRaw([===[Please do the following to activate conda: + > conda activate workflow_tools +]===]) +end + diff --git a/modulefiles/wflow_orion.lua b/modulefiles/wflow_orion.lua index 0d2a8593b..b08e03f2c 100644 --- a/modulefiles/wflow_orion.lua +++ b/modulefiles/wflow_orion.lua @@ -9,10 +9,9 @@ load("contrib") load("rocoto") load("wget") -prepend_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/hpc-stack/intel-2022.1.2/modulefiles/stack") -load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2022.1.2")) -load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver") or "2022.1.2")) +prepend_path("MODULEPATH","/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core") +load(pathJoin("stack-intel", os.getenv("stack_intel_ver") or "2022.0.2")) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.5.1")) load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) unload("python") From ec674b31c5701ece67010f915683bfacefccd09e Mon Sep 17 00:00:00 2001 From: "guoqing.ge" Date: Tue, 19 Dec 2023 23:30:43 -0700 Subject: [PATCH 3/7] more upddates for hercules --- README.md | 2 +- doc/UsersGuide/source/Quickstart.rst | 12 ++++++------ modulefiles/build_hercules_intel.lua | 6 +++--- modulefiles/tasks/hercules/python_srw.lua | 5 +++++ modulefiles/wflow_hercules.lua | 6 +++--- parm/FV3LAM_wflow.xml | 10 +++++----- parm/FV3LAM_wflow_nonDA.xml | 2 +- scripts/exrrfs_bufrsnd.sh | 5 +++++ scripts/exrrfs_calc_ensmean.sh | 6 ++++++ scripts/exrrfs_make_ics.sh | 4 ++++ scripts/exrrfs_make_lbcs.sh | 4 ++++ scripts/exrrfs_make_sfc_climo.sh | 4 ++++ scripts/exrrfs_nonvarcldanl.sh | 4 ++++ scripts/exrrfs_process_bufr.sh | 6 +++++- scripts/exrrfs_process_lightning.sh | 4 ++++ scripts/exrrfs_process_pm.sh | 4 ++++ scripts/exrrfs_process_radarref.sh | 6 +++++- scripts/exrrfs_refl2tten.sh | 4 ++++ scripts/exrrfs_run_analysis.sh | 10 ++++++++-- scripts/exrrfs_run_enkf.sh | 6 ++++++ scripts/exrrfs_run_fcst.sh | 4 ++++ scripts/exrrfs_run_gsidiag.sh | 6 ++++++ scripts/exrrfs_run_post.sh | 6 ++++++ scripts/exrrfs_run_postanal.sh | 6 ++++++ scripts/exrrfs_run_prdgen.sh | 6 ++++++ scripts/exrrfs_run_prepstart.sh | 4 ++++ scripts/exrrfs_run_prepstart_fromext.sh | 4 ++++ scripts/exrrfs_run_recenter.sh | 6 ++++++ sorc/app_build.sh | 6 +++--- ush/etc/lmod-setup.csh | 2 +- ush/etc/lmod-setup.sh | 2 +- .../smoke_dust/config.sh.RRFS-SD_PM25DA_CONUS_3km | 4 ++-- 32 files changed, 136 insertions(+), 30 deletions(-) create mode 100644 modulefiles/tasks/hercules/python_srw.lua diff --git a/README.md b/README.md index 42ce15529..7066769c1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The above command is equal to: ./manage_externals/checkout_externals ./app_build.sh -p=[machine] ``` -where `[machine]` is `wcoss2`, `hera`, `jet`, or `orion`. +where `[machine]` is `wcoss2`, `hera`, `jet`, `orion`, or `hercules`. 4. Move to the home directory (rrfs-workflow): ``` diff --git a/doc/UsersGuide/source/Quickstart.rst b/doc/UsersGuide/source/Quickstart.rst index 4d9642687..108bb5d2f 100644 --- a/doc/UsersGuide/source/Quickstart.rst +++ b/doc/UsersGuide/source/Quickstart.rst @@ -37,7 +37,7 @@ Building RRFS workflow ./app_build.sh --platform= - where ```` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. + where ```` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``. #. Move to the home directory (rrfs-workflow): @@ -58,7 +58,7 @@ Engineering Test: Non-DA module use modulefiles module load wflow_wcoss2 - * On Hera | Jet | Orion: + * On Hera | Jet | Orion | Hercules: .. code-block:: console @@ -66,7 +66,7 @@ Engineering Test: Non-DA module load wflow_ conda activate workflow_tools - where ```` is ``hera``, ``jet``, or ``orion``. + where ```` is ``hera``, ``jet``, ``orion``, or ``hercules``. #. Copy the pre-defined configuration file: @@ -75,7 +75,7 @@ Engineering Test: Non-DA cd ush cp sample_configs/non-DA_eng/config.nonDA...sh config.sh - where ```` is ``grib2`` or ``netcdf``, and ```` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``. + where ```` is ``grib2`` or ``netcdf``, and ```` is ``wcoss2``, ``hera``, ``jet``, ``orion``, or ``hercules``. Note that you may need to change ``ACCOUNT``, ``STMP``, or ``PTMP`` in the configuration file ``config.sh``. #. Generate the experiment workflow: @@ -112,7 +112,7 @@ Engineering Test: DA module use modulefiles module load wflow_wcoss2 - * On Hera | Jet | Orion: + * On Hera | Jet | Orion | Hercules : .. code-block:: console @@ -120,7 +120,7 @@ Engineering Test: DA module load wflow_ conda activate workflow_tools - where ```` is ``hera``, ``jet``, or ``orion``. + where ```` is ``hera``, ``jet``, ``orion``, or ``hercules``. #. Copy the pre-defined configuration file: diff --git a/modulefiles/build_hercules_intel.lua b/modulefiles/build_hercules_intel.lua index 550a4f9da..5b416bbb1 100644 --- a/modulefiles/build_hercules_intel.lua +++ b/modulefiles/build_hercules_intel.lua @@ -1,9 +1,9 @@ help([[ This module loads libraries for building the RRFS workflow on -the MSU machine Orion using Intel-2022.1.2 +the MSU machine Hercules using Intel-2021.9.0 ]]) -whatis([===[Loads libraries needed for building the RRFS worfklow on Orion ]===]) +whatis([===[Loads libraries needed for building the RRFS worfklow on Hercules ]===]) load("contrib") load("noaatools") @@ -61,4 +61,4 @@ unload("python/3.10.8") setenv("CMAKE_C_COMPILER","mpiicc") setenv("CMAKE_CXX_COMPILER","mpiicpc") setenv("CMAKE_Fortran_COMPILER","mpiifort") -setenv("CMAKE_Platform","orion.intel") +setenv("CMAKE_Platform","hercules.intel") diff --git a/modulefiles/tasks/hercules/python_srw.lua b/modulefiles/tasks/hercules/python_srw.lua new file mode 100644 index 000000000..157e6884e --- /dev/null +++ b/modulefiles/tasks/hercules/python_srw.lua @@ -0,0 +1,5 @@ +unload("python") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") +load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) + +setenv("SRW_ENV", "workflow_tools") diff --git a/modulefiles/wflow_hercules.lua b/modulefiles/wflow_hercules.lua index 400b20139..d16b2adbc 100644 --- a/modulefiles/wflow_hercules.lua +++ b/modulefiles/wflow_hercules.lua @@ -1,9 +1,9 @@ help([[ This module loads python environement for running RRFS workflow on -the MSU machine Orion +the MSU machine Hercules ]]) -whatis([===[Loads libraries needed for running RRFS workflow on Orion ]===]) +whatis([===[Loads libraries needed for running RRFS workflow on Hercules ]===]) load("contrib") load("rocoto") @@ -15,7 +15,7 @@ load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver") or "2021.9.0 load(pathJoin("crtm", os.getenv("crtm_ver") or "2.4.0")) unload("python") -append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/orion/miniconda3/modulefiles") +append_path("MODULEPATH","/work/noaa/epic/role-epic/contrib/hercules/miniconda3/modulefiles") load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0")) if mode() == "load" then diff --git a/parm/FV3LAM_wflow.xml b/parm/FV3LAM_wflow.xml index a22fb1f63..cf0ced671 100644 --- a/parm/FV3LAM_wflow.xml +++ b/parm/FV3LAM_wflow.xml @@ -745,7 +745,7 @@ MODULES_RUN_TASK_FP script. {%- elif machine in ["JET"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.atmf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_ics }}/@y@j@H00.gdas.t@Hz.sfcf00{{ extrn_mdl_ics_offset_hrs }}.#memNameGDAS#.nc {%- endif %} @@ -757,7 +757,7 @@ MODULES_RUN_TASK_FP script. {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} {%- elif machine in ["HERA"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_ics }}/@y@j@H000{{ "%03d" % extrn_mdl_ics_offset_hrs }} {%- endif %} {%- endif %} @@ -866,7 +866,7 @@ MODULES_RUN_TASK_FP script. {%- elif machine in ["JET"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.atmf{{ "%03d" % h }}.#memNameGDAS#.nc {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H00.gdas.t@Hz.sfcf{{ "%03d" % h }}.#memNameGDAS#.nc {%- endif %} @@ -875,7 +875,7 @@ MODULES_RUN_TASK_FP script. {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} {%- elif machine in ["HERA"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/@y@j@H000{{ "%03d" % h }} {%- endif %} {%- endif %} @@ -919,7 +919,7 @@ MODULES_RUN_TASK_FP script. {%- for h in range(extrn_mdl_lbcs_offset_hrs, boundary_long_len_hrs+extrn_mdl_lbcs_offset_hrs+1, bc_update_interval) %} {%- if machine in ["WCOSS2"] %} {{ extrn_mdl_sysbasedir_lbcs }}/gfs.@Y@m@d/@H/atmos/gfs.t@Hz.pgrb2.0p25.f{{ "%03d" % h }} - {%- elif machine in ["ORION"] %} + {%- elif machine in ["ORION","HERCULES"] %} {{ extrn_mdl_sysbasedir_lbcs }}/gdas.@Y@m@d/@H/atmos/gdas.t@Hz.atmf{{ "%03d" % h }}.nc {%- else %} {%- if extrn_mdl_name_lbcs in ["GEFS"] %} diff --git a/parm/FV3LAM_wflow_nonDA.xml b/parm/FV3LAM_wflow_nonDA.xml index 0551ee712..26ed8ec3f 100644 --- a/parm/FV3LAM_wflow_nonDA.xml +++ b/parm/FV3LAM_wflow_nonDA.xml @@ -22,7 +22,7 @@ Parameters needed by the job scheduler. - +