Add data model support to HAFS workflow (hafs-community#91)

This PR allows developers to perform HAFS runs with the Community Data models for Earth Prediction Systems (CDEPS). Developers can use the new workflow capabilities to one-way couple the HYCOM ocean model to a data atmosphere created from the ERA5 reanalysis. Or they can one-way couple the UFS weather model to a data ocean created from the OISST or GHRSST analyses. The workflow is capable of running all of the necessary steps in an automated fashion, including pre-processing the input data, generating the data model grids, and running the forecast. Scripts are provided to download ERA5, OISST, and GHRSST data, which must be run offline. Documentation is available for exercising CDEPS support within the HAFS workflow. This PR is not expected to have any impact on HAFS runs that do not use data models.

Co-authored-by: Samuel Trahan <[email protected]>
Co-authored-by: Ufuk Turuncoglu <[email protected]>
Co-authored-by: Bin.Liu <[email protected]>
Co-authored-by: john.steffen <[email protected]>
Co-authored-by: Daniel Rosen <[email protected]>

.gitignore

@@ -7,3 +7,9 @@
 *.exe
 *.x
 *.log
+*~
+*.db
+*.db.bak
+system.conf
+fix/
+rocoto/*.xml

jobs/JHAFS_ATM_PREP

@@ -62,8 +62,13 @@ mkdir -p $OUTDIR $DATA
 cd $DATA
 
 # Execute ex-script
-${HOMEhafs}/scripts/exhafs_atm_prep.sh
-export err=$?
+if [[ "${run_datm:-no}" == yes ]] ; then
+    ${HOMEhafs}/scripts/exhafs_datm_prep.sh
+    export err=$?
+else
+    ${HOMEhafs}/scripts/exhafs_atm_prep.sh
+    export err=$?
+fi
 exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}

jobs/JHAFS_OCN_PREP

@@ -51,8 +51,13 @@ mkdir -p $DATA
 cd $DATA
 
 # Execute ex-script
-${HOMEhafs}/scripts/exhafs_ocn_prep.py
-export err=$?
+if [[ "${run_docn:-no}" == yes ]] ; then
+    ${HOMEhafs}/scripts/exhafs_docn_prep.sh
+    export err=$?
+else
+    ${HOMEhafs}/scripts/exhafs_ocn_prep.py
+    export err=$?
+fi
 exit $err
 
 export KEEPDATA=${KEEPDATA:-YES}

modulefiles/modulefile.hafs.hera

@@ -69,3 +69,6 @@ module use -a /scratch1/NCEPDEV/nems/emc.nemspara/soft/modulefiles
 module load rocoto/1.3.3
 
 module load intelpython/3.6.8
+
+# For CDEPS data models:
+module load cdo/1.9.10

modulefiles/modulefile.hafs.jet

@@ -65,3 +65,6 @@ module load nco/4.9.1
 module load rocoto/1.3.3
 
 module load intelpython/3.6.5
+
+# For CDEPS data models:
+module load cdo

modulefiles/modulefile.hafs.orion

@@ -65,3 +65,6 @@ module load nco/4.9.3
 module load rocoto/1.3.3
 
 module load intelpython3/2020
+
+# For CDEPS data models:
+module load cdo

modulefiles/modulefile.hafs.wcoss_dell_p3

@@ -73,3 +73,6 @@ setenv CMAKE_Platform wcoss_dell_p3
 module use /usrx/local/dev/emc_rocoto/modulefiles
 module load ruby/2.5.1
 module load rocoto/1.3.0rc2
+
+# For CDEPS data models:
+module load cdo

parm/cdeps/datm_era5.streams

@@ -0,0 +1,15 @@
+stream_info:               ERA5_HOURLY01
+taxmode01:                 limit 
+mapalgo01:                 redist
+tInterpAlgo01:             linear
+readMode01:                single
+dtlimit01:                 1.5
+stream_offset01:           0
+yearFirst01:               _yearFirst_ 
+yearLast01:                _yearLast_
+yearAlign01:               _yearFirst_
+stream_vectors01:          "u:v"
+stream_mesh_file01:        _mesh_atm_ 
+stream_lev_dimname01:      null
+stream_data_files01:      
+stream_data_variables01:   "u10 Sa_u10m" "v10 Sa_v10m" "t2m Sa_t2m" "skt Sa_tskn" "d2m Sa_tdew" "msl Sa_pslv" "tp Faxa_rain" "cp Faxa_rainc" "lsp Faxa_rainl" "csf Faxa_snowc" "lsf Faxa_snowl" "ssrd Faxa_swdn" "ssr Faxa_swnet" "strd Faxa_lwdn" "str Faxa_lwnet" "aluvp Faxa_swvdr" "aluvd Faxa_swvdf" "alnip Faxa_swndr" "alnid Faxa_swndf" "sshf Faxa_sen" "slhf Faxa_lat" "ewss Faxa_taux" "nsss Faxa_tauy"

parm/cdeps/datm_in

@@ -0,0 +1,14 @@
+&datm_nml
+  datamode = "ERA5"
+  factorfn_data = "null"
+  factorfn_mesh = "null"
+  flds_co2 = .false.
+  flds_presaero = .false.
+  flds_wiso = .false.
+  iradsw = 1
+  model_maskfile = "_mesh_atm_"
+  model_meshfile = "_mesh_atm_"
+  nx_global = 1440
+  ny_global = 721
+  restfilm = "null"
+/

parm/cdeps/docn_ghrsst.streams

@@ -0,0 +1,15 @@
+stream_info:               PRESCRIBED01
+taxmode01:                 limit
+mapalgo01:                 redist
+tInterpAlgo01:             linear
+readMode01:                single
+dtlimit01:                 1.5
+stream_offset01:           0
+yearFirst01:               _yearFirst_
+yearLast01:                _yearLast_
+yearAlign01:               _yearFirst_
+stream_vectors01:          "null"
+stream_mesh_file01:        "INPUT/DOCN_ESMF_mesh.nc"
+stream_lev_dimname01:      null
+stream_data_files01:       
+stream_data_variables01:   "analysed_sst So_t"

parm/cdeps/docn_in

@@ -0,0 +1,9 @@
+&docn_nml
+  datamode = "sstdata"
+  model_maskfile = "_mesh_ocn_"
+  model_meshfile = "_mesh_ocn_"
+  nx_global = _nx_global_
+  ny_global = _ny_global_
+  restfilm = "null"
+  sst_constant_value = -1.0
+/

parm/cdeps/docn_oisst.streams

@@ -0,0 +1,15 @@
+stream_info:               PRESCRIBED01
+taxmode01:                 limit
+mapalgo01:                 redist
+tInterpAlgo01:             linear
+readMode01:                single
+dtlimit01:                 1.5
+stream_offset01:           0
+yearFirst01:               _yearFirst_
+yearLast01:                _yearLast_
+yearAlign01:               _yearFirst_
+stream_vectors01:          "null"
+stream_mesh_file01:        _mesh_ocn_
+stream_lev_dimname01:      null
+stream_data_files01:      
+stream_data_variables01:   "sst So_t"

parm/forecast/globnest/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/forecast/globnest_hwrf/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/forecast/regional/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/forecast/regional/nems.configure.cdeps.tmp

@@ -0,0 +1,150 @@
+##############################################
+#####  NEMS Run-Time Configuration File  #####
+##############################################
+
+# EARTH #
+EARTH_component_list: MED ATM OCN
+EARTH_attributes::
+  Verbosity = 0
+::
+
+# MED #
+MED_model:                      cmeps
+_MED_petlist_bounds_
+MED_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  ATM_model = _atm_model_
+  OCN_model = _ocn_model_
+  MED_model = cmeps
+  history_n = 1
+  history_option = ndays
+  history_ymd = -999
+  coupling_mode = hafs
+  normalization = none
+  merge_type = copy
+::
+
+# ATM #
+ATM_model:                      _atm_model_
+_ATM_petlist_bounds_
+ATM_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  mesh_atm = _mesh_atm_
+::
+
+# OCN #
+OCN_model:                      _ocn_model_
+_OCN_petlist_bounds_
+OCN_attributes::
+  Verbosity = 1
+  Diagnostic = 0
+  mesh_ocn = _mesh_ocn_
+# The following are only used by the hycom ocean model. #
+  cdf_impexp_freq = 3
+  cpl_hour = 0
+  cpl_min = 0
+  cpl_sec = _cpl_dt_
+  base_dtg = _base_dtg_
+  merge_import = _merge_import_
+  skip_first_import = .true.
+  hycom_arche_output = .false.
+  hyc_esmf_exp_output = .true.
+  hyc_esmf_imp_output = .true.
+  import_diagnostics = .false.
+  import_setting = flexible
+  hyc_impexp_file = nems.configure
+  espc_show_impexp_minmax = .true.
+  ocean_start_dtg = _ocean_start_dtg_
+  start_hour = 0
+  start_min = 0
+  start_sec = 0
+  end_hour = _end_hour_
+  end_min = 0
+  end_sec = 0
+::
+
+# Run Sequence #
+runSeq::
+@_cpl_dt_
+  ATM -> MED :remapMethod=redist
+  MED med_phases_post_atm
+  OCN -> MED :remapMethod=redist
+  MED med_phases_post_ocn
+  MED med_phases_prep_atm
+  MED med_phases_prep_ocn_accum
+  MED med_phases_prep_ocn_avg
+  MED -> ATM :remapMethod=redist
+  MED -> OCN :remapMethod=redist
+  ATM
+  OCN
+  MED med_phases_restart_write
+  MED med_phases_history_write
+@
+::
+
+# Other Attributes #
+DRIVER_attributes::
+  start_type = startup
+::
+
+ALLCOMP_attributes::
+  ATM_model = _atm_model_
+  OCN_model = _ocn_model_
+  MED_model = cmeps
+  ScalarFieldCount = 3
+  ScalarFieldIdxGridNX = 1
+  ScalarFieldIdxGridNY = 2
+  ScalarFieldIdxNextSwCday = 3
+  ScalarFieldName = cpl_scalars
+  start_type = startup
+  case_name = ufs.hafs
+  restart_n = 1
+  restart_option = ndays
+  restart_ymd = -999
+  dbug_flag = 20
+  use_coldstart = true
+  orb_eccen = 1.e36
+  orb_iyear = 2000
+  orb_iyear_align = 2000
+  orb_mode = fixed_year
+  orb_mvelp = 1.e36
+  orb_obliq = 1.e36
+  mediator_read_restart = false
+  mediator_present = true
+::
+
+ATM_modelio::
+  diro = .
+  logfile = atm.log
+::
+
+OCN_modelio::
+  diro = .
+  logfile = ocn.log
+::
+
+MED_modelio::
+  diro = .
+  logfile = med.log
+::
+
+# The following are only used by the hycom ocean model. #
+ocn_export_fields::
+  'sst'         'sea_surface_temperature'   'K'
+  'mask'        'ocean_mask'                '1'
+  'cpl_scalars' 'cpl_scalars'               '1'
+::
+
+# The following are only used by the hycom ocean model. #
+ocn_import_fields::
+  'taux10'      'mean_zonal_moment_flx_atm' 'N_m-2'
+  'tauy10'      'mean_merid_moment_flx_atm' 'N_m-2'
+  'prcp'        'mean_prec_rate'            'kg_m-2_s-1'
+  'swflxd'      'mean_net_sw_flx'           'W_m-2'
+  'lwflxd'      'mean_net_lw_flx'           'W_m-2'
+  'mslprs'      'inst_pres_height_surface'  'Pa'
+  'sensflx'     'mean_sensi_heat_flx'       'W_m-2'
+  'latflx'      'mean_laten_heat_flx'       'W_m-2'
+::

parm/forecast/regional_hwrf/model_configure.tmp

@@ -1,4 +1,4 @@
-print_esmf:              .false.
+print_esmf:              _print_esmf_
 start_year:              YR
 start_month:             MN
 start_day:               DY

parm/hafs.conf

@@ -27,6 +27,9 @@ ENS=99                ;; The ensemble number (placeholder)
 # Specifies a section (default: [hafsdata]) to use: hafsdata, wcoss_fcst_nco
 input_catalog=fcst_{GFSVER}
 
+docn_source=NONE ;; Data source for data ocean model (GHRSST, OISST, or NONE)
+datm_source=NONE ;; Data source for data atmosphere model (ERA5 or NONE)
+
 ## Configure file and directory paths
 [dir]
 HOMEhafs={CDSAVE}/{EXPT}
@@ -57,9 +60,20 @@ gsistatus2=gsi_status.{vit[stormname]}{vit[stnum]:02d}{vit[basin1lc]}.{cycle}
 PARMforecast={PARMhafs}/forecast/regional   ;; The location where the forecast job will find its parm and namelist files
 PARMgsi={PARMhafs}/hafs-gsi/                ;; GSI input data for everything except CRTM
 FIXcrtm={FIXhafs}/hafs-crtm-2.2.3/          ;; GSI CRTM input data
+FIXcdeps={FIXhafs}/fix_cdeps                ;; CDEPS fix files
+FIXmeshes={FIXcdeps}/meshes                 ;; premade CDEPS meshes
 
 utilexec={HOMEhafs}/exec                    ;; utility exe location (placeholder)
 
+# Data model locations
+DOCNdir=/work/noaa/{disk_project}/{ENV[USER]}/DOCN
+DATMdir=/work/noaa/{disk_project}/{ENV[USER]}/DATM
+
+# Processed input files and meshes during workflow execution:
+docn_input_path={intercom}/cdeps
+datm_input_path={intercom}/cdeps
+
+
 ## Executable program locations
 # Currently not used in the workflow script system
 [exe]
@@ -226,6 +240,8 @@ npx=2561
 npy=2161
 npz=64
 
+print_esmf=.false.  ;; .true. to generate ESMF log files or .false. not to
+
 # The write_grid_component related options
 quilting=.true.
 write_groups=2
@@ -272,6 +288,15 @@ ocean_tasks=60 ;; Number of PEs for the OCN component
 ocean_start_dtg=auto ;; epoch day since hycom_epoch=datetime.datetime(1900,12,31,0,0,0), e.g., 43340.00000
 merge_import=.true.
 
+# Data model defaults
+mesh_ocn_in=missing ;; premade mesh to use if make_mesh_ocn=no
+mesh_ocn_gen={WORKhafs}/intercom/cdeps/DOCN_ESMF_mesh.nc ;; do not change
+mesh_atm_in=missing ;; premade mesh to use if make_mesh_atm=no
+mesh_atm_gen={WORKhafs}/intercom/cdeps/DATM_ESMF_mesh.nc ;; do not change
+docn_mesh_nx_global=1440 ;; Dimensions of data ocean model in X direction
+docn_mesh_ny_global=720 ;; Dimensions of data ocean model in Y direction
+
+
 [forecast_ens]
 # ccpp suites
 ccpp_suite_regional_ens={forecast/ccpp_suite_regional}
@@ -368,6 +393,7 @@ COMgfs={dir/COMgfs}                 ;; input GFS com directory
 COMrtofs={dir/COMrtofs}             ;; input RTOFS com directory
 gtype={grid/gtype}                  ;; grid type: uniform, stretch, nest, or regional (currently only nest and regional have been tested and supported)
 GFSVER={config/GFSVER}              ;; Version of GFS input data, e.g., PROD2019, PROD2021
+
 # Specify the forecast job resources. Only a few combinations are provided. If
 # needed, you may add other options in the site entity files under rocoto/sites.
 #FORECAST_RESOURCES=FORECAST_RESOURCES_regional_{forecast/layoutx}x{forecast/layouty}io{forecast/write_groups}x{forecast/write_tasks_per_group}_omp2
@@ -385,9 +411,13 @@ RUN_ENSDA={run_ensda}               ;; Do we run the ensda system?
 RUN_ENKF={run_enkf}                 ;; Do we run the self-cycled ensda system with EnKF analysis
 RUN_OCEAN={run_ocean}               ;; Do we run with ocean coupling?
 RUN_WAVE={run_wave}                 ;; Do we run with wave coupling?
+RUN_DATM={run_datm}                 ;; Do we run with a data atmosphere using CDEPS?
+RUN_DOCN={run_docn}                 ;; Do we run with a data ocean using CDEPS?
+RUN_DWAV={run_dwav}                 ;; Do we run with data waves using CDEPS?
 RUN_VORTEXINIT={run_vortexinit}     ;; Do we enable vortex initialization?
 RUN_HRDGRAPHICS={run_hrdgraphics}   ;; Do we run HRD graphics?
 RUN_EMCGRAPHICS={run_emcgraphics}   ;; Do we run EMC graphics?
 SCRUB_COM={scrub_com}               ;; Should Rocoto scrub the COM directory?
 SCRUB_WORK={scrub_work}             ;; Should Rocoto scrub the WORK directory?
-
+MAKE_MESH_ATM={make_mesh_atm}       ;; Should the DATM mesh be generated by the workflow?
+MAKE_MESH_OCN={make_mesh_ocn}       ;; Should the DOCN mesh be generated by the workflow?