Make assorted updates to waves (#3190)

This PR adds the following: * converting from inp -> nml (@sbanihash) * turning on PIO for waves for restarts (@sbanihash) * enabling cycling for WW3 which required some updates to wave prep jobs + changing what restarts are being saved/etc * changed the way CEMPS, MOM6, CICE and WW3 write restarts to be in sync with FV3 for IAU, which required moving the ufs-weather-model forward one hash to use the new flexible restart feature. (UFS PR ufs-community/ufs-weather-model#2419) * adds uglo_15km - the targeted new wave grid. * Update to use new esmf_threading ufs.configure files which changes the toggle between how you use esmf threading versus traditional threading (UFS PR ufs-community/ufs-weather-model#2538) Notes on ufs-weather-model updates: | Commit date | Commit hash/ PR | Notes for g-w changes | Baseline Changes | | :------------- | :------------- | :------------- | :------------- | | Dec 11, 2024 | ufs-community/ufs-weather-model@409bc85 ufs-community/ufs-weather-model#2419 | Enables flexible restart writes - changes included in g-w PR | none| | Dec 16, 2024 | ufs-community/ufs-weather-model@6ec6b45 ufs-community/ufs-weather-model#2528 ufs-community/ufs-weather-model#2469 | n/a | HAFs test changes, no global changes | | Dec 18, 2024 |ufs-community/ufs-weather-model@e119370 ufs-community/ufs-weather-model#2448 | Adds Gaea C6 support (changes in other g-w PRs, not here) | none | |Dec 23, 2024 | ufs-community/ufs-weather-model@2950089 ufs-community/ufs-weather-model#2533 ufs-community/ufs-weather-model#2538 | changes for ESMF vs traditional threading | none | |Dec 30, 2024 | ufs-community/ufs-weather-model@241dd8e ufs-community/ufs-weather-model#2485 | n/a | changes in conus13km, no global changes| |Jan 3, 2025 | ufs-community/ufs-weather-model@76471dc ufs-community/ufs-weather-model#2530 | n/a | changes in regional tests, no global changes | Note this PR requires the following: * update to fix files to add uglo_15km * staging ICs for high resolution test case for uglo_15km Co-author: @sbanihash Related Issues: - Fixes #1457 - Fixes #3154 - Fixes #1795 - related to #1776 --------- Co-authored-by: Rahul Mahajan <[email protected]> Co-authored-by: Saeideh Banihashemi <[email protected]> Co-authored-by: David Huber <[email protected]> Co-authored-by: Walter Kolczynski - NOAA <[email protected]>
NOAA-EMC · Jan 17, 2025 · 57ce1b0 · 57ce1b0
1 parent 4c6c6a4
commit 57ce1b0
Show file tree

Hide file tree

Showing 25 changed files with 207 additions and 243 deletions.
diff --git a/.gitignore b/.gitignore
@@ -81,6 +81,7 @@ parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN
 parm/ufs/ufs.configure.*.IN
 parm/ufs/post_itag_gfs
+parm/ufs/ww3_shel.nml.IN
 parm/wafs
 
 # Ignore sorc and logs folders from externals

diff --git a/docs/source/wave.rst b/docs/source/wave.rst
@@ -21,6 +21,8 @@ Here are several regional naming conventions:
 +===========+=======================+
 | glo       | Global domain         |
 +-----------+-----------------------+
+| uglo      | Unstructured global   |
++-----------+-----------------------+
 | ak        | Alaska                |
 +-----------+-----------------------+
 | ao or aoc | Arctic Ocean          |

diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2
@@ -1,5 +1,5 @@
 gdaswave:
-    {% set head = "gdas.wave.t" + cycle_HH + "z." %}
+    {% set head = "gdaswave.t" + cycle_HH + "z." %}
     name: "GDASWAVE"
     target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar"
     required:

diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs
@@ -512,9 +512,7 @@ fi
 
 # Set the name of the UFS (previously nems) configure template to use
 # Default ufs.configure templates for supported model configurations
-if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
-  tmpl_suffix="_esmf"
-fi
+# WW3 restart field variable is different for slow vs fast loop.  Add  WW3_RSTFLDS="ice" for slow loop variables based on coupling scheme. 
 case "${model_list}" in
   atm)
     default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN"
@@ -533,9 +531,11 @@ case "${model_list}" in
     ;;
   atm.ocean.ice.wave)
     default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   atm.ocean.ice.wave.aero)
     default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   *)
     echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}"
@@ -547,6 +547,9 @@ esac
 export ufs_configure_template=${ufs_configure_template:-${default_template:-"/dev/null"}}
 unset model_list default_template
 
+#export wave restart variable: 
+export WW3_RSTFLDS=${WW3_RSTFLDS:-" "}
+
 if [[ ! -r "${ufs_configure_template}" ]]; then
   echo "FATAL ERROR: ${ufs_configure_template} either doesn't exist or is not readable."
   exit 17

diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
@@ -219,7 +219,7 @@ case "${CASE}" in
         export waveGRD='uglo_100km'
         ;;
     "C768" | "C1152")
-        export waveGRD='uglo_m1g16'
+        export waveGRD='uglo_15km'
         ;;
     *)
         echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -198,8 +198,8 @@ case ${step} in
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     NTASKS=${ntasks}
-    memory_gdas="10GB"
-    memory_gfs="10GB"
+    memory_gdas="20GB"
+    memory_gfs="20GB"
     ;;
 
   # The wavepost*pnt* jobs are I/O heavy and do not scale well to large nodes.

diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
@@ -15,7 +15,7 @@ if (( $# <= 1 )); then
     echo "--fv3 C48|C96|C192|C384|C768|C1152|C3072"
     echo "--mom6 500|100|025"
     echo "--cice6 500|100|025"
-    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_100|glo_200|glo_500|mx025|uglo_100km|uglo_m1g16"
+    echo "--ww3 gnh_10m;aoc_9km;gsh_15m|gwes_30m|glo_025|glo_100|glo_200|glo_500|mx025|uglo_100km"
     echo "--gocart"
 
     exit 1
@@ -605,12 +605,12 @@ if [[ "${skip_ww3}" == "false" ]]; then
     "mx025")
       ntasks_ww3=80
       ;;
-    "uglo_100km")
-      ntasks_ww3=40
+    "uglo_15km")
+      ntasks_ww3=1000
       nthreads_ww3=1
       ;;
-    "uglo_m1g16")
-      ntasks_ww3=1000
+    "uglo_100km")
+      ntasks_ww3=40
       nthreads_ww3=1
       ;;
     *)
@@ -630,9 +630,7 @@ fi
 
 # Set the name of the UFS (previously nems) configure template to use
 # Default ufs.configure templates for supported model configurations
-if [[ "${USE_ESMF_THREADING:-}" == "YES" ]]; then
-  tmpl_suffix="_esmf"
-fi
+# WW3 restart field variable is different for slow vs fast loop.  Add  WW3_RSTFLDS="ice" for slow loop variables based on coupling scheme.
 case "${model_list}" in
   atm)
     default_template="${PARMgfs}/ufs/ufs.configure.atm${tmpl_suffix:-}.IN"
@@ -651,9 +649,11 @@ case "${model_list}" in
     ;;
   atm.ocean.ice.wave)
     default_template="${PARMgfs}/ufs/ufs.configure.s2sw${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   atm.ocean.ice.wave.aero)
     default_template="${PARMgfs}/ufs/ufs.configure.s2swa${tmpl_suffix:-}.IN"
+    WW3_RSTFLDS="ice"
     ;;
   *)
     echo "FATAL ERROR: Unsupported UFSWM configuration for ${model_list}"
@@ -665,6 +665,9 @@ esac
 export ufs_configure_template=${ufs_configure_template:-${default_template:-"/dev/null"}}
 unset model_list default_template
 
+# export wave restart variable: 
+export WW3_RSTFLDS=${WW3_RSTFLDS:-" "}
+
 if [[ ! -r "${ufs_configure_template}" ]]; then
   echo "FATAL ERROR: ${ufs_configure_template} either doesn't exist or is not readable."
   exit 17

diff --git a/parm/config/gfs/config.wave b/parm/config/gfs/config.wave
@@ -56,18 +56,18 @@ case "${waveGRD}" in
     export wavepostGRD='glo_500'
     export waveuoutpGRD=${waveGRD}
     ;;
+  "uglo_15km")
+    # unstructured 15km grid
+    export waveinterpGRD='glo_15mxt at_10m ep_10m wc_10m glo_30m'
+    export wavepostGRD=''
+    export waveuoutpGRD=${waveGRD}
+    ;; 
   "uglo_100km")
     #unstructured 100km grid
     export waveinterpGRD='glo_200'
     export wavepostGRD=''
     export waveuoutpGRD=${waveGRD}
     ;;
-  "uglo_m1g16")
-    #unstructured m1v16 grid
-    export waveinterpGRD='glo_15mxt'
-    export wavepostGRD=''
-    export waveuoutpGRD=${waveGRD}
-    ;;
    *)
     echo "FATAL ERROR: No grid specific wave config values exist for ${waveGRD}. Aborting."
     exit 1

diff --git a/parm/wave/ak_10m_interp.inp.tmpl b/parm/wave/ak_10m_interp.inp.tmpl
@@ -3,10 +3,9 @@ $------------------------------------------------
 $ Start Time DT NSteps
  TIME DT NSTEPS
 $ Total number of grids 
- 3
+ 2
 $ Grid extensions 
-  'gnh_10m'
-  'aoc_9km'
+  'uglo_15km'
   'ak_10m'
 $
  0

diff --git a/parm/wave/at_10m_interp.inp.tmpl b/parm/wave/at_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'uglo_m1g16'
+  'uglo_15km'
   'at_10m'
 $
  0

diff --git a/parm/wave/ep_10m_interp.inp.tmpl b/parm/wave/ep_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'uglo_m1g16'
+  'uglo_15km'
   'ep_10m'
 $
  0

diff --git a/parm/wave/glo_15mxt_interp.inp.tmpl b/parm/wave/glo_15mxt_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'uglo_m1g16'
+  'uglo_15km'
   'glo_15mxt'
 $
  0

diff --git a/parm/wave/glo_30m_interp.inp.tmpl b/parm/wave/glo_30m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'uglo_m1g16'
+  'uglo_15km'
   'glo_30m'
 $
  0

diff --git a/parm/wave/wc_10m_interp.inp.tmpl b/parm/wave/wc_10m_interp.inp.tmpl
@@ -5,7 +5,7 @@ $ Start Time DT NSteps
 $ Total number of grids 
  2
 $ Grid extensions 
-  'uglo_m1g16'
+  'uglo_15km'
   'wc_10m'
 $
  0

diff --git a/parm/wave/ww3_shel.inp.tmpl b/parm/wave/ww3_shel.inp.tmpl
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
@@ -175,19 +175,13 @@ declare -a ufs_templates=("model_configure.IN" "input_global_nest.nml.IN"
   "MOM6_data_table.IN"
   "ice_in.IN"
   "ufs.configure.atm.IN"
-  "ufs.configure.atm_esmf.IN"
   "ufs.configure.atmaero.IN"
-  "ufs.configure.atmaero_esmf.IN"
   "ufs.configure.s2s.IN"
-  "ufs.configure.s2s_esmf.IN"
   "ufs.configure.s2sa.IN"
-  "ufs.configure.s2sa_esmf.IN"
   "ufs.configure.s2sw.IN"
-  "ufs.configure.s2sw_esmf.IN"
   "ufs.configure.s2swa.IN"
-  "ufs.configure.s2swa_esmf.IN"
   "ufs.configure.leapfrog_atm_wav.IN"
-  "ufs.configure.leapfrog_atm_wav_esmf.IN"
+  "ww3_shel.nml.IN"
   "post_itag_gfs")
 for file in "${ufs_templates[@]}"; do
   [[ -s "${file}" ]] && rm -f "${file}"

diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd