Stage atmospheric backgrounds and UFS cubed-sphere history files (#1236)

This PR changes the GDASApp so that backgrounds for the atmospheric analyses are staged as UFS cubed-sphere history files rather than FMS restarts. jjob tests and test data are also updated to accommodate these changes. The motivation being that FMS restarts include many data we don't need for the analyses and thus take up too much disk space/bandwidth. This is a companion PR to Global Workflow [#2792](NOAA-EMC/global-workflow#2792) and JCB-GDAS [NOAA-EMC#19](NOAA-EMC/jcb-gdas#19).
hafs-community · Aug 6, 2024 · f3fa26d · f3fa26d
1 parent cc4568d
commit f3fa26d
Show file tree

Hide file tree

Showing 8 changed files with 86 additions and 82 deletions.
diff --git a/parm/atm/jcb-base.yaml.j2 b/parm/atm/jcb-base.yaml.j2
@@ -61,6 +61,7 @@ atmosphere_forecast_timestep: "{{ BKG_TSTEP }}"
 # Write final increment on Guassian grid in variational
 atmosphere_final_increment_prefix: "./anl/atminc."
 
+atmosphere_variational_history_prefix: "{{GPREFIX}}"
 
 # Observation things
 # ------------------
@@ -97,7 +98,7 @@ atmosphere_obsbiascovout_suffix: ".satbias_cov.nc"
 # -------------------------
 local_ensemble_da_solver: GETKF
 
-increment_variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr]
+increment_variables: [ua,va,delz,delp,t,ps,sphum,ice_wat,liq_wat,o3mr]
 
 # Veritcal localization for GETKF
 vl_fraction_of_retained_variance: 0.750
@@ -118,3 +119,5 @@ driver_save_posterior_ensemble_increments: true
 # Diagnostics
 atmosphere_ensemble_increment_prefix: "./anl/mem%{member}%/atminc."
 atmosphere_posterior_output_gaussian: "./mem%{member}%/atmanl."
+
+atmosphere_ensemble_history_prefix: "enkf{{GPREFIX}}"
diff --git a/parm/io/fv3jedi_fieldmetadata_history.yaml b/parm/io/fv3jedi_fieldmetadata_history.yaml
@@ -47,8 +47,9 @@ field metadata:
 
 - long name: surface_pressure
   io name: pressfc
+  io file: atmosphere
 
-- long name: sfc_geopotential_height_times_grav
+- long name: surface_geopotential_height
   io name: hgtsfc
 
 - long name: u_component_of_native_D_grid_wind
@@ -82,7 +83,7 @@ field metadata:
 - long name: vfrac
   io name: veg
 
-- long name: totalSnowDepth
+- long name: totalSnowDepthMeters
   io name: snod
 
 - long name: surface_eastward_wind

diff --git a/parm/jcb-gdas b/parm/jcb-gdas
diff --git a/test/atm/global-workflow/3dvar.ref b/test/atm/global-workflow/3dvar.ref
@@ -1,21 +1,21 @@
 CostJb   : Nonlinear Jb = 0.0000000000000000e+00
 CostJo   : Nonlinear Jo(AMSUA N19) = 3.9490015258335385e+04, nobs = 73667, Jo/n = 5.3606112992704180e-01, err = 6.2697697603632934e+00
-CostJo   : Nonlinear Jo(sondes) = 1.0738490911289642e+04, nobs = 4255, Jo/n = 2.5237346442513848e+00, err = 1.0981085630539836e+01
-CostFunction: Nonlinear J = 5.0228506169625027e+04
-DRPCGMinimizer: reduction in residual norm = 9.6305250910880502e-01
+CostJo   : Nonlinear Jo(sondes) = 1.0738590249678713e+04, nobs = 4255, Jo/n = 2.5237579905237868e+00, err = 1.0981123140127320e+01
+CostFunction: Nonlinear J = 5.0228605508014094e+04
+DRPCGMinimizer: reduction in residual norm = 9.6305250910880724e-01
 CostFunction::addIncrement: Analysis: 
 ----------------------------------------------------------------------------------------------------
 State print | number of fields = 22 | cube sphere face size: C48
 eastward_wind                                | Min:-5.5495644220059390e+01 Max:+8.4934651204487366e+01 RMS:+1.6388250288534095e+01
 northward_wind                               | Min:-7.3712421229093422e+01 Max:+7.6984825502186652e+01 RMS:+8.1424144891570869e+00
 air_temperature                              | Min:+1.7482158077318448e+02 Max:+3.1467235233685437e+02 RMS:+2.4978424883092669e+02
 air_pressure_thickness                       | Min:+6.0599999874109178e-01 Max:+1.7760098554198755e+03 RMS:+9.5680601590204810e+02
-surface_pressure                             | Min:+5.3297486514404744e+04 Max:+1.0397927292902800e+05 RMS:+9.8896232762674656e+04
+surface_pressure                             | Min:+5.3298485514404747e+04 Max:+1.0398027192902798e+05 RMS:+9.8897229693652524e+04
 specific_humidity                            | Min:+0.0000000000000000e+00 Max:+2.0041369334954085e-02 RMS:+4.7792636816418417e-03
 cloud_liquid_ice                             | Min:+0.0000000000000000e+00 Max:+1.2024392024662985e-03 RMS:+1.4396968866569995e-05
 cloud_liquid_water                           | Min:+0.0000000000000000e+00 Max:+1.5935686618789048e-03 RMS:+4.1931345590543333e-05
 ozone_mass_mixing_ratio                      | Min:+1.1998327271379204e-08 Max:+1.7808431790670831e-05 RMS:+4.4947917149861064e-06
-sfc_geopotential_height_times_grav           | Min:-2.6495993813133236e+02 Max:+5.1678555530273436e+04 RMS:+6.1813840931281156e+03
+surface_geopotential_height                  | Min:-2.7018394470214844e+01 Max:+5.2697460937500000e+03 RMS:+6.3032575784066080e+02
 slmsk                                        | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+7.3450427349184144e-01
 sheleg                                       | Min:+0.0000000000000000e+00 Max:+1.8650088674622128e+02 RMS:+2.2457791153397761e+01
 sea_surface_temperature                      | Min:+2.1068090851678403e+02 Max:+3.3125178729099270e+02 RMS:+2.8949442340676961e+02
@@ -24,7 +24,7 @@ stype                                        | Min:+0.0000000000000000e+00 Max:+
 vfrac                                        | Min:+0.0000000000000000e+00 Max:+9.7215068788150627e-01 RMS:+2.2249321391025326e-01
 stc                                          | Min:+2.1550390656839755e+02 Max:+3.2609885436959308e+02 RMS:+2.8945169639231835e+02
 soilMoistureVolumetric                       | Min:+1.1321643448483208e-02 Max:+1.0000000000000000e+00 RMS:+8.3306585674111977e-01
-totalSnowDepth                               | Min:+0.0000000000000000e+00 Max:+1.7098998490425442e+03 RMS:+1.8885640927654018e+02
+totalSnowDepthMeters                         | Min:+0.0000000000000000e+00 Max:+1.7098998490425443e+00 RMS:+1.8885640927654021e-01
 surface_eastward_wind                        | Min:-1.6966302087131233e+01 Max:+2.1273647117120184e+01 RMS:+5.4892133439475188e+00
 surface_northward_wind                       | Min:-1.7367330522224339e+01 Max:+2.2091894088184446e+01 RMS:+4.4723535847538756e+00
 f10m                                         | Min:+9.4613586695410434e-01 Max:+1.0695628272426509e+00 RMS:+9.9526981313535046e-01
@@ -44,23 +44,23 @@ sensorScanAngle_order_2:  Min= -5.1097259521664853,  Max= 2.8632600308292808,  N
 
 
 CostJb   : Nonlinear Jb = 0.0000003247124839
-CostJo   : Nonlinear Jo(AMSUA N19) = 39487.6033905211515957, nobs = 73667, Jo/n = 0.5360283897881162, err = 6.2697697603632934
-CostJo   : Nonlinear Jo(sondes) = 10687.3021478507798747, nobs = 4255, Jo/n = 2.5117043825736265, err = 10.9810856305398357
-CostFunction: Nonlinear J = 50174.9055386966429069
-DRPCGMinimizer: reduction in residual norm = 1.0067413227975508
+CostJo   : Nonlinear Jo(AMSUA N19) = 39487.6033905210933881, nobs = 73667, Jo/n = 0.5360283897881154, err = 6.2697697603632934
+CostJo   : Nonlinear Jo(sondes) = 10687.4014846169593511, nobs = 4255, Jo/n = 2.5117277284646202, err = 10.9811231401273197
+CostFunction: Nonlinear J = 50175.0048754627641756
+DRPCGMinimizer: reduction in residual norm = 1.0067413227974593
 CostFunction::addIncrement: Analysis: 
 ----------------------------------------------------------------------------------------------------
 State print | number of fields = 22 | cube sphere face size: C48
 eastward_wind                                | Min:-5.5495644220059390e+01 Max:+8.4934651204487366e+01 RMS:+1.6388250288542654e+01
 northward_wind                               | Min:-7.3712421229093422e+01 Max:+7.6984825502186652e+01 RMS:+8.1424144891586234e+00
 air_temperature                              | Min:+1.7482158077318232e+02 Max:+3.1467235233685437e+02 RMS:+2.4978424883219918e+02
 air_pressure_thickness                       | Min:+6.0599999874109178e-01 Max:+1.7760098554198755e+03 RMS:+9.5680601590204810e+02
-surface_pressure                             | Min:+5.3297486514404744e+04 Max:+1.0397927292902800e+05 RMS:+9.8896232762674656e+04
+surface_pressure                             | Min:+5.3298485514404747e+04 Max:+1.0398027192902798e+05 RMS:+9.8897229693652524e+04
 specific_humidity                            | Min:+0.0000000000000000e+00 Max:+2.0041369334954085e-02 RMS:+4.7792637405692090e-03
 cloud_liquid_ice                             | Min:+0.0000000000000000e+00 Max:+1.2024392024662985e-03 RMS:+1.4396968866569995e-05
 cloud_liquid_water                           | Min:+0.0000000000000000e+00 Max:+1.5935686618789048e-03 RMS:+4.1931345590543333e-05
 ozone_mass_mixing_ratio                      | Min:+1.1998327271379204e-08 Max:+1.7808431790670831e-05 RMS:+4.4947917149861064e-06
-sfc_geopotential_height_times_grav           | Min:-2.6495993813133236e+02 Max:+5.1678555530273436e+04 RMS:+6.1813840931281156e+03
+surface_geopotential_height                  | Min:-2.7018394470214844e+01 Max:+5.2697460937500000e+03 RMS:+6.3032575784066080e+02
 slmsk                                        | Min:+0.0000000000000000e+00 Max:+2.0000000000000000e+00 RMS:+7.3450427349184144e-01
 sheleg                                       | Min:+0.0000000000000000e+00 Max:+1.8650088674622128e+02 RMS:+2.2457791153397761e+01
 sea_surface_temperature                      | Min:+2.1068090851678403e+02 Max:+3.3125178729099270e+02 RMS:+2.8949442340676961e+02
@@ -69,7 +69,7 @@ stype                                        | Min:+0.0000000000000000e+00 Max:+
 vfrac                                        | Min:+0.0000000000000000e+00 Max:+9.7215068788150627e-01 RMS:+2.2249321391025326e-01
 stc                                          | Min:+2.1550390656839755e+02 Max:+3.2609885436959308e+02 RMS:+2.8945169639231835e+02
 soilMoistureVolumetric                       | Min:+1.1321643448483208e-02 Max:+1.0000000000000000e+00 RMS:+8.3306585674111977e-01
-totalSnowDepth                               | Min:+0.0000000000000000e+00 Max:+1.7098998490425442e+03 RMS:+1.8885640927654018e+02
+totalSnowDepthMeters                         | Min:+0.0000000000000000e+00 Max:+1.7098998490425443e+00 RMS:+1.8885640927654021e-01
 surface_eastward_wind                        | Min:-1.6966302087131233e+01 Max:+2.1273647117120184e+01 RMS:+5.4892133439475188e+00
 surface_northward_wind                       | Min:-1.7367330522224339e+01 Max:+2.2091894088184446e+01 RMS:+4.4723535847538756e+00
 f10m                                         | Min:+9.4613586695410434e-01 Max:+1.0695628272426509e+00 RMS:+9.9526981313535046e-01

diff --git a/test/atm/global-workflow/jjob_ens_init.sh b/test/atm/global-workflow/jjob_ens_init.sh
@@ -95,17 +95,19 @@ for imem in $(seq 1 $NMEM_ENS); do
 	COM_ATMOS_RESTART_PREV_ENS:COM_ATMOS_RESTART_TMPL
     COM_ATMOS_RESTART_PREV_DIRNAME_ENS=$(dirname $COM_ATMOS_RESTART_PREV_ENS)
 
-    source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos
-    target=$COM_ATMOS_RESTART_PREV_DIRNAME_ENS
-    mkdir -p $target
-    rm -rf $target/restart
-    ln -fs $source/restart $target/
-
     source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos/history
     target=$COM_ATMOS_HISTORY_PREV_ENS
     mkdir -p $target
     rm -rf $target/enkfgdas.t${gcyc}z.atmf006.nc
     ln -fs $source/enkfgdas.t${gcyc}z.atmf006.nc $target/
+
+    source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos/history
+    target=$COM_ATMOS_HISTORY_PREV_ENS
+    flist=("cubed_sphere_grid_atmf006.nc" "cubed_sphere_grid_sfcf006.nc")
+    for file in "${flist[@]}"; do
+	rm -rf $target/enkf${gprefix}.${file}
+        ln -fs $source/enkf${gprefix}.${file} $target/
+    done
 done
 
 # Execute j-job

diff --git a/test/atm/global-workflow/jjob_var_init.sh b/test/atm/global-workflow/jjob_var_init.sh
@@ -87,24 +87,20 @@ for file in $flist; do
    ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$gprefix.${file} $COM_ATMOS_ANALYSIS_PREV/$gprefix.${file}
 done
 
-# Link atmospheric background on gaussian grid
+# Link atmospheric history on gaussian grid
 dpath=gdas.$gPDY/$gcyc/model/atmos/history
 mkdir -p $COM_ATMOS_HISTORY_PREV
 flist="atmf006.nc"
 for file in $flist; do
    ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/${gprefix}.${file}
 done
 
-# Link atmospheric bacgkround on tiles
-dpath=gdas.$gPDY/$gcyc/model/atmos
-COM_ATMOS_RESTART_PREV_DIRNAME=$(dirname $COM_ATMOS_RESTART_PREV)
-mkdir -p $COM_ATMOS_RESTART_PREV_DIRNAME
-flist="restart"
-for file in $flist; do
-   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/$file $COM_ATMOS_RESTART_PREV_DIRNAME/$file
+# Link atmospheric histories on native cubed-sphere grid
+flist=("cubed_sphere_grid_atmf006.nc" "cubed_sphere_grid_sfcf006.nc")
+for file in "${flist[@]}"; do
+   ln -fs $GDASAPP_TESTDATA/lowres/$dpath/${gprefix}.${file} $COM_ATMOS_HISTORY_PREV/${gprefix}.${file}
 done
 
-
 # Link member atmospheric background on tiles and atmf006
 dpath=enkfgdas.$gPDY/$gcyc
 for imem in $(seq 1 $NMEM_ENS); do
@@ -115,17 +111,19 @@ for imem in $(seq 1 $NMEM_ENS); do
 	COM_ATMOS_RESTART_PREV_ENS:COM_ATMOS_RESTART_TMPL
     COM_ATMOS_RESTART_PREV_DIRNAME_ENS=$(dirname $COM_ATMOS_RESTART_PREV_ENS)
 
-    source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos
-    target=$COM_ATMOS_RESTART_PREV_DIRNAME_ENS
-    mkdir -p $target
-    rm -rf $target/restart
-    ln -fs $source/restart $target/
-
     source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos/history
     target=$COM_ATMOS_HISTORY_PREV_ENS
     mkdir -p $target
     rm -rf $target/enkfgdas.t${gcyc}z.atmf006.nc
     ln -fs $source/enkfgdas.t${gcyc}z.atmf006.nc $target/
+
+    source=$GDASAPP_TESTDATA/lowres/$dpath/$memchar/model/atmos/history
+    target=$COM_ATMOS_HISTORY_PREV_ENS
+    flist=("cubed_sphere_grid_atmf006.nc" "cubed_sphere_grid_sfcf006.nc")
+    for file in "${flist[@]}"; do
+        rm -rf $target/enkf${gprefix}.${file}
+        ln -fs $source/enkf${gprefix}.${file} $target/
+    done
 done
+8 −10		algorithm/atmosphere/fv3jedi_fv3inc_lgetkf.yaml.j2
+7 −9		algorithm/atmosphere/fv3jedi_fv3inc_variational.yaml.j2
+1 −1		model/atmosphere/atmosphere_3dvar_outer_loop_1.yaml.j2
+1 −1		model/atmosphere/atmosphere_3dvar_outer_loop_2.yaml.j2
+8 −10		model/atmosphere/atmosphere_background.yaml.j2
+8 −9		model/atmosphere/atmosphere_background_ensemble.yaml.j2
+4 −7		model/atmosphere/atmosphere_background_error_hybrid_gsibec_bump.yaml.j2
+1 −1		model/atmosphere/atmosphere_geometry_background.yaml.j2