Add ctests for running GETKF in split mode and move reference files into repo

rrfs-test/CMakeLists.txt

@@ -13,10 +13,12 @@ add_to_dictionary(rrfs_fv3jedi_tests "rrfs_fv3jedi_letkf_2022052619"   "fv3jedi_
 
 # MPAS-JEDI tests 
 set(rrfs_mpasjedi_tests)
-add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_Ens3Dvar" "mpasjedi_variational.x")
-add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_letkf"    "mpasjedi_enkf.x")
-add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_getkf"    "mpasjedi_enkf.x")
-add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_bumploc"  "mpasjedi_error_covariance_toolbox.x")
+add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_Ens3Dvar"       "mpasjedi_variational.x")
+add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_letkf"          "mpasjedi_enkf.x")
+add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_getkf"          "mpasjedi_enkf.x")
+add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_getkf_observer" "mpasjedi_enkf.x")
+add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_getkf_solver"   "mpasjedi_enkf.x")
+add_to_dictionary(rrfs_mpasjedi_tests "rrfs_mpasjedi_2024052700_bumploc"        "mpasjedi_error_covariance_toolbox.x")
 
 message(STATUS "Use test data from rrfs-test-data repository")
 set (rrfs-test_data_local "${CMAKE_SOURCE_DIR}/rrfs-test-data/")
@@ -37,22 +39,23 @@ if(FV3_DYCORE)
       endif()   
 
       get_all_keys(rrfs_fv3jedi_tests caselist)
+      set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-rrfs_fv3jedi_2022052619")
+      set(src_casedir "${rrfs-test_data_local}/rrfs-data_fv3jedi_2022052619")
+      if (NOT EXISTS "${casedir}")
+        file(MAKE_DIRECTORY ${casedir})
+      endif()
+      file(CREATE_LINK ${src_casedir}/DataFix ${casedir}/DataFix SYMBOLIC)
+      file(CREATE_LINK ${src_casedir}/Data_static ${casedir}/Data_static SYMBOLIC)
+      file(CREATE_LINK ${src_casedir}/INPUT ${casedir}/INPUT SYMBOLIC)
+      file(CREATE_LINK ${src_casedir}/Data ${casedir}/Data SYMBOLIC)
+      file(CREATE_LINK ${CMAKE_SOURCE_DIR}/rrfs-test/testoutput ${casedir}/testoutput SYMBOLIC)
       foreach(case ${caselist})
          get_from_dictionary(rrfs_fv3jedi_tests case exe)
-         set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-${case}")
-         set(src_casedir "${rrfs-test_data_local}/rrfs-data_fv3jedi_2022052619")
-         if (NOT EXISTS "${casedir}")
-           file(MAKE_DIRECTORY ${casedir})
-         endif()
-         file(CREATE_LINK ${src_casedir}/DataFix ${casedir}/DataFix SYMBOLIC)
-         file(CREATE_LINK ${src_casedir}/Data_static ${casedir}/Data_static SYMBOLIC)
-         file(CREATE_LINK ${src_casedir}/INPUT ${casedir}/INPUT SYMBOLIC)
-         file(CREATE_LINK ${src_casedir}/Data ${casedir}/Data SYMBOLIC)
          file(COPY ${src_yaml}/${case}.yaml DESTINATION ${casedir} )
          set(target_test ${case})
          ecbuild_add_test( TARGET ${target_test}
                         MPI      80
-                        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${target_test}
+                        WORKING_DIRECTORY ${casedir}
                         ARGS     ${target_test}.yaml
                         COMMAND  ${exe} )
       endforeach()
@@ -75,28 +78,29 @@ if(FV3_DYCORE)
       set(MPI_ARGS "${MPI_ARGS} --ntasks-per-node=4")
 
       get_all_keys(rrfs_mpasjedi_tests caselist)
+      set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-rrfs_mpasjedi_2024052700")
+      set(src_casedir "${rrfs-test_data_local}/rrfs-data_mpasjedi_2024052700")
+      if (NOT EXISTS "${casedir}")
+        file(MAKE_DIRECTORY ${casedir})
+      endif()
+      file(CREATE_LINK ${src_casedir}/data ${casedir}/data SYMBOLIC)
+      file(CREATE_LINK ${src_casedir}/graphinfo ${casedir}/graphinfo SYMBOLIC)
+      file(CREATE_LINK ${src_casedir}/stream_list ${casedir}/stream_list SYMBOLIC)
+      file(CREATE_LINK ${CMAKE_SOURCE_DIR}/rrfs-test/testoutput ${casedir}/testoutput SYMBOLIC)
+      file(COPY ${src_casedir}/streams.atmosphere DESTINATION ${casedir})
+      file(COPY ${src_casedir}/namelist.atmosphere DESTINATION ${casedir})
+      file(COPY ${src_casedir}/geovars.yaml DESTINATION ${casedir})
+      file(COPY ${src_casedir}/keptvars.yaml DESTINATION ${casedir})
+      file(COPY ${src_casedir}/obsop_name_map.yaml DESTINATION ${casedir})
+      file(GLOB bl_FILES "${src_casedir}/*.*BL")
+      file(COPY ${bl_FILES} DESTINATION ${casedir})
       foreach(case ${caselist})
 	 get_from_dictionary(rrfs_mpasjedi_tests case exe)
-         set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-${case}")
-         set(src_casedir "${rrfs-test_data_local}/rrfs-data_mpasjedi_2024052700")
-         if (NOT EXISTS "${casedir}")
-           file(MAKE_DIRECTORY ${casedir})
-         endif()
-	 file(CREATE_LINK ${src_casedir}/data ${casedir}/data SYMBOLIC)
-	 file(CREATE_LINK ${src_casedir}/graphinfo ${casedir}/graphinfo SYMBOLIC)
-	 file(CREATE_LINK ${src_casedir}/stream_list ${casedir}/stream_list SYMBOLIC)
-	 file(COPY ${src_casedir}/streams.atmosphere DESTINATION ${casedir})
-	 file(COPY ${src_casedir}/namelist.atmosphere DESTINATION ${casedir})
-	 file(COPY ${src_casedir}/geovars.yaml DESTINATION ${casedir})
-	 file(COPY ${src_casedir}/keptvars.yaml DESTINATION ${casedir})
-	 file(COPY ${src_casedir}/obsop_name_map.yaml DESTINATION ${casedir})
-	 file(GLOB bl_FILES "${src_casedir}/*.*BL")
-	 file(COPY ${bl_FILES} DESTINATION ${casedir})
-	 file(COPY ${src_yaml}/${case}.yaml DESTINATION ${casedir})
+         file(COPY ${src_yaml}/${case}.yaml DESTINATION ${casedir})
          set(target_test ${case})
          ecbuild_add_test( TARGET ${target_test}
                         MPI      120
-                        WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${target_test}
+                        WORKING_DIRECTORY ${casedir}
                         ARGS     ${target_test}.yaml
                         COMMAND  ${exe} )
       endforeach()

rrfs-test/testinput/rrfs_fv3jedi_hyb_2022052619.yaml

@@ -225,7 +225,7 @@ output:
   prefix: hybens3dvar-fv3_lam-C775 #Ens3dvar-fv3_lam-C775
   frequency: PT1H
 test:
-  reference filename: Data/testinput/rrfs-fv3jedi-hyb.ref
+  reference filename: testoutput/rrfs-fv3jedi-hyb.ref
   test output filename: rrfs-fv3jedi-hyb.out
   float relative tolerance: 1.0e-3
   float absolute tolerance: 1.0e-6

rrfs-test/testinput/rrfs_fv3jedi_letkf_2022052619.yaml

@@ -162,7 +162,7 @@ output: # for outputting mean posterior
   prefix: letkf-meanposterior-fv3_lam-C775
 
 test:
-  reference filename: Data/testinput/rrfs-fv3jedi-letkf.ref
+  reference filename: testoutput/rrfs-fv3jedi-letkf.ref
   test output filename: ./rrfs-fv3jedi-letkf.out
   float relative tolerance: 1.0e-3
   float absolute tolerance: 1.0e-6

rrfs-test/testinput/rrfs_mpasjedi_2024052700_Ens3Dvar.yaml

@@ -125,7 +125,7 @@ cost function:
            defer to post: true
 
 test:
-  reference filename: data/ref/rrfs-mpasjedi-ens3dvar.ref
+  reference filename: testoutput/rrfs-mpasjedi-ens3dvar.ref
   test output filename: ./rrfs-mpasjedi-ens3dvar.out
   float relative tolerance: 1.0e-3
   float absolute tolerance: 1.0e-6

rrfs-test/testinput/rrfs_mpasjedi_2024052700_bumploc.yaml

@@ -45,7 +45,7 @@ background error:
           stream name: control
 
 test:
-  reference filename: data/ref/rrfs-mpasjedi-bump.ref
+  reference filename: testoutput/rrfs-mpasjedi-bump.ref
   test output filename: ./rrfs-mpasjedi-bump.out
   float relative tolerance: 1.0e-5
   float absolute tolerance: 1.0e-6

rrfs-test/testinput/rrfs_mpasjedi_2024052700_getkf.yaml

@@ -78,15 +78,15 @@ local ensemble DA:
     mult: 1.1
 
 output mean prior:
-  filename: ./bkg.$Y-$M-$D_$h.$m.$s.nc
+  filename: ./bkg_getkf.$Y-$M-$D_$h.$m.$s.nc
   stream name: background
 
 output: # for outputting mean posterior
-  filename: ./ana.$Y-$M-$D_$h.$m.$s.nc
+  filename: ./ana_getkf.$Y-$M-$D_$h.$m.$s.nc
   stream name: analysis 
 
 test:
-  reference filename: data/ref/rrfs-mpasjedi-getkf.ref
+  reference filename: testoutput/rrfs-mpasjedi-getkf.ref
   test output filename: ./rrfs-mpasjedi-getkf.out
   float relative tolerance: 1.0e-3
   float absolute tolerance: 1.0e-6

rrfs-test/testinput/rrfs_mpasjedi_2024052700_getkf_observer.yaml

@@ -0,0 +1,81 @@
+_member: &memberConfig
+  date: &analysisDate '2024-05-27T00:00:00Z'
+  state variables: [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal,theta,rho,u,qv,pressure,landmask,xice,snowc,skintemp,ivgtyp,isltyp,snowh,vegfra,u10,v10,lai,smois,tslb,pressure_p,qc,qi,qg,qr,qs,cldfrac]
+  stream name: ensemble
+
+geometry:
+  nml_file: ./namelist.atmosphere
+  streams_file: ./streams.atmosphere
+  deallocate non-da fields: true
+  interpolation type: unstructured
+  iterator dimension: 2
+
+increment variables: [temperature, spechum, uReconstructZonal, uReconstructMeridional, surface_pressure]
+
+background: 
+    members from template:
+      template:
+        <<: *memberConfig
+        filename: ./data/ens/mem%iMember%/restart.2024-05-27_00.00.00.nc
+      pattern: %iMember%
+      start: 1
+      zero padding: 2
+      nmembers: 30
+
+observations:
+     observers:
+     - obs space:
+         name: sonde
+         distribution:
+           name: RoundRobin
+         obsdatain:
+           engine:
+             type: H5File
+             obsfile: data/obs/rap.t00z.singleobs.ADPUPA.tm00.nc4
+         obsdataout:
+           engine:
+             type: H5File
+             obsfile: ./adpupa_getkf_hofx.nc4
+         simulated variables: [airTemperature]
+         observed variables: [airTemperature]
+       obs operator:
+         name: VertInterp
+       obs error:
+         covariance model: diagonal
+       obs localizations:
+         - localization method: Horizontal Gaspari-Cohn
+           lengthscale: 300e3 # orig
+       obs filters:
+         - filter: Perform Action
+           filter variables:
+           - name: airTemperature
+           action:
+             name: assign error
+             error parameter: 1.2  # K
+           defer to post: true
+
+driver:
+  run as observer only: true
+  update obs config with geometry info: false
+
+time window:
+       begin: 2024-05-26T23:00:00Z
+       length: PT2H
+
+local ensemble DA:
+  solver: GETKF
+  use linear observer: true
+  vertical localization: # current settings use 12 modulated members
+    fraction of retained variance: 0.850
+    lengthscale: 0.55
+    lengthscale units: logp
+  inflation:
+    rtps: 0.95
+    rtpp: 0.6
+    mult: 1.1
+
+test:
+  reference filename: testoutput/rrfs-mpasjedi-getkf-observer.ref
+  test output filename: ./rrfs-mpasjedi-getkf-observer.out
+  float relative tolerance: 1.0e-3
+  float absolute tolerance: 1.0e-6

rrfs-test/testinput/rrfs_mpasjedi_2024052700_getkf_solver.yaml

@@ -0,0 +1,93 @@
+_member: &memberConfig
+  date: &analysisDate '2024-05-27T00:00:00Z'
+  state variables: [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal,theta,rho,u,qv,pressure,landmask,xice,snowc,skintemp,ivgtyp,isltyp,snowh,vegfra,u10,v10,lai,smois,tslb,pressure_p,qc,qi,qg,qr,qs,cldfrac]
+  stream name: ensemble
+
+geometry:
+  nml_file: ./namelist.atmosphere
+  streams_file: ./streams.atmosphere
+  deallocate non-da fields: true
+  interpolation type: unstructured
+  iterator dimension: 2
+
+increment variables: [temperature, spechum, uReconstructZonal, uReconstructMeridional, surface_pressure]
+
+background: 
+    members from template:
+      template:
+        <<: *memberConfig
+        filename: ./data/ens/mem%iMember%/restart.2024-05-27_00.00.00.nc
+      pattern: %iMember%
+      start: 1
+      zero padding: 2
+      nmembers: 30
+
+observations:
+     observers:
+     - obs space:
+         name: sonde
+         distribution:
+           name: Halo
+           halo size: 1000e3
+         obsdatain:
+           engine:
+             type: H5File
+             obsfile: ./adpupa_getkf_hofx.nc4
+         obsdataout:
+           engine:
+             type: H5File
+             obsfile: ./adpupa_getkf_solver.nc4
+         simulated variables: [airTemperature]
+         observed variables: [airTemperature]
+       obs operator:
+         name: VertInterp
+       obs error:
+         covariance model: diagonal
+       obs localizations:
+         - localization method: Horizontal Gaspari-Cohn
+           lengthscale: 300e3 # orig
+       obs filters:
+         - filter: Perform Action
+           filter variables:
+           - name: airTemperature
+           action:
+             name: assign error
+             error parameter: 1.2  # K
+           defer to post: true
+
+driver:
+  read HX from disk: true
+  save posterior ensemble: false
+  save prior mean: true
+  save posterior mean: true
+  do posterior observer: true
+
+time window:
+       begin: 2024-05-26T23:00:00Z
+       length: PT2H
+
+local ensemble DA:
+  solver: GETKF
+  use linear observer: true
+  vertical localization: # current settings use 12 modulated members
+    fraction of retained variance: 0.850
+    lengthscale: 0.55
+    lengthscale units: logp
+  inflation:
+    rtps: 0.95
+    rtpp: 0.6
+    mult: 1.1
+
+output mean prior:
+  filename: ./bkg_getkf_split.$Y-$M-$D_$h.$m.$s.nc
+  stream name: background
+
+output: # for outputting mean posterior
+  filename: ./ana_getkf_split.$Y-$M-$D_$h.$m.$s.nc
+  stream name: analysis 
+
+test:
+  reference filename: testoutput/rrfs-mpasjedi-getkf-solver.ref
+  test output filename: ./rrfs-mpasjedi-getkf-solver.out
+  float relative tolerance: 1.0e-3
+  float absolute tolerance: 1.0e-6

rrfs-test/testinput/rrfs_mpasjedi_2024052700_letkf.yaml

@@ -79,15 +79,15 @@ local ensemble DA:
     mult: 1.1
 
 output mean prior:
-  filename: ./bkg.$Y-$M-$D_$h.$m.$s.nc
+  filename: ./bkg_letkf.$Y-$M-$D_$h.$m.$s.nc
   stream name: background
 
 output: # for outputting mean posterior
-  filename: ./ana.$Y-$M-$D_$h.$m.$s.nc
+  filename: ./ana_letkf.$Y-$M-$D_$h.$m.$s.nc
   stream name: analysis 
 
 test:
-  reference filename: data/ref/rrfs-mpasjedi-letkf.ref
+  reference filename: testoutput/rrfs-mpasjedi-letkf.ref
   test output filename: ./rrfs-mpasjedi-letkf.out
   float relative tolerance: 1.0e-3
   float absolute tolerance: 1.0e-6