Add required CX fields for screen obs (#196)

* add field names needed for screen processing * screen cx test * individual Cx writer tests for vis, surf param a/b
MetOffice · Feb 21, 2024 · a57929a · a57929a
1 parent 96ccc7f
commit a57929a
Show file tree

Hide file tree

Showing 15 changed files with 218 additions and 4 deletions.
diff --git a/etc/global/cx/screen/Surface.nl b/etc/global/cx/screen/Surface.nl
@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=2,3,4,10,24,31,33,254,407,409,3209,3210,3236,3245,3247,9217,16222,20000,20001
+/
diff --git a/src/opsinputs/opsinputs_cxfields_mod.F90 b/src/opsinputs/opsinputs_cxfields_mod.F90
@@ -20,7 +20,7 @@ module opsinputs_cxfields_mod
 character(len=*), parameter, public :: opsinputs_cxfields_rh2 = "relative_humidity_2m"
 character(len=*), parameter, public :: opsinputs_cxfields_u10 = var_sfc_u10
 character(len=*), parameter, public :: opsinputs_cxfields_v10 = var_sfc_v10
-character(len=*), parameter, public :: opsinputs_cxfields_vis = opsinputs_cxfields_unknown
+character(len=*), parameter, public :: opsinputs_cxfields_vis = "visibility_1p5m"
 character(len=*), parameter, public :: opsinputs_cxfields_WAVE_HGHT = opsinputs_cxfields_unknown
 character(len=*), parameter, public :: opsinputs_cxfields_WIND_SPED = opsinputs_cxfields_unknown
 character(len=*), parameter, public :: opsinputs_cxfields_SeaHeight = opsinputs_cxfields_unknown
@@ -31,8 +31,8 @@ module opsinputs_cxfields_mod
 character(len=*), parameter, public :: opsinputs_cxfields_SnowAmount = opsinputs_cxfields_unknown
 character(len=*), parameter, public :: opsinputs_cxfields_qt2 = opsinputs_cxfields_unknown
 character(len=*), parameter, public :: opsinputs_cxfields_aerosol = opsinputs_cxfields_unknown
-character(len=*), parameter, public :: opsinputs_cxfields_PSurfParamA = opsinputs_cxfields_unknown
-character(len=*), parameter, public :: opsinputs_cxfields_PSurfParamB = opsinputs_cxfields_unknown
+character(len=*), parameter, public :: opsinputs_cxfields_PSurfParamA = "surf_param_a"
+character(len=*), parameter, public :: opsinputs_cxfields_PSurfParamB = "surf_param_b"
 character(len=*), parameter, public :: opsinputs_cxfields_LapseRate = opsinputs_cxfields_unknown
 character(len=*), parameter, public :: opsinputs_cxfields_CloudAmount = "total_cloud_amount"
 character(len=*), parameter, public :: opsinputs_cxfields_ConvCloudAmount = opsinputs_cxfields_unknown

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -388,6 +388,10 @@ ADD_WRITER_TEST(NAME     cxwriter_006_SurfaceCxField_v10
                 YAML     006_SurfaceCxField_v10.yaml
                 NAMELIST CxWriterNamelists_006_SurfaceCxField_v10/AMSUB.nl
                 DATA     006_SurfaceCxField_v10.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_008_SurfaceCxField_visibility_1p5m
+                YAML     008_SurfaceCxField_visibility_1p5m.yaml
+                NAMELIST CxWriterNamelists_008_SurfaceCxField_visibility_1p5m/Surface.nl
+                DATA     008_SurfaceCxField_visibility_1p5m.nc4 dummy.nc4)
 ADD_WRITER_TEST(NAME     cxwriter_013_SurfaceCxField_TskinSea
                 YAML     013_SurfaceCxField_TskinSea.yaml
                 NAMELIST CxWriterNamelists_013_SurfaceCxField_TskinSea/AMSUB.nl
@@ -400,6 +404,14 @@ ADD_WRITER_TEST(NAME     cxwriter_017_SurfaceCxField_SeaIce
                 YAML     017_SurfaceCxField_SeaIce.yaml
                 NAMELIST CxWriterNamelists_017_SurfaceCxField_SeaIce/AMSUB.nl
                 DATA     017_SurfaceCxField_SeaIce.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_022_SurfaceCxField_surf_param_a
+                YAML     022_SurfaceCxField_surf_param_a.yaml
+                NAMELIST CxWriterNamelists_022_SurfaceCxField_surf_param_a/Surface.nl
+                DATA     022_SurfaceCxField_surf_param_a.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_023_SurfaceCxField_surf_param_b
+                YAML     023_SurfaceCxField_surf_param_b.yaml
+                NAMELIST CxWriterNamelists_023_SurfaceCxField_surf_param_b/Surface.nl
+                DATA     023_SurfaceCxField_surf_param_b.nc4 dummy.nc4)
 ADD_WRITER_TEST(NAME     cxwriter_024_SurfaceCxField_CloudAmount
                 YAML     024_SurfaceCxField_CloudAmount.yaml
                 NAMELIST CxWriterNamelists_024_SurfaceCxField_CloudAmount/SatTCWV.nl
@@ -635,6 +647,10 @@ ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_surface
 ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_scatwind
                 YAML     cxwriter_globalnamelist_scatwind.yaml
                 DATA     cx_globalnamelist_scatwind.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_screen
+                YAML     cxwriter_globalnamelist_screen.yaml
+                NAMELIST ../../etc/global/cx/screen/Surface.nl
+                DATA     cx_globalnamelist_screen.nc4 dummy.nc4)
 
 # Tests the ukv namelist files in the etc directory
 

diff --git a/test/generate_unittest_netcdfs.py b/test/generate_unittest_netcdfs.py
@@ -827,9 +827,12 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
     output_1d_geoval_to_netcdf       ('relative_humidity_2m',       'testinput/004_SurfaceCxField_rh2.nc4')
     output_1d_geoval_to_netcdf       ('uwind_at_10m',               'testinput/005_SurfaceCxField_u10.nc4')
     output_1d_geoval_to_netcdf       ('vwind_at_10m',               'testinput/006_SurfaceCxField_v10.nc4')
+    output_1d_geoval_to_netcdf       ('visibility_1p5m',            'testinput/008_SurfaceCxField_visibility_1p5m.nc4')
     output_1d_geoval_to_netcdf       ('skin_temperature',           'testinput/013_SurfaceCxField_TskinSea.nc4')
     output_1d_geoval_to_netcdf       ('surface_pressure_at_mean_sea_level', 'testinput/016_SurfaceCxField_pmsl.nc4')
     output_1d_geoval_to_netcdf       ('ice_area_fraction',          'testinput/017_SurfaceCxField_SeaIce.nc4')
+    output_1d_geoval_to_netcdf       ('surf_param_a',               'testinput/022_SurfaceCxField_surf_param_a.nc4')
+    output_1d_geoval_to_netcdf       ('surf_param_b',               'testinput/023_SurfaceCxField_surf_param_b.nc4')
     output_1d_geoval_to_netcdf       ('total_cloud_amount', 'testinput/024_SurfaceCxField_CloudAmount.nc4')
     output_1d_geoval_to_netcdf       ('obukhov_length',             'testinput/056_SurfaceCxField_obukhov_length.nc4')
     output_1d_geoval_to_netcdf       ('friction_velocity_over_water', 'testinput/057_SurfaceCxField_friction_velocity_over_water.nc4')
@@ -1031,6 +1034,14 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
                              'testinput/cx_globalnamelist_scatwind.nc4')
     # Unrotate 10m winds
     output_full_cx_to_netcdf         (['uwind_at_10m', 'vwind_at_10m'],[], 'testinput/CxWriter_UnRotateWinds10M.nc4')
-
     output_1d_multi_level_simulated_var_to_netcdf('relativeHumidity', 'testinput/relative_humidity_Sonde.nc4')
     output_2d_geoval_for_multi_level_obs_to_netcdf('relative_humidity', 'testinput/002_UpperAirCxFieldForMultiLevelObs_relative_humidity.nc4')
+
+    # Screen
+    output_full_cx_to_netcdf(['surface_altitude', 'skin_temperature', 'ice_area_fraction', 'surface_pressure',
+                              'uwind_at_10m', 'vwind_at_10m', 'surface_temperature', 'relative_humidity_2m',
+                              'visibility_1p5m', 'total_cloud_amount', 'surface_pressure_at_mean_sea_level',
+                              'surf_param_a', 'surf_param_b'],
+                             ['potential_temperature', 'eastward_wind', 'northward_wind', 'specific_humidity',
+                              'air_pressure_levels', 'mass_content_of_cloud_liquid_water_in_atmosphere_layer'],
+                             'testinput/cx_globalnamelist_screen.nc4')
diff --git a/test/testinput/008_SurfaceCxField_visibility_1p5m.nc4 b/test/testinput/008_SurfaceCxField_visibility_1p5m.nc4
diff --git a/test/testinput/008_SurfaceCxField_visibility_1p5m.yaml b/test/testinput/008_SurfaceCxField_visibility_1p5m.yaml
@@ -0,0 +1,41 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: Surface
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/008_SurfaceCxField_visibility_1p5m.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    # Reject observation 3: we want to check if it is omitted from the Cx file, as expected.
+    - filter: Domain Check
+      where:
+      - variable:
+          name: MetaData/latitude
+        minvalue: 0.0
+    - filter: Cx Writer
+      namelist_directory: testinput/CxWriterNamelists_008_SurfaceCxField_visibility_1p5m
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["8"] # IndexCxvisibility_1p5m
+      expected_upper_air_variables: []
+      expected_main_table_columns:
+      - # batch 1
+        - ["7.10"]       # column 1
+        - ["**********"] # column 2 (the asterisks represent a missing float)
+        - ["7.40"]       # column 3
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0
diff --git a/test/testinput/022_SurfaceCxField_surf_param_a.nc4 b/test/testinput/022_SurfaceCxField_surf_param_a.nc4
diff --git a/test/testinput/022_SurfaceCxField_surf_param_a.yaml b/test/testinput/022_SurfaceCxField_surf_param_a.yaml
@@ -0,0 +1,41 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: Surface
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/022_SurfaceCxField_surf_param_a.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    # Reject observation 3: we want to check if it is omitted from the Cx file, as expected.
+    - filter: Domain Check
+      where:
+      - variable:
+          name: MetaData/latitude
+        minvalue: 0.0
+    - filter: Cx Writer
+      namelist_directory: testinput/CxWriterNamelists_022_SurfaceCxField_surf_param_a
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["22"] # IndexCxsurf_param_a
+      expected_upper_air_variables: []
+      expected_main_table_columns:
+      - # batch 1
+        - ["7.10"]       # column 1
+        - ["**********"] # column 2 (the asterisks represent a missing float)
+        - ["7.40"]       # column 3
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0
diff --git a/test/testinput/023_SurfaceCxField_surf_param_b.nc4 b/test/testinput/023_SurfaceCxField_surf_param_b.nc4
diff --git a/test/testinput/023_SurfaceCxField_surf_param_b.yaml b/test/testinput/023_SurfaceCxField_surf_param_b.yaml
@@ -0,0 +1,41 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: Surface
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/023_SurfaceCxField_surf_param_b.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    # Reject observation 3: we want to check if it is omitted from the Cx file, as expected.
+    - filter: Domain Check
+      where:
+      - variable:
+          name: MetaData/latitude
+        minvalue: 0.0
+    - filter: Cx Writer
+      namelist_directory: testinput/CxWriterNamelists_023_SurfaceCxField_surf_param_b
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["23"] # IndexCxsurf_param_b
+      expected_upper_air_variables: []
+      expected_main_table_columns:
+      - # batch 1
+        - ["7.10"]       # column 1
+        - ["**********"] # column 2 (the asterisks represent a missing float)
+        - ["7.40"]       # column 3
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0
diff --git a/test/testinput/CxWriterNamelists_008_SurfaceCxField_visibility_1p5m/Surface.nl b/test/testinput/CxWriterNamelists_008_SurfaceCxField_visibility_1p5m/Surface.nl
@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=3247
+/
diff --git a/test/testinput/CxWriterNamelists_022_SurfaceCxField_surf_param_a/Surface.nl b/test/testinput/CxWriterNamelists_022_SurfaceCxField_surf_param_a/Surface.nl
@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=20000
+/
diff --git a/test/testinput/CxWriterNamelists_023_SurfaceCxField_surf_param_b/Surface.nl b/test/testinput/CxWriterNamelists_023_SurfaceCxField_surf_param_b/Surface.nl
@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=20001
+/
diff --git a/test/testinput/cx_globalnamelist_screen.nc4 b/test/testinput/cx_globalnamelist_screen.nc4
diff --git a/test/testinput/cxwriter_globalnamelist_screen.yaml b/test/testinput/cxwriter_globalnamelist_screen.yaml
@@ -0,0 +1,52 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: Surface  # Screen observations are processed using the name Surface
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/cx_globalnamelist_screen.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    # Reject observation 3: we want to check if it is omitted from the Cx file, as expected.
+    - filter: Domain Check
+      where:
+      - variable:
+          name: MetaData/latitude
+        minvalue: 0.0
+    - filter: Cx Writer
+      namelist_directory: ../etc/global/cx/screen  # There is a custom location for the screen CX namelist
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["1", "2", "3", "4", "5", "6", "8", "13", "16", "17", "22", "23", "24"]
+      expected_upper_air_variables: ["1", "3", "4", "5", "11", "30"]
+      expected_main_table_columns:
+      - # observation 3 is rejected by the tests above hence only 3 (1,2,4) columns
+        - ["7.10","37.10","67.10","77.10","47.10","57.10",
+        "87.10","17.10","107.10","27.10","117.10","127.10",
+        "97.10","1.10","1.20","1.30","11.10","11.20","11.30",
+        "21.10","21.20","21.30","31.10","31.20","31.30",
+        "41.10","41.20","41.30","51.10","51.20","51.30"] # column 1 - 1st observation
+        - ["**********","**********","**********","**********","**********","**********","**********",
+        "**********","**********","**********","**********","**********","**********","2.10","**********",
+        "2.30","12.10","**********","12.30","22.10","**********","22.30","32.10","**********","32.30",
+        "42.10","**********","42.30","52.10","**********","52.30"] # column 2 - 2nd observation
+        - ["7.40","37.40","67.40","77.40","47.40","57.40","87.40",
+        "17.40","107.40","27.40","117.40","127.40","97.40","4.10",
+        "4.20","4.30","14.10","14.20","14.30","24.10","24.20",
+        "24.30","34.10","34.20","34.30","44.10","44.20","44.30",
+        "54.10","54.20","54.30"] # column 3 - observation 4
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0