Add policies toward coupling and absorb policies into GDAS/atmosphere

src/jcb/__init__.py

@@ -1,24 +1,33 @@
 # --------------------------------------------------------------------------------------------------
 
 
-# Set the version
-__version__ = '0.0.1'
-
-
-# --------------------------------------------------------------------------------------------------
-
+import os
 
-# Elevate all functions and classed to being useable as jcb.<function>
-# from .driver import main as jcb
 from .observation_chronicle.observation_chronicle import ObservationChronicle
 from .observation_chronicle.satellite_chronicle import process_satellite_chronicles
 from .renderer import render as render
 from .renderer import Renderer as Renderer
 from .utilities.config_parsing import datetime_from_conf, duration_from_conf
 from .utilities.parse_channels import parse_channels, parse_channels_set
+from .utilities.testing import get_apps, apps_directory_to_dictionary, render_app_with_test_config
 from .utilities.trapping import abort, abort_if
 
-# Define the
+
+# --------------------------------------------------------------------------------------------------
+
+
+# JCB Version
+__version__ = '0.0.1'
+
+
+def version():
+    return __version__
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+# Define the visible functions and classes
 __all__ = [
     'Renderer',
     'render',
@@ -30,7 +39,21 @@
     'parse_channels_set',
     'abort_if',
     'abort',
+    'version',
+    '__version__',
+    'get_jcb_path',
+    'get_apps',
+    'apps_directory_to_dictionary',
+    'render_app_with_test_config',
 ]
 
 
 # --------------------------------------------------------------------------------------------------
+
+
+# Function that returns the path of the this file
+def get_jcb_path():
+    return os.path.dirname(os.path.realpath(__file__))
+
+
+# --------------------------------------------------------------------------------------------------

src/jcb/driver.py

@@ -8,7 +8,7 @@
 
 
 @click.group()
-@click.version_option(version=jcb.__version__, prog_name="Jedi Configuration Builder (jcb)")
+@click.version_option(version=jcb.version(), prog_name="Jedi Configuration Builder (jcb)")
 def jcb_driver():
     """
     Welcome to the Jedi Configuration Builder (jcb).

src/jcb/renderer.py

@@ -87,6 +87,10 @@ def __init__(self, template_dict: dict):
         app_path_model = self.template_dict.get('app_path_model')
         if app_path_model:
 
+            # Take the last element of the path and set this to the model_component in the
+            # dictionary. The path might end in a slash so split on / and take the last element.
+            self.template_dict['model_component'] = app_path_model.split('/')[-1] + '_'
+
             # Check if app_path_model is an absolute path
             if os.path.isabs(app_path_model):
                 self.j2_search_paths += [app_path_model]

src/jcb/utilities/testing.py

@@ -0,0 +1,76 @@
+# --------------------------------------------------------------------------------------------------
+
+
+import os
+
+import jcb
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+def get_apps():
+
+    # Get JCB path
+    jcb_path = jcb.get_jcb_path()
+
+    # Path to the apps
+    apps_path = os.path.join(jcb_path, 'configuration', 'apps')
+
+    # Return list of apps
+    return [app for app in os.listdir(apps_path) if os.path.isdir(os.path.join(apps_path, app))]
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+def apps_directory_to_dictionary():
+
+    # Get JCB path
+    jcb_path = jcb.get_jcb_path()
+
+    # Path to the apps
+    apps_path = os.path.join(jcb_path, 'configuration', 'apps')
+
+    def add_to_dict(d, path_parts, files):
+        for part in path_parts[:-1]:
+            if part not in d:
+                d[part] = {}
+            d = d[part]
+        d[path_parts[-1]] = files
+
+    directory_dict = {}
+
+    for dirpath, _, filenames in os.walk(apps_path):
+        yaml_j2_files = [f for f in filenames if f.endswith('.yaml.j2')]
+        if yaml_j2_files:
+            relative_path = os.path.relpath(dirpath, apps_path)
+            path_parts = relative_path.split(os.sep)
+            add_to_dict(directory_dict, path_parts, yaml_j2_files)
+
+    return directory_dict
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+def render_app_with_test_config(app_test_config):
+
+    # Style 1 for call: all in one API
+    # --------------------------------
+    jedi_dict_1 = jcb.render(app_test_config)
+
+    # Style 2 for call: renderer for multiple algorithms
+    # --------------------------------------------------
+    # Algorithm does not need to be in the dictionary of templates
+    algorithm = app_test_config['algorithm']
+    del app_test_config['algorithm']
+
+    jcb_obj = jcb.Renderer(app_test_config)
+    jedi_dict_2 = jcb_obj.render(algorithm)
+
+    # Assert that the two outputs match one another
+    assert jedi_dict_1 == jedi_dict_2
+
+
+# --------------------------------------------------------------------------------------------------

test/client_integration/gdas-atmosphere-templates.yaml

@@ -0,0 +1,127 @@
+# This part is for testing only. Normally this would just show algorithm: <algorithm>
+# -----------------------------------------------------------------------------------
+supported_algorithms:
+- 3dvar
+- hofx3d
+- hofx4d
+- local_ensemble_da
+
+
+# Search path for model and obs for JCB (relative for the submodule, or can be absolute)
+# --------------------------------------------------------------------------------------
+app_path_model: gdas/model/atmosphere
+app_path_observations: gdas/observations/atmosphere
+app_path_observation_chronicle: gdas/observation_chronicle/atmosphere
+
+
+# Places where we deviate from the generic file name of a yaml
+# ------------------------------------------------------------
+final_increment_file: atmosphere_final_increment_gaussian
+output_ensemble_increments_file: atmosphere_output_ensemble_increments_gaussian
+model_file: atmosphere_model_pseudo
+initial_condition_file: atmosphere_background
+background_error_file: atmosphere_background_error_hybrid_gsibec_bump
+
+
+# Global analysis things
+# ----------------------
+window_begin: '2024-02-01T00:00:00Z'
+window_length: PT6H
+bound_to_include: begin
+minimizer: DRPCG
+final_diagnostics_departures: anlmob
+analysis_variables: [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr]
+number_of_outer_loops: 2
+
+
+# Model things
+# ------------
+atmosphere_layout_x: 2
+atmosphere_layout_y: 2
+atmosphere_npx_ges: 361
+atmosphere_npy_ges: 361
+atmosphere_npz_ges: 127
+atmosphere_npx_anl: 361
+atmosphere_npy_anl: 361
+atmosphere_npz_anl: 127
+atmosphere_fv3jedi_files_path: DATA/fv3jedi
+
+# Background
+atmosphere_background_path: DATA/bkg
+atmosphere_background_ensemble_path: "DATA/ens/mem%mem%"
+
+atmosphere_background_time_iso: '2024-02-02T00:00:00Z'
+
+# Background error
+
+atmosphere_bump_data_directory: DATA/berror
+atmosphere_gsibec_path: DATA/berror
+atmosphere_number_ensemble_members: 3
+atmosphere_layout_gsib_x: 2
+atmosphere_layout_gsib_y: 2
+
+
+# Forecasting
+atmosphere_forecast_length: PT6H
+atmosphere_forecast_timestep: PT1H
+
+# Write final increment on Gaussian grid in variational
+atmosphere_final_increment_prefix: "./anl/atminc."
+
+
+# Observation things
+# ------------------
+observations: all_observations
+
+crtm_coefficient_path: "DATA/crtm/"
+
+# Naming conventions for observational files
+atmosphere_obsdatain_path: DATA/obs
+atmosphere_obsdatain_prefix: OPREFIX
+atmosphere_obsdatain_suffix: ".2024020100.nc"
+
+atmosphere_obsdataout_path: DATA/diags
+atmosphere_obsdataout_prefix: diag
+atmosphere_obsdataout_suffix: "_2024020100.nc"
+
+# Naming conventions for bias correction files
+atmosphere_obsbiasin_path: DATA/obs
+atmosphere_obsbiasin_prefix: GPREFIX
+atmosphere_obsbiasin_suffix: ".satbias.nc"
+atmosphere_obstlapsein_prefix: GPREFIX
+atmosphere_obstlapsein_suffix: ".tlapse.txt"
+atmosphere_obsbiascovin_prefix: GPREFIX
+atmosphere_obsbiascovin_suffix: ".satbias_cov.nc"
+
+atmosphere_obsbiasout_path: DATA/bc
+atmosphere_obsbiasout_prefix: APREFIX
+atmosphere_obsbiasout_suffix: ".satbias.nc"
+atmosphere_obsbiascovout_prefix: APREFIX
+atmosphere_obsbiascovout_suffix: ".satbias_cov.nc"
+
+
+# Local Ensemble DA (LETKF)
+# -------------------------
+local_ensemble_da_solver: GETKF
+
+increment_variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr]
+
+# Veritcal localization for GETKF
+vl_fraction_of_retained_variance: 0.750
+vl_lengthscale: 2.1
+vl_lengthscale_units: logp
+inflation_rtps: 0.85
+inflation_rtpp: 0.0
+inflation_mult: 1.0
+
+# Driver
+driver_update_obs_config_with_geometry_info: true
+driver_save_posterior_mean: false
+driver_save_posterior_ensemble: false
+driver_save_prior_mean: false
+driver_save_posterior_mean_increment: false
+driver_save_posterior_ensemble_increments: true
+
+# Diagnostics
+atmosphere_ensemble_increment_prefix: "./anl/mem%{member}%/atminc."
+atmosphere_posterior_output_gaussian: "./mem%{member}%/atmanl."