Difference plots and other stuff

cset-workflow/flow.cylc

@@ -5,18 +5,12 @@ description = Workflow for running CSET.
 URL = https://metoffice.github.io/CSET
 
 # Import all of our Jinja utilities for use in the workflow.
-{% from "jinja_utils" import get_models, glob, max, min, zip, restructure_field_list, sanitise_task_name %}
+{% from "jinja_utils" import get_models, glob, max, min, zip, sanitise_task_name %}
 # Load a list a model detail dictionaries.
 {% set models = get_models(ROSE_SUITE_VARIABLES) %}
 
 
 [scheduling]
-    # There is rarely a reason to manually set the runahead limit, as we will
-    # usually be limited by number of concurrent jobs.
-    {% if CSET_RUNAHEAD_LIMIT|default(False) %}
-        runahead limit = P{{CSET_RUNAHEAD_LIMIT}}
-    {% endif %}
-
     # Initial and final cycle points cover the entire period of interest.
     {% if CSET_CYCLING_MODE == "case_study" %}
         initial cycle point = {{ min(CSET_CASE_DATES) }}
@@ -101,9 +95,6 @@ URL = https://metoffice.github.io/CSET
     [[FETCH_DATA]]
     script = rose task-run -v --app-key=fetch_fcst
     execution time limit = PT1H
-        [[[environment]]]
-        CSET_ANALYSIS_OFFSET = {{CSET_ANALYSIS_OFFSET}}
-        CSET_ANALYSIS_PERIOD = {{CSET_ANALYSIS_PERIOD}}
 
     [[METPLUS]]
         [[[environment]]]
@@ -142,17 +133,17 @@ URL = https://metoffice.github.io/CSET
         WEB_DIR = {{WEB_DIR}}
 
     {% for model in models %}
-    [[fetch_fcst_m{{model["number"]}}]]
+    [[fetch_fcst_m{{model["id"]}}]]
     # Fetch data from disk or a file based archival system.
     inherit = FETCH_DATA
         [[[environment]]]
-        MODEL_NUMBER = {{model["number"]}}
+        MODEL_IDENTIFIER = {{model["id"]}}
         ROSE_APP_COMMAND_KEY = {{model["data_source"]}}
         DATA_PATH = {{model["data_path"]}}
         DATE_TYPE = {{model["date_type"]}}
-        {% if model["date_type"] != "initiation" %}
-            DATA_PERIOD = {{model["data_period"]}}
-        {% endif %}
+        DATA_PERIOD = {{model["data_period"]}}
+        ANALYSIS_OFFSET = {{model["analysis_offset"]}}
+        ANALYSIS_PERIOD = {{model["analysis_period"]}}
     {% endfor %}
 
     [[housekeeping_raw]]

cset-workflow/includes/aoa_diag.cylc

@@ -2,7 +2,7 @@
 {% for model in models %}
 {% for plev in AOA_PLEV %}
 [runtime]
-    [[aoa_diag_m{{model["number"]}}_plev{{plev}}]]
+    [[aoa_diag_m{{model["id"]}}_plev{{plev}}]]
     inherit = PROCESS
     execution time limit = PT60M
 
@@ -13,7 +13,7 @@
         [[[environment]]]
         CSET_RECIPE_NAME = "ageofair.yaml"
         CSET_ADDOPTS = "--AOA_PLEV={{plev}} --AOA_CYCLIC={{AOA_CYCLIC}} --MODEL_NAME='{{model["name"]}}'"
-        MODEL_NUMBER = {{model["number"]}}
+        MODEL_IDENTIFIERS = {{model["id"]}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/basic_qq_plot.cylc

@@ -34,17 +34,17 @@
 
 {# Example code to use as basis for multi-model cycling:
 {% for equivalent_field in restructure_field_list(PRESSURE_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[generic_basic_qq_plot_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[generic_basic_qq_plot_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_basic_qq_plot.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 #}

cset-workflow/includes/deterministic_domain_histogram_series.cylc

@@ -1,18 +1,18 @@
 {% if DOMAIN_HISTOGRAM_SERIES|default(False) %}
 {% for equivalent_field in restructure_field_list(PRESSURE_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 {% for plevel in PRESSURE_LEVELS %}
 [runtime]
-    [[generic_plevel_histogram_series_m{{model_number}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(plevel)}}]]
+    [[generic_plevel_histogram_series_m{{model_identifier}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(plevel)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_plevel_histogram_series.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
             --PLEVEL='{{plevel}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endfor %}

cset-workflow/includes/deterministic_domain_mean_surface_time_series.cylc

@@ -1,16 +1,16 @@
 {% if DOMAIN_MEAN_SURFACE_TIME_SERIES|default(False) %}
 {% for equivalent_field in restructure_field_list(SURFACE_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[generic_surface_domain_mean_time_series_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[generic_surface_domain_mean_time_series_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_surface_domain_mean_time_series.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/deterministic_domain_mean_vertical_model_level_profile_series.cylc

@@ -1,16 +1,16 @@
 {% if DOMAIN_MEAN_VERTICAL_PROFILE_MODELLEVEL_SERIES|default(False) %}
 {% for equivalent_field in restructure_field_list(MODEL_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[generic_mlevel_domain_mean_vertical_profile_series_m{{model_number}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(mlevel)}}]]
+    [[generic_mlevel_domain_mean_vertical_profile_series_m{{model_identifier}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(mlevel)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_mlevel_domain_mean_vertical_profile_series.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/deterministic_domain_mean_vertical_pressure_profile_series.cylc

@@ -1,16 +1,16 @@
 {% if DOMAIN_MEAN_VERTICAL_PROFILE_SERIES|default(False) %}
 {% for equivalent_field in restructure_field_list(PRESSURE_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[generic_plevel_domain_mean_vertical_profile_series_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[generic_plevel_domain_mean_vertical_profile_series_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_plevel_domain_mean_vertical_profile_series.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/deterministic_domain_surface_histogram_series.cylc

@@ -1,16 +1,16 @@
 {% if DOMAIN_SURFACE_HISTOGRAM_SERIES_FIELD|default(False) %}
 {% for equivalent_field in restructure_field_list(SURFACE_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[generic_surface_histogram_series_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[generic_surface_histogram_series_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_surface_histogram_series.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/deterministic_plot_cape_ratio.cylc

@@ -1,13 +1,13 @@
 {% if DETERMINISTIC_PLOT_CAPE_RATIO|default(False) %}
 {% for model in models %}
 [runtime]
-    [[plot_cape_ratio_m{{model["number"]}}]]
+    [[plot_cape_ratio_m{{model["id"]}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "CAPE_ratio_plot.yaml"
         CSET_ADDOPTS = """
             --MODEL_NAME='{{model["name"]}}'
         """
-        MODEL_NUMBER = {{model["number"]}}
+        MODEL_IDENTIFIERS = {{model["id"]}}
 {% endfor %}
 {% endif %}

cset-workflow/includes/deterministic_plot_inflow_properties.cylc

@@ -1,13 +1,13 @@
 {% if DETERMINISTIC_PLOT_INFLOW_PROPERTIES|default(False) %}
 {% for model in models %}
 [runtime]
-    [[inflow_layer_properties_plot_m{{model["number"]}}]]
+    [[inflow_layer_properties_plot_m{{model["id"]}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "inflow_layer_properties_plot.yaml"
         CSET_ADDOPTS = """
             --MODEL_NAME='{{model["name"]}}'
         """
-        MODEL_NUMBER = {{model["number"]}}
+        MODEL_IDENTIFIERS = {{model["id"]}}
 {% endfor %}
 {% endif %}

cset-workflow/includes/deterministic_single_point_surface_time_series.cylc

@@ -1,19 +1,19 @@
 {% if SURFACE_SINGLE_POINT_TIME_SERIES|default(False) %}
 {% for equivalent_field in restructure_field_list(SURFACE_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[generic_surface_single_point_time_series_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[generic_surface_single_point_time_series_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_surface_single_point_time_series.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
             --LONGITUDE_POINT='{{LONGITUDE_POINT}}'
             --LATITUDE_POINT='{{LATITUDE_POINT}}'
             --SINGLE_POINT_METHOD='{{SINGLE_POINT_METHOD}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/mlevel_transect.cylc

@@ -1,8 +1,8 @@
 {% if EXTRACT_MLEVEL_TRANSECT|default(False) %}
 {% for equivalent_field in restructure_field_list(MODEL_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[mlevel_transect_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[mlevel_transect_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
     # Transect calculation can take a while, so increase the time limit.
     execution time limit = PT60M
@@ -11,11 +11,11 @@
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
             --VERTICAL_COORDINATE='model_level_number'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
             --START_COORDS='{{MLEVEL_TRANSECT_STARTCOORDS}}'
             --FINISH_COORDS='{{MLEVEL_TRANSECT_FINISHCOORDS}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/plevel_transect.cylc

@@ -1,8 +1,8 @@
 {% if EXTRACT_PLEVEL_TRANSECT|default(False) %}
 {% for equivalent_field in restructure_field_list(PRESSURE_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 [runtime]
-    [[plevel_transect_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[plevel_transect_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
     # Transect calculation can take a while, so increase the time limit.
     execution time limit = PT60M
@@ -11,11 +11,11 @@
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
             --VERTICAL_COORDINATE='pressure'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
             --START_COORDS='{{PLEVEL_TRANSECT_STARTCOORDS}}'
             --FINISH_COORDS='{{PLEVEL_TRANSECT_FINISHCOORDS}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/plot_spatial_mlevel_model_field.cylc

@@ -1,18 +1,18 @@
 {% if PLOT_SPATIAL_MODEL_LEVEL_MODEL_FIELD|default(False) %}
 {% for equivalent_field in restructure_field_list(MODEL_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 {% for mlevel in UM_MODEL_LEVELS %}
 [runtime]
-    [[generic_mlevel_spatial_plot_sequence_m{{model_number}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(mlevel)}}]]
+    [[generic_mlevel_spatial_plot_sequence_m{{model_identifier}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(mlevel)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_mlevel_spatial_plot_sequence.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
             --MLEVEL='{{mlevel}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endfor %}

cset-workflow/includes/plot_spatial_plevel_model_field.cylc

@@ -1,9 +1,9 @@
 {% if PLOT_SPATIAL_PRESSURE_LEVEL_MODEL_FIELD|default(False) %}
 {% for equivalent_field in restructure_field_list(PRESSURE_LEVEL_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for model_identifier, field in equivalent_field.items() %}
 {% for plevel in PRESSURE_LEVELS %}
 [runtime]
-    [[generic_plevel_spatial_plot_sequence_m{{model_number}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(plevel)}}]]
+    [[generic_plevel_spatial_plot_sequence_m{{model_identifier}}_{{sanitise_task_name(field)}}_lv{{sanitise_task_name(plevel)}}]]
     inherit = PROCESS
         [[[directives]]]
         --mem=4000
@@ -12,9 +12,9 @@
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
             --PLEVEL='{{plevel}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{models[model_identifier]["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model_identifier}}
 {% endfor %}
 {% endfor %}
 {% endfor %}

cset-workflow/includes/plot_spatial_surface_model_field.cylc

@@ -1,16 +1,16 @@
 {% if PLOT_SPATIAL_SURFACE_MODEL_FIELD|default(False) %}
-{% for equivalent_field in restructure_field_list(SURFACE_MODEL_FIELDS) %}
-{% for model_number, field in equivalent_field.items() %}
+{% for field in SURFACE_MODEL_FIELDS %}
+{% for model in models %}
 [runtime]
-    [[generic_spatial_plot_time_series_m{{model_number}}_{{sanitise_task_name(field)}}]]
+    [[generic_spatial_plot_time_series_m{{model["id"]}}_{{sanitise_task_name(field)}}]]
     inherit = PROCESS
         [[[environment]]]
         CSET_RECIPE_NAME = "generic_surface_spatial_plot_sequence.yaml"
         CSET_ADDOPTS = """
             --VARNAME='{{field}}'
-            --MODEL_NAME='{{models[model_number]["name"]}}'
+            --MODEL_NAME='{{model["name"]}}'
         """
-        MODEL_NUMBER = {{model_number}}
+        MODEL_IDENTIFIERS = {{model["id"]}}
 {% endfor %}
 {% endfor %}
 {% endif %}

cset-workflow/includes/spatial_surface_difference.cylc

@@ -0,0 +1,19 @@
+{% if SPATIAL_SURFACE_DIFFERENCE|default(False) %}
+{% set base_model = models|selectattr("id", 0)|first %}
+# As we use model 0 as the base model we skip it here.
+{% for model in models|selectattr("id") %}
+{% for field in SURFACE_MODEL_FIELDS %}
+[runtime]
+    [[spatial_surface_difference_m{{model_identifier}}_{{sanitise_task_name(field)}}]]
+    inherit = PROCESS
+        [[[environment]]]
+        CSET_RECIPE_NAME = "surface_spatial_difference.yaml"
+        CSET_ADDOPTS = """
+            --VARNAME='{{field}}'
+            --BASE_MODEL='{{base_model["name"]}}'
+            --OTHER_MODEL='{{model["name"]}}'
+        """
+        MODEL_IDENTIFIERS = '0 {{model_identifier}}'
+{% endfor %}
+{% endfor %}
+{% endif %}