From f30c17981dcf61ac8034e2f59ec1f788136e9d69 Mon Sep 17 00:00:00 2001 From: Paola Petrelli Date: Mon, 28 Jun 2021 10:54:42 +1000 Subject: [PATCH] cordex now get vocabularies from json file not from ESGF query --- clef/cli.py | 1 + clef/code.py | 9 +- clef/cordex.py | 10 +- clef/data/CMIP6_activity_id.json | 6 +- clef/data/CMIP6_experiment_id.json | 6 +- clef/data/CMIP6_frequency.json | 14 +- clef/data/CMIP6_grid_label.json | 6 +- clef/data/CMIP6_institution_id.json | 6 +- clef/data/CMIP6_nominal_resolution.json | 6 +- clef/data/CMIP6_realm.json | 6 +- clef/data/CMIP6_source_id.json | 159 +++- clef/data/CMIP6_source_type.json | 6 +- clef/data/CMIP6_sub_experiment_id.json | 6 +- clef/data/CMIP6_table_id.json | 6 +- clef/data/CMIP6_validation.json | 3 + clef/data/CORDEX_validation.json | 989 ++++++++++++++++++++++++ clef/helpers.py | 25 +- 17 files changed, 1198 insertions(+), 66 deletions(-) create mode 100644 clef/data/CORDEX_validation.json diff --git a/clef/cli.py b/clef/cli.py index 94f9846..d076fb7 100644 --- a/clef/cli.py +++ b/clef/cli.py @@ -33,6 +33,7 @@ from .esdoc import citation, write_cite import clef.cordex as cordex_ + def clef_catch(): debug_logger = logging.getLogger('clef_debug') debug_logger.setLevel(logging.CRITICAL) diff --git a/clef/code.py b/clef/code.py index ff09149..e5c2414 100644 --- a/clef/code.py +++ b/clef/code.py @@ -292,10 +292,6 @@ def and_filter(df, cols, fixed, **kwargs): useful = set(['version', 'source_id', 'model', 'path','dataset_id', 'domain', 'cmor_table','table_id', 'ensemble', 'member_id', 'driving_experiment', 'model_id', 'frequency', 'driving_model', 'rcm_version']) - set(fixed) - #useful = set(['version', 'source_id', 'model', 'path','dataset_id', - # 'cmor_table','table_id', 'ensemble', 'member_id', 'driving_model_id', - # 'driving_model_ensemble_member', 'model_id', 'frequency', - # 'cordex_domain', 'driving_experiment_name', 'rcm_version_id']) - set(fixed) fields = ['comb'] + [f for f in useful if f in [c for c in df.columns.values]] # define the aggregation dictionary agg_dict = {k: set for k in fields} @@ -425,9 +421,8 @@ def ids_df(dids): 'time_frequency', 'realm', 'cmor_table', 'ensemble', 'version'] elif project == 'cordex': facets_list = ['project', 'product', 'domain', 'institute', 'driving_model', - 'driving_experiment', 'ensemble', 'model_id', 'rcm_version', 'frequency', 'variable', 'version'] - #facets_list = ['project', 'product', 'cordex_domain', 'institute_id', 'driving_model_id', - # 'driving_experiment_name', 'driving_model_ensemble_member', 'model_id', 'rcm_version_id', 'frequency', 'variable', 'version'] + 'driving_experiment', 'ensemble', 'model_id', 'rcm_version', 'frequency', + 'variable', 'version'] else: print(f'Warning: project {project} not available') return results diff --git a/clef/cordex.py b/clef/cordex.py index bc4fe3d..a2fed3f 100644 --- a/clef/cordex.py +++ b/clef/cordex.py @@ -16,10 +16,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from clef.esgf import esgf_query import functools import click +from clef.esgf import esgf_query +from clef.helpers import load_vocabularies def tidy_facet_count(v): return v[::2] @@ -60,7 +61,8 @@ def get_esgf_facets(project): "controlled_vocab": True, }, "version": {"short": ['-vrs'], "help": "Data publication version", "controlled_vocab": True}, - "cf_standard_name": {"short": ['-cf'], "help": "CF-Conventions name of the variable"}, + "cf_standard_name": {"short": ['-cf'], "help": "CF-Conventions name of the variable", + "controlled_vocab": True}, "experiment_family": {"short": ['-ef'], 'one': True, "controlled_vocab": True, "help": "Experiment family: All, Historical, RCP"}, "institute": { "short": ['-inst'], @@ -74,8 +76,8 @@ class CordexCommand(click.Command): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - facets = get_esgf_facets(project="CORDEX,CORDEX-Adjust,CORDEX-ESD,CORDEXReklies") - facets['driving_experiment'] = facets['experiment'] + #facets = get_esgf_facets(project="CORDEX,CORDEX-Adjust,CORDEX-ESD,CORDEXReklies") + facets = load_vocabularies('CORDEX') facets['rcm_name'].append('CCAM-1391M') for k, v in cli_facets.items(): opt = click.Option( diff --git a/clef/data/CMIP6_activity_id.json b/clef/data/CMIP6_activity_id.json index 5b45021..f273e5c 100644 --- a/clef/data/CMIP6_activity_id.json +++ b/clef/data/CMIP6_activity_id.json @@ -26,13 +26,13 @@ "VolMIP":"Volcanic Forcings Model Intercomparison Project" }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "activity_id_CV_modified":"Mon Mar 5 16:39:09 2018 -0800", "activity_id_CV_note":"Update activity_id to include CDRMIP and PAMIP", "author":"Paul J. Durack ", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" } } \ No newline at end of file diff --git a/clef/data/CMIP6_experiment_id.json b/clef/data/CMIP6_experiment_id.json index 0478776..b6ec588 100644 --- a/clef/data/CMIP6_experiment_id.json +++ b/clef/data/CMIP6_experiment_id.json @@ -9456,13 +9456,13 @@ } }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "experiment_id_CV_modified":"Tue Dec 15 12:25:59 2020 -0800", "experiment_id_CV_note":"Revise experiment_id historical parent experiments", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" } } \ No newline at end of file diff --git a/clef/data/CMIP6_frequency.json b/clef/data/CMIP6_frequency.json index 2af161a..d31bdee 100644 --- a/clef/data/CMIP6_frequency.json +++ b/clef/data/CMIP6_frequency.json @@ -3,9 +3,9 @@ "1hr":"sampled hourly", "1hrCM":"monthly-mean diurnal cycle resolving each day into 1-hour means", "1hrPt":"sampled hourly, at specified time point within an hour", - "3hr":"sampled every 3 hours", + "3hr":"3 hourly mean samples", "3hrPt":"sampled 3 hourly, at specified time point within the time period", - "6hr":"sampled every 6 hours", + "6hr":"6 hourly mean samples", "6hrPt":"sampled 6 hourly, at specified time point within the time period", "day":"daily mean samples", "dec":"decadal mean samples", @@ -18,13 +18,13 @@ "yrPt":"sampled yearly, at specified time point within the time period" }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", - "frequency_CV_modified":"Fri Oct 27 14:03:00 2017 -0700", - "frequency_CV_note":"Issue414 durack1 revise frequency 1hrCM definition (#418)", + "frequency_CV_modified":"Mon May 24 13:48:15 2021 00100", + "frequency_CV_note":"Update description of 3hr and 6hr frequencies", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" } } \ No newline at end of file diff --git a/clef/data/CMIP6_grid_label.json b/clef/data/CMIP6_grid_label.json index 818a7be..4e9cc74 100644 --- a/clef/data/CMIP6_grid_label.json +++ b/clef/data/CMIP6_grid_label.json @@ -47,13 +47,13 @@ "grz":"regridded zonal mean data reported on the data provider's preferred latitude target grid" }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "grid_label_CV_modified":"Fri Sep 8 18:12:00 2017 -0700", "grid_label_CV_note":"Issue395 durack1 augment grid_label with description (#401)", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" } } \ No newline at end of file diff --git a/clef/data/CMIP6_institution_id.json b/clef/data/CMIP6_institution_id.json index bc4f9c0..c16d5c1 100644 --- a/clef/data/CMIP6_institution_id.json +++ b/clef/data/CMIP6_institution_id.json @@ -55,13 +55,13 @@ "UofT":"Department of Physics, University of Toronto, 60 St George Street, Toronto, ON M5S1A7, Canada" }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "institution_id":"PCMDI", "institution_id_CV_modified":"Mon Nov 16 11:16:39 2020 -0800", "institution_id_CV_note":"Register institution_id LLNL; Py3 cleanup", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" } } \ No newline at end of file diff --git a/clef/data/CMIP6_nominal_resolution.json b/clef/data/CMIP6_nominal_resolution.json index 055a5af..0d90b0e 100644 --- a/clef/data/CMIP6_nominal_resolution.json +++ b/clef/data/CMIP6_nominal_resolution.json @@ -17,13 +17,13 @@ "5000 km" ], "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "institution_id":"PCMDI", "nominal_resolution_CV_modified":"Tues Nov 15 16:04:00 2016 -0700", "nominal_resolution_CV_note":"Issue141 durack1 update grid_resolution to nominal_resolution (#143)", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" } } \ No newline at end of file diff --git a/clef/data/CMIP6_realm.json b/clef/data/CMIP6_realm.json index c37c580..0e00324 100644 --- a/clef/data/CMIP6_realm.json +++ b/clef/data/CMIP6_realm.json @@ -10,11 +10,11 @@ "seaIce":"Sea Ice" }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "realm_CV_modified":"Tues Apr 18 12:03:00 2017 -0700", "realm_CV_note":"Issue285 durack1 update realm format (#290)", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" diff --git a/clef/data/CMIP6_source_id.json b/clef/data/CMIP6_source_id.json index f91e495..a291353 100644 --- a/clef/data/CMIP6_source_id.json +++ b/clef/data/CMIP6_source_id.json @@ -4084,6 +4084,57 @@ "release_year":"2019", "source_id":"GISS-E2-2-G" }, + "GISS-E2-2-H":{ + "activity_participation":[ + "CMIP", + "SIMIP", + "ScenarioMIP" + ], + "cohort":[ + "Registered" + ], + "institution_id":[ + "NASA-GISS" + ], + "label":"GISS-E2.2H", + "label_extended":"GISS-E2.2H", + "model_component":{ + "aerosol":{ + "description":"Varies with physics-version (p==1 none, p==3 OMA, p==4 TOMAS, p==5 MATRIX)", + "native_nominal_resolution":"250 km" + }, + "atmos":{ + "description":"GISS-E2.2 (High Top, 2.5x2 degree; 144 x 90 longitude/latitude; 102 levels; top level 0.002 hPa)", + "native_nominal_resolution":"250 km" + }, + "atmosChem":{ + "description":"Varies with physics-version (p==1 Non-interactive, p>1 GPUCCINI)", + "native_nominal_resolution":"250 km" + }, + "land":{ + "description":"GISS LSM", + "native_nominal_resolution":"250 km" + }, + "landIce":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "ocean":{ + "description":"HYCOM Ocean (~1 degree tripolar grid; 360 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)", + "native_nominal_resolution":"100 km" + }, + "ocnBgchem":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "seaIce":{ + "description":"GISS SI (same grid as atmos)", + "native_nominal_resolution":"250 km" + } + }, + "release_year":"2021", + "source_id":"GISS-E2-2-H" + }, "GISS-E3-G":{ "activity_participation":[ "AerChemMIP", @@ -5035,6 +5086,104 @@ "release_year":"2019", "source_id":"IPSL-CM6A-LR-INCA" }, + "IPSL-CM6A-MR025":{ + "activity_participation":[ + "CMIP" + ], + "cohort":[ + "Registered" + ], + "institution_id":[ + "IPSL" + ], + "label":"IPSL-CM6A-MR025", + "label_extended":"IPSL-CM6A-MR025", + "model_component":{ + "aerosol":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "atmos":{ + "description":"LMDZ (NPv6; 256 x 256 longitude/latitude; 79 levels; top level 80000 m)", + "native_nominal_resolution":"100 km" + }, + "atmosChem":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "land":{ + "description":"ORCHIDEE (v2.2, Water/Carbon/Energy mode; same grid as atmos)", + "native_nominal_resolution":"100 km" + }, + "landIce":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "ocean":{ + "description":"NEMO-OPA (eORCA025, tripolar primarily 0.25deg; 1442 x 1207 longitude/latitude; 75 levels; top grid cell 0-2 m)", + "native_nominal_resolution":"25 km" + }, + "ocnBgchem":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "seaIce":{ + "description":"NEMO-LIM3 (same grid as ocean)", + "native_nominal_resolution":"25 km" + } + }, + "release_year":"2021", + "source_id":"IPSL-CM6A-MR025" + }, + "IPSL-CM6A-MR1":{ + "activity_participation":[ + "CMIP" + ], + "cohort":[ + "Registered" + ], + "institution_id":[ + "IPSL" + ], + "label":"IPSL-CM6A-MR1", + "label_extended":"IPSL-CM6A-MR1", + "model_component":{ + "aerosol":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "atmos":{ + "description":"LMDZ (NPv6; 256 x 256 longitude/latitude; 79 levels; top level 80000 m)", + "native_nominal_resolution":"100 km" + }, + "atmosChem":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "land":{ + "description":"ORCHIDEE (v2.2, Water/Carbon/Energy mode; same grid as atmos)", + "native_nominal_resolution":"100 km" + }, + "landIce":{ + "description":"none", + "native_nominal_resolution":"none" + }, + "ocean":{ + "description":"NEMO-OPA (eORCA1.3, tripolar primarily 1deg; 362 x 332 longitude/latitude; 75 levels; top grid cell 0-2 m)", + "native_nominal_resolution":"100 km" + }, + "ocnBgchem":{ + "description":"NEMO-PISCES (same grid as ocean)", + "native_nominal_resolution":"100 km" + }, + "seaIce":{ + "description":"NEMO-LIM3 (same grid as ocean)", + "native_nominal_resolution":"100 km" + } + }, + "release_year":"2021", + "source_id":"IPSL-CM6A-MR1" + }, "IPSL-CM7A-ATM-HR":{ "activity_participation":[ "HighResMIP" @@ -7272,13 +7421,13 @@ } }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", - "source_id_CV_modified":"Tue Feb 16 10:44:59 2021 -0800", - "source_id_CV_note":"Revise source_id CMCC-ESM2", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", + "source_id_CV_modified":"Mon Jun 21 11:22:45 2021 -0700", + "source_id_CV_note":"Register source_id IPSL-CM6A-MR025", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" } } \ No newline at end of file diff --git a/clef/data/CMIP6_source_type.json b/clef/data/CMIP6_source_type.json index 78eea68..402bc20 100644 --- a/clef/data/CMIP6_source_type.json +++ b/clef/data/CMIP6_source_type.json @@ -12,11 +12,11 @@ "SLAB":"slab-ocean used with an AGCM in representing the atmosphere-ocean coupled system" }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "source_type_CV_modified":"Fri Sep 8 17:57:00 2017 -0700", "source_type_CV_note":"Issue396 durack1 augment source_type with description (#399)", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)" diff --git a/clef/data/CMIP6_sub_experiment_id.json b/clef/data/CMIP6_sub_experiment_id.json index 9359745..c45c413 100644 --- a/clef/data/CMIP6_sub_experiment_id.json +++ b/clef/data/CMIP6_sub_experiment_id.json @@ -76,11 +76,11 @@ "s2029":"initialized near end of year 2029" }, "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)", "sub_experiment_id_CV_modified":"Mon Jun 17 11:01:51 2019 -0700", "sub_experiment_id_CV_note":"Revise sub_experiment_id values" diff --git a/clef/data/CMIP6_table_id.json b/clef/data/CMIP6_table_id.json index e68e7f5..0473d25 100644 --- a/clef/data/CMIP6_table_id.json +++ b/clef/data/CMIP6_table_id.json @@ -45,11 +45,11 @@ "fx" ], "version_metadata":{ - "CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800", - "CV_collection_version":"6.2.55.10", + "CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700", + "CV_collection_version":"6.2.56.2", "author":"Paul J. Durack ", "institution_id":"PCMDI", - "previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd", + "previous_commit":"66344226746627e0b6c12db974666126c343ce27", "specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)", "table_id_CV_modified":"Fri Jan 13 09:27:00 2017 -0700", "table_id_CV_note":"Issue199 durack1 update table_id to Data Request v1.0 (#200)" diff --git a/clef/data/CMIP6_validation.json b/clef/data/CMIP6_validation.json index 4e971bf..fd34d3f 100644 --- a/clef/data/CMIP6_validation.json +++ b/clef/data/CMIP6_validation.json @@ -77,6 +77,7 @@ "GISS-E2-1-G-CC", "GISS-E2-1-H", "GISS-E2-2-G", + "GISS-E2-2-H", "GISS-E3-G", "HadGEM3-GC31-HH", "HadGEM3-GC31-HM", @@ -95,6 +96,8 @@ "IPSL-CM6A-ATM-HR", "IPSL-CM6A-LR", "IPSL-CM6A-LR-INCA", + "IPSL-CM6A-MR025", + "IPSL-CM6A-MR1", "IPSL-CM7A-ATM-HR", "IPSL-CM7A-ATM-LR", "KACE-1-0-G", diff --git a/clef/data/CORDEX_validation.json b/clef/data/CORDEX_validation.json new file mode 100644 index 0000000..56200f9 --- /dev/null +++ b/clef/data/CORDEX_validation.json @@ -0,0 +1,989 @@ +{ + "cf_standard_name": [ + "2-m_specific_humidity", + "2m_dew_point_temperature", + "accumulated_downwelling_lw_flux_at_top", + "accumulated_morton_areal_potential_evapotranspiration", + "accumulated_total_grid_scale_snow_and_ice", + "air_pressure_at_sea_level", + "air_temperature", + "atmosphere_boundary_layer_thickness", + "atmosphere_cloud_condensed_water_content", + "atmosphere_cloud_ice_content", + "atmosphere_cloud_liquid_water_content", + "atmosphere_mass_content_of_cloud_ice", + "atmosphere_mass_content_of_cloud_liquid_water", + "atmosphere_water_vapor_content", + "avg_evaporation", + "avg_latent_heat_flux", + "avg_sensible_heat_flux", + "avg_soil_moisture_1", + "cell_area", + "clear-sky_surface_downwelling_longwave_radiation", + "clear-sky_surface_downwelling_shortwave_radiation", + "clear-sky_surface_upwelling_longwave_radiation", + "clear-sky_surface_upwelling_shortwave_radiation", + "clear-sky_toa_outgoing_longwave_radiation", + "clear-sky_toa_outgoing_shortwave_radiation", + "cloud-top_effective_droplet_radius", + "cloud_area_fraction", + "cloud_area_fraction_in_atmosphere_layer", + "cloud_droplet_number_concentration_at_cloud_top", + "cloud_fraction", + "cloud_liquid_water_mixing_ratio", + "convective_available_potential_energy_(cape)", + "convective_cloud_area_fraction", + "convective_precipitation_flux", + "daily_maximum_near-surface_temperature_maximum", + "daily_minimum_near-surface_temperature_minimum", + "dew_point_temperature", + "duration_of_sunshine", + "eastward_wind", + "frictional_velocity_in_similarity_theory", + "geopotential_height", + "grid_eastward_wind", + "grid_northward_wind", + "height", + "ice_cloud_optical_depth", + "ice_mixing_ratio", + "lagrangian_tendency_of_air_pressure", + "land_area_fraction", + "land_area_fraction_", + "land_ice_area_fraction", + "large_scale_precipitation_amount", + "liquid_cloud_optical_depth", + "long_name", + "lw_at_toa", + "lw_downwelling_at_ground", + "maximum_screen_temperature", + "mean_sea_level_pressure", + "meridional_wind_at_200hpa", + "meridional_wind_at_500hpa", + "meridional_wind_at_850hpa", + "meridional_wind_stress", + "minimum_screen_temperature", + "mrro", + "near-surface_specific_humidity", + "northward_wind", + "number_concentration_of_cloud_liquid_water_particles_in_air", + "pbl_depth", + "potential_evapotranspiration", + "prc", + "precipitation", + "precipitation_flux", + "relative_humidity", + "root_depth", + "rotated_pole", + "runoff", + "runoff_flux", + "screen_temperature", + "sea_ice_area_fraction", + "sea_ice_flag", + "sea_surface_temperature", + "snow_depth_(liquid_water)", + "snowfall_flux", + "soil_frozen_water_content", + "soil_level", + "soil_liquid_water", + "soil_moisture_content", + "soil_moisture_content_at_field_capacity", + "soil_moisture_saturation_point", + "soil_moisture_wilting_point", + "soil_temperature", + "solar_downwelling_at_ground", + "solar_in_at_toa", + "solar_out_at_toa", + "specific_humidity", + "specific_humidity_at_200hpa", + "specific_humidity_at_850hpa", + "subsurface_runoff", + "sunshine_hours", + "surface_air_pressure", + "surface_albedo", + "surface_altitude", + "surface_downward_eastward_stress", + "surface_downward_northward_stress", + "surface_downwelling_longwave_flux_in_air", + "surface_downwelling_shortwave_flux_in_air", + "surface_emissivity", + "surface_runoff_flux", + "surface_snow_amount", + "surface_snow_area_fraction", + "surface_snow_melt_flux", + "surface_snow_thickness", + "surface_temperature", + "surface_upward_latent_heat_flux", + "surface_upward_sensible_heat_flux", + "surface_upward_water_vapor_flux_in_air", + "surface_upwelling_longwave_flux_in_air", + "surface_upwelling_shortwave_flux_in_air", + "surface_wind_speed", + "toa_incoming_shortwave_flux", + "toa_outgoing_longwave_flux", + "toa_outgoing_shortwave_flux", + "total_cloud_ave", + "upwelling_longwave_radiation", + "upwelling_shortwave_radiation", + "volume_fraction_of_condensed_water_in_soil", + "water_evaporation_flux", + "water_potential_evaporation_flux", + "wet-bulb_potential_temperature", + "wind_speed", + "wind_speed_of_gust", + "x-component_10m_wind", + "x_bnds", + "y-component_10m_wind", + "y_bnds", + "zg850", + "zonal_wind_at_200hpa", + "zonal_wind_at_500hpa", + "zonal_wind_at_850hpa", + "zonal_wind_stress" + ], + "domain": [ + "AFR-22", + "AFR-44", + "AFR-44i", + "ANT-44", + "ANT-44i", + "ARC-22", + "ARC-44", + "ARC-44i", + "AUS", + "AUS-22", + "AUS-44", + "AUS-44i", + "CAM-22", + "CAM-44", + "CAM-44i", + "CAS-22", + "CAS-44", + "EAS-22", + "EAS-44", + "EAS-44i", + "EUR-11", + "EUR-11i", + "EUR-22", + "EUR-25", + "EUR-44", + "EUR-44i", + "MED-11", + "MNA-22", + "MNA-22i", + "MNA-44", + "MNA-44i", + "NAM-11", + "NAM-22", + "NAM-44", + "NAM-44i", + "SAM-20", + "SAM-22", + "SAM-44", + "SAM-44i", + "SEA-22", + "WAS-22", + "WAS-44", + "WAS-44i" + ], + "driving_model": [ + "ACCESS1-0", + "ACCESS1-3", + "BNU-ESM", + "CCCma-CanESM2", + "CCSM4", + "CMCC-CMCC-CM", + "CMCC-CMS", + "CNRM-CERFACS-CNRM-CM5", + "CNRM-CM5", + "CSIRO-BOM-ACCESS1-0", + "CSIRO-BOM-ACCESS1-3", + "CSIRO-Mk3-6-0", + "CSIRO-QCCCE-CSIRO-Mk3-6-0", + "CanESM2", + "ECMWF-ERAINT", + "GFDL-ESM2G", + "GFDL-ESM2M", + "HadGEM2-CC", + "ICHEC-EC-EARTH", + "IPSL-CM5A-LR", + "IPSL-CM5A-MR", + "IPSL-CM5B-LR", + "IPSL-IPSL-CM5A-LR", + "IPSL-IPSL-CM5A-MR", + "MIROC-ESM", + "MIROC-ESM-CHEM", + "MIROC-MIROC5", + "MIROC5", + "MOHC-HadGEM2-CC", + "MOHC-HadGEM2-ES", + "MPI-ESM-LR", + "MPI-ESM-MR", + "MPI-M-MPI-ESM-LR", + "MPI-M-MPI-ESM-MR", + "MRI-CGCM3", + "NCAR-CCSM4", + "NCC-NorESM1-M", + "NOAA-GFDL-GFDL-CM3", + "NOAA-GFDL-GFDL-ESM2G", + "NOAA-GFDL-GFDL-ESM2M", + "NorESM1-M", + "bcc-csm1-1-m" + ], + "ensemble": [ + "r0i0p0", + "r12i1p1", + "r1i1p1", + "r2i1p1", + "r3i1p1", + "r4i1p1", + "r5i1p1", + "r6i1p1", + "r9i1p1" + ], + "experiment": [ + "evaluation", + "historical", + "rcp26", + "rcp45", + "rcp85" + ], + "experiment_family": [ + "All", + "Historical", + "RCP" + ], + "institute": [ + "AWI", + "BCCR", + "BOM", + "BOUN", + "CCCMA", + "CLMcom", + "CLMcom-CMCC", + "CLMcom-ETH", + "CLMcom-HZG", + "CLMcom-KIT", + "CNRM", + "CSIRO", + "CYI", + "DHMZ", + "DMI", + "DWD", + "ETH", + "GERICS", + "HMS", + "ICTP", + "IITM", + "INPE", + "IPSL", + "IPSL-INERIS", + "ISU", + "KNMI", + "KNU", + "MGO", + "MOHC", + "MPI-CSC", + "NCAR", + "NIMS-KMA", + "NUIM", + "ORNL", + "OURANOS", + "PNU", + "POSTECH", + "RMIB-UGent", + "RU-CORE", + "SMHI", + "UA", + "UCAN", + "UHOH", + "ULg", + "UNIST", + "UNSW", + "UQAM" + ], + "rcm_name": [ + "ALADIN52", + "ALADIN53", + "ALADIN63", + "ALADIN64", + "ALARO-0", + "ARPEGE51", + "BOM-SDM", + "CCAM", + "CCAM-1704", + "CCAM-2008", + "CCLM-0-9", + "CCLM4-21-2", + "CCLM4-8-17", + "CCLM4-8-17-CLM3-5", + "CCLM5-0-15", + "CCLM5-0-2", + "CCLM5-0-6", + "CCLM5-0-9", + "COSMO-crCLIM-v1-1", + "CRCM5", + "CRCM5-SN", + "CanRCM4", + "EPISODES2018", + "Eta", + "HIRHAM5", + "HadGEM3-RA", + "HadREM3-GA7-05", + "HadRM3P", + "MAR311", + "MAR36", + "RA", + "RACMO21P", + "RACMO22E", + "RACMO22T", + "RCA4", + "RCA4-SN", + "REMO2009", + "REMO2015", + "RRCM", + "RegCM4", + "RegCM4-0", + "RegCM4-2", + "RegCM4-3", + "RegCM4-4", + "RegCM4-6", + "RegCM4-7", + "SNURCM", + "VRF370", + "WRF", + "WRF331", + "WRF331F", + "WRF331G", + "WRF341E", + "WRF341I", + "WRF351", + "WRF360J", + "WRF360K", + "WRF360L", + "WRF361H", + "WRF381P", + "CCAM-1391M" + ], + "rcm_version": [ + "r2", + "v0", + "v01", + "v02", + "v1", + "v1-r1", + "v1a", + "v2", + "v201312", + "v3", + "v3-5-1", + "v4", + "v4-4-rc8", + "v5", + "x2yn2-v1" + ], + "time_frequency": [ + "1hr", + "3hr", + "6hr", + "day", + "fx", + "mon", + "sem" + ], + "variable": [ + "Rotated_Pole", + "aclwdnt", + "alb", + "apet", + "areacella", + "cape", + "cdnc200", + "cdnc500", + "cdnc700", + "cdnc850", + "cdnc925", + "cdnctop", + "clfr", + "clfr1000", + "clfr200", + "clfr300", + "clfr400", + "clfr500", + "clfr600", + "clfr700", + "clfr850", + "clfr875", + "clfr900", + "clfr925", + "clfr950", + "clfr975", + "clh", + "clice", + "clice1000", + "clice200", + "clice300", + "clice400", + "clice500", + "clice600", + "clice700", + "clice850", + "clice875", + "clice900", + "clice925", + "clice950", + "clice975", + "clivi", + "cll", + "clm", + "clt", + "cltc100", + "clwmr", + "clwmr1000", + "clwmr200", + "clwmr300", + "clwmr400", + "clwmr500", + "clwmr600", + "clwmr700", + "clwmr850", + "clwmr875", + "clwmr900", + "clwmr925", + "clwmr950", + "clwmr975", + "clwvi", + "emiss", + "evspsbl", + "evspsblpot", + "heightv", + "hfls", + "hfss", + "hufs", + "hur", + "hur1000", + "hur200", + "hur300", + "hur400", + "hur500", + "hur600", + "hur700", + "hur850", + "hur875", + "hur900", + "hur925", + "hur950", + "hur975", + "hurs", + "hus", + "hus100", + "hus1000", + "hus200", + "hus250", + "hus300", + "hus400", + "hus500", + "hus50m", + "hus600", + "hus700", + "hus850", + "hus875", + "hus900", + "hus925", + "hus950", + "hus975", + "huss", + "icod", + "lcod", + "lwp", + "mrfso", + "mross", + "mrro", + "mrros", + "mrso", + "mrsofc", + "mrsos", + "mrsosat", + "mrsosd", + "mrsowp", + "orog", + "pr", + "pr-bc", + "pr-fl", + "prAdjust", + "prc", + "prhmax", + "prls", + "prsn", + "prw", + "ps", + "psl", + "reffclwtop", + "rlds", + "rldscs", + "rlus", + "rluscs", + "rlut", + "rlutcs", + "rootd", + "rsds", + "rsdsAdjust", + "rsdscs", + "rsdt", + "rsus", + "rsuscs", + "rsut", + "rsutcs", + "sfcWind", + "sfcWindAdjust", + "sfcWindmax", + "sfcWindmaxmax", + "sftgif", + "sftlf", + "sic", + "slev", + "slw", + "snc", + "snd", + "snm", + "snownc", + "snw", + "sst", + "sund", + "ta", + "ta100", + "ta1000", + "ta200", + "ta250", + "ta300", + "ta400", + "ta500", + "ta50m", + "ta600", + "ta700", + "ta850", + "ta875", + "ta900", + "ta925", + "ta950", + "ta975", + "tas", + "tasAdjust", + "tasmax", + "tasmax-bc", + "tasmaxAdjust", + "tasmaxts", + "tasmin", + "tasmin-bc", + "tasminAdjust", + "tasmints", + "tauu", + "tauv", + "tds", + "thetapw1000", + "thetapw700", + "thetapw850", + "thetapw925", + "ts", + "tsmax", + "tsmin", + "tsos", + "u200", + "u500", + "u850", + "ua", + "ua100", + "ua1000", + "ua100m", + "ua150m", + "ua200", + "ua200m", + "ua250", + "ua250m", + "ua300", + "ua400", + "ua500", + "ua50m", + "ua600", + "ua700", + "ua850", + "ua875", + "ua900", + "ua925", + "ua950", + "ua975", + "uas", + "ustar", + "v200", + "v500", + "v850", + "va", + "va100", + "va1000", + "va100m", + "va150m", + "va200", + "va200m", + "va250", + "va250m", + "va300", + "va400", + "va500", + "va50m", + "va600", + "va700", + "va850", + "va875", + "va900", + "va925", + "va950", + "va975", + "vas", + "wap200", + "wap500", + "wap700", + "wap850", + "wap925", + "wsgsmax", + "x_bnds", + "y_bnds", + "zg", + "zg100", + "zg1000", + "zg100m", + "zg150m", + "zg200", + "zg200m", + "zg250", + "zg250m", + "zg300", + "zg350", + "zg400", + "zg450", + "zg500", + "zg50m", + "zg550", + "zg600", + "zg650", + "zg700", + "zg750", + "zg800", + "zg850", + "zg875", + "zg900", + "zg925", + "zg950", + "zg975", + "zmla" + ], + "version": [ + "20130927", + "20131026", + "20131030", + "20131101", + "20131108", + "20131119", + "20131211", + "20131212", + "20140113", + "20140123", + "20140207", + "20140218", + "20140225", + "20140301", + "20140304", + "20140313", + "20140319", + "20140324", + "20140401", + "20140402", + "20140424", + "20140515", + "20140520", + "20140529", + "20140612", + "20140616", + "20140620", + "20140804", + "20140826", + "20140915", + "20140917", + "20140918", + "20141003", + "20141024", + "20141105", + "20141114", + "20141202", + "20141216", + "20141217", + "20150114", + "20150127", + "20150213", + "20150320", + "20150325", + "20150409", + "20150421", + "20150423", + "20150504", + "20150508", + "20150520", + "20150521", + "20150522", + "20150527", + "20150609", + "20150616", + "20150701", + "20151029", + "20160112", + "20160322", + "20160408", + "20160411", + "20160412", + "20160419", + "20160502", + "20160525", + "20160531", + "20160601", + "20160607", + "20160620", + "20160624", + "20160630", + "20160704", + "20160705", + "20160728", + "20160802", + "20160803", + "20160818", + "20160823", + "20160824", + "20160825", + "20160830", + "20160831", + "20160901", + "20160902", + "20160907", + "20160912", + "20160919", + "20161011", + "20161018", + "20161020", + "20161024", + "20161027", + "20161130", + "20161201", + "20170109", + "20170113", + "20170206", + "20170207", + "20170208", + "20170301", + "20170311", + "20170314", + "20170321", + "20170403", + "20170504", + "20170508", + "20170509", + "20170523", + "20170524", + "20170606", + "20170619", + "20170620", + "20170705", + "20170713", + "20170801", + "20170804", + "20170912", + "20170914", + "20171005", + "20171009", + "20171115", + "20171212", + "20180109", + "20180117", + "20180119", + "20180129", + "20180221", + "20180226", + "20180227", + "20180228", + "20180320", + "20180321", + "20180409", + "20180417", + "20180507", + "20180509", + "20180528", + "20180614", + "20180626", + "20180707", + "20180710", + "20180717", + "20180719", + "20180813", + "20180820", + "20181008", + "20181010", + "20181017", + "20181107", + "20181126", + "20181203", + "20181211", + "20181212", + "20190102", + "20190103", + "20190108", + "20190115", + "20190121", + "20190131", + "20190208", + "20190212", + "20190220", + "20190304", + "20190410", + "20190412", + "20190415", + "20190418", + "20190419", + "20190502", + "20190509", + "20190510", + "20190512", + "20190522", + "20190601", + "20190604", + "20190619", + "20190620", + "20190625", + "20190626", + "20190702", + "20190814", + "20190820", + "20190826", + "20190828", + "20190831", + "20190919", + "20190925", + "20190926", + "20191001", + "20191004", + "20191005", + "20191014", + "20191015", + "20191017", + "20191025", + "20191028", + "20191029", + "20191030", + "20191031", + "20191101", + "20191105", + "20191116", + "20191118", + "20191119", + "20191129", + "20191130", + "2019116", + "20191201", + "20191210", + "20191211", + "20191212", + "20191217", + "20191218", + "20191219", + "20191223", + "20200102", + "20200103", + "20200106", + "20200113", + "20200114", + "20200120", + "20200204", + "20200211", + "20200219", + "20200220", + "20200221", + "20200224", + "20200229", + "20200304", + "20200306", + "20200312", + "20200327", + "20200330", + "20200424", + "20200515", + "20200518", + "20200525", + "202006", + "20200603", + "20200609", + "20200623", + "20200625", + "20200702", + "20200706", + "20200713", + "20200716", + "20200803", + "20200831", + "20200901", + "20200910", + "20200915", + "20200921", + "20200924", + "20200925", + "20200927", + "20200929", + "20201001", + "20201009", + "20201015", + "20201031", + "20201101", + "20201103", + "20201111", + "20201112", + "20201114", + "20201120", + "20201123", + "20201127", + "20201203", + "20201208", + "20201209", + "20201215", + "20210104", + "20210113", + "20210120", + "20210128", + "20210131", + "20210215", + "20210222", + "20210308", + "20210405", + "20210408", + "20210415", + "20210424", + "20210430", + "20210510", + "20210518", + "20210520" + ], + "driving_experiment": [ + "evaluation", + "historical", + "rcp26", + "rcp45", + "rcp85" + ], + "attributes": [ + "cf_standard_name", + "domain", + "driving_model", + "ensemble", + "experiment", + "experiment_family", + "institute", + "rcm_name", + "rcm_version", + "time_frequency", + "variable", + "version", + "driving_experiment"] +} diff --git a/clef/helpers.py b/clef/helpers.py index 68f6c87..da2b2de 100644 --- a/clef/helpers.py +++ b/clef/helpers.py @@ -23,7 +23,7 @@ from datetime import datetime, timedelta from .exception import ClefException -from .cordex import get_esgf_facets +#from .cordex import get_esgf_facets def get_version(path): @@ -229,11 +229,6 @@ def check_values(args, project, vocabularies): if project not in ['CMIP5', 'CMIP6', 'CORDEX']: raise ClefException(f'Query for {project} not yet implemented') facets = [v for v in get_facets(project).values() if v is not None] - # keeping this to test cmip5 and cmip6 facets when we switched to get them from esgf - #for k in facets: - # if k != 'None': - # if vocabularies[k] == []: - # print(f'facet {k} has an empty vocab') for k,v in args.items(): if k not in facets: raise ClefException(f'"{k}" is not a valid facet for project {project}') @@ -256,16 +251,14 @@ def load_vocabularies(project): """ project = project.upper() if project.split('-')[0] == 'CORDEX': - vocab = get_esgf_facets('CORDEX') - vocab['frequency'] = vocab['time_frequency'] - vocab['attributes'] = [k for k in vocab.keys()] - - else: - - vfile = pkg_resources.resource_filename(__name__, 'data/'+project+'_validation.json') - with open(vfile, 'r') as f: - data = f.read() - vocab = json.loads(data) + project = 'CORDEX' + #vocab = get_esgf_facets('CORDEX') + #vocab['frequency'] = vocab['time_frequency'] + #vocab['attributes'] = [k for k in vocab.keys()] + vfile = pkg_resources.resource_filename(__name__, 'data/'+project+'_validation.json') + with open(vfile, 'r') as f: + data = f.read() + vocab = json.loads(data) return vocab