Skip to content

Commit

Permalink
Merge pull request #151 from coecms/cordexupdate
Browse files Browse the repository at this point in the history 
cordex now get vocabularies from json file not from ESGF query
  • Loading branch information
@paolap
paolap authored Jun 28, 2021
2 parents 5504b11 + f30c179 commit aeacb08
Show file tree
Hide file tree
Showing 17 changed files with 1,198 additions and 66 deletions.
1 change: 1 addition & 0 deletions clef/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from .esdoc import citation, write_cite
import clef.cordex as cordex_


def clef_catch():
debug_logger = logging.getLogger('clef_debug')
debug_logger.setLevel(logging.CRITICAL)
Expand Down
9 changes: 2 additions & 7 deletions clef/code.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,10 +292,6 @@ def and_filter(df, cols, fixed, **kwargs):
useful = set(['version', 'source_id', 'model', 'path','dataset_id', 'domain',
'cmor_table','table_id', 'ensemble', 'member_id', 'driving_experiment',
'model_id', 'frequency', 'driving_model', 'rcm_version']) - set(fixed)
#useful = set(['version', 'source_id', 'model', 'path','dataset_id',
# 'cmor_table','table_id', 'ensemble', 'member_id', 'driving_model_id',
# 'driving_model_ensemble_member', 'model_id', 'frequency',
# 'cordex_domain', 'driving_experiment_name', 'rcm_version_id']) - set(fixed)
fields = ['comb'] + [f for f in useful if f in [c for c in df.columns.values]]
# define the aggregation dictionary
agg_dict = {k: set for k in fields}
Expand Down Expand Up @@ -425,9 +421,8 @@ def ids_df(dids):
'time_frequency', 'realm', 'cmor_table', 'ensemble', 'version']
elif project == 'cordex':
facets_list = ['project', 'product', 'domain', 'institute', 'driving_model',
'driving_experiment', 'ensemble', 'model_id', 'rcm_version', 'frequency', 'variable', 'version']
#facets_list = ['project', 'product', 'cordex_domain', 'institute_id', 'driving_model_id',
# 'driving_experiment_name', 'driving_model_ensemble_member', 'model_id', 'rcm_version_id', 'frequency', 'variable', 'version']
'driving_experiment', 'ensemble', 'model_id', 'rcm_version', 'frequency',
'variable', 'version']
else:
print(f'Warning: project {project} not available')
return results
Expand Down
10 changes: 6 additions & 4 deletions clef/cordex.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.

from clef.esgf import esgf_query
import functools
import click

from clef.esgf import esgf_query
from clef.helpers import load_vocabularies

def tidy_facet_count(v):
return v[::2]
Expand Down Expand Up @@ -60,7 +61,8 @@ def get_esgf_facets(project):
"controlled_vocab": True,
},
"version": {"short": ['-vrs'], "help": "Data publication version", "controlled_vocab": True},
"cf_standard_name": {"short": ['-cf'], "help": "CF-Conventions name of the variable"},
"cf_standard_name": {"short": ['-cf'], "help": "CF-Conventions name of the variable",
"controlled_vocab": True},
"experiment_family": {"short": ['-ef'], 'one': True, "controlled_vocab": True,
"help": "Experiment family: All, Historical, RCP"},
"institute": { "short": ['-inst'],
Expand All @@ -74,8 +76,8 @@ class CordexCommand(click.Command):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)

facets = get_esgf_facets(project="CORDEX,CORDEX-Adjust,CORDEX-ESD,CORDEXReklies")
facets['driving_experiment'] = facets['experiment']
#facets = get_esgf_facets(project="CORDEX,CORDEX-Adjust,CORDEX-ESD,CORDEXReklies")
facets = load_vocabularies('CORDEX')
facets['rcm_name'].append('CCAM-1391M')
for k, v in cli_facets.items():
opt = click.Option(
Expand Down
6 changes: 3 additions & 3 deletions clef/data/CMIP6_activity_id.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@
"VolMIP":"Volcanic Forcings Model Intercomparison Project"
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"activity_id_CV_modified":"Mon Mar 5 16:39:09 2018 -0800",
"activity_id_CV_note":"Update activity_id to include CDRMIP and PAMIP",
"author":"Paul J. Durack <[email protected]>",
"institution_id":"PCMDI",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
}
}
6 changes: 3 additions & 3 deletions clef/data/CMIP6_experiment_id.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9456,13 +9456,13 @@
}
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"experiment_id_CV_modified":"Tue Dec 15 12:25:59 2020 -0800",
"experiment_id_CV_note":"Revise experiment_id historical parent experiments",
"institution_id":"PCMDI",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
}
}
14 changes: 7 additions & 7 deletions clef/data/CMIP6_frequency.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@
"1hr":"sampled hourly",
"1hrCM":"monthly-mean diurnal cycle resolving each day into 1-hour means",
"1hrPt":"sampled hourly, at specified time point within an hour",
"3hr":"sampled every 3 hours",
"3hr":"3 hourly mean samples",
"3hrPt":"sampled 3 hourly, at specified time point within the time period",
"6hr":"sampled every 6 hours",
"6hr":"6 hourly mean samples",
"6hrPt":"sampled 6 hourly, at specified time point within the time period",
"day":"daily mean samples",
"dec":"decadal mean samples",
Expand All @@ -18,13 +18,13 @@
"yrPt":"sampled yearly, at specified time point within the time period"
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"frequency_CV_modified":"Fri Oct 27 14:03:00 2017 -0700",
"frequency_CV_note":"Issue414 durack1 revise frequency 1hrCM definition (#418)",
"frequency_CV_modified":"Mon May 24 13:48:15 2021 00100",
"frequency_CV_note":"Update description of 3hr and 6hr frequencies",
"institution_id":"PCMDI",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
}
}
6 changes: 3 additions & 3 deletions clef/data/CMIP6_grid_label.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,13 @@
"grz":"regridded zonal mean data reported on the data provider's preferred latitude target grid"
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"grid_label_CV_modified":"Fri Sep 8 18:12:00 2017 -0700",
"grid_label_CV_note":"Issue395 durack1 augment grid_label with description (#401)",
"institution_id":"PCMDI",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
}
}
6 changes: 3 additions & 3 deletions clef/data/CMIP6_institution_id.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,13 +55,13 @@
"UofT":"Department of Physics, University of Toronto, 60 St George Street, Toronto, ON M5S1A7, Canada"
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"institution_id":"PCMDI",
"institution_id_CV_modified":"Mon Nov 16 11:16:39 2020 -0800",
"institution_id_CV_note":"Register institution_id LLNL; Py3 cleanup",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
}
}
6 changes: 3 additions & 3 deletions clef/data/CMIP6_nominal_resolution.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@
"5000 km"
],
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"institution_id":"PCMDI",
"nominal_resolution_CV_modified":"Tues Nov 15 16:04:00 2016 -0700",
"nominal_resolution_CV_note":"Issue141 durack1 update grid_resolution to nominal_resolution (#143)",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
}
}
6 changes: 3 additions & 3 deletions clef/data/CMIP6_realm.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@
"seaIce":"Sea Ice"
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"institution_id":"PCMDI",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"realm_CV_modified":"Tues Apr 18 12:03:00 2017 -0700",
"realm_CV_note":"Issue285 durack1 update realm format (#290)",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
Expand Down
159 changes: 154 additions & 5 deletions clef/data/CMIP6_source_id.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4084,6 +4084,57 @@
"release_year":"2019",
"source_id":"GISS-E2-2-G"
},
"GISS-E2-2-H":{
"activity_participation":[
"CMIP",
"SIMIP",
"ScenarioMIP"
],
"cohort":[
"Registered"
],
"institution_id":[
"NASA-GISS"
],
"label":"GISS-E2.2H",
"label_extended":"GISS-E2.2H",
"model_component":{
"aerosol":{
"description":"Varies with physics-version (p==1 none, p==3 OMA, p==4 TOMAS, p==5 MATRIX)",
"native_nominal_resolution":"250 km"
},
"atmos":{
"description":"GISS-E2.2 (High Top, 2.5x2 degree; 144 x 90 longitude/latitude; 102 levels; top level 0.002 hPa)",
"native_nominal_resolution":"250 km"
},
"atmosChem":{
"description":"Varies with physics-version (p==1 Non-interactive, p>1 GPUCCINI)",
"native_nominal_resolution":"250 km"
},
"land":{
"description":"GISS LSM",
"native_nominal_resolution":"250 km"
},
"landIce":{
"description":"none",
"native_nominal_resolution":"none"
},
"ocean":{
"description":"HYCOM Ocean (~1 degree tripolar grid; 360 x 180 longitude/latitude; 32 levels; top grid cell 0-10 m)",
"native_nominal_resolution":"100 km"
},
"ocnBgchem":{
"description":"none",
"native_nominal_resolution":"none"
},
"seaIce":{
"description":"GISS SI (same grid as atmos)",
"native_nominal_resolution":"250 km"
}
},
"release_year":"2021",
"source_id":"GISS-E2-2-H"
},
"GISS-E3-G":{
"activity_participation":[
"AerChemMIP",
Expand Down Expand Up @@ -5035,6 +5086,104 @@
"release_year":"2019",
"source_id":"IPSL-CM6A-LR-INCA"
},
"IPSL-CM6A-MR025":{
"activity_participation":[
"CMIP"
],
"cohort":[
"Registered"
],
"institution_id":[
"IPSL"
],
"label":"IPSL-CM6A-MR025",
"label_extended":"IPSL-CM6A-MR025",
"model_component":{
"aerosol":{
"description":"none",
"native_nominal_resolution":"none"
},
"atmos":{
"description":"LMDZ (NPv6; 256 x 256 longitude/latitude; 79 levels; top level 80000 m)",
"native_nominal_resolution":"100 km"
},
"atmosChem":{
"description":"none",
"native_nominal_resolution":"none"
},
"land":{
"description":"ORCHIDEE (v2.2, Water/Carbon/Energy mode; same grid as atmos)",
"native_nominal_resolution":"100 km"
},
"landIce":{
"description":"none",
"native_nominal_resolution":"none"
},
"ocean":{
"description":"NEMO-OPA (eORCA025, tripolar primarily 0.25deg; 1442 x 1207 longitude/latitude; 75 levels; top grid cell 0-2 m)",
"native_nominal_resolution":"25 km"
},
"ocnBgchem":{
"description":"none",
"native_nominal_resolution":"none"
},
"seaIce":{
"description":"NEMO-LIM3 (same grid as ocean)",
"native_nominal_resolution":"25 km"
}
},
"release_year":"2021",
"source_id":"IPSL-CM6A-MR025"
},
"IPSL-CM6A-MR1":{
"activity_participation":[
"CMIP"
],
"cohort":[
"Registered"
],
"institution_id":[
"IPSL"
],
"label":"IPSL-CM6A-MR1",
"label_extended":"IPSL-CM6A-MR1",
"model_component":{
"aerosol":{
"description":"none",
"native_nominal_resolution":"none"
},
"atmos":{
"description":"LMDZ (NPv6; 256 x 256 longitude/latitude; 79 levels; top level 80000 m)",
"native_nominal_resolution":"100 km"
},
"atmosChem":{
"description":"none",
"native_nominal_resolution":"none"
},
"land":{
"description":"ORCHIDEE (v2.2, Water/Carbon/Energy mode; same grid as atmos)",
"native_nominal_resolution":"100 km"
},
"landIce":{
"description":"none",
"native_nominal_resolution":"none"
},
"ocean":{
"description":"NEMO-OPA (eORCA1.3, tripolar primarily 1deg; 362 x 332 longitude/latitude; 75 levels; top grid cell 0-2 m)",
"native_nominal_resolution":"100 km"
},
"ocnBgchem":{
"description":"NEMO-PISCES (same grid as ocean)",
"native_nominal_resolution":"100 km"
},
"seaIce":{
"description":"NEMO-LIM3 (same grid as ocean)",
"native_nominal_resolution":"100 km"
}
},
"release_year":"2021",
"source_id":"IPSL-CM6A-MR1"
},
"IPSL-CM7A-ATM-HR":{
"activity_participation":[
"HighResMIP"
Expand Down Expand Up @@ -7272,13 +7421,13 @@
}
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"institution_id":"PCMDI",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"source_id_CV_modified":"Tue Feb 16 10:44:59 2021 -0800",
"source_id_CV_note":"Revise source_id CMCC-ESM2",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"source_id_CV_modified":"Mon Jun 21 11:22:45 2021 -0700",
"source_id_CV_note":"Register source_id IPSL-CM6A-MR025",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
}
}
6 changes: 3 additions & 3 deletions clef/data/CMIP6_source_type.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@
"SLAB":"slab-ocean used with an AGCM in representing the atmosphere-ocean coupled system"
},
"version_metadata":{
"CV_collection_modified":"Tue Mar 9 11:08:57 2021 -0800",
"CV_collection_version":"6.2.55.10",
"CV_collection_modified":"Mon Jun 21 11:28:21 2021 -0700",
"CV_collection_version":"6.2.56.2",
"author":"Paul J. Durack <[email protected]>",
"institution_id":"PCMDI",
"previous_commit":"beb5c86f29724e40b3c8fc9755f321cf236230bd",
"previous_commit":"66344226746627e0b6c12db974666126c343ce27",
"source_type_CV_modified":"Fri Sep 8 17:57:00 2017 -0700",
"source_type_CV_note":"Issue396 durack1 augment source_type with description (#399)",
"specs_doc":"v6.2.7 (10th September 2018; https://goo.gl/v1drZl)"
Expand Down
Loading

0 comments on commit aeacb08

Please sign in to comment.