From d94aabe63ccf49c8d95429fc0e2f3edec549dd0c Mon Sep 17 00:00:00 2001 From: Alice Bertini Date: Sun, 11 Dec 2016 12:22:07 -0700 Subject: [PATCH] - update ice and lnd avg generator to allow netcdf_format XML input - bug fix for ice diags_bc.py "ice_vol*" symlink - add valid_values to env_*.xml description comment - change all postprocessing default queues to geyser in preparation for CMIP6 runs where allocation on YS will be needed for the runs. Test suite: for ice diags tested with SO2_feedbackrun1_2020-2100_serial.001 Fixes [Github issue #]: Issue #39 User interface changes: None Input data changes: None Code review: --- Config/config_postprocess.xml | 20 +++--- Machines/machine_postprocess.xml | 12 ++-- Templates/env_diags.tmpl | 3 +- Templates/env_postprocess.tmpl | 2 +- cesm_utils/cesm_utils/create_postprocess | 56 +++++++++------ .../ice/Config/config_diags_ice.xml | 10 +++ .../diagnostics/ice/ice_avg_generator.py | 45 +++++++----- diagnostics/diagnostics/ice/ice_diags_bc.py | 70 ++++++++++--------- .../lnd/Config/config_diags_lnd.xml | 10 +++ .../diagnostics/lnd/lnd_avg_generator.py | 44 ++++++++---- 10 files changed, 170 insertions(+), 102 deletions(-) diff --git a/Config/config_postprocess.xml b/Config/config_postprocess.xml index c2df49b0..524cd247 100644 --- a/Config/config_postprocess.xml +++ b/Config/config_postprocess.xml @@ -53,7 +53,7 @@ - 128 - 16 + 16 + 16 6 /glade/p/cesm/amwg/amwg_data - 128 + 16 4 /glade/p/cesm/pcwg/ice/data - 128 + 16 12 6 /glade/p/cesm/lmwg/diag/lnd_diag_data - 128 + 16 16 - /glade/p/cesm + /glade/p/cesm/ diff --git a/Templates/env_diags.tmpl b/Templates/env_diags.tmpl index 75bb6bd4..90f10c7f 100644 --- a/Templates/env_diags.tmpl +++ b/Templates/env_diags.tmpl @@ -23,7 +23,8 @@ {% for xml in v %} - + + {% endfor %} {% endif %} diff --git a/Templates/env_postprocess.tmpl b/Templates/env_postprocess.tmpl index 7291717a..6027400e 100644 --- a/Templates/env_postprocess.tmpl +++ b/Templates/env_postprocess.tmpl @@ -22,7 +22,7 @@ {% for xml in v %} - + {% endfor %} {% endif %} diff --git a/cesm_utils/cesm_utils/create_postprocess b/cesm_utils/cesm_utils/create_postprocess index 786bd9bd..3d2a5d5e 100755 --- a/cesm_utils/cesm_utils/create_postprocess +++ b/cesm_utils/cesm_utils/create_postprocess @@ -94,10 +94,11 @@ import jinja2 # define a class to be used for the xml entry, id, desc values # ------------------------------------------------------------------------------- class XmlEntry(object): - def __init__(self, id, value, desc): + def __init__(self, id, value, desc, valid_values): self._id = id self._value = value self._desc = desc + self._valid_values = valid_values def id(self): return self._id @@ -108,6 +109,9 @@ class XmlEntry(object): def desc(self): return self._desc + def valid_values(self): + return self._valid_values + # ------------------------------------------------------------------------------- # commandline_options - parse any command line options # ------------------------------------------------------------------------------- @@ -204,9 +208,11 @@ def create_env_file(envDict, configFile, tmplFile, envFile, obs_root, comp, stan for entry_tag in group_tag.findall('entry'): # check if the value needs to be inherited from the envDict if entry_tag.get('value') == 'inherit': - xml_list.append(XmlEntry(entry_tag.get('id'), envDict[entry_tag.get('id')], entry_tag.get('desc'))) + xml_list.append(XmlEntry(entry_tag.get('id'), envDict[entry_tag.get('id')], + entry_tag.get('desc'), entry_tag.get('valid_values'))) else: - xml_list.append(XmlEntry(entry_tag.get('id'), entry_tag.get('value'), entry_tag.get('desc'))) + xml_list.append(XmlEntry(entry_tag.get('id'), entry_tag.get('value'), + entry_tag.get('desc'), entry_tag.get('valid_values'))) group_dict = { 'order' : order, 'name' : name, 'comment' : comment, 'xml_list' : xml_list } group_list.append(group_dict) @@ -217,9 +223,8 @@ def create_env_file(envDict, configFile, tmplFile, envFile, obs_root, comp, stan xml_list = list() if obs_root: if len(obs_root) > 0: - xml_obs = XmlEntry('{0}DIAG_DIAGOBSROOT'.format(comp.upper()), - obs_root, - 'Machine dependent diagnostics observation files root path') + xml_obs = XmlEntry('{0}DIAG_DIAGOBSROOT'.format(comp.upper()), obs_root, + 'Machine dependent diagnostics observation files root path', '') xml_list.append(xml_obs) # the xml_list now contains a list of XmlEntry classes that can be written to the template @@ -562,7 +567,8 @@ def main(options): config_file = '{0}/diagnostics/diagnostics/{1}/Config/config_diags_{1}.xml'.format(envDict['POSTPROCESS_PATH'], comp) tmpl_file = 'env_diags.tmpl' env_file = '{0}/env_diags_{1}.xml'.format(envDict['PP_CASE_PATH'], comp) - create_env_file(envDict, config_file, tmpl_file, env_file, machine['{0}_obs_root'.format(comp)], comp, standalone) + create_env_file(envDict, config_file, tmpl_file, env_file, machine['{0}_obs_root'.format(comp)], + comp, standalone) # define the template files for the batch scripts batch_tmpl = 'batch_{0}.tmpl'.format(envDict['MACH']) @@ -575,8 +581,9 @@ def main(options): create_batch(envDict['POSTPROCESS_PATH'], machine['timeseries_pes'], batch_tmpl, run_tmpl, postProcessCmd, machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'], - envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['timeseries_queue'], - machine['timeseries_ppn'], machine['timeseries_wallclock'], options, standalone) + envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], + machine['timeseries_queue'], machine['timeseries_ppn'], machine['timeseries_wallclock'], + options, standalone) # generate the diagnostics batch submit scripts from template files for comp in compList: @@ -584,19 +591,24 @@ def main(options): postProcessCmd = '{0}_avg_generator.py'.format(comp) processName = '{0}_averages'.format(comp) outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'], processName) - create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_averages_pes'.format(comp)], batch_tmpl, run_tmpl, postProcessCmd, - machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'], - envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['{0}_averages_queue'.format(comp)], - machine['{0}_averages_ppn'.format(comp)], machine['{0}_averages_wallclock'.format(comp)], options, standalone) + create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_averages_pes'.format(comp)], batch_tmpl, + run_tmpl, postProcessCmd, machine['mpi_command'], outFile, processName, + machine['project'], machine['pythonpath'], envDict['PP_CASE_PATH'], + machine['reset_modules'], machine['modules'], machine['{0}_averages_queue'.format(comp)], + machine['{0}_averages_ppn'.format(comp)], machine['{0}_averages_wallclock'.format(comp)], + options, standalone) # generate the diagnostics batch submit script postProcessCmd = '{0}_diags_generator.py'.format(comp) processName = '{0}_diagnostics'.format(comp) outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'], processName) - create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_diagnostics_pes'.format(comp)], batch_tmpl, run_tmpl, postProcessCmd, - machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'], - envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['{0}_diagnostics_queue'.format(comp)], - machine['{0}_diagnostics_ppn'.format(comp)], machine['{0}_diagnostics_wallclock'.format(comp)], options, standalone) + create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_diagnostics_pes'.format(comp)], batch_tmpl, + run_tmpl, postProcessCmd, machine['mpi_command'], outFile, processName, + machine['project'], machine['pythonpath'], envDict['PP_CASE_PATH'], + machine['reset_modules'], machine['modules'], + machine['{0}_diagnostics_queue'.format(comp)], + machine['{0}_diagnostics_ppn'.format(comp)], + machine['{0}_diagnostics_wallclock'.format(comp)], options, standalone) # generate the regrid batch submit scripts from template files for comp in regridList: @@ -604,10 +616,12 @@ def main(options): postProcessCmd = '{0}_regrid_generator.py'.format(comp) processName = '{0}_regrid'.format(comp) outFile = '{0}/{1}'.format(envDict['PP_CASE_PATH'], processName) - create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_regrid_pes'.format(comp)], batch_tmpl, run_tmpl, postProcessCmd, - machine['mpi_command'], outFile, processName, machine['project'], machine['pythonpath'], - envDict['PP_CASE_PATH'], machine['reset_modules'], machine['modules'], machine['{0}_regrid_queue'.format(comp)], - machine['{0}_regrid_ppn'.format(comp)], machine['{0}_regrid_wallclock'.format(comp)], options, standalone) + create_batch(envDict['POSTPROCESS_PATH'], machine['{0}_regrid_pes'.format(comp)], batch_tmpl, + run_tmpl, postProcessCmd, machine['mpi_command'], outFile, processName, + machine['project'], machine['pythonpath'], envDict['PP_CASE_PATH'], + machine['reset_modules'], machine['modules'], machine['{0}_regrid_queue'.format(comp)], + machine['{0}_regrid_ppn'.format(comp)], machine['{0}_regrid_wallclock'.format(comp)], + options, standalone) # copy some tools to the PP_CASE_PATH files_to_copy = ['copy_html', 'pp_config', 'create_env_script', 'env_file.xsd'] diff --git a/diagnostics/diagnostics/ice/Config/config_diags_ice.xml b/diagnostics/diagnostics/ice/Config/config_diags_ice.xml index 7f7d6579..d8f79051 100644 --- a/diagnostics/diagnostics/ice/Config/config_diags_ice.xml +++ b/diagnostics/diagnostics/ice/Config/config_diags_ice.xml @@ -45,6 +45,16 @@ desc="Controls whether to generate model vs. model diagnostics" > + + 0: # call the pyAverager with the inVarList - callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile, out_dir, case_prefix, averageList, inVarList, - envDict, stream, reg_file, year0, year1, split, split_size, main_comm, debugMsg) + callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile, out_dir, + case_prefix, averageList, inVarList, envDict, stream, reg_file, year0, + year1, split, split_size, netcdf_format, main_comm, debugMsg) #============================================ @@ -271,9 +282,6 @@ def initialize_envDict(envDict, caseroot, debugMsg, standalone): envDict (dictionary) - environment dictionary """ # setup envDict['id'] = 'value' parsed from the CASEROOT/[env_file_list] files - # TODO put this file list into the config_postprocess definition -## env_file_list = ['../env_case.xml', '../env_run.xml', '../env_build.xml', '../env_mach_pes.xml', './env_postprocess.xml', './env_diags_ice.xml'] -## if standalone: env_file_list = ['./env_postprocess.xml', './env_diags_ice.xml'] envDict = cesmEnvLib.readXML(caseroot, env_file_list) @@ -288,7 +296,7 @@ def initialize_envDict(envDict, caseroot, debugMsg, standalone): # add the os.environ['PATH'] to the envDict['PATH'] envDict['ICEDIAG_PATH'] = os.pathsep + os.environ['PATH'] - # strip the ATMDIAG_ prefix from the envDict entries before setting the + # strip the ICEDIAG_ prefix from the envDict entries before setting the # enviroment to allow for compatibility with all the diag routine calls envDict = diagUtilsLib.strip_prefix(envDict, 'ICEDIAG_') @@ -379,8 +387,11 @@ def main(options, main_comm, debugMsg): avg_BEGYR = (int(envDict['ENDYR_CONT']) - int(envDict['YRS_TO_AVG'])) + 1 createClimFiles(avg_BEGYR, envDict['ENDYR_CONT'], h_path, split, split_size, - envDict['cont_htype'], envDict['cont_key_infile'], envDict['PATH_CLIMO_CONT'], envDict['CASE_TO_CONT'], - 'cice.h', varList, envDict, envDict['GRIDFILECONT'], envDict['BEGYR_CONT'], envDict['ENDYR_CONT'], main_comm, debugMsg) + envDict['cont_htype'], envDict['cont_key_infile'], + envDict['PATH_CLIMO_CONT'], envDict['CASE_TO_CONT'], + 'cice.h', varList, envDict, envDict['GRIDFILECONT'], + envDict['BEGYR_CONT'], envDict['ENDYR_CONT'], + envDict['netcdf_format'], main_comm, debugMsg) except Exception as error: print(str(error)) traceback.print_exc() @@ -411,8 +422,11 @@ def main(options, main_comm, debugMsg): avg_BEGYR_DIFF = (int(envDict['ENDYR_DIFF']) - int(envDict['YRS_TO_AVG'])) + 1 createClimFiles(avg_BEGYR_DIFF, envDict['ENDYR_DIFF'], h_path, split, split_size, - envDict['diff_htype'], envDict['diff_key_infile'], envDict['PATH_CLIMO_DIFF'], envDict['CASE_TO_DIFF'], - 'cice.h', varList, envDict, envDict['GRIDFILEDIFF'], envDict['BEGYR_DIFF'], envDict['ENDYR_DIFF'], main_comm, debugMsg) + envDict['diff_htype'], envDict['diff_key_infile'], + envDict['PATH_CLIMO_DIFF'], envDict['CASE_TO_DIFF'], + 'cice.h', varList, envDict, envDict['GRIDFILEDIFF'], + envDict['BEGYR_DIFF'], envDict['ENDYR_DIFF'], + envDict['netcdf_format'], main_comm, debugMsg) except Exception as error: print(str(error)) traceback.print_exc() @@ -441,9 +455,6 @@ def main(options, main_comm, debugMsg): print(' Successfully completed generating ice climatology averages') print('*************************************************************') sys.exit(status) - -## except RunTimeError as error: - except Exception as error: print(str(error)) if options.backtrace: diff --git a/diagnostics/diagnostics/ice/ice_diags_bc.py b/diagnostics/diagnostics/ice/ice_diags_bc.py index 4fa77575..c13d8cd8 100755 --- a/diagnostics/diagnostics/ice/ice_diags_bc.py +++ b/diagnostics/diagnostics/ice/ice_diags_bc.py @@ -74,44 +74,50 @@ def setup_workdir(self, env, t, scomm): if (scomm.is_manager()): climo_files = glob.glob(old_workdir+'/*.nc') for climo_file in climo_files: - name_split = climo_file.split('.') # Split on '.' - if ('-' in name_split[-3]): - fn = str.join('.',name_split[:len(name_split)-3] + name_split[-2:]) #Piece together w/o the date, but still has old path + if ('ice_vol_' in climo_file): + new_fn = workdir + '/' + os.path.basename(climo_file) ## if (scomm.is_manager()): -## print('0. fn = {0}'.format(fn)) - path_split = fn.split('/') # Remove the path - if ('jfm_climo' in path_split[-1]): - s = 'jfm' - elif ('amj_climo' in path_split[-1]): - s = 'amj' - elif ('jas_climo' in path_split[-1]): - s = 'jas' - elif ('ond_climo' in path_split[-1]): - s = 'ond' - elif ('fm_climo' in path_split[-1]): - s = 'fm' - elif ('on_climo' in path_split[-1]): - s = 'on' - elif ('_ANN_climo' in path_split[-1]): - s = 'ann' - else: - s = None - if s is not None: - new_fn = workdir + '/' + s + '_avg_' + str(avg_BEGYR).zfill(4) + '-' + env['ENDYR_'+t].zfill(4) + '.nc' -## if (scomm.is_manager()): -## print('1. ice_diags_bc.py s = {0}: new_fn = {1}'.format(s, new_fn)) +## print('1. ice_diags_bc.py: new_fn = {0}'.format(new_fn)) + else: + name_split = climo_file.split('.') # Split on '.' + if ('-' in name_split[-3]): + fn = str.join('.',name_split[:len(name_split)-3] + name_split[-2:]) #Piece together w/o the date, but still has old path + if (scomm.is_manager()): + print('1. fn = {0}'.format(fn)) + path_split = fn.split('/') # Remove the path + if ('jfm_climo' in path_split[-1]): + s = 'jfm' + elif ('amj_climo' in path_split[-1]): + s = 'amj' + elif ('jas_climo' in path_split[-1]): + s = 'jas' + elif ('ond_climo' in path_split[-1]): + s = 'ond' + elif ('fm_climo' in path_split[-1]): + s = 'fm' + elif ('on_climo' in path_split[-1]): + s = 'on' + elif ('_ANN_climo' in path_split[-1]): + s = 'ann' + else: + s = None + if s is not None: + new_fn = workdir + '/' + s + '_avg_' + str(avg_BEGYR).zfill(4) + '-' + env['ENDYR_'+t].zfill(4) + '.nc' +## if (scomm.is_manager()): +## print('2. ice_diags_bc.py s = {0}: new_fn = {1}'.format(s, new_fn)) + else: + new_fn = workdir + '/' +path_split[-1] # Take file name and add it to new path +## if (scomm.is_manager()): +## print('3. ice_diags_bc.py: new_fn = {0}'.format(new_fn)) else: - new_fn = workdir + '/' +path_split[-1] # Take file name and add it to new path + new_fn = workdir + '/' + os.path.basename(climo_file) ## if (scomm.is_manager()): -## print('2. ice_diags_bc.py: new_fn = {0}'.format(new_fn)) - else: - new_fn = workdir + '/' + os.path.basename(climo_file) -## if (scomm.is_manager()): -## print('3. ice_diags_bc.py: new_fn = {0}'.format(new_fn)) - +## print('4. ice_diags_bc.py: new_fn = {0}'.format(new_fn)) rc1, err_msg1 = cesmEnvLib.checkFile(new_fn, 'read') if not rc1: os.symlink(climo_file,new_fn) + else: + print('ice_diags_bc.py: unable to create link to file {0}'.format(new_fn)) return env def check_prerequisites(self, env, scomm): diff --git a/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml b/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml index 7aaaf658..d7386947 100644 --- a/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml +++ b/diagnostics/diagnostics/lnd/Config/config_diags_lnd.xml @@ -191,6 +191,16 @@ desc="Controls whether to generate model vs. model diagnostics" > + + 0: # call the pyAverager with the inVarList - callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile, out_dir, case_prefix, averageList, inVarList, - envDict, stream, trend_year0, trend_year1, main_comm, debugMsg) + callPyAverager(avg_start_year, avg_stop_year, in_dir, htype, key_infile, + out_dir, case_prefix, averageList, inVarList, + envDict, stream, trend_year0, trend_year1, + netcdf_format, main_comm, debugMsg) #============================================ @@ -394,9 +406,11 @@ def main(options, main_comm, debugMsg): case1_climo_dir = envDict['PTMPDIR_1']+'/climo/'+ envDict['caseid_1']+'/' createClimFiles(envDict['clim_first_yr_1'], envDict['clim_last_yr_1'], h_path, - envDict['case1_htype'], envDict['case1_key_infile'], case1_climo_dir, envDict['caseid_1'], - suffix, varList, envDict, envDict['trends_first_yr_1'], envDict['trends_last_yr_1'], - envDict['climo_'+model+'_1'], envDict['trends_'+model+'_1'], model, main_comm, debugMsg) + envDict['case1_htype'], envDict['case1_key_infile'], case1_climo_dir, + envDict['caseid_1'], suffix, varList, envDict, + envDict['trends_first_yr_1'], envDict['trends_last_yr_1'], + envDict['climo_'+model+'_1'], envDict['trends_'+model+'_1'], model, + envDict['netcdf_large'], main_comm, debugMsg) except Exception as error: print(str(error)) traceback.print_exc() @@ -446,9 +460,11 @@ def main(options, main_comm, debugMsg): case2_climo_dir = envDict['PTMPDIR_2']+'/climo/'+ envDict['caseid_2']+'/' createClimFiles(envDict['clim_first_yr_2'], envDict['clim_last_yr_2'], h_path, - envDict['case2_htype'], envDict['case2_key_infile'], case2_climo_dir, envDict['caseid_2'], - suffix, varList, envDict, envDict['trends_first_yr_2'], envDict['trends_last_yr_2'], - envDict['climo_'+model+'_2'], envDict['trends_'+model+'_2'], model, main_comm, debugMsg) + envDict['case2_htype'], envDict['case2_key_infile'], case2_climo_dir, + envDict['caseid_2'], suffix, varList, envDict, + envDict['trends_first_yr_2'], envDict['trends_last_yr_2'], + envDict['climo_'+model+'_2'], envDict['trends_'+model+'_2'], model, + envDict['netcdf_format'], main_comm, debugMsg) except Exception as error: print(str(error)) traceback.print_exc()