diff --git a/catalogbuilder/intakebuilder/builderconfig.py b/catalogbuilder/intakebuilder/builderconfig.py index 0ca71ae..d6c9ad1 100644 --- a/catalogbuilder/intakebuilder/builderconfig.py +++ b/catalogbuilder/intakebuilder/builderconfig.py @@ -15,7 +15,7 @@ headerlist = ["activity_id", "institution_id", "source_id", "experiment_id", "frequency", "realm", "table_id", "member_id", "grid_label", "variable_id", - "time_range", "chunk_freq","grid_label","platform","dimensions","cell_methods","path"] + "time_range", "chunk_freq","grid_label","platform","dimensions","cell_methods","standard_name","path"] #what kind of directory structure to expect? #For a directory structure like /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp diff --git a/catalogbuilder/intakebuilder/gfdlcrawler.py b/catalogbuilder/intakebuilder/gfdlcrawler.py index 4071040..00455f2 100644 --- a/catalogbuilder/intakebuilder/gfdlcrawler.py +++ b/catalogbuilder/intakebuilder/gfdlcrawler.py @@ -25,7 +25,6 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow): headerlist = configyaml.headerlist else: headerlist = builderconfig.headerlist - #For those columns that we cannot find in output path template or output file template from config yaml, we have hooks #now to look up the netcdf dataset if slow option is True #todo catch exceptions upon furhter testing @@ -38,8 +37,18 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow): if (configyaml.output_path_template is not None) & (configyaml.output_file_template is not None) : list_ptemplate = configyaml.output_path_template list_ftemplate = configyaml.output_file_template - set_ptemplate = set(list_ptemplate) - set_ftemplate = set(list_ftemplate) + else: + #if it is none, the user is likely using default config which may be phased out, or redesigned to use a config template json rather than builderconfig + try: + list_ptemplate = builderconfig.output_path_template + except: + sys.exit("output_path_template is not set. Check your configuration") + try: + list_ftemplate = builderconfig.output_file_template + except: + sys.exit("output_file_template is not set. Check your configuration") + set_ptemplate = set(list_ptemplate) + set_ftemplate = set(list_ftemplate) #print(headerlist) #print(list_ptemplate) #print(list_ftemplate)