Merge pull request #65 from aradhakrishnanGFDL/64-dontskip

64 dontskip

catalogbuilder/intakebuilder/builderconfig.py

@@ -15,7 +15,7 @@
 headerlist = ["activity_id", "institution_id", "source_id", "experiment_id",
                   "frequency", "realm", "table_id",
                   "member_id", "grid_label", "variable_id",
-                  "time_range", "chunk_freq","grid_label","platform","dimensions","cell_methods","path"]
+                  "time_range", "chunk_freq","grid_label","platform","dimensions","cell_methods","standard_name","path"]
 
 #what kind of directory structure to expect?
 #For a directory structure like /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp

catalogbuilder/intakebuilder/gfdlcrawler.py

@@ -25,7 +25,6 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow):
        headerlist = configyaml.headerlist
     else:
        headerlist = builderconfig.headerlist
-
     #For those columns that we cannot find in output path template or output file template from config yaml, we have hooks
     #now to look up the netcdf dataset if slow option is True
     #todo catch exceptions upon furhter testing
@@ -38,8 +37,18 @@ def crawlLocal(projectdir, dictFilter,dictFilterIgnore,logger,configyaml,slow):
         if (configyaml.output_path_template is not None) & (configyaml.output_file_template is not None) :
           list_ptemplate = configyaml.output_path_template
           list_ftemplate = configyaml.output_file_template
-        set_ptemplate = set(list_ptemplate)
-        set_ftemplate = set(list_ftemplate)
+    else:
+        #if it is none, the user is likely using default config which may be phased out, or redesigned to use a config template json rather than builderconfig
+        try:
+           list_ptemplate = builderconfig.output_path_template
+        except:
+           sys.exit("output_path_template is not set. Check your configuration")
+        try:
+           list_ftemplate = builderconfig.output_file_template
+        except:
+           sys.exit("output_file_template is not set. Check your configuration")
+    set_ptemplate = set(list_ptemplate)
+    set_ftemplate = set(list_ftemplate)
     #print(headerlist)
     #print(list_ptemplate)
     #print(list_ftemplate)