add option to select which products to generate (default all)

aodn · Feb 6, 2020 · b6d04f9 · b6d04f9
1 parent d79a6b1
commit b6d04f9
Showing 1 changed file with 14 additions and 2 deletions.
diff --git a/aodndata/moorings/products_handler.py b/aodndata/moorings/products_handler.py
@@ -17,6 +17,7 @@
 
 
 PRODUCT_TYPE_PATTERN = re.compile(r'FV0[12]_([^_]+)_END')
+VALID_PRODUCTS = ['aggregated', 'hourly']
 
 
 def get_product_type(file_path):
@@ -109,6 +110,7 @@ def __init__(self, *args, **kwargs):
         self.allowed_extensions = ['.json_manifest', '.nc', '.zip']
         self.product_site_code = None
         self.product_variables = None
+        self.products_to_create = VALID_PRODUCTS
         self.input_file_collection = None
         self.input_file_variables = None
         self.excluded_files = dict()
@@ -126,6 +128,14 @@ def _read_manifest(self):
             raise InvalidFileContentError(
                 "manifest file '{self.input_file}' missing information (site_code, variables)".format(self=self)
             )
+        if 'products' in manifest:
+            invalid_products = set(manifest['products']) - set(VALID_PRODUCTS)
+            if invalid_products:
+                raise InvalidFileContentError(
+                    "invalid product(s) {invalid_products} requested "
+                    "in manifest file '{self.input_file}'".format(invalid_products=invalid_products, self=self)
+                )
+            self.products_to_create = manifest['products']
 
     def get_wfs_features(self, filter_list, propertyname='*'):
         """Query the file index WFS layer with the given filters and return a list of features.
@@ -292,8 +302,10 @@ def preprocess(self):
 
         # TODO: Run compliance checks and remove non-compliant files from the input list (log them).
 
-        self._make_aggregated_timeseries()
-        self._make_hourly_timeseries()
+        if 'aggregated' in self.products_to_create:
+            self._make_aggregated_timeseries()
+        if 'hourly' in self.products_to_create:
+            self._make_hourly_timeseries()
 
         # TODO: Include the list of excluded files as another table in the notification email (instead of the log)
         if self.excluded_files: