Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create hourly product in moorings products pipeline #206

Merged
merged 11 commits into from
Feb 24, 2020
16 changes: 14 additions & 2 deletions aodndata/moorings/products_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@


PRODUCT_TYPE_PATTERN = re.compile(r'FV0[12]_([^_]+)_END')
VALID_PRODUCTS = {'aggregated', 'hourly'}


def get_product_type(file_path):
Expand Down Expand Up @@ -109,6 +110,7 @@ def __init__(self, *args, **kwargs):
self.allowed_extensions = ['.json_manifest', '.nc', '.zip']
self.product_site_code = None
self.product_variables = None
self.products_to_create = VALID_PRODUCTS
mhidas marked this conversation as resolved.
Show resolved Hide resolved
self.input_file_collection = None
self.input_file_variables = None
self.excluded_files = dict()
Expand All @@ -126,6 +128,14 @@ def _read_manifest(self):
raise InvalidFileContentError(
"manifest file '{self.input_file}' missing information (site_code, variables)".format(self=self)
)
if 'products' in manifest:
mhidas marked this conversation as resolved.
Show resolved Hide resolved
invalid_products = set(manifest['products']) - VALID_PRODUCTS
if invalid_products:
raise InvalidFileContentError(
"invalid product(s) {invalid_products} requested "
"in manifest file '{self.input_file}'".format(invalid_products=invalid_products, self=self)
)
self.products_to_create = set(manifest['products'])

def get_wfs_features(self, filter_list, propertyname='*'):
"""Query the file index WFS layer with the given filters and return a list of features.
Expand Down Expand Up @@ -292,8 +302,10 @@ def preprocess(self):

# TODO: Run compliance checks and remove non-compliant files from the input list (log them).

self._make_aggregated_timeseries()
self._make_hourly_timeseries()
if 'aggregated' in self.products_to_create:
self._make_aggregated_timeseries()
if 'hourly' in self.products_to_create:
self._make_hourly_timeseries()

# TODO: Include the list of excluded files as another table in the notification email (instead of the log)
if self.excluded_files:
Expand Down
3 changes: 2 additions & 1 deletion test_aodndata/moorings/test_product.json_manifest
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"site_code": "NRSROT",
"variables": ["TEMP", "PSAL", "CHLF"]
"variables": ["TEMP", "PSAL", "CHLF"],
"products":["aggregated", "hourly"]
}