From 886e85253befda671fb1e40b67fa048b23683698 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 12:20:44 +1100 Subject: [PATCH 1/9] Fix WCS multi-time requests. --- datacube_ows/wcs1.py | 4 ---- datacube_ows/wcs1_utils.py | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/datacube_ows/wcs1.py b/datacube_ows/wcs1.py index 8eddb8503..626c29304 100644 --- a/datacube_ows/wcs1.py +++ b/datacube_ows/wcs1.py @@ -113,10 +113,6 @@ def desc_coverages(args): def get_coverage(args): cfg = get_config() req = WCS1GetCoverageRequest(args) - if req.format.multi_time and len(req.times) > 1: - raise WCS1Exception("Multi-time requests are not currently supported for WCS1", - WCS1Exception.INVALID_PARAMETER_VALUE, - locator="Time parameter") qprof = QueryProfiler(req.ows_stats) n_datasets, data = get_coverage_data(req, qprof) if req.ows_stats: diff --git a/datacube_ows/wcs1_utils.py b/datacube_ows/wcs1_utils.py index d213b613b..7f7490435 100644 --- a/datacube_ows/wcs1_utils.py +++ b/datacube_ows/wcs1_utils.py @@ -147,7 +147,7 @@ def __init__(self, args): locator="TIME parameter", valid_keys = [d.strftime('%Y-%m-%d') for d in self.product.ranges["time_set"]] ) - elif len(self.times) > 1 and not self.format["multi-time"]: + elif len(self.times) > 1 and not self.format.multi_time: raise WCS1Exception( "Cannot select more than one time slice with the %s format" % self.format["name"], WCS1Exception.INVALID_PARAMETER_VALUE, From 656c1744698b7cb1aaf5d69e9d7cefad750e4b1e Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 14:20:36 +1100 Subject: [PATCH 2/9] Fix WCS2 multi-time requests. --- datacube_ows/wcs2_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datacube_ows/wcs2_utils.py b/datacube_ows/wcs2_utils.py index bfafd5db3..75b69eae2 100644 --- a/datacube_ows/wcs2_utils.py +++ b/datacube_ows/wcs2_utils.py @@ -274,10 +274,10 @@ def get_coverage_data(request, qprof): qprof.end_event("fetch-datasets") if qprof.active: qprof["datasets"] = {str(q): ids for q, ids in stacker.datasets(dc.index, mode=MVSelectOpts.IDS).items()} - if fmt.multi_time and len(times) > 1: + # if fmt.multi_time and len(times) > 1: # Group by solar day - group_by = datacube.api.query.query_group_by(time=times, group_by='solar_day') - datasets = dc.group_datasets(datasets, group_by) + # group_by = datacube.api.query.query_group_by(time=times, group_by='solar_day') + # datasets = dc.group_datasets(datasets, group_by) qprof.start_event("load-data") output = stacker.data(datasets, skip_corrections=True) From ca96f60805cd3297ac1734e12cf188ce05b6b347 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 15:16:51 +1100 Subject: [PATCH 3/9] Clearer error message. --- datacube_ows/wcs2_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/datacube_ows/wcs2_utils.py b/datacube_ows/wcs2_utils.py index 75b69eae2..285c6f455 100644 --- a/datacube_ows/wcs2_utils.py +++ b/datacube_ows/wcs2_utils.py @@ -112,6 +112,11 @@ def get_coverage_data(request, qprof): dimension = subset.dimension.lower() if dimension == 'time': if isinstance(subset, Trim): + if "," in subset.high: + raise WCS2Exception( + "Subsets can only contain 2 elements - the lower and upper bounds. For arbitrary date lists, use WCS1", + WCS2Exception.INVALID_SUBSETTING, + locator="time") low = parse(subset.low).date() if subset.low is not None else None high = parse(subset.high).date() if subset.high is not None else None if low is not None: @@ -127,7 +132,6 @@ def get_coverage_data(request, qprof): elif isinstance(subset, Slice): point = parse(subset.point).date() times = [point] - else: try: if isinstance(subset, Trim): From 6e84a433d20827147e7cc737f9943d900bb5d744 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 15:22:34 +1100 Subject: [PATCH 4/9] Remove unused import. --- datacube_ows/wcs2_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datacube_ows/wcs2_utils.py b/datacube_ows/wcs2_utils.py index 285c6f455..8978e745d 100644 --- a/datacube_ows/wcs2_utils.py +++ b/datacube_ows/wcs2_utils.py @@ -8,7 +8,6 @@ import collections import logging -import datacube from datacube.utils import geometry from dateutil.parser import parse from ows.wcs.v20 import ScaleAxis, ScaleExtent, ScaleSize, Slice, Trim From fe1d5dc38efea336218de74088bc578416890e76 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 15:54:08 +1100 Subject: [PATCH 5/9] Remove un-needed commented-out code. --- datacube_ows/wcs2_utils.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datacube_ows/wcs2_utils.py b/datacube_ows/wcs2_utils.py index 8978e745d..5b9479209 100644 --- a/datacube_ows/wcs2_utils.py +++ b/datacube_ows/wcs2_utils.py @@ -277,10 +277,6 @@ def get_coverage_data(request, qprof): qprof.end_event("fetch-datasets") if qprof.active: qprof["datasets"] = {str(q): ids for q, ids in stacker.datasets(dc.index, mode=MVSelectOpts.IDS).items()} - # if fmt.multi_time and len(times) > 1: - # Group by solar day - # group_by = datacube.api.query.query_group_by(time=times, group_by='solar_day') - # datasets = dc.group_datasets(datasets, group_by) qprof.start_event("load-data") output = stacker.data(datasets, skip_corrections=True) From 9f8320099c84f98387b58eef6b0667b1cf8ffc1e Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 16:56:11 +1100 Subject: [PATCH 6/9] Attempt partial test of multidate code. --- integration_tests/test_wcs_server.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_wcs_server.py b/integration_tests/test_wcs_server.py index a0a35b98b..bd717a509 100644 --- a/integration_tests/test_wcs_server.py +++ b/integration_tests/test_wcs_server.py @@ -282,7 +282,7 @@ def test_wcs1_multi_time_exceptions(ows_server): "crs": "EPSG:4326", "format": "GeoTIFF", "bbox": extents["bbox"], - "time": ",".join(extents["times"]), + "time": extents['times'], "resx": 0.01, "resy": 0.01, }, @@ -1138,7 +1138,6 @@ def test_wcs20_getcoverage_multidate_netcdf(ows_server): scalesize="x(400),y(300)", ) - def test_wcs21_server(ows_server): # N.B. At time of writing owslib does not support WCS 2.1, so we have to make requests manually. r = requests.get( @@ -1342,6 +1341,12 @@ def test_wcs2_getcov_trim_time(ows_server): subsets = extent.raw_wcs2_subsets( ODCExtent.OFFSET_SUBSET_FOR_TIMES, ODCExtent.FIRST_TWO ) + if len(subsets[2].split(",")) == 2: + subsets = [ + subsets[0], + subsets[1], + 'time("2019-01-05","2019-01-09")' + ] r = requests.get( ows_server.url + "/wcs", From 6e5a0481de20dd9113fb49ee2396dc0b03c489d4 Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 19:41:27 +1100 Subject: [PATCH 7/9] At least cover the trim handling code. --- integration_tests/test_wcs_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/test_wcs_server.py b/integration_tests/test_wcs_server.py index bd717a509..6f7e1d7f8 100644 --- a/integration_tests/test_wcs_server.py +++ b/integration_tests/test_wcs_server.py @@ -1341,7 +1341,7 @@ def test_wcs2_getcov_trim_time(ows_server): subsets = extent.raw_wcs2_subsets( ODCExtent.OFFSET_SUBSET_FOR_TIMES, ODCExtent.FIRST_TWO ) - if len(subsets[2].split(",")) == 2: + if len(subsets[2].split(",")) == 1: subsets = [ subsets[0], subsets[1], From 21c16a6fba777862276b71d4f2c4e8d0aa1d24da Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 20:02:30 +1100 Subject: [PATCH 8/9] Check error message for subset with >2 times. --- integration_tests/test_wcs_server.py | 36 +++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/integration_tests/test_wcs_server.py b/integration_tests/test_wcs_server.py index 6f7e1d7f8..0553a011f 100644 --- a/integration_tests/test_wcs_server.py +++ b/integration_tests/test_wcs_server.py @@ -1360,9 +1360,43 @@ def test_wcs2_getcov_trim_time(ows_server): "scalesize": "x(400),y(400)", "subset": subsets, }, - ) + ) assert r.status_code == 200 +def test_wcs2_getcov_badtrim_time(ows_server): + cfg = get_config(refresh=True) + layer = None + for lyr in cfg.product_index.values(): + if lyr.ready and not lyr.hide: + layer = lyr + break + assert layer + extent = ODCExtent(layer) + subsets = extent.raw_wcs2_subsets( + ODCExtent.OFFSET_SUBSET_FOR_TIMES, ODCExtent.FIRST_TWO + ) + subsets = [ + subsets[0], + subsets[1], + 'time("2019-01-05","2019-01-09","2019-01-15")' + ] + + check_wcs_error( + ows_server.url + "/wcs", + params={ + "request": "GetCoverage", + "coverageid": layer.name, + "version": "2.1.0", + "service": "WCS", + "format": "application/x-netcdf", + "subsettingcrs": "EPSG:4326", + "scalesize": "x(400),y(400)", + "subset": subsets, + }, + expected_error_message="Subsets can only contain 2 elements - the lower and upper bounds. For arbitrary date lists, use WCS1", + expected_status_code=400, + ) + def test_wcs2_getcov_slice_space(ows_server): cfg = get_config(refresh=True) From f292e611dbbf34c831e6855238df98aecb7e2fce Mon Sep 17 00:00:00 2001 From: Paul Haesler Date: Thu, 3 Feb 2022 20:41:53 +1100 Subject: [PATCH 9/9] indentation fixes for flake8 --- integration_tests/test_wcs_server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/test_wcs_server.py b/integration_tests/test_wcs_server.py index 0553a011f..89de92117 100644 --- a/integration_tests/test_wcs_server.py +++ b/integration_tests/test_wcs_server.py @@ -1360,7 +1360,7 @@ def test_wcs2_getcov_trim_time(ows_server): "scalesize": "x(400),y(400)", "subset": subsets, }, - ) + ) assert r.status_code == 200 def test_wcs2_getcov_badtrim_time(ows_server): @@ -1395,7 +1395,7 @@ def test_wcs2_getcov_badtrim_time(ows_server): }, expected_error_message="Subsets can only contain 2 elements - the lower and upper bounds. For arbitrary date lists, use WCS1", expected_status_code=400, - ) + ) def test_wcs2_getcov_slice_space(ows_server):