From 95d2f27113d72a30280818bf156f0ef33331ad3d Mon Sep 17 00:00:00 2001 From: James Date: Fri, 15 Dec 2023 10:28:39 -0500 Subject: [PATCH 1/9] LIKE wildcard search --- .../stac_fastapi/elasticsearch/app.py | 6 +- .../elasticsearch/extensions/filter.py | 43 ++++++++- .../tests/extensions/cql2/example19.json | 9 ++ .../tests/extensions/cql2/example20.json | 10 ++ .../tests/extensions/test_filter.py | 94 +++++++++++++++++++ 5 files changed, 160 insertions(+), 2 deletions(-) create mode 100755 stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json create mode 100755 stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index e3c4bc64..c7569e87 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -19,11 +19,15 @@ TokenPaginationExtension, TransactionExtension, ) + from stac_fastapi.extensions.third_party import BulkTransactionExtension settings = ElasticsearchSettings() session = Session.create_from_settings(settings) +filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient()) +filter_extension.conformance_classes.append("http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators") + extensions = [ TransactionExtension(client=TransactionsClient(session=session), settings=settings), BulkTransactionExtension(client=BulkTransactionsClient(session=session)), @@ -32,7 +36,7 @@ SortExtension(), TokenPaginationExtension(), ContextExtension(), - FilterExtension(client=EsAsyncBaseFiltersClient()), + filter_extension ] post_request_model = create_post_request_model(extensions) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py index 6cc4ac28..690f9ddc 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py @@ -78,6 +78,15 @@ def to_es(self): ) +class AdvancedComparisonOp(str, Enum): + """Advanced Comparison operator. + + CQL2 advanced comparison operator like (~). + """ + + like = "like" + + class SpatialIntersectsOp(str, Enum): """Spatial intersections operator s_intersects.""" @@ -152,7 +161,7 @@ def validate(cls, v): class Clause(BaseModel): """Filter extension clause.""" - op: Union[LogicalOp, ComparisonOp, SpatialIntersectsOp] + op: Union[LogicalOp, ComparisonOp, AdvancedComparisonOp, SpatialIntersectsOp] args: List[Arg] def to_es(self): @@ -171,6 +180,16 @@ def to_es(self): "must_not": [{"term": {to_es(self.args[0]): to_es(self.args[1])}}] } } + elif self.op == AdvancedComparisonOp.like: + return { + "wildcard": { + to_es(self.args[0]): { + "value": cql2_like_to_es(str(to_es(self.args[1]))), + "boost": 1.0, + "case_insensitive": "true" + } + } + } elif ( self.op == ComparisonOp.lt or self.op == ComparisonOp.lte @@ -210,3 +229,25 @@ def to_es(arg: Arg): return arg else: raise RuntimeError(f"unknown arg {repr(arg)}") + + +def cql2_like_to_es(input_string): + """ + Convert arugument in CQL2 ('_' and '%') to Elasticsearch wildcard operators ('?' and '*', respectively). Handle escape characters and + handle Elasticsearch wildcards directly. + """ + es_string = "" + escape = False + + for char in input_string: + if char == "\\": + escape = True + elif char == '_' and not escape: + es_string += '?' + elif char == '%' and not escape: + es_string += '*' + else: + es_string += char + escape = False + + return es_string diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json b/stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json new file mode 100755 index 00000000..0e4306fb --- /dev/null +++ b/stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json @@ -0,0 +1,9 @@ +{ + "op": "like", + "args": [ + { + "property": "scene_id" + }, + "LC82030282019133%" + ] +} diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json b/stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json new file mode 100755 index 00000000..f7412fc0 --- /dev/null +++ b/stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json @@ -0,0 +1,10 @@ +{ + "op": "like", + "args": [ + { + "property": "scene_id" + }, + "LC82030282019133LGN0_" + ] + } + \ No newline at end of file diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index d9db48cd..050b7e60 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -213,3 +213,97 @@ async def test_search_filter_extension_floats_post(app_client, ctx): assert resp.status_code == 200 assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_extension_wildcard_cql2(app_client, ctx): + single_char = ctx.item["id"][:-1] + "_" + multi_char = ctx.item["id"][:-3] + "%" + + params = { + "filter": { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, + { + "op": "like", + "args": [ + {"property": "id"}, + single_char, + ], + }, + { + "op": "like", + "args": [ + {"property": "id"}, + multi_char, + ], + }, + ], + } + } + + resp = await app_client.post("/search", json=params) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_extension_wildcard_es(app_client, ctx): + single_char = ctx.item["id"][:-1] + "?" + multi_char = ctx.item["id"][:-3] + "*" + + params = { + "filter": { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, + { + "op": "like", + "args": [ + {"property": "id"}, + single_char, + ], + }, + { + "op": "like", + "args": [ + {"property": "id"}, + multi_char, + ], + }, + ], + } + } + + resp = await app_client.post("/search", json=params) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_extension_escape_chars(app_client, ctx): + esc_chars = ctx.item["properties"]["landsat:product_id"].replace("_", "\_")[:-1] + "_" + + params = { + "filter": { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, + { + "op": "like", + "args": [ + {"property": "properties.landsat:product_id"}, + esc_chars, + ], + } + ], + } + } + + resp = await app_client.post("/search", json=params) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 From c8edf52b742e0714328960530c24ae3b3b7d9e2f Mon Sep 17 00:00:00 2001 From: James Date: Fri, 15 Dec 2023 10:45:55 -0500 Subject: [PATCH 2/9] Update documentation --- .../stac_fastapi/elasticsearch/extensions/filter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py index 690f9ddc..83cbc0eb 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py @@ -4,6 +4,9 @@ Basic CQL2 (AND, OR, NOT), comparison operators (=, <>, <, <=, >, >=), and IS NULL. The comparison operators are allowed against string, numeric, boolean, date, and datetime types. +Advanced CQL2 LIKE comparison operator (http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators). +The LIKE comparison operator is allowed against string types. + Basic Spatial Operators (http://www.opengis.net/spec/cql2/1.0/conf/basic-spatial-operators) defines the intersects operator (S_INTERSECTS). """ From 3f1a3e50d24fe07bb1bef0468d4b234229123d40 Mon Sep 17 00:00:00 2001 From: James Date: Fri, 15 Dec 2023 10:56:17 -0500 Subject: [PATCH 3/9] formatting fixes --- .../stac_fastapi/elasticsearch/app.py | 7 +++--- .../elasticsearch/extensions/filter.py | 25 ++++++++----------- .../tests/extensions/test_filter.py | 6 +++-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index c7569e87..692ce959 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -19,14 +19,15 @@ TokenPaginationExtension, TransactionExtension, ) - from stac_fastapi.extensions.third_party import BulkTransactionExtension settings = ElasticsearchSettings() session = Session.create_from_settings(settings) filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient()) -filter_extension.conformance_classes.append("http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators") +filter_extension.conformance_classes.append( + "http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators" +) extensions = [ TransactionExtension(client=TransactionsClient(session=session), settings=settings), @@ -36,7 +37,7 @@ SortExtension(), TokenPaginationExtension(), ContextExtension(), - filter_extension + filter_extension, ] post_request_model = create_post_request_model(extensions) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py index 83cbc0eb..344a04ce 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py @@ -184,12 +184,12 @@ def to_es(self): } } elif self.op == AdvancedComparisonOp.like: - return { - "wildcard": { - to_es(self.args[0]): { - "value": cql2_like_to_es(str(to_es(self.args[1]))), + return { + "wildcard": { + to_es(self.args[0]): { + "value": cql2_like_to_es(str(to_es(self.args[1]))), "boost": 1.0, - "case_insensitive": "true" + "case_insensitive": "true", } } } @@ -232,23 +232,20 @@ def to_es(arg: Arg): return arg else: raise RuntimeError(f"unknown arg {repr(arg)}") - + def cql2_like_to_es(input_string): - """ - Convert arugument in CQL2 ('_' and '%') to Elasticsearch wildcard operators ('?' and '*', respectively). Handle escape characters and - handle Elasticsearch wildcards directly. - """ + """Convert arugument in CQL2 ('_' and '%') to Elasticsearch wildcard operators ('?' and '*', respectively). Handle escape characters and handle Elasticsearch wildcards directly.""" es_string = "" escape = False for char in input_string: if char == "\\": escape = True - elif char == '_' and not escape: - es_string += '?' - elif char == '%' and not escape: - es_string += '*' + elif char == "_" and not escape: + es_string += "?" + elif char == "%" and not escape: + es_string += "*" else: es_string += char escape = False diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index 050b7e60..1f5d2eb9 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -285,7 +285,9 @@ async def test_search_filter_extension_wildcard_es(app_client, ctx): @pytest.mark.asyncio async def test_search_filter_extension_escape_chars(app_client, ctx): - esc_chars = ctx.item["properties"]["landsat:product_id"].replace("_", "\_")[:-1] + "_" + esc_chars = ( + ctx.item["properties"]["landsat:product_id"].replace("_", "\\_")[:-1] + "_" + ) params = { "filter": { @@ -298,7 +300,7 @@ async def test_search_filter_extension_escape_chars(app_client, ctx): {"property": "properties.landsat:product_id"}, esc_chars, ], - } + }, ], } } From 8ed4d357daa0e5c4f3d05651a0c5746e0580881c Mon Sep 17 00:00:00 2001 From: James Date: Fri, 15 Dec 2023 11:10:36 -0500 Subject: [PATCH 4/9] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94bec4de..99aa940b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- LIKE search operator to Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178) + ### Changed - Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169) From 73f6dc403223286be59a2fbbac4d032ec185cacb Mon Sep 17 00:00:00 2001 From: James Date: Tue, 19 Dec 2023 15:04:40 -0500 Subject: [PATCH 5/9] IN and BETWEEN operators --- CHANGELOG.md | 2 +- .../elasticsearch/extensions/filter.py | 28 ++++- .../tests/extensions/cql2/example21.json | 35 ++++++ .../tests/extensions/cql2/example22.json | 13 +++ .../tests/extensions/test_filter.py | 107 ++++++++++++++++++ 5 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json create mode 100644 stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 99aa940b..03160fe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added -- LIKE search operator to Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178) +- Advanced comparison (LIKE, IN, BETWEEN) operators to the Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178) ### Changed diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py index 344a04ce..f28d2507 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py @@ -4,8 +4,8 @@ Basic CQL2 (AND, OR, NOT), comparison operators (=, <>, <, <=, >, >=), and IS NULL. The comparison operators are allowed against string, numeric, boolean, date, and datetime types. -Advanced CQL2 LIKE comparison operator (http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators). -The LIKE comparison operator is allowed against string types. +Advanced comparison operators (http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators) +defines the LIKE, IN, and BETWEEN operators. Basic Spatial Operators (http://www.opengis.net/spec/cql2/1.0/conf/basic-spatial-operators) defines the intersects operator (S_INTERSECTS). @@ -84,10 +84,12 @@ def to_es(self): class AdvancedComparisonOp(str, Enum): """Advanced Comparison operator. - CQL2 advanced comparison operator like (~). + CQL2 advanced comparison operators like (~), between, and in. """ like = "like" + between = "between" + _in = "in" class SpatialIntersectsOp(str, Enum): @@ -165,7 +167,7 @@ class Clause(BaseModel): """Filter extension clause.""" op: Union[LogicalOp, ComparisonOp, AdvancedComparisonOp, SpatialIntersectsOp] - args: List[Arg] + args: List[Union[Arg, List[Arg]]] def to_es(self): """Generate an Elasticsearch expression for this Clause.""" @@ -188,11 +190,27 @@ def to_es(self): "wildcard": { to_es(self.args[0]): { "value": cql2_like_to_es(str(to_es(self.args[1]))), - "boost": 1.0, "case_insensitive": "true", } } } + elif self.op == AdvancedComparisonOp.between: + if not isinstance(self.args[1], List): + raise RuntimeError(f"Arg {self.args[1]} is not a list") + return { + "range": { + to_es(self.args[0]): { + "gte": to_es(self.args[1][0]), + "lte": to_es(self.args[1][1]), + } + } + } + elif self.op == AdvancedComparisonOp._in: + if not isinstance(self.args[1], List): + raise RuntimeError(f"Arg {self.args[1]} is not a list") + return { + "terms": {to_es(self.args[0]): [to_es(arg) for arg in self.args[1]]} + } elif ( self.op == ComparisonOp.lt or self.op == ComparisonOp.lte diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json b/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json new file mode 100644 index 00000000..7005f6e2 --- /dev/null +++ b/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json @@ -0,0 +1,35 @@ +{ + "op": "and", + "args": [ + { + "op": "between", + "args": [ + { + "property": "cloud_cover" + }, + [ + 0.1, + 0.2 + ] + ] + }, + { + "op": "=", + "args": [ + { + "property": "landsat:wrs_row" + }, + 28 + ] + }, + { + "op": "=", + "args": [ + { + "property": "landsat:wrs_path" + }, + 203 + ] + } + ] +} \ No newline at end of file diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json b/stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json new file mode 100644 index 00000000..880c90c3 --- /dev/null +++ b/stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json @@ -0,0 +1,13 @@ +{ + "op": "and", + "args": [ + { + "op": "in", + "args": [ + {"property": "id"}, + ["LC08_L1TP_060247_20180905_20180912_01_T1_L1TP"] + ] + }, + {"op": "=", "args": [{"property": "collection"}, "landsat8_l1tp"]} + ] +} \ No newline at end of file diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index 1f5d2eb9..b113d219 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -309,3 +309,110 @@ async def test_search_filter_extension_escape_chars(app_client, ctx): assert resp.status_code == 200 assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_extension_in(app_client, ctx): + product_id = ctx.item["properties"]["landsat:product_id"] + + params = { + "filter": { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, + { + "op": "in", + "args": [ + {"property": "properties.landsat:product_id"}, + [product_id], + ], + }, + ], + } + } + + resp = await app_client.post("/search", json=params) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_extension_in_no_list(app_client, ctx): + product_id = ctx.item["properties"]["landsat:product_id"] + + params = { + "filter": { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, + { + "op": "in", + "args": [ + {"property": "properties.landsat:product_id"}, + product_id, + ], + }, + ], + } + } + + resp = await app_client.post("/search", json=params) + + assert resp.status_code == 400 + assert resp.json() == { + "detail": f"Error with cql2_json filter: Arg {product_id} is not a list" + } + + +@pytest.mark.asyncio +async def test_search_filter_extension_between(app_client, ctx): + sun_elevation = ctx.item["properties"]["view:sun_elevation"] + + params = { + "filter": { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, + { + "op": "between", + "args": [ + {"property": "properties.view:sun_elevation"}, + [sun_elevation - 0.01, sun_elevation + 0.01], + ], + }, + ], + } + } + resp = await app_client.post("/search", json=params) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_extension_between_no_list(app_client, ctx): + sun_elevation = ctx.item["properties"]["view:sun_elevation"] + + params = { + "filter": { + "op": "and", + "args": [ + {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, + { + "op": "between", + "args": [ + {"property": "properties.view:sun_elevation"}, + sun_elevation - 0.01, + sun_elevation + 0.01, + ], + }, + ], + } + } + resp = await app_client.post("/search", json=params) + + assert resp.status_code == 400 + assert resp.json() == { + "detail": f"Error with cql2_json filter: Arg {sun_elevation - 0.01} is not a list" + } From 4e4b5eb3684b707a46a4df1fd2aa36a8327edc95 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Wed, 20 Dec 2023 09:03:13 +0100 Subject: [PATCH 6/9] #179: filter extension - set filter-lang=cql2-text as default for GET search --- .../stac_fastapi/elasticsearch/core.py | 6 +++--- .../tests/extensions/test_filter.py | 17 ++++++++++++++--- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 4fb9f174..12cc6b2c 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -380,12 +380,12 @@ async def get_search( base_args["sortby"] = sort_param if filter: - if filter_lang == "cql2-text": + if filter_lang == "cql2-json": base_args["filter-lang"] = "cql2-json" - base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter))) + base_args["filter"] = orjson.loads(unquote_plus(filter)) else: base_args["filter-lang"] = "cql2-json" - base_args["filter"] = orjson.loads(unquote_plus(filter)) + base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter))) if fields: includes = set() diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index d9db48cd..39fb8c25 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -101,6 +101,17 @@ async def test_search_filter_ext_and_get(app_client, ctx): assert len(resp.json()["features"]) == 1 +@pytest.mark.asyncio +async def test_search_filter_ext_and_get_id(app_client, ctx): + collection = ctx.item["collection"] + id = ctx.item["id"] + filter = f"id='{id}' AND collection='{collection}'" + resp = await app_client.get(f"/search?&filter={filter}") + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + @pytest.mark.asyncio async def test_search_filter_ext_and_get_cql2text_id(app_client, ctx): collection = ctx.item["collection"] @@ -162,21 +173,21 @@ async def test_search_filter_ext_and_post(app_client, ctx): @pytest.mark.asyncio async def test_search_filter_extension_floats_get(app_client, ctx): resp = await app_client.get( - """/search?filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30891534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30691534"]}]}""" + """/search?filter-lang=cql2-json&filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30891534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30691534"]}]}""" ) assert resp.status_code == 200 assert len(resp.json()["features"]) == 1 resp = await app_client.get( - """/search?filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item-7"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30891534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30691534"]}]}""" + """/search?filter-lang=cql2-json&filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item-7"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30891534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30691534"]}]}""" ) assert resp.status_code == 200 assert len(resp.json()["features"]) == 0 resp = await app_client.get( - """/search?filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30591534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30491534"]}]}""" + """/search?filter-lang=cql2-json&filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30591534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30491534"]}]}""" ) assert resp.status_code == 200 From 85eb50b04515d009746d91d53a0e02660a54426a Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Wed, 20 Dec 2023 09:55:50 +0100 Subject: [PATCH 7/9] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94bec4de..eabf6e7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166) - Upgrade stac-fastapi to v2.4.9 [#172](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/172) +- Set correct default filter-lang for GET /search requests [#179](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/179) ## [v1.0.0] From 7bf41c67e64b5d3f8317dc75827d693b512d3a0c Mon Sep 17 00:00:00 2001 From: James Date: Wed, 20 Dec 2023 12:24:11 -0500 Subject: [PATCH 8/9] Wildcard replacement with regex --- .../elasticsearch/extensions/filter.py | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py index f28d2507..a0ab3777 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py @@ -13,6 +13,7 @@ from __future__ import annotations import datetime +import re from enum import Enum from typing import List, Union @@ -252,20 +253,17 @@ def to_es(arg: Arg): raise RuntimeError(f"unknown arg {repr(arg)}") -def cql2_like_to_es(input_string): - """Convert arugument in CQL2 ('_' and '%') to Elasticsearch wildcard operators ('?' and '*', respectively). Handle escape characters and handle Elasticsearch wildcards directly.""" - es_string = "" - escape = False +def cql2_like_to_es(string): + """Convert wildcard characters in CQL2 ('_' and '%') to Elasticsearch wildcard characters ('?' and '*', respectively). Handle escape characters and pass through Elasticsearch wildcards.""" + percent_pattern = r"(? Date: Thu, 21 Dec 2023 10:38:02 -0500 Subject: [PATCH 9/9] update CQL2 BETWEEN syntax --- .../stac_fastapi/elasticsearch/app.py | 2 +- .../elasticsearch/extensions/filter.py | 8 ++--- .../tests/extensions/cql2/example21.json | 6 ++-- .../tests/extensions/test_filter.py | 31 ++----------------- 4 files changed, 8 insertions(+), 39 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 692ce959..8adcece4 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -26,7 +26,7 @@ filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient()) filter_extension.conformance_classes.append( - "http://www.opengis.net/spec/cql2/1.0/req/advanced-comparison-operators" + "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" ) extensions = [ diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py index a0ab3777..fe691ddf 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py @@ -191,18 +191,16 @@ def to_es(self): "wildcard": { to_es(self.args[0]): { "value": cql2_like_to_es(str(to_es(self.args[1]))), - "case_insensitive": "true", + "case_insensitive": "false", } } } elif self.op == AdvancedComparisonOp.between: - if not isinstance(self.args[1], List): - raise RuntimeError(f"Arg {self.args[1]} is not a list") return { "range": { to_es(self.args[0]): { - "gte": to_es(self.args[1][0]), - "lte": to_es(self.args[1][1]), + "gte": to_es(self.args[1]), + "lte": to_es(self.args[2]), } } } diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json b/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json index 7005f6e2..175b8732 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json +++ b/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json @@ -7,10 +7,8 @@ { "property": "cloud_cover" }, - [ - 0.1, - 0.2 - ] + 0.1, + 0.2 ] }, { diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index b113d219..c5e521bc 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -369,31 +369,6 @@ async def test_search_filter_extension_in_no_list(app_client, ctx): async def test_search_filter_extension_between(app_client, ctx): sun_elevation = ctx.item["properties"]["view:sun_elevation"] - params = { - "filter": { - "op": "and", - "args": [ - {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}, - { - "op": "between", - "args": [ - {"property": "properties.view:sun_elevation"}, - [sun_elevation - 0.01, sun_elevation + 0.01], - ], - }, - ], - } - } - resp = await app_client.post("/search", json=params) - - assert resp.status_code == 200 - assert len(resp.json()["features"]) == 1 - - -@pytest.mark.asyncio -async def test_search_filter_extension_between_no_list(app_client, ctx): - sun_elevation = ctx.item["properties"]["view:sun_elevation"] - params = { "filter": { "op": "and", @@ -412,7 +387,5 @@ async def test_search_filter_extension_between_no_list(app_client, ctx): } resp = await app_client.post("/search", json=params) - assert resp.status_code == 400 - assert resp.json() == { - "detail": f"Error with cql2_json filter: Arg {sun_elevation - 0.01} is not a list" - } + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1