From dca31ba1285214a143d3c8002318db974402f37b Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 11 Dec 2023 09:04:12 +0000 Subject: [PATCH 01/11] Moving collection id to query parameter. --- .../stac_fastapi/elasticsearch/core.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index e2af91a6..1342b653 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -413,9 +413,9 @@ async def post_search( ) if search_request.query: - for (field_name, expr) in search_request.query.items(): + for field_name, expr in search_request.query.items(): field = "properties__" + field_name - for (op, value) in expr.items(): + for op, value in expr.items(): search = self.database.apply_stacql_filter( search=search, op=op, field=field, value=value ) @@ -627,9 +627,18 @@ async def update_collection( """ base_url = str(kwargs["request"].base_url) - await self.database.find_collection(collection_id=collection["id"]) - await self.delete_collection(collection["id"]) - await self.create_collection(collection, **kwargs) + collection_id = kwargs["request"].query_params.get( + "collection_id", collection["id"] + ) + + collection_links = CollectionLinks( + collection_id=collection["id"], base_url=base_url + ).create_links() + collection["links"] = collection_links + + await self.database.update_collection( + collection_id=collection_id, collection=collection + ) return CollectionSerializer.db_to_stac(collection, base_url) From c0235469e874e70f6165631a7b4b3ffdf9337d8c Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 11 Dec 2023 09:05:05 +0000 Subject: [PATCH 02/11] Moving items if collection id is changed. --- .../elasticsearch/database_logic.py | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index d641738a..d1190243 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -734,6 +734,53 @@ async def find_collection(self, collection_id: str) -> Collection: return collection["_source"] + async def update_collection( + self, collection_id: str, collection: Collection, refresh: bool = False + ): + """Update a collection from the database. + + Args: + self: The instance of the object calling this function. + collection_id (str): The ID of the collection to be updated. + collection (Collection): The Collection object to be updated. + + Raises: + NotFoundError: If the collection with the given `collection_id` is not + found in the database. + + Notes: + This function updates the collection in the database using the specified + `collection_id` and with the collection specified in the `Collection` object. + If the collection is not found, a `NotFoundError` is raised. + """ + await self.find_collection(collection_id=collection_id) + + if collection_id != collection["id"]: + await self.create_collection(collection) + + await self.client.reindex( + body={ + "dest": {"index": f"items_{collection['id']}"}, + "source": {"index": f"items_{collection_id}"}, + "script": { + "lang": "painless", + "source": f"""ctx._source.collection = '{collection["id"]}'""", + }, + }, + wait_for_completion=True, + refresh=refresh, + ) + + await self.delete_collection(collection_id) + + else: + await self.client.index( + index=COLLECTIONS_INDEX, + id=collection_id, + document=collection, + refresh=refresh, + ) + async def delete_collection(self, collection_id: str, refresh: bool = False): """Delete a collection from the database. From 203b88bab86102d2ea2add58c049a636774b86ab Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Wed, 3 Jan 2024 08:39:02 +0000 Subject: [PATCH 03/11] Adding API key option to ES config. --- .../elasticsearch/stac_fastapi/elasticsearch/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 8634d3b9..22d6b77a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -39,6 +39,9 @@ def _es_config() -> Dict[str, Any]: if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): config["http_auth"] = (u, p) + if api_key := os.getenv("ES_API_KEY"): + config["headers"]["x-api-key"] = api_key + return config From 4e6c71134e3f192f40695fb53c249a0bc0f445ce Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Wed, 3 Jan 2024 08:39:42 +0000 Subject: [PATCH 04/11] Adding query_params to MockRequest Class attributes. --- stac_fastapi/elasticsearch/tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index fa093af2..13956329 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -39,6 +39,7 @@ def __init__(self, item, collection): class MockRequest: base_url = "http://test-server" + query_params = {} def __init__( self, @@ -50,7 +51,7 @@ def __init__( self.method = method self.url = url self.app = app - self.query_params = query_params or {} + self.query_params = query_params class TestSettings(AsyncElasticsearchSettings): From 0ed59475bc011668ee5675075568b8eb7db26390 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Wed, 3 Jan 2024 08:42:57 +0000 Subject: [PATCH 05/11] Adding to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94bec4de..e416f34d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169) +- Collection update endpoint no longer delete all sub items [#177](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/177) ### Fixed From 9cbce133c6dd9913263e89c78b573dbe7612b60f Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Wed, 3 Jan 2024 08:58:07 +0000 Subject: [PATCH 06/11] Updating doc strings. --- .../elasticsearch/stac_fastapi/elasticsearch/core.py | 7 +++++-- .../stac_fastapi/elasticsearch/database_logic.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 64ad6cf1..a4f5cc1f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -660,8 +660,11 @@ async def update_collection( Update a collection. This method updates an existing collection in the database by first finding - the collection by its id, then deleting the old version, and finally creating - a new version of the updated collection. The updated collection is then returned. + the collection by the id given in the keyword argument `collection_id`. + If no `collection_id` is given the id of the given collection object is used. + If the object and keyword collection ids don't match the sub items + collection id is updated else the items are left unchanged. + The updated collection is then returned. Args: collection: A STAC collection that needs to be updated. diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 35430e7d..9ed4b84a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -771,7 +771,7 @@ async def update_collection( Args: self: The instance of the object calling this function. collection_id (str): The ID of the collection to be updated. - collection (Collection): The Collection object to be updated. + collection (Collection): The Collection object to be used for the update. Raises: NotFoundError: If the collection with the given `collection_id` is not From 705b9cbe18131b7e18d8a90f2a3db070faffee85 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Fri, 5 Jan 2024 11:54:32 +0000 Subject: [PATCH 07/11] Fixing mypy error. --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index 22d6b77a..b727d9ff 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -40,7 +40,7 @@ def _es_config() -> Dict[str, Any]: config["http_auth"] = (u, p) if api_key := os.getenv("ES_API_KEY"): - config["headers"]["x-api-key"] = api_key + config |= {"headers": {"x-api-key": api_key}} return config From 54a487f446a4f2666c18d4d2ef899c86e3ffa4db Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 8 Jan 2024 08:41:28 +0000 Subject: [PATCH 08/11] Removing | operand for 3.8 mypy. --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index b727d9ff..be64d02a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -40,7 +40,7 @@ def _es_config() -> Dict[str, Any]: config["http_auth"] = (u, p) if api_key := os.getenv("ES_API_KEY"): - config |= {"headers": {"x-api-key": api_key}} + config = {**config, **{"headers": {"x-api-key": api_key}}} return config From 8c9fc5e509165f818d9e8ec61b532c3b75317764 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 8 Jan 2024 09:35:51 +0000 Subject: [PATCH 09/11] Adding dict type check for mypy. --- .../elasticsearch/stac_fastapi/elasticsearch/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index be64d02a..e34667ab 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -40,8 +40,15 @@ def _es_config() -> Dict[str, Any]: config["http_auth"] = (u, p) if api_key := os.getenv("ES_API_KEY"): - config = {**config, **{"headers": {"x-api-key": api_key}}} + if isinstance(config["headers"], dict): + headers = {**config["headers"], "x-api-key": api_key} + else: + headers = {"x-api-key": api_key} + + config["headers"] = headers + + print(config) return config From fe9d5d5e03bc8c76e08404970764ec876e28d093 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Tue, 9 Jan 2024 08:17:41 +0000 Subject: [PATCH 10/11] Removing testing print statement. --- .../elasticsearch/stac_fastapi/elasticsearch/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index e34667ab..10cf95e9 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -44,11 +44,10 @@ def _es_config() -> Dict[str, Any]: headers = {**config["headers"], "x-api-key": api_key} else: - headers = {"x-api-key": api_key} + config["headers"] = {"x-api-key": api_key} config["headers"] = headers - print(config) return config From 4e11011d953b5775fb42ea63f005be03d0f03982 Mon Sep 17 00:00:00 2001 From: rhysrevans3 Date: Mon, 29 Jan 2024 15:29:19 +0000 Subject: [PATCH 11/11] Adding tests for new collection update workflow. --- docker-compose.yml | 3 +- .../elasticsearch/database_logic.py | 8 +- .../tests/clients/test_elasticsearch.py | 86 +++++++++++++++++-- 3 files changed, 86 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index db3352fb..03698654 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,7 +14,7 @@ services: - RELOAD=true - ENVIRONMENT=local - WEB_CONCURRENCY=10 - - ES_HOST=172.17.0.1 + - ES_HOST=elasticsearch - ES_PORT=9200 - ES_USE_SSL=false - ES_VERIFY_CERTS=false @@ -32,6 +32,7 @@ services: elasticsearch: container_name: es-container image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0} + hostname: elasticsearch environment: ES_JAVA_OPTS: -Xms512m -Xmx1g volumes: diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 9ed4b84a..8b8911d1 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -785,15 +785,15 @@ async def update_collection( await self.find_collection(collection_id=collection_id) if collection_id != collection["id"]: - await self.create_collection(collection) + await self.create_collection(collection, refresh=refresh) await self.client.reindex( body={ - "dest": {"index": f"items_{collection['id']}"}, - "source": {"index": f"items_{collection_id}"}, + "dest": {"index": f"{ITEMS_INDEX_PREFIX}{collection['id']}"}, + "source": {"index": f"{ITEMS_INDEX_PREFIX}{collection_id}"}, "script": { "lang": "painless", - "source": f"""ctx._source.collection = '{collection["id"]}'""", + "source": f"""ctx._id = ctx._id.replace('{collection_id}', '{collection["id"]}'); ctx._source.collection = '{collection["id"]}' ;""", }, }, wait_for_completion=True, diff --git a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py index 3da8f86d..41fcf26d 100644 --- a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py +++ b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py @@ -40,16 +40,90 @@ async def test_update_collection( txn_client, load_test_data: Callable, ): - data = load_test_data("test_collection.json") + collection_data = load_test_data("test_collection.json") + item_data = load_test_data("test_item.json") - await txn_client.create_collection(data, request=MockRequest) - data["keywords"].append("new keyword") - await txn_client.update_collection(data, request=MockRequest) + await txn_client.create_collection(collection_data, request=MockRequest) + await txn_client.create_item( + collection_id=collection_data["id"], + item=item_data, + request=MockRequest, + refresh=True, + ) - coll = await core_client.get_collection(data["id"], request=MockRequest) + collection_data["keywords"].append("new keyword") + await txn_client.update_collection(collection_data, request=MockRequest) + + coll = await core_client.get_collection(collection_data["id"], request=MockRequest) assert "new keyword" in coll["keywords"] - await txn_client.delete_collection(data["id"]) + item = await core_client.get_item( + item_id=item_data["id"], + collection_id=collection_data["id"], + request=MockRequest, + ) + assert item["id"] == item_data["id"] + assert item["collection"] == item_data["collection"] + + await txn_client.delete_collection(collection_data["id"]) + + +@pytest.mark.asyncio +async def test_update_collection_id( + core_client, + txn_client, + load_test_data: Callable, +): + collection_data = load_test_data("test_collection.json") + item_data = load_test_data("test_item.json") + new_collection_id = "new-test-collection" + + await txn_client.create_collection(collection_data, request=MockRequest) + await txn_client.create_item( + collection_id=collection_data["id"], + item=item_data, + request=MockRequest, + refresh=True, + ) + + old_collection_id = collection_data["id"] + collection_data["id"] = new_collection_id + + await txn_client.update_collection( + collection=collection_data, + request=MockRequest( + query_params={ + "collection_id": old_collection_id, + "limit": "10", + } + ), + refresh=True, + ) + + with pytest.raises(NotFoundError): + await core_client.get_collection(old_collection_id, request=MockRequest) + + coll = await core_client.get_collection(collection_data["id"], request=MockRequest) + assert coll["id"] == new_collection_id + + with pytest.raises(NotFoundError): + await core_client.get_item( + item_id=item_data["id"], + collection_id=old_collection_id, + request=MockRequest, + ) + + item = await core_client.get_item( + item_id=item_data["id"], + collection_id=collection_data["id"], + request=MockRequest, + refresh=True, + ) + + assert item["id"] == item_data["id"] + assert item["collection"] == new_collection_id + + await txn_client.delete_collection(collection_data["id"]) @pytest.mark.asyncio