From 02b896a9812f831c4f61fd6d5dd1fb088da9e5c4 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 5 Dec 2023 09:53:17 +0100 Subject: [PATCH 1/3] upgrade stac-fastapi to v2.4.9, implement support for BulkTransaction method parameter --- stac_fastapi/elasticsearch/setup.py | 6 +++--- .../elasticsearch/stac_fastapi/elasticsearch/core.py | 12 +++++++----- .../stac_fastapi/elasticsearch/database_logic.py | 10 ++++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 6ff8cd86..fa39a276 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -10,9 +10,9 @@ "attrs", "pydantic[dotenv]<2", "stac_pydantic==2.0.*", - "stac-fastapi.types==2.4.8", - "stac-fastapi.api==2.4.8", - "stac-fastapi.extensions==2.4.8", + "stac-fastapi.types==2.4.9", + "stac-fastapi.api==2.4.9", + "stac-fastapi.extensions==2.4.9", "elasticsearch[async]==7.17.9", "elasticsearch-dsl==7.4.1", "pystac[validation]", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 76bb04cd..53940aa5 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -26,7 +26,7 @@ from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, - Items, + Items, BulkTransactionMethod, ) from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings @@ -568,7 +568,7 @@ async def create_item( if item["type"] == "FeatureCollection": bulk_client = BulkTransactionsClient() processed_items = [ - bulk_client.preprocess_item(item, base_url) for item in item["features"] # type: ignore + bulk_client.preprocess_item(item, base_url, BulkTransactionMethod.INSERT) for item in item["features"] # type: ignore ] await self.database.bulk_async( @@ -718,17 +718,19 @@ def __attrs_post_init__(self): settings = ElasticsearchSettings() self.client = settings.create_client - def preprocess_item(self, item: stac_types.Item, base_url) -> stac_types.Item: + def preprocess_item(self, item: stac_types.Item, base_url, method: BulkTransactionMethod) -> stac_types.Item: """Preprocess an item to match the data model. Args: item: The item to preprocess. base_url: The base URL of the request. + method: The bulk transaction method. Returns: The preprocessed item. """ - return self.database.sync_prep_create_item(item=item, base_url=base_url) + exist_ok = (method == BulkTransactionMethod.UPSERT) + return self.database.sync_prep_create_item(item=item, base_url=base_url, exist_ok=exist_ok) @overrides def bulk_item_insert( @@ -751,7 +753,7 @@ def bulk_item_insert( base_url = "" processed_items = [ - self.preprocess_item(item, base_url) for item in items.items.values() + self.preprocess_item(item, base_url, items.method) for item in items.items.values() ] # not a great way to get the collection_id-- should be part of the method signature diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 6b2cd433..d9cafa54 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -591,13 +591,14 @@ async def check_collection_exists(self, collection_id: str): if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): raise NotFoundError(f"Collection {collection_id} does not exist") - async def prep_create_item(self, item: Item, base_url: str) -> Item: + async def prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: """ Preps an item for insertion into the database. Args: item (Item): The item to be prepped for insertion. base_url (str): The base URL used to create the item's self URL. + exist_ok (bool): Indicates whether the item can exist already. Returns: Item: The prepped item. @@ -608,7 +609,7 @@ async def prep_create_item(self, item: Item, base_url: str) -> Item: """ await self.check_collection_exists(collection_id=item["collection"]) - if await self.client.exists( + if not exist_ok and await self.client.exists( index=index_by_collection_id(item["collection"]), id=mk_item_id(item["id"], item["collection"]), ): @@ -618,7 +619,7 @@ async def prep_create_item(self, item: Item, base_url: str) -> Item: return self.item_serializer.stac_to_db(item, base_url) - def sync_prep_create_item(self, item: Item, base_url: str) -> Item: + def sync_prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: """ Prepare an item for insertion into the database. @@ -629,6 +630,7 @@ def sync_prep_create_item(self, item: Item, base_url: str) -> Item: Args: item (Item): The item to be inserted into the database. base_url (str): The base URL used for constructing URLs for the item. + exist_ok (bool): Indicates whether the item can exist already. Returns: Item: The item after preparation is done. @@ -642,7 +644,7 @@ def sync_prep_create_item(self, item: Item, base_url: str) -> Item: if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=collection_id): raise NotFoundError(f"Collection {collection_id} does not exist") - if self.sync_client.exists( + if not exist_ok and self.sync_client.exists( index=index_by_collection_id(collection_id), id=mk_item_id(item_id, collection_id), ): From 1281805bc86d04a761948a194a79870d840706d0 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 5 Dec 2023 09:55:28 +0100 Subject: [PATCH 2/3] fix formatting --- .../stac_fastapi/elasticsearch/core.py | 16 +++++++++++----- .../stac_fastapi/elasticsearch/database_logic.py | 8 ++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 53940aa5..4fb9f174 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -26,7 +26,8 @@ from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, - Items, BulkTransactionMethod, + BulkTransactionMethod, + Items, ) from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings @@ -718,7 +719,9 @@ def __attrs_post_init__(self): settings = ElasticsearchSettings() self.client = settings.create_client - def preprocess_item(self, item: stac_types.Item, base_url, method: BulkTransactionMethod) -> stac_types.Item: + def preprocess_item( + self, item: stac_types.Item, base_url, method: BulkTransactionMethod + ) -> stac_types.Item: """Preprocess an item to match the data model. Args: @@ -729,8 +732,10 @@ def preprocess_item(self, item: stac_types.Item, base_url, method: BulkTransacti Returns: The preprocessed item. """ - exist_ok = (method == BulkTransactionMethod.UPSERT) - return self.database.sync_prep_create_item(item=item, base_url=base_url, exist_ok=exist_ok) + exist_ok = method == BulkTransactionMethod.UPSERT + return self.database.sync_prep_create_item( + item=item, base_url=base_url, exist_ok=exist_ok + ) @overrides def bulk_item_insert( @@ -753,7 +758,8 @@ def bulk_item_insert( base_url = "" processed_items = [ - self.preprocess_item(item, base_url, items.method) for item in items.items.values() + self.preprocess_item(item, base_url, items.method) + for item in items.items.values() ] # not a great way to get the collection_id-- should be part of the method signature diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index d9cafa54..20d5c901 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -591,7 +591,9 @@ async def check_collection_exists(self, collection_id: str): if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): raise NotFoundError(f"Collection {collection_id} does not exist") - async def prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: + async def prep_create_item( + self, item: Item, base_url: str, exist_ok: bool = False + ) -> Item: """ Preps an item for insertion into the database. @@ -619,7 +621,9 @@ async def prep_create_item(self, item: Item, base_url: str, exist_ok: bool = Fal return self.item_serializer.stac_to_db(item, base_url) - def sync_prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: + def sync_prep_create_item( + self, item: Item, base_url: str, exist_ok: bool = False + ) -> Item: """ Prepare an item for insertion into the database. From ea2294b147011518c1863c834b9279ed5db676e5 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 5 Dec 2023 10:01:03 +0100 Subject: [PATCH 3/3] update changelog --- CHANGELOG.md | 1 + .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d50d6ff9..2cd268b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166) +- Upgrade stac-fastapi to v2.4.9 [#172](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/172) ## [v1.0.0] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 20d5c901..1e8c5333 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -629,7 +629,7 @@ def sync_prep_create_item( This method performs pre-insertion preparation on the given `item`, such as checking if the collection the item belongs to exists, - and verifying that an item with the same ID does not already exist in the database. + and optionally verifying that an item with the same ID does not already exist in the database. Args: item (Item): The item to be inserted into the database.