From ff4a919fc61dd48b637a65d0d134cdc82d5f6381 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 20 Nov 2023 19:33:28 +0800 Subject: [PATCH 01/23] database logic to folder --- .../stac_fastapi/elasticsearch/app.py | 2 +- .../stac_fastapi/elasticsearch/core.py | 2 +- .../database_logic.py | 16 +--------------- stac_fastapi/elasticsearch/tests/conftest.py | 2 +- 4 files changed, 4 insertions(+), 18 deletions(-) rename stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/{ => database_elasticsearch}/database_logic.py (97%) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index e3c4bc64..7044be46 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -8,7 +8,7 @@ EsAsyncBaseFiltersClient, TransactionsClient, ) -from stac_fastapi.elasticsearch.database_logic import create_collection_index +from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.core import ( diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 65e9a458..2dfbab20 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -20,7 +20,7 @@ from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings -from stac_fastapi.elasticsearch.database_logic import DatabaseLogic +from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic from stac_fastapi.elasticsearch.models.links import PagingLinks from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.elasticsearch.session import Session diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py similarity index 97% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py rename to stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py index 6b2cd433..f8db5226 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py @@ -17,6 +17,7 @@ from stac_fastapi.elasticsearch.extensions import filter from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item +from stac_fastapi.elasticsearch.utilities import bbox2polygon logger = logging.getLogger(__name__) @@ -229,21 +230,6 @@ async def delete_item_index(collection_id: str): await client.close() -def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: - """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. - - Args: - b0 (float): The x-coordinate of the lower-left corner of the bounding box. - b1 (float): The y-coordinate of the lower-left corner of the bounding box. - b2 (float): The x-coordinate of the upper-right corner of the bounding box. - b3 (float): The y-coordinate of the upper-right corner of the bounding box. - - Returns: - List[List[List[float]]]: A polygon represented as a list of lists of coordinates. - """ - return [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]] - - def mk_item_id(item_id: str, collection_id: str): """Create the document id for an Item in Elasticsearch. diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index f4b49928..67d72135 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -16,7 +16,7 @@ CoreClient, TransactionsClient, ) -from stac_fastapi.elasticsearch.database_logic import create_collection_index +from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, From a4f4cd41ad469de335ef001a4e8ec191561a831f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 20 Nov 2023 19:33:43 +0800 Subject: [PATCH 02/23] add utilities --- .../stac_fastapi/elasticsearch/utilities.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py new file mode 100644 index 00000000..7be319a7 --- /dev/null +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py @@ -0,0 +1,15 @@ +from typing import List + +def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: + """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. + + Args: + b0 (float): The x-coordinate of the lower-left corner of the bounding box. + b1 (float): The y-coordinate of the lower-left corner of the bounding box. + b2 (float): The x-coordinate of the upper-right corner of the bounding box. + b3 (float): The y-coordinate of the upper-right corner of the bounding box. + + Returns: + List[List[List[float]]]: A polygon represented as a list of lists of coordinates. + """ + return [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]] From 45ce18b0b462d43e308b9445a5ba642f147ad0ae Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 20 Nov 2023 23:27:33 +0800 Subject: [PATCH 03/23] add opensearch to docker-compose --- docker-compose.yml | 40 ++++++++++++++++++++++++++++++++ opensearch/config/opensearch.yml | 35 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 opensearch/config/opensearch.yml diff --git a/docker-compose.yml b/docker-compose.yml index 1cad4dee..ac163674 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,34 @@ services: command: bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app" + app-opensearch: + container_name: stac-fastapi-os + image: stac-utils/stac-fastapi + restart: always + build: + context: . + dockerfile: Dockerfile.dev + environment: + - APP_HOST=0.0.0.0 + - APP_PORT=8080 + - RELOAD=true + - ENVIRONMENT=local + - WEB_CONCURRENCY=10 + - OS_HOST=172.17.0.1 + - OS_PORT=9200 + - OS_USE_SSL=false + - OS_VERIFY_CERTS=false + ports: + - "8082:8080" + volumes: + - ./stac_fastapi:/app/stac_fastapi + - ./scripts:/app/scripts + - ./osdata:/usr/share/opensearch/data + depends_on: + - opensearch + command: + bash -c "./scripts/wait-for-it-os.sh os-container:9200 && python -m stac_fastapi.elasticsearch.app" + elasticsearch: container_name: es-container image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.10.4} @@ -39,3 +67,15 @@ services: - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots ports: - "9200:9200" + + opensearch: + container_name: os-container + image: opensearchproject/opensearch:latest + environment: + - "discovery.type=single-node" + - "plugins.security.disabled=true" + volumes: + - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml + - ./opensearch/snapshots:/usr/share/opensearch/snapshots + ports: + - "9202:9200" diff --git a/opensearch/config/opensearch.yml b/opensearch/config/opensearch.yml new file mode 100644 index 00000000..dc69776a --- /dev/null +++ b/opensearch/config/opensearch.yml @@ -0,0 +1,35 @@ +## Cluster Settings +cluster.name: stac-cluster +node.name: es01 +network.host: 0.0.0.0 +transport.host: 0.0.0.0 +discovery.type: single-node +http.port: 9200 + +path: + repo: + - /usr/share/opensearch/snapshots + +######## Start OpenSearch Security Demo Configuration ######## +# WARNING: revise all the lines below before you go into production +plugins.security.ssl.transport.pemcert_filepath: esnode.pem +plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem +plugins.security.ssl.transport.enforce_hostname_verification: false +plugins.security.ssl.http.enabled: true +plugins.security.ssl.http.pemcert_filepath: esnode.pem +plugins.security.ssl.http.pemkey_filepath: esnode-key.pem +plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem +plugins.security.allow_unsafe_democertificates: true +plugins.security.allow_default_init_securityindex: true +plugins.security.authcz.admin_dn: + - CN=kirk,OU=client,O=client,L=test, C=de + +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.system_indices.enabled: true +plugins.security.system_indices.indices: [".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"] +node.max_local_storage_nodes: 3 +######## End OpenSearch Security Demo Configuration ######## From 6a9b66601b880926c912f824a4e2e43f31fe2a26 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 21 Nov 2023 12:08:52 +0800 Subject: [PATCH 04/23] update compose --- docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index ac163674..e8058b45 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ version: '3.9' services: app-elasticsearch: container_name: stac-fastapi-es - image: stac-utils/stac-fastapi + image: stac-utils/stac-fastapi-es restart: always build: context: . @@ -31,7 +31,7 @@ services: app-opensearch: container_name: stac-fastapi-os - image: stac-utils/stac-fastapi + image: stac-utils/stac-fastapi-os restart: always build: context: . @@ -55,7 +55,7 @@ services: depends_on: - opensearch command: - bash -c "./scripts/wait-for-it-os.sh os-container:9200 && python -m stac_fastapi.elasticsearch.app" + bash -c "./scripts/wait-for-it-es.sh os-container:9200 && python -m stac_fastapi.elasticsearch.app" elasticsearch: container_name: es-container @@ -67,7 +67,7 @@ services: - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots ports: - "9200:9200" - + opensearch: container_name: os-container image: opensearchproject/opensearch:latest From 969a56f14bdc9a4ef6fc1e0e24052ccc8a18e380 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 29 Jan 2024 12:42:40 +0800 Subject: [PATCH 05/23] update docker-compose --- docker-compose.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 96b7f493..3eaae906 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ services: - ES_PORT=9200 - ES_USE_SSL=false - ES_VERIFY_CERTS=false + - DB_TYPE=elasticsearch ports: - "8080:8080" volumes: @@ -42,10 +43,11 @@ services: - RELOAD=true - ENVIRONMENT=local - WEB_CONCURRENCY=10 - - OS_HOST=172.17.0.1 - - OS_PORT=9200 - - OS_USE_SSL=false - - OS_VERIFY_CERTS=false + - ES_HOST=172.17.0.1 + - ES_PORT=9200 + - ES_USE_SSL=false + - ES_VERIFY_CERTS=false + - DB_TYPE=opensearch ports: - "8082:8080" volumes: From 3aad1e5851c1c03902d448182711b8af17c50f9b Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 29 Jan 2024 12:43:11 +0800 Subject: [PATCH 06/23] opensearch db logic scratch --- .../database_opensearch/database_logic.py | 838 ++++++++++++++++++ 1 file changed, 838 insertions(+) create mode 100644 stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py new file mode 100644 index 00000000..26d830f0 --- /dev/null +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py @@ -0,0 +1,838 @@ +"""Database logic.""" +import asyncio +import logging +import os +from base64 import urlsafe_b64decode, urlsafe_b64encode +from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, Union + +import attr +from elasticsearch_dsl import Q, Search + +from elasticsearch import exceptions, helpers # type: ignore +from stac_fastapi.elasticsearch import serializers +from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings +from stac_fastapi.elasticsearch.config import ( + ElasticsearchSettings as SyncElasticsearchSettings, +) +from stac_fastapi.elasticsearch.extensions import filter +from stac_fastapi.types.errors import ConflictError, NotFoundError +from stac_fastapi.types.stac import Collection, Item +from stac_fastapi.elasticsearch.utilities import bbox2polygon + +logger = logging.getLogger(__name__) + +NumType = Union[float, int] + +COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections") +ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_") +ES_INDEX_NAME_UNSUPPORTED_CHARS = { + "\\", + "/", + "*", + "?", + '"', + "<", + ">", + "|", + " ", + ",", + "#", + ":", +} + +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*" + +DEFAULT_SORT = { + "properties.datetime": {"order": "desc"}, + "id": {"order": "desc"}, + "collection": {"order": "desc"}, +} + +ES_ITEMS_SETTINGS = { + "index": { + "sort.field": list(DEFAULT_SORT.keys()), + "sort.order": [v["order"] for v in DEFAULT_SORT.values()], + } +} + +ES_MAPPINGS_DYNAMIC_TEMPLATES = [ + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + { + "descriptions": { + "match_mapping_type": "string", + "match": "description", + "mapping": {"type": "text"}, + } + }, + { + "titles": { + "match_mapping_type": "string", + "match": "title", + "mapping": {"type": "text"}, + } + }, + # Projection Extension https://github.com/stac-extensions/projection + {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, + { + "proj_projjson": { + "match": "proj:projjson", + "mapping": {"type": "object", "enabled": False}, + } + }, + { + "proj_centroid": { + "match": "proj:centroid", + "mapping": {"type": "geo_point"}, + } + }, + { + "proj_geometry": { + "match": "proj:geometry", + "mapping": {"type": "object", "enabled": False}, + } + }, + { + "no_index_href": { + "match": "href", + "mapping": {"type": "text", "index": False}, + } + }, + # Default all other strings not otherwise specified to keyword + {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, + {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, +] + +ES_ITEMS_MAPPINGS = { + "numeric_detection": False, + "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, + "properties": { + "id": {"type": "keyword"}, + "collection": {"type": "keyword"}, + "geometry": {"type": "geo_shape"}, + "assets": {"type": "object", "enabled": False}, + "links": {"type": "object", "enabled": False}, + "properties": { + "type": "object", + "properties": { + # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md + "datetime": {"type": "date"}, + "start_datetime": {"type": "date"}, + "end_datetime": {"type": "date"}, + "created": {"type": "date"}, + "updated": {"type": "date"}, + # Satellite Extension https://github.com/stac-extensions/sat + "sat:absolute_orbit": {"type": "integer"}, + "sat:relative_orbit": {"type": "integer"}, + }, + }, + }, +} + +ES_COLLECTIONS_MAPPINGS = { + "numeric_detection": False, + "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, + "properties": { + "extent.spatial.bbox": {"type": "long"}, + "extent.temporal.interval": {"type": "date"}, + "providers": {"type": "object", "enabled": False}, + "links": {"type": "object", "enabled": False}, + "item_assets": {"type": "object", "enabled": False}, + }, +} + + +def index_by_collection_id(collection_id: str) -> str: + """ + Translate a collection id into an Elasticsearch index name. + + Args: + collection_id (str): The collection id to translate into an index name. + + Returns: + str: The index name derived from the collection id. + """ + return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id.lower() if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}" + + +def indices(collection_ids: Optional[List[str]]) -> str: + """ + Get a comma-separated string of index names for a given list of collection ids. + + Args: + collection_ids: A list of collection ids. + + Returns: + A string of comma-separated index names. If `collection_ids` is None, returns the default indices. + """ + if collection_ids is None: + return ITEM_INDICES + else: + return ",".join([index_by_collection_id(c) for c in collection_ids]) + + +async def create_collection_index() -> None: + """ + Create the index for a Collection. + + Returns: + None + + """ + client = AsyncElasticsearchSettings().create_client + + await client.options(ignore_status=400).indices.create( + index=f"{COLLECTIONS_INDEX}-000001", + aliases={COLLECTIONS_INDEX: {}}, + mappings=ES_COLLECTIONS_MAPPINGS, + ) + await client.close() + + +async def create_item_index(collection_id: str): + """ + Create the index for Items. + + Args: + collection_id (str): Collection identifier. + + Returns: + None + + """ + client = AsyncElasticsearchSettings().create_client + index_name = index_by_collection_id(collection_id) + + await client.options(ignore_status=400).indices.create( + index=f"{index_by_collection_id(collection_id)}-000001", + aliases={index_name: {}}, + mappings=ES_ITEMS_MAPPINGS, + settings=ES_ITEMS_SETTINGS, + ) + await client.close() + + +async def delete_item_index(collection_id: str): + """Delete the index for items in a collection. + + Args: + collection_id (str): The ID of the collection whose items index will be deleted. + """ + client = AsyncElasticsearchSettings().create_client + + name = index_by_collection_id(collection_id) + resolved = await client.indices.resolve_index(name=name) + if "aliases" in resolved and resolved["aliases"]: + [alias] = resolved["aliases"] + await client.indices.delete_alias(index=alias["indices"], name=alias["name"]) + await client.indices.delete(index=alias["indices"]) + else: + await client.indices.delete(index=name) + await client.close() + + +def mk_item_id(item_id: str, collection_id: str): + """Create the document id for an Item in Elasticsearch. + + Args: + item_id (str): The id of the Item. + collection_id (str): The id of the Collection that the Item belongs to. + + Returns: + str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character. + """ + return f"{item_id}|{collection_id}" + + +def mk_actions(collection_id: str, processed_items: List[Item]): + """Create Elasticsearch bulk actions for a list of processed items. + + Args: + collection_id (str): The identifier for the collection the items belong to. + processed_items (List[Item]): The list of processed items to be bulk indexed. + + Returns: + List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed, + each action being a dictionary with the following keys: + - `_index`: the index to store the document in. + - `_id`: the document's identifier. + - `_source`: the source of the document. + """ + return [ + { + "_index": index_by_collection_id(collection_id), + "_id": mk_item_id(item["id"], item["collection"]), + "_source": item, + } + for item in processed_items + ] + + +# stac_pydantic classes extend _GeometryBase, which doesn't have a type field, +# So create our own Protocol for typing +# Union[ Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon, GeometryCollection] +class Geometry(Protocol): # noqa + type: str + coordinates: Any + + +@attr.s +class DatabaseLogic: + """Database logic.""" + + client = AsyncElasticsearchSettings().create_client + sync_client = SyncElasticsearchSettings().create_client + + item_serializer: Type[serializers.ItemSerializer] = attr.ib( + default=serializers.ItemSerializer + ) + collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( + default=serializers.CollectionSerializer + ) + + """CORE LOGIC""" + + async def get_all_collections( + self, token: Optional[str], limit: int + ) -> Iterable[Dict[str, Any]]: + """Retrieve a list of all collections from the database. + + Args: + token (Optional[str]): The token used to return the next set of results. + limit (int): Number of results to return + + Returns: + collections (Iterable[Dict[str, Any]]): A list of dictionaries containing the source data for each collection. + + Notes: + The collections are retrieved from the Elasticsearch database using the `client.search` method, + with the `COLLECTIONS_INDEX` as the target index and `size=limit` to retrieve records. + The result is a generator of dictionaries containing the source data for each collection. + """ + search_after = None + if token: + search_after = urlsafe_b64decode(token.encode()).decode().split(",") + collections = await self.client.search( + index=COLLECTIONS_INDEX, + search_after=search_after, + size=limit, + sort={"id": {"order": "asc"}}, + ) + hits = collections["hits"]["hits"] + return hits + + async def get_one_item(self, collection_id: str, item_id: str) -> Dict: + """Retrieve a single item from the database. + + Args: + collection_id (str): The id of the Collection that the Item belongs to. + item_id (str): The id of the Item. + + Returns: + item (Dict): A dictionary containing the source data for the Item. + + Raises: + NotFoundError: If the specified Item does not exist in the Collection. + + Notes: + The Item is retrieved from the Elasticsearch database using the `client.get` method, + with the index for the Collection as the target index and the combined `mk_item_id` as the document id. + """ + try: + item = await self.client.get( + index=index_by_collection_id(collection_id), + id=mk_item_id(item_id, collection_id), + ) + except exceptions.NotFoundError: + raise NotFoundError( + f"Item {item_id} does not exist in Collection {collection_id}" + ) + return item["_source"] + + @staticmethod + def make_search(): + """Database logic to create a Search instance.""" + return Search().sort(*DEFAULT_SORT) + + @staticmethod + def apply_ids_filter(search: Search, item_ids: List[str]): + """Database logic to search a list of STAC item ids.""" + return search.filter("terms", id=item_ids) + + @staticmethod + def apply_collections_filter(search: Search, collection_ids: List[str]): + """Database logic to search a list of STAC collection ids.""" + return search.filter("terms", collection=collection_ids) + + @staticmethod + def apply_datetime_filter(search: Search, datetime_search): + """Apply a filter to search based on datetime field. + + Args: + search (Search): The search object to filter. + datetime_search (dict): The datetime filter criteria. + + Returns: + Search: The filtered search object. + """ + if "eq" in datetime_search: + search = search.filter( + "term", **{"properties__datetime": datetime_search["eq"]} + ) + else: + search = search.filter( + "range", properties__datetime={"lte": datetime_search["lte"]} + ) + search = search.filter( + "range", properties__datetime={"gte": datetime_search["gte"]} + ) + return search + + @staticmethod + def apply_bbox_filter(search: Search, bbox: List): + """Filter search results based on bounding box. + + Args: + search (Search): The search object to apply the filter to. + bbox (List): The bounding box coordinates, represented as a list of four values [minx, miny, maxx, maxy]. + + Returns: + search (Search): The search object with the bounding box filter applied. + + Notes: + The bounding box is transformed into a polygon using the `bbox2polygon` function and + a geo_shape filter is added to the search object, set to intersect with the specified polygon. + """ + return search.filter( + Q( + { + "geo_shape": { + "geometry": { + "shape": { + "type": "polygon", + "coordinates": bbox2polygon(*bbox), + }, + "relation": "intersects", + } + } + } + ) + ) + + @staticmethod + def apply_intersects_filter( + search: Search, + intersects: Geometry, + ): + """Filter search results based on intersecting geometry. + + Args: + search (Search): The search object to apply the filter to. + intersects (Geometry): The intersecting geometry, represented as a GeoJSON-like object. + + Returns: + search (Search): The search object with the intersecting geometry filter applied. + + Notes: + A geo_shape filter is added to the search object, set to intersect with the specified geometry. + """ + return search.filter( + Q( + { + "geo_shape": { + "geometry": { + "shape": { + "type": intersects.type.lower(), + "coordinates": intersects.coordinates, + }, + "relation": "intersects", + } + } + } + ) + ) + + @staticmethod + def apply_stacql_filter(search: Search, op: str, field: str, value: float): + """Filter search results based on a comparison between a field and a value. + + Args: + search (Search): The search object to apply the filter to. + op (str): The comparison operator to use. Can be 'eq' (equal), 'gt' (greater than), 'gte' (greater than or equal), + 'lt' (less than), or 'lte' (less than or equal). + field (str): The field to perform the comparison on. + value (float): The value to compare the field against. + + Returns: + search (Search): The search object with the specified filter applied. + """ + if op != "eq": + key_filter = {field: {f"{op}": value}} + search = search.filter(Q("range", **key_filter)) + else: + search = search.filter("term", **{field: value}) + + return search + + @staticmethod + def apply_cql2_filter(search: Search, _filter: Optional[Dict[str, Any]]): + """Database logic to perform query for search endpoint.""" + if _filter is not None: + search = search.filter(filter.Clause.parse_obj(_filter).to_es()) + return search + + @staticmethod + def populate_sort(sortby: List) -> Optional[Dict[str, Dict[str, str]]]: + """Database logic to sort search instance.""" + if sortby: + return {s.field: {"order": s.direction} for s in sortby} + else: + return None + + async def execute_search( + self, + search: Search, + limit: int, + token: Optional[str], + sort: Optional[Dict[str, Dict[str, str]]], + collection_ids: Optional[List[str]], + ignore_unavailable: bool = True, + ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: + """Execute a search query with limit and other optional parameters. + + Args: + search (Search): The search query to be executed. + limit (int): The maximum number of results to be returned. + token (Optional[str]): The token used to return the next set of results. + sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted. + collection_ids (Optional[List[str]]): The collection ids to search. + ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True. + + Returns: + Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: A tuple containing: + - An iterable of search results, where each result is a dictionary with keys and values representing the + fields and values of each document. + - The total number of results (if the count could be computed), or None if the count could not be + computed. + - The token to be used to retrieve the next set of results, or None if there are no more results. + + Raises: + NotFoundError: If the collections specified in `collection_ids` do not exist. + """ + search_after = None + if token: + search_after = urlsafe_b64decode(token.encode()).decode().split(",") + + query = search.query.to_dict() if search.query else None + + index_param = indices(collection_ids) + + search_task = asyncio.create_task( + self.client.search( + index=index_param, + ignore_unavailable=ignore_unavailable, + query=query, + sort=sort or DEFAULT_SORT, + search_after=search_after, + size=limit, + ) + ) + + count_task = asyncio.create_task( + self.client.count( + index=index_param, + ignore_unavailable=ignore_unavailable, + body=search.to_dict(count=True), + ) + ) + + try: + es_response = await search_task + except exceptions.NotFoundError: + raise NotFoundError(f"Collections '{collection_ids}' do not exist") + + hits = es_response["hits"]["hits"] + items = (hit["_source"] for hit in hits) + + next_token = None + if hits and (sort_array := hits[-1].get("sort")): + next_token = urlsafe_b64encode( + ",".join([str(x) for x in sort_array]).encode() + ).decode() + + # (1) count should not block returning results, so don't wait for it to be done + # (2) don't cancel the task so that it will populate the ES cache for subsequent counts + maybe_count = None + if count_task.done(): + try: + maybe_count = count_task.result().get("count") + except Exception as e: + logger.error(f"Count task failed: {e}") + + return items, maybe_count, next_token + + """ TRANSACTION LOGIC """ + + async def check_collection_exists(self, collection_id: str): + """Database logic to check if a collection exists.""" + if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): + raise NotFoundError(f"Collection {collection_id} does not exist") + + async def prep_create_item( + self, item: Item, base_url: str, exist_ok: bool = False + ) -> Item: + """ + Preps an item for insertion into the database. + + Args: + item (Item): The item to be prepped for insertion. + base_url (str): The base URL used to create the item's self URL. + exist_ok (bool): Indicates whether the item can exist already. + + Returns: + Item: The prepped item. + + Raises: + ConflictError: If the item already exists in the database. + + """ + await self.check_collection_exists(collection_id=item["collection"]) + + if not exist_ok and await self.client.exists( + index=index_by_collection_id(item["collection"]), + id=mk_item_id(item["id"], item["collection"]), + ): + raise ConflictError( + f"Item {item['id']} in collection {item['collection']} already exists" + ) + + return self.item_serializer.stac_to_db(item, base_url) + + def sync_prep_create_item( + self, item: Item, base_url: str, exist_ok: bool = False + ) -> Item: + """ + Prepare an item for insertion into the database. + + This method performs pre-insertion preparation on the given `item`, + such as checking if the collection the item belongs to exists, + and optionally verifying that an item with the same ID does not already exist in the database. + + Args: + item (Item): The item to be inserted into the database. + base_url (str): The base URL used for constructing URLs for the item. + exist_ok (bool): Indicates whether the item can exist already. + + Returns: + Item: The item after preparation is done. + + Raises: + NotFoundError: If the collection that the item belongs to does not exist in the database. + ConflictError: If an item with the same ID already exists in the collection. + """ + item_id = item["id"] + collection_id = item["collection"] + if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=collection_id): + raise NotFoundError(f"Collection {collection_id} does not exist") + + if not exist_ok and self.sync_client.exists( + index=index_by_collection_id(collection_id), + id=mk_item_id(item_id, collection_id), + ): + raise ConflictError( + f"Item {item_id} in collection {collection_id} already exists" + ) + + return self.item_serializer.stac_to_db(item, base_url) + + async def create_item(self, item: Item, refresh: bool = False): + """Database logic for creating one item. + + Args: + item (Item): The item to be created. + refresh (bool, optional): Refresh the index after performing the operation. Defaults to False. + + Raises: + ConflictError: If the item already exists in the database. + + Returns: + None + """ + # todo: check if collection exists, but cache + item_id = item["id"] + collection_id = item["collection"] + es_resp = await self.client.index( + index=index_by_collection_id(collection_id), + id=mk_item_id(item_id, collection_id), + document=item, + refresh=refresh, + ) + + if (meta := es_resp.get("meta")) and meta.get("status") == 409: + raise ConflictError( + f"Item {item_id} in collection {collection_id} already exists" + ) + + async def delete_item( + self, item_id: str, collection_id: str, refresh: bool = False + ): + """Delete a single item from the database. + + Args: + item_id (str): The id of the Item to be deleted. + collection_id (str): The id of the Collection that the Item belongs to. + refresh (bool, optional): Whether to refresh the index after the deletion. Default is False. + + Raises: + NotFoundError: If the Item does not exist in the database. + """ + try: + await self.client.delete( + index=index_by_collection_id(collection_id), + id=mk_item_id(item_id, collection_id), + refresh=refresh, + ) + except exceptions.NotFoundError: + raise NotFoundError( + f"Item {item_id} in collection {collection_id} not found" + ) + + async def create_collection(self, collection: Collection, refresh: bool = False): + """Create a single collection in the database. + + Args: + collection (Collection): The Collection object to be created. + refresh (bool, optional): Whether to refresh the index after the creation. Default is False. + + Raises: + ConflictError: If a Collection with the same id already exists in the database. + + Notes: + A new index is created for the items in the Collection using the `create_item_index` function. + """ + collection_id = collection["id"] + + if await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): + raise ConflictError(f"Collection {collection_id} already exists") + + await self.client.index( + index=COLLECTIONS_INDEX, + id=collection_id, + document=collection, + refresh=refresh, + ) + + await create_item_index(collection_id) + + async def find_collection(self, collection_id: str) -> Collection: + """Find and return a collection from the database. + + Args: + self: The instance of the object calling this function. + collection_id (str): The ID of the collection to be found. + + Returns: + Collection: The found collection, represented as a `Collection` object. + + Raises: + NotFoundError: If the collection with the given `collection_id` is not found in the database. + + Notes: + This function searches for a collection in the database using the specified `collection_id` and returns the found + collection as a `Collection` object. If the collection is not found, a `NotFoundError` is raised. + """ + try: + collection = await self.client.get( + index=COLLECTIONS_INDEX, id=collection_id + ) + except exceptions.NotFoundError: + raise NotFoundError(f"Collection {collection_id} not found") + + return collection["_source"] + + async def delete_collection(self, collection_id: str, refresh: bool = False): + """Delete a collection from the database. + + Parameters: + self: The instance of the object calling this function. + collection_id (str): The ID of the collection to be deleted. + refresh (bool): Whether to refresh the index after the deletion (default: False). + + Raises: + NotFoundError: If the collection with the given `collection_id` is not found in the database. + + Notes: + This function first verifies that the collection with the specified `collection_id` exists in the database, and then + deletes the collection. If `refresh` is set to True, the index is refreshed after the deletion. Additionally, this + function also calls `delete_item_index` to delete the index for the items in the collection. + """ + await self.find_collection(collection_id=collection_id) + await self.client.delete( + index=COLLECTIONS_INDEX, id=collection_id, refresh=refresh + ) + await delete_item_index(collection_id) + + async def bulk_async( + self, collection_id: str, processed_items: List[Item], refresh: bool = False + ) -> None: + """Perform a bulk insert of items into the database asynchronously. + + Args: + self: The instance of the object calling this function. + collection_id (str): The ID of the collection to which the items belong. + processed_items (List[Item]): A list of `Item` objects to be inserted into the database. + refresh (bool): Whether to refresh the index after the bulk insert (default: False). + + Notes: + This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`. The + insert is performed asynchronously, and the event loop is used to run the operation in a separate executor. The + `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True, the + index is refreshed after the bulk insert. The function does not return any value. + """ + await helpers.async_bulk( + self.client, + mk_actions(collection_id, processed_items), + refresh=refresh, + raise_on_error=False, + ) + + def bulk_sync( + self, collection_id: str, processed_items: List[Item], refresh: bool = False + ) -> None: + """Perform a bulk insert of items into the database synchronously. + + Args: + self: The instance of the object calling this function. + collection_id (str): The ID of the collection to which the items belong. + processed_items (List[Item]): A list of `Item` objects to be inserted into the database. + refresh (bool): Whether to refresh the index after the bulk insert (default: False). + + Notes: + This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`. The + insert is performed synchronously and blocking, meaning that the function does not return until the insert has + completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to + True, the index is refreshed after the bulk insert. The function does not return any value. + """ + helpers.bulk( + self.sync_client, + mk_actions(collection_id, processed_items), + refresh=refresh, + raise_on_error=False, + ) + + # DANGER + async def delete_items(self) -> None: + """Danger. this is only for tests.""" + await self.client.delete_by_query( + index=ITEM_INDICES, + body={"query": {"match_all": {}}}, + wait_for_completion=True, + ) + + # DANGER + async def delete_collections(self) -> None: + """Danger. this is only for tests.""" + await self.client.delete_by_query( + index=COLLECTIONS_INDEX, + body={"query": {"match_all": {}}}, + wait_for_completion=True, + ) From c1b1da1487ac893e9b82d59b7e69fab9b83fa9ec Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 29 Jan 2024 12:43:30 +0800 Subject: [PATCH 07/23] add opensearch to setup --- stac_fastapi/elasticsearch/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 3106c512..bd38a3a3 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -36,6 +36,9 @@ ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], "server": ["uvicorn[standard]==0.19.0"], + "elasticsearch": [], + "opensearch": [], + } setup( From fc3cefb4d3d179b2772a4c1b2df17533c9257cb9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 29 Jan 2024 12:44:35 +0800 Subject: [PATCH 08/23] app/ core plumbing --- .../stac_fastapi/elasticsearch/app.py | 16 ++++++++++++++-- .../stac_fastapi/elasticsearch/core.py | 11 +++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index fabfbb4f..820c5e0f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -1,4 +1,6 @@ """FastAPI application.""" +import os + from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model from stac_fastapi.elasticsearch.config import ElasticsearchSettings @@ -8,7 +10,8 @@ EsAsyncBaseFiltersClient, TransactionsClient, ) -from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index +# from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index +# from stac_fastapi.elasticsearch.database_opensearch.database_logic import create_collection_index from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.core import ( @@ -21,6 +24,13 @@ ) from stac_fastapi.extensions.third_party import BulkTransactionExtension +db_type = os.getenv("DB_TYPE", "elasticsearch").lower() + +if db_type == "opensearch": + from stac_fastapi.elasticsearch.database_opensearch.database_logic import create_collection_index +else: + from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index + settings = ElasticsearchSettings() session = Session.create_from_settings(settings) @@ -41,11 +51,13 @@ ] post_request_model = create_post_request_model(extensions) +core_client = CoreClient(session=session, post_request_model=post_request_model) +core_client.set_database_logic(db_type) api = StacApi( settings=settings, extensions=extensions, - client=CoreClient(session=session, post_request_model=post_request_model), + client=core_client, search_get_request_model=create_get_request_model(extensions), search_post_request_model=post_request_model, ) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index d65190fd..ed753f64 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -20,7 +20,7 @@ from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings -from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic +# from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic from stac_fastapi.elasticsearch.models.links import PagingLinks from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.elasticsearch.session import Session @@ -71,8 +71,15 @@ class CoreClient(AsyncBaseCoreClient): collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( default=serializers.CollectionSerializer ) - database = DatabaseLogic() + def set_database_logic(self, db_type): + if db_type == "opensearch": + from stac_fastapi.elasticsearch.database_opensearch.database_logic import DatabaseLogic + else: + from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic + + self.database = DatabaseLogic() + @overrides async def all_collections(self, **kwargs) -> Collections: """Read all collections from the database. From b5e45695171edf0e541152184332d5147e50a922 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 29 Jan 2024 12:58:39 +0800 Subject: [PATCH 09/23] pre-commit fixes --- stac_fastapi/elasticsearch/setup.py | 1 - .../stac_fastapi/elasticsearch/app.py | 19 ++++-- .../stac_fastapi/elasticsearch/core.py | 66 +++++++++++++++++-- .../database_elasticsearch/database_logic.py | 2 +- .../database_opensearch/database_logic.py | 2 +- .../stac_fastapi/elasticsearch/utilities.py | 7 ++ stac_fastapi/elasticsearch/tests/conftest.py | 4 +- 7 files changed, 87 insertions(+), 14 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index bd38a3a3..12dd3121 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -38,7 +38,6 @@ "server": ["uvicorn[standard]==0.19.0"], "elasticsearch": [], "opensearch": [], - } setup( diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 820c5e0f..dbc0174e 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -10,6 +10,7 @@ EsAsyncBaseFiltersClient, TransactionsClient, ) + # from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index # from stac_fastapi.elasticsearch.database_opensearch.database_logic import create_collection_index from stac_fastapi.elasticsearch.extensions import QueryExtension @@ -27,9 +28,13 @@ db_type = os.getenv("DB_TYPE", "elasticsearch").lower() if db_type == "opensearch": - from stac_fastapi.elasticsearch.database_opensearch.database_logic import create_collection_index + from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( + create_collection_index, + ) else: - from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index + from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( + create_collection_index, + ) settings = ElasticsearchSettings() session = Session.create_from_settings(settings) @@ -39,9 +44,15 @@ "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" ) +transactions_client = TransactionsClient(session=session) +transactions_client.set_database_logic(db_type) + +bulk_transactions_client = BulkTransactionsClient(session=session) +bulk_transactions_client.set_database_logic(db_type) + extensions = [ - TransactionExtension(client=TransactionsClient(session=session), settings=settings), - BulkTransactionExtension(client=BulkTransactionsClient(session=session)), + TransactionExtension(client=transactions_client, settings=settings), + BulkTransactionExtension(client=bulk_transactions_client), FieldsExtension(), QueryExtension(), SortExtension(), diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index ed753f64..25659a1e 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -20,6 +20,7 @@ from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings + # from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic from stac_fastapi.elasticsearch.models.links import PagingLinks from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer @@ -73,13 +74,26 @@ class CoreClient(AsyncBaseCoreClient): ) def set_database_logic(self, db_type): + """ + Set the database logic for the CoreClient. + + This method dynamically sets the database logic based on the specified database type. + It supports switching between Elasticsearch and OpenSearch databases. + + Args: + db_type (str): The database type, either 'elasticsearch' or 'opensearch'. + """ if db_type == "opensearch": - from stac_fastapi.elasticsearch.database_opensearch.database_logic import DatabaseLogic + from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( + DatabaseLogic, + ) else: - from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic - + from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( + DatabaseLogic, + ) + self.database = DatabaseLogic() - + @overrides async def all_collections(self, **kwargs) -> Collections: """Read all collections from the database. @@ -549,7 +563,27 @@ class TransactionsClient(AsyncBaseTransactionsClient): """Transactions extension specific CRUD operations.""" session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database = DatabaseLogic() + + def set_database_logic(self, db_type): + """ + Set the database logic for the CoreClient. + + This method dynamically sets the database logic based on the specified database type. + It supports switching between Elasticsearch and OpenSearch databases. + + Args: + db_type (str): The database type, either 'elasticsearch' or 'opensearch'. + """ + if db_type == "opensearch": + from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( + DatabaseLogic, + ) + else: + from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( + DatabaseLogic, + ) + + self.database = DatabaseLogic() @overrides async def create_item( @@ -719,7 +753,27 @@ class BulkTransactionsClient(BaseBulkTransactionsClient): """ session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database = DatabaseLogic() + + def set_database_logic(self, db_type): + """ + Set the database logic for the CoreClient. + + This method dynamically sets the database logic based on the specified database type. + It supports switching between Elasticsearch and OpenSearch databases. + + Args: + db_type (str): The database type, either 'elasticsearch' or 'opensearch'. + """ + if db_type == "opensearch": + from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( + DatabaseLogic, + ) + else: + from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( + DatabaseLogic, + ) + + self.database = DatabaseLogic() def __attrs_post_init__(self): """Create es engine.""" diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py index 26d830f0..3b709ff2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py @@ -15,9 +15,9 @@ ElasticsearchSettings as SyncElasticsearchSettings, ) from stac_fastapi.elasticsearch.extensions import filter +from stac_fastapi.elasticsearch.utilities import bbox2polygon from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item -from stac_fastapi.elasticsearch.utilities import bbox2polygon logger = logging.getLogger(__name__) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py index 26d830f0..3b709ff2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py @@ -15,9 +15,9 @@ ElasticsearchSettings as SyncElasticsearchSettings, ) from stac_fastapi.elasticsearch.extensions import filter +from stac_fastapi.elasticsearch.utilities import bbox2polygon from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item -from stac_fastapi.elasticsearch.utilities import bbox2polygon logger = logging.getLogger(__name__) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py index 7be319a7..4646cfe0 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py @@ -1,5 +1,12 @@ +"""Module for geospatial processing functions. + +This module contains functions for transforming geospatial coordinates, +such as converting bounding boxes to polygon representations. +""" + from typing import List + def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index 888ec8e3..2ec7df42 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -16,7 +16,9 @@ CoreClient, TransactionsClient, ) -from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index +from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( + create_collection_index, +) from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, From 6cc3b5c62c93ce73d5868b3a949b8769f28bd231 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 29 Jan 2024 18:26:08 +0800 Subject: [PATCH 10/23] use attr post init --- .../stac_fastapi/elasticsearch/app.py | 17 ++----- .../stac_fastapi/elasticsearch/core.py | 48 +++++++------------ 2 files changed, 21 insertions(+), 44 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index dbc0174e..a050f23a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -10,9 +10,6 @@ EsAsyncBaseFiltersClient, TransactionsClient, ) - -# from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import create_collection_index -# from stac_fastapi.elasticsearch.database_opensearch.database_logic import create_collection_index from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.core import ( @@ -44,15 +41,9 @@ "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" ) -transactions_client = TransactionsClient(session=session) -transactions_client.set_database_logic(db_type) - -bulk_transactions_client = BulkTransactionsClient(session=session) -bulk_transactions_client.set_database_logic(db_type) - extensions = [ - TransactionExtension(client=transactions_client, settings=settings), - BulkTransactionExtension(client=bulk_transactions_client), + TransactionExtension(client=TransactionsClient(session=session), settings=settings), + BulkTransactionExtension(client=BulkTransactionsClient(session=session)), FieldsExtension(), QueryExtension(), SortExtension(), @@ -62,13 +53,11 @@ ] post_request_model = create_post_request_model(extensions) -core_client = CoreClient(session=session, post_request_model=post_request_model) -core_client.set_database_logic(db_type) api = StacApi( settings=settings, extensions=extensions, - client=core_client, + client=CoreClient(session=session, post_request_model=post_request_model), search_get_request_model=create_get_request_model(extensions), search_post_request_model=post_request_model, ) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 25659a1e..8aee30c5 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -1,5 +1,6 @@ """Item crud client.""" import logging +import os import re from base64 import urlsafe_b64encode from datetime import datetime as datetime_type @@ -20,8 +21,6 @@ from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings - -# from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import DatabaseLogic from stac_fastapi.elasticsearch.models.links import PagingLinks from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer from stac_fastapi.elasticsearch.session import Session @@ -73,16 +72,15 @@ class CoreClient(AsyncBaseCoreClient): default=serializers.CollectionSerializer ) - def set_database_logic(self, db_type): + def __attrs_post_init__(self): """ - Set the database logic for the CoreClient. - - This method dynamically sets the database logic based on the specified database type. - It supports switching between Elasticsearch and OpenSearch databases. + Post-initialization method for CoreClient. - Args: - db_type (str): The database type, either 'elasticsearch' or 'opensearch'. + This method is automatically called after the CoreClient instance is initialized. + It sets the 'database' attribute based on the DB_TYPE environment variable, + choosing between Elasticsearch and OpenSearch database logic. """ + db_type = os.getenv("DB_TYPE", "elasticsearch").lower() if db_type == "opensearch": from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( DatabaseLogic, @@ -564,16 +562,15 @@ class TransactionsClient(AsyncBaseTransactionsClient): session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - def set_database_logic(self, db_type): + def __attrs_post_init__(self): """ - Set the database logic for the CoreClient. + Post-initialization method for CoreClient. - This method dynamically sets the database logic based on the specified database type. - It supports switching between Elasticsearch and OpenSearch databases. - - Args: - db_type (str): The database type, either 'elasticsearch' or 'opensearch'. + This method is automatically called after the CoreClient instance is initialized. + It sets the 'database' attribute based on the DB_TYPE environment variable, + choosing between Elasticsearch and OpenSearch database logic. """ + db_type = os.getenv("DB_TYPE", "elasticsearch").lower() if db_type == "opensearch": from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( DatabaseLogic, @@ -754,16 +751,12 @@ class BulkTransactionsClient(BaseBulkTransactionsClient): session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - def set_database_logic(self, db_type): - """ - Set the database logic for the CoreClient. - - This method dynamically sets the database logic based on the specified database type. - It supports switching between Elasticsearch and OpenSearch databases. + def __attrs_post_init__(self): + """Create es engine, database logic type.""" + settings = ElasticsearchSettings() + self.client = settings.create_client - Args: - db_type (str): The database type, either 'elasticsearch' or 'opensearch'. - """ + db_type = os.getenv("DB_TYPE", "elasticsearch").lower() if db_type == "opensearch": from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( DatabaseLogic, @@ -775,11 +768,6 @@ def set_database_logic(self, db_type): self.database = DatabaseLogic() - def __attrs_post_init__(self): - """Create es engine.""" - settings = ElasticsearchSettings() - self.client = settings.create_client - def preprocess_item( self, item: stac_types.Item, base_url, method: BulkTransactionMethod ) -> stac_types.Item: From bb3b0a0a375b5cc4fbb8280bdaa9d1918dff5478 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 29 Jan 2024 19:31:51 +0800 Subject: [PATCH 11/23] install opensearch-py --- stac_fastapi/elasticsearch/setup.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 12dd3121..6f50446c 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -1,4 +1,4 @@ -"""stac_fastapi: elasticsearch module.""" +"""stac_fastapi: elasticsearch. opensearch module.""" from setuptools import find_namespace_packages, setup @@ -8,13 +8,13 @@ install_requires = [ "fastapi", "attrs", + "elasticsearch[async]==8.11.0", + "elasticsearch-dsl==8.11.0", "pydantic[dotenv]<2", "stac_pydantic==2.0.*", "stac-fastapi.types==2.4.9", "stac-fastapi.api==2.4.9", "stac-fastapi.extensions==2.4.9", - "elasticsearch[async]==8.11.0", - "elasticsearch-dsl==8.11.0", "pystac[validation]", "uvicorn", "orjson", @@ -36,8 +36,7 @@ ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], "server": ["uvicorn[standard]==0.19.0"], - "elasticsearch": [], - "opensearch": [], + "opensearch": ["opensearch-py==2.4.2"], } setup( From 6b1db4143a75c9054c5481868b1a31b6ce6874dc Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 30 Jan 2024 01:16:00 +0800 Subject: [PATCH 12/23] separate modules --- Dockerfile.dev | 4 +- docker-compose.yml | 4 +- examples/pip_docker/docker-compose.yml | 2 +- stac_fastapi/common/__init__.py | 1 + stac_fastapi/common/base_database_logic.py | 37 + .../elasticsearch => common}/core.py | 72 +- .../datetime_utils.py | 0 .../extensions/__init__.py | 0 .../extensions/filter.py | 0 .../extensions/query.py | 0 .../models/__init__.py | 0 .../elasticsearch => common}/models/links.py | 0 .../elasticsearch => common}/models/search.py | 0 .../elasticsearch => common}/types/search.py | 0 .../README.md | 0 .../__init__.py | 0 .../elasticsearch => elastic_search}/app.py | 27 +- .../config.py | 0 .../database_logic.py | 10 +- .../pytest.ini | 0 .../serializers.py | 2 +- .../session.py | 0 stac_fastapi/elastic_search/setup.cfg | 2 + .../setup.py | 10 +- .../tests/__init__.py | 0 .../tests/api/__init__.py | 0 .../tests/api/test_api.py | 0 .../tests/clients/__init__.py | 0 .../tests/clients/test_elasticsearch.py | 0 .../tests/conftest.py | 8 +- .../tests/data/test_collection.json | 0 .../tests/data/test_item.json | 0 .../tests/extensions/cql2/example01.json | 0 .../tests/extensions/cql2/example04.json | 0 .../tests/extensions/cql2/example05a.json | 0 .../tests/extensions/cql2/example06b.json | 0 .../tests/extensions/cql2/example08.json | 0 .../tests/extensions/cql2/example09.json | 0 .../tests/extensions/cql2/example1.json | 0 .../tests/extensions/cql2/example10.json | 0 .../tests/extensions/cql2/example14.json | 0 .../tests/extensions/cql2/example15.json | 0 .../tests/extensions/cql2/example17.json | 0 .../tests/extensions/cql2/example18.json | 0 .../tests/extensions/cql2/example19.json | 0 .../tests/extensions/cql2/example20.json | 0 .../tests/extensions/cql2/example21.json | 0 .../tests/extensions/cql2/example22.json | 0 .../tests/extensions/test_filter.py | 0 .../tests/resources/__init__.py | 0 .../tests/resources/test_collection.py | 0 .../tests/resources/test_conformance.py | 0 .../tests/resources/test_item.py | 4 +- .../tests/resources/test_mgmt.py | 0 .../utilities.py | 0 .../version.py | 0 stac_fastapi/elasticsearch/setup.cfg | 2 - .../database_elasticsearch/database_logic.py | 838 ------------------ stac_fastapi/open_search/app.py | 93 ++ stac_fastapi/open_search/setup.cfg | 2 + stac_fastapi/open_search/setup.py | 72 ++ stac_fastapi/open_search/version.py | 2 + 62 files changed, 250 insertions(+), 942 deletions(-) create mode 100644 stac_fastapi/common/__init__.py create mode 100644 stac_fastapi/common/base_database_logic.py rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/core.py (92%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/datetime_utils.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/extensions/__init__.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/extensions/filter.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/extensions/query.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/models/__init__.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/models/links.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/models/search.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => common}/types/search.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/README.md (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => elastic_search}/__init__.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => elastic_search}/app.py (78%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => elastic_search}/config.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch/database_opensearch => elastic_search}/database_logic.py (99%) rename stac_fastapi/{elasticsearch => elastic_search}/pytest.ini (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => elastic_search}/serializers.py (98%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => elastic_search}/session.py (100%) create mode 100644 stac_fastapi/elastic_search/setup.cfg rename stac_fastapi/{elasticsearch => elastic_search}/setup.py (87%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/__init__.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/api/__init__.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/api/test_api.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/clients/__init__.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/clients/test_elasticsearch.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/conftest.py (94%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/data/test_collection.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/data/test_item.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example01.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example04.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example05a.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example06b.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example08.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example09.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example1.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example10.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example14.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example15.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example17.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example18.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example19.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example20.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example21.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/cql2/example22.json (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/extensions/test_filter.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/resources/__init__.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/resources/test_collection.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/resources/test_conformance.py (100%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/resources/test_item.py (99%) rename stac_fastapi/{elasticsearch => elastic_search}/tests/resources/test_mgmt.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => elastic_search}/utilities.py (100%) rename stac_fastapi/{elasticsearch/stac_fastapi/elasticsearch => elastic_search}/version.py (100%) delete mode 100644 stac_fastapi/elasticsearch/setup.cfg delete mode 100644 stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py create mode 100644 stac_fastapi/open_search/app.py create mode 100644 stac_fastapi/open_search/setup.cfg create mode 100644 stac_fastapi/open_search/setup.py create mode 100644 stac_fastapi/open_search/version.py diff --git a/Dockerfile.dev b/Dockerfile.dev index 4e2f0f4b..e283e4c1 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -13,6 +13,6 @@ ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt WORKDIR /app -COPY . /app +COPY /stac_fastapi /app -RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server] +RUN pip install --no-cache-dir -e ./elastic_search[dev,server] diff --git a/docker-compose.yml b/docker-compose.yml index 3eaae906..ddc3c6f6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,7 +28,7 @@ services: depends_on: - elasticsearch command: - bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app" + bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elastic_search.app" app-opensearch: container_name: stac-fastapi-os @@ -57,7 +57,7 @@ services: depends_on: - opensearch command: - bash -c "./scripts/wait-for-it-es.sh os-container:9200 && python -m stac_fastapi.elasticsearch.app" + bash -c "./scripts/wait-for-it-es.sh os-container:9200 && python -m stac_fastapi.elastic_search.app" elasticsearch: container_name: es-container diff --git a/examples/pip_docker/docker-compose.yml b/examples/pip_docker/docker-compose.yml index 2c2d289e..8c76967d 100644 --- a/examples/pip_docker/docker-compose.yml +++ b/examples/pip_docker/docker-compose.yml @@ -27,7 +27,7 @@ services: depends_on: - elasticsearch command: - bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app" + bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elastic_search.app" elasticsearch: container_name: es-container diff --git a/stac_fastapi/common/__init__.py b/stac_fastapi/common/__init__.py new file mode 100644 index 00000000..69f055ab --- /dev/null +++ b/stac_fastapi/common/__init__.py @@ -0,0 +1 @@ +"""elasticsearch/ opensearch shared code.""" diff --git a/stac_fastapi/common/base_database_logic.py b/stac_fastapi/common/base_database_logic.py new file mode 100644 index 00000000..2478722b --- /dev/null +++ b/stac_fastapi/common/base_database_logic.py @@ -0,0 +1,37 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict, List, Optional, Tuple, Union +from stac_fastapi.types.stac import Collection, Item + +class BaseDatabaseLogic(ABC): + """ + Abstract base class for database logic. + This class defines the interface for database operations. + """ + + @abstractmethod + async def get_all_collections(self, token: Optional[str], limit: int) -> Any: + pass + + @abstractmethod + async def get_one_item(self, collection_id: str, item_id: str) -> Dict: + pass + + @abstractmethod + async def create_item(self, item: Item, refresh: bool = False) -> None: + pass + + @abstractmethod + async def delete_item(self, item_id: str, collection_id: str, refresh: bool = False) -> None: + pass + + @abstractmethod + async def create_collection(self, collection: Collection, refresh: bool = False) -> None: + pass + + @abstractmethod + async def find_collection(self, collection_id: str) -> Collection: + pass + + @abstractmethod + async def delete_collection(self, collection_id: str, refresh: bool = False) -> None: + pass \ No newline at end of file diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/common/core.py similarity index 92% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py rename to stac_fastapi/common/core.py index 8aee30c5..927273f3 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/common/core.py @@ -1,6 +1,5 @@ """Item crud client.""" import logging -import os import re from base64 import urlsafe_b64encode from datetime import datetime as datetime_type @@ -11,6 +10,7 @@ import attr import orjson import stac_pydantic +from elastic_search.database_logic import DatabaseLogic as BaseDatabaseLogic from fastapi import HTTPException, Request from overrides import overrides from pydantic import ValidationError @@ -19,11 +19,11 @@ from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes -from stac_fastapi.elasticsearch import serializers -from stac_fastapi.elasticsearch.config import ElasticsearchSettings -from stac_fastapi.elasticsearch.models.links import PagingLinks -from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer -from stac_fastapi.elasticsearch.session import Session +from elastic_search import serializers +from elastic_search.config import ElasticsearchSettings +from common.models.links import PagingLinks +from elastic_search.serializers import CollectionSerializer, ItemSerializer +from elastic_search.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, BulkTransactionMethod, @@ -71,26 +71,7 @@ class CoreClient(AsyncBaseCoreClient): collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( default=serializers.CollectionSerializer ) - - def __attrs_post_init__(self): - """ - Post-initialization method for CoreClient. - - This method is automatically called after the CoreClient instance is initialized. - It sets the 'database' attribute based on the DB_TYPE environment variable, - choosing between Elasticsearch and OpenSearch database logic. - """ - db_type = os.getenv("DB_TYPE", "elasticsearch").lower() - if db_type == "opensearch": - from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( - DatabaseLogic, - ) - else: - from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( - DatabaseLogic, - ) - - self.database = DatabaseLogic() + database = BaseDatabaseLogic() @overrides async def all_collections(self, **kwargs) -> Collections: @@ -561,26 +542,7 @@ class TransactionsClient(AsyncBaseTransactionsClient): """Transactions extension specific CRUD operations.""" session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - - def __attrs_post_init__(self): - """ - Post-initialization method for CoreClient. - - This method is automatically called after the CoreClient instance is initialized. - It sets the 'database' attribute based on the DB_TYPE environment variable, - choosing between Elasticsearch and OpenSearch database logic. - """ - db_type = os.getenv("DB_TYPE", "elasticsearch").lower() - if db_type == "opensearch": - from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( - DatabaseLogic, - ) - else: - from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( - DatabaseLogic, - ) - - self.database = DatabaseLogic() + database = BaseDatabaseLogic() @overrides async def create_item( @@ -750,23 +712,7 @@ class BulkTransactionsClient(BaseBulkTransactionsClient): """ session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - - def __attrs_post_init__(self): - """Create es engine, database logic type.""" - settings = ElasticsearchSettings() - self.client = settings.create_client - - db_type = os.getenv("DB_TYPE", "elasticsearch").lower() - if db_type == "opensearch": - from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( - DatabaseLogic, - ) - else: - from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( - DatabaseLogic, - ) - - self.database = DatabaseLogic() + database = BaseDatabaseLogic() def preprocess_item( self, item: stac_types.Item, base_url, method: BulkTransactionMethod diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/datetime_utils.py b/stac_fastapi/common/datetime_utils.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/datetime_utils.py rename to stac_fastapi/common/datetime_utils.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/__init__.py b/stac_fastapi/common/extensions/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/__init__.py rename to stac_fastapi/common/extensions/__init__.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_fastapi/common/extensions/filter.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py rename to stac_fastapi/common/extensions/filter.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/query.py b/stac_fastapi/common/extensions/query.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/query.py rename to stac_fastapi/common/extensions/query.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/__init__.py b/stac_fastapi/common/models/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/__init__.py rename to stac_fastapi/common/models/__init__.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/links.py b/stac_fastapi/common/models/links.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/links.py rename to stac_fastapi/common/models/links.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/search.py b/stac_fastapi/common/models/search.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/search.py rename to stac_fastapi/common/models/search.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/types/search.py b/stac_fastapi/common/types/search.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/types/search.py rename to stac_fastapi/common/types/search.py diff --git a/stac_fastapi/elasticsearch/README.md b/stac_fastapi/elastic_search/README.md similarity index 100% rename from stac_fastapi/elasticsearch/README.md rename to stac_fastapi/elastic_search/README.md diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/__init__.py b/stac_fastapi/elastic_search/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/__init__.py rename to stac_fastapi/elastic_search/__init__.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elastic_search/app.py similarity index 78% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py rename to stac_fastapi/elastic_search/app.py index a050f23a..294c8312 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elastic_search/app.py @@ -1,17 +1,17 @@ """FastAPI application.""" -import os - from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_fastapi.elasticsearch.config import ElasticsearchSettings -from stac_fastapi.elasticsearch.core import ( +from elastic_search.config import ElasticsearchSettings +from stac_fastapi.common.core import ( BulkTransactionsClient, CoreClient, EsAsyncBaseFiltersClient, TransactionsClient, ) -from stac_fastapi.elasticsearch.extensions import QueryExtension -from stac_fastapi.elasticsearch.session import Session +from elastic_search.database_logic import DatabaseLogic +from stac_fastapi.common.extensions import QueryExtension +from elastic_search.session import Session +from elastic_search.database_logic import create_collection_index from stac_fastapi.extensions.core import ( ContextExtension, FieldsExtension, @@ -22,17 +22,6 @@ ) from stac_fastapi.extensions.third_party import BulkTransactionExtension -db_type = os.getenv("DB_TYPE", "elasticsearch").lower() - -if db_type == "opensearch": - from stac_fastapi.elasticsearch.database_opensearch.database_logic import ( - create_collection_index, - ) -else: - from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( - create_collection_index, - ) - settings = ElasticsearchSettings() session = Session.create_from_settings(settings) @@ -41,6 +30,8 @@ "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" ) +database_logic = DatabaseLogic() + extensions = [ TransactionExtension(client=TransactionsClient(session=session), settings=settings), BulkTransactionExtension(client=BulkTransactionsClient(session=session)), @@ -75,7 +66,7 @@ def run() -> None: import uvicorn uvicorn.run( - "stac_fastapi.elasticsearch.app:app", + "stac_fastapi.elastic_search.app:app", host=settings.app_host, port=settings.app_port, log_level="info", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elastic_search/config.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py rename to stac_fastapi/elastic_search/config.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py b/stac_fastapi/elastic_search/database_logic.py similarity index 99% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py rename to stac_fastapi/elastic_search/database_logic.py index 3b709ff2..88d46328 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_opensearch/database_logic.py +++ b/stac_fastapi/elastic_search/database_logic.py @@ -9,13 +9,13 @@ from elasticsearch_dsl import Q, Search from elasticsearch import exceptions, helpers # type: ignore -from stac_fastapi.elasticsearch import serializers -from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings -from stac_fastapi.elasticsearch.config import ( +from elastic_search import serializers +from elastic_search.config import AsyncElasticsearchSettings +from elastic_search.config import ( ElasticsearchSettings as SyncElasticsearchSettings, ) -from stac_fastapi.elasticsearch.extensions import filter -from stac_fastapi.elasticsearch.utilities import bbox2polygon +from common.extensions import filter +from elastic_search.utilities import bbox2polygon from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item diff --git a/stac_fastapi/elasticsearch/pytest.ini b/stac_fastapi/elastic_search/pytest.ini similarity index 100% rename from stac_fastapi/elasticsearch/pytest.ini rename to stac_fastapi/elastic_search/pytest.ini diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py b/stac_fastapi/elastic_search/serializers.py similarity index 98% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py rename to stac_fastapi/elastic_search/serializers.py index 725e8f65..fe08c9b3 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py +++ b/stac_fastapi/elastic_search/serializers.py @@ -4,7 +4,7 @@ import attr -from stac_fastapi.elasticsearch.datetime_utils import now_to_rfc3339_str +from common.datetime_utils import now_to_rfc3339_str from stac_fastapi.types import stac as stac_types from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/session.py b/stac_fastapi/elastic_search/session.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/session.py rename to stac_fastapi/elastic_search/session.py diff --git a/stac_fastapi/elastic_search/setup.cfg b/stac_fastapi/elastic_search/setup.cfg new file mode 100644 index 00000000..f1023827 --- /dev/null +++ b/stac_fastapi/elastic_search/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +version = attr: version.__version__ diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elastic_search/setup.py similarity index 87% rename from stac_fastapi/elasticsearch/setup.py rename to stac_fastapi/elastic_search/setup.py index 6f50446c..033d10d4 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elastic_search/setup.py @@ -1,4 +1,4 @@ -"""stac_fastapi: elasticsearch. opensearch module.""" +"""stac_fastapi: elasticsearch module.""" from setuptools import find_namespace_packages, setup @@ -36,7 +36,6 @@ ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], "server": ["uvicorn[standard]==0.19.0"], - "opensearch": ["opensearch-py==2.4.2"], } setup( @@ -57,14 +56,17 @@ ], url="https://github.com/stac-utils/stac-fastapi-elasticsearch", license="MIT", - packages=find_namespace_packages(exclude=["alembic", "tests", "scripts"]), + packages=find_namespace_packages( + include=["elasticsearch", "elasticsearch.*", "common", "common.*"], + exclude=["tests", "scripts"], + ), zip_safe=False, install_requires=install_requires, tests_require=extra_reqs["dev"], extras_require=extra_reqs, entry_points={ "console_scripts": [ - "stac-fastapi-elasticsearch=stac_fastapi.elasticsearch.app:run" + "stac-fastapi-elasticsearch=app:run" ] }, ) diff --git a/stac_fastapi/elasticsearch/tests/__init__.py b/stac_fastapi/elastic_search/tests/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/__init__.py rename to stac_fastapi/elastic_search/tests/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/api/__init__.py b/stac_fastapi/elastic_search/tests/api/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/api/__init__.py rename to stac_fastapi/elastic_search/tests/api/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elastic_search/tests/api/test_api.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/api/test_api.py rename to stac_fastapi/elastic_search/tests/api/test_api.py diff --git a/stac_fastapi/elasticsearch/tests/clients/__init__.py b/stac_fastapi/elastic_search/tests/clients/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/clients/__init__.py rename to stac_fastapi/elastic_search/tests/clients/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py b/stac_fastapi/elastic_search/tests/clients/test_elasticsearch.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py rename to stac_fastapi/elastic_search/tests/clients/test_elasticsearch.py diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elastic_search/tests/conftest.py similarity index 94% rename from stac_fastapi/elasticsearch/tests/conftest.py rename to stac_fastapi/elastic_search/tests/conftest.py index 2ec7df42..442894f4 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elastic_search/tests/conftest.py @@ -10,16 +10,16 @@ from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings -from stac_fastapi.elasticsearch.core import ( +from elastic_search.config import AsyncElasticsearchSettings +from common.core import ( BulkTransactionsClient, CoreClient, TransactionsClient, ) -from stac_fastapi.elasticsearch.database_elasticsearch.database_logic import ( +from elastic_search.database_logic import ( create_collection_index, ) -from stac_fastapi.elasticsearch.extensions import QueryExtension +from common.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, FieldsExtension, diff --git a/stac_fastapi/elasticsearch/tests/data/test_collection.json b/stac_fastapi/elastic_search/tests/data/test_collection.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/data/test_collection.json rename to stac_fastapi/elastic_search/tests/data/test_collection.json diff --git a/stac_fastapi/elasticsearch/tests/data/test_item.json b/stac_fastapi/elastic_search/tests/data/test_item.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/data/test_item.json rename to stac_fastapi/elastic_search/tests/data/test_item.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example01.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example01.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example01.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example01.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example04.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example04.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example04.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example04.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example05a.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example05a.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example05a.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example05a.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example06b.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example06b.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example06b.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example06b.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example08.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example08.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example08.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example08.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example09.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example09.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example09.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example09.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example1.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example1.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example1.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example1.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example10.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example10.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example10.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example10.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example14.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example14.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example14.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example14.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example15.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example15.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example15.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example15.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example17.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example17.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example17.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example17.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example18.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example18.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example18.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example18.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example19.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example19.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example20.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example20.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example21.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example21.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json b/stac_fastapi/elastic_search/tests/extensions/cql2/example22.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json rename to stac_fastapi/elastic_search/tests/extensions/cql2/example22.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elastic_search/tests/extensions/test_filter.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/test_filter.py rename to stac_fastapi/elastic_search/tests/extensions/test_filter.py diff --git a/stac_fastapi/elasticsearch/tests/resources/__init__.py b/stac_fastapi/elastic_search/tests/resources/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/__init__.py rename to stac_fastapi/elastic_search/tests/resources/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_collection.py b/stac_fastapi/elastic_search/tests/resources/test_collection.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/test_collection.py rename to stac_fastapi/elastic_search/tests/resources/test_collection.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_conformance.py b/stac_fastapi/elastic_search/tests/resources/test_conformance.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/test_conformance.py rename to stac_fastapi/elastic_search/tests/resources/test_conformance.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_item.py b/stac_fastapi/elastic_search/tests/resources/test_item.py similarity index 99% rename from stac_fastapi/elasticsearch/tests/resources/test_item.py rename to stac_fastapi/elastic_search/tests/resources/test_item.py index 5b382873..d13d139d 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_item.py +++ b/stac_fastapi/elastic_search/tests/resources/test_item.py @@ -12,8 +12,8 @@ from geojson_pydantic.geometries import Polygon from pystac.utils import datetime_to_str -from stac_fastapi.elasticsearch.core import CoreClient -from stac_fastapi.elasticsearch.datetime_utils import now_to_rfc3339_str +from stac_fastapi.common.core import CoreClient +from common.datetime_utils import now_to_rfc3339_str from stac_fastapi.types.core import LandingPageMixin from ..conftest import create_item, refresh_indices diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_fastapi/elastic_search/tests/resources/test_mgmt.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/test_mgmt.py rename to stac_fastapi/elastic_search/tests/resources/test_mgmt.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py b/stac_fastapi/elastic_search/utilities.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/utilities.py rename to stac_fastapi/elastic_search/utilities.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elastic_search/version.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py rename to stac_fastapi/elastic_search/version.py diff --git a/stac_fastapi/elasticsearch/setup.cfg b/stac_fastapi/elasticsearch/setup.cfg deleted file mode 100644 index 7a42432c..00000000 --- a/stac_fastapi/elasticsearch/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: stac_fastapi.elasticsearch.version.__version__ diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py deleted file mode 100644 index 3b709ff2..00000000 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_elasticsearch/database_logic.py +++ /dev/null @@ -1,838 +0,0 @@ -"""Database logic.""" -import asyncio -import logging -import os -from base64 import urlsafe_b64decode, urlsafe_b64encode -from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, Union - -import attr -from elasticsearch_dsl import Q, Search - -from elasticsearch import exceptions, helpers # type: ignore -from stac_fastapi.elasticsearch import serializers -from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings -from stac_fastapi.elasticsearch.config import ( - ElasticsearchSettings as SyncElasticsearchSettings, -) -from stac_fastapi.elasticsearch.extensions import filter -from stac_fastapi.elasticsearch.utilities import bbox2polygon -from stac_fastapi.types.errors import ConflictError, NotFoundError -from stac_fastapi.types.stac import Collection, Item - -logger = logging.getLogger(__name__) - -NumType = Union[float, int] - -COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections") -ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_") -ES_INDEX_NAME_UNSUPPORTED_CHARS = { - "\\", - "/", - "*", - "?", - '"', - "<", - ">", - "|", - " ", - ",", - "#", - ":", -} - -ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*" - -DEFAULT_SORT = { - "properties.datetime": {"order": "desc"}, - "id": {"order": "desc"}, - "collection": {"order": "desc"}, -} - -ES_ITEMS_SETTINGS = { - "index": { - "sort.field": list(DEFAULT_SORT.keys()), - "sort.order": [v["order"] for v in DEFAULT_SORT.values()], - } -} - -ES_MAPPINGS_DYNAMIC_TEMPLATES = [ - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - { - "descriptions": { - "match_mapping_type": "string", - "match": "description", - "mapping": {"type": "text"}, - } - }, - { - "titles": { - "match_mapping_type": "string", - "match": "title", - "mapping": {"type": "text"}, - } - }, - # Projection Extension https://github.com/stac-extensions/projection - {"proj_epsg": {"match": "proj:epsg", "mapping": {"type": "integer"}}}, - { - "proj_projjson": { - "match": "proj:projjson", - "mapping": {"type": "object", "enabled": False}, - } - }, - { - "proj_centroid": { - "match": "proj:centroid", - "mapping": {"type": "geo_point"}, - } - }, - { - "proj_geometry": { - "match": "proj:geometry", - "mapping": {"type": "object", "enabled": False}, - } - }, - { - "no_index_href": { - "match": "href", - "mapping": {"type": "text", "index": False}, - } - }, - # Default all other strings not otherwise specified to keyword - {"strings": {"match_mapping_type": "string", "mapping": {"type": "keyword"}}}, - {"numerics": {"match_mapping_type": "long", "mapping": {"type": "float"}}}, -] - -ES_ITEMS_MAPPINGS = { - "numeric_detection": False, - "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, - "properties": { - "id": {"type": "keyword"}, - "collection": {"type": "keyword"}, - "geometry": {"type": "geo_shape"}, - "assets": {"type": "object", "enabled": False}, - "links": {"type": "object", "enabled": False}, - "properties": { - "type": "object", - "properties": { - # Common https://github.com/radiantearth/stac-spec/blob/master/item-spec/common-metadata.md - "datetime": {"type": "date"}, - "start_datetime": {"type": "date"}, - "end_datetime": {"type": "date"}, - "created": {"type": "date"}, - "updated": {"type": "date"}, - # Satellite Extension https://github.com/stac-extensions/sat - "sat:absolute_orbit": {"type": "integer"}, - "sat:relative_orbit": {"type": "integer"}, - }, - }, - }, -} - -ES_COLLECTIONS_MAPPINGS = { - "numeric_detection": False, - "dynamic_templates": ES_MAPPINGS_DYNAMIC_TEMPLATES, - "properties": { - "extent.spatial.bbox": {"type": "long"}, - "extent.temporal.interval": {"type": "date"}, - "providers": {"type": "object", "enabled": False}, - "links": {"type": "object", "enabled": False}, - "item_assets": {"type": "object", "enabled": False}, - }, -} - - -def index_by_collection_id(collection_id: str) -> str: - """ - Translate a collection id into an Elasticsearch index name. - - Args: - collection_id (str): The collection id to translate into an index name. - - Returns: - str: The index name derived from the collection id. - """ - return f"{ITEMS_INDEX_PREFIX}{''.join(c for c in collection_id.lower() if c not in ES_INDEX_NAME_UNSUPPORTED_CHARS)}" - - -def indices(collection_ids: Optional[List[str]]) -> str: - """ - Get a comma-separated string of index names for a given list of collection ids. - - Args: - collection_ids: A list of collection ids. - - Returns: - A string of comma-separated index names. If `collection_ids` is None, returns the default indices. - """ - if collection_ids is None: - return ITEM_INDICES - else: - return ",".join([index_by_collection_id(c) for c in collection_ids]) - - -async def create_collection_index() -> None: - """ - Create the index for a Collection. - - Returns: - None - - """ - client = AsyncElasticsearchSettings().create_client - - await client.options(ignore_status=400).indices.create( - index=f"{COLLECTIONS_INDEX}-000001", - aliases={COLLECTIONS_INDEX: {}}, - mappings=ES_COLLECTIONS_MAPPINGS, - ) - await client.close() - - -async def create_item_index(collection_id: str): - """ - Create the index for Items. - - Args: - collection_id (str): Collection identifier. - - Returns: - None - - """ - client = AsyncElasticsearchSettings().create_client - index_name = index_by_collection_id(collection_id) - - await client.options(ignore_status=400).indices.create( - index=f"{index_by_collection_id(collection_id)}-000001", - aliases={index_name: {}}, - mappings=ES_ITEMS_MAPPINGS, - settings=ES_ITEMS_SETTINGS, - ) - await client.close() - - -async def delete_item_index(collection_id: str): - """Delete the index for items in a collection. - - Args: - collection_id (str): The ID of the collection whose items index will be deleted. - """ - client = AsyncElasticsearchSettings().create_client - - name = index_by_collection_id(collection_id) - resolved = await client.indices.resolve_index(name=name) - if "aliases" in resolved and resolved["aliases"]: - [alias] = resolved["aliases"] - await client.indices.delete_alias(index=alias["indices"], name=alias["name"]) - await client.indices.delete(index=alias["indices"]) - else: - await client.indices.delete(index=name) - await client.close() - - -def mk_item_id(item_id: str, collection_id: str): - """Create the document id for an Item in Elasticsearch. - - Args: - item_id (str): The id of the Item. - collection_id (str): The id of the Collection that the Item belongs to. - - Returns: - str: The document id for the Item, combining the Item id and the Collection id, separated by a `|` character. - """ - return f"{item_id}|{collection_id}" - - -def mk_actions(collection_id: str, processed_items: List[Item]): - """Create Elasticsearch bulk actions for a list of processed items. - - Args: - collection_id (str): The identifier for the collection the items belong to. - processed_items (List[Item]): The list of processed items to be bulk indexed. - - Returns: - List[Dict[str, Union[str, Dict]]]: The list of bulk actions to be executed, - each action being a dictionary with the following keys: - - `_index`: the index to store the document in. - - `_id`: the document's identifier. - - `_source`: the source of the document. - """ - return [ - { - "_index": index_by_collection_id(collection_id), - "_id": mk_item_id(item["id"], item["collection"]), - "_source": item, - } - for item in processed_items - ] - - -# stac_pydantic classes extend _GeometryBase, which doesn't have a type field, -# So create our own Protocol for typing -# Union[ Point, MultiPoint, LineString, MultiLineString, Polygon, MultiPolygon, GeometryCollection] -class Geometry(Protocol): # noqa - type: str - coordinates: Any - - -@attr.s -class DatabaseLogic: - """Database logic.""" - - client = AsyncElasticsearchSettings().create_client - sync_client = SyncElasticsearchSettings().create_client - - item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer - ) - collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer - ) - - """CORE LOGIC""" - - async def get_all_collections( - self, token: Optional[str], limit: int - ) -> Iterable[Dict[str, Any]]: - """Retrieve a list of all collections from the database. - - Args: - token (Optional[str]): The token used to return the next set of results. - limit (int): Number of results to return - - Returns: - collections (Iterable[Dict[str, Any]]): A list of dictionaries containing the source data for each collection. - - Notes: - The collections are retrieved from the Elasticsearch database using the `client.search` method, - with the `COLLECTIONS_INDEX` as the target index and `size=limit` to retrieve records. - The result is a generator of dictionaries containing the source data for each collection. - """ - search_after = None - if token: - search_after = urlsafe_b64decode(token.encode()).decode().split(",") - collections = await self.client.search( - index=COLLECTIONS_INDEX, - search_after=search_after, - size=limit, - sort={"id": {"order": "asc"}}, - ) - hits = collections["hits"]["hits"] - return hits - - async def get_one_item(self, collection_id: str, item_id: str) -> Dict: - """Retrieve a single item from the database. - - Args: - collection_id (str): The id of the Collection that the Item belongs to. - item_id (str): The id of the Item. - - Returns: - item (Dict): A dictionary containing the source data for the Item. - - Raises: - NotFoundError: If the specified Item does not exist in the Collection. - - Notes: - The Item is retrieved from the Elasticsearch database using the `client.get` method, - with the index for the Collection as the target index and the combined `mk_item_id` as the document id. - """ - try: - item = await self.client.get( - index=index_by_collection_id(collection_id), - id=mk_item_id(item_id, collection_id), - ) - except exceptions.NotFoundError: - raise NotFoundError( - f"Item {item_id} does not exist in Collection {collection_id}" - ) - return item["_source"] - - @staticmethod - def make_search(): - """Database logic to create a Search instance.""" - return Search().sort(*DEFAULT_SORT) - - @staticmethod - def apply_ids_filter(search: Search, item_ids: List[str]): - """Database logic to search a list of STAC item ids.""" - return search.filter("terms", id=item_ids) - - @staticmethod - def apply_collections_filter(search: Search, collection_ids: List[str]): - """Database logic to search a list of STAC collection ids.""" - return search.filter("terms", collection=collection_ids) - - @staticmethod - def apply_datetime_filter(search: Search, datetime_search): - """Apply a filter to search based on datetime field. - - Args: - search (Search): The search object to filter. - datetime_search (dict): The datetime filter criteria. - - Returns: - Search: The filtered search object. - """ - if "eq" in datetime_search: - search = search.filter( - "term", **{"properties__datetime": datetime_search["eq"]} - ) - else: - search = search.filter( - "range", properties__datetime={"lte": datetime_search["lte"]} - ) - search = search.filter( - "range", properties__datetime={"gte": datetime_search["gte"]} - ) - return search - - @staticmethod - def apply_bbox_filter(search: Search, bbox: List): - """Filter search results based on bounding box. - - Args: - search (Search): The search object to apply the filter to. - bbox (List): The bounding box coordinates, represented as a list of four values [minx, miny, maxx, maxy]. - - Returns: - search (Search): The search object with the bounding box filter applied. - - Notes: - The bounding box is transformed into a polygon using the `bbox2polygon` function and - a geo_shape filter is added to the search object, set to intersect with the specified polygon. - """ - return search.filter( - Q( - { - "geo_shape": { - "geometry": { - "shape": { - "type": "polygon", - "coordinates": bbox2polygon(*bbox), - }, - "relation": "intersects", - } - } - } - ) - ) - - @staticmethod - def apply_intersects_filter( - search: Search, - intersects: Geometry, - ): - """Filter search results based on intersecting geometry. - - Args: - search (Search): The search object to apply the filter to. - intersects (Geometry): The intersecting geometry, represented as a GeoJSON-like object. - - Returns: - search (Search): The search object with the intersecting geometry filter applied. - - Notes: - A geo_shape filter is added to the search object, set to intersect with the specified geometry. - """ - return search.filter( - Q( - { - "geo_shape": { - "geometry": { - "shape": { - "type": intersects.type.lower(), - "coordinates": intersects.coordinates, - }, - "relation": "intersects", - } - } - } - ) - ) - - @staticmethod - def apply_stacql_filter(search: Search, op: str, field: str, value: float): - """Filter search results based on a comparison between a field and a value. - - Args: - search (Search): The search object to apply the filter to. - op (str): The comparison operator to use. Can be 'eq' (equal), 'gt' (greater than), 'gte' (greater than or equal), - 'lt' (less than), or 'lte' (less than or equal). - field (str): The field to perform the comparison on. - value (float): The value to compare the field against. - - Returns: - search (Search): The search object with the specified filter applied. - """ - if op != "eq": - key_filter = {field: {f"{op}": value}} - search = search.filter(Q("range", **key_filter)) - else: - search = search.filter("term", **{field: value}) - - return search - - @staticmethod - def apply_cql2_filter(search: Search, _filter: Optional[Dict[str, Any]]): - """Database logic to perform query for search endpoint.""" - if _filter is not None: - search = search.filter(filter.Clause.parse_obj(_filter).to_es()) - return search - - @staticmethod - def populate_sort(sortby: List) -> Optional[Dict[str, Dict[str, str]]]: - """Database logic to sort search instance.""" - if sortby: - return {s.field: {"order": s.direction} for s in sortby} - else: - return None - - async def execute_search( - self, - search: Search, - limit: int, - token: Optional[str], - sort: Optional[Dict[str, Dict[str, str]]], - collection_ids: Optional[List[str]], - ignore_unavailable: bool = True, - ) -> Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: - """Execute a search query with limit and other optional parameters. - - Args: - search (Search): The search query to be executed. - limit (int): The maximum number of results to be returned. - token (Optional[str]): The token used to return the next set of results. - sort (Optional[Dict[str, Dict[str, str]]]): Specifies how the results should be sorted. - collection_ids (Optional[List[str]]): The collection ids to search. - ignore_unavailable (bool, optional): Whether to ignore unavailable collections. Defaults to True. - - Returns: - Tuple[Iterable[Dict[str, Any]], Optional[int], Optional[str]]: A tuple containing: - - An iterable of search results, where each result is a dictionary with keys and values representing the - fields and values of each document. - - The total number of results (if the count could be computed), or None if the count could not be - computed. - - The token to be used to retrieve the next set of results, or None if there are no more results. - - Raises: - NotFoundError: If the collections specified in `collection_ids` do not exist. - """ - search_after = None - if token: - search_after = urlsafe_b64decode(token.encode()).decode().split(",") - - query = search.query.to_dict() if search.query else None - - index_param = indices(collection_ids) - - search_task = asyncio.create_task( - self.client.search( - index=index_param, - ignore_unavailable=ignore_unavailable, - query=query, - sort=sort or DEFAULT_SORT, - search_after=search_after, - size=limit, - ) - ) - - count_task = asyncio.create_task( - self.client.count( - index=index_param, - ignore_unavailable=ignore_unavailable, - body=search.to_dict(count=True), - ) - ) - - try: - es_response = await search_task - except exceptions.NotFoundError: - raise NotFoundError(f"Collections '{collection_ids}' do not exist") - - hits = es_response["hits"]["hits"] - items = (hit["_source"] for hit in hits) - - next_token = None - if hits and (sort_array := hits[-1].get("sort")): - next_token = urlsafe_b64encode( - ",".join([str(x) for x in sort_array]).encode() - ).decode() - - # (1) count should not block returning results, so don't wait for it to be done - # (2) don't cancel the task so that it will populate the ES cache for subsequent counts - maybe_count = None - if count_task.done(): - try: - maybe_count = count_task.result().get("count") - except Exception as e: - logger.error(f"Count task failed: {e}") - - return items, maybe_count, next_token - - """ TRANSACTION LOGIC """ - - async def check_collection_exists(self, collection_id: str): - """Database logic to check if a collection exists.""" - if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): - raise NotFoundError(f"Collection {collection_id} does not exist") - - async def prep_create_item( - self, item: Item, base_url: str, exist_ok: bool = False - ) -> Item: - """ - Preps an item for insertion into the database. - - Args: - item (Item): The item to be prepped for insertion. - base_url (str): The base URL used to create the item's self URL. - exist_ok (bool): Indicates whether the item can exist already. - - Returns: - Item: The prepped item. - - Raises: - ConflictError: If the item already exists in the database. - - """ - await self.check_collection_exists(collection_id=item["collection"]) - - if not exist_ok and await self.client.exists( - index=index_by_collection_id(item["collection"]), - id=mk_item_id(item["id"], item["collection"]), - ): - raise ConflictError( - f"Item {item['id']} in collection {item['collection']} already exists" - ) - - return self.item_serializer.stac_to_db(item, base_url) - - def sync_prep_create_item( - self, item: Item, base_url: str, exist_ok: bool = False - ) -> Item: - """ - Prepare an item for insertion into the database. - - This method performs pre-insertion preparation on the given `item`, - such as checking if the collection the item belongs to exists, - and optionally verifying that an item with the same ID does not already exist in the database. - - Args: - item (Item): The item to be inserted into the database. - base_url (str): The base URL used for constructing URLs for the item. - exist_ok (bool): Indicates whether the item can exist already. - - Returns: - Item: The item after preparation is done. - - Raises: - NotFoundError: If the collection that the item belongs to does not exist in the database. - ConflictError: If an item with the same ID already exists in the collection. - """ - item_id = item["id"] - collection_id = item["collection"] - if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=collection_id): - raise NotFoundError(f"Collection {collection_id} does not exist") - - if not exist_ok and self.sync_client.exists( - index=index_by_collection_id(collection_id), - id=mk_item_id(item_id, collection_id), - ): - raise ConflictError( - f"Item {item_id} in collection {collection_id} already exists" - ) - - return self.item_serializer.stac_to_db(item, base_url) - - async def create_item(self, item: Item, refresh: bool = False): - """Database logic for creating one item. - - Args: - item (Item): The item to be created. - refresh (bool, optional): Refresh the index after performing the operation. Defaults to False. - - Raises: - ConflictError: If the item already exists in the database. - - Returns: - None - """ - # todo: check if collection exists, but cache - item_id = item["id"] - collection_id = item["collection"] - es_resp = await self.client.index( - index=index_by_collection_id(collection_id), - id=mk_item_id(item_id, collection_id), - document=item, - refresh=refresh, - ) - - if (meta := es_resp.get("meta")) and meta.get("status") == 409: - raise ConflictError( - f"Item {item_id} in collection {collection_id} already exists" - ) - - async def delete_item( - self, item_id: str, collection_id: str, refresh: bool = False - ): - """Delete a single item from the database. - - Args: - item_id (str): The id of the Item to be deleted. - collection_id (str): The id of the Collection that the Item belongs to. - refresh (bool, optional): Whether to refresh the index after the deletion. Default is False. - - Raises: - NotFoundError: If the Item does not exist in the database. - """ - try: - await self.client.delete( - index=index_by_collection_id(collection_id), - id=mk_item_id(item_id, collection_id), - refresh=refresh, - ) - except exceptions.NotFoundError: - raise NotFoundError( - f"Item {item_id} in collection {collection_id} not found" - ) - - async def create_collection(self, collection: Collection, refresh: bool = False): - """Create a single collection in the database. - - Args: - collection (Collection): The Collection object to be created. - refresh (bool, optional): Whether to refresh the index after the creation. Default is False. - - Raises: - ConflictError: If a Collection with the same id already exists in the database. - - Notes: - A new index is created for the items in the Collection using the `create_item_index` function. - """ - collection_id = collection["id"] - - if await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): - raise ConflictError(f"Collection {collection_id} already exists") - - await self.client.index( - index=COLLECTIONS_INDEX, - id=collection_id, - document=collection, - refresh=refresh, - ) - - await create_item_index(collection_id) - - async def find_collection(self, collection_id: str) -> Collection: - """Find and return a collection from the database. - - Args: - self: The instance of the object calling this function. - collection_id (str): The ID of the collection to be found. - - Returns: - Collection: The found collection, represented as a `Collection` object. - - Raises: - NotFoundError: If the collection with the given `collection_id` is not found in the database. - - Notes: - This function searches for a collection in the database using the specified `collection_id` and returns the found - collection as a `Collection` object. If the collection is not found, a `NotFoundError` is raised. - """ - try: - collection = await self.client.get( - index=COLLECTIONS_INDEX, id=collection_id - ) - except exceptions.NotFoundError: - raise NotFoundError(f"Collection {collection_id} not found") - - return collection["_source"] - - async def delete_collection(self, collection_id: str, refresh: bool = False): - """Delete a collection from the database. - - Parameters: - self: The instance of the object calling this function. - collection_id (str): The ID of the collection to be deleted. - refresh (bool): Whether to refresh the index after the deletion (default: False). - - Raises: - NotFoundError: If the collection with the given `collection_id` is not found in the database. - - Notes: - This function first verifies that the collection with the specified `collection_id` exists in the database, and then - deletes the collection. If `refresh` is set to True, the index is refreshed after the deletion. Additionally, this - function also calls `delete_item_index` to delete the index for the items in the collection. - """ - await self.find_collection(collection_id=collection_id) - await self.client.delete( - index=COLLECTIONS_INDEX, id=collection_id, refresh=refresh - ) - await delete_item_index(collection_id) - - async def bulk_async( - self, collection_id: str, processed_items: List[Item], refresh: bool = False - ) -> None: - """Perform a bulk insert of items into the database asynchronously. - - Args: - self: The instance of the object calling this function. - collection_id (str): The ID of the collection to which the items belong. - processed_items (List[Item]): A list of `Item` objects to be inserted into the database. - refresh (bool): Whether to refresh the index after the bulk insert (default: False). - - Notes: - This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`. The - insert is performed asynchronously, and the event loop is used to run the operation in a separate executor. The - `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True, the - index is refreshed after the bulk insert. The function does not return any value. - """ - await helpers.async_bulk( - self.client, - mk_actions(collection_id, processed_items), - refresh=refresh, - raise_on_error=False, - ) - - def bulk_sync( - self, collection_id: str, processed_items: List[Item], refresh: bool = False - ) -> None: - """Perform a bulk insert of items into the database synchronously. - - Args: - self: The instance of the object calling this function. - collection_id (str): The ID of the collection to which the items belong. - processed_items (List[Item]): A list of `Item` objects to be inserted into the database. - refresh (bool): Whether to refresh the index after the bulk insert (default: False). - - Notes: - This function performs a bulk insert of `processed_items` into the database using the specified `collection_id`. The - insert is performed synchronously and blocking, meaning that the function does not return until the insert has - completed. The `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to - True, the index is refreshed after the bulk insert. The function does not return any value. - """ - helpers.bulk( - self.sync_client, - mk_actions(collection_id, processed_items), - refresh=refresh, - raise_on_error=False, - ) - - # DANGER - async def delete_items(self) -> None: - """Danger. this is only for tests.""" - await self.client.delete_by_query( - index=ITEM_INDICES, - body={"query": {"match_all": {}}}, - wait_for_completion=True, - ) - - # DANGER - async def delete_collections(self) -> None: - """Danger. this is only for tests.""" - await self.client.delete_by_query( - index=COLLECTIONS_INDEX, - body={"query": {"match_all": {}}}, - wait_for_completion=True, - ) diff --git a/stac_fastapi/open_search/app.py b/stac_fastapi/open_search/app.py new file mode 100644 index 00000000..9230d2b1 --- /dev/null +++ b/stac_fastapi/open_search/app.py @@ -0,0 +1,93 @@ +"""FastAPI application.""" +from stac_fastapi.api.app import StacApi +from stac_fastapi.api.models import create_get_request_model, create_post_request_model +from elastic_search.config import ElasticsearchSettings +from stac_fastapi.common.core import ( + BulkTransactionsClient, + CoreClient, + EsAsyncBaseFiltersClient, + TransactionsClient, +) +from elastic_search.database_logic import DatabaseLogic +from stac_fastapi.common.extensions import QueryExtension +from elastic_search.session import Session +from elastic_search.database_logic import create_collection_index +from stac_fastapi.extensions.core import ( + ContextExtension, + FieldsExtension, + FilterExtension, + SortExtension, + TokenPaginationExtension, + TransactionExtension, +) +from stac_fastapi.extensions.third_party import BulkTransactionExtension + +settings = ElasticsearchSettings() +session = Session.create_from_settings(settings) + +filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient()) +filter_extension.conformance_classes.append( + "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" +) + +database_logic = DatabaseLogic() + +extensions = [ + TransactionExtension(client=TransactionsClient(session=session, database=database_logic), settings=settings), + BulkTransactionExtension(client=BulkTransactionsClient(session=session, database=database_logic)), + FieldsExtension(), + QueryExtension(), + SortExtension(), + TokenPaginationExtension(), + ContextExtension(), + filter_extension, +] + +post_request_model = create_post_request_model(extensions) + +api = StacApi( + settings=settings, + extensions=extensions, + client=CoreClient(session=session, post_request_model=post_request_model, database=database_logic), + search_get_request_model=create_get_request_model(extensions), + search_post_request_model=post_request_model, +) +app = api.app + + +@app.on_event("startup") +async def _startup_event() -> None: + await create_collection_index() + + +def run() -> None: + """Run app from command line using uvicorn if available.""" + try: + import uvicorn + + uvicorn.run( + "stac_fastapi.elasticsearch.app:app", + host=settings.app_host, + port=settings.app_port, + log_level="info", + reload=settings.reload, + ) + except ImportError: + raise RuntimeError("Uvicorn must be installed in order to use command") + + +if __name__ == "__main__": + run() + + +def create_handler(app): + """Create a handler to use with AWS Lambda if mangum available.""" + try: + from mangum import Mangum + + return Mangum(app) + except ImportError: + return None + + +handler = create_handler(app) diff --git a/stac_fastapi/open_search/setup.cfg b/stac_fastapi/open_search/setup.cfg new file mode 100644 index 00000000..f1023827 --- /dev/null +++ b/stac_fastapi/open_search/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +version = attr: version.__version__ diff --git a/stac_fastapi/open_search/setup.py b/stac_fastapi/open_search/setup.py new file mode 100644 index 00000000..033d10d4 --- /dev/null +++ b/stac_fastapi/open_search/setup.py @@ -0,0 +1,72 @@ +"""stac_fastapi: elasticsearch module.""" + +from setuptools import find_namespace_packages, setup + +with open("README.md") as f: + desc = f.read() + +install_requires = [ + "fastapi", + "attrs", + "elasticsearch[async]==8.11.0", + "elasticsearch-dsl==8.11.0", + "pydantic[dotenv]<2", + "stac_pydantic==2.0.*", + "stac-fastapi.types==2.4.9", + "stac-fastapi.api==2.4.9", + "stac-fastapi.extensions==2.4.9", + "pystac[validation]", + "uvicorn", + "orjson", + "overrides", + "starlette", + "geojson-pydantic", + "pygeofilter==0.2.1", +] + +extra_reqs = { + "dev": [ + "pytest", + "pytest-cov", + "pytest-asyncio", + "pre-commit", + "requests", + "ciso8601", + "httpx", + ], + "docs": ["mkdocs", "mkdocs-material", "pdocs"], + "server": ["uvicorn[standard]==0.19.0"], +} + +setup( + name="stac-fastapi.elasticsearch", + description="An implementation of STAC API based on the FastAPI framework with Elasticsearch.", + long_description=desc, + long_description_content_type="text/markdown", + python_requires=">=3.8", + classifiers=[ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: MIT License", + ], + url="https://github.com/stac-utils/stac-fastapi-elasticsearch", + license="MIT", + packages=find_namespace_packages( + include=["elasticsearch", "elasticsearch.*", "common", "common.*"], + exclude=["tests", "scripts"], + ), + zip_safe=False, + install_requires=install_requires, + tests_require=extra_reqs["dev"], + extras_require=extra_reqs, + entry_points={ + "console_scripts": [ + "stac-fastapi-elasticsearch=app:run" + ] + }, +) diff --git a/stac_fastapi/open_search/version.py b/stac_fastapi/open_search/version.py new file mode 100644 index 00000000..1eeef171 --- /dev/null +++ b/stac_fastapi/open_search/version.py @@ -0,0 +1,2 @@ +"""library version.""" +__version__ = "1.0.0" From 2d3e5019139688258b1d2135d13b79ddb7aca2c0 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 30 Jan 2024 12:33:48 +0800 Subject: [PATCH 13/23] update docker files --- Dockerfile.dev => Dockerfile.dev.es | 3 ++- docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) rename Dockerfile.dev => Dockerfile.dev.es (81%) diff --git a/Dockerfile.dev b/Dockerfile.dev.es similarity index 81% rename from Dockerfile.dev rename to Dockerfile.dev.es index e283e4c1..132eaea8 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev.es @@ -13,6 +13,7 @@ ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt WORKDIR /app -COPY /stac_fastapi /app +COPY ./stac_fastapi/common /app/common +COPY ./stac_fastapi/elastic_search /app/elastic_search RUN pip install --no-cache-dir -e ./elastic_search[dev,server] diff --git a/docker-compose.yml b/docker-compose.yml index ddc3c6f6..853a85cf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,7 @@ services: restart: always build: context: . - dockerfile: Dockerfile.dev + dockerfile: Dockerfile.dev.es environment: - APP_HOST=0.0.0.0 - APP_PORT=8080 From a2b03d03abb6dee4304db8e43788fb40f44f7e26 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 30 Jan 2024 12:58:30 +0800 Subject: [PATCH 14/23] dynamically import --- stac_fastapi/common/base_database_logic.py | 84 +++++++++-------- stac_fastapi/common/core.py | 78 ++++++++++++++-- stac_fastapi/elastic_search/app.py | 12 +-- stac_fastapi/elastic_search/database_logic.py | 12 +-- stac_fastapi/elastic_search/serializers.py | 2 +- stac_fastapi/elastic_search/setup.py | 6 +- stac_fastapi/elastic_search/tests/conftest.py | 14 +-- .../tests/resources/test_item.py | 2 +- stac_fastapi/open_search/app.py | 93 ------------------- stac_fastapi/open_search/setup.py | 6 +- 10 files changed, 134 insertions(+), 175 deletions(-) delete mode 100644 stac_fastapi/open_search/app.py diff --git a/stac_fastapi/common/base_database_logic.py b/stac_fastapi/common/base_database_logic.py index 2478722b..09bb01a6 100644 --- a/stac_fastapi/common/base_database_logic.py +++ b/stac_fastapi/common/base_database_logic.py @@ -1,37 +1,47 @@ -from abc import ABC, abstractmethod -from typing import Any, Dict, List, Optional, Tuple, Union -from stac_fastapi.types.stac import Collection, Item - -class BaseDatabaseLogic(ABC): - """ - Abstract base class for database logic. - This class defines the interface for database operations. - """ - - @abstractmethod - async def get_all_collections(self, token: Optional[str], limit: int) -> Any: - pass - - @abstractmethod - async def get_one_item(self, collection_id: str, item_id: str) -> Dict: - pass - - @abstractmethod - async def create_item(self, item: Item, refresh: bool = False) -> None: - pass - - @abstractmethod - async def delete_item(self, item_id: str, collection_id: str, refresh: bool = False) -> None: - pass - - @abstractmethod - async def create_collection(self, collection: Collection, refresh: bool = False) -> None: - pass - - @abstractmethod - async def find_collection(self, collection_id: str) -> Collection: - pass - - @abstractmethod - async def delete_collection(self, collection_id: str, refresh: bool = False) -> None: - pass \ No newline at end of file +"""base class for database logic.""" + +# from abc import ABC, abstractmethod +# from typing import Any, Dict, List, Optional, Tuple, Union + +# from stac_fastapi.types.stac import Collection, Item + + +# class BaseDatabaseLogic(ABC): +# """ +# Abstract base class for database logic. +# This class defines the interface for database operations. +# """ + +# @abstractmethod +# async def get_all_collections(self, token: Optional[str], limit: int) -> Any: +# pass + +# @abstractmethod +# async def get_one_item(self, collection_id: str, item_id: str) -> Dict: +# pass + +# @abstractmethod +# async def create_item(self, item: Item, refresh: bool = False) -> None: +# pass + +# @abstractmethod +# async def delete_item( +# self, item_id: str, collection_id: str, refresh: bool = False +# ) -> None: +# pass + +# @abstractmethod +# async def create_collection( +# self, collection: Collection, refresh: bool = False +# ) -> None: +# pass + +# @abstractmethod +# async def find_collection(self, collection_id: str) -> Collection: +# pass + +# @abstractmethod +# async def delete_collection( +# self, collection_id: str, refresh: bool = False +# ) -> None: +# pass diff --git a/stac_fastapi/common/core.py b/stac_fastapi/common/core.py index 927273f3..71e6fd3f 100644 --- a/stac_fastapi/common/core.py +++ b/stac_fastapi/common/core.py @@ -1,4 +1,5 @@ """Item crud client.""" +import importlib import logging import re from base64 import urlsafe_b64encode @@ -10,7 +11,11 @@ import attr import orjson import stac_pydantic -from elastic_search.database_logic import DatabaseLogic as BaseDatabaseLogic +from common.base_database_logic import BaseDatabaseLogic +from common.models.links import PagingLinks +from elastic_search import serializers +from elastic_search.serializers import CollectionSerializer, ItemSerializer +from elastic_search.session import Session from fastapi import HTTPException, Request from overrides import overrides from pydantic import ValidationError @@ -19,11 +24,6 @@ from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes -from elastic_search import serializers -from elastic_search.config import ElasticsearchSettings -from common.models.links import PagingLinks -from elastic_search.serializers import CollectionSerializer, ItemSerializer -from elastic_search.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, BulkTransactionMethod, @@ -71,7 +71,27 @@ class CoreClient(AsyncBaseCoreClient): collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( default=serializers.CollectionSerializer ) - database = BaseDatabaseLogic() + database: BaseDatabaseLogic = attr.ib(init=False) + + def __attrs_post_init__(self): + """ + Post-initialization method for CoreClient. + + This method is automatically called after CoreClient's instance is initialized. + It's responsible for setting up the database logic dynamically based on the + environment or configuration, ensuring that the appropriate database logic + (Elasticsearch or OpenSearch) is used. + """ + try: + # Dynamically import the database logic based on installed package + database_module = importlib.import_module("elastic_search.database_logic") + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + except ImportError: + # Fall back to OpenSearch if Elasticsearch is not available + database_module = importlib.import_module("open_search.database_logic") + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + + self.database = DatabaseLogicClass() @overrides async def all_collections(self, **kwargs) -> Collections: @@ -542,7 +562,27 @@ class TransactionsClient(AsyncBaseTransactionsClient): """Transactions extension specific CRUD operations.""" session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database = BaseDatabaseLogic() + database: BaseDatabaseLogic = attr.ib(init=False) + + def __attrs_post_init__(self): + """ + Post-initialization method for TransactionsClient. + + This method is automatically called after the instance is initialized. + It's responsible for setting up the database logic dynamically based on the + environment or configuration, ensuring that the appropriate database logic + (Elasticsearch or OpenSearch) is used. + """ + try: + # Dynamically import the database logic based on installed package + database_module = importlib.import_module("elastic_search.database_logic") + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + except ImportError: + # Fall back to OpenSearch if Elasticsearch is not available + database_module = importlib.import_module("opensearch.database_logic") + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + + self.database = DatabaseLogicClass() @overrides async def create_item( @@ -712,7 +752,27 @@ class BulkTransactionsClient(BaseBulkTransactionsClient): """ session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database = BaseDatabaseLogic() + database: BaseDatabaseLogic = attr.ib(init=False) + + def __attrs_post_init__(self): + """ + Post-initialization method for BulkTransactionsClient. + + This method is automatically called after the instance is initialized. + It's responsible for setting up the database logic dynamically based on the + environment or configuration, ensuring that the appropriate database logic + (Elasticsearch or OpenSearch) is used. + """ + try: + # Dynamically import the database logic based on installed package + database_module = importlib.import_module("elastic_search.database_logic") + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + except ImportError: + # Fall back to OpenSearch if Elasticsearch is not available + database_module = importlib.import_module("opensearch.database_logic") + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + + self.database = DatabaseLogicClass() def preprocess_item( self, item: stac_types.Item, base_url, method: BulkTransactionMethod diff --git a/stac_fastapi/elastic_search/app.py b/stac_fastapi/elastic_search/app.py index 294c8312..61e1015b 100644 --- a/stac_fastapi/elastic_search/app.py +++ b/stac_fastapi/elastic_search/app.py @@ -1,17 +1,17 @@ """FastAPI application.""" +from elastic_search.config import ElasticsearchSettings +from elastic_search.database_logic import create_collection_index +from elastic_search.session import Session + from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from elastic_search.config import ElasticsearchSettings from stac_fastapi.common.core import ( BulkTransactionsClient, CoreClient, EsAsyncBaseFiltersClient, TransactionsClient, ) -from elastic_search.database_logic import DatabaseLogic from stac_fastapi.common.extensions import QueryExtension -from elastic_search.session import Session -from elastic_search.database_logic import create_collection_index from stac_fastapi.extensions.core import ( ContextExtension, FieldsExtension, @@ -30,8 +30,6 @@ "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" ) -database_logic = DatabaseLogic() - extensions = [ TransactionExtension(client=TransactionsClient(session=session), settings=settings), BulkTransactionExtension(client=BulkTransactionsClient(session=session)), @@ -48,7 +46,7 @@ api = StacApi( settings=settings, extensions=extensions, - client=CoreClient(session=session, post_request_model=post_request_model), + client=CoreClient(session=session), search_get_request_model=create_get_request_model(extensions), search_post_request_model=post_request_model, ) diff --git a/stac_fastapi/elastic_search/database_logic.py b/stac_fastapi/elastic_search/database_logic.py index 88d46328..305ab72c 100644 --- a/stac_fastapi/elastic_search/database_logic.py +++ b/stac_fastapi/elastic_search/database_logic.py @@ -6,16 +6,14 @@ from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, Union import attr -from elasticsearch_dsl import Q, Search - -from elasticsearch import exceptions, helpers # type: ignore +from common.extensions import filter from elastic_search import serializers from elastic_search.config import AsyncElasticsearchSettings -from elastic_search.config import ( - ElasticsearchSettings as SyncElasticsearchSettings, -) -from common.extensions import filter +from elastic_search.config import ElasticsearchSettings as SyncElasticsearchSettings from elastic_search.utilities import bbox2polygon +from elasticsearch_dsl import Q, Search + +from elasticsearch import exceptions, helpers # type: ignore from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item diff --git a/stac_fastapi/elastic_search/serializers.py b/stac_fastapi/elastic_search/serializers.py index fe08c9b3..786e2c98 100644 --- a/stac_fastapi/elastic_search/serializers.py +++ b/stac_fastapi/elastic_search/serializers.py @@ -3,8 +3,8 @@ from typing import Any import attr - from common.datetime_utils import now_to_rfc3339_str + from stac_fastapi.types import stac as stac_types from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links diff --git a/stac_fastapi/elastic_search/setup.py b/stac_fastapi/elastic_search/setup.py index 033d10d4..2c0607fc 100644 --- a/stac_fastapi/elastic_search/setup.py +++ b/stac_fastapi/elastic_search/setup.py @@ -64,9 +64,5 @@ install_requires=install_requires, tests_require=extra_reqs["dev"], extras_require=extra_reqs, - entry_points={ - "console_scripts": [ - "stac-fastapi-elasticsearch=app:run" - ] - }, + entry_points={"console_scripts": ["stac-fastapi-elasticsearch=app:run"]}, ) diff --git a/stac_fastapi/elastic_search/tests/conftest.py b/stac_fastapi/elastic_search/tests/conftest.py index 442894f4..2817fda6 100644 --- a/stac_fastapi/elastic_search/tests/conftest.py +++ b/stac_fastapi/elastic_search/tests/conftest.py @@ -6,20 +6,14 @@ import pytest import pytest_asyncio +from common.core import BulkTransactionsClient, CoreClient, TransactionsClient +from common.extensions import QueryExtension +from elastic_search.config import AsyncElasticsearchSettings +from elastic_search.database_logic import create_collection_index from httpx import AsyncClient from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from elastic_search.config import AsyncElasticsearchSettings -from common.core import ( - BulkTransactionsClient, - CoreClient, - TransactionsClient, -) -from elastic_search.database_logic import ( - create_collection_index, -) -from common.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, FieldsExtension, diff --git a/stac_fastapi/elastic_search/tests/resources/test_item.py b/stac_fastapi/elastic_search/tests/resources/test_item.py index d13d139d..b66b3572 100644 --- a/stac_fastapi/elastic_search/tests/resources/test_item.py +++ b/stac_fastapi/elastic_search/tests/resources/test_item.py @@ -9,11 +9,11 @@ import ciso8601 import pystac import pytest +from common.datetime_utils import now_to_rfc3339_str from geojson_pydantic.geometries import Polygon from pystac.utils import datetime_to_str from stac_fastapi.common.core import CoreClient -from common.datetime_utils import now_to_rfc3339_str from stac_fastapi.types.core import LandingPageMixin from ..conftest import create_item, refresh_indices diff --git a/stac_fastapi/open_search/app.py b/stac_fastapi/open_search/app.py deleted file mode 100644 index 9230d2b1..00000000 --- a/stac_fastapi/open_search/app.py +++ /dev/null @@ -1,93 +0,0 @@ -"""FastAPI application.""" -from stac_fastapi.api.app import StacApi -from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from elastic_search.config import ElasticsearchSettings -from stac_fastapi.common.core import ( - BulkTransactionsClient, - CoreClient, - EsAsyncBaseFiltersClient, - TransactionsClient, -) -from elastic_search.database_logic import DatabaseLogic -from stac_fastapi.common.extensions import QueryExtension -from elastic_search.session import Session -from elastic_search.database_logic import create_collection_index -from stac_fastapi.extensions.core import ( - ContextExtension, - FieldsExtension, - FilterExtension, - SortExtension, - TokenPaginationExtension, - TransactionExtension, -) -from stac_fastapi.extensions.third_party import BulkTransactionExtension - -settings = ElasticsearchSettings() -session = Session.create_from_settings(settings) - -filter_extension = FilterExtension(client=EsAsyncBaseFiltersClient()) -filter_extension.conformance_classes.append( - "http://www.opengis.net/spec/cql2/1.0/conf/advanced-comparison-operators" -) - -database_logic = DatabaseLogic() - -extensions = [ - TransactionExtension(client=TransactionsClient(session=session, database=database_logic), settings=settings), - BulkTransactionExtension(client=BulkTransactionsClient(session=session, database=database_logic)), - FieldsExtension(), - QueryExtension(), - SortExtension(), - TokenPaginationExtension(), - ContextExtension(), - filter_extension, -] - -post_request_model = create_post_request_model(extensions) - -api = StacApi( - settings=settings, - extensions=extensions, - client=CoreClient(session=session, post_request_model=post_request_model, database=database_logic), - search_get_request_model=create_get_request_model(extensions), - search_post_request_model=post_request_model, -) -app = api.app - - -@app.on_event("startup") -async def _startup_event() -> None: - await create_collection_index() - - -def run() -> None: - """Run app from command line using uvicorn if available.""" - try: - import uvicorn - - uvicorn.run( - "stac_fastapi.elasticsearch.app:app", - host=settings.app_host, - port=settings.app_port, - log_level="info", - reload=settings.reload, - ) - except ImportError: - raise RuntimeError("Uvicorn must be installed in order to use command") - - -if __name__ == "__main__": - run() - - -def create_handler(app): - """Create a handler to use with AWS Lambda if mangum available.""" - try: - from mangum import Mangum - - return Mangum(app) - except ImportError: - return None - - -handler = create_handler(app) diff --git a/stac_fastapi/open_search/setup.py b/stac_fastapi/open_search/setup.py index 033d10d4..2c0607fc 100644 --- a/stac_fastapi/open_search/setup.py +++ b/stac_fastapi/open_search/setup.py @@ -64,9 +64,5 @@ install_requires=install_requires, tests_require=extra_reqs["dev"], extras_require=extra_reqs, - entry_points={ - "console_scripts": [ - "stac-fastapi-elasticsearch=app:run" - ] - }, + entry_points={"console_scripts": ["stac-fastapi-elasticsearch=app:run"]}, ) From 0cc0c337dc2772019e04f8151ee8790e107d0725 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 30 Jan 2024 23:28:35 +0800 Subject: [PATCH 15/23] update dockerfiles --- Dockerfile.dev.es | 11 ++++++----- docker-compose.yml | 3 ++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile.dev.es b/Dockerfile.dev.es index 132eaea8..b64e2217 100644 --- a/Dockerfile.dev.es +++ b/Dockerfile.dev.es @@ -1,6 +1,5 @@ FROM python:3.10-slim - # update apt pkgs, and install build-essential for ciso8601 RUN apt-get update && \ apt-get -y upgrade && \ @@ -9,11 +8,13 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # update certs used by Requests -ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +ENV CURL_CA BUNDLE=/etc/ssl/certs/ca-certificates.crt WORKDIR /app -COPY ./stac_fastapi/common /app/common -COPY ./stac_fastapi/elastic_search /app/elastic_search +# Copy the contents of common and elastic_search directories directly into /app +COPY ./stac_fastapi/common /app/stac_fastapi/common +COPY ./stac_fastapi/elastic_search /app/stac_fastapi/elastic_search -RUN pip install --no-cache-dir -e ./elastic_search[dev,server] +# Install dependencies +RUN pip install --no-cache-dir -e ./stac_fastapi/elastic_search[dev,server] diff --git a/docker-compose.yml b/docker-compose.yml index 853a85cf..84a36cf5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,7 +22,8 @@ services: ports: - "8080:8080" volumes: - - ./stac_fastapi:/app/stac_fastapi + - ./stac_fastapi/elastic_search:/app/stac_fastapi/elastic_search + - ./stac_fastapi/common:/app/stac_fastapi/common - ./scripts:/app/scripts - ./esdata:/usr/share/elasticsearch/data depends_on: From 022dcb7106d93229a133f485c832d0c1492f8649 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 00:01:30 +0800 Subject: [PATCH 16/23] reorg --- .pre-commit-config.yaml | 2 +- stac_fastapi/common/core.py | 44 ++++++++++++------- stac_fastapi/elastic_search/app.py | 11 +++-- stac_fastapi/elastic_search/database_logic.py | 16 ++++--- stac_fastapi/elastic_search/pytest.ini | 2 +- stac_fastapi/elastic_search/serializers.py | 2 +- stac_fastapi/elastic_search/setup.py | 6 ++- stac_fastapi/elastic_search/tests/conftest.py | 15 ++++++- .../tests/resources/test_item.py | 2 +- 9 files changed, 65 insertions(+), 35 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index bef7f1c2..374db30b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: # E501 let black handle all line length decisions # W503 black conflicts with "line break before operator" rule # E203 black conflicts with "whitespace before ':'" rule - '--ignore=E501,W503,E203,C901' ] + '--ignore=E501,W503,E203,C901,E402' ] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.991 hooks: diff --git a/stac_fastapi/common/core.py b/stac_fastapi/common/core.py index 71e6fd3f..29c653a5 100644 --- a/stac_fastapi/common/core.py +++ b/stac_fastapi/common/core.py @@ -11,11 +11,6 @@ import attr import orjson import stac_pydantic -from common.base_database_logic import BaseDatabaseLogic -from common.models.links import PagingLinks -from elastic_search import serializers -from elastic_search.serializers import CollectionSerializer, ItemSerializer -from elastic_search.session import Session from fastapi import HTTPException, Request from overrides import overrides from pydantic import ValidationError @@ -24,6 +19,11 @@ from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes +# from common.base_database_logic import BaseDatabaseLogic +from stac_fastapi.common.models.links import PagingLinks +from stac_fastapi.elastic_search import serializers +from stac_fastapi.elastic_search.serializers import CollectionSerializer, ItemSerializer +from stac_fastapi.elastic_search.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, BulkTransactionMethod, @@ -66,12 +66,12 @@ class CoreClient(AsyncBaseCoreClient): session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer + default=serializers.ItemSerializer # type: ignore ) collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer + default=serializers.CollectionSerializer # type: ignore ) - database: BaseDatabaseLogic = attr.ib(init=False) + post_request_model = attr.ib(default=BaseSearchPostRequest) def __attrs_post_init__(self): """ @@ -84,11 +84,15 @@ def __attrs_post_init__(self): """ try: # Dynamically import the database logic based on installed package - database_module = importlib.import_module("elastic_search.database_logic") + database_module = importlib.import_module( + "stac_fastapi.elastic_search.database_logic" + ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") except ImportError: # Fall back to OpenSearch if Elasticsearch is not available - database_module = importlib.import_module("open_search.database_logic") + database_module = importlib.import_module( + "stac_fastapi.open_search.database_logic" + ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") self.database = DatabaseLogicClass() @@ -562,7 +566,7 @@ class TransactionsClient(AsyncBaseTransactionsClient): """Transactions extension specific CRUD operations.""" session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database: BaseDatabaseLogic = attr.ib(init=False) + # database: BaseDatabaseLogic = attr.ib(init=False) def __attrs_post_init__(self): """ @@ -575,11 +579,15 @@ def __attrs_post_init__(self): """ try: # Dynamically import the database logic based on installed package - database_module = importlib.import_module("elastic_search.database_logic") + database_module = importlib.import_module( + "stac_fastapi.elastic_search.database_logic" + ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") except ImportError: # Fall back to OpenSearch if Elasticsearch is not available - database_module = importlib.import_module("opensearch.database_logic") + database_module = importlib.import_module( + "stac_fastapi.opensearch.database_logic" + ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") self.database = DatabaseLogicClass() @@ -752,7 +760,7 @@ class BulkTransactionsClient(BaseBulkTransactionsClient): """ session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database: BaseDatabaseLogic = attr.ib(init=False) + # database: BaseDatabaseLogic = attr.ib(init=False) def __attrs_post_init__(self): """ @@ -765,11 +773,15 @@ def __attrs_post_init__(self): """ try: # Dynamically import the database logic based on installed package - database_module = importlib.import_module("elastic_search.database_logic") + database_module = importlib.import_module( + "stac_fastapi.elastic_search.database_logic" + ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") except ImportError: # Fall back to OpenSearch if Elasticsearch is not available - database_module = importlib.import_module("opensearch.database_logic") + database_module = importlib.import_module( + "stac_fastapi.opensearch.database_logic" + ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") self.database = DatabaseLogicClass() diff --git a/stac_fastapi/elastic_search/app.py b/stac_fastapi/elastic_search/app.py index 61e1015b..7b251025 100644 --- a/stac_fastapi/elastic_search/app.py +++ b/stac_fastapi/elastic_search/app.py @@ -1,7 +1,5 @@ """FastAPI application.""" -from elastic_search.config import ElasticsearchSettings -from elastic_search.database_logic import create_collection_index -from elastic_search.session import Session +import sys from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model @@ -12,6 +10,9 @@ TransactionsClient, ) from stac_fastapi.common.extensions import QueryExtension +from stac_fastapi.elastic_search.config import ElasticsearchSettings +from stac_fastapi.elastic_search.database_logic import create_collection_index +from stac_fastapi.elastic_search.session import Session from stac_fastapi.extensions.core import ( ContextExtension, FieldsExtension, @@ -22,6 +23,8 @@ ) from stac_fastapi.extensions.third_party import BulkTransactionExtension +print("API sys.path:", sys.path) + settings = ElasticsearchSettings() session = Session.create_from_settings(settings) @@ -46,7 +49,7 @@ api = StacApi( settings=settings, extensions=extensions, - client=CoreClient(session=session), + client=CoreClient(session=session, post_request_model=post_request_model), search_get_request_model=create_get_request_model(extensions), search_post_request_model=post_request_model, ) diff --git a/stac_fastapi/elastic_search/database_logic.py b/stac_fastapi/elastic_search/database_logic.py index 305ab72c..afcb3a30 100644 --- a/stac_fastapi/elastic_search/database_logic.py +++ b/stac_fastapi/elastic_search/database_logic.py @@ -6,14 +6,16 @@ from typing import Any, Dict, Iterable, List, Optional, Protocol, Tuple, Type, Union import attr -from common.extensions import filter -from elastic_search import serializers -from elastic_search.config import AsyncElasticsearchSettings -from elastic_search.config import ElasticsearchSettings as SyncElasticsearchSettings -from elastic_search.utilities import bbox2polygon from elasticsearch_dsl import Q, Search from elasticsearch import exceptions, helpers # type: ignore +from stac_fastapi.common.extensions import filter +from stac_fastapi.elastic_search import serializers +from stac_fastapi.elastic_search.config import AsyncElasticsearchSettings +from stac_fastapi.elastic_search.config import ( + ElasticsearchSettings as SyncElasticsearchSettings, +) +from stac_fastapi.elastic_search.utilities import bbox2polygon from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item @@ -281,10 +283,10 @@ class DatabaseLogic: sync_client = SyncElasticsearchSettings().create_client item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer + default=serializers.ItemSerializer # type: ignore ) collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer + default=serializers.CollectionSerializer # type: ignore ) """CORE LOGIC""" diff --git a/stac_fastapi/elastic_search/pytest.ini b/stac_fastapi/elastic_search/pytest.ini index db0353ef..f770b9c9 100644 --- a/stac_fastapi/elastic_search/pytest.ini +++ b/stac_fastapi/elastic_search/pytest.ini @@ -1,4 +1,4 @@ [pytest] -testpaths = tests +testpaths = elastic_search/tests addopts = -sv asyncio_mode = auto \ No newline at end of file diff --git a/stac_fastapi/elastic_search/serializers.py b/stac_fastapi/elastic_search/serializers.py index 786e2c98..ce5d1f32 100644 --- a/stac_fastapi/elastic_search/serializers.py +++ b/stac_fastapi/elastic_search/serializers.py @@ -3,8 +3,8 @@ from typing import Any import attr -from common.datetime_utils import now_to_rfc3339_str +from stac_fastapi.common.datetime_utils import now_to_rfc3339_str from stac_fastapi.types import stac as stac_types from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links diff --git a/stac_fastapi/elastic_search/setup.py b/stac_fastapi/elastic_search/setup.py index 2c0607fc..b2999b2b 100644 --- a/stac_fastapi/elastic_search/setup.py +++ b/stac_fastapi/elastic_search/setup.py @@ -57,12 +57,14 @@ url="https://github.com/stac-utils/stac-fastapi-elasticsearch", license="MIT", packages=find_namespace_packages( - include=["elasticsearch", "elasticsearch.*", "common", "common.*"], + include=["elastic_search", "elastic_search.*", "common", "common.*"], exclude=["tests", "scripts"], ), zip_safe=False, install_requires=install_requires, tests_require=extra_reqs["dev"], extras_require=extra_reqs, - entry_points={"console_scripts": ["stac-fastapi-elasticsearch=app:run"]}, + entry_points={ + "console_scripts": ["stac-fastapi-elasticsearch=elastic_search.app:run"] + }, ) diff --git a/stac_fastapi/elastic_search/tests/conftest.py b/stac_fastapi/elastic_search/tests/conftest.py index 2817fda6..026020ae 100644 --- a/stac_fastapi/elastic_search/tests/conftest.py +++ b/stac_fastapi/elastic_search/tests/conftest.py @@ -1,3 +1,7 @@ +import sys + +sys.path.insert(0, "/app") + import asyncio import copy import json @@ -6,14 +10,18 @@ import pytest import pytest_asyncio -from common.core import BulkTransactionsClient, CoreClient, TransactionsClient -from common.extensions import QueryExtension from elastic_search.config import AsyncElasticsearchSettings from elastic_search.database_logic import create_collection_index from httpx import AsyncClient from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model +from stac_fastapi.common.core import ( + BulkTransactionsClient, + CoreClient, + TransactionsClient, +) +from stac_fastapi.common.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, FieldsExtension, @@ -24,6 +32,9 @@ ) from stac_fastapi.types.config import Settings +# # Assuming your tests are in the 'tests' directory at the same level as 'stac_fastapi' +# sys.path.append(str(Path(__file__).parent.parent)) + DATA_DIR = os.path.join(os.path.dirname(__file__), "data") diff --git a/stac_fastapi/elastic_search/tests/resources/test_item.py b/stac_fastapi/elastic_search/tests/resources/test_item.py index b66b3572..88f23402 100644 --- a/stac_fastapi/elastic_search/tests/resources/test_item.py +++ b/stac_fastapi/elastic_search/tests/resources/test_item.py @@ -9,11 +9,11 @@ import ciso8601 import pystac import pytest +from common.core import CoreClient from common.datetime_utils import now_to_rfc3339_str from geojson_pydantic.geometries import Polygon from pystac.utils import datetime_to_str -from stac_fastapi.common.core import CoreClient from stac_fastapi.types.core import LandingPageMixin from ..conftest import create_item, refresh_indices From 002106a0f7de19f59897afd2600a173b66d84670 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 00:02:48 +0800 Subject: [PATCH 17/23] update workflow --- .github/workflows/cicd.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 9a50e28f..40d347ab 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -74,7 +74,7 @@ jobs: - name: Run test suite against Elasticsearch 7.x run: | - cd stac_fastapi/elasticsearch && pipenv run pytest -svvv + cd stac_fastapi/elastic_search && pipenv run pytest -svvv env: ENVIRONMENT: testing ES_PORT: 9200 @@ -84,7 +84,7 @@ jobs: - name: Run test suite against Elasticsearch 8.x run: | - cd stac_fastapi/elasticsearch && pipenv run pytest -svvv + cd stac_fastapi/elastic_search && pipenv run pytest -svvv env: ENVIRONMENT: testing ES_PORT: 9400 From d1e3d4c8e9b7d6338c342b6eee76df0352cc76c9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 00:08:34 +0800 Subject: [PATCH 18/23] update workflow --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 40d347ab..0d154347 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -70,7 +70,7 @@ jobs: - name: Install elasticsearch stac-fastapi run: | - pip install ./stac_fastapi/elasticsearch[dev,server] + pip install ./stac_fastapi/elastic_search[dev,server] - name: Run test suite against Elasticsearch 7.x run: | From 023ef9be332e7d64633c25cf09050f88d4d57d1d Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 00:14:01 +0800 Subject: [PATCH 19/23] remove path insert --- stac_fastapi/elastic_search/tests/conftest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elastic_search/tests/conftest.py b/stac_fastapi/elastic_search/tests/conftest.py index 026020ae..6723c198 100644 --- a/stac_fastapi/elastic_search/tests/conftest.py +++ b/stac_fastapi/elastic_search/tests/conftest.py @@ -1,6 +1,6 @@ -import sys +# import sys -sys.path.insert(0, "/app") +# sys.path.insert(0, "/app") import asyncio import copy From 34310f1a712ed9d00dd693305535a397617afb55 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 00:20:45 +0800 Subject: [PATCH 20/23] import change --- stac_fastapi/elastic_search/database_logic.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/elastic_search/database_logic.py b/stac_fastapi/elastic_search/database_logic.py index afcb3a30..a4f5a3db 100644 --- a/stac_fastapi/elastic_search/database_logic.py +++ b/stac_fastapi/elastic_search/database_logic.py @@ -9,13 +9,13 @@ from elasticsearch_dsl import Q, Search from elasticsearch import exceptions, helpers # type: ignore -from stac_fastapi.common.extensions import filter -from stac_fastapi.elastic_search import serializers -from stac_fastapi.elastic_search.config import AsyncElasticsearchSettings -from stac_fastapi.elastic_search.config import ( +from common.extensions import filter +from elastic_search import serializers +from elastic_search.config import AsyncElasticsearchSettings +from elastic_search.config import ( ElasticsearchSettings as SyncElasticsearchSettings, ) -from stac_fastapi.elastic_search.utilities import bbox2polygon +from elastic_search.utilities import bbox2polygon from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item From 311fabdcc607a281f529991a8476228613c0087b Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 11:20:01 +0800 Subject: [PATCH 21/23] reorganize code, tests --- Dockerfile.dev.es | 7 ++++--- docker-compose.yml | 7 ++++--- .../tests => stac_api}/__init__.py | 0 {stac_fastapi => stac_api}/common/__init__.py | 0 .../common/base_database_logic.py | 0 {stac_fastapi => stac_api}/common/core.py | 20 +++++++++---------- .../common/datetime_utils.py | 0 .../common/extensions/__init__.py | 0 .../common/extensions/filter.py | 0 .../common/extensions/query.py | 0 .../common/models/__init__.py | 0 .../common/models/links.py | 0 .../common/models/search.py | 0 .../common/types/search.py | 2 +- .../elastic_search/README.md | 0 .../elastic_search/__init__.py | 0 .../elastic_search/app.py | 12 +++++------ .../elastic_search/config.py | 0 .../elastic_search/database_logic.py | 14 ++++++------- .../elastic_search/serializers.py | 2 +- .../elastic_search/session.py | 0 .../elastic_search/setup.cfg | 0 .../elastic_search/setup.py | 2 +- .../elastic_search/utilities.py | 0 .../elastic_search/version.py | 0 .../open_search/setup.cfg | 0 .../open_search/setup.py | 0 .../open_search/version.py | 0 .../elastic_search => stac_api}/pytest.ini | 0 .../tests/api => stac_api/tests}/__init__.py | 0 .../tests/api}/__init__.py | 0 .../tests/api/test_api.py | 0 .../tests/clients}/__init__.py | 0 .../tests/clients/test_elasticsearch.py | 0 .../tests/conftest.py | 12 +++++------ .../tests/data/test_collection.json | 0 .../tests/data/test_item.json | 0 .../tests/extensions/cql2/example01.json | 0 .../tests/extensions/cql2/example04.json | 0 .../tests/extensions/cql2/example05a.json | 0 .../tests/extensions/cql2/example06b.json | 0 .../tests/extensions/cql2/example08.json | 0 .../tests/extensions/cql2/example09.json | 0 .../tests/extensions/cql2/example1.json | 0 .../tests/extensions/cql2/example10.json | 0 .../tests/extensions/cql2/example14.json | 0 .../tests/extensions/cql2/example15.json | 0 .../tests/extensions/cql2/example17.json | 0 .../tests/extensions/cql2/example18.json | 0 .../tests/extensions/cql2/example19.json | 0 .../tests/extensions/cql2/example20.json | 0 .../tests/extensions/cql2/example21.json | 0 .../tests/extensions/cql2/example22.json | 0 .../tests/extensions/test_filter.py | 0 stac_api/tests/resources/__init__.py | 0 .../tests/resources/test_collection.py | 0 .../tests/resources/test_conformance.py | 0 .../tests/resources/test_item.py | 0 .../tests/resources/test_mgmt.py | 0 59 files changed, 40 insertions(+), 38 deletions(-) rename {stac_fastapi/elastic_search/tests => stac_api}/__init__.py (100%) rename {stac_fastapi => stac_api}/common/__init__.py (100%) rename {stac_fastapi => stac_api}/common/base_database_logic.py (100%) rename {stac_fastapi => stac_api}/common/core.py (98%) rename {stac_fastapi => stac_api}/common/datetime_utils.py (100%) rename {stac_fastapi => stac_api}/common/extensions/__init__.py (100%) rename {stac_fastapi => stac_api}/common/extensions/filter.py (100%) rename {stac_fastapi => stac_api}/common/extensions/query.py (100%) rename {stac_fastapi => stac_api}/common/models/__init__.py (100%) rename {stac_fastapi => stac_api}/common/models/links.py (100%) rename {stac_fastapi => stac_api}/common/models/search.py (100%) rename {stac_fastapi => stac_api}/common/types/search.py (97%) rename {stac_fastapi => stac_api}/elastic_search/README.md (100%) rename {stac_fastapi => stac_api}/elastic_search/__init__.py (100%) rename {stac_fastapi => stac_api}/elastic_search/app.py (86%) rename {stac_fastapi => stac_api}/elastic_search/config.py (100%) rename {stac_fastapi => stac_api}/elastic_search/database_logic.py (98%) rename {stac_fastapi => stac_api}/elastic_search/serializers.py (98%) rename {stac_fastapi => stac_api}/elastic_search/session.py (100%) rename {stac_fastapi => stac_api}/elastic_search/setup.cfg (100%) rename {stac_fastapi => stac_api}/elastic_search/setup.py (95%) rename {stac_fastapi => stac_api}/elastic_search/utilities.py (100%) rename {stac_fastapi => stac_api}/elastic_search/version.py (100%) rename {stac_fastapi => stac_api}/open_search/setup.cfg (100%) rename {stac_fastapi => stac_api}/open_search/setup.py (100%) rename {stac_fastapi => stac_api}/open_search/version.py (100%) rename {stac_fastapi/elastic_search => stac_api}/pytest.ini (100%) rename {stac_fastapi/elastic_search/tests/api => stac_api/tests}/__init__.py (100%) rename {stac_fastapi/elastic_search/tests/clients => stac_api/tests/api}/__init__.py (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/api/test_api.py (100%) rename {stac_fastapi/elastic_search/tests/resources => stac_api/tests/clients}/__init__.py (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/clients/test_elasticsearch.py (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/conftest.py (94%) rename {stac_fastapi/elastic_search => stac_api}/tests/data/test_collection.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/data/test_item.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example01.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example04.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example05a.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example06b.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example08.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example09.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example1.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example10.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example14.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example15.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example17.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example18.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example19.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example20.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example21.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/cql2/example22.json (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/extensions/test_filter.py (100%) create mode 100644 stac_api/tests/resources/__init__.py rename {stac_fastapi/elastic_search => stac_api}/tests/resources/test_collection.py (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/resources/test_conformance.py (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/resources/test_item.py (100%) rename {stac_fastapi/elastic_search => stac_api}/tests/resources/test_mgmt.py (100%) diff --git a/Dockerfile.dev.es b/Dockerfile.dev.es index b64e2217..f1194d1d 100644 --- a/Dockerfile.dev.es +++ b/Dockerfile.dev.es @@ -13,8 +13,9 @@ ENV CURL_CA BUNDLE=/etc/ssl/certs/ca-certificates.crt WORKDIR /app # Copy the contents of common and elastic_search directories directly into /app -COPY ./stac_fastapi/common /app/stac_fastapi/common -COPY ./stac_fastapi/elastic_search /app/stac_fastapi/elastic_search +COPY ./stac_api/common /app/stac_api/common +COPY ./stac_api/elastic_search /app/stac_api/elastic_search +COPY ./stac_api/tests /app/stac_api/tests # Install dependencies -RUN pip install --no-cache-dir -e ./stac_fastapi/elastic_search[dev,server] +RUN pip install --no-cache-dir -e ./stac_api/elastic_search[dev,server] diff --git a/docker-compose.yml b/docker-compose.yml index 84a36cf5..6cfe18ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,14 +22,15 @@ services: ports: - "8080:8080" volumes: - - ./stac_fastapi/elastic_search:/app/stac_fastapi/elastic_search - - ./stac_fastapi/common:/app/stac_fastapi/common + - ./stac_api/elastic_search:/app/stac_api/elastic_search + - ./stac_api/common:/app/stac_api/common + - ./stac_api/tests:/app/stac_api/tests - ./scripts:/app/scripts - ./esdata:/usr/share/elasticsearch/data depends_on: - elasticsearch command: - bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elastic_search.app" + bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_api.elastic_search.app" app-opensearch: container_name: stac-fastapi-os diff --git a/stac_fastapi/elastic_search/tests/__init__.py b/stac_api/__init__.py similarity index 100% rename from stac_fastapi/elastic_search/tests/__init__.py rename to stac_api/__init__.py diff --git a/stac_fastapi/common/__init__.py b/stac_api/common/__init__.py similarity index 100% rename from stac_fastapi/common/__init__.py rename to stac_api/common/__init__.py diff --git a/stac_fastapi/common/base_database_logic.py b/stac_api/common/base_database_logic.py similarity index 100% rename from stac_fastapi/common/base_database_logic.py rename to stac_api/common/base_database_logic.py diff --git a/stac_fastapi/common/core.py b/stac_api/common/core.py similarity index 98% rename from stac_fastapi/common/core.py rename to stac_api/common/core.py index 29c653a5..918b5ce0 100644 --- a/stac_fastapi/common/core.py +++ b/stac_api/common/core.py @@ -20,10 +20,10 @@ from stac_pydantic.shared import MimeTypes # from common.base_database_logic import BaseDatabaseLogic -from stac_fastapi.common.models.links import PagingLinks -from stac_fastapi.elastic_search import serializers -from stac_fastapi.elastic_search.serializers import CollectionSerializer, ItemSerializer -from stac_fastapi.elastic_search.session import Session +from stac_api.common.models.links import PagingLinks +from stac_api.elastic_search import serializers +from stac_api.elastic_search.serializers import CollectionSerializer, ItemSerializer +from stac_api.elastic_search.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, BulkTransactionMethod, @@ -85,13 +85,13 @@ def __attrs_post_init__(self): try: # Dynamically import the database logic based on installed package database_module = importlib.import_module( - "stac_fastapi.elastic_search.database_logic" + "stac_api.elastic_search.database_logic" ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") except ImportError: # Fall back to OpenSearch if Elasticsearch is not available database_module = importlib.import_module( - "stac_fastapi.open_search.database_logic" + "stac_api.open_search.database_logic" ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") @@ -580,13 +580,13 @@ def __attrs_post_init__(self): try: # Dynamically import the database logic based on installed package database_module = importlib.import_module( - "stac_fastapi.elastic_search.database_logic" + "stac_api.elastic_search.database_logic" ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") except ImportError: # Fall back to OpenSearch if Elasticsearch is not available database_module = importlib.import_module( - "stac_fastapi.opensearch.database_logic" + "stac_api.opensearch.database_logic" ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") @@ -774,13 +774,13 @@ def __attrs_post_init__(self): try: # Dynamically import the database logic based on installed package database_module = importlib.import_module( - "stac_fastapi.elastic_search.database_logic" + "stac_api.elastic_search.database_logic" ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") except ImportError: # Fall back to OpenSearch if Elasticsearch is not available database_module = importlib.import_module( - "stac_fastapi.opensearch.database_logic" + "stac_api.opensearch.database_logic" ) DatabaseLogicClass = getattr(database_module, "DatabaseLogic") diff --git a/stac_fastapi/common/datetime_utils.py b/stac_api/common/datetime_utils.py similarity index 100% rename from stac_fastapi/common/datetime_utils.py rename to stac_api/common/datetime_utils.py diff --git a/stac_fastapi/common/extensions/__init__.py b/stac_api/common/extensions/__init__.py similarity index 100% rename from stac_fastapi/common/extensions/__init__.py rename to stac_api/common/extensions/__init__.py diff --git a/stac_fastapi/common/extensions/filter.py b/stac_api/common/extensions/filter.py similarity index 100% rename from stac_fastapi/common/extensions/filter.py rename to stac_api/common/extensions/filter.py diff --git a/stac_fastapi/common/extensions/query.py b/stac_api/common/extensions/query.py similarity index 100% rename from stac_fastapi/common/extensions/query.py rename to stac_api/common/extensions/query.py diff --git a/stac_fastapi/common/models/__init__.py b/stac_api/common/models/__init__.py similarity index 100% rename from stac_fastapi/common/models/__init__.py rename to stac_api/common/models/__init__.py diff --git a/stac_fastapi/common/models/links.py b/stac_api/common/models/links.py similarity index 100% rename from stac_fastapi/common/models/links.py rename to stac_api/common/models/links.py diff --git a/stac_fastapi/common/models/search.py b/stac_api/common/models/search.py similarity index 100% rename from stac_fastapi/common/models/search.py rename to stac_api/common/models/search.py diff --git a/stac_fastapi/common/types/search.py b/stac_api/common/types/search.py similarity index 97% rename from stac_fastapi/common/types/search.py rename to stac_api/common/types/search.py index 26a2dbb6..b2536021 100644 --- a/stac_fastapi/common/types/search.py +++ b/stac_api/common/types/search.py @@ -8,7 +8,7 @@ from stac_pydantic.api.extensions.fields import FieldsExtension as FieldsBase -from stac_fastapi.types.config import Settings +from stac_api.types.config import Settings logger = logging.getLogger("uvicorn") logger.setLevel(logging.INFO) diff --git a/stac_fastapi/elastic_search/README.md b/stac_api/elastic_search/README.md similarity index 100% rename from stac_fastapi/elastic_search/README.md rename to stac_api/elastic_search/README.md diff --git a/stac_fastapi/elastic_search/__init__.py b/stac_api/elastic_search/__init__.py similarity index 100% rename from stac_fastapi/elastic_search/__init__.py rename to stac_api/elastic_search/__init__.py diff --git a/stac_fastapi/elastic_search/app.py b/stac_api/elastic_search/app.py similarity index 86% rename from stac_fastapi/elastic_search/app.py rename to stac_api/elastic_search/app.py index 7b251025..0f89c92d 100644 --- a/stac_fastapi/elastic_search/app.py +++ b/stac_api/elastic_search/app.py @@ -3,16 +3,16 @@ from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_fastapi.common.core import ( +from stac_api.common.core import ( BulkTransactionsClient, CoreClient, EsAsyncBaseFiltersClient, TransactionsClient, ) -from stac_fastapi.common.extensions import QueryExtension -from stac_fastapi.elastic_search.config import ElasticsearchSettings -from stac_fastapi.elastic_search.database_logic import create_collection_index -from stac_fastapi.elastic_search.session import Session +from stac_api.common.extensions import QueryExtension +from stac_api.elastic_search.config import ElasticsearchSettings +from stac_api.elastic_search.database_logic import create_collection_index +from stac_api.elastic_search.session import Session from stac_fastapi.extensions.core import ( ContextExtension, FieldsExtension, @@ -67,7 +67,7 @@ def run() -> None: import uvicorn uvicorn.run( - "stac_fastapi.elastic_search.app:app", + "stac_api.elastic_search.app:app", host=settings.app_host, port=settings.app_port, log_level="info", diff --git a/stac_fastapi/elastic_search/config.py b/stac_api/elastic_search/config.py similarity index 100% rename from stac_fastapi/elastic_search/config.py rename to stac_api/elastic_search/config.py diff --git a/stac_fastapi/elastic_search/database_logic.py b/stac_api/elastic_search/database_logic.py similarity index 98% rename from stac_fastapi/elastic_search/database_logic.py rename to stac_api/elastic_search/database_logic.py index a4f5a3db..b0a4a02d 100644 --- a/stac_fastapi/elastic_search/database_logic.py +++ b/stac_api/elastic_search/database_logic.py @@ -9,13 +9,13 @@ from elasticsearch_dsl import Q, Search from elasticsearch import exceptions, helpers # type: ignore -from common.extensions import filter -from elastic_search import serializers -from elastic_search.config import AsyncElasticsearchSettings -from elastic_search.config import ( +from stac_api.common.extensions import filter +from stac_api.elastic_search import serializers +from stac_api.elastic_search.config import AsyncElasticsearchSettings +from stac_api.elastic_search.config import ( ElasticsearchSettings as SyncElasticsearchSettings, ) -from elastic_search.utilities import bbox2polygon +from stac_api.elastic_search.utilities import bbox2polygon from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item @@ -283,10 +283,10 @@ class DatabaseLogic: sync_client = SyncElasticsearchSettings().create_client item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer # type: ignore + default=serializers.ItemSerializer # type: ignore ) collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer # type: ignore + default=serializers.CollectionSerializer # type: ignore ) """CORE LOGIC""" diff --git a/stac_fastapi/elastic_search/serializers.py b/stac_api/elastic_search/serializers.py similarity index 98% rename from stac_fastapi/elastic_search/serializers.py rename to stac_api/elastic_search/serializers.py index ce5d1f32..58b9f458 100644 --- a/stac_fastapi/elastic_search/serializers.py +++ b/stac_api/elastic_search/serializers.py @@ -4,7 +4,7 @@ import attr -from stac_fastapi.common.datetime_utils import now_to_rfc3339_str +from stac_api.common.datetime_utils import now_to_rfc3339_str from stac_fastapi.types import stac as stac_types from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links diff --git a/stac_fastapi/elastic_search/session.py b/stac_api/elastic_search/session.py similarity index 100% rename from stac_fastapi/elastic_search/session.py rename to stac_api/elastic_search/session.py diff --git a/stac_fastapi/elastic_search/setup.cfg b/stac_api/elastic_search/setup.cfg similarity index 100% rename from stac_fastapi/elastic_search/setup.cfg rename to stac_api/elastic_search/setup.cfg diff --git a/stac_fastapi/elastic_search/setup.py b/stac_api/elastic_search/setup.py similarity index 95% rename from stac_fastapi/elastic_search/setup.py rename to stac_api/elastic_search/setup.py index b2999b2b..842baddf 100644 --- a/stac_fastapi/elastic_search/setup.py +++ b/stac_api/elastic_search/setup.py @@ -65,6 +65,6 @@ tests_require=extra_reqs["dev"], extras_require=extra_reqs, entry_points={ - "console_scripts": ["stac-fastapi-elasticsearch=elastic_search.app:run"] + "console_scripts": ["stac-fastapi-elasticsearch=stac_api.elastic_search.app:run"] }, ) diff --git a/stac_fastapi/elastic_search/utilities.py b/stac_api/elastic_search/utilities.py similarity index 100% rename from stac_fastapi/elastic_search/utilities.py rename to stac_api/elastic_search/utilities.py diff --git a/stac_fastapi/elastic_search/version.py b/stac_api/elastic_search/version.py similarity index 100% rename from stac_fastapi/elastic_search/version.py rename to stac_api/elastic_search/version.py diff --git a/stac_fastapi/open_search/setup.cfg b/stac_api/open_search/setup.cfg similarity index 100% rename from stac_fastapi/open_search/setup.cfg rename to stac_api/open_search/setup.cfg diff --git a/stac_fastapi/open_search/setup.py b/stac_api/open_search/setup.py similarity index 100% rename from stac_fastapi/open_search/setup.py rename to stac_api/open_search/setup.py diff --git a/stac_fastapi/open_search/version.py b/stac_api/open_search/version.py similarity index 100% rename from stac_fastapi/open_search/version.py rename to stac_api/open_search/version.py diff --git a/stac_fastapi/elastic_search/pytest.ini b/stac_api/pytest.ini similarity index 100% rename from stac_fastapi/elastic_search/pytest.ini rename to stac_api/pytest.ini diff --git a/stac_fastapi/elastic_search/tests/api/__init__.py b/stac_api/tests/__init__.py similarity index 100% rename from stac_fastapi/elastic_search/tests/api/__init__.py rename to stac_api/tests/__init__.py diff --git a/stac_fastapi/elastic_search/tests/clients/__init__.py b/stac_api/tests/api/__init__.py similarity index 100% rename from stac_fastapi/elastic_search/tests/clients/__init__.py rename to stac_api/tests/api/__init__.py diff --git a/stac_fastapi/elastic_search/tests/api/test_api.py b/stac_api/tests/api/test_api.py similarity index 100% rename from stac_fastapi/elastic_search/tests/api/test_api.py rename to stac_api/tests/api/test_api.py diff --git a/stac_fastapi/elastic_search/tests/resources/__init__.py b/stac_api/tests/clients/__init__.py similarity index 100% rename from stac_fastapi/elastic_search/tests/resources/__init__.py rename to stac_api/tests/clients/__init__.py diff --git a/stac_fastapi/elastic_search/tests/clients/test_elasticsearch.py b/stac_api/tests/clients/test_elasticsearch.py similarity index 100% rename from stac_fastapi/elastic_search/tests/clients/test_elasticsearch.py rename to stac_api/tests/clients/test_elasticsearch.py diff --git a/stac_fastapi/elastic_search/tests/conftest.py b/stac_api/tests/conftest.py similarity index 94% rename from stac_fastapi/elastic_search/tests/conftest.py rename to stac_api/tests/conftest.py index 6723c198..99886d8c 100644 --- a/stac_fastapi/elastic_search/tests/conftest.py +++ b/stac_api/tests/conftest.py @@ -1,6 +1,6 @@ -# import sys +import sys -# sys.path.insert(0, "/app") +sys.path.insert(0, "/app") import asyncio import copy @@ -10,18 +10,18 @@ import pytest import pytest_asyncio -from elastic_search.config import AsyncElasticsearchSettings -from elastic_search.database_logic import create_collection_index +from stac_api.elastic_search.config import AsyncElasticsearchSettings +from stac_api.elastic_search.database_logic import create_collection_index from httpx import AsyncClient from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_fastapi.common.core import ( +from stac_api.common.core import ( BulkTransactionsClient, CoreClient, TransactionsClient, ) -from stac_fastapi.common.extensions import QueryExtension +from stac_api.common.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, FieldsExtension, diff --git a/stac_fastapi/elastic_search/tests/data/test_collection.json b/stac_api/tests/data/test_collection.json similarity index 100% rename from stac_fastapi/elastic_search/tests/data/test_collection.json rename to stac_api/tests/data/test_collection.json diff --git a/stac_fastapi/elastic_search/tests/data/test_item.json b/stac_api/tests/data/test_item.json similarity index 100% rename from stac_fastapi/elastic_search/tests/data/test_item.json rename to stac_api/tests/data/test_item.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example01.json b/stac_api/tests/extensions/cql2/example01.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example01.json rename to stac_api/tests/extensions/cql2/example01.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example04.json b/stac_api/tests/extensions/cql2/example04.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example04.json rename to stac_api/tests/extensions/cql2/example04.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example05a.json b/stac_api/tests/extensions/cql2/example05a.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example05a.json rename to stac_api/tests/extensions/cql2/example05a.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example06b.json b/stac_api/tests/extensions/cql2/example06b.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example06b.json rename to stac_api/tests/extensions/cql2/example06b.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example08.json b/stac_api/tests/extensions/cql2/example08.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example08.json rename to stac_api/tests/extensions/cql2/example08.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example09.json b/stac_api/tests/extensions/cql2/example09.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example09.json rename to stac_api/tests/extensions/cql2/example09.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example1.json b/stac_api/tests/extensions/cql2/example1.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example1.json rename to stac_api/tests/extensions/cql2/example1.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example10.json b/stac_api/tests/extensions/cql2/example10.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example10.json rename to stac_api/tests/extensions/cql2/example10.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example14.json b/stac_api/tests/extensions/cql2/example14.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example14.json rename to stac_api/tests/extensions/cql2/example14.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example15.json b/stac_api/tests/extensions/cql2/example15.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example15.json rename to stac_api/tests/extensions/cql2/example15.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example17.json b/stac_api/tests/extensions/cql2/example17.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example17.json rename to stac_api/tests/extensions/cql2/example17.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example18.json b/stac_api/tests/extensions/cql2/example18.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example18.json rename to stac_api/tests/extensions/cql2/example18.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example19.json b/stac_api/tests/extensions/cql2/example19.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example19.json rename to stac_api/tests/extensions/cql2/example19.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example20.json b/stac_api/tests/extensions/cql2/example20.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example20.json rename to stac_api/tests/extensions/cql2/example20.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example21.json b/stac_api/tests/extensions/cql2/example21.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example21.json rename to stac_api/tests/extensions/cql2/example21.json diff --git a/stac_fastapi/elastic_search/tests/extensions/cql2/example22.json b/stac_api/tests/extensions/cql2/example22.json similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/cql2/example22.json rename to stac_api/tests/extensions/cql2/example22.json diff --git a/stac_fastapi/elastic_search/tests/extensions/test_filter.py b/stac_api/tests/extensions/test_filter.py similarity index 100% rename from stac_fastapi/elastic_search/tests/extensions/test_filter.py rename to stac_api/tests/extensions/test_filter.py diff --git a/stac_api/tests/resources/__init__.py b/stac_api/tests/resources/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/stac_fastapi/elastic_search/tests/resources/test_collection.py b/stac_api/tests/resources/test_collection.py similarity index 100% rename from stac_fastapi/elastic_search/tests/resources/test_collection.py rename to stac_api/tests/resources/test_collection.py diff --git a/stac_fastapi/elastic_search/tests/resources/test_conformance.py b/stac_api/tests/resources/test_conformance.py similarity index 100% rename from stac_fastapi/elastic_search/tests/resources/test_conformance.py rename to stac_api/tests/resources/test_conformance.py diff --git a/stac_fastapi/elastic_search/tests/resources/test_item.py b/stac_api/tests/resources/test_item.py similarity index 100% rename from stac_fastapi/elastic_search/tests/resources/test_item.py rename to stac_api/tests/resources/test_item.py diff --git a/stac_fastapi/elastic_search/tests/resources/test_mgmt.py b/stac_api/tests/resources/test_mgmt.py similarity index 100% rename from stac_fastapi/elastic_search/tests/resources/test_mgmt.py rename to stac_api/tests/resources/test_mgmt.py From 653b9c4eede8a101c99093469971ae9fd70d22f8 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 11:25:09 +0800 Subject: [PATCH 22/23] workflow, pre-commit --- .github/workflows/cicd.yml | 6 +++--- stac_api/__init__.py | 1 + stac_api/elastic_search/app.py | 4 ++-- stac_api/elastic_search/database_logic.py | 4 ++-- stac_api/elastic_search/setup.py | 4 +++- stac_api/tests/conftest.py | 12 ++++-------- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 0d154347..38172df8 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -70,11 +70,11 @@ jobs: - name: Install elasticsearch stac-fastapi run: | - pip install ./stac_fastapi/elastic_search[dev,server] + pip install ./stac_api/elastic_search[dev,server] - name: Run test suite against Elasticsearch 7.x run: | - cd stac_fastapi/elastic_search && pipenv run pytest -svvv + cd stac_api && pipenv run pytest -svvv env: ENVIRONMENT: testing ES_PORT: 9200 @@ -84,7 +84,7 @@ jobs: - name: Run test suite against Elasticsearch 8.x run: | - cd stac_fastapi/elastic_search && pipenv run pytest -svvv + cd stac_api && pipenv run pytest -svvv env: ENVIRONMENT: testing ES_PORT: 9400 diff --git a/stac_api/__init__.py b/stac_api/__init__.py index e69de29b..11a01537 100644 --- a/stac_api/__init__.py +++ b/stac_api/__init__.py @@ -0,0 +1 @@ +"""stac api.""" diff --git a/stac_api/elastic_search/app.py b/stac_api/elastic_search/app.py index 0f89c92d..b0fbc709 100644 --- a/stac_api/elastic_search/app.py +++ b/stac_api/elastic_search/app.py @@ -1,8 +1,6 @@ """FastAPI application.""" import sys -from stac_fastapi.api.app import StacApi -from stac_fastapi.api.models import create_get_request_model, create_post_request_model from stac_api.common.core import ( BulkTransactionsClient, CoreClient, @@ -13,6 +11,8 @@ from stac_api.elastic_search.config import ElasticsearchSettings from stac_api.elastic_search.database_logic import create_collection_index from stac_api.elastic_search.session import Session +from stac_fastapi.api.app import StacApi +from stac_fastapi.api.models import create_get_request_model, create_post_request_model from stac_fastapi.extensions.core import ( ContextExtension, FieldsExtension, diff --git a/stac_api/elastic_search/database_logic.py b/stac_api/elastic_search/database_logic.py index b0a4a02d..3693158c 100644 --- a/stac_api/elastic_search/database_logic.py +++ b/stac_api/elastic_search/database_logic.py @@ -283,10 +283,10 @@ class DatabaseLogic: sync_client = SyncElasticsearchSettings().create_client item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer # type: ignore + default=serializers.ItemSerializer # type: ignore ) collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer # type: ignore + default=serializers.CollectionSerializer # type: ignore ) """CORE LOGIC""" diff --git a/stac_api/elastic_search/setup.py b/stac_api/elastic_search/setup.py index 842baddf..d8420886 100644 --- a/stac_api/elastic_search/setup.py +++ b/stac_api/elastic_search/setup.py @@ -65,6 +65,8 @@ tests_require=extra_reqs["dev"], extras_require=extra_reqs, entry_points={ - "console_scripts": ["stac-fastapi-elasticsearch=stac_api.elastic_search.app:run"] + "console_scripts": [ + "stac-fastapi-elasticsearch=stac_api.elastic_search.app:run" + ] }, ) diff --git a/stac_api/tests/conftest.py b/stac_api/tests/conftest.py index 99886d8c..1c1b687d 100644 --- a/stac_api/tests/conftest.py +++ b/stac_api/tests/conftest.py @@ -10,18 +10,14 @@ import pytest import pytest_asyncio -from stac_api.elastic_search.config import AsyncElasticsearchSettings -from stac_api.elastic_search.database_logic import create_collection_index from httpx import AsyncClient +from stac_api.common.core import BulkTransactionsClient, CoreClient, TransactionsClient +from stac_api.common.extensions import QueryExtension +from stac_api.elastic_search.config import AsyncElasticsearchSettings +from stac_api.elastic_search.database_logic import create_collection_index from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_api.common.core import ( - BulkTransactionsClient, - CoreClient, - TransactionsClient, -) -from stac_api.common.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, FieldsExtension, From 635eb64827610062a095fa6f38765349fc05532d Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 31 Jan 2024 11:35:23 +0800 Subject: [PATCH 23/23] revert --- .pre-commit-config.yaml | 2 +- stac_api/tests/conftest.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 374db30b..bef7f1c2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: # E501 let black handle all line length decisions # W503 black conflicts with "line break before operator" rule # E203 black conflicts with "whitespace before ':'" rule - '--ignore=E501,W503,E203,C901,E402' ] + '--ignore=E501,W503,E203,C901' ] - repo: https://github.com/pre-commit/mirrors-mypy rev: v0.991 hooks: diff --git a/stac_api/tests/conftest.py b/stac_api/tests/conftest.py index 1c1b687d..157cabce 100644 --- a/stac_api/tests/conftest.py +++ b/stac_api/tests/conftest.py @@ -1,6 +1,6 @@ -import sys +# import sys -sys.path.insert(0, "/app") +# sys.path.insert(0, "/app") import asyncio import copy