diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 9a50e28f..38172df8 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -70,11 +70,11 @@ jobs: - name: Install elasticsearch stac-fastapi run: | - pip install ./stac_fastapi/elasticsearch[dev,server] + pip install ./stac_api/elastic_search[dev,server] - name: Run test suite against Elasticsearch 7.x run: | - cd stac_fastapi/elasticsearch && pipenv run pytest -svvv + cd stac_api && pipenv run pytest -svvv env: ENVIRONMENT: testing ES_PORT: 9200 @@ -84,7 +84,7 @@ jobs: - name: Run test suite against Elasticsearch 8.x run: | - cd stac_fastapi/elasticsearch && pipenv run pytest -svvv + cd stac_api && pipenv run pytest -svvv env: ENVIRONMENT: testing ES_PORT: 9400 diff --git a/Dockerfile.dev b/Dockerfile.dev deleted file mode 100644 index 4e2f0f4b..00000000 --- a/Dockerfile.dev +++ /dev/null @@ -1,18 +0,0 @@ -FROM python:3.10-slim - - -# update apt pkgs, and install build-essential for ciso8601 -RUN apt-get update && \ - apt-get -y upgrade && \ - apt-get install -y build-essential && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# update certs used by Requests -ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt - -WORKDIR /app - -COPY . /app - -RUN pip install --no-cache-dir -e ./stac_fastapi/elasticsearch[dev,server] diff --git a/Dockerfile.dev.es b/Dockerfile.dev.es new file mode 100644 index 00000000..f1194d1d --- /dev/null +++ b/Dockerfile.dev.es @@ -0,0 +1,21 @@ +FROM python:3.10-slim + +# update apt pkgs, and install build-essential for ciso8601 +RUN apt-get update && \ + apt-get -y upgrade && \ + apt-get install -y build-essential && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# update certs used by Requests +ENV CURL_CA BUNDLE=/etc/ssl/certs/ca-certificates.crt + +WORKDIR /app + +# Copy the contents of common and elastic_search directories directly into /app +COPY ./stac_api/common /app/stac_api/common +COPY ./stac_api/elastic_search /app/stac_api/elastic_search +COPY ./stac_api/tests /app/stac_api/tests + +# Install dependencies +RUN pip install --no-cache-dir -e ./stac_api/elastic_search[dev,server] diff --git a/docker-compose.yml b/docker-compose.yml index db3352fb..6cfe18ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,11 @@ version: '3.9' services: app-elasticsearch: container_name: stac-fastapi-es - image: stac-utils/stac-fastapi + image: stac-utils/stac-fastapi-es restart: always build: context: . - dockerfile: Dockerfile.dev + dockerfile: Dockerfile.dev.es environment: - APP_HOST=0.0.0.0 - APP_PORT=8080 @@ -18,16 +18,48 @@ services: - ES_PORT=9200 - ES_USE_SSL=false - ES_VERIFY_CERTS=false + - DB_TYPE=elasticsearch ports: - "8080:8080" volumes: - - ./stac_fastapi:/app/stac_fastapi + - ./stac_api/elastic_search:/app/stac_api/elastic_search + - ./stac_api/common:/app/stac_api/common + - ./stac_api/tests:/app/stac_api/tests - ./scripts:/app/scripts - ./esdata:/usr/share/elasticsearch/data depends_on: - elasticsearch command: - bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app" + bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_api.elastic_search.app" + + app-opensearch: + container_name: stac-fastapi-os + image: stac-utils/stac-fastapi-os + restart: always + build: + context: . + dockerfile: Dockerfile.dev + environment: + - APP_HOST=0.0.0.0 + - APP_PORT=8080 + - RELOAD=true + - ENVIRONMENT=local + - WEB_CONCURRENCY=10 + - ES_HOST=172.17.0.1 + - ES_PORT=9200 + - ES_USE_SSL=false + - ES_VERIFY_CERTS=false + - DB_TYPE=opensearch + ports: + - "8082:8080" + volumes: + - ./stac_fastapi:/app/stac_fastapi + - ./scripts:/app/scripts + - ./osdata:/usr/share/opensearch/data + depends_on: + - opensearch + command: + bash -c "./scripts/wait-for-it-es.sh os-container:9200 && python -m stac_fastapi.elastic_search.app" elasticsearch: container_name: es-container @@ -39,3 +71,15 @@ services: - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots ports: - "9200:9200" + + opensearch: + container_name: os-container + image: opensearchproject/opensearch:latest + environment: + - "discovery.type=single-node" + - "plugins.security.disabled=true" + volumes: + - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml + - ./opensearch/snapshots:/usr/share/opensearch/snapshots + ports: + - "9202:9200" diff --git a/examples/pip_docker/docker-compose.yml b/examples/pip_docker/docker-compose.yml index 2c2d289e..8c76967d 100644 --- a/examples/pip_docker/docker-compose.yml +++ b/examples/pip_docker/docker-compose.yml @@ -27,7 +27,7 @@ services: depends_on: - elasticsearch command: - bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app" + bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elastic_search.app" elasticsearch: container_name: es-container diff --git a/opensearch/config/opensearch.yml b/opensearch/config/opensearch.yml new file mode 100644 index 00000000..dc69776a --- /dev/null +++ b/opensearch/config/opensearch.yml @@ -0,0 +1,35 @@ +## Cluster Settings +cluster.name: stac-cluster +node.name: es01 +network.host: 0.0.0.0 +transport.host: 0.0.0.0 +discovery.type: single-node +http.port: 9200 + +path: + repo: + - /usr/share/opensearch/snapshots + +######## Start OpenSearch Security Demo Configuration ######## +# WARNING: revise all the lines below before you go into production +plugins.security.ssl.transport.pemcert_filepath: esnode.pem +plugins.security.ssl.transport.pemkey_filepath: esnode-key.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem +plugins.security.ssl.transport.enforce_hostname_verification: false +plugins.security.ssl.http.enabled: true +plugins.security.ssl.http.pemcert_filepath: esnode.pem +plugins.security.ssl.http.pemkey_filepath: esnode-key.pem +plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem +plugins.security.allow_unsafe_democertificates: true +plugins.security.allow_default_init_securityindex: true +plugins.security.authcz.admin_dn: + - CN=kirk,OU=client,O=client,L=test, C=de + +plugins.security.audit.type: internal_opensearch +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.system_indices.enabled: true +plugins.security.system_indices.indices: [".plugins-ml-config", ".plugins-ml-connector", ".plugins-ml-model-group", ".plugins-ml-model", ".plugins-ml-task", ".plugins-ml-conversation-meta", ".plugins-ml-conversation-interactions", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".ql-datasources", ".opendistro-asynchronous-search-response*", ".replication-metadata-store", ".opensearch-knn-models", ".geospatial-ip2geo-data*"] +node.max_local_storage_nodes: 3 +######## End OpenSearch Security Demo Configuration ######## diff --git a/stac_api/__init__.py b/stac_api/__init__.py new file mode 100644 index 00000000..11a01537 --- /dev/null +++ b/stac_api/__init__.py @@ -0,0 +1 @@ +"""stac api.""" diff --git a/stac_api/common/__init__.py b/stac_api/common/__init__.py new file mode 100644 index 00000000..69f055ab --- /dev/null +++ b/stac_api/common/__init__.py @@ -0,0 +1 @@ +"""elasticsearch/ opensearch shared code.""" diff --git a/stac_api/common/base_database_logic.py b/stac_api/common/base_database_logic.py new file mode 100644 index 00000000..09bb01a6 --- /dev/null +++ b/stac_api/common/base_database_logic.py @@ -0,0 +1,47 @@ +"""base class for database logic.""" + +# from abc import ABC, abstractmethod +# from typing import Any, Dict, List, Optional, Tuple, Union + +# from stac_fastapi.types.stac import Collection, Item + + +# class BaseDatabaseLogic(ABC): +# """ +# Abstract base class for database logic. +# This class defines the interface for database operations. +# """ + +# @abstractmethod +# async def get_all_collections(self, token: Optional[str], limit: int) -> Any: +# pass + +# @abstractmethod +# async def get_one_item(self, collection_id: str, item_id: str) -> Dict: +# pass + +# @abstractmethod +# async def create_item(self, item: Item, refresh: bool = False) -> None: +# pass + +# @abstractmethod +# async def delete_item( +# self, item_id: str, collection_id: str, refresh: bool = False +# ) -> None: +# pass + +# @abstractmethod +# async def create_collection( +# self, collection: Collection, refresh: bool = False +# ) -> None: +# pass + +# @abstractmethod +# async def find_collection(self, collection_id: str) -> Collection: +# pass + +# @abstractmethod +# async def delete_collection( +# self, collection_id: str, refresh: bool = False +# ) -> None: +# pass diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_api/common/core.py similarity index 89% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py rename to stac_api/common/core.py index 12cc6b2c..918b5ce0 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_api/common/core.py @@ -1,4 +1,5 @@ """Item crud client.""" +import importlib import logging import re from base64 import urlsafe_b64encode @@ -18,12 +19,11 @@ from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes -from stac_fastapi.elasticsearch import serializers -from stac_fastapi.elasticsearch.config import ElasticsearchSettings -from stac_fastapi.elasticsearch.database_logic import DatabaseLogic -from stac_fastapi.elasticsearch.models.links import PagingLinks -from stac_fastapi.elasticsearch.serializers import CollectionSerializer, ItemSerializer -from stac_fastapi.elasticsearch.session import Session +# from common.base_database_logic import BaseDatabaseLogic +from stac_api.common.models.links import PagingLinks +from stac_api.elastic_search import serializers +from stac_api.elastic_search.serializers import CollectionSerializer, ItemSerializer +from stac_api.elastic_search.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, BulkTransactionMethod, @@ -66,12 +66,36 @@ class CoreClient(AsyncBaseCoreClient): session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer + default=serializers.ItemSerializer # type: ignore ) collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer + default=serializers.CollectionSerializer # type: ignore ) - database = DatabaseLogic() + post_request_model = attr.ib(default=BaseSearchPostRequest) + + def __attrs_post_init__(self): + """ + Post-initialization method for CoreClient. + + This method is automatically called after CoreClient's instance is initialized. + It's responsible for setting up the database logic dynamically based on the + environment or configuration, ensuring that the appropriate database logic + (Elasticsearch or OpenSearch) is used. + """ + try: + # Dynamically import the database logic based on installed package + database_module = importlib.import_module( + "stac_api.elastic_search.database_logic" + ) + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + except ImportError: + # Fall back to OpenSearch if Elasticsearch is not available + database_module = importlib.import_module( + "stac_api.open_search.database_logic" + ) + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + + self.database = DatabaseLogicClass() @overrides async def all_collections(self, **kwargs) -> Collections: @@ -542,7 +566,31 @@ class TransactionsClient(AsyncBaseTransactionsClient): """Transactions extension specific CRUD operations.""" session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database = DatabaseLogic() + # database: BaseDatabaseLogic = attr.ib(init=False) + + def __attrs_post_init__(self): + """ + Post-initialization method for TransactionsClient. + + This method is automatically called after the instance is initialized. + It's responsible for setting up the database logic dynamically based on the + environment or configuration, ensuring that the appropriate database logic + (Elasticsearch or OpenSearch) is used. + """ + try: + # Dynamically import the database logic based on installed package + database_module = importlib.import_module( + "stac_api.elastic_search.database_logic" + ) + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + except ImportError: + # Fall back to OpenSearch if Elasticsearch is not available + database_module = importlib.import_module( + "stac_api.opensearch.database_logic" + ) + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + + self.database = DatabaseLogicClass() @overrides async def create_item( @@ -712,12 +760,31 @@ class BulkTransactionsClient(BaseBulkTransactionsClient): """ session: Session = attr.ib(default=attr.Factory(Session.create_from_env)) - database = DatabaseLogic() + # database: BaseDatabaseLogic = attr.ib(init=False) def __attrs_post_init__(self): - """Create es engine.""" - settings = ElasticsearchSettings() - self.client = settings.create_client + """ + Post-initialization method for BulkTransactionsClient. + + This method is automatically called after the instance is initialized. + It's responsible for setting up the database logic dynamically based on the + environment or configuration, ensuring that the appropriate database logic + (Elasticsearch or OpenSearch) is used. + """ + try: + # Dynamically import the database logic based on installed package + database_module = importlib.import_module( + "stac_api.elastic_search.database_logic" + ) + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + except ImportError: + # Fall back to OpenSearch if Elasticsearch is not available + database_module = importlib.import_module( + "stac_api.opensearch.database_logic" + ) + DatabaseLogicClass = getattr(database_module, "DatabaseLogic") + + self.database = DatabaseLogicClass() def preprocess_item( self, item: stac_types.Item, base_url, method: BulkTransactionMethod diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/datetime_utils.py b/stac_api/common/datetime_utils.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/datetime_utils.py rename to stac_api/common/datetime_utils.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/__init__.py b/stac_api/common/extensions/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/__init__.py rename to stac_api/common/extensions/__init__.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py b/stac_api/common/extensions/filter.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/filter.py rename to stac_api/common/extensions/filter.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/query.py b/stac_api/common/extensions/query.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/extensions/query.py rename to stac_api/common/extensions/query.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/__init__.py b/stac_api/common/models/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/__init__.py rename to stac_api/common/models/__init__.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/links.py b/stac_api/common/models/links.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/links.py rename to stac_api/common/models/links.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/search.py b/stac_api/common/models/search.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/models/search.py rename to stac_api/common/models/search.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/types/search.py b/stac_api/common/types/search.py similarity index 97% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/types/search.py rename to stac_api/common/types/search.py index 26a2dbb6..b2536021 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/types/search.py +++ b/stac_api/common/types/search.py @@ -8,7 +8,7 @@ from stac_pydantic.api.extensions.fields import FieldsExtension as FieldsBase -from stac_fastapi.types.config import Settings +from stac_api.types.config import Settings logger = logging.getLogger("uvicorn") logger.setLevel(logging.INFO) diff --git a/stac_fastapi/elasticsearch/README.md b/stac_api/elastic_search/README.md similarity index 100% rename from stac_fastapi/elasticsearch/README.md rename to stac_api/elastic_search/README.md diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/__init__.py b/stac_api/elastic_search/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/__init__.py rename to stac_api/elastic_search/__init__.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_api/elastic_search/app.py similarity index 85% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py rename to stac_api/elastic_search/app.py index 8adcece4..b0fbc709 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_api/elastic_search/app.py @@ -1,16 +1,18 @@ """FastAPI application.""" -from stac_fastapi.api.app import StacApi -from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_fastapi.elasticsearch.config import ElasticsearchSettings -from stac_fastapi.elasticsearch.core import ( +import sys + +from stac_api.common.core import ( BulkTransactionsClient, CoreClient, EsAsyncBaseFiltersClient, TransactionsClient, ) -from stac_fastapi.elasticsearch.database_logic import create_collection_index -from stac_fastapi.elasticsearch.extensions import QueryExtension -from stac_fastapi.elasticsearch.session import Session +from stac_api.common.extensions import QueryExtension +from stac_api.elastic_search.config import ElasticsearchSettings +from stac_api.elastic_search.database_logic import create_collection_index +from stac_api.elastic_search.session import Session +from stac_fastapi.api.app import StacApi +from stac_fastapi.api.models import create_get_request_model, create_post_request_model from stac_fastapi.extensions.core import ( ContextExtension, FieldsExtension, @@ -21,6 +23,8 @@ ) from stac_fastapi.extensions.third_party import BulkTransactionExtension +print("API sys.path:", sys.path) + settings = ElasticsearchSettings() session = Session.create_from_settings(settings) @@ -63,7 +67,7 @@ def run() -> None: import uvicorn uvicorn.run( - "stac_fastapi.elasticsearch.app:app", + "stac_api.elastic_search.app:app", host=settings.app_host, port=settings.app_port, log_level="info", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_api/elastic_search/config.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py rename to stac_api/elastic_search/config.py diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_api/elastic_search/database_logic.py similarity index 96% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py rename to stac_api/elastic_search/database_logic.py index 336c8d07..3693158c 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_api/elastic_search/database_logic.py @@ -9,12 +9,13 @@ from elasticsearch_dsl import Q, Search from elasticsearch import exceptions, helpers # type: ignore -from stac_fastapi.elasticsearch import serializers -from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings -from stac_fastapi.elasticsearch.config import ( +from stac_api.common.extensions import filter +from stac_api.elastic_search import serializers +from stac_api.elastic_search.config import AsyncElasticsearchSettings +from stac_api.elastic_search.config import ( ElasticsearchSettings as SyncElasticsearchSettings, ) -from stac_fastapi.elasticsearch.extensions import filter +from stac_api.elastic_search.utilities import bbox2polygon from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item @@ -229,21 +230,6 @@ async def delete_item_index(collection_id: str): await client.close() -def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: - """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. - - Args: - b0 (float): The x-coordinate of the lower-left corner of the bounding box. - b1 (float): The y-coordinate of the lower-left corner of the bounding box. - b2 (float): The x-coordinate of the upper-right corner of the bounding box. - b3 (float): The y-coordinate of the upper-right corner of the bounding box. - - Returns: - List[List[List[float]]]: A polygon represented as a list of lists of coordinates. - """ - return [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]] - - def mk_item_id(item_id: str, collection_id: str): """Create the document id for an Item in Elasticsearch. @@ -297,10 +283,10 @@ class DatabaseLogic: sync_client = SyncElasticsearchSettings().create_client item_serializer: Type[serializers.ItemSerializer] = attr.ib( - default=serializers.ItemSerializer + default=serializers.ItemSerializer # type: ignore ) collection_serializer: Type[serializers.CollectionSerializer] = attr.ib( - default=serializers.CollectionSerializer + default=serializers.CollectionSerializer # type: ignore ) """CORE LOGIC""" diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py b/stac_api/elastic_search/serializers.py similarity index 98% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py rename to stac_api/elastic_search/serializers.py index 725e8f65..58b9f458 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/serializers.py +++ b/stac_api/elastic_search/serializers.py @@ -4,7 +4,7 @@ import attr -from stac_fastapi.elasticsearch.datetime_utils import now_to_rfc3339_str +from stac_api.common.datetime_utils import now_to_rfc3339_str from stac_fastapi.types import stac as stac_types from stac_fastapi.types.links import CollectionLinks, ItemLinks, resolve_links diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/session.py b/stac_api/elastic_search/session.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/session.py rename to stac_api/elastic_search/session.py diff --git a/stac_api/elastic_search/setup.cfg b/stac_api/elastic_search/setup.cfg new file mode 100644 index 00000000..f1023827 --- /dev/null +++ b/stac_api/elastic_search/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +version = attr: version.__version__ diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_api/elastic_search/setup.py similarity index 88% rename from stac_fastapi/elasticsearch/setup.py rename to stac_api/elastic_search/setup.py index 3106c512..d8420886 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_api/elastic_search/setup.py @@ -8,13 +8,13 @@ install_requires = [ "fastapi", "attrs", + "elasticsearch[async]==8.11.0", + "elasticsearch-dsl==8.11.0", "pydantic[dotenv]<2", "stac_pydantic==2.0.*", "stac-fastapi.types==2.4.9", "stac-fastapi.api==2.4.9", "stac-fastapi.extensions==2.4.9", - "elasticsearch[async]==8.11.0", - "elasticsearch-dsl==8.11.0", "pystac[validation]", "uvicorn", "orjson", @@ -56,14 +56,17 @@ ], url="https://github.com/stac-utils/stac-fastapi-elasticsearch", license="MIT", - packages=find_namespace_packages(exclude=["alembic", "tests", "scripts"]), + packages=find_namespace_packages( + include=["elastic_search", "elastic_search.*", "common", "common.*"], + exclude=["tests", "scripts"], + ), zip_safe=False, install_requires=install_requires, tests_require=extra_reqs["dev"], extras_require=extra_reqs, entry_points={ "console_scripts": [ - "stac-fastapi-elasticsearch=stac_fastapi.elasticsearch.app:run" + "stac-fastapi-elasticsearch=stac_api.elastic_search.app:run" ] }, ) diff --git a/stac_api/elastic_search/utilities.py b/stac_api/elastic_search/utilities.py new file mode 100644 index 00000000..4646cfe0 --- /dev/null +++ b/stac_api/elastic_search/utilities.py @@ -0,0 +1,22 @@ +"""Module for geospatial processing functions. + +This module contains functions for transforming geospatial coordinates, +such as converting bounding boxes to polygon representations. +""" + +from typing import List + + +def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]: + """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon. + + Args: + b0 (float): The x-coordinate of the lower-left corner of the bounding box. + b1 (float): The y-coordinate of the lower-left corner of the bounding box. + b2 (float): The x-coordinate of the upper-right corner of the bounding box. + b3 (float): The y-coordinate of the upper-right corner of the bounding box. + + Returns: + List[List[List[float]]]: A polygon represented as a list of lists of coordinates. + """ + return [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_api/elastic_search/version.py similarity index 100% rename from stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py rename to stac_api/elastic_search/version.py diff --git a/stac_api/open_search/setup.cfg b/stac_api/open_search/setup.cfg new file mode 100644 index 00000000..f1023827 --- /dev/null +++ b/stac_api/open_search/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +version = attr: version.__version__ diff --git a/stac_api/open_search/setup.py b/stac_api/open_search/setup.py new file mode 100644 index 00000000..2c0607fc --- /dev/null +++ b/stac_api/open_search/setup.py @@ -0,0 +1,68 @@ +"""stac_fastapi: elasticsearch module.""" + +from setuptools import find_namespace_packages, setup + +with open("README.md") as f: + desc = f.read() + +install_requires = [ + "fastapi", + "attrs", + "elasticsearch[async]==8.11.0", + "elasticsearch-dsl==8.11.0", + "pydantic[dotenv]<2", + "stac_pydantic==2.0.*", + "stac-fastapi.types==2.4.9", + "stac-fastapi.api==2.4.9", + "stac-fastapi.extensions==2.4.9", + "pystac[validation]", + "uvicorn", + "orjson", + "overrides", + "starlette", + "geojson-pydantic", + "pygeofilter==0.2.1", +] + +extra_reqs = { + "dev": [ + "pytest", + "pytest-cov", + "pytest-asyncio", + "pre-commit", + "requests", + "ciso8601", + "httpx", + ], + "docs": ["mkdocs", "mkdocs-material", "pdocs"], + "server": ["uvicorn[standard]==0.19.0"], +} + +setup( + name="stac-fastapi.elasticsearch", + description="An implementation of STAC API based on the FastAPI framework with Elasticsearch.", + long_description=desc, + long_description_content_type="text/markdown", + python_requires=">=3.8", + classifiers=[ + "Intended Audience :: Developers", + "Intended Audience :: Information Technology", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "License :: OSI Approved :: MIT License", + ], + url="https://github.com/stac-utils/stac-fastapi-elasticsearch", + license="MIT", + packages=find_namespace_packages( + include=["elasticsearch", "elasticsearch.*", "common", "common.*"], + exclude=["tests", "scripts"], + ), + zip_safe=False, + install_requires=install_requires, + tests_require=extra_reqs["dev"], + extras_require=extra_reqs, + entry_points={"console_scripts": ["stac-fastapi-elasticsearch=app:run"]}, +) diff --git a/stac_api/open_search/version.py b/stac_api/open_search/version.py new file mode 100644 index 00000000..1eeef171 --- /dev/null +++ b/stac_api/open_search/version.py @@ -0,0 +1,2 @@ +"""library version.""" +__version__ = "1.0.0" diff --git a/stac_fastapi/elasticsearch/pytest.ini b/stac_api/pytest.ini similarity index 56% rename from stac_fastapi/elasticsearch/pytest.ini rename to stac_api/pytest.ini index db0353ef..f770b9c9 100644 --- a/stac_fastapi/elasticsearch/pytest.ini +++ b/stac_api/pytest.ini @@ -1,4 +1,4 @@ [pytest] -testpaths = tests +testpaths = elastic_search/tests addopts = -sv asyncio_mode = auto \ No newline at end of file diff --git a/stac_fastapi/elasticsearch/tests/__init__.py b/stac_api/tests/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/__init__.py rename to stac_api/tests/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/api/__init__.py b/stac_api/tests/api/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/api/__init__.py rename to stac_api/tests/api/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_api/tests/api/test_api.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/api/test_api.py rename to stac_api/tests/api/test_api.py diff --git a/stac_fastapi/elasticsearch/tests/clients/__init__.py b/stac_api/tests/clients/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/clients/__init__.py rename to stac_api/tests/clients/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py b/stac_api/tests/clients/test_elasticsearch.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py rename to stac_api/tests/clients/test_elasticsearch.py diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_api/tests/conftest.py similarity index 90% rename from stac_fastapi/elasticsearch/tests/conftest.py rename to stac_api/tests/conftest.py index fa093af2..157cabce 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_api/tests/conftest.py @@ -1,3 +1,7 @@ +# import sys + +# sys.path.insert(0, "/app") + import asyncio import copy import json @@ -8,16 +12,12 @@ import pytest_asyncio from httpx import AsyncClient +from stac_api.common.core import BulkTransactionsClient, CoreClient, TransactionsClient +from stac_api.common.extensions import QueryExtension +from stac_api.elastic_search.config import AsyncElasticsearchSettings +from stac_api.elastic_search.database_logic import create_collection_index from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings -from stac_fastapi.elasticsearch.core import ( - BulkTransactionsClient, - CoreClient, - TransactionsClient, -) -from stac_fastapi.elasticsearch.database_logic import create_collection_index -from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, FieldsExtension, @@ -28,6 +28,9 @@ ) from stac_fastapi.types.config import Settings +# # Assuming your tests are in the 'tests' directory at the same level as 'stac_fastapi' +# sys.path.append(str(Path(__file__).parent.parent)) + DATA_DIR = os.path.join(os.path.dirname(__file__), "data") diff --git a/stac_fastapi/elasticsearch/tests/data/test_collection.json b/stac_api/tests/data/test_collection.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/data/test_collection.json rename to stac_api/tests/data/test_collection.json diff --git a/stac_fastapi/elasticsearch/tests/data/test_item.json b/stac_api/tests/data/test_item.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/data/test_item.json rename to stac_api/tests/data/test_item.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example01.json b/stac_api/tests/extensions/cql2/example01.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example01.json rename to stac_api/tests/extensions/cql2/example01.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example04.json b/stac_api/tests/extensions/cql2/example04.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example04.json rename to stac_api/tests/extensions/cql2/example04.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example05a.json b/stac_api/tests/extensions/cql2/example05a.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example05a.json rename to stac_api/tests/extensions/cql2/example05a.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example06b.json b/stac_api/tests/extensions/cql2/example06b.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example06b.json rename to stac_api/tests/extensions/cql2/example06b.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example08.json b/stac_api/tests/extensions/cql2/example08.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example08.json rename to stac_api/tests/extensions/cql2/example08.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example09.json b/stac_api/tests/extensions/cql2/example09.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example09.json rename to stac_api/tests/extensions/cql2/example09.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example1.json b/stac_api/tests/extensions/cql2/example1.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example1.json rename to stac_api/tests/extensions/cql2/example1.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example10.json b/stac_api/tests/extensions/cql2/example10.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example10.json rename to stac_api/tests/extensions/cql2/example10.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example14.json b/stac_api/tests/extensions/cql2/example14.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example14.json rename to stac_api/tests/extensions/cql2/example14.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example15.json b/stac_api/tests/extensions/cql2/example15.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example15.json rename to stac_api/tests/extensions/cql2/example15.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example17.json b/stac_api/tests/extensions/cql2/example17.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example17.json rename to stac_api/tests/extensions/cql2/example17.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example18.json b/stac_api/tests/extensions/cql2/example18.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example18.json rename to stac_api/tests/extensions/cql2/example18.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json b/stac_api/tests/extensions/cql2/example19.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example19.json rename to stac_api/tests/extensions/cql2/example19.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json b/stac_api/tests/extensions/cql2/example20.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example20.json rename to stac_api/tests/extensions/cql2/example20.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json b/stac_api/tests/extensions/cql2/example21.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example21.json rename to stac_api/tests/extensions/cql2/example21.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json b/stac_api/tests/extensions/cql2/example22.json similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/cql2/example22.json rename to stac_api/tests/extensions/cql2/example22.json diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_api/tests/extensions/test_filter.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/extensions/test_filter.py rename to stac_api/tests/extensions/test_filter.py diff --git a/stac_fastapi/elasticsearch/tests/resources/__init__.py b/stac_api/tests/resources/__init__.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/__init__.py rename to stac_api/tests/resources/__init__.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_collection.py b/stac_api/tests/resources/test_collection.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/test_collection.py rename to stac_api/tests/resources/test_collection.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_conformance.py b/stac_api/tests/resources/test_conformance.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/test_conformance.py rename to stac_api/tests/resources/test_conformance.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_item.py b/stac_api/tests/resources/test_item.py similarity index 99% rename from stac_fastapi/elasticsearch/tests/resources/test_item.py rename to stac_api/tests/resources/test_item.py index 5b382873..88f23402 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_item.py +++ b/stac_api/tests/resources/test_item.py @@ -9,11 +9,11 @@ import ciso8601 import pystac import pytest +from common.core import CoreClient +from common.datetime_utils import now_to_rfc3339_str from geojson_pydantic.geometries import Polygon from pystac.utils import datetime_to_str -from stac_fastapi.elasticsearch.core import CoreClient -from stac_fastapi.elasticsearch.datetime_utils import now_to_rfc3339_str from stac_fastapi.types.core import LandingPageMixin from ..conftest import create_item, refresh_indices diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_api/tests/resources/test_mgmt.py similarity index 100% rename from stac_fastapi/elasticsearch/tests/resources/test_mgmt.py rename to stac_api/tests/resources/test_mgmt.py diff --git a/stac_fastapi/elasticsearch/setup.cfg b/stac_fastapi/elasticsearch/setup.cfg deleted file mode 100644 index 7a42432c..00000000 --- a/stac_fastapi/elasticsearch/setup.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[metadata] -version = attr: stac_fastapi.elasticsearch.version.__version__