Merge AtomicMaps branch into common_core, reorg code

.github/workflows/cicd.yml

@@ -46,6 +46,22 @@ jobs:
           ES_JAVA_OPTS: -Xms512m -Xmx1g
         ports:
           - 9400:9400
+
+      opensearch_2_11:
+        image: opensearchproject/opensearch:2.11.1
+        env:
+          cluster.name: stac-cluster
+          node.name: os01
+          network.host: 0.0.0.0
+          transport.host: 0.0.0.0
+          discovery.type: single-node
+          http.port: 9202
+          http.cors.enabled: true
+          plugins.security.disabled: true
+          plugins.security.ssl.http.enabled: true
+          OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+        ports:
+          - 9202:9202
     strategy:
       matrix:
         python-version: [ "3.8", "3.9", "3.10", "3.11"]
@@ -94,4 +110,15 @@ jobs:
           ES_PORT: 9400
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
-          ES_VERIFY_CERTS: false
+          ES_VERIFY_CERTS: false
+
+      - name: Run test suite against OpenSearch 2.11.1
+        run: |
+          cd stac_fastapi/elasticsearch && pipenv run pytest -svvv
+        env:
+          ENVIRONMENT: testing
+          ES_PORT: 9202
+          ES_HOST: 172.17.0.1
+          ES_USE_SSL: false
+          ES_VERIFY_CERTS: false
+          BACKEND: opensearch

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Added
 
+- OpenSearch 2.11.1 support [#187](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/187)
 - Advanced comparison (LIKE, IN, BETWEEN) operators to the Filter extension [#178](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/178)
 
 ### Changed

Makefile

@@ -1,20 +1,32 @@
 #!make
 APP_HOST ?= 0.0.0.0
-APP_PORT ?= 8080
+ES_APP_PORT ?= 8080
 EXTERNAL_APP_PORT ?= ${APP_PORT}
 
-APP_PORT ?= 8080
+ES_APP_PORT ?= 8080
 ES_HOST ?= docker.for.mac.localhost
 ES_PORT ?= 9200
 
+OS_APP_PORT ?= 8082
+ES_HOST ?= docker.for.mac.localhost
+OS_PORT ?= 9202
+
 run_es = docker-compose \
 	run \
-	-p ${EXTERNAL_APP_PORT}:${APP_PORT} \
+	-p ${EXTERNAL_APP_PORT}:${ES_APP_PORT} \
 	-e PY_IGNORE_IMPORTMISMATCH=1 \
 	-e APP_HOST=${APP_HOST} \
-	-e APP_PORT=${APP_PORT} \
+	-e APP_PORT=${ES_APP_PORT} \
 	app-elasticsearch
 
+run_os = docker-compose \
+	run \
+	-p ${EXTERNAL_APP_PORT}:${OS_APP_PORT} \
+	-e PY_IGNORE_IMPORTMISMATCH=1 \
+	-e APP_HOST=${APP_HOST} \
+	-e APP_PORT=${OS_APP_PORT} \
+	app-opensearch
+
 .PHONY: image-deploy
 image-deploy:
 	docker build -f Dockerfile.deploy -t stac-fastapi-elasticsearch:latest .
@@ -40,15 +52,32 @@ docker-run: image-dev
 docker-shell:
 	$(run_es) /bin/bash
 
+.PHONY: test-elasticsearch
+test:
+	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
+.PHONY: test-opensearch
+test-opensearch:
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
 .PHONY: test
 test:
 	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
 	docker-compose down
 
-.PHONY: run-database
-run-database:
+	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd /app/stac_fastapi/elasticsearch/tests/ && pytest'
+	docker-compose down
+
+.PHONY: run-database-es
+run-database-es:
 	docker-compose run --rm elasticsearch
 
+.PHONY: run-database-os
+run-database-os:
+	docker-compose run --rm opensearch
+
 .PHONY: pybase-install
 pybase-install:
 	pip install wheel && \

README.md

@@ -42,13 +42,15 @@ docker-compose build
 docker-compose up
 ```
 
-By default, docker-compose uses Elasticsearch 8.x. However, most recent 7.x versions should also work. 
+By default, docker-compose uses Elasticsearch 8.x and OpenSearch 2.11.1. 
 If you wish to use a different version, put the following in a 
 file named `.env` in the same directory you run docker-compose from:
 
 ```shell
 ELASTICSEARCH_VERSION=7.17.1
+OPENSEARCH_VERSION=2.11.0
 ```
+The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information.
 
 To create a new Collection:
 
@@ -78,7 +80,18 @@ curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token"
 ```shell
 make test
 ```
-
+Test against OpenSearch only
+
+```shell
+make test-opensearch
+```
+
+Test against Elasticsearch only
+
+```shell
+make test-elasticsearch
+```  
+
 ## Ingest sample data
 
 ```shell

data_loader/data_loader.py

@@ -1,12 +1,24 @@
 """Database ingestion script."""
 import json
 import os
+import sys
 
 import click
 import requests
 
+if len(sys.argv) != 2:
+    print("Usage: python data_loader.py <opensearch|elasticsearch>")
+    sys.exit(1)
+
 DATA_DIR = os.path.join(os.path.dirname(__file__), "setup_data/")
-STAC_API_BASE_URL = "http://localhost:8080"
+
+backend = sys.argv[1].lower()
+if backend == "opensearch":
+    STAC_API_BASE_URL = "http://localhost:8082"
+elif backend == "elasticsearch":
+    STAC_API_BASE_URL = "http://localhost:8080"
+else:
+    print("Invalid backend tag. Enter either 'opensearch' or 'elasticsearch'.")
 
 
 def load_data(filename):

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   app-elasticsearch:
     container_name: stac-fastapi-es
-    image: stac-utils/stac-fastapi
+    image: stac-utils/stac-fastapi-es
     restart: always
     build:
       context: .
@@ -18,6 +18,7 @@ services:
       - ES_PORT=9200
       - ES_USE_SSL=false
       - ES_VERIFY_CERTS=false
+      - BACKEND=elasticsearch
     ports:
       - "8080:8080"
     volumes:
@@ -29,6 +30,35 @@ services:
     command:
       bash -c "./scripts/wait-for-it-es.sh es-container:9200 && python -m stac_fastapi.elasticsearch.app"
 
+  app-opensearch:
+    container_name: stac-fastapi-os
+    image: stac-utils/stac-fastapi-os
+    restart: always
+    build:
+      context: .
+      dockerfile: Dockerfile.dev
+    environment:
+      - APP_HOST=0.0.0.0
+      - APP_PORT=8082
+      - RELOAD=true
+      - ENVIRONMENT=local
+      - WEB_CONCURRENCY=10
+      - ES_HOST=172.17.0.1
+      - ES_PORT=9202
+      - ES_USE_SSL=false
+      - ES_VERIFY_CERTS=false
+      - BACKEND=opensearch
+    ports:
+      - "8082:8082"
+    volumes:
+      - ./stac_fastapi:/app/stac_fastapi
+      - ./scripts:/app/scripts
+      - ./osdata:/usr/share/opensearch/data
+    depends_on:
+      - opensearch
+    command:
+      bash -c "./scripts/wait-for-it-es.sh os-container:9202 && python -m stac_fastapi.elasticsearch.app"
+
   elasticsearch:
     container_name: es-container
     image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0}
@@ -40,3 +70,16 @@ services:
       - ./elasticsearch/snapshots:/usr/share/elasticsearch/snapshots
     ports:
       - "9200:9200"
+
+  opensearch:
+    container_name: os-container
+    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-2.11.1}
+    environment:
+      - discovery.type=single-node
+      - plugins.security.disabled=true
+      - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m
+    volumes:
+      - ./opensearch/config/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
+      - ./opensearch/snapshots:/usr/share/opensearch/snapshots
+    ports:
+      - "9202:9202"

opensearch/config/opensearch.yml

@@ -0,0 +1,19 @@
+## Cluster Settings
+cluster.name: stac-cluster
+node.name: os01
+network.host: 0.0.0.0
+transport.host: 0.0.0.0
+discovery.type: single-node
+http.port: 9202
+http.cors.enabled: true
+http.cors.allow-headers: X-Requested-With,Content-Type,Content-Length,Accept,Authorization
+
+path:
+  repo:
+    - /usr/share/opensearch/snapshots
+
+# Security
+plugins.security.disabled: true
+plugins.security.ssl.http.enabled: true
+
+node.max_local_storage_nodes: 3

stac_fastapi/core/stac_fastapi/core/core.py

@@ -218,6 +218,7 @@ async def all_collections(self, **kwargs) -> Collections:
         next_link = None
         if len(hits) == limit:
             last_hit = hits[-1]
+            logger.info(last_hit)
             next_search_after = last_hit["sort"]
             next_token = urlsafe_b64encode(
                 ",".join(map(str, next_search_after)).encode()
@@ -848,6 +849,8 @@ def __attrs_post_init__(self):
         """Create es engine."""
         # settings = BaseSettings()
         self.client = self.settings.create_client
+        # settings = SearchSettings()
+        # self.client = settings.create_client
 
     def preprocess_item(
         self, item: stac_types.Item, base_url, method: BulkTransactionMethod

stac_fastapi/core/stac_fastapi/core/utilities.py

@@ -0,0 +1,21 @@
+"""Module for geospatial processing functions.
+
+This module contains functions for transforming geospatial coordinates,
+such as converting bounding boxes to polygon representations.
+"""
+from typing import List
+
+
+def bbox2polygon(b0: float, b1: float, b2: float, b3: float) -> List[List[List[float]]]:
+    """Transform a bounding box represented by its four coordinates `b0`, `b1`, `b2`, and `b3` into a polygon.
+
+    Args:
+        b0 (float): The x-coordinate of the lower-left corner of the bounding box.
+        b1 (float): The y-coordinate of the lower-left corner of the bounding box.
+        b2 (float): The x-coordinate of the upper-right corner of the bounding box.
+        b3 (float): The y-coordinate of the upper-right corner of the bounding box.
+
+    Returns:
+        List[List[List[float]]]: A polygon represented as a list of lists of coordinates.
+    """
+    return [[[b0, b1], [b2, b1], [b2, b3], [b0, b3], [b0, b1]]]

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py

@@ -1,4 +1,5 @@
 """FastAPI application."""
+
 from stac_fastapi.api.app import StacApi
 from stac_fastapi.api.models import create_get_request_model, create_post_request_model
 from stac_fastapi.core.core import (