From 11e5685e0b392e199be2fc2b85acd499f378bb04 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Wed, 9 Oct 2024 13:33:14 +0800 Subject: [PATCH 1/8] Publish to PyPI in gh actions (#305) - add publish.yml to automatically push new releases to PyPI - update sfeos to v3.2.0 **Description:** **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- .github/workflows/publish.yml | 60 +++++++++++++++++++ CHANGELOG.md | 1 + .../core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- .../stac_fastapi/opensearch/version.py | 2 +- 7 files changed, 66 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..e28da570 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,60 @@ +name: Publish + +on: + push: + tags: + - "v*.*.*" # Triggers when a tag like 'v3.2.0' is pushed + +jobs: + build-and-publish: + name: Build and Publish Packages + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install build dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Build and publish stac-fastapi-core + working-directory: stac_fastapi/core + env: + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + # Build package + python setup.py sdist bdist_wheel + + # Publish to PyPI + twine upload dist/* + + - name: Build and publish stac-fastapi-elasticsearch + working-directory: stac_fastapi/elasticsearch + env: + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + # Build package + python setup.py sdist bdist_wheel + + # Publish to PyPI + twine upload dist/* + + - name: Build and publish stac-fastapi-opensearch + working-directory: stac_fastapi/opensearch + env: + TWINE_USERNAME: "__token__" + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + run: | + # Build package + python setup.py sdist bdist_wheel + + # Publish to PyPI + twine upload dist/* diff --git a/CHANGELOG.md b/CHANGELOG.md index b96a2a5b..87a26625 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Added `datetime_frequency_interval` parameter for `datetime_frequency` aggregation. [#294](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/294) - Added rate limiting functionality with configurable limits using environment variable `STAC_FASTAPI_RATE_LIMIT`, example: `500/minute`. [#303](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/303) +- Added publish.yml to automatically publish new releases to PyPI [#305](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/305) ### Changed diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index 30b3d6c9..fad2b4e4 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.1.0" +__version__ = "3.2.0" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 0d2a9895..a23cbe7c 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.1.0", + "stac-fastapi.core==3.2.0", "elasticsearch[async]==8.11.0", "elasticsearch-dsl==8.11.0", "uvicorn", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 30b3d6c9..fad2b4e4 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.1.0" +__version__ = "3.2.0" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index ee47cdeb..2cd43da3 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.1.0", + "stac-fastapi.core==3.2.0", "opensearch-py==2.4.2", "opensearch-py[async]==2.4.2", "uvicorn", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index 30b3d6c9..fad2b4e4 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.1.0" +__version__ = "3.2.0" From 059da7e977368e1d3f6064a3ad00aaacc99470f6 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Wed, 9 Oct 2024 13:41:50 +0800 Subject: [PATCH 2/8] Update CHANGELOG.md --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87a26625..c3994220 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v3.2.0] - 2024-10-09 + ### Added - Added `datetime_frequency_interval` parameter for `datetime_frequency` aggregation. [#294](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/294) @@ -263,7 +265,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.1.0...main +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.0...main +[v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.1.0...v3.2.0 [v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0...v3.1.0 [v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.1...v3.0.0 [v2.4.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.0...v2.4.1 From 5ab4c4bde6c7d86e4f4abd3e19036ec7954b72f3 Mon Sep 17 00:00:00 2001 From: James Fisher <85769594+jamesfisher-geo@users.noreply.github.com> Date: Fri, 29 Nov 2024 22:13:30 -0500 Subject: [PATCH 3/8] Aggregation Bugfix and Documentation Update (#314) **Related Issue(s):** - #290 **Description:** Includes a bugfix for Elasticsearch aggregation, the indices() function was only checking if the input was None. But in POST requests the input is an empty list ({}). So that was leading to some aggregations to search through all indices in an Elasticsearch cluster, not just the items indices. **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 2 + README.md | 104 +---------------- docs/mkdocs.yml | 1 + docs/src/aggregation.md | 105 ++++++++++++++++++ .../elasticsearch/database_logic.py | 2 +- 5 files changed, 110 insertions(+), 104 deletions(-) create mode 100644 docs/src/aggregation.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c3994220..95c427a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +- Aggregation ElasticSearch `total_count` bugfix, moved aggregation text to docs. [#314](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/314) + ## [v3.2.0] - 2024-10-09 ### Added diff --git a/README.md b/README.md index 6ec94b72..0ff4f254 100644 --- a/README.md +++ b/README.md @@ -279,111 +279,9 @@ The modified Items with lowercase identifiers will now be visible to users acces Authentication is an optional feature that can be enabled through `Route Dependencies` examples can be found and a more detailed explanation in [examples/auth](examples/auth). - ## Aggregation -Sfeos supports the STAC API [Aggregation Extension](https://github.com/stac-api-extensions/aggregation). This enables geospatial aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`//aggregations`). The `/aggregate` route also fully supports base search and the STAC API [Filter Extension](https://github.com/stac-api-extensions/filter). Any query made with `/search` may also be executed with `/aggregate`, provided that the relevant aggregation fields are available, - - -A field named `aggregations` should be added to the Collection object for the collection for which the aggregations are available, for example: - -```json -"aggregations": [ - { - "name": "total_count", - "data_type": "integer" - }, - { - "name": "datetime_max", - "data_type": "datetime" - }, - { - "name": "datetime_min", - "data_type": "datetime" - }, - { - "name": "datetime_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "datetime" - }, - { - "name": "sun_elevation_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "numeric" - }, - { - "name": "platform_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "string" - }, - { - "name": "sun_azimuth_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "numeric" - }, - { - "name": "off_nadir_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "numeric" - }, - { - "name": "cloud_cover_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "numeric" - }, - { - "name": "grid_code_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "string" - }, - { - "name": "centroid_geohash_grid_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "string" - }, - { - "name": "centroid_geohex_grid_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "string" - }, - { - "name": "centroid_geotile_grid_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "string" - }, - { - "name": "geometry_geohash_grid_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "numeric" - }, - { - "name": "geometry_geotile_grid_frequency", - "data_type": "frequency_distribution", - "frequency_distribution_data_type": "string" - } -] - ``` - -Available aggregations are: - -- total_count (count of total items) -- collection_frequency (Item `collection` field) -- platform_frequency (Item.Properties.platform) -- cloud_cover_frequency (Item.Properties.eo:cloud_cover) -- datetime_frequency (Item.Properties.datetime, monthly interval) -- datetime_min (earliest Item.Properties.datetime) -- datetime_max (latest Item.Properties.datetime) -- sun_elevation_frequency (Item.Properties.view:sun_elevation) -- sun_azimuth_frequency (Item.Properties.view:sun_azimuth) -- off_nadir_frequency (Item.Properties.view:off_nadir) -- grid_code_frequency (Item.Properties.grid:code) -- centroid_geohash_grid_frequency ([geohash grid](https://opensearch.org/docs/latest/aggregations/bucket/geohash-grid/) on Item.Properties.proj:centroid) -- centroid_geohex_grid_frequency ([geohex grid](https://opensearch.org/docs/latest/aggregations/bucket/geohex-grid/) on Item.Properties.proj:centroid) -- centroid_geotile_grid_frequency (geotile on Item.Properties.proj:centroid) -- geometry_geohash_grid_frequency ([geohash grid](https://opensearch.org/docs/latest/aggregations/bucket/geohash-grid/) on Item.geometry) -- geometry_geotile_grid_frequency ([geotile grid](https://opensearch.org/docs/latest/aggregations/bucket/geotile-grid/) on Item.geometry) - -Support for additional fields and new aggregations can be added in the associated `database_logic.py` file. +Aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates is supported in stac-fatsapi-elasticsearch-opensearch. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`//aggregations`). Details for supported aggregations can be found at [./docs/src/aggregation.md](./docs/src/aggregation.md) ## Rate Limiting diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 5968ffa6..67764805 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -52,6 +52,7 @@ nav: - session: api/stac_fastapi/core/session.md - utilities: api/stac_fastapi/core/utilities.md - version: api/stac_fastapi/core/version.md + - Aggregation: "aggregation.md" - Development - Contributing: "contributing.md" - Release Notes: "release-notes.md" diff --git a/docs/src/aggregation.md b/docs/src/aggregation.md new file mode 100644 index 00000000..be09fa40 --- /dev/null +++ b/docs/src/aggregation.md @@ -0,0 +1,105 @@ +## Aggregation + +Stac-fatsapi-elasticsearch-opensearch supports the STAC API [Aggregation Extension](https://github.com/stac-api-extensions/aggregation). This enables aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`//aggregations`). The [Filter Extension](https://github.com/stac-api-extensions/filter) is also fully supported, enabling aggregated returns of search queries. Any query made with `/search` may also be executed with `/aggregate`, provided that the relevant aggregation fields are available, + +A field named `aggregations` should be added to the Collection object for the collection for which the aggregations are available, for example: + +Available aggregations are: + +- total_count (count of total items) +- collection_frequency (Item `collection` field) +- platform_frequency (Item.Properties.platform) +- cloud_cover_frequency (Item.Properties.eo:cloud_cover) +- datetime_frequency (Item.Properties.datetime, monthly interval) +- datetime_min (earliest Item.Properties.datetime) +- datetime_max (latest Item.Properties.datetime) +- sun_elevation_frequency (Item.Properties.view:sun_elevation) +- sun_azimuth_frequency (Item.Properties.view:sun_azimuth) +- off_nadir_frequency (Item.Properties.view:off_nadir) +- grid_code_frequency (Item.Properties.grid:code) +- centroid_geohash_grid_frequency ([geohash grid](https://opensearch.org/docs/latest/aggregations/bucket/geohash-grid/) on Item.Properties.proj:centroid) +- centroid_geohex_grid_frequency ([geohex grid](https://opensearch.org/docs/latest/aggregations/bucket/geohex-grid/) on Item.Properties.proj:centroid) +- centroid_geotile_grid_frequency (geotile on Item.Properties.proj:centroid) +- geometry_geohash_grid_frequency ([geohash grid](https://opensearch.org/docs/latest/aggregations/bucket/geohash-grid/) on Item.geometry) +- geometry_geotile_grid_frequency ([geotile grid](https://opensearch.org/docs/latest/aggregations/bucket/geotile-grid/) on Item.geometry) + +Support for additional fields and new aggregations can be added in the [OpenSearch database_logic.py](../../stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py) and [ElasticSearch database_logic.py](../../stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py) files. + +```json +"aggregations": [ + { + "name": "total_count", + "data_type": "integer" + }, + { + "name": "datetime_max", + "data_type": "datetime" + }, + { + "name": "datetime_min", + "data_type": "datetime" + }, + { + "name": "datetime_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "datetime" + }, + { + "name": "sun_elevation_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "numeric" + }, + { + "name": "platform_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "string" + }, + { + "name": "sun_azimuth_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "numeric" + }, + { + "name": "off_nadir_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "numeric" + }, + { + "name": "cloud_cover_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "numeric" + }, + { + "name": "grid_code_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "string" + }, + { + "name": "centroid_geohash_grid_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "string" + }, + { + "name": "centroid_geohex_grid_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "string" + }, + { + "name": "centroid_geotile_grid_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "string" + }, + { + "name": "geometry_geohash_grid_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "numeric" + }, + { + "name": "geometry_geotile_grid_frequency", + "data_type": "frequency_distribution", + "frequency_distribution_data_type": "string" + } +] + ``` + + diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index da6d6880..cd723cfb 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -168,7 +168,7 @@ def indices(collection_ids: Optional[List[str]]) -> str: Returns: A string of comma-separated index names. If `collection_ids` is None, returns the default indices. """ - if collection_ids is None: + if collection_ids is None or collection_ids == []: return ITEM_INDICES else: return ",".join([index_by_collection_id(c) for c in collection_ids]) From 2fceed14cdbef23b4fabc430eaabfe4e91c8d678 Mon Sep 17 00:00:00 2001 From: Mo <109725814+mo-dkrz@users.noreply.github.com> Date: Sat, 30 Nov 2024 06:10:34 +0100 Subject: [PATCH 4/8] add build push docker image image in the publish ci (#311) **Related Issue(s):** - # **Description:** Since I needed to have this docker images of elastic search and open search, I added a new publish docker image step in publish.yml ping @jonhealy1 **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --------- Co-authored-by: Jonathan Healy --- .github/workflows/publish.yml | 68 ++++++++++++++++++- CHANGELOG.md | 8 +++ README.md | 46 +++++++++++-- dockerfiles/Dockerfile.ci.es | 19 ++++++ dockerfiles/Dockerfile.ci.os | 19 ++++++ .../core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- .../stac_fastapi/opensearch/version.py | 2 +- 10 files changed, 159 insertions(+), 11 deletions(-) create mode 100644 dockerfiles/Dockerfile.ci.es create mode 100644 dockerfiles/Dockerfile.ci.os diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e28da570..eb84e7fc 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,7 +6,7 @@ on: - "v*.*.*" # Triggers when a tag like 'v3.2.0' is pushed jobs: - build-and-publish: + build-and-publish-pypi: name: Build and Publish Packages runs-on: ubuntu-latest @@ -58,3 +58,69 @@ jobs: # Publish to PyPI twine upload dist/* + + build-and-push-images: + name: Build and Push Docker Images + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Elasticsearch image + id: meta-es + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/stac-fastapi-es + tags: | + type=raw,value=latest + type=ref,event=tag + + - name: Push Elasticsearch image + uses: docker/build-push-action@v6 + with: + context: . + file: dockerfiles/Dockerfile.ci.es + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta-es.outputs.tags }} + labels: ${{ steps.meta-es.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Extract metadata for OpenSearch image + id: meta-os + uses: docker/metadata-action@v5 + with: + images: ghcr.io/${{ github.repository_owner }}/stac-fastapi-os + tags: | + type=raw,value=latest + type=ref,event=tag + + - name: Push OpenSearch image + uses: docker/build-push-action@v6 + with: + context: . + file: dockerfiles/Dockerfile.ci.os + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta-os.outputs.tags }} + labels: ${{ steps.meta-os.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 95c427a7..2469eb46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v3.2.1] - 2024-11-14 + +### Added +- Added `dockerfiles/Dockerfile.ci.os` and `dockerfiles/Dockerfile.ci.es`, along with their respective entrypoints [#311](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/311) + +### Changed +- Updated the `publish.yml` workflow to include Docker image publishing to GitHub Container Registry [#311](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/311) +- Improved the README with detailed descriptions of the new Docker images, providing guidance for images. [#311](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/311) - Aggregation ElasticSearch `total_count` bugfix, moved aggregation text to docs. [#314](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/314) ## [v3.2.0] - 2024-10-09 diff --git a/README.md b/README.md index 0ff4f254..1783422c 100644 --- a/README.md +++ b/README.md @@ -43,17 +43,24 @@ or pip install stac_fastapi.opensearch ``` -## Build Elasticsearch API backend +### To install and run via pre-built Docker Images + +We provide ready-to-use Docker images through GitHub Container Registry ([ElasticSearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-es) and [OpenSearch](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pkgs/container/stac-fastapi-os) backends). You can easily pull and run these images: ```shell -docker-compose up elasticsearch -docker-compose build app-elasticsearch +# For Elasticsearch backend +docker pull ghcr.io/stac-utils/stac-fastapi-es:latest + +# For OpenSearch backend +docker pull ghcr.io/stac-utils/stac-fastapi-os:latest ``` -## Running Elasticsearch API on localhost:8080 +## Run Elasticsearch API backend on localhost:8080 + +You need to ensure [**Docker Compose**](https://docs.docker.com/compose/install/) or [**Podman Compose**](https://podman-desktop.io/docs/compose) installed and running on your machine. In the follwoing command instead of `docker-compose` you can use `podman-compose` as well. ```shell -docker-compose up app-elasticsearch +docker-compose up elasticsearch app-elasticsearch ``` By default, docker-compose uses Elasticsearch 8.x and OpenSearch 2.11.1. @@ -66,6 +73,35 @@ OPENSEARCH_VERSION=2.11.0 ``` The most recent Elasticsearch 7.x versions should also work. See the [opensearch-py docs](https://github.com/opensearch-project/opensearch-py/blob/main/COMPATIBILITY.md) for compatibility information. +#### **Configuration reference keys:** + +You can customize additional settings in your `.env` file: +###### Key variables to configure: + +| Variable | Description | Default | Required | +|------------------------------|--------------------------------------------------------------------------------------|--------------------------|---------------------------------------------------------------------------------------------| +| `ES_HOST` | Hostname for external Elasticsearch/OpenSearch. | `localhost` | Optional | +| `ES_PORT` | Port for Elasticsearch/OpenSearch. | `9200` (ES) / `9202` (OS)| Optional | +| `ES_USE_SSL` | Use SSL for connecting to Elasticsearch/OpenSearch. | `false` | Optional | +| `ES_VERIFY_CERTS` | Verify SSL certificates when connecting. | `false` | Optional | +| `STAC_FASTAPI_TITLE` | Title of the API in the documentation. | `stac-fastapi-elasticsearch` or `stac-fastapi-opensearch` | Optional | +| `STAC_FASTAPI_DESCRIPTION` | Description of the API in the documentation. | N/A | Optional | +| `STAC_FASTAPI_VERSION` | API version. | `2.1` | Optional | +| `APP_HOST` | Server bind address. | `0.0.0.0` | Optional | +| `APP_PORT` | Server port. | `8080` | Optional | +| `ENVIRONMENT` | Runtime environment. | `local` | Optional | +| `WEB_CONCURRENCY` | Number of worker processes. | `10` | Optional | +| `RELOAD` | Enable auto-reload for development. | `true` | Optional | +| `STAC_FASTAPI_RATE_LIMIT` | API rate limit per client. | `200/minute` | Optional | +| `BACKEND` | Tests-related variable | `elasticsearch` or `opensearch` based on the backend | Optional | +| `ELASTICSEARCH_VERSION` | ElasticSearch version | `7.17.1` | Optional | +| `OPENSEARCH_VERSION` | OpenSearch version | `2.11.0` | Optional | + +> [!NOTE] +> The variables `ES_HOST`, `ES_PORT`, `ES_USE_SSL`, and `ES_VERIFY_CERTS` apply to both Elasticsearch and OpenSearch backends, so there is no need to rename the key names to `OS_` even if you're using OpenSearch. + +## Interacting with the API + To create a new Collection: ```shell diff --git a/dockerfiles/Dockerfile.ci.es b/dockerfiles/Dockerfile.ci.es new file mode 100644 index 00000000..a6fb6a53 --- /dev/null +++ b/dockerfiles/Dockerfile.ci.es @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gcc \ + curl \ + && apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY . /app/ + +RUN pip3 install --no-cache-dir -e ./stac_fastapi/core && \ + pip3 install --no-cache-dir ./stac_fastapi/elasticsearch[server] + +USER root + +CMD ["python", "-m", "stac_fastapi.elasticsearch.app"] \ No newline at end of file diff --git a/dockerfiles/Dockerfile.ci.os b/dockerfiles/Dockerfile.ci.os new file mode 100644 index 00000000..a046a3b6 --- /dev/null +++ b/dockerfiles/Dockerfile.ci.os @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + gcc \ + curl \ + && apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY . /app/ + +RUN pip3 install --no-cache-dir -e ./stac_fastapi/core && \ + pip3 install --no-cache-dir ./stac_fastapi/opensearch[server] + +USER root + +CMD ["python", "-m", "stac_fastapi.opensearch.app"] \ No newline at end of file diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index fad2b4e4..59d4d1a3 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.0" +__version__ = "3.2.1" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index a23cbe7c..9239d4fa 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.2.0", + "stac-fastapi.core==3.2.1", "elasticsearch[async]==8.11.0", "elasticsearch-dsl==8.11.0", "uvicorn", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index fad2b4e4..59d4d1a3 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.0" +__version__ = "3.2.1" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 2cd43da3..3ea0b9ca 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.2.0", + "stac-fastapi.core==3.2.1", "opensearch-py==2.4.2", "opensearch-py[async]==2.4.2", "uvicorn", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index fad2b4e4..59d4d1a3 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.0" +__version__ = "3.2.1" From a9e15eed208a99f7332e18dbc6a62e6e09657e92 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Sat, 30 Nov 2024 13:16:41 +0800 Subject: [PATCH 5/8] Pin httpx to <= 0.27.2 to fix tests (#316) --- stac_fastapi/elasticsearch/setup.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 9239d4fa..f6e28790 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -21,7 +21,7 @@ "pre-commit", "requests", "ciso8601", - "httpx", + "httpx<=0.27.2", ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], "server": ["uvicorn[standard]==0.19.0"], diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index 3ea0b9ca..b6ebd70b 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -21,7 +21,7 @@ "pre-commit", "requests", "ciso8601", - "httpx", + "httpx<=0.27.2", ], "docs": ["mkdocs", "mkdocs-material", "pdocs"], "server": ["uvicorn[standard]==0.19.0"], From 106fabbdf1aa4afbcad01dce3a87f62a0f525729 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Sat, 30 Nov 2024 14:00:24 +0800 Subject: [PATCH 6/8] Update for pep625 - pypi deprecation warnings (#317) --- stac_fastapi/core/setup.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/core/setup.py b/stac_fastapi/core/setup.py index 9a359afc..a83e62dc 100644 --- a/stac_fastapi/core/setup.py +++ b/stac_fastapi/core/setup.py @@ -23,7 +23,7 @@ ] setup( - name="stac-fastapi.core", + name="stac_fastapi.core", description="Core library for the Elasticsearch and Opensearch stac-fastapi backends.", long_description=desc, long_description_content_type="text/markdown", diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index f6e28790..5f85b610 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -28,7 +28,7 @@ } setup( - name="stac-fastapi.elasticsearch", + name="stac_fastapi.elasticsearch", description="An implementation of STAC API based on the FastAPI framework with both Elasticsearch and Opensearch.", long_description=desc, long_description_content_type="text/markdown", diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index b6ebd70b..c1acf2c8 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -28,7 +28,7 @@ } setup( - name="stac-fastapi.opensearch", + name="stac_fastapi.opensearch", description="Opensearch stac-fastapi backend.", long_description=desc, long_description_content_type="text/markdown", From 7e848b45121f39b8b883dbec985e23c15f985495 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Fri, 13 Dec 2024 12:34:10 +0100 Subject: [PATCH 7/8] fix: use base64 encoded JSON string of sort keys as pagination token (#323) **Related Issue(s):** - #322 **Description:** Use base64 encoded JSON string of sort keys as pagination token instead of comma separated string. --- CHANGELOG.md | 2 ++ .../stac_fastapi/elasticsearch/database_logic.py | 7 +++---- .../opensearch/stac_fastapi/opensearch/database_logic.py | 7 +++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2469eb46..f2b31791 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Use base64 encoded JSON string of sort keys as pagination token instead of comma-separated string [#323](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/323) ## [v3.2.1] - 2024-11-14 diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index cd723cfb..c404b5e5 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -1,6 +1,7 @@ """Database logic.""" import asyncio +import json import logging import os from base64 import urlsafe_b64decode, urlsafe_b64encode @@ -660,7 +661,7 @@ async def execute_search( search_after = None if token: - search_after = urlsafe_b64decode(token.encode()).decode().split(",") + search_after = json.loads(urlsafe_b64decode(token).decode()) query = search.query.to_dict() if search.query else None @@ -700,9 +701,7 @@ async def execute_search( next_token = None if len(hits) > limit and limit < max_result_window: if hits and (sort_array := hits[limit - 1].get("sort")): - next_token = urlsafe_b64encode( - ",".join([str(x) for x in sort_array]).encode() - ).decode() + next_token = urlsafe_b64encode(json.dumps(sort_array).encode()).decode() matched = ( es_response["hits"]["total"]["value"] diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 778cfe03..63a42427 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -1,6 +1,7 @@ """Database logic.""" import asyncio +import json import logging import os from base64 import urlsafe_b64decode, urlsafe_b64encode @@ -692,7 +693,7 @@ async def execute_search( search_after = None if token: - search_after = urlsafe_b64decode(token.encode()).decode().split(",") + search_after = json.loads(urlsafe_b64decode(token).decode()) if search_after: search_body["search_after"] = search_after @@ -732,9 +733,7 @@ async def execute_search( next_token = None if len(hits) > limit and limit < max_result_window: if hits and (sort_array := hits[limit - 1].get("sort")): - next_token = urlsafe_b64encode( - ",".join([str(x) for x in sort_array]).encode() - ).decode() + next_token = urlsafe_b64encode(json.dumps(sort_array).encode()).decode() matched = ( es_response["hits"]["total"]["value"] From e45cfea64871978a90e99cdfc8d185dec5c6f30f Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Sun, 15 Dec 2024 00:11:53 +0800 Subject: [PATCH 8/8] Release v3.2.2 (#324) **Related Issue(s):** - # **Description:** **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 12 ++++++++++-- stac_fastapi/core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 2 +- .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 2 +- .../opensearch/stac_fastapi/opensearch/version.py | 2 +- 6 files changed, 15 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2b31791..e41a1462 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,17 +6,23 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). ## [Unreleased] + +## [v3.2.2] - 2024-12-15 + ### Changed + - Use base64 encoded JSON string of sort keys as pagination token instead of comma-separated string [#323](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/323) ## [v3.2.1] - 2024-11-14 ### Added + - Added `dockerfiles/Dockerfile.ci.os` and `dockerfiles/Dockerfile.ci.es`, along with their respective entrypoints [#311](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/311) ### Changed + - Updated the `publish.yml` workflow to include Docker image publishing to GitHub Container Registry [#311](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/311) -- Improved the README with detailed descriptions of the new Docker images, providing guidance for images. [#311](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/311) +- Improved the README with detailed descriptions of the new Docker images, providing guidance for images. [#311](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/311) - Aggregation ElasticSearch `total_count` bugfix, moved aggregation text to docs. [#314](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/314) ## [v3.2.0] - 2024-10-09 @@ -277,7 +283,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.0...main +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.2...main +[v3.2.2]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.1...v3.2.2 +[v3.2.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.2.0...v3.2.1 [v3.2.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.1.0...v3.2.0 [v3.1.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v3.0.0...v3.1.0 [v3.0.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch/tree/v2.4.1...v3.0.0 diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index 59d4d1a3..05a55a44 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.1" +__version__ = "3.2.2" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 5f85b610..fd2723de 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.2.1", + "stac-fastapi.core==3.2.2", "elasticsearch[async]==8.11.0", "elasticsearch-dsl==8.11.0", "uvicorn", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 59d4d1a3..05a55a44 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.1" +__version__ = "3.2.2" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index c1acf2c8..948e2e44 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==3.2.1", + "stac-fastapi.core==3.2.2", "opensearch-py==2.4.2", "opensearch-py[async]==2.4.2", "uvicorn", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index 59d4d1a3..05a55a44 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "3.2.1" +__version__ = "3.2.2"