fix ci

stac-utils · Oct 31, 2024 · e462253 · e462253
1 parent 0b726a8
commit e462253
Show file tree

Hide file tree

Showing 8 changed files with 177 additions and 137 deletions.
diff --git a/.github/workflows/civ2.yml b/.github/workflows/civ2.yml
@@ -82,32 +82,118 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha, mode=max
 
-  # test:
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - uses: docker/setup-buildx-action@v3
-  #     - name: Build and Cache Docker layers
-  #       uses: docker/bake-action@master
-  #       with:
-  #         push: false
-  #         load: true
-  #         allow:
-  #           network.host
-  #         files: |-
-  #           docker-compose.yml
-  #           docker-compose-cache.json
-  #     - name: Run docker compose
-  #       run: |
-  #         docker compose up -d pgstac
+  check-pypgstac-base:
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    outputs:
+      pypgstac-image-exists: ${{ steps.pypgstac_image_check.outputs.pypgstac-image-exists }}
+      pypgstac-image-hash: ${{ steps.pypgstac_image_check.outputs.pypgstac-image-hash }}
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Check if pypgstac-base docker image needs to be updated
+        id: pypgstac_image_check
+        run: |
+          set -e
+          IMAGE=stac-utils/pypgstac-base
+          HASH=$(git log -1 --pretty="format:%h" -- docker/pypgstac)
+          TAG="\"$HASH\""
+          TOKEN=$(curl -s https://ghcr.io/token\?scope\="repository:$IMAGE:pull" | jq -r .token)
+          EXISTS=$(curl -s -H "Authorization: Bearer $TOKEN" https://ghcr.io/v2/$IMAGE/tags/list | jq "try(any(.tags[]; . == $TAG))")
+          echo "pypgstac-image-exists=$EXISTS" >>$GITHUB_OUTPUT
+          echo "pypgstac-image-hash=$HASH" >>$GITHUB_OUTPUT
+          echo "pypgstac-image-exists: $EXISTS"
+          echo "pypgstac-image-hash: $HASH"
 
-  #     - name: Execute tests in the running services
-  #       run: |
-  #         scripts/test --nobuild
+  buildpypgstacbase:
+    name: Build and push base postgres image
+    needs: [check-pypgstac-base]
+    if: ${{ needs.check-pypgstac-base.outputs.pypgstac-image-exists != 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Display inputs
+        run: |
+          echo "Inputs:"
+          echo ${{ needs.check-pypgstac-base.outputs.pypgstac-image-exists }}
+          echo ${{ needs.check-pypgstac-base.outputs.pypgstac-image-hash }}
+      - uses: actions/checkout@v4
+      - uses: docker/setup-buildx-action@v3
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and Push Base Postgres
+        uses: docker/build-push-action@v6
+        with:
+          platforms: linux/amd64
+          context: .
+          target: pypgstac-base
+          file: docker/pgstac/Dockerfile
+          tags: ghcr.io/stac-utils/pypgstac-base:${{ needs.check-pypgstac-base.outputs.pypgstac-image-hash }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha, mode=max
+  wheels:
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v4
+        - name: Build wheels
+          uses: PyO3/maturin-action@v1
+          with:
+            working-directory: src/pypgstac
+            target: x86_64
+            args: --release --out /home/runner/work/pgstac/pgstac/dist
+            sccache: 'true'
+            manylinux: auto
+        - name: Upload wheels
+          uses: actions/upload-artifact@v4
+          with:
+            name: wheels
+            path: /home/runner/work/pgstac/pgstac/dist/*
+            if-no-files-found: error
+  test:
+    needs: [buildpg, buildpypgstacbase, wheels]
+    if: ${{ always() && !failure() && !cancelled() }}
+    runs-on: ubuntu-latest
+    container: ghcr.io/stac-utils/pypgstac-base:${{ needs.check-pypgstac-base.outputs.pypgstac-image-hash }}
+    env:
+      PGHOST: localhost
+      PGUSER: postgres
+      PGPASSWORD: postgres
+      PGDATABASE: postgres
+    services:
+      pg:
+        image: ghcr.io/stac-utils/pgstac-base:${{ needs.check-pgstac-base.outputs.pgstac-image-hash }}
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+          POSTGRES_DATABASE: postgres
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
-  #     - name: Stop Docker
-  #       run: |
-  #         scripts/nuke
+    steps:
+      - uses: actions/checkout@v4
+      - name: Get Wheel
+        uses: actions/download-artifact@v4
+        with:
+            name: wheels
+            path: /tmp/wheels
+      - name: Install pypgstac
+        run:  |
+          cd /home/runner/work/pgstac/pgstac/src/pypgstac
+          source $UV_PROJECT_ENVIRONMENT/bin/activate
+          uv pip install --offline --find-links /tmp/wheels pypgstac
+      - name: Run tests
+        run:  |
+          cd /home/runner/work/pgstac/pgstac/src/pypgstac
+          source $UV_PROJECT_ENVIRONMENT/bin/activate
+          $SCRIPTS/test
 
   # linux_x86_64:
   #   runs-on: ubuntu-latest
@@ -218,54 +304,4 @@ jobs:
   #         path: /home/runner/work/pgstac/pgstac/dist/*
   #         if-no-files-found: error
 
-  # test:
-  #   name: test
-  #   needs: [buildpg, linux_x86_64]
-  #   runs-on: ubuntu-latest
-  #   container:
-  #   env:
-  #     PGHOST: localhost
-  #     UV_FROZEN: true
-  #     UV_FIND_LINKS: /tmp/wheels
-  #     UV_PROJECT_ENVIRONMENT: /home/runner/work/pgstac/pgstac/src/pypgstac/.venv
-  #     VIRTUAL_ENVIRONMENT: /home/runner/work/pgstac/pgstac/src/pypgstac/.venv
-
-  #   services:
-  #     postgres:
-  #       image: ghcr.io/stac-utils/pgstac-postgres:latest
-  #       env:
-  #         POSTGRES_PASSWORD: postgres
-  #         POSTGRES_USER: postgres
-  #         POSTGRES_DATABASE: postgres
-  #       options: >-
-  #         --health-cmd pg_isready
-  #         --health-interval 10s
-  #         --health-timeout 5s
-  #         --health-retries 5
-
-  #   steps:
-  #     - uses: actions/checkout@v4
-  #     - name: Setup UV
-  #       uses: astral-sh/setup-uv@v3
-  #       with:
-  #         enable-cache: true
-  #     - name: Get Wheel
-  #       uses: actions/download-artifact@v4
-  #       with:
-  #           name: wheels-x86_64
-  #           path: /tmp/wheels
-  #     - name: Install pypgstac
-  #       run:  |
-  #         cd /home/runner/work/pgstac/pgstac/src/pypgstac
-  #         uv venv
-  #         source $UV_PROJECT_ENVIRONMENT/bin/activate
-  #         uv pip install psycopg[binary] psycopg-pool
-  #         uv sync --frozen --no-install-project --extra test --extra dev
-  #         uv pip install --offline --find-links /tmp/wheels pypgstac
-
-
-  #     - name: Run tests
-  #       run:  |
-  #         cd /home/runner/work/pgstac/pgstac/src/pypgstac
-  #         source $UV_PROJECT_ENVIRONMENT/bin/activate
-  #         $SCRIPTS/test
+  #
diff --git a/docker/pgstac/Dockerfile b/docker/pgstac/Dockerfile
@@ -1,8 +1,8 @@
-ARG PG_MAJOR=15
+ARG PG_MAJOR=16
 ARG POSTGIS_MAJOR=3
 
 # Base postgres image that pgstac can be installed onto
-FROM postgres:${PG_MAJOR}-bullseye AS pgstacbase
+FROM postgres:${PG_MAJOR}-bookworm AS pgstacbase
 ARG POSTGIS_MAJOR
 RUN  \
     apt-get update \
@@ -20,11 +20,12 @@ COPY docker/pgstac/dbinit/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.
 
 # Base postgres image with plrust installed that can be used for future development using plrust
 FROM pgstacbase AS pgstacbase-plrust
-ENV PLRUSTVERSION=1.2.7
-ENV RUSTVERSION=1.72.0
-ENV PLRUSTDOWNLOADURL=https://github.com/tcdi/plrust/releases/download/
-ENV PLRUSTFILE=plrust-trusted-${PLRUSTVERSION}_${RUSTVERSION}-debian-pg${PG_MAJOR}-amd64.deb
-ENV PLRUSTURL=${PLRUSTDOWNLOADURL}v${PLRUSTVERSION}/${PLRUSTFILE}
+ENV \
+    PLRUSTVERSION=1.2.7 \
+    RUSTVERSION=1.72.0 \
+    PLRUSTDOWNLOADURL=https://github.com/tcdi/plrust/releases/download/ \
+    PLRUSTFILE=plrust-trusted-${PLRUSTVERSION}_${RUSTVERSION}-debian-pg${PG_MAJOR}-amd64.deb \
+    PLRUSTURL=${PLRUSTDOWNLOADURL}v${PLRUSTVERSION}/${PLRUSTFILE}
 ADD $PLRUSTURL .
 ENV PATH=/home/postgres/.cargo/bin:$PATH
 RUN  \

diff --git a/docker/pypgstac/Dockerfile b/docker/pypgstac/Dockerfile
@@ -1,6 +1,4 @@
-ARG UV_VERSION=latest
-ARG PY_VERSION=3.12
-
+# Image used to build wheels for pypgstac
 FROM ghcr.io/pyo3/maturin AS pypgstacbuilder
 WORKDIR /io
 RUN --mount=type=bind,source=./src/pypgstac,target=/io \
@@ -10,7 +8,8 @@ RUN --mount=type=bind,source=./src/pypgstac,target=/io \
 ENTRYPOINT []
 CMD []
 
-FROM ghcr.io/astral-sh/uv:python3.9-bookworm-slim AS pypgstac-slim
+# Image that contains pypgstac but not psql, dev, or test
+FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS pypgstac-slim-base
 ENV \
     UV_LINK_MODE=copy \
     UV_COMPILE_BYTECODE=1 \
@@ -24,22 +23,14 @@ RUN \
     --mount=type=bind,source=./src/pypgstac/uv.lock,target=/opt/src/pypgstac/uv.lock \
     --mount=type=bind,source=./src/pypgstac/pyproject.toml,target=/opt/src/pypgstac/pyproject.toml \
     uv venv --allow-existing --relocatable /venv \
-    && uv sync --frozen --no-install-project \
-    && uv pip install psycopg[binary] psycopg-pool
-
-RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/wheels \
-    for i in /wheels/pypgstac*x86_64.whl; do uv pip install pypgstac@$i; done
+    && uv sync --frozen --no-install-project --extra psycopg
 
 ENTRYPOINT []
 CMD ["/bin/true"]
 
-FROM ghcr.io/astral-sh/uv:python3.9-bookworm-slim AS pypgstac
+# Base image with psql and all dependencies
+FROM pypgstac-slim-base AS pypgstac-base
 ENV \
-    UV_LINK_MODE=copy \
-    UV_COMPILE_BYTECODE=1 \
-    UV_PROJECT_ENVIRONMENT=/venv \
-    UV_PYTHON_INSTALL_DIR=/venv/bin \
-    VIRTUAL_ENVIRONMENT=/venv \
     PATH=/scripts:/venv/bin:$PATH \
     SCRIPTS=/scripts \
     PGSTACDIR=/src/pgstac \
@@ -48,27 +39,34 @@ ENV \
     TESTS=/src/pypgstac/tests
 WORKDIR /opt/src/pypgstac
 RUN \
+    --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
     --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=./src/pypgstac/uv.lock,target=/opt/src/pypgstac/uv.lock \
     --mount=type=bind,source=./src/pypgstac/pyproject.toml,target=/opt/src/pypgstac/pyproject.toml \
     apt-get update \
-    && apt-get upgrade -y \
     && apt-get install -y --no-install-recommends \
         postgresql-client libpq-dev python3-dev \
     && apt-get remove -y apt-transport-https \
     && apt-get clean && apt-get -y autoremove \
-    && uv venv --allow-existing --relocatable /venv \
-    && uv sync --frozen --no-install-project --extra test --extra dev --extra migrations \
-    && uv pip install psycopg[binary] psycopg-pool setuptools
+    && uv sync --frozen --no-install-project --extra test --extra dev --extra migrations --extra psycopg \
+    && uv pip install setuptools
+
 
+# slim image for running pypgstac
+FROM pypgstac-slim-base AS pypgstac-slim
 COPY ./docker/pypgstac/bin/ /scripts/
 COPY ./src /src
+RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/tmp/wheels \
+    uv pip install --offline --find-links /tmp/wheels pypgstac
 
-RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/wheels \
-    for i in /wheels/pypgstac*x86_64.whl; do echo $i; uv pip install pypgstac@$i; done
+# Loaded image for running pypgstac and test suite
+FROM pypgstac-base AS pypgstac
+COPY ./docker/pypgstac/bin/ /scripts/
+COPY ./src /src
+RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/tmp/wheels \
+    uv pip install --offline --find-links /tmp/wheels pypgstac
 
-ENTRYPOINT []
-CMD ["/bin/true"]
 
 
 

diff --git a/src/pgstac/tests/pgtap.sql b/src/pgstac/tests/pgtap.sql
@@ -34,7 +34,7 @@ SELECT plan(229);
 \i tests/pgtap/004a_collectionsearch.sql
 \i tests/pgtap/005_tileutils.sql
 \i tests/pgtap/006_tilesearch.sql
-\i tests/pgtap/999_version.sql
+\i tests/pgtap/999_testreadonly.sql
 
 -- Finish the tests and clean up.
 SELECT * FROM finish();

diff --git a/src/pgstac/tests/pgtap/001_core.sql b/src/pgstac/tests/pgtap/001_core.sql
@@ -27,37 +27,6 @@ SELECT lives_ok(
     'Search works with readonly mode set to off in readwrite mode.'
 );
 
-SET transaction_read_only TO 'on';
-
-SELECT results_eq(
-    $$ SHOW transaction_read_only; $$,
-    $$ SELECT 'on'; $$,
-    'Transaction set to read only'
-);
-
-SELECT throws_ok(
-    $$ SELECT search('{}'); $$,
-    '25006'
-);
-
-SET pgstac.readonly to 'true';
-SELECT results_eq(
-    $$ SELECT pgstac.readonly(); $$,
-    $$ SELECT TRUE; $$,
-    'Readonly is set to true'
-);
-
-SELECT lives_ok(
-    $$ SELECT search('{}'); $$,
-    'Search works with readonly mode set to on in readonly mode.'
-);
-
-SET pgstac.context TO 'on';
-SELECT lives_ok(
-    $$ SELECT search('{}'); $$,
-    'Search works with readonly mode set to on in readonly mode and the context extension enabled.'
-);
-RESET transaction_read_only;
 RESET pgstac.readonly;
 RESET pgstac.context;
 SELECT is_definer('update_partition_stats');

diff --git a/src/pgstac/tests/pgtap/004_search.sql b/src/pgstac/tests/pgtap/004_search.sql
@@ -705,6 +705,12 @@ SELECT * FROM pg_temp.testpaging('desc','asc');
 
 \copy items_staging (content) FROM 'tests/testdata/items_duplicate_ids.ndjson'
 
+SELECT is(
+    (SELECT count(*) FROM items WHERE id='pgstac-test-item-duplicated'),
+    '2',
+    'Check count of items with dupe id'
+);
+
 SELECT is(
     (SELECT jsonb_array_length(search('{"ids": ["pgstac-test-item-duplicated"]}')->'features')),
     '2',