From e4622538fe028f824ed7f4e3a20b745d35d8b3ee Mon Sep 17 00:00:00 2001 From: David W Bitner Date: Thu, 31 Oct 2024 15:11:15 -0500 Subject: [PATCH] fix ci --- .github/workflows/civ2.yml | 186 ++++++++++++-------- docker/pgstac/Dockerfile | 15 +- docker/pypgstac/Dockerfile | 44 +++-- src/pgstac/tests/pgtap.sql | 2 +- src/pgstac/tests/pgtap/001_core.sql | 31 ---- src/pgstac/tests/pgtap/004_search.sql | 6 + src/pgstac/tests/pgtap/999_testreadonly.sql | 30 ++++ src/pgstac/tests/pgtap/999_version.sql | 0 8 files changed, 177 insertions(+), 137 deletions(-) create mode 100644 src/pgstac/tests/pgtap/999_testreadonly.sql delete mode 100644 src/pgstac/tests/pgtap/999_version.sql diff --git a/.github/workflows/civ2.yml b/.github/workflows/civ2.yml index 535bb495..b328fb6e 100644 --- a/.github/workflows/civ2.yml +++ b/.github/workflows/civ2.yml @@ -82,32 +82,118 @@ jobs: cache-from: type=gha cache-to: type=gha, mode=max - # test: - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # - uses: docker/setup-buildx-action@v3 - # - name: Build and Cache Docker layers - # uses: docker/bake-action@master - # with: - # push: false - # load: true - # allow: - # network.host - # files: |- - # docker-compose.yml - # docker-compose-cache.json - # - name: Run docker compose - # run: | - # docker compose up -d pgstac + check-pypgstac-base: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + pypgstac-image-exists: ${{ steps.pypgstac_image_check.outputs.pypgstac-image-exists }} + pypgstac-image-hash: ${{ steps.pypgstac_image_check.outputs.pypgstac-image-hash }} + + steps: + - uses: actions/checkout@v4 + - name: Check if pypgstac-base docker image needs to be updated + id: pypgstac_image_check + run: | + set -e + IMAGE=stac-utils/pypgstac-base + HASH=$(git log -1 --pretty="format:%h" -- docker/pypgstac) + TAG="\"$HASH\"" + TOKEN=$(curl -s https://ghcr.io/token\?scope\="repository:$IMAGE:pull" | jq -r .token) + EXISTS=$(curl -s -H "Authorization: Bearer $TOKEN" https://ghcr.io/v2/$IMAGE/tags/list | jq "try(any(.tags[]; . == $TAG))") + echo "pypgstac-image-exists=$EXISTS" >>$GITHUB_OUTPUT + echo "pypgstac-image-hash=$HASH" >>$GITHUB_OUTPUT + echo "pypgstac-image-exists: $EXISTS" + echo "pypgstac-image-hash: $HASH" - # - name: Execute tests in the running services - # run: | - # scripts/test --nobuild + buildpypgstacbase: + name: Build and push base postgres image + needs: [check-pypgstac-base] + if: ${{ needs.check-pypgstac-base.outputs.pypgstac-image-exists != 'true' }} + runs-on: ubuntu-latest + steps: + - name: Display inputs + run: | + echo "Inputs:" + echo ${{ needs.check-pypgstac-base.outputs.pypgstac-image-exists }} + echo ${{ needs.check-pypgstac-base.outputs.pypgstac-image-hash }} + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and Push Base Postgres + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + context: . + target: pypgstac-base + file: docker/pgstac/Dockerfile + tags: ghcr.io/stac-utils/pypgstac-base:${{ needs.check-pypgstac-base.outputs.pypgstac-image-hash }} + push: true + cache-from: type=gha + cache-to: type=gha, mode=max + wheels: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + working-directory: src/pypgstac + target: x86_64 + args: --release --out /home/runner/work/pgstac/pgstac/dist + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels + path: /home/runner/work/pgstac/pgstac/dist/* + if-no-files-found: error + test: + needs: [buildpg, buildpypgstacbase, wheels] + if: ${{ always() && !failure() && !cancelled() }} + runs-on: ubuntu-latest + container: ghcr.io/stac-utils/pypgstac-base:${{ needs.check-pypgstac-base.outputs.pypgstac-image-hash }} + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + PGDATABASE: postgres + services: + pg: + image: ghcr.io/stac-utils/pgstac-base:${{ needs.check-pgstac-base.outputs.pgstac-image-hash }} + env: + POSTGRES_PASSWORD: postgres + POSTGRES_USER: postgres + POSTGRES_DATABASE: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 - # - name: Stop Docker - # run: | - # scripts/nuke + steps: + - uses: actions/checkout@v4 + - name: Get Wheel + uses: actions/download-artifact@v4 + with: + name: wheels + path: /tmp/wheels + - name: Install pypgstac + run: | + cd /home/runner/work/pgstac/pgstac/src/pypgstac + source $UV_PROJECT_ENVIRONMENT/bin/activate + uv pip install --offline --find-links /tmp/wheels pypgstac + - name: Run tests + run: | + cd /home/runner/work/pgstac/pgstac/src/pypgstac + source $UV_PROJECT_ENVIRONMENT/bin/activate + $SCRIPTS/test # linux_x86_64: # runs-on: ubuntu-latest @@ -218,54 +304,4 @@ jobs: # path: /home/runner/work/pgstac/pgstac/dist/* # if-no-files-found: error - # test: - # name: test - # needs: [buildpg, linux_x86_64] - # runs-on: ubuntu-latest - # container: - # env: - # PGHOST: localhost - # UV_FROZEN: true - # UV_FIND_LINKS: /tmp/wheels - # UV_PROJECT_ENVIRONMENT: /home/runner/work/pgstac/pgstac/src/pypgstac/.venv - # VIRTUAL_ENVIRONMENT: /home/runner/work/pgstac/pgstac/src/pypgstac/.venv - - # services: - # postgres: - # image: ghcr.io/stac-utils/pgstac-postgres:latest - # env: - # POSTGRES_PASSWORD: postgres - # POSTGRES_USER: postgres - # POSTGRES_DATABASE: postgres - # options: >- - # --health-cmd pg_isready - # --health-interval 10s - # --health-timeout 5s - # --health-retries 5 - - # steps: - # - uses: actions/checkout@v4 - # - name: Setup UV - # uses: astral-sh/setup-uv@v3 - # with: - # enable-cache: true - # - name: Get Wheel - # uses: actions/download-artifact@v4 - # with: - # name: wheels-x86_64 - # path: /tmp/wheels - # - name: Install pypgstac - # run: | - # cd /home/runner/work/pgstac/pgstac/src/pypgstac - # uv venv - # source $UV_PROJECT_ENVIRONMENT/bin/activate - # uv pip install psycopg[binary] psycopg-pool - # uv sync --frozen --no-install-project --extra test --extra dev - # uv pip install --offline --find-links /tmp/wheels pypgstac - - - # - name: Run tests - # run: | - # cd /home/runner/work/pgstac/pgstac/src/pypgstac - # source $UV_PROJECT_ENVIRONMENT/bin/activate - # $SCRIPTS/test + # diff --git a/docker/pgstac/Dockerfile b/docker/pgstac/Dockerfile index 7035df7d..6a371f1d 100644 --- a/docker/pgstac/Dockerfile +++ b/docker/pgstac/Dockerfile @@ -1,8 +1,8 @@ -ARG PG_MAJOR=15 +ARG PG_MAJOR=16 ARG POSTGIS_MAJOR=3 # Base postgres image that pgstac can be installed onto -FROM postgres:${PG_MAJOR}-bullseye AS pgstacbase +FROM postgres:${PG_MAJOR}-bookworm AS pgstacbase ARG POSTGIS_MAJOR RUN \ apt-get update \ @@ -20,11 +20,12 @@ COPY docker/pgstac/dbinit/docker-entrypoint.sh /usr/local/bin/docker-entrypoint. # Base postgres image with plrust installed that can be used for future development using plrust FROM pgstacbase AS pgstacbase-plrust -ENV PLRUSTVERSION=1.2.7 -ENV RUSTVERSION=1.72.0 -ENV PLRUSTDOWNLOADURL=https://github.com/tcdi/plrust/releases/download/ -ENV PLRUSTFILE=plrust-trusted-${PLRUSTVERSION}_${RUSTVERSION}-debian-pg${PG_MAJOR}-amd64.deb -ENV PLRUSTURL=${PLRUSTDOWNLOADURL}v${PLRUSTVERSION}/${PLRUSTFILE} +ENV \ + PLRUSTVERSION=1.2.7 \ + RUSTVERSION=1.72.0 \ + PLRUSTDOWNLOADURL=https://github.com/tcdi/plrust/releases/download/ \ + PLRUSTFILE=plrust-trusted-${PLRUSTVERSION}_${RUSTVERSION}-debian-pg${PG_MAJOR}-amd64.deb \ + PLRUSTURL=${PLRUSTDOWNLOADURL}v${PLRUSTVERSION}/${PLRUSTFILE} ADD $PLRUSTURL . ENV PATH=/home/postgres/.cargo/bin:$PATH RUN \ diff --git a/docker/pypgstac/Dockerfile b/docker/pypgstac/Dockerfile index 3f267d66..ae185127 100644 --- a/docker/pypgstac/Dockerfile +++ b/docker/pypgstac/Dockerfile @@ -1,6 +1,4 @@ -ARG UV_VERSION=latest -ARG PY_VERSION=3.12 - +# Image used to build wheels for pypgstac FROM ghcr.io/pyo3/maturin AS pypgstacbuilder WORKDIR /io RUN --mount=type=bind,source=./src/pypgstac,target=/io \ @@ -10,7 +8,8 @@ RUN --mount=type=bind,source=./src/pypgstac,target=/io \ ENTRYPOINT [] CMD [] -FROM ghcr.io/astral-sh/uv:python3.9-bookworm-slim AS pypgstac-slim +# Image that contains pypgstac but not psql, dev, or test +FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS pypgstac-slim-base ENV \ UV_LINK_MODE=copy \ UV_COMPILE_BYTECODE=1 \ @@ -24,22 +23,14 @@ RUN \ --mount=type=bind,source=./src/pypgstac/uv.lock,target=/opt/src/pypgstac/uv.lock \ --mount=type=bind,source=./src/pypgstac/pyproject.toml,target=/opt/src/pypgstac/pyproject.toml \ uv venv --allow-existing --relocatable /venv \ - && uv sync --frozen --no-install-project \ - && uv pip install psycopg[binary] psycopg-pool - -RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/wheels \ - for i in /wheels/pypgstac*x86_64.whl; do uv pip install pypgstac@$i; done + && uv sync --frozen --no-install-project --extra psycopg ENTRYPOINT [] CMD ["/bin/true"] -FROM ghcr.io/astral-sh/uv:python3.9-bookworm-slim AS pypgstac +# Base image with psql and all dependencies +FROM pypgstac-slim-base AS pypgstac-base ENV \ - UV_LINK_MODE=copy \ - UV_COMPILE_BYTECODE=1 \ - UV_PROJECT_ENVIRONMENT=/venv \ - UV_PYTHON_INSTALL_DIR=/venv/bin \ - VIRTUAL_ENVIRONMENT=/venv \ PATH=/scripts:/venv/bin:$PATH \ SCRIPTS=/scripts \ PGSTACDIR=/src/pgstac \ @@ -48,27 +39,34 @@ ENV \ TESTS=/src/pypgstac/tests WORKDIR /opt/src/pypgstac RUN \ + --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,source=./src/pypgstac/uv.lock,target=/opt/src/pypgstac/uv.lock \ --mount=type=bind,source=./src/pypgstac/pyproject.toml,target=/opt/src/pypgstac/pyproject.toml \ apt-get update \ - && apt-get upgrade -y \ && apt-get install -y --no-install-recommends \ postgresql-client libpq-dev python3-dev \ && apt-get remove -y apt-transport-https \ && apt-get clean && apt-get -y autoremove \ - && uv venv --allow-existing --relocatable /venv \ - && uv sync --frozen --no-install-project --extra test --extra dev --extra migrations \ - && uv pip install psycopg[binary] psycopg-pool setuptools + && uv sync --frozen --no-install-project --extra test --extra dev --extra migrations --extra psycopg \ + && uv pip install setuptools + +# slim image for running pypgstac +FROM pypgstac-slim-base AS pypgstac-slim COPY ./docker/pypgstac/bin/ /scripts/ COPY ./src /src +RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/tmp/wheels \ + uv pip install --offline --find-links /tmp/wheels pypgstac -RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/wheels \ - for i in /wheels/pypgstac*x86_64.whl; do echo $i; uv pip install pypgstac@$i; done +# Loaded image for running pypgstac and test suite +FROM pypgstac-base AS pypgstac +COPY ./docker/pypgstac/bin/ /scripts/ +COPY ./src /src +RUN --mount=type=bind,from=pypgstacbuilder,source=/wheels,target=/tmp/wheels \ + uv pip install --offline --find-links /tmp/wheels pypgstac -ENTRYPOINT [] -CMD ["/bin/true"] diff --git a/src/pgstac/tests/pgtap.sql b/src/pgstac/tests/pgtap.sql index a2819eaa..8e9adfad 100644 --- a/src/pgstac/tests/pgtap.sql +++ b/src/pgstac/tests/pgtap.sql @@ -34,7 +34,7 @@ SELECT plan(229); \i tests/pgtap/004a_collectionsearch.sql \i tests/pgtap/005_tileutils.sql \i tests/pgtap/006_tilesearch.sql -\i tests/pgtap/999_version.sql +\i tests/pgtap/999_testreadonly.sql -- Finish the tests and clean up. SELECT * FROM finish(); diff --git a/src/pgstac/tests/pgtap/001_core.sql b/src/pgstac/tests/pgtap/001_core.sql index d7da02df..d4aa9d58 100644 --- a/src/pgstac/tests/pgtap/001_core.sql +++ b/src/pgstac/tests/pgtap/001_core.sql @@ -27,37 +27,6 @@ SELECT lives_ok( 'Search works with readonly mode set to off in readwrite mode.' ); -SET transaction_read_only TO 'on'; - -SELECT results_eq( - $$ SHOW transaction_read_only; $$, - $$ SELECT 'on'; $$, - 'Transaction set to read only' -); - -SELECT throws_ok( - $$ SELECT search('{}'); $$, - '25006' -); - -SET pgstac.readonly to 'true'; -SELECT results_eq( - $$ SELECT pgstac.readonly(); $$, - $$ SELECT TRUE; $$, - 'Readonly is set to true' -); - -SELECT lives_ok( - $$ SELECT search('{}'); $$, - 'Search works with readonly mode set to on in readonly mode.' -); - -SET pgstac.context TO 'on'; -SELECT lives_ok( - $$ SELECT search('{}'); $$, - 'Search works with readonly mode set to on in readonly mode and the context extension enabled.' -); -RESET transaction_read_only; RESET pgstac.readonly; RESET pgstac.context; SELECT is_definer('update_partition_stats'); diff --git a/src/pgstac/tests/pgtap/004_search.sql b/src/pgstac/tests/pgtap/004_search.sql index cbbc6678..b9507715 100644 --- a/src/pgstac/tests/pgtap/004_search.sql +++ b/src/pgstac/tests/pgtap/004_search.sql @@ -705,6 +705,12 @@ SELECT * FROM pg_temp.testpaging('desc','asc'); \copy items_staging (content) FROM 'tests/testdata/items_duplicate_ids.ndjson' +SELECT is( + (SELECT count(*) FROM items WHERE id='pgstac-test-item-duplicated'), + '2', + 'Check count of items with dupe id' +); + SELECT is( (SELECT jsonb_array_length(search('{"ids": ["pgstac-test-item-duplicated"]}')->'features')), '2', diff --git a/src/pgstac/tests/pgtap/999_testreadonly.sql b/src/pgstac/tests/pgtap/999_testreadonly.sql new file mode 100644 index 00000000..cc5333db --- /dev/null +++ b/src/pgstac/tests/pgtap/999_testreadonly.sql @@ -0,0 +1,30 @@ +SET transaction_read_only TO 'on'; + +SELECT results_eq( + $$ SHOW transaction_read_only; $$, + $$ SELECT 'on'; $$, + 'Transaction set to read only' +); + +SELECT throws_ok( + $$ SELECT search('{}'); $$, + '25006' +); + +SET pgstac.readonly to 'true'; +SELECT results_eq( + $$ SELECT pgstac.readonly(); $$, + $$ SELECT TRUE; $$, + 'Readonly is set to true' +); + +SELECT lives_ok( + $$ SELECT search('{}'); $$, + 'Search works with readonly mode set to on in readonly mode.' +); + +SET pgstac.context TO 'on'; +SELECT lives_ok( + $$ SELECT search('{}'); $$, + 'Search works with readonly mode set to on in readonly mode and the context extension enabled.' +); diff --git a/src/pgstac/tests/pgtap/999_version.sql b/src/pgstac/tests/pgtap/999_version.sql deleted file mode 100644 index e69de29b..00000000