Merge branch 'master' into ci/upload-files-when-failure

.github/dependabot.yml

@@ -1,5 +1,9 @@
 version: 2
 updates:
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      interval: daily
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/containers.yml

@@ -17,7 +17,7 @@ jobs:
       AWS_CI_ROLE: ${{ secrets.AWS_CI_ROLE }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           fetch-depth: 0
 
@@ -30,27 +30,27 @@ jobs:
           { echo "version=${GIT_VERSION}"; echo "version_major=${GIT_VERSION_MAJOR}"; echo "version_major_minor=${GIT_VERSION_MAJOR_MINOR}"; } >> "$GITHUB_OUTPUT"
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v2
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
         with:
           images: ${{ github.repository }}
           labels: |
             org.opencontainers.image.version=${{ steps.version.outputs.version }}
 
       - name: Login to GitHub Container Registry
         if: ${{(github.ref == 'refs/heads/master') && !(startsWith(github.event.head_commit.message, 'release:'))}}
-        uses: docker/login-action@v2
+        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Configure AWS Credentials
         if: ${{env.AWS_CI_ROLE != '' && (github.ref == 'refs/heads/master') && !(startsWith(github.event.head_commit.message, 'release:'))}}
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2
         with:
           aws-region: ap-southeast-2
           mask-aws-account-id: true
@@ -59,11 +59,11 @@ jobs:
       - name: Login to Amazon ECR
         if: ${{env.AWS_CI_ROLE != '' && (github.ref == 'refs/heads/master') && !(startsWith(github.event.head_commit.message, 'release:'))}}
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1
 
       - name: Setup docker tags
         id: tags
-        uses: actions/github-script@v6
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6
         with:
           result-encoding: string
           script: |
@@ -77,7 +77,7 @@ jobs:
             return tags.join(', ')
 
       - name: Build and push container
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4
         with:
           context: .
           tags: ${{ steps.tags.outputs.result }}

.github/workflows/format-tests.yml

@@ -5,10 +5,10 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
 
       - name: Use Python "3.10.6"
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4
         with:
           python-version: "3.10.6"
       - name: Install
@@ -63,6 +63,12 @@ jobs:
           docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 translate_ascii.py --from-file ./tests/data/elevation_ascii.json --target /tmp/
           cmp --silent "${{ runner.temp }}/elevation_ascii.tiff" ./scripts/tests/data/output/elevation_ascii.tiff
 
+      - name: End to end test - Remove empty files
+        run: |
+          docker run  -v "${{ runner.temp }}/tmp-empty/:/tmp/" topo-imagery python3 standardise_validate.py --from-file=./tests/data/empty.json --preset=webp --target-epsg=2193 --source-epsg=2193 --target=/tmp --collection-id=123 --start-datetime=2023-01-01 --end-datetime=2023-01-01
+          empty_target_directory="$(find "${{ runner.temp }}/tmp-empty" -maxdepth 0 -type d -empty)"
+          [[ -n "$empty_target_directory" ]]
+
       - uses: actions/[email protected]
         with:
           name: gdal-output

.github/workflows/release-please.yml

@@ -13,7 +13,7 @@ jobs:
     outputs:
       release_created: ${{ steps.release.outputs.release_created }}
     steps:
-      - uses: google-github-actions/release-please-action@v3
+      - uses: google-github-actions/release-please-action@db8f2c60ee802b3748b512940dde88eabd7b7e01 # v3
         id: release
         with:
           release-type: python
@@ -32,7 +32,7 @@ jobs:
     if: ${{ needs.release-please.outputs.release_created }}
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
         with:
           fetch-depth: 0
 
@@ -45,25 +45,25 @@ jobs:
           { echo "version=${GIT_VERSION}"; echo "version_major=${GIT_VERSION_MAJOR}"; echo "version_major_minor=${GIT_VERSION_MAJOR_MINOR}"; } >> "$GITHUB_OUTPUT"
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@885d1462b80bc1c1c7f0b00334ad271f09369c55 # v2
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v4
         with:
           images: ${{ github.repository }}
           labels: |
             org.opencontainers.image.version=${{ steps.version.outputs.version }}
 
       - name: Login to GitHub Container Registry
-        uses: docker/login-action@v2
+        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v2
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2
         if: ${{env.AWS_CI_ROLE != ''}}
         with:
           aws-region: ap-southeast-2
@@ -72,12 +72,12 @@ jobs:
 
       - name: Login to Amazon ECR
         id: login-ecr
-        uses: aws-actions/amazon-ecr-login@v1
+        uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d # v1
         if: ${{env.AWS_CI_ROLE != ''}}
 
       - name: Setup docker tags
         id: tags
-        uses: actions/github-script@v6
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6
         with:
           result-encoding: string
           script: |
@@ -95,7 +95,7 @@ jobs:
             return tags.join(', ')
 
       - name: Build and push container
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 # v4
         with:
           context: .
           tags: ${{ steps.tags.outputs.result }}

CHANGELOG.md

@@ -1,5 +1,42 @@
 # Changelog
 
+## [3.5.2](https://github.com/linz/topo-imagery/compare/v3.5.1...v3.5.2) (2023-12-21)
+
+
+### Bug Fixes
+
+* minor changes to be consistent with current data ([#790](https://github.com/linz/topo-imagery/issues/790)) ([4892b6b](https://github.com/linz/topo-imagery/commit/4892b6b30dfc5cb7cc3a00a5eef52b6fde8447ee))
+
+## [3.5.1](https://github.com/linz/topo-imagery/compare/v3.5.0...v3.5.1) (2023-12-19)
+
+
+### Bug Fixes
+
+* typo in prod ([#788](https://github.com/linz/topo-imagery/issues/788)) ([275a665](https://github.com/linz/topo-imagery/commit/275a6658f2144f2a4cf2a690a2ea2b701aafc44b))
+
+## [3.5.0](https://github.com/linz/topo-imagery/compare/v3.4.0...v3.5.0) (2023-12-19)
+
+
+### Features
+
+* Dependabot for Docker TDE-963 ([#746](https://github.com/linz/topo-imagery/issues/746)) ([250b82c](https://github.com/linz/topo-imagery/commit/250b82c196c1f3db601e9712d6834d6da28e6952))
+* Pin Docker image TDE-958 ([#744](https://github.com/linz/topo-imagery/issues/744)) ([de8a129](https://github.com/linz/topo-imagery/commit/de8a129d60bb7fc16f7872542ee93bc3ce3c9570))
+* title and description by arguments TDE-960 ([#757](https://github.com/linz/topo-imagery/issues/757)) ([dd3e282](https://github.com/linz/topo-imagery/commit/dd3e282660cc0e652667adf54fe3e5ad3b0b0bea))
+
+
+### Documentation
+
+* add explaination about collection provider tests ([#764](https://github.com/linz/topo-imagery/issues/764)) ([4e84901](https://github.com/linz/topo-imagery/commit/4e84901abe30ca0294ae570caef7fba511fe0ce5))
+
+## [3.4.0](https://github.com/linz/topo-imagery/compare/v3.3.1...v3.4.0) (2023-11-23)
+
+
+### Features
+
+* force LERC overview max Z error threshold to 0.1 (10cm) TDE-873 ([#740](https://github.com/linz/topo-imagery/issues/740)) ([0924c13](https://github.com/linz/topo-imagery/commit/0924c132fd01221cb0467cba2126dbc8ba80b269))
+* lint GitHub Actions workflows TDE-919 ([#720](https://github.com/linz/topo-imagery/issues/720)) ([1d32588](https://github.com/linz/topo-imagery/commit/1d325881607c69022dec6c93e178f85e9e05705e))
+* Pin actions to hashes ([#729](https://github.com/linz/topo-imagery/issues/729)) ([e2343d5](https://github.com/linz/topo-imagery/commit/e2343d5f1959e71d3634f134cd7d61c1447ab261))
+
 ## [3.3.1](https://github.com/linz/topo-imagery/compare/v3.3.0...v3.3.1) (2023-10-04)
 
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.2
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.0@sha256:3ca7c26ef3aff8c8d134bbaa62e13d1a4c97c8a7b2853f1db0ff34c2f939a4de
 
 RUN apt-get update
 # Install pip

pyproject.toml

@@ -25,7 +25,7 @@ ignore_missing_imports = true
 
 [tool.poetry]
 name = "topo-imagery"
-version = "3.3.1"
+version = "3.5.2"
 description = "A collection of scripts for processing imagery"
 authors = [
     "Blayne Chard <[email protected]>",
@@ -41,6 +41,7 @@ boto3 = "^1.28.70"
 linz-logger = "^0.11.0"
 py-multihash = "^2.0.1"
 shapely = "^2.0.1"
+tifffile = "^2023.12.9"
 
 [tool.poetry.dev-dependencies]
 black = "^23.11.0"

scripts/aws/aws_helper.py

@@ -4,18 +4,18 @@
 from typing import Any, Dict, List, NamedTuple, Optional
 from urllib.parse import urlparse
 
-import boto3
-import botocore
+from boto3 import Session
 from botocore.credentials import AssumeRoleCredentialFetcher, DeferredRefreshableCredentials, ReadOnlyCredentials
+from botocore.session import Session as BotocoreSession
 from linz_logger import get_log
 
 from scripts.aws.aws_credential_source import CredentialSource
 
 S3Path = NamedTuple("S3Path", [("bucket", str), ("key", str)])
 
 aws_profile = environ.get("AWS_PROFILE")
-session = boto3.Session(profile_name=aws_profile)
-sessions: Dict[str, boto3.Session] = {}
+session = Session(profile_name=aws_profile)
+sessions: Dict[str, Session] = {}
 
 bucket_roles: List[CredentialSource] = []
 
@@ -40,14 +40,14 @@ def _init_roles() -> None:
     get_log().debug("bucket_config_loaded", config=bucket_config_path, prefix_count=len(bucket_roles))
 
 
-def _get_client_creator(local_session: boto3.Session) -> Any:
+def _get_client_creator(local_session: Session) -> Any:
     def client_creator(service_name: str, **kwargs: Any) -> Any:
         return local_session.client(service_name, **kwargs)
 
     return client_creator
 
 
-def get_session(prefix: str) -> boto3.Session:
+def get_session(prefix: str) -> Session:
     """Get an AWS session to deal with an object on `s3`.
 
     Args:
@@ -78,14 +78,14 @@ def get_session(prefix: str) -> boto3.Session:
         role_arn=cfg.roleArn,
         extra_args=extra_args,
     )
-    botocore_session = botocore.session.Session()
+    botocore_session = BotocoreSession()
 
     # pylint:disable=protected-access
     botocore_session._credentials = DeferredRefreshableCredentials(
         method="assume-role", refresh_using=fetcher.fetch_credentials
     )
 
-    current_session = boto3.Session(botocore_session=botocore_session)
+    current_session = Session(botocore_session=botocore_session)
     sessions[cfg.roleArn] = current_session
 
     get_log().info("role_assume", prefix=prefix, bucket=cfg.bucket, role_arn=cfg.roleArn)