Merge branch 'master' into feat/update-collection

linz · Feb 4, 2025 · a3dd3cd · a3dd3cd
2 parents de61f27 + 7b0f43a
commit a3dd3cd
Show file tree

Hide file tree

Showing 9 changed files with 101 additions and 99 deletions.
diff --git a/.github/workflows/format-tests.yml b/.github/workflows/format-tests.yml
@@ -78,7 +78,7 @@ jobs:
           empty_target_directory="$(find "${{ runner.temp }}/tmp-empty" -maxdepth 0 -type d -empty)"
           [[ -n "$empty_target_directory" ]]
 
-      - uses: actions/upload-artifact@v3.1.3
+      - uses: actions/upload-artifact@v4.6.0
         with:
           name: gdal-output
           path: ${{ runner.temp }}/*.tiff

diff --git a/scripts/files/file_tiff.py b/scripts/files/file_tiff.py
@@ -6,8 +6,7 @@
 from urllib.parse import unquote
 
 from scripts.gdal.gdal_helper import GDALExecutionException, gdal_info, run_gdal
-from scripts.gdal.gdal_preset import DEFAULT_NO_DATA_VALUE
-from scripts.gdal.gdal_presets import Preset
+from scripts.gdal.gdal_presets import DEFAULT_NO_DATA_VALUE, Preset
 from scripts.gdal.gdalinfo import GdalInfo
 
 

diff --git a/scripts/gdal/gdal_preset.py → scripts/gdal/gdal_commands.py b/scripts/gdal/gdal_preset.py → scripts/gdal/gdal_commands.py
@@ -1,92 +1,17 @@
-from decimal import Decimal
-from typing import Annotated
-
 from linz_logger import get_log
 
 from scripts.gdal.gdal_bands import get_gdal_band_offset
-from scripts.gdal.gdal_presets import Preset
+from scripts.gdal.gdal_presets import (
+    BASE_COG,
+    COMPRESS_LZW,
+    COMPRESS_WEBP_LOSSLESS,
+    DEM_LERC,
+    SCALE_254_ADD_NO_DATA,
+    WEBP_OVERVIEWS,
+    Preset,
+)
 from scripts.gdal.gdalinfo import GdalInfo
 
-SCALE_254_ADD_NO_DATA = ["-scale", "0", "255", "0", "254", "-a_nodata", "255"]
-""" Scale imagery from 0-255 to 0-254 then set 255 as NO_DATA. 
-Useful for imagery that does not have a alpha band.
-"""
-
-BASE_COG = [
-    # Suppress progress monitor and other non-error output.
-    "-q",
-    # Output to a COG
-    "-of",
-    "COG",
-    "-stats",
-    # Tile the image int 512x512px images
-    "-co",
-    "blocksize=512",
-    # Ensure all CPUs are used for gdal translate
-    "-co",
-    "num_threads=all_cpus",
-    # If not all tiles are needed in the tiff, instead of writing empty images write a null byte
-    # this significantly reduces the size of tiffs which are very sparse
-    "-co",
-    "sparse_ok=true",
-    # Do not create BIGTIFF
-    # An error will be raise by GDAL if it fails creating a tiff > 4GB in size
-    "-co",
-    "bigtiff=no",
-    # Always ignore existing overviews so they are not created from already compressed overviews
-    "-co",
-    "overviews=ignore_existing",
-]
-
-DEFAULT_NO_DATA_VALUE: Annotated[Decimal, "From the New Zealand National Aerial LiDAR Base Specification"] = Decimal(-9999)
-DEM_LERC = [
-    "-co",
-    "compress=lerc",
-    "-co",
-    # Set Max Z Error to 1mm
-    "max_z_error=0.001",
-    "-co",
-    # Set MAX Z ERROR OVERVIEW to 10cm
-    "max_z_error_overview=0.1",
-    # Force all DEMS to AREA to be consistent
-    # input tiffs vary between AREA or POINT
-    "-mo",
-    "AREA_OR_POINT=Area",
-    "-a_nodata",
-    str(DEFAULT_NO_DATA_VALUE),
-]
-
-COMPRESS_LZW = [
-    # Compress as LZW
-    "-co",
-    "compress=lzw",
-    # Predictor creates smaller files, for RGB imagery
-    "-co",
-    "predictor=2",
-]
-
-COMPRESS_WEBP_LOSSLESS = [
-    # Compress into webp
-    "-co",
-    "compress=webp",
-    # Compress losslessly
-    "-co",
-    "quality=100",
-]
-
-WEBP_OVERVIEWS = [
-    # When creating overviews also compress them into Webp
-    "-co",
-    "overview_compress=webp",
-    # When resampling overviews use lanczos
-    # see https://github.com/linz/basemaps/blob/master/docs/imagery/cog.quality.md
-    "-co",
-    "overview_resampling=lanczos",
-    # Reduce quality of overviews to 90%
-    "-co",
-    "overview_quality=90",
-]
-
 
 def get_gdal_command(preset: str, epsg: int) -> list[str]:
     """Build a `gdal_translate` command based on the `preset`, `epsg` code, with conversion to 8bits if required.

diff --git a/scripts/gdal/gdal_presets.py b/scripts/gdal/gdal_presets.py
@@ -1,4 +1,82 @@
+from decimal import Decimal
 from enum import Enum
+from typing import Annotated
+
+DEFAULT_NO_DATA_VALUE: Annotated[Decimal, "From the New Zealand National Aerial LiDAR Base Specification"] = Decimal(-9999)
+
+SCALE_254_ADD_NO_DATA = ["-scale", "0", "255", "0", "254", "-a_nodata", "255"]
+""" Scale imagery from 0-255 to 0-254 then set 255 as NO_DATA. 
+Useful for imagery that does not have a alpha band.
+"""
+BASE_COG = [
+    # Suppress progress monitor and other non-error output.
+    "-q",
+    # Output to a COG
+    "-of",
+    "COG",
+    "-stats",
+    # Tile the image int 512x512px images
+    "-co",
+    "blocksize=512",
+    # Ensure all CPUs are used for gdal translate
+    "-co",
+    "num_threads=all_cpus",
+    # If not all tiles are needed in the tiff, instead of writing empty images write a null byte
+    # this significantly reduces the size of tiffs which are very sparse
+    "-co",
+    "sparse_ok=true",
+    # Do not create BIGTIFF
+    # An error will be raise by GDAL if it fails creating a tiff > 4GB in size
+    "-co",
+    "bigtiff=no",
+    # Always ignore existing overviews so they are not created from already compressed overviews
+    "-co",
+    "overviews=ignore_existing",
+]
+DEM_LERC = [
+    "-co",
+    "compress=lerc",
+    "-co",
+    # Set Max Z Error to 1mm
+    "max_z_error=0.001",
+    "-co",
+    # Set MAX Z ERROR OVERVIEW to 10cm
+    "max_z_error_overview=0.1",
+    # Force all DEMS to AREA to be consistent
+    # input tiffs vary between AREA or POINT
+    "-mo",
+    "AREA_OR_POINT=Area",
+    "-a_nodata",
+    str(DEFAULT_NO_DATA_VALUE),
+]
+COMPRESS_LZW = [
+    # Compress as LZW
+    "-co",
+    "compress=lzw",
+    # Predictor creates smaller files, for RGB imagery
+    "-co",
+    "predictor=2",
+]
+COMPRESS_WEBP_LOSSLESS = [
+    # Compress into webp
+    "-co",
+    "compress=webp",
+    # Compress losslessly
+    "-co",
+    "quality=100",
+]
+WEBP_OVERVIEWS = [
+    # When creating overviews also compress them into Webp
+    "-co",
+    "overview_compress=webp",
+    # When resampling overviews use lanczos
+    # see https://github.com/linz/basemaps/blob/master/docs/operator-guide/cog-quality.md
+    "-co",
+    "overview_resampling=lanczos",
+    # Reduce quality of overviews to 90%
+    "-co",
+    "overview_quality=90",
+]
 
 
 class Preset(str, Enum):

diff --git a/scripts/gdal/tests/gdal_preset_test.py → scripts/gdal/tests/gdal_commands_test.py b/scripts/gdal/tests/gdal_preset_test.py → scripts/gdal/tests/gdal_commands_test.py
@@ -1,7 +1,7 @@
 from pytest_subtests import SubTests
 
+from scripts.gdal.gdal_commands import get_cutline_command, get_gdal_command
 from scripts.gdal.gdal_helper import EpsgNumber
-from scripts.gdal.gdal_preset import get_cutline_command, get_gdal_command
 from scripts.gdal.gdal_presets import Preset
 
 

diff --git a/scripts/stac/imagery/item.py b/scripts/stac/imagery/item.py
@@ -50,15 +50,15 @@ def from_file(cls, file_name: str) -> "ImageryItem":
             ImageryItem: The new ImageryItem.
         """
         file_content = read(file_name)
-        stac_dict_from_s3 = json.loads(file_content.decode("UTF-8"))
-        if (bbox := stac_dict_from_s3.get("bbox")) is not None:
-            stac_dict_from_s3["bbox"] = tuple(bbox)
+        stac_from_file = json.loads(file_content.decode("UTF-8"))
+        if (bbox := stac_from_file.get("bbox")) is not None:
+            stac_from_file["bbox"] = tuple(bbox)
         new_item = cls(
-            id_=stac_dict_from_s3["id"],
-            stac_asset=stac_dict_from_s3["assets"]["visual"],
-            stac_processing=stac_dict_from_s3["properties"],
+            id_=stac_from_file["id"],
+            stac_asset=stac_from_file["assets"]["visual"],
+            stac_processing=stac_from_file["properties"],
         )
-        new_item.stac = stac_dict_from_s3
+        new_item.stac = stac_from_file
 
         return new_item
 

diff --git a/scripts/standardising.py b/scripts/standardising.py
@@ -13,14 +13,14 @@
 from scripts.files.files_helper import SUFFIX_FOOTPRINT, ContentType, is_tiff
 from scripts.files.fs import exists, read, write, write_all, write_sidecars
 from scripts.gdal.gdal_bands import get_gdal_band_offset
-from scripts.gdal.gdal_helper import EpsgNumber, gdal_info, run_gdal
-from scripts.gdal.gdal_preset import (
+from scripts.gdal.gdal_commands import (
     get_alpha_command,
     get_build_vrt_command,
     get_cutline_command,
     get_gdal_command,
     get_transform_srs_command,
 )
+from scripts.gdal.gdal_helper import EpsgNumber, gdal_info, run_gdal
 from scripts.logging.time_helper import time_in_ms
 from scripts.stac.imagery.capture_area import get_buffer_distance
 from scripts.tile.tile_index import Bounds, get_bounds_from_name

diff --git a/scripts/thumbnails.py b/scripts/thumbnails.py
@@ -10,8 +10,8 @@
 from scripts.files.files_helper import ContentType, get_file_name_from_path, is_tiff
 from scripts.files.fs import exists, read, write
 from scripts.gdal import gdal_helper
+from scripts.gdal.gdal_commands import get_thumbnail_command
 from scripts.gdal.gdal_helper import is_geotiff, run_gdal
-from scripts.gdal.gdal_preset import get_thumbnail_command
 from scripts.logging.time_helper import time_in_ms
 
 

diff --git a/scripts/translate_ascii.py b/scripts/translate_ascii.py
@@ -10,8 +10,8 @@
 from scripts.cli.cli_helper import is_argo
 from scripts.files.files_helper import get_file_name_from_path
 from scripts.files.fs import read, write_all, write_sidecars
+from scripts.gdal.gdal_commands import get_ascii_translate_command
 from scripts.gdal.gdal_helper import run_gdal
-from scripts.gdal.gdal_preset import get_ascii_translate_command
 from scripts.logging.time_helper import time_in_ms