diff --git a/.github/workflows/format-tests.yml b/.github/workflows/format-tests.yml index 765a7b6d5..d575e8690 100644 --- a/.github/workflows/format-tests.yml +++ b/.github/workflows/format-tests.yml @@ -78,7 +78,7 @@ jobs: empty_target_directory="$(find "${{ runner.temp }}/tmp-empty" -maxdepth 0 -type d -empty)" [[ -n "$empty_target_directory" ]] - - uses: actions/upload-artifact@v3.1.3 + - uses: actions/upload-artifact@v4.6.0 with: name: gdal-output path: ${{ runner.temp }}/*.tiff diff --git a/scripts/files/file_tiff.py b/scripts/files/file_tiff.py index aa359e40c..3d8b8a2c1 100644 --- a/scripts/files/file_tiff.py +++ b/scripts/files/file_tiff.py @@ -6,8 +6,7 @@ from urllib.parse import unquote from scripts.gdal.gdal_helper import GDALExecutionException, gdal_info, run_gdal -from scripts.gdal.gdal_preset import DEFAULT_NO_DATA_VALUE -from scripts.gdal.gdal_presets import Preset +from scripts.gdal.gdal_presets import DEFAULT_NO_DATA_VALUE, Preset from scripts.gdal.gdalinfo import GdalInfo diff --git a/scripts/gdal/gdal_preset.py b/scripts/gdal/gdal_commands.py similarity index 69% rename from scripts/gdal/gdal_preset.py rename to scripts/gdal/gdal_commands.py index 7f4033619..ecad86dea 100644 --- a/scripts/gdal/gdal_preset.py +++ b/scripts/gdal/gdal_commands.py @@ -1,92 +1,17 @@ -from decimal import Decimal -from typing import Annotated - from linz_logger import get_log from scripts.gdal.gdal_bands import get_gdal_band_offset -from scripts.gdal.gdal_presets import Preset +from scripts.gdal.gdal_presets import ( + BASE_COG, + COMPRESS_LZW, + COMPRESS_WEBP_LOSSLESS, + DEM_LERC, + SCALE_254_ADD_NO_DATA, + WEBP_OVERVIEWS, + Preset, +) from scripts.gdal.gdalinfo import GdalInfo -SCALE_254_ADD_NO_DATA = ["-scale", "0", "255", "0", "254", "-a_nodata", "255"] -""" Scale imagery from 0-255 to 0-254 then set 255 as NO_DATA. -Useful for imagery that does not have a alpha band. -""" - -BASE_COG = [ - # Suppress progress monitor and other non-error output. - "-q", - # Output to a COG - "-of", - "COG", - "-stats", - # Tile the image int 512x512px images - "-co", - "blocksize=512", - # Ensure all CPUs are used for gdal translate - "-co", - "num_threads=all_cpus", - # If not all tiles are needed in the tiff, instead of writing empty images write a null byte - # this significantly reduces the size of tiffs which are very sparse - "-co", - "sparse_ok=true", - # Do not create BIGTIFF - # An error will be raise by GDAL if it fails creating a tiff > 4GB in size - "-co", - "bigtiff=no", - # Always ignore existing overviews so they are not created from already compressed overviews - "-co", - "overviews=ignore_existing", -] - -DEFAULT_NO_DATA_VALUE: Annotated[Decimal, "From the New Zealand National Aerial LiDAR Base Specification"] = Decimal(-9999) -DEM_LERC = [ - "-co", - "compress=lerc", - "-co", - # Set Max Z Error to 1mm - "max_z_error=0.001", - "-co", - # Set MAX Z ERROR OVERVIEW to 10cm - "max_z_error_overview=0.1", - # Force all DEMS to AREA to be consistent - # input tiffs vary between AREA or POINT - "-mo", - "AREA_OR_POINT=Area", - "-a_nodata", - str(DEFAULT_NO_DATA_VALUE), -] - -COMPRESS_LZW = [ - # Compress as LZW - "-co", - "compress=lzw", - # Predictor creates smaller files, for RGB imagery - "-co", - "predictor=2", -] - -COMPRESS_WEBP_LOSSLESS = [ - # Compress into webp - "-co", - "compress=webp", - # Compress losslessly - "-co", - "quality=100", -] - -WEBP_OVERVIEWS = [ - # When creating overviews also compress them into Webp - "-co", - "overview_compress=webp", - # When resampling overviews use lanczos - # see https://github.com/linz/basemaps/blob/master/docs/imagery/cog.quality.md - "-co", - "overview_resampling=lanczos", - # Reduce quality of overviews to 90% - "-co", - "overview_quality=90", -] - def get_gdal_command(preset: str, epsg: int) -> list[str]: """Build a `gdal_translate` command based on the `preset`, `epsg` code, with conversion to 8bits if required. diff --git a/scripts/gdal/gdal_presets.py b/scripts/gdal/gdal_presets.py index 655df4f46..0b833df59 100644 --- a/scripts/gdal/gdal_presets.py +++ b/scripts/gdal/gdal_presets.py @@ -1,4 +1,82 @@ +from decimal import Decimal from enum import Enum +from typing import Annotated + +DEFAULT_NO_DATA_VALUE: Annotated[Decimal, "From the New Zealand National Aerial LiDAR Base Specification"] = Decimal(-9999) + +SCALE_254_ADD_NO_DATA = ["-scale", "0", "255", "0", "254", "-a_nodata", "255"] +""" Scale imagery from 0-255 to 0-254 then set 255 as NO_DATA. +Useful for imagery that does not have a alpha band. +""" +BASE_COG = [ + # Suppress progress monitor and other non-error output. + "-q", + # Output to a COG + "-of", + "COG", + "-stats", + # Tile the image int 512x512px images + "-co", + "blocksize=512", + # Ensure all CPUs are used for gdal translate + "-co", + "num_threads=all_cpus", + # If not all tiles are needed in the tiff, instead of writing empty images write a null byte + # this significantly reduces the size of tiffs which are very sparse + "-co", + "sparse_ok=true", + # Do not create BIGTIFF + # An error will be raise by GDAL if it fails creating a tiff > 4GB in size + "-co", + "bigtiff=no", + # Always ignore existing overviews so they are not created from already compressed overviews + "-co", + "overviews=ignore_existing", +] +DEM_LERC = [ + "-co", + "compress=lerc", + "-co", + # Set Max Z Error to 1mm + "max_z_error=0.001", + "-co", + # Set MAX Z ERROR OVERVIEW to 10cm + "max_z_error_overview=0.1", + # Force all DEMS to AREA to be consistent + # input tiffs vary between AREA or POINT + "-mo", + "AREA_OR_POINT=Area", + "-a_nodata", + str(DEFAULT_NO_DATA_VALUE), +] +COMPRESS_LZW = [ + # Compress as LZW + "-co", + "compress=lzw", + # Predictor creates smaller files, for RGB imagery + "-co", + "predictor=2", +] +COMPRESS_WEBP_LOSSLESS = [ + # Compress into webp + "-co", + "compress=webp", + # Compress losslessly + "-co", + "quality=100", +] +WEBP_OVERVIEWS = [ + # When creating overviews also compress them into Webp + "-co", + "overview_compress=webp", + # When resampling overviews use lanczos + # see https://github.com/linz/basemaps/blob/master/docs/operator-guide/cog-quality.md + "-co", + "overview_resampling=lanczos", + # Reduce quality of overviews to 90% + "-co", + "overview_quality=90", +] class Preset(str, Enum): diff --git a/scripts/gdal/tests/gdal_preset_test.py b/scripts/gdal/tests/gdal_commands_test.py similarity index 98% rename from scripts/gdal/tests/gdal_preset_test.py rename to scripts/gdal/tests/gdal_commands_test.py index 0ce05fd2d..99a54281d 100644 --- a/scripts/gdal/tests/gdal_preset_test.py +++ b/scripts/gdal/tests/gdal_commands_test.py @@ -1,7 +1,7 @@ from pytest_subtests import SubTests +from scripts.gdal.gdal_commands import get_cutline_command, get_gdal_command from scripts.gdal.gdal_helper import EpsgNumber -from scripts.gdal.gdal_preset import get_cutline_command, get_gdal_command from scripts.gdal.gdal_presets import Preset diff --git a/scripts/stac/imagery/item.py b/scripts/stac/imagery/item.py index 8a9f46dc8..56be74f95 100644 --- a/scripts/stac/imagery/item.py +++ b/scripts/stac/imagery/item.py @@ -50,15 +50,15 @@ def from_file(cls, file_name: str) -> "ImageryItem": ImageryItem: The new ImageryItem. """ file_content = read(file_name) - stac_dict_from_s3 = json.loads(file_content.decode("UTF-8")) - if (bbox := stac_dict_from_s3.get("bbox")) is not None: - stac_dict_from_s3["bbox"] = tuple(bbox) + stac_from_file = json.loads(file_content.decode("UTF-8")) + if (bbox := stac_from_file.get("bbox")) is not None: + stac_from_file["bbox"] = tuple(bbox) new_item = cls( - id_=stac_dict_from_s3["id"], - stac_asset=stac_dict_from_s3["assets"]["visual"], - stac_processing=stac_dict_from_s3["properties"], + id_=stac_from_file["id"], + stac_asset=stac_from_file["assets"]["visual"], + stac_processing=stac_from_file["properties"], ) - new_item.stac = stac_dict_from_s3 + new_item.stac = stac_from_file return new_item diff --git a/scripts/standardising.py b/scripts/standardising.py index 3c990a45e..4d7ce4ca7 100644 --- a/scripts/standardising.py +++ b/scripts/standardising.py @@ -13,14 +13,14 @@ from scripts.files.files_helper import SUFFIX_FOOTPRINT, ContentType, is_tiff from scripts.files.fs import exists, read, write, write_all, write_sidecars from scripts.gdal.gdal_bands import get_gdal_band_offset -from scripts.gdal.gdal_helper import EpsgNumber, gdal_info, run_gdal -from scripts.gdal.gdal_preset import ( +from scripts.gdal.gdal_commands import ( get_alpha_command, get_build_vrt_command, get_cutline_command, get_gdal_command, get_transform_srs_command, ) +from scripts.gdal.gdal_helper import EpsgNumber, gdal_info, run_gdal from scripts.logging.time_helper import time_in_ms from scripts.stac.imagery.capture_area import get_buffer_distance from scripts.tile.tile_index import Bounds, get_bounds_from_name diff --git a/scripts/thumbnails.py b/scripts/thumbnails.py index 23c3a9857..b941f751e 100644 --- a/scripts/thumbnails.py +++ b/scripts/thumbnails.py @@ -10,8 +10,8 @@ from scripts.files.files_helper import ContentType, get_file_name_from_path, is_tiff from scripts.files.fs import exists, read, write from scripts.gdal import gdal_helper +from scripts.gdal.gdal_commands import get_thumbnail_command from scripts.gdal.gdal_helper import is_geotiff, run_gdal -from scripts.gdal.gdal_preset import get_thumbnail_command from scripts.logging.time_helper import time_in_ms diff --git a/scripts/translate_ascii.py b/scripts/translate_ascii.py index c1e7bbd31..ba0636817 100644 --- a/scripts/translate_ascii.py +++ b/scripts/translate_ascii.py @@ -10,8 +10,8 @@ from scripts.cli.cli_helper import is_argo from scripts.files.files_helper import get_file_name_from_path from scripts.files.fs import read, write_all, write_sidecars +from scripts.gdal.gdal_commands import get_ascii_translate_command from scripts.gdal.gdal_helper import run_gdal -from scripts.gdal.gdal_preset import get_ascii_translate_command from scripts.logging.time_helper import time_in_ms