feat: generate hillshade TDE-1369 (#1253)

### Motivation

LINZ wants to offer some standard "out-of-the-box" hillshades to the
elevation data consumers. This feature gives the ability to generate two
standard hillshades from existing DEM TIFF(s):
<div align="center">

| [:world_map: Monodirectional or "Greyscale"
hillshade](https://basemaps.linz.govt.nz/@-40.6547814,176.2198739,z3.69?style=23-test-hillshade-7w5ph&i=23-test-hillshade-7w5ph&tileMatrix=NZTM2000Quad&config=6sUk4Za2WKysFTtUuDLY8VZKUUvfvkXoNXGCqB3PC6uKQH1kbhniZaS5JVmTxb8WBZ1RBvwbK3xk8XrDdC37MwSj1sn3XBLwcFxEJf1mR6fHJdoP8LYt624nphascfFHdL8v83CksZqqWKvhcGBLTpHJsL1PeuHMMuCGqUG6eNUAu9FWxajyLD&debug=true&debug.layer=basemaps-23-test-hillshade-7w5ph-color-ramp)
| [:world_map: "Igor"
hillshade](https://basemaps.linz.govt.nz/@-40.1252942,174.5734854,z5?style=24-test-hillshade-fpps6&i=24-test-hillshade-fpps6&tileMatrix=NZTM2000Quad&config=6sUk4Za2WKysFTtUuDLY8VZKUUvfvkXoNXGCqB3PC6uKQH1kbhniuZFFm2BPvcgeALCZJgJ3NRK6Y3kgEH7pKGK5LadeRw3LZDiivVeEE4zGRufribx7q1EAHzMgwvF6vQ6ZqePJHk4Gb54aXzBs4Ey8WXhgp2hUARXq1F6o1AzJ9fHe3anphw&debug=true&debug.layer=basemaps-24-test-hillshade-fpps6-color-ramp)
|
| ------------- | ------------- |
| <img
src=https://github.com/user-attachments/assets/9f22e13b-d006-4a22-a4be-877e13fc2e79
width=300> | <img
src=https://github.com/user-attachments/assets/2b6876db-8b46-44fc-9223-eb39be0ab4bf
width=300> |

</div>

### Modifications

- add a script `generate_hillshade.py`
```
usage: generate_hillshade.py [-h] --from-file FROM_FILE --preset
                             {greyscale,igor} --target TARGET [--force]

options:
  -h, --help            show this help message and exit
  --from-file FROM_FILE
                        Specify the path to a json file containing the input
                        tiffs. Format: [{'output': 'tile1', 'inputs':
                        ['path/input1.tiff', 'path/input2.tiff']}]
  --preset {greyscale,igor}
                        Type of hillshade to generate.
  --target TARGET       Specify the path to save the generated hillshade to.
  --force               Regenerate the hillshade TIFF files if already exist.
                        Defaults to False.
```

### Verification

Manual, automated end-to-end tests, and Argo Workflows runs.

.github/workflows/format-tests.yml

@@ -40,6 +40,16 @@ jobs:
           cmp --silent "${{ runner.temp }}/BK39_10000_0102.tiff" ./scripts/tests/data/output/BK39_10000_0102.tiff
           cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101.tiff
 
+      - name: End to end test - Hillshade Greyscale
+        run: |
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 generate_hillshade.py --from-file ./tests/data/hillshade.json --preset greyscale --target /tmp/ --force
+          cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101_greyscale.tiff
+
+      - name: End to end test - Hillshade Igor
+        run: |
+          docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 generate_hillshade.py --from-file ./tests/data/hillshade.json --preset igor --target /tmp/ --force
+          cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101_igor.tiff
+
       - name: End to end test - Historical Aerial Imagery
         run: |
           docker run  -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 60 --create-footprints=true --current-datetime=2010-09-18T12:34:56Z

README.md

@@ -30,7 +30,7 @@ The scripts have been implemented to be run inside the Docker container only. Th
 docker build --tag=topo-imagery .
 ```
 
-- Running `standardising_validate.py` script
+- Example: running `standardising_validate.py` script
 
 This script standardises TIFF files to [COGs](https://www.cogeo.org/) with a creation of a [STAC](https://stacspec.org/) Item file per TIFF containing the metadata.
 The input TIFF file paths have to be passed through a `json` file in the following format:

scripts/files/file_tiff.py

@@ -6,7 +6,7 @@
 from urllib.parse import unquote
 
 from scripts.gdal.gdal_helper import GDALExecutionException, gdal_info, run_gdal
-from scripts.gdal.gdal_presets import DEFAULT_NO_DATA_VALUE, Preset
+from scripts.gdal.gdal_presets import DEFAULT_NO_DATA_VALUE, CompressionPreset
 from scripts.gdal.gdalinfo import GdalInfo
 
 
@@ -50,7 +50,7 @@ def __init__(
         self._errors: list[dict[str, Any]] = []
         self._gdalinfo: GdalInfo | None = None
         self._srs: bytes | None = None
-        if preset == Preset.DEM_LERC.value:
+        if preset == CompressionPreset.DEM_LERC.value:
             self._tiff_type = FileTiffType.DEM
         else:
             self._tiff_type = FileTiffType.IMAGERY

scripts/files/tests/file_tiff_test.py

@@ -1,5 +1,5 @@
 from scripts.files.file_tiff import FileTiff, FileTiffErrorType
-from scripts.gdal.gdal_presets import Preset
+from scripts.gdal.gdal_presets import CompressionPreset
 from scripts.gdal.tests.gdalinfo import add_band, add_palette_band, fake_gdal_info
 
 
@@ -75,7 +75,7 @@ def test_check_band_count_valid_1_dem() -> None:
     gdalinfo = fake_gdal_info()
     add_band(gdalinfo)
 
-    file_tiff = FileTiff(["test"], Preset.DEM_LERC.value)
+    file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value)
     file_tiff.check_band_count(gdalinfo)
 
     assert not file_tiff.get_errors()
@@ -90,7 +90,7 @@ def test_check_band_count_invalid_alpha_dem() -> None:
     add_band(gdalinfo)
     add_band(gdalinfo, color_interpretation="Alpha")
 
-    file_tiff = FileTiff(["test"], Preset.DEM_LERC.value)
+    file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value)
     file_tiff.check_band_count(gdalinfo)
 
     assert file_tiff.get_errors()
@@ -106,7 +106,7 @@ def test_check_band_count_invalid_3_dem() -> None:
     add_band(gdalinfo)
     add_band(gdalinfo)
 
-    file_tiff = FileTiff(["test"], Preset.DEM_LERC.value)
+    file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value)
     file_tiff.check_band_count(gdalinfo)
 
     assert file_tiff.get_errors()
@@ -165,7 +165,7 @@ def test_check_color_interpretation_valid_dem() -> None:
     gdalinfo = fake_gdal_info()
     add_band(gdalinfo, color_interpretation="Gray")
 
-    file_tiff = FileTiff(["test"], Preset.DEM_LERC.value)
+    file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value)
     file_tiff.check_color_interpretation(gdalinfo)
 
     assert not file_tiff.get_errors()
@@ -180,7 +180,7 @@ def test_check_color_interpretation_invalid_dem() -> None:
     add_band(gdalinfo, color_interpretation="Green")
     add_band(gdalinfo, color_interpretation="Blue")
 
-    file_tiff = FileTiff(["test"], Preset.DEM_LERC.value)
+    file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value)
     file_tiff.check_color_interpretation(gdalinfo)
 
     assert file_tiff.get_errors()
@@ -322,7 +322,7 @@ def test_should_throw_when_encountering_non_integer_no_data_value() -> None:
     gdalinfo = fake_gdal_info()
     add_palette_band(gdalinfo, colour_table_entries=[[x, x, x, 255] for x in reversed(range(256))], no_data_value="-9999.1")
 
-    file_tiff = FileTiff(["test"], Preset.DEM_LERC.value)
+    file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value)
     file_tiff.check_no_data(gdalinfo)
 
     assert file_tiff.get_errors() == [

scripts/gdal/gdal_bands.py

@@ -1,7 +1,7 @@
 from linz_logger import get_log
 
 from scripts.gdal.gdal_helper import gdal_info
-from scripts.gdal.gdal_presets import Preset
+from scripts.gdal.gdal_presets import CompressionPreset
 from scripts.gdal.gdalinfo import GdalInfo, GdalInfoBand
 
 
@@ -44,7 +44,7 @@ def get_gdal_band_offset(file: str, info: GdalInfo | None = None, preset: str |
 
     if band_grey := find_band(bands, "Gray"):
         band_grey_index = str(band_grey["band"])
-        if preset == Preset.DEM_LERC.value:
+        if preset == CompressionPreset.DEM_LERC.value:
             # return single band if DEM/DSM
             return ["-b", band_grey_index]
         # Grey scale imagery, set R,G and B to just the grey_band

scripts/gdal/gdal_commands.py

@@ -8,7 +8,8 @@
     DEM_LERC,
     SCALE_254_ADD_NO_DATA,
     WEBP_OVERVIEWS,
-    Preset,
+    CompressionPreset,
+    HillshadePreset,
 )
 from scripts.gdal.gdalinfo import GdalInfo
 
@@ -30,16 +31,16 @@ def get_gdal_command(preset: str, epsg: int) -> list[str]:
     # Force the source projection to an input EPSG
     gdal_command.extend(["-a_srs", f"EPSG:{epsg}"])
 
-    if preset == Preset.LZW.value:
+    if preset == CompressionPreset.LZW.value:
         gdal_command.extend(SCALE_254_ADD_NO_DATA)
         gdal_command.extend(COMPRESS_LZW)
         gdal_command.extend(WEBP_OVERVIEWS)
 
-    elif preset == Preset.WEBP.value:
+    elif preset == CompressionPreset.WEBP.value:
         gdal_command.extend(COMPRESS_WEBP_LOSSLESS)
         gdal_command.extend(WEBP_OVERVIEWS)
 
-    elif preset == Preset.DEM_LERC.value:
+    elif preset == CompressionPreset.DEM_LERC.value:
         gdal_command.extend(DEM_LERC)
 
     return gdal_command
@@ -195,3 +196,38 @@ def get_ascii_translate_command() -> list[str]:
         "-co",
         "COMPRESS=lzw",
     ]
+
+
+def get_hillshade_command(preset: str) -> list[str]:
+    """Get a `gdaldem` command to create a hillshade based on the provided HillshadePreset.
+
+    Args:
+        preset: a HillshadePreset
+
+    Returns:
+        a `gdaldem` command
+    """
+    gdal_command: list[str] = [
+        "gdaldem",
+        "hillshade",
+        "-compute_edges",
+        "-of",
+        "COG",
+        "-co",
+        "COMPRESS=lerc",
+        "-co",
+        "OVERVIEW_COMPRESS=lerc",
+        "-co",
+        "MAX_Z_ERROR_OVERVIEW=0",
+        "-co",
+        "NUM_THREADS=ALL_CPUS",
+        "-co",
+        "MAX_Z_ERROR=0",
+    ]
+
+    if preset == HillshadePreset.GREYSCALE.value:
+        gdal_command.extend(["-az", "315", "-alt", "45"])
+    elif preset == HillshadePreset.IGOR.value:
+        gdal_command.extend(["-igor"])
+
+    return gdal_command

scripts/gdal/gdal_presets.py

@@ -79,7 +79,16 @@
 ]
 
 
-class Preset(str, Enum):
+class CompressionPreset(str, Enum):
+    """Enum for the different compression presets available for standardising TIFFs."""
+
     DEM_LERC = "dem_lerc"
     LZW = "lzw"
     WEBP = "webp"
+
+
+class HillshadePreset(str, Enum):
+    """Enum for the different type of hillshade available for generating from a DEM."""
+
+    GREYSCALE = "greyscale"  # Standard/default mono-directional hillshade
+    IGOR = "igor"  # Whiter hillshade (see http://maperitive.net/docs/Commands/GenerateReliefImageIgor.html)

scripts/gdal/tests/gdal_bands_test.py

@@ -1,5 +1,5 @@
 from scripts.gdal.gdal_bands import get_gdal_band_offset, get_gdal_band_type
-from scripts.gdal.gdal_presets import Preset
+from scripts.gdal.gdal_presets import CompressionPreset
 from scripts.gdal.tests.gdalinfo import add_band, add_palette_band, fake_gdal_info
 
 
@@ -26,7 +26,7 @@ def test_gdal_grey_bands_dem_detection() -> None:
     gdalinfo = fake_gdal_info()
     add_band(gdalinfo, color_interpretation="Gray")
 
-    bands = get_gdal_band_offset("some_file.tiff", gdalinfo, Preset.DEM_LERC.value)
+    bands = get_gdal_band_offset("some_file.tiff", gdalinfo, CompressionPreset.DEM_LERC.value)
 
     assert " ".join(bands) == "-b 1"
 

scripts/gdal/tests/gdal_commands_test.py

@@ -2,11 +2,11 @@
 
 from scripts.gdal.gdal_commands import get_cutline_command, get_gdal_command
 from scripts.gdal.gdal_helper import EpsgNumber
-from scripts.gdal.gdal_presets import Preset
+from scripts.gdal.gdal_presets import CompressionPreset
 
 
 def test_preset_webp(subtests: SubTests) -> None:
-    gdal_command = get_gdal_command(Preset.WEBP.value, epsg=EpsgNumber.NZTM_2000.value)
+    gdal_command = get_gdal_command(CompressionPreset.WEBP.value, epsg=EpsgNumber.NZTM_2000.value)
 
     # Basic cog creation
     with subtests.test():
@@ -46,7 +46,7 @@ def test_preset_webp(subtests: SubTests) -> None:
 
 
 def test_preset_lzw(subtests: SubTests) -> None:
-    gdal_command = get_gdal_command(Preset.LZW.value, epsg=EpsgNumber.NZTM_2000.value)
+    gdal_command = get_gdal_command(CompressionPreset.LZW.value, epsg=EpsgNumber.NZTM_2000.value)
 
     # Basic cog creation
     with subtests.test():
@@ -86,7 +86,7 @@ def test_preset_lzw(subtests: SubTests) -> None:
 
 
 def test_preset_dem_lerc(subtests: SubTests) -> None:
-    gdal_command = get_gdal_command(Preset.DEM_LERC.value, epsg=EpsgNumber.NZTM_2000.value)
+    gdal_command = get_gdal_command(CompressionPreset.DEM_LERC.value, epsg=EpsgNumber.NZTM_2000.value)
     # Basic cog creation
     with subtests.test():
         assert "COG" in gdal_command