diff --git a/.github/workflows/format-tests.yml b/.github/workflows/format-tests.yml index d575e8690..5171bd959 100644 --- a/.github/workflows/format-tests.yml +++ b/.github/workflows/format-tests.yml @@ -40,6 +40,16 @@ jobs: cmp --silent "${{ runner.temp }}/BK39_10000_0102.tiff" ./scripts/tests/data/output/BK39_10000_0102.tiff cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101.tiff + - name: End to end test - Hillshade Greyscale + run: | + docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 generate_hillshade.py --from-file ./tests/data/hillshade.json --preset greyscale --target /tmp/ --force + cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101_greyscale.tiff + + - name: End to end test - Hillshade Igor + run: | + docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 generate_hillshade.py --from-file ./tests/data/hillshade.json --preset igor --target /tmp/ --force + cmp --silent "${{ runner.temp }}/BK39_10000_0101.tiff" ./scripts/tests/data/output/BK39_10000_0101_igor.tiff + - name: End to end test - Historical Aerial Imagery run: | docker run -v "${{ runner.temp }}:/tmp/" topo-imagery python3 standardise_validate.py --from-file ./tests/data/hi.json --preset webp --target-epsg 2193 --source-epsg 2193 --target /tmp/ --collection-id 123 --start-datetime 2023-01-01 --end-datetime 2023-01-01 --gsd 60 --create-footprints=true --current-datetime=2010-09-18T12:34:56Z diff --git a/README.md b/README.md index 18e352af9..0dc1f2451 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ The scripts have been implemented to be run inside the Docker container only. Th docker build --tag=topo-imagery . ``` -- Running `standardising_validate.py` script +- Example: running `standardising_validate.py` script This script standardises TIFF files to [COGs](https://www.cogeo.org/) with a creation of a [STAC](https://stacspec.org/) Item file per TIFF containing the metadata. The input TIFF file paths have to be passed through a `json` file in the following format: diff --git a/scripts/files/file_tiff.py b/scripts/files/file_tiff.py index 3d8b8a2c1..5eae4177a 100644 --- a/scripts/files/file_tiff.py +++ b/scripts/files/file_tiff.py @@ -6,7 +6,7 @@ from urllib.parse import unquote from scripts.gdal.gdal_helper import GDALExecutionException, gdal_info, run_gdal -from scripts.gdal.gdal_presets import DEFAULT_NO_DATA_VALUE, Preset +from scripts.gdal.gdal_presets import DEFAULT_NO_DATA_VALUE, CompressionPreset from scripts.gdal.gdalinfo import GdalInfo @@ -50,7 +50,7 @@ def __init__( self._errors: list[dict[str, Any]] = [] self._gdalinfo: GdalInfo | None = None self._srs: bytes | None = None - if preset == Preset.DEM_LERC.value: + if preset == CompressionPreset.DEM_LERC.value: self._tiff_type = FileTiffType.DEM else: self._tiff_type = FileTiffType.IMAGERY diff --git a/scripts/files/tests/file_tiff_test.py b/scripts/files/tests/file_tiff_test.py index 7d6571a2e..fd5b950f2 100644 --- a/scripts/files/tests/file_tiff_test.py +++ b/scripts/files/tests/file_tiff_test.py @@ -1,5 +1,5 @@ from scripts.files.file_tiff import FileTiff, FileTiffErrorType -from scripts.gdal.gdal_presets import Preset +from scripts.gdal.gdal_presets import CompressionPreset from scripts.gdal.tests.gdalinfo import add_band, add_palette_band, fake_gdal_info @@ -75,7 +75,7 @@ def test_check_band_count_valid_1_dem() -> None: gdalinfo = fake_gdal_info() add_band(gdalinfo) - file_tiff = FileTiff(["test"], Preset.DEM_LERC.value) + file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value) file_tiff.check_band_count(gdalinfo) assert not file_tiff.get_errors() @@ -90,7 +90,7 @@ def test_check_band_count_invalid_alpha_dem() -> None: add_band(gdalinfo) add_band(gdalinfo, color_interpretation="Alpha") - file_tiff = FileTiff(["test"], Preset.DEM_LERC.value) + file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value) file_tiff.check_band_count(gdalinfo) assert file_tiff.get_errors() @@ -106,7 +106,7 @@ def test_check_band_count_invalid_3_dem() -> None: add_band(gdalinfo) add_band(gdalinfo) - file_tiff = FileTiff(["test"], Preset.DEM_LERC.value) + file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value) file_tiff.check_band_count(gdalinfo) assert file_tiff.get_errors() @@ -165,7 +165,7 @@ def test_check_color_interpretation_valid_dem() -> None: gdalinfo = fake_gdal_info() add_band(gdalinfo, color_interpretation="Gray") - file_tiff = FileTiff(["test"], Preset.DEM_LERC.value) + file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value) file_tiff.check_color_interpretation(gdalinfo) assert not file_tiff.get_errors() @@ -180,7 +180,7 @@ def test_check_color_interpretation_invalid_dem() -> None: add_band(gdalinfo, color_interpretation="Green") add_band(gdalinfo, color_interpretation="Blue") - file_tiff = FileTiff(["test"], Preset.DEM_LERC.value) + file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value) file_tiff.check_color_interpretation(gdalinfo) assert file_tiff.get_errors() @@ -322,7 +322,7 @@ def test_should_throw_when_encountering_non_integer_no_data_value() -> None: gdalinfo = fake_gdal_info() add_palette_band(gdalinfo, colour_table_entries=[[x, x, x, 255] for x in reversed(range(256))], no_data_value="-9999.1") - file_tiff = FileTiff(["test"], Preset.DEM_LERC.value) + file_tiff = FileTiff(["test"], CompressionPreset.DEM_LERC.value) file_tiff.check_no_data(gdalinfo) assert file_tiff.get_errors() == [ diff --git a/scripts/gdal/gdal_bands.py b/scripts/gdal/gdal_bands.py index dab968d6e..6cd26b838 100644 --- a/scripts/gdal/gdal_bands.py +++ b/scripts/gdal/gdal_bands.py @@ -1,7 +1,7 @@ from linz_logger import get_log from scripts.gdal.gdal_helper import gdal_info -from scripts.gdal.gdal_presets import Preset +from scripts.gdal.gdal_presets import CompressionPreset from scripts.gdal.gdalinfo import GdalInfo, GdalInfoBand @@ -44,7 +44,7 @@ def get_gdal_band_offset(file: str, info: GdalInfo | None = None, preset: str | if band_grey := find_band(bands, "Gray"): band_grey_index = str(band_grey["band"]) - if preset == Preset.DEM_LERC.value: + if preset == CompressionPreset.DEM_LERC.value: # return single band if DEM/DSM return ["-b", band_grey_index] # Grey scale imagery, set R,G and B to just the grey_band diff --git a/scripts/gdal/gdal_commands.py b/scripts/gdal/gdal_commands.py index ecad86dea..f010fd950 100644 --- a/scripts/gdal/gdal_commands.py +++ b/scripts/gdal/gdal_commands.py @@ -8,7 +8,8 @@ DEM_LERC, SCALE_254_ADD_NO_DATA, WEBP_OVERVIEWS, - Preset, + CompressionPreset, + HillshadePreset, ) from scripts.gdal.gdalinfo import GdalInfo @@ -30,16 +31,16 @@ def get_gdal_command(preset: str, epsg: int) -> list[str]: # Force the source projection to an input EPSG gdal_command.extend(["-a_srs", f"EPSG:{epsg}"]) - if preset == Preset.LZW.value: + if preset == CompressionPreset.LZW.value: gdal_command.extend(SCALE_254_ADD_NO_DATA) gdal_command.extend(COMPRESS_LZW) gdal_command.extend(WEBP_OVERVIEWS) - elif preset == Preset.WEBP.value: + elif preset == CompressionPreset.WEBP.value: gdal_command.extend(COMPRESS_WEBP_LOSSLESS) gdal_command.extend(WEBP_OVERVIEWS) - elif preset == Preset.DEM_LERC.value: + elif preset == CompressionPreset.DEM_LERC.value: gdal_command.extend(DEM_LERC) return gdal_command @@ -195,3 +196,38 @@ def get_ascii_translate_command() -> list[str]: "-co", "COMPRESS=lzw", ] + + +def get_hillshade_command(preset: str) -> list[str]: + """Get a `gdaldem` command to create a hillshade based on the provided HillshadePreset. + + Args: + preset: a HillshadePreset + + Returns: + a `gdaldem` command + """ + gdal_command: list[str] = [ + "gdaldem", + "hillshade", + "-compute_edges", + "-of", + "COG", + "-co", + "COMPRESS=lerc", + "-co", + "OVERVIEW_COMPRESS=lerc", + "-co", + "MAX_Z_ERROR_OVERVIEW=0", + "-co", + "NUM_THREADS=ALL_CPUS", + "-co", + "MAX_Z_ERROR=0", + ] + + if preset == HillshadePreset.GREYSCALE.value: + gdal_command.extend(["-az", "315", "-alt", "45"]) + elif preset == HillshadePreset.IGOR.value: + gdal_command.extend(["-igor"]) + + return gdal_command diff --git a/scripts/gdal/gdal_presets.py b/scripts/gdal/gdal_presets.py index 0b833df59..b0972f6fc 100644 --- a/scripts/gdal/gdal_presets.py +++ b/scripts/gdal/gdal_presets.py @@ -79,7 +79,16 @@ ] -class Preset(str, Enum): +class CompressionPreset(str, Enum): + """Enum for the different compression presets available for standardising TIFFs.""" + DEM_LERC = "dem_lerc" LZW = "lzw" WEBP = "webp" + + +class HillshadePreset(str, Enum): + """Enum for the different type of hillshade available for generating from a DEM.""" + + GREYSCALE = "greyscale" # Standard/default mono-directional hillshade + IGOR = "igor" # Whiter hillshade (see http://maperitive.net/docs/Commands/GenerateReliefImageIgor.html) diff --git a/scripts/gdal/tests/gdal_bands_test.py b/scripts/gdal/tests/gdal_bands_test.py index 7a7157565..ec051879c 100644 --- a/scripts/gdal/tests/gdal_bands_test.py +++ b/scripts/gdal/tests/gdal_bands_test.py @@ -1,5 +1,5 @@ from scripts.gdal.gdal_bands import get_gdal_band_offset, get_gdal_band_type -from scripts.gdal.gdal_presets import Preset +from scripts.gdal.gdal_presets import CompressionPreset from scripts.gdal.tests.gdalinfo import add_band, add_palette_band, fake_gdal_info @@ -26,7 +26,7 @@ def test_gdal_grey_bands_dem_detection() -> None: gdalinfo = fake_gdal_info() add_band(gdalinfo, color_interpretation="Gray") - bands = get_gdal_band_offset("some_file.tiff", gdalinfo, Preset.DEM_LERC.value) + bands = get_gdal_band_offset("some_file.tiff", gdalinfo, CompressionPreset.DEM_LERC.value) assert " ".join(bands) == "-b 1" diff --git a/scripts/gdal/tests/gdal_commands_test.py b/scripts/gdal/tests/gdal_commands_test.py index 99a54281d..4fff4c202 100644 --- a/scripts/gdal/tests/gdal_commands_test.py +++ b/scripts/gdal/tests/gdal_commands_test.py @@ -2,11 +2,11 @@ from scripts.gdal.gdal_commands import get_cutline_command, get_gdal_command from scripts.gdal.gdal_helper import EpsgNumber -from scripts.gdal.gdal_presets import Preset +from scripts.gdal.gdal_presets import CompressionPreset def test_preset_webp(subtests: SubTests) -> None: - gdal_command = get_gdal_command(Preset.WEBP.value, epsg=EpsgNumber.NZTM_2000.value) + gdal_command = get_gdal_command(CompressionPreset.WEBP.value, epsg=EpsgNumber.NZTM_2000.value) # Basic cog creation with subtests.test(): @@ -46,7 +46,7 @@ def test_preset_webp(subtests: SubTests) -> None: def test_preset_lzw(subtests: SubTests) -> None: - gdal_command = get_gdal_command(Preset.LZW.value, epsg=EpsgNumber.NZTM_2000.value) + gdal_command = get_gdal_command(CompressionPreset.LZW.value, epsg=EpsgNumber.NZTM_2000.value) # Basic cog creation with subtests.test(): @@ -86,7 +86,7 @@ def test_preset_lzw(subtests: SubTests) -> None: def test_preset_dem_lerc(subtests: SubTests) -> None: - gdal_command = get_gdal_command(Preset.DEM_LERC.value, epsg=EpsgNumber.NZTM_2000.value) + gdal_command = get_gdal_command(CompressionPreset.DEM_LERC.value, epsg=EpsgNumber.NZTM_2000.value) # Basic cog creation with subtests.test(): assert "COG" in gdal_command diff --git a/scripts/generate_hillshade.py b/scripts/generate_hillshade.py new file mode 100644 index 000000000..cfa02d424 --- /dev/null +++ b/scripts/generate_hillshade.py @@ -0,0 +1,158 @@ +import argparse +import os +import sys +import tempfile +from functools import partial +from multiprocessing import Pool + +from linz_logger import get_log + +from scripts.cli.cli_helper import InputParameterError, TileFiles, is_argo, load_input_files +from scripts.files.files_helper import ContentType, is_tiff +from scripts.files.fs import exists, read, write, write_all +from scripts.gdal.gdal_commands import get_hillshade_command +from scripts.gdal.gdal_helper import run_gdal +from scripts.gdal.gdal_presets import HillshadePreset +from scripts.logging.time_helper import time_in_ms +from scripts.standardising import create_vrt + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + "--from-file", + dest="from_file", + required=True, + help="Specify the path to a json file containing the input tiffs. " + "Format: [{'output': 'tile1', 'inputs': ['path/input1.tiff', 'path/input2.tiff']}]", + ) + parser.add_argument( + "--preset", + dest="preset", + required=True, + choices=[preset.value for preset in HillshadePreset], + help="Type of hillshade to generate.", + ) + parser.add_argument("--target", dest="target", required=True, help="Specify the path to save the generated hillshade to.") + parser.add_argument( + "--force", + dest="force", + help="Regenerate the hillshade TIFF files if already exist. Defaults to False.", + action="store_true", + ) + + return parser.parse_args() + + +def create_hillshade( + tile: TileFiles, + preset: str, + target_output: str = "/tmp/", + force: bool = False, +) -> str | None: + """Create a hillshade TIFF file from a `TileFiles` which include an output tile with its input TIFFs. + + Args: + tile: a TileFiles object with the input TIFFs and the output tile name. + preset: a `HillshadePreset` to use. See `gdal.gdal_presets.py`. + target_output: path where the output files need to be saved to. Defaults to "/tmp/". + force: overwrite existing output file. Defaults to False. + + Returns: + The filename of the hillshade TIFF file if created. + """ + hillshade_file_name = tile.output + ".tiff" + + hillshade_file_path = os.path.join(target_output, hillshade_file_name) + + # Already processed can skip processing + if exists(hillshade_file_path): + if not force: + get_log().info("Skipping: hillshade TIFF already exists.", path=hillshade_file_path) + return None + get_log().info("Overwriting: hillshade TIFF already exists.", path=hillshade_file_path) + + # Download any needed file from S3 ["/foo/bar.tiff", "s3://foo"] => "/tmp/bar.tiff", "/tmp/foo.tiff" + with tempfile.TemporaryDirectory() as tmp_path: + hillshade_working_path = os.path.join(tmp_path, hillshade_file_name) + + source_files = write_all(tile.inputs, f"{tmp_path}/source/") + source_tiffs = [file for file in source_files if is_tiff(file)] + + # Start from base VRT + input_file = create_vrt(source_tiffs, tmp_path) + + # Need GDAL to write to temporary location so no broken files end up in the final folder. + run_gdal(get_hillshade_command(preset), input_file=input_file, output_file=hillshade_working_path) + + write(hillshade_file_path, read(hillshade_working_path), content_type=ContentType.GEOTIFF.value) + + return hillshade_file_path + + +def run_create_hillshade( + todo: list[TileFiles], + preset: str, + concurrency: int, + target_output: str = "/tmp/", + force: bool = False, +) -> list[str]: + """Run `create_hillshade()` in parallel (see `concurrency`). + + Args: + todo: list of TileFiles (tile name and input files) to hillshade + preset: `HillshadePreset` to use. See `gdal.gdal_presets.py` + concurrency: number of concurrent tiles to process + target_output: output directory path. Defaults to "/tmp/" + force: overwrite existing files. Defaults to False. + + Returns: + the list of generated hillshade TIFF paths. + """ + with Pool(concurrency) as p: + results = [ + entry + for entry in p.map( + partial( + create_hillshade, + preset=preset, + target_output=target_output, + force=force, + ), + todo, + ) + if entry is not None + ] + p.close() + p.join() + + return results + + +def main() -> None: + start_time = time_in_ms() + arguments = parse_args() + + try: + tile_files = load_input_files(arguments.from_file) + except InputParameterError as e: + get_log().error("An error occurred when loading the input file.", error=str(e)) + sys.exit(1) + + get_log().info( + "generate_hillshade_start", gdalVersion=os.environ["GDAL_VERSION"], fileCount=len(tile_files), preset=arguments.preset + ) + + concurrency: int = 1 + if is_argo(): + concurrency = 4 + + file_paths = run_create_hillshade(tile_files, arguments.preset, concurrency, arguments.target, arguments.force) + + get_log().info( + "generate_hillshade_end", duration=time_in_ms() - start_time, fileCount=len(file_paths), path=arguments.target + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/tests/data/hillshade.json b/scripts/tests/data/hillshade.json new file mode 100644 index 000000000..030915137 --- /dev/null +++ b/scripts/tests/data/hillshade.json @@ -0,0 +1,6 @@ +[ + { + "output": "BK39_10000_0101", + "input": ["./tests/data/output/BK39_10000_0101.tiff"] + } +] diff --git a/scripts/tests/data/output/BK39_10000_0101_greyscale.tiff b/scripts/tests/data/output/BK39_10000_0101_greyscale.tiff new file mode 100644 index 000000000..42b5bca3a Binary files /dev/null and b/scripts/tests/data/output/BK39_10000_0101_greyscale.tiff differ diff --git a/scripts/tests/data/output/BK39_10000_0101_igor.tiff b/scripts/tests/data/output/BK39_10000_0101_igor.tiff new file mode 100644 index 000000000..a7dc429b7 Binary files /dev/null and b/scripts/tests/data/output/BK39_10000_0101_igor.tiff differ