Merge pull request #40 from Geodan/39-tiler-parallel-warnings-for-unzip

release 1.1, add --jobs parameter for parallel (default 5)

README.md

@@ -125,10 +125,11 @@ $ docker run -v $(pwd):/data -it geodan/terrainwarp
 The script takes as input parameters:
 
 ```
-Syntax: [-c|m|h]
+Syntax: [-c|m|j|h]
 options:
 c Source s_srs - default EPSG:7415
 m fillnodata maxdistance (in pixels) - default 100
+j number of parallel jobs - default 5
 h Print this help
 ```
 
@@ -142,11 +143,12 @@ A the vertical EPSG code is optional but needed to transform from geoid to ellip
 Sample output:
 
 ```
-Terrain tiler 1.0 - Warp
+Terrain tiler 1.1 - Warp
 Start: Tue Jul 25 12:52:27 UTC 2023
 Temp directory: tmp
 s_src input images:
 Fillnodata maxdistance: 100
+Jobs: 6
 Delete tmp directory...
 tmp directory created.
 s_srs not set, trying to detect it from first GeoTIFF
@@ -178,6 +180,7 @@ o Output directory - default tiles
 b Break zoomlevel - default 9
 s Start zoomlevel - default 15
 e End zoomlevel - default 0
+j number of parallel jobs - default 5
 h Print this help
 ```
 
@@ -190,13 +193,14 @@ $ docker run -v $(pwd):/data -it geodan/terraintiler -s 10
 Sample output:
 
 ```
-Terrain tiler 1.0
+Terrain tiler 1.1
 Start: Wed Jun 21 09:24:39 UTC 2023
 Output directory: tiles
 Tif extension: TIF
 Start zoomlevel: 15
 Break zoomlevel: 9
 End zoomlevel: 0
+Jobs: 6
 Delete output directory...
 Directory created: tiles
 Running ctb-tile from 15 to level 9...
@@ -247,6 +251,12 @@ $ sh build_all.sh
 
 ## History
 
+2023-08-08: release 1.1:
+
+- add -j option for number of parallel jobs
+
+- removed parallel progress bar in tiler
+
 2023-07-26: release 1.0: 
 
 - performance improvement by using parallel processes;

tiler/process.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 
-version=1.0
+version=1.1
 
 # Set default values 
 output_dir=tiles
 tmp_dir=tmp
 start_zoom=15
 break_zoom=9
 end_zoom=0
+jobs=5
 # Sample reading parameter
 echo Terrain tiler $version - Step 2/2 Tiling
 start_time=$(date +%s)
@@ -16,24 +17,26 @@ echo Start: $(date)
 print_usage()
 {
    # Display Help
-   echo Syntax: '[b|s|e|o|h]'
+   echo Syntax: '[b|s|e|o|j|h]'
    echo options:
    echo o     Output directory - default $output_dir
    echo s     Start zoomlevel - default $start_zoom   
    echo b     Break zoomlevel - default $break_zoom
    echo e     End zoomlevel - default $end_zoom
+   echo j     Number of jobs - default $jobs
    echo h     Print this help
    echo
 }
 
 # Parse input arguments (flags)
-while getopts s:e:b:o:h flag
+while getopts s:e:b:o:j:h flag
 do
     case $flag in
         o) output_dir=$OPTARG;;
         s) start_zoom=$OPTARG;;
         b) break_zoom=$OPTARG;;
         e) end_zoom=$OPTARG;;
+        j) jobs=$OPTARG;;
         h) print_usage; exit 0;;
     esac
 done
@@ -42,6 +45,7 @@ echo Output directory: $output_dir
 echo Start zoomlevel: $start_zoom
 echo Break zoomlevel: $break_zoom
 echo End zoomlevel: $end_zoom
+echo Jobs: $jobs
 
 # Check on tmp dir
 if [ ! -d "$tmp_dir" ];
@@ -107,8 +111,11 @@ ctb-tile -v -f Mesh -C -N -e ${end_zoom} -s $((break_zoom-1)) -o ${output_dir} $
 
 # end workaround for level break_zoom - 0
 
+echo
 echo Unzip terrain files...
-find ${output_dir} -name '*.terrain' | parallel --bar 'mv {} {}.gz && gunzip -f -S terrain {}.gz'
+echo Warning: this may take a while...
+echo No progressbar is shown because performance.
+find ${output_dir} -name '*.terrain' | parallel -j ${jobs} 'mv {} {}.gz && gunzip -f -S terrain {}.gz'
 
 end_time=$(date +%s)
 echo End: $(date)

warp/process.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
 
-version=1.0
+version=1.1
 tmp_dir=tmp
 default_s_srs=EPSG:7415
 s_srs=""
 md=100
+jobs=5
 
 echo Terrain tiler $version - Warp
 start_time=$(date +%s)
@@ -13,10 +14,11 @@ echo Start: $(date)
 print_usage()
 {
    # Display Help
-   echo Syntax: '[-c|m|h]'
+   echo Syntax: '[-c|m|j|h]'
    echo options:
    echo c     Source s_srs - default $s_srs
    echo m     fillnodata maxdistance in pixels - default $md
+   echo j     Number of jobs - default $jobs
    echo h     Print this help
    echo
 }
@@ -38,22 +40,24 @@ warp_tiff()
 warp_tiffs()
 {
     echo Start processing ${tiffs} GeoTIFFS...
-    find ./ -maxdepth 1 -type f -iname '*.tif' | parallel --bar warp_tiff ${tmp_dir} ${md} ${s_srs}
+    find ./ -maxdepth 1 -type f -iname '*.tif' | parallel --bar -j ${jobs} warp_tiff ${tmp_dir} ${md} ${s_srs}
 }
 export -f warp_tiff
 
 # Parse input arguments (flags)
-while getopts c:m:h flag
+while getopts c:m:j:h flag
 do
     case $flag in
         c) s_srs=$OPTARG;;
         m) md=$OPTARG;;
+        j) jobs=$OPTARG;;
         h) print_usage; exit 0;;
     esac
 done
 
 echo Temp directory: $tmp_dir
 echo Fillnodata maxdistance: $md
+echo Jobs: $jobs
 
 # Check if input directory has .tif files
 tiffs=`find ./ -maxdepth 1 -type f -iname '*.tif' 2> /dev/null | wc -l`