GTC-2631 Update datapump for GADM 4.1

src/Dockerfile

@@ -15,7 +15,7 @@ RUN pip install . -t python
 # to change the hash of the file and get TF to realize it needs to be
 # redeployed. Ticket for a better solution:
 # https://gfw.atlassian.net/browse/GTC-1250
-# change 14
+# change 15
 
 RUN yum install -y zip geos-devel
 

src/datapump/clients/data_api.py

@@ -52,6 +52,8 @@ def get_1x1_asset(self, dataset: str, version: str) -> str:
             )
         elif dataset == "gadm" and version == "v3.6":
             return "s3://gfw-files/2018_update/tsv/gadm36_adm2_1_1.csv"
+        elif dataset == "gadm" and version == "v4.1":
+            return "s3://gfw-pipelines/geotrellis/features/gadm41_adm2_1x1.tsv"
 
         return self.get_asset(dataset, version, "1x1 grid")["asset_uri"]
 

src/datapump/jobs/geotrellis.py

@@ -821,8 +821,7 @@ def _run_job_flow(self, name, instances, steps, applications, configurations):
                 {
                     "Name": "Install GDAL",
                     "ScriptBootstrapAction": {
-                        "Path": f"s3://{GLOBALS.s3_bucket_pipeline}/geotrellis/bootstrap/gdal.sh",
-                        "Args": ["3.1.2"],
+                        "Path": f"s3://{GLOBALS.s3_bucket_pipeline}/geotrellis/bootstrap/gdal-3.8.3.sh"
                     },
                 },
             ],
@@ -834,6 +833,15 @@ def _run_job_flow(self, name, instances, steps, applications, configurations):
         if GLOBALS.emr_service_role:
             request["ServiceRole"] = GLOBALS.emr_service_role
 
+        # If using version 2.4.1 or earlier, use older GDAL version
+        if self.geotrellis_version < "2.4.1":
+            request["BootstrapActions"] = {
+                "Name": "Install GDAL",
+                "ScriptBootstrapAction": {
+                    "Path": f"s3://{GLOBALS.s3_bucket_pipeline}/geotrellis/bootstrap/gdal.sh",
+                },
+            },
+
         LOGGER.info(f"Sending EMR request:\n{pformat(request)}")
 
         response = client.run_job_flow(**request)