diff --git a/src/Dockerfile b/src/Dockerfile index a07a959..692a22c 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -15,7 +15,7 @@ RUN pip install . -t python # to change the hash of the file and get TF to realize it needs to be # redeployed. Ticket for a better solution: # https://gfw.atlassian.net/browse/GTC-1250 -# change 14 +# change 15 RUN yum install -y zip geos-devel diff --git a/src/datapump/clients/data_api.py b/src/datapump/clients/data_api.py index c14cb94..adefb37 100644 --- a/src/datapump/clients/data_api.py +++ b/src/datapump/clients/data_api.py @@ -52,6 +52,8 @@ def get_1x1_asset(self, dataset: str, version: str) -> str: ) elif dataset == "gadm" and version == "v3.6": return "s3://gfw-files/2018_update/tsv/gadm36_adm2_1_1.csv" + elif dataset == "gadm" and version == "v4.1": + return "s3://gfw-pipelines/geotrellis/features/gadm41_adm2_1x1.tsv" return self.get_asset(dataset, version, "1x1 grid")["asset_uri"] diff --git a/src/datapump/jobs/geotrellis.py b/src/datapump/jobs/geotrellis.py index 6b9a9d6..734aeb9 100644 --- a/src/datapump/jobs/geotrellis.py +++ b/src/datapump/jobs/geotrellis.py @@ -821,8 +821,7 @@ def _run_job_flow(self, name, instances, steps, applications, configurations): { "Name": "Install GDAL", "ScriptBootstrapAction": { - "Path": f"s3://{GLOBALS.s3_bucket_pipeline}/geotrellis/bootstrap/gdal.sh", - "Args": ["3.1.2"], + "Path": f"s3://{GLOBALS.s3_bucket_pipeline}/geotrellis/bootstrap/gdal-3.8.3.sh" }, }, ], @@ -834,6 +833,15 @@ def _run_job_flow(self, name, instances, steps, applications, configurations): if GLOBALS.emr_service_role: request["ServiceRole"] = GLOBALS.emr_service_role + # If using version 2.4.1 or earlier, use older GDAL version + if self.geotrellis_version < "2.4.1": + request["BootstrapActions"] = { + "Name": "Install GDAL", + "ScriptBootstrapAction": { + "Path": f"s3://{GLOBALS.s3_bucket_pipeline}/geotrellis/bootstrap/gdal.sh", + }, + }, + LOGGER.info(f"Sending EMR request:\n{pformat(request)}") response = client.run_job_flow(**request)