Merge remote-tracking branch 'origin/master' into disable_export_to_gee

wri · Aug 28, 2024 · 7e19abe · 7e19abe
2 parents 548048d + 70d0aef
commit 7e19abe
Show file tree

Hide file tree

Showing 7 changed files with 35 additions and 6 deletions.
diff --git a/src/datapump/sync/fire_alerts.py b/src/datapump/sync/fire_alerts.py
@@ -10,7 +10,7 @@
 from ..clients.aws import get_s3_client
 from ..globals import LOGGER
 
-ACTIVE_FIRE_ALERTS_48HR_CSV_URLS = {
+ACTIVE_FIRE_ALERTS_7D_SHAPEFILE_URLS = {
     "modis": "https://firms2.modaps.eosdis.nasa.gov/data/active_fire/modis-c6.1/shapes/zips/MODIS_C6_1_Global_7d.zip",
     "viirs": "https://firms2.modaps.eosdis.nasa.gov/data/active_fire/suomi-npp-viirs-c2/shapes/zips/SUOMI_VIIRS_C2_Global_7d.zip",
 }
@@ -31,18 +31,24 @@
 
 def process_active_fire_alerts(alert_type):
     LOGGER.info(f"Retrieving fire alerts for {alert_type}")
-    response = requests.get(ACTIVE_FIRE_ALERTS_48HR_CSV_URLS[alert_type])
+    response = requests.get(ACTIVE_FIRE_ALERTS_7D_SHAPEFILE_URLS[alert_type])
 
     if response.status_code != 200:
         raise Exception(
             f"Unable to get active {alert_type} fire alerts, FIRMS returned status code {response.status_code}"
         )
 
-    LOGGER.info("Successfully download alerts from NASA")
+    LOGGER.info("Successfully downloaded alerts from NASA")
 
     zip = zipfile.ZipFile(io.BytesIO(response.content))
     shp_dir = f"{TEMP_DIR}/fire_alerts_{alert_type}"
     zip.extractall(shp_dir)
+
+    if not os.path.isfile(f"{shp_dir}/{SHP_NAMES[alert_type]}"):
+        raise Exception(
+            f"{alert_type} fire alerts zip downloaded, but contains no .shp file!"
+        )
+
     sf = shapefile.Reader(f"{shp_dir}/{SHP_NAMES[alert_type]}")
 
     rows = []

diff --git a/terraform/main.tf b/terraform/main.tf
@@ -49,8 +49,9 @@ module "datapump" {
   sfn_wait_time = 120
   data_api_uri = var.data_api_uri
   data_lake_bucket = data.terraform_remote_state.core.outputs.data-lake_bucket
-  rasterio_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_rasterio_138_arn
-  shapely_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_shapely_185_arn
+  numpy_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_numpy_arn
+  rasterio_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_rasterio_no_numpy_arn
+  shapely_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_shapely_no_numpy_arn
   glad_path = local.glad_path
   emr_instance_profile_name = data.terraform_remote_state.core.outputs.emr_instance_profile_name
   emr_service_role_name = data.terraform_remote_state.core.outputs.emr_service_role_name

diff --git a/terraform/modules/datapump/lambdas.tf b/terraform/modules/datapump/lambdas.tf
@@ -11,6 +11,7 @@ resource "aws_lambda_function" "dispatcher" {
   tags             = local.tags
   layers           = [
     module.py310_datapump_021.layer_arn,
+    var.numpy_lambda_layer_arn,
     var.rasterio_lambda_layer_arn,
     var.shapely_lambda_layer_arn
   ]
@@ -68,6 +69,7 @@ resource "aws_lambda_function" "postprocessor" {
   tags             = local.tags
   layers           = [
     module.py310_datapump_021.layer_arn,
+    var.numpy_lambda_layer_arn,
     var.rasterio_lambda_layer_arn,
     var.shapely_lambda_layer_arn
   ]

diff --git a/terraform/modules/datapump/variables.tf b/terraform/modules/datapump/variables.tf
@@ -88,6 +88,11 @@ variable "sfn_wait_time" {
   description = "Time to wait in between steps of step function"
 }
 
+variable "numpy_lambda_layer_arn" {
+  type        = string
+  description = "ARN of the numpy lambda layer"
+}
+
 variable "rasterio_lambda_layer_arn" {
   type        = string
   description = "ARN of the rasterio lambda layer"

diff --git a/tests/files/numpy.zip b/tests/files/numpy.zip
diff --git a/tests/terraform/main.tf b/tests/terraform/main.tf
@@ -35,6 +35,7 @@ module "datapump" {
   sfn_wait_time = 1
   data_api_uri = var.data_api_uri
   data_lake_bucket = aws_s3_bucket.data_lake_test.id
+  numpy_lambda_layer_arn = aws_lambda_layer_version.numpy.arn
   rasterio_lambda_layer_arn = aws_lambda_layer_version.rasterio.arn
   shapely_lambda_layer_arn = aws_lambda_layer_version.shapely.arn
   glad_path = var.glad_path

diff --git a/tests/terraform/test.tf b/tests/terraform/test.tf
@@ -64,6 +64,13 @@ resource "aws_s3_bucket_object" "glad_status" {
   etag   = filemd5("../files/status")
 }
 
+resource "aws_s3_bucket_object" "numpy" {
+  bucket = aws_s3_bucket.pipelines_test.id
+  key    = "lambda_layers/numpy.zip"
+  source = "../files/numpy.zip"
+  etag   = filemd5("../files/numpy.zip")
+}
+
 resource "aws_s3_bucket_object" "rasterio" {
   bucket = aws_s3_bucket.pipelines_test.id
   key    = "lambda_layers/rasterio.zip"
@@ -78,6 +85,14 @@ resource "aws_s3_bucket_object" "shapely" {
   etag   = filemd5("../files/shapely.zip")
 }
 
+resource "aws_lambda_layer_version" "numpy" {
+  layer_name          = substr("test-numpy", 0, 64)
+  s3_bucket           = aws_s3_bucket_object.numpy.bucket
+  s3_key              = aws_s3_bucket_object.numpy.key
+  compatible_runtimes = ["python3.10"]
+  source_code_hash    = filebase64sha256("../files/numpy.zip")
+}
+
 resource "aws_lambda_layer_version" "rasterio" {
   layer_name          = substr("test-rasterio", 0, 64)
   s3_bucket           = aws_s3_bucket_object.rasterio.bucket
@@ -101,7 +116,6 @@ module "api_token_secret" {
   secret_string = jsonencode({ "token" = "test_token", "email" = "[email protected]" })
 }
 
-
 module "slack_secret" {
   source        = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/secrets?ref=master"
   project       = "test_proj"