diff --git a/src/datapump/sync/fire_alerts.py b/src/datapump/sync/fire_alerts.py index e2132f2..fb8f9eb 100644 --- a/src/datapump/sync/fire_alerts.py +++ b/src/datapump/sync/fire_alerts.py @@ -10,7 +10,7 @@ from ..clients.aws import get_s3_client from ..globals import LOGGER -ACTIVE_FIRE_ALERTS_48HR_CSV_URLS = { +ACTIVE_FIRE_ALERTS_7D_SHAPEFILE_URLS = { "modis": "https://firms2.modaps.eosdis.nasa.gov/data/active_fire/modis-c6.1/shapes/zips/MODIS_C6_1_Global_7d.zip", "viirs": "https://firms2.modaps.eosdis.nasa.gov/data/active_fire/suomi-npp-viirs-c2/shapes/zips/SUOMI_VIIRS_C2_Global_7d.zip", } @@ -31,18 +31,24 @@ def process_active_fire_alerts(alert_type): LOGGER.info(f"Retrieving fire alerts for {alert_type}") - response = requests.get(ACTIVE_FIRE_ALERTS_48HR_CSV_URLS[alert_type]) + response = requests.get(ACTIVE_FIRE_ALERTS_7D_SHAPEFILE_URLS[alert_type]) if response.status_code != 200: raise Exception( f"Unable to get active {alert_type} fire alerts, FIRMS returned status code {response.status_code}" ) - LOGGER.info("Successfully download alerts from NASA") + LOGGER.info("Successfully downloaded alerts from NASA") zip = zipfile.ZipFile(io.BytesIO(response.content)) shp_dir = f"{TEMP_DIR}/fire_alerts_{alert_type}" zip.extractall(shp_dir) + + if not os.path.isfile(f"{shp_dir}/{SHP_NAMES[alert_type]}"): + raise Exception( + f"{alert_type} fire alerts zip downloaded, but contains no .shp file!" + ) + sf = shapefile.Reader(f"{shp_dir}/{SHP_NAMES[alert_type]}") rows = [] diff --git a/terraform/main.tf b/terraform/main.tf index 5f24c3a..a16583b 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -49,8 +49,9 @@ module "datapump" { sfn_wait_time = 120 data_api_uri = var.data_api_uri data_lake_bucket = data.terraform_remote_state.core.outputs.data-lake_bucket - rasterio_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_rasterio_138_arn - shapely_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_shapely_185_arn + numpy_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_numpy_arn + rasterio_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_rasterio_no_numpy_arn + shapely_lambda_layer_arn = data.terraform_remote_state.lambda-layers.outputs.py310_shapely_no_numpy_arn glad_path = local.glad_path emr_instance_profile_name = data.terraform_remote_state.core.outputs.emr_instance_profile_name emr_service_role_name = data.terraform_remote_state.core.outputs.emr_service_role_name diff --git a/terraform/modules/datapump/lambdas.tf b/terraform/modules/datapump/lambdas.tf index ea880fd..4fb147c 100644 --- a/terraform/modules/datapump/lambdas.tf +++ b/terraform/modules/datapump/lambdas.tf @@ -11,6 +11,7 @@ resource "aws_lambda_function" "dispatcher" { tags = local.tags layers = [ module.py310_datapump_021.layer_arn, + var.numpy_lambda_layer_arn, var.rasterio_lambda_layer_arn, var.shapely_lambda_layer_arn ] @@ -68,6 +69,7 @@ resource "aws_lambda_function" "postprocessor" { tags = local.tags layers = [ module.py310_datapump_021.layer_arn, + var.numpy_lambda_layer_arn, var.rasterio_lambda_layer_arn, var.shapely_lambda_layer_arn ] diff --git a/terraform/modules/datapump/variables.tf b/terraform/modules/datapump/variables.tf index 62c9364..d197fa5 100644 --- a/terraform/modules/datapump/variables.tf +++ b/terraform/modules/datapump/variables.tf @@ -88,6 +88,11 @@ variable "sfn_wait_time" { description = "Time to wait in between steps of step function" } +variable "numpy_lambda_layer_arn" { + type = string + description = "ARN of the numpy lambda layer" +} + variable "rasterio_lambda_layer_arn" { type = string description = "ARN of the rasterio lambda layer" diff --git a/tests/files/numpy.zip b/tests/files/numpy.zip new file mode 100644 index 0000000..5841998 Binary files /dev/null and b/tests/files/numpy.zip differ diff --git a/tests/terraform/main.tf b/tests/terraform/main.tf index d672632..d50435a 100644 --- a/tests/terraform/main.tf +++ b/tests/terraform/main.tf @@ -35,6 +35,7 @@ module "datapump" { sfn_wait_time = 1 data_api_uri = var.data_api_uri data_lake_bucket = aws_s3_bucket.data_lake_test.id + numpy_lambda_layer_arn = aws_lambda_layer_version.numpy.arn rasterio_lambda_layer_arn = aws_lambda_layer_version.rasterio.arn shapely_lambda_layer_arn = aws_lambda_layer_version.shapely.arn glad_path = var.glad_path diff --git a/tests/terraform/test.tf b/tests/terraform/test.tf index 4a38af7..fc0f936 100644 --- a/tests/terraform/test.tf +++ b/tests/terraform/test.tf @@ -64,6 +64,13 @@ resource "aws_s3_bucket_object" "glad_status" { etag = filemd5("../files/status") } +resource "aws_s3_bucket_object" "numpy" { + bucket = aws_s3_bucket.pipelines_test.id + key = "lambda_layers/numpy.zip" + source = "../files/numpy.zip" + etag = filemd5("../files/numpy.zip") +} + resource "aws_s3_bucket_object" "rasterio" { bucket = aws_s3_bucket.pipelines_test.id key = "lambda_layers/rasterio.zip" @@ -78,6 +85,14 @@ resource "aws_s3_bucket_object" "shapely" { etag = filemd5("../files/shapely.zip") } +resource "aws_lambda_layer_version" "numpy" { + layer_name = substr("test-numpy", 0, 64) + s3_bucket = aws_s3_bucket_object.numpy.bucket + s3_key = aws_s3_bucket_object.numpy.key + compatible_runtimes = ["python3.10"] + source_code_hash = filebase64sha256("../files/numpy.zip") +} + resource "aws_lambda_layer_version" "rasterio" { layer_name = substr("test-rasterio", 0, 64) s3_bucket = aws_s3_bucket_object.rasterio.bucket @@ -101,7 +116,6 @@ module "api_token_secret" { secret_string = jsonencode({ "token" = "test_token", "email" = "gfw-sync@test.org" }) } - module "slack_secret" { source = "git::https://github.com/wri/gfw-terraform-modules.git//terraform/modules/secrets?ref=master" project = "test_proj"