Merge pull request #2541 from cisagov/AL-SQS-Local

Add more memory to Fargate tasks and cleanup logging
cisagov · Feb 15, 2024 · e58a23e · e58a23e
2 parents f759846 + bc1d319
commit e58a23e
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 14 deletions.
diff --git a/backend/worker/generate_config.sh b/backend/worker/generate_config.sh
@@ -54,6 +54,4 @@ pe_reports_path="${pe_reports_path%/pe-reports}/pe_reports"
 # Copy database.ini to the module's installation directory
 cp /app/pe-reports/src/pe_reports/data/database.ini "${pe_reports_path}/data/"
 
-cat /app/pe-reports/src/pe_reports/data/database.ini
-
 exec "$@"
diff --git a/backend/worker/pe-worker-entry.sh b/backend/worker/pe-worker-entry.sh
@@ -50,7 +50,7 @@ while true; do
   if [ "$IS_LOCAL" = true ]; then
     ORG=$(echo "$MESSAGE" | jq -r '.org')
   else
-    ORG=$(echo "$MESSAGE" | jq -r '.Messages[0].Body | capture("org: \"(?<org>[^\"]+)\"") | .org')
+    ORG=$(echo "$MESSAGE" | jq -r '.Messages[0].Body | fromjson | .org')
   fi
 
   if [[ "$SERVICE_TYPE" = *"shodan"*  ]]; then

diff --git a/infrastructure/pe_worker.tf b/infrastructure/pe_worker.tf
@@ -168,9 +168,8 @@ resource "aws_ecs_task_definition" "pe_dnstwist_worker" {
 
   # CPU and memory values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
 
-  cpu    = 256 # .25 vCPU
-  memory = 512 # 512 MB
-
+  cpu    = 2048
+  memory = 16384
   tags = {
     Project = var.project
     Stage   = var.stage
@@ -190,6 +189,9 @@ resource "aws_ecs_service" "dnstwist_service" {
     security_groups  = [aws_security_group.worker.id]
     assign_public_ip = true
   }
+  lifecycle {
+    ignore_changes = [desired_count]
+  }
 }
 
 # P&E Shodan Task Definition
@@ -322,8 +324,8 @@ resource "aws_ecs_task_definition" "pe_shodan_worker" {
 
   # CPU and memory values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
 
-  cpu    = 256 # .25 vCPU
-  memory = 512 # 512 MB
+  cpu    = 2048
+  memory = 16384
 
   tags = {
     Project = var.project
@@ -343,6 +345,9 @@ resource "aws_ecs_service" "shodan_service" {
     security_groups  = [aws_security_group.worker.id]
     assign_public_ip = true
   }
+  lifecycle {
+    ignore_changes = [desired_count]
+  }
 }
 
 # P&E HIBP Task Definition
@@ -475,8 +480,8 @@ resource "aws_ecs_task_definition" "pe_hibp_worker" {
 
   # CPU and memory values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
 
-  cpu    = 256 # .25 vCPU
-  memory = 512 # 512 MB
+  cpu    = 2048
+  memory = 16384
 
   tags = {
     Project = var.project
@@ -497,6 +502,9 @@ resource "aws_ecs_service" "hibp_service" {
     security_groups  = [aws_security_group.worker.id]
     assign_public_ip = true
   }
+  lifecycle {
+    ignore_changes = [desired_count]
+  }
 }
 
 # P&E IntelX Task Definition
@@ -629,8 +637,8 @@ resource "aws_ecs_task_definition" "pe_intelx_worker" {
 
   # CPU and memory values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
 
-  cpu    = 256 # .25 vCPU
-  memory = 512 # 512 MB
+  cpu    = 2048
+  memory = 16384
 
   tags = {
     Project = var.project
@@ -651,6 +659,10 @@ resource "aws_ecs_service" "intelx_service" {
     security_groups  = [aws_security_group.worker.id]
     assign_public_ip = true
   }
+
+  lifecycle {
+    ignore_changes = [desired_count]
+  }
 }
 
 # P&E Cybersixgill Task Definition
@@ -783,8 +795,8 @@ resource "aws_ecs_task_definition" "pe_cybersixgill_worker" {
 
   # CPU and memory values: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
 
-  cpu    = 256 # .25 vCPU
-  memory = 512 # 512 MB
+  cpu    = 2048
+  memory = 16384
 
   tags = {
     Project = var.project
@@ -805,6 +817,9 @@ resource "aws_ecs_service" "cybersixgill_service" {
     security_groups  = [aws_security_group.worker.id]
     assign_public_ip = true
   }
+  lifecycle {
+    ignore_changes = [desired_count]
+  }
 }
 
 # Create the  log group