fix(user-list-search): replace kinesis with event bridge events for search indexing

[bassrock]

Goal

Stop using our old Unified Events Kinesis stream for processing search indexing events.

[github-actions]

Plan Result (user-list-search-production)

CI link

⚠️ Resource Deletion will happen ⚠️

This plan contains resource delete operation. Please check the plan result very carefully!

Plan: 1 to add, 3 to change, 9 to destroy.

• Create
  • aws_sns_topic_subscription.list_events_sns_topic_subscription
• Update
  • aws_sqs_queue_policy.user_events_sns_topic_dlq_policy
  • aws_sqs_queue_policy.user_events_sqs_policy
  • module.dashboard_alarm.aws_cloudwatch_dashboard.dashboard["sla"]
• Delete
  • aws_cloudwatch_log_group.unified_events_consumer
  • aws_codedeploy_app.lambda_unified_events_consumer
  • aws_codedeploy_deployment_group.lambda_unified_events_consumer
  • aws_codestarnotifications_notification_rule.lambda_unified_events_consumer_notifications
  • aws_iam_role_policy.lambda_execution_policy
  • aws_lambda_alias.unified_events_consumer
  • aws_lambda_event_source_mapping.kinesis_consumer
  • aws_lambda_function.unified_events_consumer
  • module.dashboard_alarm.aws_cloudwatch_metric_alarm.metric_alarms["event_consumer_lambda_errors"]

Change Result (Click me)

  # aws_cloudwatch_log_group.unified_events_consumer will be destroyed
  # (because aws_cloudwatch_log_group.unified_events_consumer is not in configuration)
  - resource "aws_cloudwatch_log_group" "unified_events_consumer" {
      - arn               = "arn:aws:logs:us-east-1:996905175585:log-group:/aws/lambda/UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - id                = "/aws/lambda/UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - log_group_class   = "STANDARD" -> null
      - name              = "/aws/lambda/UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - retention_in_days = 14 -> null
      - skip_destroy      = false -> null
      - tags              = {} -> null
      - tags_all          = {
          - "app_code"       = "pocket"
          - "component_code" = "pocket-userlistsearch"
          - "costCenter"     = "Pocket"
          - "env_code"       = "prod"
          - "environment"    = "Prod"
          - "owner"          = "Pocket"
          - "service"        = "UserListSearch"
        } -> null
        # (2 unchanged attributes hidden)
    }

  # aws_codedeploy_app.lambda_unified_events_consumer will be destroyed
  # (because aws_codedeploy_app.lambda_unified_events_consumer is not in configuration)
  - resource "aws_codedeploy_app" "lambda_unified_events_consumer" {
      - application_id      = "5dd2b608-4987-4821-981c-872e051a168b" -> null
      - arn                 = "arn:aws:codedeploy:us-east-1:996905175585:application:UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - compute_platform    = "Lambda" -> null
      - id                  = "5dd2b608-4987-4821-981c-872e051a168b:UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - linked_to_github    = false -> null
      - name                = "UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - tags                = {} -> null
      - tags_all            = {
          - "app_code"       = "pocket"
          - "component_code" = "pocket-userlistsearch"
          - "costCenter"     = "Pocket"
          - "env_code"       = "prod"
          - "environment"    = "Prod"
          - "owner"          = "Pocket"
          - "service"        = "UserListSearch"
        } -> null
        # (1 unchanged attribute hidden)
    }

  # aws_codedeploy_deployment_group.lambda_unified_events_consumer will be destroyed
  # (because aws_codedeploy_deployment_group.lambda_unified_events_consumer is not in configuration)
  - resource "aws_codedeploy_deployment_group" "lambda_unified_events_consumer" {
      - app_name                    = "UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - arn                         = "arn:aws:codedeploy:us-east-1:996905175585:deploymentgroup:UserListSearch-Prod-UnifiedEventsConsumer/UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - autoscaling_groups          = [] -> null
      - compute_platform            = "Lambda" -> null
      - deployment_config_name      = "CodeDeployDefault.LambdaAllAtOnce" -> null
      - deployment_group_id         = "09a4ba63-459f-44ed-af24-007f0d37e196" -> null
      - deployment_group_name       = "UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - id                          = "09a4ba63-459f-44ed-af24-007f0d37e196" -> null
      - outdated_instances_strategy = "UPDATE" -> null
      - service_role_arn            = "arn:aws:iam::996905175585:role/UserListSearch-Prod-LambdaCodeDeployRole" -> null
      - tags                        = {} -> null
      - tags_all                    = {
          - "app_code"       = "pocket"
          - "component_code" = "pocket-userlistsearch"
          - "costCenter"     = "Pocket"
          - "env_code"       = "prod"
          - "environment"    = "Prod"
          - "owner"          = "Pocket"
          - "service"        = "UserListSearch"
        } -> null

      - auto_rollback_configuration {
          - enabled = true -> null
          - events  = [
              - "DEPLOYMENT_FAILURE",
            ] -> null
        }

      - deployment_style {
          - deployment_option = "WITH_TRAFFIC_CONTROL" -> null
          - deployment_type   = "BLUE_GREEN" -> null
        }
    }

  # aws_codestarnotifications_notification_rule.lambda_unified_events_consumer_notifications will be destroyed
  # (because aws_codestarnotifications_notification_rule.lambda_unified_events_consumer_notifications is not in configuration)
  - resource "aws_codestarnotifications_notification_rule" "lambda_unified_events_consumer_notifications" {
      - arn            = "arn:aws:codestar-notifications:us-east-1:996905175585:notificationrule/3279b5719fa1ea8fc929dea926bff3620bbf2bbd" -> null
      - detail_type    = "BASIC" -> null
      - event_type_ids = [
          - "codedeploy-application-deployment-failed",
        ] -> null
      - id             = "arn:aws:codestar-notifications:us-east-1:996905175585:notificationrule/3279b5719fa1ea8fc929dea926bff3620bbf2bbd" -> null
      - name           = "UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - resource       = "arn:aws:codedeploy:us-east-1:996905175585:application:UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - status         = "ENABLED" -> null
      - tags           = {} -> null
      - tags_all       = {
          - "app_code"       = "pocket"
          - "component_code" = "pocket-userlistsearch"
          - "costCenter"     = "Pocket"
          - "env_code"       = "prod"
          - "environment"    = "Prod"
          - "owner"          = "Pocket"
          - "service"        = "UserListSearch"
        } -> null

      - target {
          - address = "arn:aws:sns:us-east-1:996905175585:Backend-Prod-ChatBot" -> null
          - status  = "ACTIVE" -> null
          - type    = "SNS" -> null
        }
    }

  # aws_iam_role_policy.lambda_execution_policy will be destroyed
  # (because aws_iam_role_policy.lambda_execution_policy is not in configuration)
  - resource "aws_iam_role_policy" "lambda_execution_policy" {
      - id          = "UserListSearch-Prod-LambdaExecutionRole:UserListSearch-Prod-KinesisEventAccessPolicy" -> null
      - name        = "UserListSearch-Prod-KinesisEventAccessPolicy" -> null
      - policy      = jsonencode(
            {
              - Statement = [
                  - {
                      - Action   = [
                          - "sqs:SendMessageBatch",
                          - "sqs:SendMessage",
                        ]
                      - Effect   = "Allow"
                      - Resource = [
                          - "arn:aws:sqs:us-east-1:996905175585:UserListSearch-Prod-UserListImport",
                          - "arn:aws:sqs:us-east-1:996905175585:UserListSearch-Prod-UserItemsUpdate",
                          - "arn:aws:sqs:us-east-1:996905175585:UserListSearch-Prod-UserItemsDelete",
                        ]
                      - Sid      = ""
                    },
                  - {
                      - Action   = [
                          - "logs:PutLogEvents",
                          - "logs:DescribeLogStreams",
                          - "logs:CreateLogStream",
                          - "logs:CreateLogGroup",
                        ]
                      - Effect   = "Allow"
                      - Resource = "arn:aws:logs:*:*:*"
                      - Sid      = ""
                    },
                  - {
                      - Action   = [
                          - "kinesis:ListStreams",
                          - "kinesis:ListStreamConsumers",
                          - "kinesis:ListShards",
                          - "kinesis:DescribeLimits",
                        ]
                      - Effect   = "Allow"
                      - Resource = "*"
                      - Sid      = ""
                    },
                  - {
                      - Action   = [
                          - "kinesis:SubscribeToShard",
                          - "kinesis:ListTagsForStream",
                          - "kinesis:Get*",
                          - "kinesis:Describe*",
                        ]
                      - Effect   = "Allow"
                      - Resource = "arn:aws:kinesis:us-east-1:996905175585:stream/unified_event"
                      - Sid      = ""
                    },
                  - {
                      - Action   = "ssm:GetParameter*"
                      - Effect   = "Allow"
                      - Resource = [
                          - "arn:aws:ssm:us-east-1:996905175585:parameter/UserListSearch/Prod/*",
                          - "arn:aws:ssm:us-east-1:996905175585:parameter/UserListSearch/Prod",
                        ]
                      - Sid      = ""
                    },
                  - {
                      - Action   = [
                          - "secretsmanager:GetSecretValue",
                          - "kms:Decrypt",
                        ]
                      - Effect   = "Allow"
                      - Resource = [
                          - "arn:aws:secretsmanager:us-east-1:996905175585:secret:UserListSearch/Prod/*",
                          - "arn:aws:secretsmanager:us-east-1:996905175585:secret:UserListSearch/Prod",
                          - "arn:aws:secretsmanager:us-east-1:996905175585:secret:UserListSearch/Default/*",
                          - "arn:aws:secretsmanager:us-east-1:996905175585:secret:UserListSearch/Default",
                          - "arn:aws:kms:us-east-1:996905175585:key/fee9bd90-a01f-4212-8b69-9c149a7cdb15",
                        ]
                      - Sid      = ""
                    },
                ]
              - Version   = "2012-10-17"
            }
        ) -> null
      - role        = "UserListSearch-Prod-LambdaExecutionRole" -> null
        # (1 unchanged attribute hidden)
    }

  # aws_lambda_alias.unified_events_consumer will be destroyed
  # (because aws_lambda_alias.unified_events_consumer is not in configuration)
  - resource "aws_lambda_alias" "unified_events_consumer" {
      - arn              = "arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer:DEPLOYED" -> null
      - function_name    = "arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - function_version = "805" -> null
      - id               = "arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer:DEPLOYED" -> null
      - invoke_arn       = "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer:DEPLOYED/invocations" -> null
      - name             = "DEPLOYED" -> null
        # (1 unchanged attribute hidden)
    }

  # aws_lambda_event_source_mapping.kinesis_consumer will be destroyed
  # (because aws_lambda_event_source_mapping.kinesis_consumer is not in configuration)
  - resource "aws_lambda_event_source_mapping" "kinesis_consumer" {
      - batch_size                         = 10000 -> null
      - bisect_batch_on_function_error     = false -> null
      - enabled                            = true -> null
      - event_source_arn                   = "arn:aws:kinesis:us-east-1:996905175585:stream/unified_event" -> null
      - function_arn                       = "arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer:DEPLOYED" -> null
      - function_name                      = "arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer:DEPLOYED" -> null
      - function_response_types            = [] -> null
      - id                                 = "feee073a-f840-4cd5-ab95-bfbe79f565c1" -> null
      - last_modified                      = "2025-01-04T11:12:00Z" -> null
      - last_processing_result             = "OK" -> null
      - maximum_batching_window_in_seconds = 0 -> null
      - maximum_record_age_in_seconds      = 60 -> null
      - maximum_retry_attempts             = -1 -> null
      - parallelization_factor             = 1 -> null
      - queues                             = [] -> null
      - starting_position                  = "LATEST" -> null
      - state                              = "Enabled" -> null
      - state_transition_reason            = "User action" -> null
      - topics                             = [] -> null
      - tumbling_window_in_seconds         = 0 -> null
      - uuid                               = "feee073a-f840-4cd5-ab95-bfbe79f565c1" -> null
        # (1 unchanged attribute hidden)

      - destination_config {
          - on_failure {}
        }
    }

  # aws_lambda_function.unified_events_consumer will be destroyed
  # (because aws_lambda_function.unified_events_consumer is not in configuration)
  - resource "aws_lambda_function" "unified_events_consumer" {
      - architectures                  = [
          - "x86_64",
        ] -> null
      - arn                            = "arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - function_name                  = "UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - handler                        = "index.handler" -> null
      - id                             = "UserListSearch-Prod-UnifiedEventsConsumer" -> null
      - invoke_arn                     = "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer/invocations" -> null
      - last_modified                  = "2025-01-06T20:47:21.000+0000" -> null
      - layers                         = [] -> null
      - memory_size                    = 256 -> null
      - package_type                   = "Zip" -> null
      - publish                        = true -> null
      - qualified_arn                  = "arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer:805" -> null
      - qualified_invoke_arn           = "arn:aws:apigateway:us-east-1:lambda:path/2015-03-31/functions/arn:aws:lambda:us-east-1:996905175585:function:UserListSearch-Prod-UnifiedEventsConsumer:805/invocations" -> null
      - reserved_concurrent_executions = -1 -> null
      - role                           = "arn:aws:iam::996905175585:role/UserListSearch-Prod-LambdaExecutionRole" -> null
      - runtime                        = "nodejs20.x" -> null
      - skip_destroy                   = false -> null
      - source_code_hash               = "MJLeY/8tNG0SrRJa+jvBcHGStyoaLr8GcxAV0sjV8L4=" -> null
      - source_code_size               = 15313137 -> null
      - tags                           = {
          - "app_code"       = "pocket"
          - "component_code" = "pocket-userlistsearch"
          - "costCenter"     = "Pocket"
          - "env_code"       = "prod"
          - "environment"    = "Prod"
          - "owner"          = "Pocket"
          - "service"        = "UserListSearch"
        } -> null
      - tags_all                       = {
          - "app_code"       = "pocket"
          - "component_code" = "pocket-userlistsearch"
          - "costCenter"     = "Pocket"
          - "env_code"       = "prod"
          - "environment"    = "Prod"
          - "owner"          = "Pocket"
          - "service"        = "UserListSearch"
        } -> null
      - timeout                        = 300 -> null
      - version                        = "805" -> null
        # (6 unchanged attributes hidden)

      - environment {
          - variables = {
              - "GIT_SHA"                   = "862e48cd464b5b08453efe542a1efff0caf57e87"
              - "NODE_ENV"                  = "production"
              - "SENTRY_DSN"                = (sensitive value)
              - "SQS_USER_ITEMS_DELETE_URL" = "https://sqs.us-east-1.amazonaws.com/996905175585/UserListSearch-Prod-UserItemsDelete"
              - "SQS_USER_ITEMS_UPDATE_URL" = "https://sqs.us-east-1.amazonaws.com/996905175585/UserListSearch-Prod-UserItemsUpdate"
              - "SQS_USER_LIST_IMPORT_URL"  = "https://sqs.us-east-1.amazonaws.com/996905175585/UserListSearch-Prod-UserListImport"
              - "USER_LIST_SEARCH_URI"      = "https://user-list-search.readitlater.com"
            } -> null
        }

      - ephemeral_storage {
          - size = 512 -> null
        }

      - logging_config {
          - log_format            = "Text" -> null
          - log_group             = "/aws/lambda/UserListSearch-Prod-UnifiedEventsConsumer" -> null
            # (2 unchanged attributes hidden)
        }

      - tracing_config {
          - mode = "Active" -> null
        }
    }

  # aws_sns_topic_subscription.list_events_sns_topic_subscription will be created
  + resource "aws_sns_topic_subscription" "list_events_sns_topic_subscription" {
      + arn                             = (known after apply)
      + confirmation_timeout_in_minutes = 1
      + confirmation_was_authenticated  = (known after apply)
      + endpoint                        = "arn:aws:sqs:us-east-1:996905175585:UserListSearch-Prod-UserListEvents"
      + endpoint_auto_confirms          = false
      + filter_policy_scope             = (known after apply)
      + id                              = (known after apply)
      + owner_id                        = (known after apply)
      + pending_confirmation            = (known after apply)
      + protocol                        = "sqs"
      + raw_message_delivery            = false
      + topic_arn                       = "arn:aws:sns:us-east-1:996905175585:PocketEventBridge-Prod-ListEvents"
    }

  # aws_sqs_queue_policy.user_events_sns_topic_dlq_policy will be updated in-place
  ~ resource "aws_sqs_queue_policy" "user_events_sns_topic_dlq_policy" {
        id        = "https://sqs.us-east-1.amazonaws.com/996905175585/UserListSearch-Prod-SNS-Topic-DLQ"
      ~ policy    = jsonencode(
          ~ {
              ~ Statement = [
                  ~ {
                      ~ Condition = {
                          ~ ArnEquals = {
                              ~ "aws:SourceArn" = "arn:aws:sns:us-east-1:996905175585:PocketEventBridge-Prod-UserEvents" -> [
                                  + "arn:aws:sns:us-east-1:996905175585:PocketEventBridge-Prod-UserEvents",
                                  + "arn:aws:sns:us-east-1:996905175585:PocketEventBridge-Prod-ListEvents",
                                ]
                            }
                        }
                        # (4 unchanged attributes hidden)
                    },
                ]
                # (1 unchanged attribute hidden)
            }
        )
        # (1 unchanged attribute hidden)
    }

  # aws_sqs_queue_policy.user_events_sqs_policy will be updated in-place
  ~ resource "aws_sqs_queue_policy" "user_events_sqs_policy" {
        id        = "https://sqs.us-east-1.amazonaws.com/996905175585/UserListSearch-Prod-UserListEvents"
      ~ policy    = jsonencode(
          ~ {
              ~ Statement = [
                  ~ {
                      ~ Condition = {
                          ~ ArnEquals = {
                              ~ "aws:SourceArn" = "arn:aws:sns:us-east-1:996905175585:PocketEventBridge-Prod-UserEvents" -> [
                                  + "arn:aws:sns:us-east-1:996905175585:PocketEventBridge-Prod-UserEvents",
                                  + "arn:aws:sns:us-east-1:996905175585:PocketEventBridge-Prod-ListEvents",
                                ]
                            }
                        }
                        # (4 unchanged attributes hidden)
                    },
                ]
                # (1 unchanged attribute hidden)
            }
        )
        # (1 unchanged attribute hidden)
    }

  # module.dashboard_alarm.aws_cloudwatch_dashboard.dashboard["sla"] will be updated in-place
  ~ resource "aws_cloudwatch_dashboard" "dashboard" {
      ~ dashboard_body = jsonencode(
          ~ {
              ~ widgets = [
                    # (7 unchanged elements hidden)
                    {
                        height     = 6
                        properties = {
                            metrics = [
                                [
                                    "AWS/Lambda",
                                    "Invocations",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImport",
                                    {
                                        color = "#1f77b4"
                                        id    = "list_item_import_lambda_invocations"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Errors",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImport",
                                    {
                                        color = "#d62728"
                                        id    = "list_item_import_lambda_errors"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "ConcurrentExecutions",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImport",
                                    {
                                        color = "#2ca02c"
                                        id    = "list_item_import_lambda_concurrent_executions"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Throttles",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImport",
                                    {
                                        color = "#ff7f0e"
                                        id    = "list_item_import_lambda_throttles"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Invocations",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdate",
                                    {
                                        color = "#1f77b4"
                                        id    = "list_item_update_lambda_invocations"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Errors",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdate",
                                    {
                                        color = "#d62728"
                                        id    = "list_item_update_lambda_errors"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "ConcurrentExecutions",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdate",
                                    {
                                        color = "#2ca02c"
                                        id    = "list_item_update_lambda_concurrent_executions"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Throttles",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdate",
                                    {
                                        color = "#ff7f0e"
                                        id    = "list_item_update_lambda_throttles"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                            ]
                            period  = 60
                            region  = "us-east-1"
                            stacked = false
                            stat    = "Average"
                            title   = "Lambda (Kinesis)"
                        }
                        type       = "metric"
                        width      = 12
                        x          = 12
                        y          = 18
                    },
                  - {
                      - height     = 6
                      - properties = {
                          - metrics = [
                              - [
                                  - "AWS/Lambda",
                                  - "Duration",
                                  - "FunctionName",
                                  - "UserListSearch-Prod-UnifiedEventsConsumer",
                                  - {
                                      - id   = "event_consumer_lambda_duration"
                                      - stat = "Sum"
                                    },
                                ],
                              - [
                                  - "AWS/Lambda",
                                  - "Errors",
                                  - "FunctionName",
                                  - "UserListSearch-Prod-UnifiedEventsConsumer",
                                  - {
                                      - id   = "event_consumer_lambda_errors"
                                      - stat = "Sum"
                                    },
                                ],
                              - [
                                  - "AWS/Lambda",
                                  - "IteratorAge",
                                  - "FunctionName",
                                  - "UserListSearch-Prod-UnifiedEventsConsumer",
                                  - {
                                      - id    = "event_consumer_lambda_iterator_age"
                                      - stat  = "Sum"
                                      - yAxis = "right"
                                    },
                                ],
                            ]
                          - period  = 60
                          - region  = "us-east-1"
                          - stacked = false
                          - stat    = "Average"
                          - title   = "Event Kinesis Consumer"
                        }
                      - type       = "metric"
                      - width      = 12
                      - x          = 0
                      - y          = 24
                    },
                    {
                        height     = 6
                        properties = {
                            metrics = [
                                [
                                    "AWS/Lambda",
                                    "Invocations",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImportBackfill",
                                    {
                                        color = "#1f77b4"
                                        id    = "list_item_import_backfill_lambda_invocations"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Errors",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImportBackfill",
                                    {
                                        color = "#d62728"
                                        id    = "list_item_import_backfill_lambda_errors"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "ConcurrentExecutions",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImportBackfill",
                                    {
                                        color = "#2ca02c"
                                        id    = "list_item_import_backfill_lambda_concurrent_executions"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Throttles",
                                    "FunctionName",
                                    "UserListSearch-Prod-UserListImportBackfill",
                                    {
                                        color = "#ff7f0e"
                                        id    = "list_item_import_backfill_lambda_throttles"
                                        stat  = "Sum"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Invocations",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdateBackfill",
                                    {
                                        color = "#1f77b4"
                                        id    = "list_item_update_backfill_lambda_invocations"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Errors",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdateBackfill",
                                    {
                                        color = "#d62728"
                                        id    = "list_item_update_backfill_lambda_errors"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "ConcurrentExecutions",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdateBackfill",
                                    {
                                        color = "#2ca02c"
                                        id    = "list_item_update_backfill_lambda_concurrent_executions"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                                [
                                    "AWS/Lambda",
                                    "Throttles",
                                    "FunctionName",
                                    "UserListSearch-Prod-ItemUpdateBackfill",
                                    {
                                        color = "#ff7f0e"
                                        id    = "list_item_update_backfill_lambda_throttles"
                                        stat  = "Sum"
                                        yAxis = "right"
                                    },
                                ],
                            ]
                            period  = 60
                            region  = "us-east-1"
                            stacked = false
                            stat    = "Average"
                            title   = "Lambda (Seperate Backfill Process)"
                        }
                        type       = "metric"
                        width      = 12
                        x          = 12
                        y          = 24
                    },
                  ~ {
                      ~ properties = {
                          ~ alarms  = [
                                # (1 unchanged element hidden)
                                "arn:aws:cloudwatch:us-east-1:996905175585:alarm:UserListSearch-Prod-ElasticSearchWriteBlockedCritical",
                              - "arn:aws:cloudwatch:us-east-1:996905175585:alarm:UserListSearch-Prod-EventConsumerLambdaErrors",
                                "arn:aws:cloudwatch:us-east-1:996905175585:alarm:UserListSearch-Prod-UserListImportErrors",
                                # (3 unchanged elements hidden)
                            ]
                            # (3 unchanged attributes hidden)
                        }
                        # (5 unchanged attributes hidden)
                    },
                ]
            }
        )
        id             = "UserListSearch-Prod"
        # (2 unchanged attributes hidden)
    }

  # module.dashboard_alarm.aws_cloudwatch_metric_alarm.metric_alarms["event_consumer_lambda_errors"] will be destroyed
  # (because key ["event_consumer_lambda_errors"] is not in for_each map)
  - resource "aws_cloudwatch_metric_alarm" "metric_alarms" {
      - actions_enabled                       = true -> null
      - alarm_actions                         = [] -> null
      - alarm_description                     = "More than 1 error for 3 consecutive minutes" -> null
      - alarm_name                            = "UserListSearch-Prod-EventConsumerLambdaErrors" -> null
      - arn                                   = "arn:aws:cloudwatch:us-east-1:996905175585:alarm:UserListSearch-Prod-EventConsumerLambdaErrors" -> null
      - comparison_operator                   = "GreaterThanThreshold" -> null
      - datapoints_to_alarm                   = 0 -> null
      - dimensions                            = {} -> null
      - evaluation_periods                    = 10 -> null
      - id                                    = "UserListSearch-Prod-EventConsumerLambdaErrors" -> null
      - insufficient_data_actions             = [] -> null
      - ok_actions                            = [] -> null
      - period                                = 0 -> null
      - tags                                  = {} -> null
      - tags_all                              = {
          - "app_code"       = "pocket"
          - "component_code" = "pocket-userlistsearch"
          - "costCenter"     = "Pocket"
          - "env_code"       = "prod"
          - "environment"    = "Prod"
          - "owner"          = "Pocket"
          - "service"        = "UserListSearch"
        } -> null
      - threshold                             = 10 -> null
      - treat_missing_data                    = "missing" -> null
        # (7 unchanged attributes hidden)

      - metric_query {
          - id          = "event_consumer_lambda_duration" -> null
          - period      = 0 -> null
          - return_data = false -> null
            # (3 unchanged attributes hidden)

          - metric {
              - dimensions  = {
                  - "FunctionName" = "UserListSearch-Prod-UnifiedEventsConsumer"
                } -> null
              - metric_name = "Duration" -> null
              - namespace   = "AWS/Lambda" -> null
              - period      = 60 -> null
              - stat        = "Sum" -> null
                # (1 unchanged attribute hidden)
            }
        }
      - metric_query {
          - id          = "event_consumer_lambda_errors" -> null
          - period      = 0 -> null
          - return_data = true -> null
            # (3 unchanged attributes hidden)

          - metric {
              - dimensions  = {
                  - "FunctionName" = "UserListSearch-Prod-UnifiedEventsConsumer"
                } -> null
              - metric_name = "Errors" -> null
              - namespace   = "AWS/Lambda" -> null
              - period      = 60 -> null
              - stat        = "Sum" -> null
                # (1 unchanged attribute hidden)
            }
        }
      - metric_query {
          - id          = "event_consumer_lambda_iterator_age" -> null
          - period      = 0 -> null
          - return_data = false -> null
            # (3 unchanged attributes hidden)

          - metric {
              - dimensions  = {
                  - "FunctionName" = "UserListSearch-Prod-UnifiedEventsConsumer"
                } -> null
              - metric_name = "IteratorAge" -> null
              - namespace   = "AWS/Lambda" -> null
              - period      = 60 -> null
              - stat        = "Sum" -> null
                # (1 unchanged attribute hidden)
            }
        }
    }

Plan: 1 to add, 3 to change, 9 to destroy.