Added alert terraform for token errors

Co-Authored-By: Samuel Aquino <[email protected]> Co-Authored-By: Sylvie <[email protected]> Co-Authored-By: Jorge Lopez <[email protected]>
CDCgov · May 16, 2024 · d801154 · d801154
1 parent 81170db
commit d801154
Show file tree

Hide file tree

Showing 17 changed files with 49 additions and 17 deletions.
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
@@ -29,7 +29,7 @@ jobs:
       VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }}
       VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}}
       VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }}
-      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}"
+      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}"
 
   staging-deploy:
     name: Staging Application Deploy

diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml
@@ -22,7 +22,7 @@ jobs:
       VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }}
       VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}}
       VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }}
-      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}"
+      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}"
 
   dev-deploy:
     name: Dev Application Deploy

diff --git a/.github/workflows/internal-deploy.yml b/.github/workflows/internal-deploy.yml
@@ -18,7 +18,7 @@ jobs:
       AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
       AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
       AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}"
+      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}"
 
   internal-deploy:
     name: Internal Application Deploy

diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml
@@ -26,7 +26,7 @@ jobs:
       VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }}
       VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}}
       VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }}
-      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}"
+      TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}"
 
   prod-deploy:
     name: Prod Application Deploy

diff --git a/.github/workflows/terraform-ci-deploy.yml b/.github/workflows/terraform-ci-deploy.yml
@@ -40,7 +40,7 @@ jobs:
       AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
       AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
       AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
-      TERRAFORM_APPLY_PARAMETERS: -var="pr_number=${{ github.event.number }}" -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}"
+      TERRAFORM_APPLY_PARAMETERS: -var="pr_number=${{ github.event.number }}" -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}"
 
 
   terraform-deploy-skip:  # runs when the PR doesn't have any changes that require the PR deploy; this ensures we get the appropriate required PR checks

diff --git a/operations/environments/dev/main.tf b/operations/environments/dev/main.tf
@@ -30,5 +30,5 @@ module "template" {
   environment          = "dev"
   deployer_id          = "f5feabe7-5d37-40ba-94f2-e5c0760b4561"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 //github app registration in CDC Azure Entra
   vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIWvb3sLkOQtcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1MjM0OFoXDTI3MDMwODE1MjM0OFowETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt1v2bAATE7IOJkqUrbwQw6X99fi3Ywf1bv0uZ0gGDjG10H+PB2BUzZ94RNcB4Oezi6t+/WAQUkhRozemFkegSkfKHEehAT6nu6OBXKt2rH/oJtpKR791ab9H9aQ6e5LO9OZ237QL6XikhGG7HXqG9ndYnhBYPy2/pd8VV6ZwqMR3PkfBJaC4tKy4d8dim+PMpT5rqPGbsf9H+dydvG6JOKZiHb3/yqi6fqoise1yY64aDwFC9MbEbtgXpvmBFsei2PA/XH5FqE6F/kyCg7mO5TSYYEqx0PCTPmICAT4iw5ELMyAhVKL2OpMjqw5YAYr/TGqlfyEYpBBQMvC3K9OmUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU5idI+AFsHn8BjvNBE5ShE+aFor0wDQYJKoZIhvcNAQELBQADggEBAJkDtuHj4QGyXtooiM7xfHZ/lGdDZvF+KAVfFKAlsIO8y1NS2iAeNT6MmampwzWzXIUMk9vxvALUoh2MJkWP5CX2e3vDj2lGpbhK5//rfWDin1/jj28+KZzSVsk4i/EkdBWW7eCKU401rafVOjSLmM5mfDTAHNrFxzQWJF5WL7TxrQw7chrnpy4v0V7/y4h+QsQja8LXx9keEdB2BQSjndAqxB9dblFALantpuEOM2pS3GCaC2REXSnKsgSEQoVL07MSndpCpdv5bsEkppM5LBC6gL66a43Lho3kSCm4ZU51mjJtNwadeBXpHjkJ1yiBA7CG/Roa+THAiV+VMP75g3E=" # pragma: allowlist secret
-  alert_slack_webhook = var.alert_slack_webhook
+  alert_slack_email = var.alert_slack_email
 }
diff --git a/operations/environments/dev/variables.tf b/operations/environments/dev/variables.tf
@@ -1,4 +1,4 @@
-variable "alert_slack_webhook" {
+variable "alert_slack_email" {
   type = string
   nullable = false
   sensitive = true

diff --git a/operations/environments/internal/main.tf b/operations/environments/internal/main.tf
@@ -29,5 +29,5 @@ module "template" {
 
   environment = "internal"
   deployer_id = "d59c2c86-de5e-41b7-a752-0869a73f5a60" //github app registration in Flexion Azure Entra
-  alert_slack_webhook = var.alert_slack_webhook
+  alert_slack_email = var.alert_slack_email
 }
diff --git a/operations/environments/internal/variables.tf b/operations/environments/internal/variables.tf
@@ -1,4 +1,4 @@
-variable "alert_slack_webhook" {
+variable "alert_slack_email" {
   type = string
   nullable = false
   sensitive = true

diff --git a/operations/environments/pr/main.tf b/operations/environments/pr/main.tf
@@ -41,7 +41,7 @@ module "template" {
 
   environment = "pr${var.pr_number}"
   deployer_id = "d59c2c86-de5e-41b7-a752-0869a73f5a60" //github app registration in Flexion Azure Entra
-  alert_slack_webhook = var.alert_slack_webhook
+  alert_slack_email = var.alert_slack_email
 
   depends_on = [azurerm_resource_group.group, azurerm_virtual_network.vnet]
 }
diff --git a/operations/environments/pr/variables.tf b/operations/environments/pr/variables.tf
@@ -3,7 +3,7 @@ variable "pr_number" {
   nullable = false
 }
 
-variable "alert_slack_webhook" {
+variable "alert_slack_email" {
   type = string
   nullable = false
   sensitive = true

diff --git a/operations/environments/prd/main.tf b/operations/environments/prd/main.tf
@@ -30,5 +30,5 @@ module "template" {
   environment          = "prd"
   deployer_id          = "f5feabe7-5d37-40ba-94f2-e5c0760b4561"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 //github app registration in CDC Azure Entra
   vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIeHnOQDhz00AwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1NDA1M1oXDTI3MDMwODE1NDA1M1owETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAkgeXZ6ReEQ5HAqlXULUUdVfCMtMPmlTeCFFkhD9i5E5lRg78PyJqczHMzCB6l83O/PrLWXjT3/s/R58cfeHJg/SndGwt/2uKhj1kNW7Ivc8kF0pgSL3lDR+NSj5OPda45EY30ZlTjgygmb9MjfCT2BmgjGcfUbgm0jzgDZsk7bLUUJkL38DJP+v2M6sDxyxMjoY9gJ1Kq5Fg81serJlZHaACShuuhgiKqH3+hwvIPluK8Y40FWfiKpGRjdkAXGTmB+afMeA4L1amyticIPzzOytIHFIDMOKgJRL62UQe+alzubXkYbDtEgDCOwF8k5TRiu9MUwID34CLkp2VWnLnUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUHyrypPmh+KVb2sspeGsxboG1hQwwDQYJKoZIhvcNAQELBQADggEBAGmfFRLgqLQxedGHeXQoajHzhCvk+62lDR1xy0s2mklA3eRxzOyaXRPgmM6lbGBm6LdLxo5nxGgfD4h2vOBZl4MXOFLryLm97QtDZ34YkxGn+tugUAXpWBB/EJIynib1Ywyg6Kv6g3oYjf2bc8Ae9bOWGR0FtOGn8TvmSzKLXoUwQd0u9DEA774YtpvPxHxw69uyf8x2nekpyWNyFbR6DWJEA9M+BHeR0oGEGoc5FH6zTgstbdeNVou3NNQlRKlWD26vWeCeQvbKDK5+KuOPjjDTimGdx1GfA9z/ai/pX+K/NKvvC4JXQdW7jYYu3QFglP70esT9mBCxVQbXd49oD9M=" # pragma: allowlist secret
-  alert_slack_webhook = var.alert_slack_webhook
+  alert_slack_email = var.alert_slack_email
 }
diff --git a/operations/environments/prd/variables.tf b/operations/environments/prd/variables.tf
@@ -1,4 +1,4 @@
-variable "alert_slack_webhook" {
+variable "alert_slack_email" {
   type = string
   nullable = false
   sensitive = true

diff --git a/operations/environments/stg/main.tf b/operations/environments/stg/main.tf
@@ -30,5 +30,5 @@ module "template" {
   environment          = "stg"
   deployer_id          = "f5feabe7-5d37-40ba-94f2-e5c0760b4561"                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 //github app registration in CDC Azure Entra
   vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIUC720RvICDQwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1MzY1MloXDTI3MDMwODE1MzY1MlowETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtn+H6PbB2y/JmoTTNuxljVY7I02BblPmnzwzQhEPAZjoMVQTsEvS5rr/ILLFFF5FdTcyPYJpD5Tvd+w1v62xV4QhZSFpSSyfRvsi6uzOLOyDOhVN++GWAjKyTvaOO654JwX/qj7nHSYQLQFtnf9OkixZazO8o0snXpGCSYKgxhBox6+XyZpjjwoFt+wMrNalrAOWCtAp/pgIB7xyStcWyGEi7vACiV+7rzI2Kxh+PfaltS4wU1vWN7jN2GxMbVG3539ybiT4fpoGuDWjZ7t7tp1LgQa1n7tlvNR0W01pdt7U/fPL9ynfyuP8Wph8eetW9THYtJkBTNk7KyhE+z36TwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU85kmRn9Cnq9LjFeKjwrMUhgo3xowDQYJKoZIhvcNAQELBQADggEBAHjzfciR/TqJojJ3xd2AmMQev5Aw6Wf9gFfhv0eb9bmqyeJ23bYhOvqWxIxb01TBp5CNhWgWuUE68cQpEqafu9JOITDk9GtQ9m6/4sHOhzqM11beGqKlomQuT+I/M/gS0pUcr//W7riTkOQQI6DHKgpoGoRXpk9/V5GrwQauZjy1hRyRpVlg4xDgJJqRr5PKUErtA07DYck+AblJW4msglfyM2HTvvMLNdsmiZmjFdU1osT0WT/W9nY+RGadAo47x6qknpFoDoVtIQ3XNH3C5Scl1bGphfQdmEjNVhg7a8gSWat7n1OjFiz3OvTqy5MsssmRz4WlOM5+xOhiT2OambA=" # pragma: allowlist secret
-  alert_slack_webhook = var.alert_slack_webhook
+  alert_slack_email = var.alert_slack_email
 }
diff --git a/operations/environments/stg/variables.tf b/operations/environments/stg/variables.tf
@@ -1,4 +1,4 @@
-variable "alert_slack_webhook" {
+variable "alert_slack_email" {
   type = string
   nullable = false
   sensitive = true

diff --git a/operations/template/alert.tf b/operations/template/alert.tf
@@ -5,7 +5,7 @@ resource "azurerm_monitor_action_group" "monitor" {
 
   email_receiver {
     name          = "cdcti-flexion-slack-email-receiver"
-    email_address = var.alert_slack_webhook
+    email_address = var.alert_slack_email
   }
 }
 
@@ -29,3 +29,35 @@ resource "azurerm_monitor_metric_alert" "alert" {
     action_group_id = azurerm_monitor_action_group.monitor.id
   }
 }
+
+resource "azurerm_monitor_scheduled_query_rules_alert" "example" {
+  name                = "cdcti-${var.environment}-api-log-token-alert"
+  location            = data.azurerm_resource_group.group.location
+  resource_group_name = data.azurerm_resource_group.group.name
+
+  action {
+  action_group           = [azurerm_monitor_action_group.monitor.id]
+  email_subject          = "FATAL: The access token has expired!"
+  }
+
+  data_source_id = azurerm_linux_web_app.api.id
+  description    = "Alert when total results cross threshold"
+  enabled        = true
+
+  query       = <<-QUERY
+      AppServiceConsoleLogs
+      | where ResultDescription has "FATAL: The access token has expired."
+      and TimeGenerated >= ago(30m)
+      and TimeGenerated <= now()
+      | summarize count()
+    QUERY
+
+  severity    = 3
+  frequency   = 10
+  time_window = 30
+
+  trigger {
+    operator  = "GreaterThan"
+    threshold = 5
+  }
+}
diff --git a/operations/template/variables.tf b/operations/template/variables.tf
@@ -14,7 +14,7 @@ variable "vpn_root_certificate" {
   default  = null
 }
 
-variable "alert_slack_webhook" {
+variable "alert_slack_email" {
   type = string
   nullable = false
   sensitive = true