From d801154cf2ddf5f0fc00117f83291d40abc52409 Mon Sep 17 00:00:00 2001 From: James Herr Date: Thu, 16 May 2024 12:16:56 -0500 Subject: [PATCH] Added alert terraform for token errors Co-Authored-By: Samuel Aquino Co-Authored-By: Sylvie <38440028+somesylvie@users.noreply.github.com> Co-Authored-By: Jorge Lopez <49923512+jorg3lopez@users.noreply.github.com> --- .github/workflows/cicd.yml | 2 +- .github/workflows/dev-deploy.yml | 2 +- .github/workflows/internal-deploy.yml | 2 +- .github/workflows/prod-deploy.yml | 2 +- .github/workflows/terraform-ci-deploy.yml | 2 +- operations/environments/dev/main.tf | 2 +- operations/environments/dev/variables.tf | 2 +- operations/environments/internal/main.tf | 2 +- operations/environments/internal/variables.tf | 2 +- operations/environments/pr/main.tf | 2 +- operations/environments/pr/variables.tf | 2 +- operations/environments/prd/main.tf | 2 +- operations/environments/prd/variables.tf | 2 +- operations/environments/stg/main.tf | 2 +- operations/environments/stg/variables.tf | 2 +- operations/template/alert.tf | 34 ++++++++++++++++++- operations/template/variables.tf | 2 +- 17 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 796a46af5..7d974b073 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -29,7 +29,7 @@ jobs: VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }} VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}} VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} - TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}" + TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}" staging-deploy: name: Staging Application Deploy diff --git a/.github/workflows/dev-deploy.yml b/.github/workflows/dev-deploy.yml index 9d5a4483a..c9a301c25 100644 --- a/.github/workflows/dev-deploy.yml +++ b/.github/workflows/dev-deploy.yml @@ -22,7 +22,7 @@ jobs: VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }} VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}} VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} - TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}" + TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}" dev-deploy: name: Dev Application Deploy diff --git a/.github/workflows/internal-deploy.yml b/.github/workflows/internal-deploy.yml index 02ce84272..5228125ae 100644 --- a/.github/workflows/internal-deploy.yml +++ b/.github/workflows/internal-deploy.yml @@ -18,7 +18,7 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}" + TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}" internal-deploy: name: Internal Application Deploy diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml index 637f8c7b3..e3bc2a729 100644 --- a/.github/workflows/prod-deploy.yml +++ b/.github/workflows/prod-deploy.yml @@ -26,7 +26,7 @@ jobs: VPN_CA_CERTIFICATE: ${{ secrets.VPN_CA_CERTIFICATE }} VPN_GITHUB_CERTIFICATE: ${{ secrets.VPN_GITHUB_CERTIFICATE}} VPN_GITHUB_SECRET_KEY: ${{ secrets.VPN_GITHUB_SECRET_KEY }} - TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}" + TERRAFORM_APPLY_PARAMETERS: -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}" prod-deploy: name: Prod Application Deploy diff --git a/.github/workflows/terraform-ci-deploy.yml b/.github/workflows/terraform-ci-deploy.yml index a78da6b1e..692befa26 100644 --- a/.github/workflows/terraform-ci-deploy.yml +++ b/.github/workflows/terraform-ci-deploy.yml @@ -40,7 +40,7 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - TERRAFORM_APPLY_PARAMETERS: -var="pr_number=${{ github.event.number }}" -var="alert_slack_webhook=${{ secrets.ALERT_SLACK_WEBHOOK }}" + TERRAFORM_APPLY_PARAMETERS: -var="pr_number=${{ github.event.number }}" -var="alert_slack_email=${{ secrets.ALERT_SLACK_EMAIL }}" terraform-deploy-skip: # runs when the PR doesn't have any changes that require the PR deploy; this ensures we get the appropriate required PR checks diff --git a/operations/environments/dev/main.tf b/operations/environments/dev/main.tf index d3cbc5d11..6a876bbc1 100644 --- a/operations/environments/dev/main.tf +++ b/operations/environments/dev/main.tf @@ -30,5 +30,5 @@ module "template" { environment = "dev" deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIWvb3sLkOQtcwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1MjM0OFoXDTI3MDMwODE1MjM0OFowETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt1v2bAATE7IOJkqUrbwQw6X99fi3Ywf1bv0uZ0gGDjG10H+PB2BUzZ94RNcB4Oezi6t+/WAQUkhRozemFkegSkfKHEehAT6nu6OBXKt2rH/oJtpKR791ab9H9aQ6e5LO9OZ237QL6XikhGG7HXqG9ndYnhBYPy2/pd8VV6ZwqMR3PkfBJaC4tKy4d8dim+PMpT5rqPGbsf9H+dydvG6JOKZiHb3/yqi6fqoise1yY64aDwFC9MbEbtgXpvmBFsei2PA/XH5FqE6F/kyCg7mO5TSYYEqx0PCTPmICAT4iw5ELMyAhVKL2OpMjqw5YAYr/TGqlfyEYpBBQMvC3K9OmUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU5idI+AFsHn8BjvNBE5ShE+aFor0wDQYJKoZIhvcNAQELBQADggEBAJkDtuHj4QGyXtooiM7xfHZ/lGdDZvF+KAVfFKAlsIO8y1NS2iAeNT6MmampwzWzXIUMk9vxvALUoh2MJkWP5CX2e3vDj2lGpbhK5//rfWDin1/jj28+KZzSVsk4i/EkdBWW7eCKU401rafVOjSLmM5mfDTAHNrFxzQWJF5WL7TxrQw7chrnpy4v0V7/y4h+QsQja8LXx9keEdB2BQSjndAqxB9dblFALantpuEOM2pS3GCaC2REXSnKsgSEQoVL07MSndpCpdv5bsEkppM5LBC6gL66a43Lho3kSCm4ZU51mjJtNwadeBXpHjkJ1yiBA7CG/Roa+THAiV+VMP75g3E=" # pragma: allowlist secret - alert_slack_webhook = var.alert_slack_webhook + alert_slack_email = var.alert_slack_email } diff --git a/operations/environments/dev/variables.tf b/operations/environments/dev/variables.tf index 92f15375a..5eaae8a8d 100644 --- a/operations/environments/dev/variables.tf +++ b/operations/environments/dev/variables.tf @@ -1,4 +1,4 @@ -variable "alert_slack_webhook" { +variable "alert_slack_email" { type = string nullable = false sensitive = true diff --git a/operations/environments/internal/main.tf b/operations/environments/internal/main.tf index 128ce8bdc..8b552b115 100644 --- a/operations/environments/internal/main.tf +++ b/operations/environments/internal/main.tf @@ -29,5 +29,5 @@ module "template" { environment = "internal" deployer_id = "d59c2c86-de5e-41b7-a752-0869a73f5a60" //github app registration in Flexion Azure Entra - alert_slack_webhook = var.alert_slack_webhook + alert_slack_email = var.alert_slack_email } diff --git a/operations/environments/internal/variables.tf b/operations/environments/internal/variables.tf index 92f15375a..5eaae8a8d 100644 --- a/operations/environments/internal/variables.tf +++ b/operations/environments/internal/variables.tf @@ -1,4 +1,4 @@ -variable "alert_slack_webhook" { +variable "alert_slack_email" { type = string nullable = false sensitive = true diff --git a/operations/environments/pr/main.tf b/operations/environments/pr/main.tf index 6709e55fc..1cfb9d327 100644 --- a/operations/environments/pr/main.tf +++ b/operations/environments/pr/main.tf @@ -41,7 +41,7 @@ module "template" { environment = "pr${var.pr_number}" deployer_id = "d59c2c86-de5e-41b7-a752-0869a73f5a60" //github app registration in Flexion Azure Entra - alert_slack_webhook = var.alert_slack_webhook + alert_slack_email = var.alert_slack_email depends_on = [azurerm_resource_group.group, azurerm_virtual_network.vnet] } diff --git a/operations/environments/pr/variables.tf b/operations/environments/pr/variables.tf index 24c11111c..256ff5b22 100644 --- a/operations/environments/pr/variables.tf +++ b/operations/environments/pr/variables.tf @@ -3,7 +3,7 @@ variable "pr_number" { nullable = false } -variable "alert_slack_webhook" { +variable "alert_slack_email" { type = string nullable = false sensitive = true diff --git a/operations/environments/prd/main.tf b/operations/environments/prd/main.tf index d9c3743a8..1592e6411 100644 --- a/operations/environments/prd/main.tf +++ b/operations/environments/prd/main.tf @@ -30,5 +30,5 @@ module "template" { environment = "prd" deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIeHnOQDhz00AwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1NDA1M1oXDTI3MDMwODE1NDA1M1owETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAkgeXZ6ReEQ5HAqlXULUUdVfCMtMPmlTeCFFkhD9i5E5lRg78PyJqczHMzCB6l83O/PrLWXjT3/s/R58cfeHJg/SndGwt/2uKhj1kNW7Ivc8kF0pgSL3lDR+NSj5OPda45EY30ZlTjgygmb9MjfCT2BmgjGcfUbgm0jzgDZsk7bLUUJkL38DJP+v2M6sDxyxMjoY9gJ1Kq5Fg81serJlZHaACShuuhgiKqH3+hwvIPluK8Y40FWfiKpGRjdkAXGTmB+afMeA4L1amyticIPzzOytIHFIDMOKgJRL62UQe+alzubXkYbDtEgDCOwF8k5TRiu9MUwID34CLkp2VWnLnUwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUHyrypPmh+KVb2sspeGsxboG1hQwwDQYJKoZIhvcNAQELBQADggEBAGmfFRLgqLQxedGHeXQoajHzhCvk+62lDR1xy0s2mklA3eRxzOyaXRPgmM6lbGBm6LdLxo5nxGgfD4h2vOBZl4MXOFLryLm97QtDZ34YkxGn+tugUAXpWBB/EJIynib1Ywyg6Kv6g3oYjf2bc8Ae9bOWGR0FtOGn8TvmSzKLXoUwQd0u9DEA774YtpvPxHxw69uyf8x2nekpyWNyFbR6DWJEA9M+BHeR0oGEGoc5FH6zTgstbdeNVou3NNQlRKlWD26vWeCeQvbKDK5+KuOPjjDTimGdx1GfA9z/ai/pX+K/NKvvC4JXQdW7jYYu3QFglP70esT9mBCxVQbXd49oD9M=" # pragma: allowlist secret - alert_slack_webhook = var.alert_slack_webhook + alert_slack_email = var.alert_slack_email } diff --git a/operations/environments/prd/variables.tf b/operations/environments/prd/variables.tf index 92f15375a..5eaae8a8d 100644 --- a/operations/environments/prd/variables.tf +++ b/operations/environments/prd/variables.tf @@ -1,4 +1,4 @@ -variable "alert_slack_webhook" { +variable "alert_slack_email" { type = string nullable = false sensitive = true diff --git a/operations/environments/stg/main.tf b/operations/environments/stg/main.tf index ad16b4640..113cdfe04 100644 --- a/operations/environments/stg/main.tf +++ b/operations/environments/stg/main.tf @@ -30,5 +30,5 @@ module "template" { environment = "stg" deployer_id = "f5feabe7-5d37-40ba-94f2-e5c0760b4561" //github app registration in CDC Azure Entra vpn_root_certificate = "MIIC5jCCAc6gAwIBAgIIUC720RvICDQwDQYJKoZIhvcNAQELBQAwETEPMA0GA1UEAxMGVlBOIENBMB4XDTI0MDMwODE1MzY1MloXDTI3MDMwODE1MzY1MlowETEPMA0GA1UEAxMGVlBOIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAtn+H6PbB2y/JmoTTNuxljVY7I02BblPmnzwzQhEPAZjoMVQTsEvS5rr/ILLFFF5FdTcyPYJpD5Tvd+w1v62xV4QhZSFpSSyfRvsi6uzOLOyDOhVN++GWAjKyTvaOO654JwX/qj7nHSYQLQFtnf9OkixZazO8o0snXpGCSYKgxhBox6+XyZpjjwoFt+wMrNalrAOWCtAp/pgIB7xyStcWyGEi7vACiV+7rzI2Kxh+PfaltS4wU1vWN7jN2GxMbVG3539ybiT4fpoGuDWjZ7t7tp1LgQa1n7tlvNR0W01pdt7U/fPL9ynfyuP8Wph8eetW9THYtJkBTNk7KyhE+z36TwIDAQABo0IwQDAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQU85kmRn9Cnq9LjFeKjwrMUhgo3xowDQYJKoZIhvcNAQELBQADggEBAHjzfciR/TqJojJ3xd2AmMQev5Aw6Wf9gFfhv0eb9bmqyeJ23bYhOvqWxIxb01TBp5CNhWgWuUE68cQpEqafu9JOITDk9GtQ9m6/4sHOhzqM11beGqKlomQuT+I/M/gS0pUcr//W7riTkOQQI6DHKgpoGoRXpk9/V5GrwQauZjy1hRyRpVlg4xDgJJqRr5PKUErtA07DYck+AblJW4msglfyM2HTvvMLNdsmiZmjFdU1osT0WT/W9nY+RGadAo47x6qknpFoDoVtIQ3XNH3C5Scl1bGphfQdmEjNVhg7a8gSWat7n1OjFiz3OvTqy5MsssmRz4WlOM5+xOhiT2OambA=" # pragma: allowlist secret - alert_slack_webhook = var.alert_slack_webhook + alert_slack_email = var.alert_slack_email } diff --git a/operations/environments/stg/variables.tf b/operations/environments/stg/variables.tf index 92f15375a..5eaae8a8d 100644 --- a/operations/environments/stg/variables.tf +++ b/operations/environments/stg/variables.tf @@ -1,4 +1,4 @@ -variable "alert_slack_webhook" { +variable "alert_slack_email" { type = string nullable = false sensitive = true diff --git a/operations/template/alert.tf b/operations/template/alert.tf index cce6f17c8..480fbd88c 100644 --- a/operations/template/alert.tf +++ b/operations/template/alert.tf @@ -5,7 +5,7 @@ resource "azurerm_monitor_action_group" "monitor" { email_receiver { name = "cdcti-flexion-slack-email-receiver" - email_address = var.alert_slack_webhook + email_address = var.alert_slack_email } } @@ -29,3 +29,35 @@ resource "azurerm_monitor_metric_alert" "alert" { action_group_id = azurerm_monitor_action_group.monitor.id } } + +resource "azurerm_monitor_scheduled_query_rules_alert" "example" { + name = "cdcti-${var.environment}-api-log-token-alert" + location = data.azurerm_resource_group.group.location + resource_group_name = data.azurerm_resource_group.group.name + + action { + action_group = [azurerm_monitor_action_group.monitor.id] + email_subject = "FATAL: The access token has expired!" + } + + data_source_id = azurerm_linux_web_app.api.id + description = "Alert when total results cross threshold" + enabled = true + + query = <<-QUERY + AppServiceConsoleLogs + | where ResultDescription has "FATAL: The access token has expired." + and TimeGenerated >= ago(30m) + and TimeGenerated <= now() + | summarize count() + QUERY + + severity = 3 + frequency = 10 + time_window = 30 + + trigger { + operator = "GreaterThan" + threshold = 5 + } +} diff --git a/operations/template/variables.tf b/operations/template/variables.tf index 28bef311c..2e492ee87 100644 --- a/operations/template/variables.tf +++ b/operations/template/variables.tf @@ -14,7 +14,7 @@ variable "vpn_root_certificate" { default = null } -variable "alert_slack_webhook" { +variable "alert_slack_email" { type = string nullable = false sensitive = true