Merge pull request #8 from adinhodovic/allow-disabling-alerts

feat(alerts): Allow disabling alerts
adinhodovic · Jan 19, 2025 · 33b1e35 · 33b1e35
2 parents c28f829 + f4fcbce
commit 33b1e35
Show file tree

Hide file tree

Showing 8 changed files with 101 additions and 103 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,4 @@
 vendor
 tmp
 jsonnetfile.lock.json
-./dashboards_out/lint
+dashboards_out/.lint
diff --git a/alerts/alerts.libsonnet b/alerts/alerts.libsonnet
@@ -3,29 +3,32 @@
     groups+: [
       {
         name: 'argo-cd',
-        rules: [
+        rules: std.prune([
           {
-            alert: 'ArgoCdAppOutOfSync',
+            alert: 'ArgoCdAppSyncFailed',
             expr: |||
               sum(
-                argocd_app_info{
-                  %(argoCdSelector)s,
-                  sync_status!="Synced"
-                }
-              ) by (job, dest_server, project, name, sync_status)
-              > 0
+                round(
+                  increase(
+                    argocd_app_sync_total{
+                      %(argoCdSelector)s,
+                      phase!="Succeeded"
+                    }[%(argoCdAppSyncInterval)s]
+                  )
+                )
+              ) by (job, dest_server, project, name, phase) > 0
             ||| % $._config,
             labels: {
               severity: 'warning',
             },
-            'for': $._config.argoCdAppOutOfSyncFor,
+            'for': '1m',
             annotations: {
-              summary: 'An ArgoCD Application is Out Of Sync.',
-              description: 'The application {{ $labels.dest_server }}/{{ $labels.project }}/{{ $labels.name }} is out of sync with the sync status {{ $labels.sync_status }} for the past %s.' % $._config.argoCdAppOutOfSyncFor,
+              summary: 'An ArgoCD Application has Failed to Sync.',
+              description: 'The application {{ $labels.dest_server }}/{{ $labels.project }}/{{ $labels.name }} has failed to sync with the status {{ $labels.phase }} the past %s.' % $._config.argoCdAppSyncInterval,
               dashboard_url: $._config.applicationOverviewDashboardUrl + '?var-dest_server={{ $labels.dest_server }}&var-project={{ $labels.project }}&var-application={{ $labels.name }}',
             },
           },
-          {
+          if $._config.argoCdAppUnhealthyEnabled then {
             alert: 'ArgoCdAppUnhealthy',
             expr: |||
               sum(
@@ -46,49 +49,46 @@
               dashboard_url: $._config.applicationOverviewDashboardUrl + '?var-dest_server={{ $labels.dest_server }}&var-project={{ $labels.project }}&var-application={{ $labels.name }}',
             },
           },
-          {
-            alert: 'ArgoCdAppAutoSyncDisabled',
+          if $._config.argoCdAppOutOfSyncEnabled then {
+            alert: 'ArgoCdAppOutOfSync',
             expr: |||
               sum(
                 argocd_app_info{
                   %(argoCdSelector)s,
-                  autosync_enabled!="true",
-                  name!~"%(argoAutoSyncDisabledIgnoredApps)s"
+                  sync_status!="Synced"
                 }
-              ) by (job, dest_server, project, name, autosync_enabled)
+              ) by (job, dest_server, project, name, sync_status)
               > 0
             ||| % $._config,
             labels: {
               severity: 'warning',
             },
-            'for': $._config.argoCdAppAutoSyncDisabledFor,
+            'for': $._config.argoCdAppOutOfSyncFor,
             annotations: {
-              summary: 'An ArgoCD Application has AutoSync Disabled.',
-              description: 'The application {{ $labels.dest_server }}/{{ $labels.project }}/{{ $labels.name }} has autosync disabled for the past %s.' % $._config.argoCdAppAutoSyncDisabledFor,
+              summary: 'An ArgoCD Application is Out Of Sync.',
+              description: 'The application {{ $labels.dest_server }}/{{ $labels.project }}/{{ $labels.name }} is out of sync with the sync status {{ $labels.sync_status }} for the past %s.' % $._config.argoCdAppOutOfSyncFor,
               dashboard_url: $._config.applicationOverviewDashboardUrl + '?var-dest_server={{ $labels.dest_server }}&var-project={{ $labels.project }}&var-application={{ $labels.name }}',
             },
           },
-          {
-            alert: 'ArgoCdAppSyncFailed',
+          if $._config.argoCdAppAutoSyncDisabledEnabled then {
+            alert: 'ArgoCdAppAutoSyncDisabled',
             expr: |||
               sum(
-                round(
-                  increase(
-                    argocd_app_sync_total{
-                      %(argoCdSelector)s,
-                      phase!="Succeeded"
-                    }[%(argoCdAppSyncInterval)s]
-                  )
-                )
-              ) by (job, dest_server, project, name, phase) > 0
+                argocd_app_info{
+                  %(argoCdSelector)s,
+                  autosync_enabled!="true",
+                  name!~"%(argoAutoSyncDisabledIgnoredApps)s"
+                }
+              ) by (job, dest_server, project, name, autosync_enabled)
+              > 0
             ||| % $._config,
             labels: {
               severity: 'warning',
             },
-            'for': '1m',
+            'for': $._config.argoCdAppAutoSyncDisabledFor,
             annotations: {
-              summary: 'An ArgoCD Application has Failed to Sync.',
-              description: 'The application {{ $labels.dest_server }}/{{ $labels.project }}/{{ $labels.name }} has failed to sync with the status {{ $labels.phase }} the past %s.' % $._config.argoCdAppSyncInterval,
+              summary: 'An ArgoCD Application has AutoSync Disabled.',
+              description: 'The application {{ $labels.dest_server }}/{{ $labels.project }}/{{ $labels.name }} has autosync disabled for the past %s.' % $._config.argoCdAppAutoSyncDisabledFor,
               dashboard_url: $._config.applicationOverviewDashboardUrl + '?var-dest_server={{ $labels.dest_server }}&var-project={{ $labels.project }}&var-application={{ $labels.name }}',
             },
           },
@@ -116,7 +116,7 @@
               dashboard_url: $._config.notificationsOverviewDashboardUrl + '?var-job={{ $labels.job }}&var-exported_service={{ $labels.exported_service }}',
             },
           },
-        ],
+        ]),
       },
     ],
   },

diff --git a/config.libsonnet b/config.libsonnet
@@ -23,8 +23,11 @@ local annotation = g.dashboard.annotation;
 
     tags: ['ci/cd', 'argo-cd'],
 
+    argoCdAppOutOfSyncEnabled: true,
     argoCdAppOutOfSyncFor: '15m',
+    argoCdAppUnhealthyEnabled: true,
     argoCdAppUnhealthyFor: '15m',
+    argoCdAppAutoSyncDisabledEnabled: true,
     argoCdAppAutoSyncDisabledFor: '2h',
     argoCdAppSyncInterval: '10m',
     argoCdNotificationDeliveryInterval: '10m',

diff --git a/dashboards_out/.lint b/dashboards_out/.lint
diff --git a/dashboards_out/argo-cd-application-overview.json b/dashboards_out/argo-cd-application-overview.json
@@ -64,7 +64,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -115,7 +115,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -166,7 +166,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -217,7 +217,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -247,7 +247,7 @@
             "content": "No applications defined",
             "mode": "markdown"
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "title": "Application Badges",
          "type": "text"
       },
@@ -329,7 +329,7 @@
                }
             ]
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -435,7 +435,7 @@
                }
             ]
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -541,7 +541,7 @@
                }
             ]
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -647,7 +647,7 @@
                }
             ]
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -735,7 +735,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -785,7 +785,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -835,7 +835,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {

diff --git a/dashboards_out/argo-cd-notifications-overview.json b/dashboards_out/argo-cd-notifications-overview.json
@@ -64,7 +64,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {
@@ -115,7 +115,7 @@
                "sort": "desc"
             }
          },
-         "pluginVersion": "v11.0.0",
+         "pluginVersion": "v11.4.0",
          "targets": [
             {
                "datasource": {