monodb-pgsql-health.sh: added more features to check_active_connectio…

…ns. variable CONN_LIMIT changed to CONN_LIMIT_PERCENT. removed check active services from all health scripts.
monobilisim · Jun 14, 2024 · 523a1b8 · 523a1b8
1 parent d231ddd
commit 523a1b8
Show file tree

Hide file tree

Showing 9 changed files with 29 additions and 129 deletions.
diff --git a/.bin/monodb-pgsql-health.conf b/.bin/monodb-pgsql-health.conf
@@ -9,4 +9,4 @@ SEND_ALARM=1
 SEND_DM_ALARM=1
 PROCESS_LIMIT=50
 QUERY_LIMIT=25
-CONN_LIMIT=0.7
+CONN_LIMIT_PERCENT=70 #percentage 70%
diff --git a/monocloud/monocloud-health.sh b/monocloud/monocloud-health.sh
@@ -140,9 +140,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[Mono Cloud - $alarm_hostname] [:red_circle:] $2"
-                        if [ $3 == "service" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -176,18 +173,6 @@ function alarm_check_up() {
     fi
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
-
 #~ check partitions
 check_partitions() {
     local partitions="$(df -l -T | awk '{print $1,$2,$7}' | sed '1d' | sort | uniq | grep -E $(echo ${FILESYSTEMS[@]} | sed 's/ /|/g') | awk '$2 != "zfs" {print} $2 == "zfs" && $1 !~ /\//')"

diff --git a/monocloud/monocloud-wppconnect-health.sh b/monocloud/monocloud-wppconnect-health.sh
@@ -124,9 +124,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[Mono Cloud WPPConnect - $IDENTIFIER] [:red_circle:] $2"
-                        if [ $3 == "service" ] || [ $3 == "queue" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -184,17 +181,6 @@ function wpp_check() {
     done
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
 
 function main() {
     printf '\n'

diff --git a/monodb/monodb-mysql-health.sh b/monodb/monodb-mysql-health.sh
@@ -121,9 +121,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[MySQL - $IDENTIFIER] [:red_circle:] $2"
-                        if [ $3 == "service" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -156,18 +153,6 @@ function alarm_check_up() {
     fi
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
-
 function select_now() {
     echo_status "MySQL Access:"
     if mysql -sNe "SELECT NOW();" >/dev/null; then

diff --git a/monodb/monodb-pgsql-health.sh b/monodb/monodb-pgsql-health.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 ###~ description: Checks the status of PostgreSQL and Patroni cluster
-VERSION=v1.0.0
+VERSION=v1.1.0
 
 [[ "$1" == '-v' ]] || [[ "$1" == '--version' ]] && {
     echo "$VERSION"
@@ -147,9 +147,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[PostgreSQL - $IDENTIFIER] [:red_circle:] $2"
-                        if [ $3 == "service" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -183,21 +180,9 @@ function alarm_check_up() {
     fi
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
-
 function postgresql_status() {
     echo_status "PostgreSQL Status"
-    if systemctl status postgresql.service >/dev/null || systemctl status postgresql*.service >/dev/null; then
+    if systemctl status postgresql.service &>/dev/null || systemctl status postgresql*.service >/dev/null; then
         print_colour "PostgreSQL" "Active"
         alarm_check_up "postgresql" "PostgreSQL is active again!"
     else
@@ -244,14 +229,32 @@ function check_active_connections() {
 
     fi
 
+    file="/tmp/monodb-pgsql-health/last-connection-above-limit.txt"
     max_conn="$(echo "$max_and_used" | awk '{print $1}')"
     used_conn="$(echo "$max_and_used" | awk '{print $3}')"
-    if eval "$(echo "$max_conn $used_conn" | awk '{if ($2 >= $1 * 0.9) print "true"; else print "false"}')"; then
-        alarm_check_down "active_conn" "Number of Active Connections is $used_conn and Above %90"
-        print_colour "Number of Active Connections" "$used_conn and Above %90" "error"
+
+    used_percentage=$(echo "$max_conn $used_conn" | awk '{print ($2*100/$1)}')
+    if [ -f "$file" ]; then
+        increase=$(cat $file)
     else
-        alarm_check_up "active_conn" "Number of Active Connections is $used_conn and Below %90"
-        print_colour "Number of Active Connections" "$used_conn and Below %90"
+        increase=1
+    fi
+
+    if eval "$(echo "$used_percentage $CONN_LIMIT_PERCENT" | awk '{if ($1 >= $2) print "true"; else print "false"}')"; then
+        alarm_check_down "active_conn" "Number of Active Connections is $used_conn ($used_percentage)% and Above $CONN_LIMIT_PERCENT%"
+        print_colour "Number of Active Connections" "$used_conn ($used_percentage)% and Above $CONN_LIMIT_PERCENT%" "error"
+        difference=$(((${used_percentage%.*} - ${CONN_LIMIT_PERCENT%.*}) / 10))
+        if [[ $difference -ge $increase ]]; then
+            increase=$((difference + 1))
+            if [ -f "$file" ]; then
+                alarm "[PostgreSQL - $IDENTIFIER] [:red_circle:] Number of Active Connections has passed $((CONN_LIMIT_PERCENT + (increase * 10)))% - It is now $used_conn%"
+            fi
+        fi
+        echo "$increase" >$file
+    else
+        alarm_check_up "active_conn" "Number of Active Connections is $used_conn ($used_percentage)% and Below $CONN_LIMIT_PERCENT%"
+        print_colour "Number of Active Connections" "$used_conn ($used_percentage)% and Below $CONN_LIMIT_PERCENT%"
+        rm -f $file
     fi
 }
 
@@ -263,15 +266,14 @@ function check_running_queries() {
         queries=$(gitlab-psql -c "SELECT COUNT(*) AS active_queries_count FROM pg_stat_activity WHERE state = 'active';" | awk 'NR==3 {print $1}')
     else
         queries=$(su - postgres -c "psql -c \"SELECT COUNT(*) AS active_queries_count FROM pg_stat_activity WHERE state = 'active';\"" | awk 'NR==3 {print $1}')
-
     fi
 
     # SELECT COUNT(*) AS active_queries_count FROM pg_stat_activity WHERE state = 'active';
     if [[ "$queries" -gt "$QUERY_LIMIT" ]]; then
-        alarm_check_down "query_limit" "Number of Active Queries is $queries/$QUERY_LIMIT"
+        alarm_check_down "query_limit" "Number of Active Queries is $queries/$QUERY_LIMIT" "active_queries"
         print_colour "Number of Active Queries" "$queries/$QUERY_LIMIT" "error"
     else
-        alarm_check_up "query_limit" "Number of Active Queries is $queries/$QUERY_LIMIT"
+        alarm_check_up "query_limit" "Number of Active Queries is $queries/$QUERY_LIMIT" "active_queries"
         print_colour "Number of Active Queries" "$queries/$QUERY_LIMIT"
     fi
 }
@@ -320,7 +322,7 @@ function cluster_status() {
                         if [[ "$(curl -s "$PATRONI_API" | jq -r .role)" == "master" ]]; then
                             eval "$LEADER_SWITCH_HOOK"
                             EXIT_CODE=$?
-                            if [ $? -eq 0 ]; then
+                            if [ $EXIT_CODE -eq 0 ]; then
                                 alarm "[Patroni - $IDENTIFIER] [:check:] Leader switch hook executed successfully"
                             else
                                 alarm "[Patroni - $IDENTIFIER] [:red_circle:] Leader switch hook failed with exit code $EXIT_CODE"

diff --git a/monok8s/monok8s-s3-uploader.sh b/monok8s/monok8s-s3-uploader.sh
@@ -187,9 +187,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[monok8s-s3-uploader - $IDENTIFIER] [:red_circle:] $2"
-                        if [ $3 == "service" ] || [ $3 == "queue" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -223,17 +220,6 @@ function alarm_check_up() {
     fi
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
 
 function main() {
     printf '\n'

diff --git a/monomail/monomail-pmg-health.sh b/monomail/monomail-pmg-health.sh
@@ -108,9 +108,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[PMG - $IDENTIFIER] [:red_circle:] $2"
-                        if [ $3 == "service" ] || [ $3 == "queue" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -144,18 +141,6 @@ function alarm_check_up() {
     fi
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
-
 function check_pmg_services() {
     echo_status "PMG Services"
     for i in "${pmg_services[@]}"; do

diff --git a/monomail/monomail-postal-health.sh b/monomail/monomail-postal-health.sh
@@ -130,9 +130,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[Postal - $IDENTIFIER] [:red_circle:] $2"
-                        if [ $3 == "service" ] || [ $3 == "queue" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -167,18 +164,6 @@ function alarm_check_up() {
     fi
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
-
 if [ -z "$(command -v mysql)" ]; then
     echo "Couldn't find mysql on the server - Aborting"
     alarm_check_down "mysql" "Can't find mysql on $IDENTIFIER - Aborting"

diff --git a/monomail/monomail-zimbra-health.sh b/monomail/monomail-zimbra-health.sh
@@ -108,9 +108,6 @@ function alarm_check_down() {
                     if ((time_diff >= ALARM_INTERVAL)); then
                         date "+%Y-%m-%d %H:%M locked" >"${file_path}"
                         alarm "[Zimbra - $IDENTIFIER] [:red_circle:] $2"
-                        if [ $3 == "service" ] || [ $3 == "queue" ]; then
-                            check_active_sessions
-                        fi
                     fi
                 fi
             fi
@@ -144,17 +141,6 @@ function alarm_check_up() {
     fi
 }
 
-function check_active_sessions() {
-    active_sessions=($(ls /var/run/ssh-session))
-    if [[ ${#active_sessions[@]} == 0 ]]; then
-        return
-    else
-        for session in "${active_sessions[@]}"; do
-            user=$(jq -r .username /var/run/ssh-session/"$session")
-            alarm_check_down "session_$session" "User *$user* is connected to host"
-        done
-    fi
-}
 
 ZIMBRA_SERVICES=(
     "amavis:zmamavisdctl"