chore: Added script to export RDS deadlocks to datadog

requirements/jenkins.in

@@ -3,6 +3,7 @@
 awscli
 boto
 boto3
+datadog
 futures ; python_version == "2.7"   # via s3transfer
 s3cmd
 pyyaml

util/jenkins/export_dead_locks/export_dead_locks_dd.py

@@ -0,0 +1,124 @@
+import boto3
+from botocore.exceptions import ClientError
+import sys
+import backoff
+import pymysql
+import time
+import uuid
+import click
+import re
+from datadog import initialize, api
+
+
+MAX_TRIES = 5
+
+
+class EC2BotoWrapper:
+    def __init__(self):
+        self.client = boto3.client("ec2")
+
+    @backoff.on_exception(backoff.expo, ClientError, max_tries=MAX_TRIES)
+    def describe_regions(self):
+        return self.client.describe_regions()
+
+
+class RDSBotoWrapper:
+    def __init__(self, **kwargs):
+        self.client = boto3.client("rds", **kwargs)
+
+    @backoff.on_exception(backoff.expo, ClientError, max_tries=MAX_TRIES)
+    def describe_db_instances(self):
+        return self.client.describe_db_instances()
+
+
+def rds_extractor(environment, whitelistregions):
+    """
+    Return list of all RDS instances across all the regions
+    Returns:
+        [
+            {
+                'name': name,
+                'ARN': RDS ARN,
+                'Region': Region of RDS
+            }
+        ]
+    """
+    client_region = EC2BotoWrapper()
+    rds_list = []
+    try:
+        regions_list = client_region.describe_regions()
+    except ClientError as e:
+        print(f"Unable to connect to AWS with error :{e}")
+        sys.exit(1)
+    if whitelistregions:
+        regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in whitelistregions]}
+    for region in regions_list["Regions"]:
+        try:
+            rds_client = RDSBotoWrapper(region_name=region["RegionName"])
+            response = rds_client.describe_db_instances()
+            for instance in response.get('DBInstances'):
+                if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]:
+                    temp_dict = {}
+                    temp_dict["name"] = instance["DBInstanceIdentifier"]
+                    temp_dict["ARN"] = instance["DBInstanceArn"]
+                    temp_dict["Region"] = region["RegionName"]
+                    temp_dict["Endpoint"] = instance.get("Endpoint").get("Address")
+                    temp_dict["Username"] = instance.get("MasterUsername")
+                    temp_dict["Port"] = instance.get("Port")
+                    rds_list.append(temp_dict)
+        except ClientError as e:
+            print(f"Unable to get RDS from this region error :{e}")
+            sys.exit(1)
+    return rds_list
+
+
+def rds_controller(rds_list, username, password, hostname, dd_apikey, port, indexname, environment):
+    for item in rds_list:
+        rds_host_endpoint = item["Endpoint"]
+        rds_port = item["Port"]
+        connection = pymysql.connect(host=rds_host_endpoint, port=rds_port,
+                                     user=username, password=password)
+        cursor = connection.cursor()
+        cursor.execute("""
+                      SHOW ENGINE INNODB STATUS;
+                    """)
+        rds_result = cursor.fetchall()
+        cursor.close()
+        connection.close()
+        regex = r"-{4,}\sLATEST DETECTED DEADLOCK\s-{4,}\s((.*)\s)*?-{4,}"
+        global_str = ""
+        for row in rds_result:
+            matches = re.finditer(regex, row[2])
+            for matchNum, match in enumerate(matches, start=1):
+                global_str = match.group()
+        expr = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}")
+        global_str = re.sub(expr, '', global_str)
+        #to avoid empty dead locks
+        if len(global_str) > 0:
+            options = {
+                'api_key': dd_apikey
+            }
+            initialize(**options)
+
+            api.Event.create(
+                    title="RDS INNODB STATUS", 
+                    text=global_str,
+                    tags=[f"env:{environment}", f"index:{index}", "source:INNODB-STATUS", "service:rds"]
+                )
+
+@click.command()
+@click.option('--username', envvar='USERNAME', required=True)
+@click.option('--password', envvar='PASSWORD', required=True)
+@click.option('--environment', required=True, help='Use to identify the environment')
+@click.option('--dd_api_key', envvar='DDAPIKEY', required=True)
+@click.option('--indexname', required=True, help='Use to identify the DD index name')
+@click.option('--rdsignore', '-i', multiple=True, help='RDS name tags to not check, can be specified multiple times')
+@click.option('--whitelistregions', '-r', multiple=True, help='Regions to check, can be specified multiple times')
+def main(username, password, environment, hostname, dd_apikey, indexname, rdsignore, whitelistregions):
+    rds_list = rds_extractor(environment, whitelistregions)
+    filtered_rds_list = list([x for x in rds_list if x['name'] not in rdsignore])
+    rds_controller(filtered_rds_list, username, password, hostname, dd_apikey, indexname, environment)
+
+
+if __name__ == '__main__':
+    main()

util/jenkins/requirements.txt

@@ -6,34 +6,34 @@
 #
 amqp==5.2.0
     # via kombu
-argcomplete==3.2.1
+argcomplete==3.5.1
     # via yq
-awscli==1.32.2
+awscli==1.35.10
     # via -r requirements/jenkins.in
 backoff==1.4.3
     # via -r requirements/jenkins.in
 backports-zoneinfo[tzdata]==0.2.1
     # via
     #   celery
     #   kombu
-billiard==4.2.0
+billiard==4.2.1
     # via celery
 boto==2.49.0
     # via -r requirements/jenkins.in
-boto3==1.34.2
+boto3==1.35.44
     # via -r requirements/jenkins.in
-botocore==1.34.2
+botocore==1.35.44
     # via
     #   awscli
     #   boto3
     #   s3transfer
-celery==5.3.6
+celery==5.4.0
     # via -r requirements/jenkins.in
-certifi==2023.11.17
+certifi==2024.8.30
     # via
     #   opsgenie-sdk
     #   requests
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
 click==8.1.7
     # via
@@ -42,60 +42,64 @@ click==8.1.7
     #   click-didyoumean
     #   click-plugins
     #   click-repl
-click-didyoumean==0.3.0
+click-didyoumean==0.3.1
     # via celery
 click-plugins==1.1.1
     # via celery
 click-repl==0.3.0
     # via celery
-colorama==0.4.4
+colorama==0.4.6
     # via awscli
+datadog==0.50.1
+    # via -r requirements/jenkins.in
 docutils==0.16
     # via awscli
-idna==3.6
+idna==3.10
     # via requests
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jq==1.6.0
+jq==1.8.0
     # via -r requirements/jenkins.in
-kombu==5.3.4
+kombu==5.4.2
     # via celery
 opsgenie-sdk==0.3.1
     # via -r requirements/jenkins.in
-prompt-toolkit==3.0.43
+prompt-toolkit==3.0.48
     # via click-repl
-pyasn1==0.5.1
+pyasn1==0.6.1
     # via rsa
 pymysql==0.9.3
     # via -r requirements/jenkins.in
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via
     #   botocore
     #   celery
     #   opsgenie-sdk
     #   s3cmd
-python-gnupg==0.5.2
+python-gnupg==0.5.3
     # via -r requirements/jenkins.in
 python-magic==0.4.27
     # via s3cmd
-pytz==2023.3.post1
+pytz==2024.2
     # via opsgenie-sdk
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via
     #   -r requirements/jenkins.in
     #   awscli
     #   yq
 redis==2.10.6
     # via -r requirements/jenkins.in
-requests==2.31.0
-    # via opsgenie-sdk
+requests==2.32.3
+    # via
+    #   datadog
+    #   opsgenie-sdk
 rsa==4.7.2
     # via awscli
 s3cmd==2.4.0
     # via -r requirements/jenkins.in
-s3transfer==0.9.0
+s3transfer==0.10.3
     # via
     #   awscli
     #   boto3
@@ -105,15 +109,15 @@ six==1.16.0
     #   python-dateutil
 splunk-sdk==1.6.16
     # via -r requirements/jenkins.in
-tomlkit==0.12.3
+tomlkit==0.13.2
     # via yq
-typing-extensions==4.9.0
+typing-extensions==4.12.2
     # via kombu
-tzdata==2023.3
+tzdata==2024.2
     # via
     #   backports-zoneinfo
     #   celery
-urllib3==1.26.18
+urllib3==1.26.20
     # via
     #   botocore
     #   opsgenie-sdk
@@ -123,11 +127,11 @@ vine==5.1.0
     #   amqp
     #   celery
     #   kombu
-wcwidth==0.2.12
+wcwidth==0.2.13
     # via prompt-toolkit
-xmltodict==0.13.0
+xmltodict==0.14.2
     # via yq
-yq==3.2.3
+yq==3.4.3
     # via -r requirements/jenkins.in
 
 # The following packages are considered to be unsafe in a requirements file: