diff --git a/requirements/jenkins.in b/requirements/jenkins.in index 0f5728b084b..1540d96d935 100644 --- a/requirements/jenkins.in +++ b/requirements/jenkins.in @@ -3,6 +3,7 @@ awscli boto boto3 +datadog futures ; python_version == "2.7" # via s3transfer s3cmd pyyaml diff --git a/util/jenkins/export_dead_locks/export_dead_locks_dd.py b/util/jenkins/export_dead_locks/export_dead_locks_dd.py new file mode 100644 index 00000000000..13ad2046c04 --- /dev/null +++ b/util/jenkins/export_dead_locks/export_dead_locks_dd.py @@ -0,0 +1,124 @@ +import boto3 +from botocore.exceptions import ClientError +import sys +import backoff +import pymysql +import time +import uuid +import click +import re +from datadog import initialize, api + + +MAX_TRIES = 5 + + +class EC2BotoWrapper: + def __init__(self): + self.client = boto3.client("ec2") + + @backoff.on_exception(backoff.expo, ClientError, max_tries=MAX_TRIES) + def describe_regions(self): + return self.client.describe_regions() + + +class RDSBotoWrapper: + def __init__(self, **kwargs): + self.client = boto3.client("rds", **kwargs) + + @backoff.on_exception(backoff.expo, ClientError, max_tries=MAX_TRIES) + def describe_db_instances(self): + return self.client.describe_db_instances() + + +def rds_extractor(environment, whitelistregions): + """ + Return list of all RDS instances across all the regions + Returns: + [ + { + 'name': name, + 'ARN': RDS ARN, + 'Region': Region of RDS + } + ] + """ + client_region = EC2BotoWrapper() + rds_list = [] + try: + regions_list = client_region.describe_regions() + except ClientError as e: + print(f"Unable to connect to AWS with error :{e}") + sys.exit(1) + if whitelistregions: + regions_list = {'Regions': [region for region in regions_list['Regions'] if region['RegionName'] in whitelistregions]} + for region in regions_list["Regions"]: + try: + rds_client = RDSBotoWrapper(region_name=region["RegionName"]) + response = rds_client.describe_db_instances() + for instance in response.get('DBInstances'): + if environment in instance.get("Endpoint").get("Address") and "test" not in instance["DBInstanceIdentifier"]: + temp_dict = {} + temp_dict["name"] = instance["DBInstanceIdentifier"] + temp_dict["ARN"] = instance["DBInstanceArn"] + temp_dict["Region"] = region["RegionName"] + temp_dict["Endpoint"] = instance.get("Endpoint").get("Address") + temp_dict["Username"] = instance.get("MasterUsername") + temp_dict["Port"] = instance.get("Port") + rds_list.append(temp_dict) + except ClientError as e: + print(f"Unable to get RDS from this region error :{e}") + sys.exit(1) + return rds_list + + +def rds_controller(rds_list, username, password, hostname, dd_apikey, port, indexname, environment): + for item in rds_list: + rds_host_endpoint = item["Endpoint"] + rds_port = item["Port"] + connection = pymysql.connect(host=rds_host_endpoint, port=rds_port, + user=username, password=password) + cursor = connection.cursor() + cursor.execute(""" + SHOW ENGINE INNODB STATUS; + """) + rds_result = cursor.fetchall() + cursor.close() + connection.close() + regex = r"-{4,}\sLATEST DETECTED DEADLOCK\s-{4,}\s((.*)\s)*?-{4,}" + global_str = "" + for row in rds_result: + matches = re.finditer(regex, row[2]) + for matchNum, match in enumerate(matches, start=1): + global_str = match.group() + expr = re.compile(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}") + global_str = re.sub(expr, '', global_str) + #to avoid empty dead locks + if len(global_str) > 0: + options = { + 'api_key': dd_apikey + } + initialize(**options) + + api.Event.create( + title="RDS INNODB STATUS", + text=global_str, + tags=[f"env:{environment}", f"index:{index}", "source:INNODB-STATUS", "service:rds"] + ) + +@click.command() +@click.option('--username', envvar='USERNAME', required=True) +@click.option('--password', envvar='PASSWORD', required=True) +@click.option('--environment', required=True, help='Use to identify the environment') +@click.option('--dd_api_key', envvar='DDAPIKEY', required=True) +@click.option('--indexname', required=True, help='Use to identify the DD index name') +@click.option('--rdsignore', '-i', multiple=True, help='RDS name tags to not check, can be specified multiple times') +@click.option('--whitelistregions', '-r', multiple=True, help='Regions to check, can be specified multiple times') +def main(username, password, environment, hostname, dd_apikey, indexname, rdsignore, whitelistregions): + rds_list = rds_extractor(environment, whitelistregions) + filtered_rds_list = list([x for x in rds_list if x['name'] not in rdsignore]) + rds_controller(filtered_rds_list, username, password, hostname, dd_apikey, indexname, environment) + + +if __name__ == '__main__': + main() diff --git a/util/jenkins/requirements.txt b/util/jenkins/requirements.txt index 4ddf612445b..2373fbfbccd 100644 --- a/util/jenkins/requirements.txt +++ b/util/jenkins/requirements.txt @@ -6,9 +6,9 @@ # amqp==5.2.0 # via kombu -argcomplete==3.2.1 +argcomplete==3.5.1 # via yq -awscli==1.32.2 +awscli==1.35.10 # via -r requirements/jenkins.in backoff==1.4.3 # via -r requirements/jenkins.in @@ -16,24 +16,24 @@ backports-zoneinfo[tzdata]==0.2.1 # via # celery # kombu -billiard==4.2.0 +billiard==4.2.1 # via celery boto==2.49.0 # via -r requirements/jenkins.in -boto3==1.34.2 +boto3==1.35.44 # via -r requirements/jenkins.in -botocore==1.34.2 +botocore==1.35.44 # via # awscli # boto3 # s3transfer -celery==5.3.6 +celery==5.4.0 # via -r requirements/jenkins.in -certifi==2023.11.17 +certifi==2024.8.30 # via # opsgenie-sdk # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests click==8.1.7 # via @@ -42,60 +42,64 @@ click==8.1.7 # click-didyoumean # click-plugins # click-repl -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -colorama==0.4.4 +colorama==0.4.6 # via awscli +datadog==0.50.1 + # via -r requirements/jenkins.in docutils==0.16 # via awscli -idna==3.6 +idna==3.10 # via requests jmespath==1.0.1 # via # boto3 # botocore -jq==1.6.0 +jq==1.8.0 # via -r requirements/jenkins.in -kombu==5.3.4 +kombu==5.4.2 # via celery opsgenie-sdk==0.3.1 # via -r requirements/jenkins.in -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.48 # via click-repl -pyasn1==0.5.1 +pyasn1==0.6.1 # via rsa pymysql==0.9.3 # via -r requirements/jenkins.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # botocore # celery # opsgenie-sdk # s3cmd -python-gnupg==0.5.2 +python-gnupg==0.5.3 # via -r requirements/jenkins.in python-magic==0.4.27 # via s3cmd -pytz==2023.3.post1 +pytz==2024.2 # via opsgenie-sdk -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/jenkins.in # awscli # yq redis==2.10.6 # via -r requirements/jenkins.in -requests==2.31.0 - # via opsgenie-sdk +requests==2.32.3 + # via + # datadog + # opsgenie-sdk rsa==4.7.2 # via awscli s3cmd==2.4.0 # via -r requirements/jenkins.in -s3transfer==0.9.0 +s3transfer==0.10.3 # via # awscli # boto3 @@ -105,15 +109,15 @@ six==1.16.0 # python-dateutil splunk-sdk==1.6.16 # via -r requirements/jenkins.in -tomlkit==0.12.3 +tomlkit==0.13.2 # via yq -typing-extensions==4.9.0 +typing-extensions==4.12.2 # via kombu -tzdata==2023.3 +tzdata==2024.2 # via # backports-zoneinfo # celery -urllib3==1.26.18 +urllib3==1.26.20 # via # botocore # opsgenie-sdk @@ -123,11 +127,11 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.12 +wcwidth==0.2.13 # via prompt-toolkit -xmltodict==0.13.0 +xmltodict==0.14.2 # via yq -yq==3.2.3 +yq==3.4.3 # via -r requirements/jenkins.in # The following packages are considered to be unsafe in a requirements file: