From a0a6cbad10b1950051065754f423f8a8c6d10256 Mon Sep 17 00:00:00 2001 From: dehort Date: Tue, 16 Apr 2019 12:16:47 -0500 Subject: [PATCH] Add a watchtower logging adapter (#204) * Add a watchtower logging adapter * Read the openshift namespace from a file and use that as the logging stream name * This works, but it needs to be be cleaned up --- Pipfile | 2 + Pipfile.lock | 159 +++++++++++++++++++++++++++++++------------------ app/logging.py | 59 ++++++++++++++++-- 3 files changed, 157 insertions(+), 63 deletions(-) diff --git a/Pipfile b/Pipfile index 4ac2a305f..59743bb80 100644 --- a/Pipfile +++ b/Pipfile @@ -25,6 +25,8 @@ pytest = "*" validators = "*" marshmallow = "*" ujson = "*" +watchtower = "*" +"boto3" = "*" [dev-packages] pytest = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 775c903b5..9dd036860 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8c8d4d19a0179f4656c106e884af3f2fb08509d47b44425a613132bb04b87dc2" + "sha256": "6300e1edbb44a2a766f57aaed547280aa045148f174659410d67e52dd3b42fff" }, "pipfile-spec": 6, "requires": { @@ -18,9 +18,9 @@ "default": { "alembic": { "hashes": [ - "sha256:505d41e01dc0c9e6d85c116d0d35dbb0a833dcb490bf483b75abeb06648864e8" + "sha256:40b9a619aa5f25ea1e1508adcda88b33704ef28e02c9cfa6471e5c772ecf0829" ], - "version": "==1.0.8" + "version": "==1.0.9" }, "aniso8601": { "hashes": [ @@ -43,6 +43,21 @@ ], "version": "==19.1.0" }, + "boto3": { + "hashes": [ + "sha256:bb69628f933a8dba22817c85289b3421b23ac643ff3202b13dd2e933c2717109", + "sha256:c75c45bae9dbdb2ff3fc3482d421a3901e552574a882dba1cffa064715acfbe7" + ], + "index": "pypi", + "version": "==1.9.130" + }, + "botocore": { + "hashes": [ + "sha256:128130b12f8ba4bf07a673b119135264060eb98f6a4a7419cbd1f2c6dc926827", + "sha256:59376112fdee707927b644dd44a1771861f8fe354a48d596131ced83d7a3c05b" + ], + "version": "==1.12.130" + }, "certifi": { "hashes": [ "sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5", @@ -86,6 +101,14 @@ ], "version": "==4.4.0" }, + "docutils": { + "hashes": [ + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + ], + "version": "==0.14" + }, "flask": { "hashes": [ "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", @@ -143,10 +166,9 @@ }, "get": { "hashes": [ - "sha256:ca946cedbb68e368e77f0bddf5488dea5709d488e0278be589e6b212f639c636", - "sha256:fcb91193e7f1d0d441b9419044a36d3e84877b5c4abea131759b12086340b8fd" + "sha256:688268840f923255932154a52bdd40ffac467de4126835c43846e9e6f112844c" ], - "version": "==2019.3.22" + "version": "==2019.4.13" }, "gunicorn": { "hashes": [ @@ -184,6 +206,13 @@ ], "version": "==2.10.1" }, + "jmespath": { + "hashes": [ + "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", + "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" + ], + "version": "==0.9.4" + }, "jsonify": { "hashes": [ "sha256:f340032753577575e9777835809b283fdc9b251867d5d5600389131647f8bfe1" @@ -214,9 +243,9 @@ }, "mako": { "hashes": [ - "sha256:04092940c0df49b01f43daea4f5adcecd0e50ef6a4b222be5ac003d5d84b2843" + "sha256:0728c404877cd4ca72c409c0ea372dc5f3b53fa1ad2bb434e1d216c0444ff1fd" ], - "version": "==1.0.8" + "version": "==1.0.9" }, "markupsafe": { "hashes": [ @@ -290,10 +319,9 @@ }, "post": { "hashes": [ - "sha256:e15dd550bd930f936cbfb77a58445d155740ffe9959dda52242be59b2d69a9c1", - "sha256:f9e9305da35f1f963446d842904c57c6c9aa650330b2f1164aace57894971204" + "sha256:8fd57fb16f2c90ef4a76b04b8701c4db8ded170e53fac9514b5ea0272da5d36f" ], - "version": "==2019.3.22" + "version": "==2019.4.13" }, "prometheus-client": { "hashes": [ @@ -304,44 +332,43 @@ }, "psycopg2-binary": { "hashes": [ - "sha256:163d3ee445a0b4c0109877da9e46271aacf4e5e3d60ae7368669555c30f13e7c", - "sha256:1af0bfe7b0c13a0e613a27311fd4f9c5d024e8fc0f4b3d284e7df02a58a11fc0", - "sha256:2169c3a1bf52d5b30cc98625b5919a964c571a32e8646be20be6c7e3e82079de", - "sha256:218f079fa48e2ef812dc3d3ce6ec2f67ac56427ba4b038d5d6331f2cceb489c2", - "sha256:26a958930687e94c4c6c73c171e4d4783b82ae4e16aa3424e6bcd4529bceedf0", - "sha256:2c7c195aef3acdbc853942bc674844031a732890d2fee88a324298ed376b6c2b", - "sha256:2ecdbfed7004669472bfa27c8d51012c717c241c7154ae17e4c8f93024043525", - "sha256:345fc31b71a90ada1b51826537917b19a1af685a91c0f066787069c184d7d00f", - "sha256:378a06649503f548be5f1e9eec2e94cc1d6138250b82a08dcc6151bca8cec107", - "sha256:3f300bf2930e501dde09605de85cb2b84c2638e2c954be02a3c86f28176d3525", - "sha256:6c2f66c653ce8bbd7e789d0f7f92c3f9fea881b55226f0ae5ee550cce9e3cf0e", - "sha256:6fccbac2633831b877a8fbf865f7082d34895e82a015795a9f80f99a2efe2576", - "sha256:7a166f8ccb6888358d3e67795b057540ea7caa71ab9e089b0cb0097f01088965", - "sha256:8f6b84f887ec6fef6c1796779f8ec2603dc7e9ef52bc9269de719d4bcbdaebbb", - "sha256:92cf3ceb7bb90cf35b8bd993c640b15d4832ba0e142a3b9da5006ef217da595d", - "sha256:a20dfdf73f56da674926a3811929cff9fd23b9af90be9a6c36ac246a3486eef3", - "sha256:a84415df4689251556c961e4fe3b25d30e32f00faa8064ce0909458dbe0d67b2", - "sha256:ab1aa1cd50df3860f624c9713ee9e690eefd4e049d3a4d86577bab6e741e9616", - "sha256:abc9dcf85e75a8687f2a6d560c0c1a2593e8e34ba6f9ad6721f8212c5de179a2", - "sha256:c10454710a81a2f4b1ff4d1c83ac2cec63e0e55845a56324991514af5b1299d0", - "sha256:c38f80719e4dfae7a6311a4f091f07f4fb2fb5d602352015d5639f63f8fabb68", - "sha256:d75cf00605630b2cfefa5c62373c605dcda1cc0d607902847dbb8e8e9b67c1ce", - "sha256:dce15cb6ef604c9e38fdaa848f58f83153ade9f4aa5e4cf5812aa27163561750", - "sha256:e7e0db4311bb76bf3f6e0380f71912cfa6d0be7cc635e3772476050b0dabdabd", - "sha256:eac59cae78dfe3fbf7ece25c170d7a152f88df7643381aa5e7344c2028a8d8d4", - "sha256:ead7b3e1567bd14cacd44279c5e42cd19f54b9feed39180220253f4fbe3abd56", - "sha256:ed772a5e8e7e5dd6bede960a86940c17cf653c7f158dafa5d52e919b676f10ba", - "sha256:f2d73131acb94afa45de8b6b8a4bfb21bbe3736633d6478e53247f19dd8c299c" + "sha256:007ca0df127b1862fc010125bc4100b7a630efc6841047bd11afceadb4754611", + "sha256:03c49e02adf0b4d68f422fdbd98f7a7c547beb27e99a75ed02298f85cb48406a", + "sha256:0a1232cdd314e08848825edda06600455ad2a7adaa463ebfb12ece2d09f3370e", + "sha256:131c80d0958c89273d9720b9adf9df1d7600bb3120e16019a7389ab15b079af5", + "sha256:2de34cc3b775724623f86617d2601308083176a495f5b2efc2bbb0da154f483a", + "sha256:2eddc31500f73544a2a54123d4c4b249c3c711d31e64deddb0890982ea37397a", + "sha256:484f6c62bdc166ee0e5be3aa831120423bf399786d1f3b0304526c86180fbc0b", + "sha256:4c2d9369ed40b4a44a8ccd6bc3a7db6272b8314812d2d1091f95c4c836d92e06", + "sha256:70f570b5fa44413b9f30dbc053d17ef3ce6a4100147a10822f8662e58d473656", + "sha256:7a2b5b095f3bd733aab101c89c0e1a3f0dfb4ebdc26f6374805c086ffe29d5b2", + "sha256:804914a669186e2843c1f7fbe12b55aad1b36d40a28274abe6027deffad9433d", + "sha256:8520c03172da18345d012949a53617a963e0191ccb3c666f23276d5326af27b5", + "sha256:90da901fc33ea393fc644607e4a3916b509387e9339ec6ebc7bfded45b7a0ae9", + "sha256:a582416ad123291a82c300d1d872bdc4136d69ad0b41d57dc5ca3df7ef8e3088", + "sha256:ac8c5e20309f4989c296d62cac20ee456b69c41fd1bc03829e27de23b6fa9dd0", + "sha256:b2cf82f55a619879f8557fdaae5cec7a294fac815e0087c4f67026fdf5259844", + "sha256:b59d6f8cfca2983d8fdbe457bf95d2192f7b7efdb2b483bf5fa4e8981b04e8b2", + "sha256:be08168197021d669b9964bd87628fa88f910b1be31e7010901070f2540c05fd", + "sha256:be0f952f1c365061041bad16e27e224e29615d4eb1fb5b7e7760a1d3d12b90b6", + "sha256:c1c9a33e46d7c12b9c96cf2d4349d783e3127163fd96254dcd44663cf0a1d438", + "sha256:d18c89957ac57dd2a2724ecfe9a759912d776f96ecabba23acb9ecbf5c731035", + "sha256:d7e7b0ff21f39433c50397e60bf0995d078802c591ca3b8d99857ea18a7496ee", + "sha256:da0929b2bf0d1f365345e5eb940d8713c1d516312e010135b14402e2a3d2404d", + "sha256:de24a4962e361c512d3e528ded6c7480eab24c655b8ca1f0b761d3b3650d2f07", + "sha256:e45f93ff3f7dae2202248cf413a87aeb330821bf76998b3cf374eda2fc893dd7", + "sha256:f046aeae1f7a845041b8661bb7a52449202b6c5d3fb59eb4724e7ca088811904", + "sha256:f1dc2b7b2748084b890f5d05b65a47cd03188824890e9a60818721fd492249fb", + "sha256:fcbe7cf3a786572b73d2cd5f34ed452a5f5fac47c9c9d1e0642c457a148f9f88" ], "index": "pypi", - "version": "==2.8.1" + "version": "==2.8.2" }, "public": { "hashes": [ - "sha256:5e7612316a29c1d6ff4445f120278184389e307d8fa8d98715d47ce71db1e39c", - "sha256:5f62a1f5a4511b96413c01d473f0663282ab17041da4d0815b4937ad9f24e8ec" + "sha256:e1436a8a99693a9849dfe40b9158f3837b7c309c163b2d3f5b8e9fce23876db1" ], - "version": "==2019.3.22" + "version": "==2019.4.13" }, "py": { "hashes": [ @@ -352,11 +379,11 @@ }, "pytest": { "hashes": [ - "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", - "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], "index": "pypi", - "version": "==4.4.0" + "version": "==4.4.1" }, "python-dateutil": { "hashes": [ @@ -401,18 +428,16 @@ }, "query-string": { "hashes": [ - "sha256:9084be7d299ca96a8f0b17146a8d4ae394c967d637e1dc2fccab236f066d6906", - "sha256:c55143803f12e0415287b89637afcba731e848b82d0d2383513d71dc7e0d9ad4" + "sha256:bb24e4f58849ef6f8219b2446c2bed076d86c97720ae9c3ae918625807394ca8" ], - "version": "==2019.3.22" + "version": "==2019.4.13" }, "request": { "hashes": [ - "sha256:0ec9f1df66c5c44e9e0221f02bc7405e9852bd42e66fb6c4d68399aeb0b69d36", - "sha256:1c8f3f991e248dd2d8690a635a2ac4a1f95a4b7a3745831c26336d03646bc320" + "sha256:6297b53c29a4928f7735034df2ab60aa1c7c044df989f605a81b7dc2d3713732" ], "index": "pypi", - "version": "==2019.3.22" + "version": "==2019.4.13" }, "requests": { "hashes": [ @@ -422,6 +447,13 @@ "index": "pypi", "version": "==2.21.0" }, + "s3transfer": { + "hashes": [ + "sha256:7b9ad3213bff7d357f888e0fab5101b56fa1a0548ee77d121c3a3dbfbef4cb2e", + "sha256:f23d5cb7d862b104401d9021fc82e5fa0e0cf57b7660a1331425aab0c691d021" + ], + "version": "==0.2.0" + }, "six": { "hashes": [ "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", @@ -431,9 +463,9 @@ }, "sqlalchemy": { "hashes": [ - "sha256:d5432832f91d200c3d8b473a266d59442d825f9ea744c467e68c5d9a9479fbce" + "sha256:91c54ca8345008fceaec987e10924bf07dcab36c442925357e5a467b36a38319" ], - "version": "==1.3.2" + "version": "==1.3.3" }, "swagger-ui-bundle": { "hashes": [ @@ -455,14 +487,23 @@ "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" ], + "markers": "python_version >= '3.4'", "version": "==1.24.1" }, "validators": { "hashes": [ - "sha256:68e4b74889aac1270d83636cb1dbcce3d2271e291ab14023cf95e7dbfbbce09d" + "sha256:df3dda070965519283bae72249a36927ee3ea9c206f9ee6f234a71cf19b36136" ], "index": "pypi", - "version": "==0.12.4" + "version": "==0.12.5" + }, + "watchtower": { + "hashes": [ + "sha256:d7cc8e74f4451ae5b16525851a74a5aa8ab7fb2c65a8e3be63bd475e6e49c71f", + "sha256:f53a1a386934638a2a1f4f57d72e3cb17e255906d86d98684dc539ee252eb9f0" + ], + "index": "pypi", + "version": "==0.5.5" }, "werkzeug": { "hashes": [ @@ -558,11 +599,11 @@ }, "pytest": { "hashes": [ - "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", - "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" + "sha256:3773f4c235918987d51daf1db66d51c99fac654c81d6f2f709a046ab446d5e5d", + "sha256:b7802283b70ca24d7119b32915efa7c409982f59913c1a6c0640aacf118b95f5" ], "index": "pypi", - "version": "==4.4.0" + "version": "==4.4.1" }, "pytest-cov": { "hashes": [ diff --git a/app/logging.py b/app/logging.py index 2f02cf294..4e7211272 100644 --- a/app/logging.py +++ b/app/logging.py @@ -2,12 +2,16 @@ import logging.config import logstash_formatter import os +import watchtower +from boto3.session import Session from gunicorn import glogging from flask import request REQUEST_ID_HEADER = "x-rh-insights-request-id" UNKNOWN_REQUEST_ID_VALUE = "-1" +OPENSHIFT_ENVIRONMENT_NAME_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" +DEFAULT_AWS_LOGGING_NAMESPACE = "inventory-dev" def configure_logging(config_name): @@ -28,14 +32,61 @@ def configure_logging(config_name): logging.config.fileConfig(fname=log_config_file) if config_name != "testing": - # Only enable the contextual filter if not in "testing" mode + _configure_watchtower_logging_handler() + _configure_contextual_logging_filter() + + +def _configure_watchtower_logging_handler(): + aws_access_key_id = os.getenv("AWS_ACCESS_KEY_ID", None) + aws_secret_access_key = os.getenv("AWS_SECRET_ACCESS_KEY", None) + aws_region_name = os.getenv("AWS_REGION_NAME", None) + log_group = "inventory" + stream_name = _get_aws_logging_stream_name(OPENSHIFT_ENVIRONMENT_NAME_FILE) + log_level = os.getenv("INVENTORY_LOG_LEVEL", "WARNING").upper() + + if all([aws_access_key_id, aws_secret_access_key, + aws_region_name, stream_name]): + print(f"Configuring watchtower logging (log_group={log_group}, stream_name={stream_name})") + boto3_session = Session(aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + region_name=aws_region_name) + root = logging.getLogger() - root.addFilter(ContextualFilter()) + handler = watchtower.CloudWatchLogHandler(boto3_session=boto3_session, + log_group=log_group, + stream_name=stream_name) + handler.setFormatter(logstash_formatter.LogstashFormatterV1()) + root.addHandler(handler) - # FIXME: Figure out a better way to load the list of modules/submodules for logger_name in ("app", "app.models", "api", "api.host"): app_logger = logging.getLogger(logger_name) - app_logger.addFilter(ContextualFilter()) + app_logger.setLevel(log_level) + + else: + print("Unable to configure watchtower logging. Please " + "verify watchtower logging configuration!") + + +def _get_aws_logging_stream_name(namespace_filename): + try: + with open(namespace_filename) as namespace_fh: + return namespace_fh.read() + except FileNotFoundError: + namespace = DEFAULT_AWS_LOGGING_NAMESPACE + print(f"Error reading the OpenShift namepsace file. " + f"Using {namespace} as aws logging stream name") + return namespace + + +def _configure_contextual_logging_filter(): + # Only enable the contextual filter if not in "testing" mode + root = logging.getLogger() + root.addFilter(ContextualFilter()) + + # FIXME: Figure out a better way to load the list of modules/submodules + for logger_name in ("app", "app.models", "api", "api.host"): + app_logger = logging.getLogger(logger_name) + app_logger.addFilter(ContextualFilter()) class ContextualFilter(logging.Filter):