From 2e23d19a0f10a0ad40afe6b7a452d782e605d029 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Thu, 1 Jun 2023 15:35:50 +0200 Subject: [PATCH 1/5] refactored Logger and config.py and metricprofile-webapi, topology-gocdb, weight-vapor --- exec/metricprofile-webapi-connector.py | 7 +++++-- exec/topology-gocdb-connector.py | 7 +++++-- exec/weights-vapor-connector.py | 8 ++++++-- modules/config.py | 4 ++-- modules/log.py | 11 ++++++++--- modules/utils.py | 8 +++++++- 6 files changed, 33 insertions(+), 12 deletions(-) diff --git a/exec/metricprofile-webapi-connector.py b/exec/metricprofile-webapi-connector.py index 33f43542..387d8b20 100755 --- a/exec/metricprofile-webapi-connector.py +++ b/exec/metricprofile-webapi-connector.py @@ -10,7 +10,7 @@ from argo_connectors.config import CustomerConf, Global from argo_connectors.log import Logger from argo_connectors.tasks.webapi_metricprofile import TaskWebApiMetricProfile -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None @@ -26,7 +26,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -38,6 +37,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 19409265..8d30ca99 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state from argo_connectors.tasks.gocdb_topology import TaskGocdbTopology -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None globopts = {} @@ -58,7 +58,6 @@ def main(): help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = [], [] - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -71,6 +70,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/weights-vapor-connector.py b/exec/weights-vapor-connector.py index b5aa728c..e7096454 100755 --- a/exec/weights-vapor-connector.py +++ b/exec/weights-vapor-connector.py @@ -13,7 +13,7 @@ from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path globopts = {} logger = None @@ -31,7 +31,6 @@ def main(): help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -43,6 +42,11 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/modules/config.py b/modules/config.py index 0ca4afb6..a01c3bf5 100644 --- a/modules/config.py +++ b/modules/config.py @@ -30,7 +30,7 @@ class Global(object): def __init__(self, caller, confpath=None, **kwargs): self.optional = dict() - self.logger = Logger(str(self.__class__)) + self.logger = Logger(str(self.__class__), tenant_name=None) self._filename = '/etc/argo-connectors/global.conf' if not confpath else confpath self._checkpath = kwargs['checkpath'] if 'checkpath' in kwargs.keys( ) else False @@ -228,7 +228,7 @@ class CustomerConf(object): deftopofeed = 'https://goc.egi.eu/gocdbpi/' def __init__(self, caller, confpath, **kwargs): - self.logger = Logger(str(self.__class__)) + self.logger = Logger(str(self.__class__), tenant_name=None) self._filename = '/etc/argo-connectors/customer.conf' if not confpath else confpath if not kwargs: self._jobattrs = self._defjobattrs[os.path.basename(caller)] diff --git a/modules/log.py b/modules/log.py index 119ef899..5bb0d6bf 100644 --- a/modules/log.py +++ b/modules/log.py @@ -2,16 +2,21 @@ import logging.handlers import sys import socket - -LOGFILE = "/var/log/argo-connectors/connectors.log" +import os class Logger: - def __init__(self, connector): + def __init__(self, connector, tenant_name): lfs = '%(name)s[%(process)s]: %(levelname)s %(message)s' logformat = logging.Formatter(lfs) logverbose = logging.INFO self.connector = connector + self.tenant_name = tenant_name + + LOGFILE = f"/var/log/argo-connectors/{self.tenant_name}/connectors.log" + + if self.tenant_name is not None and not os.path.exists(LOGFILE): + os.makedirs(f'/var/log/argo-connectors/{self.tenant_name}/') logging.basicConfig(format=lfs, level=logging.INFO, stream=sys.stdout) self.logger = logging.getLogger(connector) diff --git a/modules/utils.py b/modules/utils.py index 963c18e1..a4b524d0 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -59,4 +59,10 @@ def remove_non_utf(string): if '"' in string: string = string.replace('"', '') - return unidecode(string) \ No newline at end of file + return unidecode(string) + + +def parse_conf_path(path): + tenant = path.split('/')[3].split('-')[0] + + return tenant.upper() From ed641b7988754e91f2dc38704c75a2154106c89a Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Thu, 1 Jun 2023 15:58:57 +0200 Subject: [PATCH 2/5] changes for downtimes-csv, downtimes-gocdb, service-types-csv, service-types-gocdb, topology-csv connectors --- exec/downtimes-csv-connector.py | 6 +++++- exec/downtimes-gocdb-connector.py | 6 +++++- exec/service-types-csv-connector.py | 7 +++++-- exec/service-types-gocdb-connector.py | 7 +++++-- exec/topology-csv-connector.py | 7 +++++-- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py index 154e9480..ddbc3995 100755 --- a/exec/downtimes-csv-connector.py +++ b/exec/downtimes-csv-connector.py @@ -12,6 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes from argo_connectors.tasks.common import write_state +from argo_connectors.utils import parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -37,13 +38,16 @@ def main(): parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/downtimes-gocdb-connector.py b/exec/downtimes-gocdb-connector.py index f6f3dc41..13ffba1a 100755 --- a/exec/downtimes-gocdb-connector.py +++ b/exec/downtimes-gocdb-connector.py @@ -12,6 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.gocdb_downtimes import TaskGocdbDowntimes from argo_connectors.tasks.common import write_state +from argo_connectors.utils import parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -42,13 +43,16 @@ def main(): help='path to global configuration file', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/service-types-csv-connector.py b/exec/service-types-csv-connector.py index 078045da..77996d50 100755 --- a/exec/service-types-csv-connector.py +++ b/exec/service-types-csv-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes from argo_connectors.tasks.common import write_state -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -43,13 +43,16 @@ def main(): if args.date and date_check(args.date): fixed_date = args.date - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/service-types-gocdb-connector.py b/exec/service-types-gocdb-connector.py index 2977afae..17271bf6 100755 --- a/exec/service-types-gocdb-connector.py +++ b/exec/service-types-gocdb-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.gocdb_servicetypes import TaskGocdbServiceTypes from argo_connectors.tasks.common import write_state -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -43,13 +43,16 @@ def main(): if args.date and date_check(args.date): fixed_date = args.date - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-csv-connector.py b/exec/topology-csv-connector.py index 082b528d..62887436 100755 --- a/exec/topology-csv-connector.py +++ b/exec/topology-csv-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state from argo_connectors.tasks.flat_topology import TaskFlatTopology -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None @@ -41,7 +41,6 @@ def main(): parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -53,6 +52,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) From df37670af7725f8c38266b0a300bc6b9197444e3 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Thu, 1 Jun 2023 16:10:40 +0200 Subject: [PATCH 3/5] changes to service-types-json, topology-agora, topology-json, topology-provider --- exec/service-types-json-connector.py | 7 +++++-- exec/topology-agora-connector.py | 7 +++++-- exec/topology-json-connector.py | 7 +++++-- exec/topology-provider-connector.py | 7 +++++-- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/exec/service-types-json-connector.py b/exec/service-types-json-connector.py index 1e30063f..03134eb5 100755 --- a/exec/service-types-json-connector.py +++ b/exec/service-types-json-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes from argo_connectors.tasks.common import write_state -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -43,13 +43,16 @@ def main(): if args.date and date_check(args.date): fixed_date = args.date - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-agora-connector.py b/exec/topology-agora-connector.py index 53f04ac7..7dacb7b6 100755 --- a/exec/topology-agora-connector.py +++ b/exec/topology-agora-connector.py @@ -10,7 +10,7 @@ from argo_connectors.exceptions import ConnectorError, ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.tasks.agora_topology import TaskProviderTopology from argo_connectors.tasks.common import write_state @@ -39,7 +39,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date @@ -50,6 +49,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-json-connector.py b/exec/topology-json-connector.py index e95f5bd9..b86dfac1 100755 --- a/exec/topology-json-connector.py +++ b/exec/topology-json-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state from argo_connectors.tasks.flat_topology import TaskFlatTopology -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None globopts = {} @@ -40,7 +40,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = list(), list() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -52,6 +51,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-provider-connector.py b/exec/topology-provider-connector.py index 5669230e..e768c7de 100755 --- a/exec/topology-provider-connector.py +++ b/exec/topology-provider-connector.py @@ -12,7 +12,7 @@ from argo_connectors.io.statewrite import state_write from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf -from argo_connectors.utils import filename_date, datestamp, date_check +from argo_connectors.utils import filename_date, datestamp, date_check, parse_conf_path from argo_connectors.tasks.provider_topology import TaskProviderTopology from argo_connectors.tasks.common import write_state @@ -41,7 +41,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = list(), list() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -53,6 +52,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) From b811ed9d81351dd90ba4b51dc04d34f3abe8b904 Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Fri, 2 Jun 2023 09:06:53 +0200 Subject: [PATCH 4/5] removed tenant folders instead added tenant name to connectors.log file --- modules/log.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/log.py b/modules/log.py index 5bb0d6bf..e9362aea 100644 --- a/modules/log.py +++ b/modules/log.py @@ -13,10 +13,8 @@ def __init__(self, connector, tenant_name): self.connector = connector self.tenant_name = tenant_name - LOGFILE = f"/var/log/argo-connectors/{self.tenant_name}/connectors.log" - - if self.tenant_name is not None and not os.path.exists(LOGFILE): - os.makedirs(f'/var/log/argo-connectors/{self.tenant_name}/') + if self.tenant_name is not None: + LOGFILE = f"/var/log/argo-connectors/connectors-{self.tenant_name}.log" logging.basicConfig(format=lfs, level=logging.INFO, stream=sys.stdout) self.logger = logging.getLogger(connector) From 6df13f788e7388f0481b7394e05301c11f8864ab Mon Sep 17 00:00:00 2001 From: Dean Hudek Date: Fri, 2 Jun 2023 14:08:18 +0200 Subject: [PATCH 5/5] added mock tenant to logger in tests --- tests/test_bdiisepath.py | 2 +- tests/test_contactfeed.py | 2 +- tests/test_retry.py | 2 +- tests/test_topofeed.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_bdiisepath.py b/tests/test_bdiisepath.py index 6692227b..8237e070 100644 --- a/tests/test_bdiisepath.py +++ b/tests/test_bdiisepath.py @@ -9,7 +9,7 @@ from bonsai import LDAPEntry -logger = Logger('test_contactfeed.py') +logger = Logger('test_contactfeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' diff --git a/tests/test_contactfeed.py b/tests/test_contactfeed.py index d0bbfcc9..54934278 100644 --- a/tests/test_contactfeed.py +++ b/tests/test_contactfeed.py @@ -9,7 +9,7 @@ from argo_connectors.parse.provider_contacts import ParseResourcesContacts, ParseProvidersContacts -logger = Logger('test_contactfeed.py') +logger = Logger('test_contactfeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' diff --git a/tests/test_retry.py b/tests/test_retry.py index 3c7526ab..5c666511 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -9,7 +9,7 @@ from argo_connectors.log import Logger from argo_connectors.exceptions import ConnectorHttpError -logger = Logger('test_topofeed.py') +logger = Logger('test_topofeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' diff --git a/tests/test_topofeed.py b/tests/test_topofeed.py index 95801d81..d7bdb86b 100644 --- a/tests/test_topofeed.py +++ b/tests/test_topofeed.py @@ -8,7 +8,7 @@ from argo_connectors.exceptions import ConnectorParseError from argo_connectors.mesh.contacts import attach_contacts_topodata -logger = Logger('test_topofeed.py') +logger = Logger('test_topofeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' # Help function - check if any of endpoints contains extensions