diff --git a/exec/downtimes-csv-connector.py b/exec/downtimes-csv-connector.py index 154e9480..ddbc3995 100755 --- a/exec/downtimes-csv-connector.py +++ b/exec/downtimes-csv-connector.py @@ -12,6 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes from argo_connectors.tasks.common import write_state +from argo_connectors.utils import parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -37,13 +38,16 @@ def main(): parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/downtimes-gocdb-connector.py b/exec/downtimes-gocdb-connector.py index f6f3dc41..13ffba1a 100755 --- a/exec/downtimes-gocdb-connector.py +++ b/exec/downtimes-gocdb-connector.py @@ -12,6 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.gocdb_downtimes import TaskGocdbDowntimes from argo_connectors.tasks.common import write_state +from argo_connectors.utils import parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -42,13 +43,16 @@ def main(): help='path to global configuration file', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/metricprofile-webapi-connector.py b/exec/metricprofile-webapi-connector.py index 33f43542..387d8b20 100755 --- a/exec/metricprofile-webapi-connector.py +++ b/exec/metricprofile-webapi-connector.py @@ -10,7 +10,7 @@ from argo_connectors.config import CustomerConf, Global from argo_connectors.log import Logger from argo_connectors.tasks.webapi_metricprofile import TaskWebApiMetricProfile -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None @@ -26,7 +26,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -38,6 +37,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/service-types-csv-connector.py b/exec/service-types-csv-connector.py index 078045da..77996d50 100755 --- a/exec/service-types-csv-connector.py +++ b/exec/service-types-csv-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes from argo_connectors.tasks.common import write_state -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -43,13 +43,16 @@ def main(): if args.date and date_check(args.date): fixed_date = args.date - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/service-types-gocdb-connector.py b/exec/service-types-gocdb-connector.py index 2977afae..17271bf6 100755 --- a/exec/service-types-gocdb-connector.py +++ b/exec/service-types-gocdb-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.gocdb_servicetypes import TaskGocdbServiceTypes from argo_connectors.tasks.common import write_state -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -43,13 +43,16 @@ def main(): if args.date and date_check(args.date): fixed_date = args.date - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/service-types-json-connector.py b/exec/service-types-json-connector.py index 1e30063f..03134eb5 100755 --- a/exec/service-types-json-connector.py +++ b/exec/service-types-json-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes from argo_connectors.tasks.common import write_state -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.config import Global, CustomerConf @@ -43,13 +43,16 @@ def main(): if args.date and date_check(args.date): fixed_date = args.date - logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-agora-connector.py b/exec/topology-agora-connector.py index 53f04ac7..7dacb7b6 100755 --- a/exec/topology-agora-connector.py +++ b/exec/topology-agora-connector.py @@ -10,7 +10,7 @@ from argo_connectors.exceptions import ConnectorError, ConnectorHttpError, ConnectorParseError from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path from argo_connectors.tasks.agora_topology import TaskProviderTopology from argo_connectors.tasks.common import write_state @@ -39,7 +39,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date @@ -50,6 +49,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-csv-connector.py b/exec/topology-csv-connector.py index 082b528d..62887436 100755 --- a/exec/topology-csv-connector.py +++ b/exec/topology-csv-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state from argo_connectors.tasks.flat_topology import TaskFlatTopology -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None @@ -41,7 +41,6 @@ def main(): parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -53,6 +52,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-gocdb-connector.py b/exec/topology-gocdb-connector.py index 19409265..8d30ca99 100755 --- a/exec/topology-gocdb-connector.py +++ b/exec/topology-gocdb-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state from argo_connectors.tasks.gocdb_topology import TaskGocdbTopology -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None globopts = {} @@ -58,7 +58,6 @@ def main(): help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = [], [] - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -71,6 +70,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-json-connector.py b/exec/topology-json-connector.py index e95f5bd9..b86dfac1 100755 --- a/exec/topology-json-connector.py +++ b/exec/topology-json-connector.py @@ -12,7 +12,7 @@ from argo_connectors.log import Logger from argo_connectors.tasks.common import write_state from argo_connectors.tasks.flat_topology import TaskFlatTopology -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path logger = None globopts = {} @@ -40,7 +40,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = list(), list() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -52,6 +51,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/topology-provider-connector.py b/exec/topology-provider-connector.py index 5669230e..e768c7de 100755 --- a/exec/topology-provider-connector.py +++ b/exec/topology-provider-connector.py @@ -12,7 +12,7 @@ from argo_connectors.io.statewrite import state_write from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf -from argo_connectors.utils import filename_date, datestamp, date_check +from argo_connectors.utils import filename_date, datestamp, date_check, parse_conf_path from argo_connectors.tasks.provider_topology import TaskProviderTopology from argo_connectors.tasks.common import write_state @@ -41,7 +41,6 @@ def main(): parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = list(), list() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -53,6 +52,10 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/exec/weights-vapor-connector.py b/exec/weights-vapor-connector.py index b5aa728c..e7096454 100755 --- a/exec/weights-vapor-connector.py +++ b/exec/weights-vapor-connector.py @@ -13,7 +13,7 @@ from argo_connectors.log import Logger from argo_connectors.config import Global, CustomerConf -from argo_connectors.utils import date_check +from argo_connectors.utils import date_check, parse_conf_path globopts = {} logger = None @@ -31,7 +31,6 @@ def main(): help='write data for this date', type=str, required=False) args = parser.parse_args() - logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): @@ -43,6 +42,11 @@ def main(): confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) + + tenant_name = parse_conf_path(confpath) + + logger = Logger(os.path.basename(sys.argv[0]), tenant_name) + confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) diff --git a/modules/config.py b/modules/config.py index 0ca4afb6..a01c3bf5 100644 --- a/modules/config.py +++ b/modules/config.py @@ -30,7 +30,7 @@ class Global(object): def __init__(self, caller, confpath=None, **kwargs): self.optional = dict() - self.logger = Logger(str(self.__class__)) + self.logger = Logger(str(self.__class__), tenant_name=None) self._filename = '/etc/argo-connectors/global.conf' if not confpath else confpath self._checkpath = kwargs['checkpath'] if 'checkpath' in kwargs.keys( ) else False @@ -228,7 +228,7 @@ class CustomerConf(object): deftopofeed = 'https://goc.egi.eu/gocdbpi/' def __init__(self, caller, confpath, **kwargs): - self.logger = Logger(str(self.__class__)) + self.logger = Logger(str(self.__class__), tenant_name=None) self._filename = '/etc/argo-connectors/customer.conf' if not confpath else confpath if not kwargs: self._jobattrs = self._defjobattrs[os.path.basename(caller)] diff --git a/modules/log.py b/modules/log.py index 119ef899..e9362aea 100644 --- a/modules/log.py +++ b/modules/log.py @@ -2,16 +2,19 @@ import logging.handlers import sys import socket - -LOGFILE = "/var/log/argo-connectors/connectors.log" +import os class Logger: - def __init__(self, connector): + def __init__(self, connector, tenant_name): lfs = '%(name)s[%(process)s]: %(levelname)s %(message)s' logformat = logging.Formatter(lfs) logverbose = logging.INFO self.connector = connector + self.tenant_name = tenant_name + + if self.tenant_name is not None: + LOGFILE = f"/var/log/argo-connectors/connectors-{self.tenant_name}.log" logging.basicConfig(format=lfs, level=logging.INFO, stream=sys.stdout) self.logger = logging.getLogger(connector) diff --git a/modules/utils.py b/modules/utils.py index 963c18e1..a4b524d0 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -59,4 +59,10 @@ def remove_non_utf(string): if '"' in string: string = string.replace('"', '') - return unidecode(string) \ No newline at end of file + return unidecode(string) + + +def parse_conf_path(path): + tenant = path.split('/')[3].split('-')[0] + + return tenant.upper() diff --git a/tests/test_bdiisepath.py b/tests/test_bdiisepath.py index 6692227b..8237e070 100644 --- a/tests/test_bdiisepath.py +++ b/tests/test_bdiisepath.py @@ -9,7 +9,7 @@ from bonsai import LDAPEntry -logger = Logger('test_contactfeed.py') +logger = Logger('test_contactfeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' diff --git a/tests/test_contactfeed.py b/tests/test_contactfeed.py index d0bbfcc9..54934278 100644 --- a/tests/test_contactfeed.py +++ b/tests/test_contactfeed.py @@ -9,7 +9,7 @@ from argo_connectors.parse.provider_contacts import ParseResourcesContacts, ParseProvidersContacts -logger = Logger('test_contactfeed.py') +logger = Logger('test_contactfeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' diff --git a/tests/test_retry.py b/tests/test_retry.py index 3c7526ab..5c666511 100644 --- a/tests/test_retry.py +++ b/tests/test_retry.py @@ -9,7 +9,7 @@ from argo_connectors.log import Logger from argo_connectors.exceptions import ConnectorHttpError -logger = Logger('test_topofeed.py') +logger = Logger('test_topofeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' diff --git a/tests/test_topofeed.py b/tests/test_topofeed.py index 95801d81..d7bdb86b 100644 --- a/tests/test_topofeed.py +++ b/tests/test_topofeed.py @@ -8,7 +8,7 @@ from argo_connectors.exceptions import ConnectorParseError from argo_connectors.mesh.contacts import attach_contacts_topodata -logger = Logger('test_topofeed.py') +logger = Logger('test_topofeed.py', 'foo_tenant') CUSTOMER_NAME = 'CUSTOMERFOO' # Help function - check if any of endpoints contains extensions