ARGO-2682 Tenant separated logs files

exec/downtimes-csv-connector.py

@@ -12,6 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.flat_downtimes import TaskCsvDowntimes
 from argo_connectors.tasks.common import write_state
+from argo_connectors.utils import parse_conf_path
 
 from argo_connectors.config import Global, CustomerConf
 
@@ -37,13 +38,16 @@ def main():
     parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
     confpath = args.gloconf[0] if args.gloconf else None
     cglob = Global(sys.argv[0], confpath)
     globopts = cglob.parse()
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/downtimes-gocdb-connector.py

@@ -12,6 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.gocdb_downtimes import TaskGocdbDowntimes
 from argo_connectors.tasks.common import write_state
+from argo_connectors.utils import parse_conf_path
 
 from argo_connectors.config import Global, CustomerConf
 
@@ -42,13 +43,16 @@ def main():
                         help='path to global configuration file', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
     confpath = args.gloconf[0] if args.gloconf else None
     cglob = Global(sys.argv[0], confpath)
     globopts = cglob.parse()
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/metricprofile-webapi-connector.py

@@ -10,7 +10,7 @@
 from argo_connectors.config import CustomerConf, Global
 from argo_connectors.log import Logger
 from argo_connectors.tasks.webapi_metricprofile import TaskWebApiMetricProfile
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 logger = None
 
@@ -26,7 +26,6 @@ def main():
     parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
 
     fixed_date = None
     if args.date and date_check(args.date):
@@ -38,6 +37,10 @@ def main():
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/service-types-csv-connector.py

@@ -12,7 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes
 from argo_connectors.tasks.common import write_state
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 from argo_connectors.config import Global, CustomerConf
 
@@ -43,13 +43,16 @@ def main():
     if args.date and date_check(args.date):
         fixed_date = args.date
 
-    logger = Logger(os.path.basename(sys.argv[0]))
     confpath = args.gloconf[0] if args.gloconf else None
     cglob = Global(sys.argv[0], confpath)
     globopts = cglob.parse()
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/service-types-gocdb-connector.py

@@ -12,7 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.gocdb_servicetypes import TaskGocdbServiceTypes
 from argo_connectors.tasks.common import write_state
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 from argo_connectors.config import Global, CustomerConf
 
@@ -43,13 +43,16 @@ def main():
     if args.date and date_check(args.date):
         fixed_date = args.date
 
-    logger = Logger(os.path.basename(sys.argv[0]))
     confpath = args.gloconf[0] if args.gloconf else None
     cglob = Global(sys.argv[0], confpath)
     globopts = cglob.parse()
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/service-types-json-connector.py

@@ -12,7 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.flat_servicetypes import TaskFlatServiceTypes
 from argo_connectors.tasks.common import write_state
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 from argo_connectors.config import Global, CustomerConf
 
@@ -43,13 +43,16 @@ def main():
     if args.date and date_check(args.date):
         fixed_date = args.date
 
-    logger = Logger(os.path.basename(sys.argv[0]))
     confpath = args.gloconf[0] if args.gloconf else None
     cglob = Global(sys.argv[0], confpath)
     globopts = cglob.parse()
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/topology-agora-connector.py

@@ -10,7 +10,7 @@
 from argo_connectors.exceptions import ConnectorError, ConnectorHttpError, ConnectorParseError
 from argo_connectors.log import Logger
 from argo_connectors.config import Global, CustomerConf
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 from argo_connectors.tasks.agora_topology import TaskProviderTopology
 from argo_connectors.tasks.common import write_state
 
@@ -39,7 +39,6 @@ def main():
     parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
     fixed_date = None
     if args.date and date_check(args.date):
         fixed_date = args.date
@@ -50,6 +49,10 @@ def main():
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/topology-csv-connector.py

@@ -12,7 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.common import write_state
 from argo_connectors.tasks.flat_topology import TaskFlatTopology
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 logger = None
 
@@ -41,7 +41,6 @@ def main():
     parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False)
     parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
     args = parser.parse_args()
-    logger = Logger(os.path.basename(sys.argv[0]))
 
     fixed_date = None
     if args.date and date_check(args.date):
@@ -53,6 +52,10 @@ def main():
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/topology-gocdb-connector.py

@@ -12,7 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.common import write_state
 from argo_connectors.tasks.gocdb_topology import TaskGocdbTopology
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 logger = None
 globopts = {}
@@ -58,7 +58,6 @@ def main():
                         help='write data for this date', type=str, required=False)
     args = parser.parse_args()
     group_endpoints, group_groups = [], []
-    logger = Logger(os.path.basename(sys.argv[0]))
 
     fixed_date = None
     if args.date and date_check(args.date):
@@ -71,6 +70,10 @@ def main():
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/topology-json-connector.py

@@ -12,7 +12,7 @@
 from argo_connectors.log import Logger
 from argo_connectors.tasks.common import write_state
 from argo_connectors.tasks.flat_topology import TaskFlatTopology
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 logger = None
 globopts = {}
@@ -40,7 +40,6 @@ def main():
     parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
     args = parser.parse_args()
     group_endpoints, group_groups = list(), list()
-    logger = Logger(os.path.basename(sys.argv[0]))
 
     fixed_date = None
     if args.date and date_check(args.date):
@@ -52,6 +51,10 @@ def main():
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/topology-provider-connector.py

@@ -12,7 +12,7 @@
 from argo_connectors.io.statewrite import state_write
 from argo_connectors.log import Logger
 from argo_connectors.config import Global, CustomerConf
-from argo_connectors.utils import filename_date, datestamp, date_check
+from argo_connectors.utils import filename_date, datestamp, date_check, parse_conf_path
 from argo_connectors.tasks.provider_topology import TaskProviderTopology
 from argo_connectors.tasks.common import write_state
 
@@ -41,7 +41,6 @@ def main():
     parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
     args = parser.parse_args()
     group_endpoints, group_groups = list(), list()
-    logger = Logger(os.path.basename(sys.argv[0]))
 
     fixed_date = None
     if args.date and date_check(args.date):
@@ -53,6 +52,10 @@ def main():
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

exec/weights-vapor-connector.py

@@ -13,7 +13,7 @@
 from argo_connectors.log import Logger
 
 from argo_connectors.config import Global, CustomerConf
-from argo_connectors.utils import date_check
+from argo_connectors.utils import date_check, parse_conf_path
 
 globopts = {}
 logger = None
@@ -31,7 +31,6 @@ def main():
                         help='write data for this date', type=str, required=False)
     args = parser.parse_args()
 
-    logger = Logger(os.path.basename(sys.argv[0]))
 
     fixed_date = None
     if args.date and date_check(args.date):
@@ -43,6 +42,11 @@ def main():
 
     confpath = args.custconf[0] if args.custconf else None
     confcust = CustomerConf(sys.argv[0], confpath)
+
+    tenant_name = parse_conf_path(confpath)
+
+    logger = Logger(os.path.basename(sys.argv[0]), tenant_name)
+
     confcust.parse()
     confcust.make_dirstruct()
     confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])

modules/config.py

@@ -30,7 +30,7 @@ class Global(object):
     def __init__(self, caller, confpath=None, **kwargs):
         self.optional = dict()
 
-        self.logger = Logger(str(self.__class__))
+        self.logger = Logger(str(self.__class__), tenant_name=None)
         self._filename = '/etc/argo-connectors/global.conf' if not confpath else confpath
         self._checkpath = kwargs['checkpath'] if 'checkpath' in kwargs.keys(
         ) else False
@@ -228,7 +228,7 @@ class CustomerConf(object):
     deftopofeed = 'https://goc.egi.eu/gocdbpi/'
 
     def __init__(self, caller, confpath, **kwargs):
-        self.logger = Logger(str(self.__class__))
+        self.logger = Logger(str(self.__class__), tenant_name=None)
         self._filename = '/etc/argo-connectors/customer.conf' if not confpath else confpath
         if not kwargs:
             self._jobattrs = self._defjobattrs[os.path.basename(caller)]

modules/log.py

@@ -2,16 +2,19 @@
 import logging.handlers
 import sys
 import socket
-
-LOGFILE = "/var/log/argo-connectors/connectors.log"
+import os
 
 
 class Logger:
-    def __init__(self, connector):
+    def __init__(self, connector, tenant_name):
         lfs = '%(name)s[%(process)s]: %(levelname)s %(message)s'
         logformat = logging.Formatter(lfs)
         logverbose = logging.INFO
         self.connector = connector
+        self.tenant_name = tenant_name
+
+        if self.tenant_name is not None:
+            LOGFILE = f"/var/log/argo-connectors/connectors-{self.tenant_name}.log"
 
         logging.basicConfig(format=lfs, level=logging.INFO, stream=sys.stdout)
         self.logger = logging.getLogger(connector)

modules/utils.py

@@ -59,4 +59,10 @@ def remove_non_utf(string):
     if '"' in string:
         string = string.replace('"', '')
 
-    return unidecode(string)
+    return unidecode(string)
+
+
+def parse_conf_path(path):
+    tenant = path.split('/')[3].split('-')[0]
+
+    return tenant.upper()

tests/test_bdiisepath.py

@@ -9,7 +9,7 @@
 from bonsai import LDAPEntry
 
 
-logger = Logger('test_contactfeed.py')
+logger = Logger('test_contactfeed.py', 'foo_tenant')
 CUSTOMER_NAME = 'CUSTOMERFOO'