diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py new file mode 100755 index 00000000..848e10ae --- /dev/null +++ b/aodncore/bin/logview.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python + +""" +Script to parse and view logs generated by pipelines. +""" + +import argparse +import os +import re + +# location of logs +from aodncore.util.logviewer import LOG_WATCH, LOGDIR_PROCESS, LogViewer + + +def find_log(input_file): + """ + Given the name of an uploaded file, find the log file(s) from the pipeline process that handled it. + + :param str input_file: Name of uploaded file + :return: List of full paths to log files + + """ + # TODO: implement find_log + # Things to try: + # Read all process logs in LOGDIR_PROCESS and use pattern match + + # first, if the input file name includes a task_id at the end, remove it + filename = os.path.basename(input_file) + assert filename, 'No input file name provided!' + match = re.match( + r"(.+?)([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?$", + filename + ) + filename, task_id = match.groups() + task_name_pattern = re.compile(r"task_name='(tasks.\w+)'.*pathname='.*{fn}'".format(fn=filename)) + + logfiles = [] + # read LOG_WATCH file and find the file name + with open(LOG_WATCH) as watchlog: + for line in watchlog: + match = task_name_pattern.search(line) + if match: + logfiles.append( + os.path.join(LOGDIR_PROCESS, '{}.log'.format(match.group(1))) + ) + + return None + + +def parse_args(): + """Parse the command line""" + parser = argparse.ArgumentParser() + parser.add_argument('-l', '--logfile', help='path to pipeline log file') + parser.add_argument('-t', '--task_name', help='log for pipeline task') + parser.add_argument('-i', '--task_id', help='filter by task_id', metavar='ID') + parser.add_argument('-e', '--errors', help='error lines only', action='store_true') + parser.add_argument('-w', '--warnings', help='warning & error lines only', action='store_true') + parser.add_argument('-p', '--pattern', help='lines matching regex pattern', metavar='REGEX') + parser.add_argument('-f', '--file', help='name of processed file') + + args = parser.parse_args() + + if not args.logfile: + if args.task_name: + args.logfile = os.path.join(LOGDIR_PROCESS, 'tasks.{}.log'.format(args.task_name)) + if args.file: + args.logfile = find_log(args.file) + + args.levels = None + if args.errors: + args.levels = ('ERROR', 'CRITICAL') + if args.warnings: + args.levels = ('WARNING', 'ERROR', 'CRITICAL') + + print('Args: {}\n'.format(args)) + + return args + + +if __name__ == '__main__': + args = parse_args() + + # TODO: filter by file name (parent or child) + + lv = LogViewer(args.logfile) + lv.show(task_id=args.task_id, levels=args.levels, pattern=args.pattern) + + exit(0) diff --git a/aodncore/util/logviewer.py b/aodncore/util/logviewer.py new file mode 100644 index 00000000..c1c50cc3 --- /dev/null +++ b/aodncore/util/logviewer.py @@ -0,0 +1,91 @@ +import os +import re +import sys +from collections import OrderedDict + +LOGDIR_BASE = '/sw/chef/src/tmp/p2_logs' +LOG_WATCH = LOGDIR_BASE + '/watchservice/pipeline_watchservice-stderr.log' +LOGDIR_CELERY = LOGDIR_BASE + '/celery' +LOGDIR_PROCESS = LOGDIR_BASE + '/process' + +# regular expressions to match log format and define fields extracted from log +LOG_FIELDS = OrderedDict([ + ('time', r"(?P