From 4034d60655a8af5728392dd60a864ccfc6af35a4 Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Wed, 15 Nov 2017 17:59:59 +1100
Subject: [PATCH 1/8] WIP: add logview.py script to parse and review pipeline
 logs

---
 aodncore/bin/logview.py | 98 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 98 insertions(+)
 create mode 100755 aodncore/bin/logview.py

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
new file mode 100755
index 00000000..99239264
--- /dev/null
+++ b/aodncore/bin/logview.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+
+"""
+Script to parse and view logs generated by pipelines.
+"""
+
+import argparse
+from collections import OrderedDict
+import re
+import sys
+
+
+# regular expressions to match log format and define fields extracted from log
+LOG_FIELDS = OrderedDict([
+    ('time', r"(?P<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+"),
+    ('level', r"(?P<level>[A-Z]+)\s+"),
+    ('task_name', r"tasks.(?P<task_name>\w+)"),
+    ('task_id', r"\[(?P<task_id>[0-9a-f-]+)\]\s+"),
+    ('message', r"(?P<message>.*)")
+])
+INPUT_REGEX = re.compile(''.join(LOG_FIELDS.values()))
+OUTPUT_FORMAT = '{time:20} {level:>9} {message}\n'
+
+
+def parse_args():
+    """Parse the command line"""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('logfile', help='pipeline task log file')
+    parser.add_argument('task_id')
+
+    args = parser.parse_args()
+
+    return args
+
+
+def parse_log(logfile):
+    """
+    Parse the log returning a list of log data. Each list item is a dictionary of fieldname: value pairs.
+
+    :param str logfile: Path to log file
+    :return: list of log details from each line.
+
+    """
+    # read all log lines
+    with open(logfile) as log:
+        lines = log.readlines()
+
+    # parse each line
+    log_data = []
+    for line in lines:
+        line = line.strip()
+
+        m = INPUT_REGEX.match(line)
+        if m is not None:
+            log_data.append(m.groupdict())
+
+        # TODO: deal with unformatted lines
+
+    return log_data
+
+
+def print_output(log_data, fmt=OUTPUT_FORMAT):
+    """
+    Print the log data to stdout.
+
+    :param list log_data: List of log data as returned by parse_log
+    :param str fmt: format string to convert log fields into a line of output
+
+    """
+    output = []
+    for data in log_data:
+        output.append(fmt.format(**data))
+        # TODO: wrap long lines
+
+    try:
+        sys.stdout.writelines(output)
+        sys.stdout.flush()
+    except IOError:
+        # this can happen if output is piped to `head` or `less`
+        pass
+
+
+def view_log(args):
+
+    log_data = parse_log(args.logfile)
+
+    # TODO: filtering
+
+    # format & print output
+    print_output(log_data)
+
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    view_log(args)
+
+    exit(0)

From 1b146275ae12c735ba2d72f54f498982da4eeadd Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Wed, 15 Nov 2017 18:30:10 +1100
Subject: [PATCH 2/8] WIP: add filtering by task_id and levels

---
 aodncore/bin/logview.py | 22 +++++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
index 99239264..2be4b0ec 100755
--- a/aodncore/bin/logview.py
+++ b/aodncore/bin/logview.py
@@ -26,7 +26,9 @@ def parse_args():
     """Parse the command line"""
     parser = argparse.ArgumentParser()
     parser.add_argument('logfile', help='pipeline task log file')
-    parser.add_argument('task_id')
+    parser.add_argument('-t', '--task_id', help='print lines for task_id', metavar='ID')
+    parser.add_argument('-e', '--errors', help='print error lines only', action='store_true')
+    parser.add_argument('-w', '--warnings', help='print warning & error lines only', action='store_true')
 
     args = parser.parse_args()
 
@@ -82,12 +84,26 @@ def print_output(log_data, fmt=OUTPUT_FORMAT):
 
 def view_log(args):
 
-    log_data = parse_log(args.logfile)
+    all_data = parse_log(args.logfile)
 
     # TODO: filtering
+    out_data = []
+    task_id = getattr(args, 'task_id', None)
+    levels = None
+    if args.errors:
+        levels = ('ERROR', 'CRITICAL')
+    if args.warnings:
+        levels = ('WARNING', 'ERROR', 'CRITICAL')
+    for data in all_data:
+        if task_id and data['task_id'] != task_id:
+            continue
+        if levels and data['level'] not in levels:
+            continue
+
+        out_data.append(data)
 
     # format & print output
-    print_output(log_data)
+    print_output(out_data)
 
 
 if __name__ == '__main__':

From 570d3ee032c16333b8d49ff39f0305968d34b316 Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Wed, 15 Nov 2017 18:41:57 +1100
Subject: [PATCH 3/8] WIP: add filtering by regex

---
 aodncore/bin/logview.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
index 2be4b0ec..a51726c3 100755
--- a/aodncore/bin/logview.py
+++ b/aodncore/bin/logview.py
@@ -26,9 +26,10 @@ def parse_args():
     """Parse the command line"""
     parser = argparse.ArgumentParser()
     parser.add_argument('logfile', help='pipeline task log file')
-    parser.add_argument('-t', '--task_id', help='print lines for task_id', metavar='ID')
-    parser.add_argument('-e', '--errors', help='print error lines only', action='store_true')
-    parser.add_argument('-w', '--warnings', help='print warning & error lines only', action='store_true')
+    parser.add_argument('-t', '--task_id', help='lines for task_id', metavar='ID')
+    parser.add_argument('-e', '--errors', help='error lines only', action='store_true')
+    parser.add_argument('-w', '--warnings', help='warning & error lines only', action='store_true')
+    parser.add_argument('-p', '--pattern', help='lines matching regex pattern', metavar='REGEX')
 
     args = parser.parse_args()
 
@@ -87,18 +88,26 @@ def view_log(args):
     all_data = parse_log(args.logfile)
 
     # TODO: filtering
-    out_data = []
     task_id = getattr(args, 'task_id', None)
+
     levels = None
     if args.errors:
         levels = ('ERROR', 'CRITICAL')
     if args.warnings:
         levels = ('WARNING', 'ERROR', 'CRITICAL')
+
+    pattern = None
+    if hasattr(args, 'pattern'):
+        pattern = re.compile(args.pattern)
+
+    out_data = []
     for data in all_data:
         if task_id and data['task_id'] != task_id:
             continue
         if levels and data['level'] not in levels:
             continue
+        if pattern and not pattern.search(data['message']):
+            continue
 
         out_data.append(data)
 

From 918aaa5b6d6948952461337da15963fdb604af59 Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Tue, 28 Nov 2017 14:54:43 +1100
Subject: [PATCH 4/8] WIP: logview add TODOs, start of find_log

---
 aodncore/bin/logview.py | 47 +++++++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 11 deletions(-)

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
index a51726c3..f55e4984 100755
--- a/aodncore/bin/logview.py
+++ b/aodncore/bin/logview.py
@@ -10,6 +10,12 @@
 import sys
 
 
+# location of logs
+LOGDIR_BASE = '/sw/chef/src/tmp/p2_logs'
+LOG_WATCH = LOGDIR_BASE + '/watchservice/pipeline_watchservice-stderr.log'
+LOGDIR_CELERY = LOGDIR_BASE + '/celery'
+LOGDIR_PROCESS = LOGDIR_BASE + '/process'
+
 # regular expressions to match log format and define fields extracted from log
 LOG_FIELDS = OrderedDict([
     ('time', r"(?P<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+"),
@@ -22,6 +28,20 @@
 OUTPUT_FORMAT = '{time:20} {level:>9} {message}\n'
 
 
+def find_log(input_file):
+    """
+    Given the name of an uploaded file, find the log file from the pipeline process that handled it.
+
+    :param str input_file: Name of uploaded file
+    :return: Full path to log file.
+    """
+    # TODO: implement find_log
+    # Things to try:
+    #   Read LOG_WATCH file and find the file name
+    #   Read all process logs in LOGDIR_PROCESS and use pattern match
+    return ''
+
+
 def parse_args():
     """Parse the command line"""
     parser = argparse.ArgumentParser()
@@ -30,9 +50,15 @@ def parse_args():
     parser.add_argument('-e', '--errors', help='error lines only', action='store_true')
     parser.add_argument('-w', '--warnings', help='warning & error lines only', action='store_true')
     parser.add_argument('-p', '--pattern', help='lines matching regex pattern', metavar='REGEX')
+    parser.add_argument('-f', '--file', help='name of processed file')
 
     args = parser.parse_args()
 
+    print('Args: {}\n'.format(args))
+
+    if args.file:
+        args.logfile = find_log(args.file)
+
     return args
 
 
@@ -73,7 +99,7 @@ def print_output(log_data, fmt=OUTPUT_FORMAT):
     output = []
     for data in log_data:
         output.append(fmt.format(**data))
-        # TODO: wrap long lines
+        # TODO: wrap long lines?
 
     try:
         sys.stdout.writelines(output)
@@ -83,21 +109,17 @@ def print_output(log_data, fmt=OUTPUT_FORMAT):
         pass
 
 
-def view_log(args):
+def view_log(logfile, task_id=None, errors=False, warnings=False, pattern=None):
 
-    all_data = parse_log(args.logfile)
-
-    # TODO: filtering
-    task_id = getattr(args, 'task_id', None)
+    all_data = parse_log(logfile)
 
     levels = None
-    if args.errors:
+    if errors:
         levels = ('ERROR', 'CRITICAL')
-    if args.warnings:
+    if warnings:
         levels = ('WARNING', 'ERROR', 'CRITICAL')
 
-    pattern = None
-    if hasattr(args, 'pattern'):
+    if pattern:
         pattern = re.compile(args.pattern)
 
     out_data = []
@@ -108,6 +130,7 @@ def view_log(args):
             continue
         if pattern and not pattern.search(data['message']):
             continue
+        # TODO: filter by handler step?
 
         out_data.append(data)
 
@@ -118,6 +141,8 @@ def view_log(args):
 if __name__ == '__main__':
     args = parse_args()
 
-    view_log(args)
+    # TODO: filter by file name (parent or child)
+
+    view_log(args.logfile, task_id=args.task_id, errors=args.errors, warnings=args.warnings, pattern=args.pattern)
 
     exit(0)

From 7af3b23ea9bf45ac51cfb6b4c7944fb87a3b783b Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Tue, 28 Nov 2017 16:55:30 +1100
Subject: [PATCH 5/8] WIP: * process log one line at a time * create LogViewer
 class

---
 aodncore/bin/logview.py | 138 +++++++++++++++++-----------------------
 1 file changed, 60 insertions(+), 78 deletions(-)

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
index f55e4984..84beb9a5 100755
--- a/aodncore/bin/logview.py
+++ b/aodncore/bin/logview.py
@@ -25,7 +25,64 @@
     ('message', r"(?P<message>.*)")
 ])
 INPUT_REGEX = re.compile(''.join(LOG_FIELDS.values()))
-OUTPUT_FORMAT = '{time:20} {level:>9} {message}\n'
+DEFAULT_FORMAT = '{time:20} {level:>9} {message}\n'
+
+
+class LogViewer(object):
+    """
+    Class to parse logs written by pipelines and output various filtered or summary views.
+    """
+
+    def __init__(self, logfile):
+        self.logfile = logfile
+
+    def show(self, task_id=None, errors=False, warnings=False, pattern=None, fmt=DEFAULT_FORMAT):
+        """
+        Print a filtered & re-formatted view of the log to stdout
+
+        :param str task_id: only include log for given task uuid
+        :param bool errors: only include error log lines
+        :param bool warnings: only include warning & error lines
+        :param str pattern: only include log messages matching pattern (regular expression)
+        :param str fmt: output format (fmt.format() applied to dict of LOG_FIELDS extracted from log)
+        """
+
+        levels = None
+        if errors:
+            levels = ('ERROR', 'CRITICAL')
+        if warnings:
+            levels = ('WARNING', 'ERROR', 'CRITICAL')
+        if pattern:
+            pattern = re.compile(pattern)
+
+        with open(self.logfile) as log:
+            for line in log:
+                line = line.strip()
+
+                # parse a line of log data
+                m = INPUT_REGEX.match(line)
+                if m is None:
+                    # TODO: deal with unformatted lines
+                    continue
+                data = m.groupdict()
+
+                # filter -- should we include this line?
+                if task_id and data['task_id'] != task_id:
+                    continue
+                if levels and data['level'] not in levels:
+                    continue
+                if pattern and not pattern.search(data['message']):
+                    continue
+                # TODO: filter by handler step?
+
+                # format & print the line
+                line_out = fmt.format(**data)
+                try:
+                    sys.stdout.write(line_out)
+                    sys.stdout.flush()
+                except IOError:
+                    # this can happen if output is piped to `head` or `less`
+                    pass
 
 
 def find_log(input_file):
@@ -62,87 +119,12 @@ def parse_args():
     return args
 
 
-def parse_log(logfile):
-    """
-    Parse the log returning a list of log data. Each list item is a dictionary of fieldname: value pairs.
-
-    :param str logfile: Path to log file
-    :return: list of log details from each line.
-
-    """
-    # read all log lines
-    with open(logfile) as log:
-        lines = log.readlines()
-
-    # parse each line
-    log_data = []
-    for line in lines:
-        line = line.strip()
-
-        m = INPUT_REGEX.match(line)
-        if m is not None:
-            log_data.append(m.groupdict())
-
-        # TODO: deal with unformatted lines
-
-    return log_data
-
-
-def print_output(log_data, fmt=OUTPUT_FORMAT):
-    """
-    Print the log data to stdout.
-
-    :param list log_data: List of log data as returned by parse_log
-    :param str fmt: format string to convert log fields into a line of output
-
-    """
-    output = []
-    for data in log_data:
-        output.append(fmt.format(**data))
-        # TODO: wrap long lines?
-
-    try:
-        sys.stdout.writelines(output)
-        sys.stdout.flush()
-    except IOError:
-        # this can happen if output is piped to `head` or `less`
-        pass
-
-
-def view_log(logfile, task_id=None, errors=False, warnings=False, pattern=None):
-
-    all_data = parse_log(logfile)
-
-    levels = None
-    if errors:
-        levels = ('ERROR', 'CRITICAL')
-    if warnings:
-        levels = ('WARNING', 'ERROR', 'CRITICAL')
-
-    if pattern:
-        pattern = re.compile(args.pattern)
-
-    out_data = []
-    for data in all_data:
-        if task_id and data['task_id'] != task_id:
-            continue
-        if levels and data['level'] not in levels:
-            continue
-        if pattern and not pattern.search(data['message']):
-            continue
-        # TODO: filter by handler step?
-
-        out_data.append(data)
-
-    # format & print output
-    print_output(out_data)
-
-
 if __name__ == '__main__':
     args = parse_args()
 
     # TODO: filter by file name (parent or child)
 
-    view_log(args.logfile, task_id=args.task_id, errors=args.errors, warnings=args.warnings, pattern=args.pattern)
+    lv = LogViewer(args.logfile)
+    lv.show(task_id=args.task_id, errors=args.errors, warnings=args.warnings, pattern=args.pattern)
 
     exit(0)

From da2670b9d24a1668246843e2d8ca28c2f696821f Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Thu, 14 Dec 2017 18:06:50 +1100
Subject: [PATCH 6/8] WIP: * add separate log parsing and filtering methods of
 LogViewer class that return generatorss * start implementing find_log()
 function

---
 aodncore/bin/logview.py | 138 +++++++++++++++++++++++++++-------------
 1 file changed, 93 insertions(+), 45 deletions(-)

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
index 84beb9a5..899ae558 100755
--- a/aodncore/bin/logview.py
+++ b/aodncore/bin/logview.py
@@ -6,6 +6,7 @@
 
 import argparse
 from collections import OrderedDict
+import os
 import re
 import sys
 
@@ -34,76 +35,114 @@ class LogViewer(object):
     """
 
     def __init__(self, logfile):
+        assert logfile, 'No log file specified!'
+        assert os.path.isfile(logfile), '{logfile}: no such file!'.format(logfile=logfile)
         self.logfile = logfile
 
-    def show(self, task_id=None, errors=False, warnings=False, pattern=None, fmt=DEFAULT_FORMAT):
-        """
-        Print a filtered & re-formatted view of the log to stdout
+    def log_entries(self):
+        """Parse the log and return a tuple (raw, data) for one log entry at a time, where
+        raw is te full text from the log, and data is a dictionary of extracted fields as
+        per INPUT_REGEX.
 
-        :param str task_id: only include log for given task uuid
-        :param bool errors: only include error log lines
-        :param bool warnings: only include warning & error lines
-        :param str pattern: only include log messages matching pattern (regular expression)
-        :param str fmt: output format (fmt.format() applied to dict of LOG_FIELDS extracted from log)
         """
-
-        levels = None
-        if errors:
-            levels = ('ERROR', 'CRITICAL')
-        if warnings:
-            levels = ('WARNING', 'ERROR', 'CRITICAL')
-        if pattern:
-            pattern = re.compile(pattern)
-
+        # TODO: option to read from stdin
         with open(self.logfile) as log:
             for line in log:
                 line = line.strip()
-
-                # parse a line of log data
                 m = INPUT_REGEX.match(line)
                 if m is None:
                     # TODO: deal with unformatted lines
                     continue
                 data = m.groupdict()
 
-                # filter -- should we include this line?
-                if task_id and data['task_id'] != task_id:
-                    continue
-                if levels and data['level'] not in levels:
-                    continue
-                if pattern and not pattern.search(data['message']):
-                    continue
-                # TODO: filter by handler step?
+                yield line, data
 
-                # format & print the line
-                line_out = fmt.format(**data)
-                try:
-                    sys.stdout.write(line_out)
-                    sys.stdout.flush()
-                except IOError:
-                    # this can happen if output is piped to `head` or `less`
-                    pass
+    def filtered_entries(self, task_id=None, levels=None, pattern=None):
+        """
+        Filter the tuples returned by log_entries according to the filters specified.
+
+        :param str task_id: only include log for given task uuid
+        :param bool errors: only include error log lines
+        :param bool warnings: only include warning & error lines
+        :param str pattern: only include log messages matching pattern (regular expression)
+        :return: tuple (raw, data) as for log_entries
+
+        """
+        if pattern:
+            pattern = re.compile(pattern)
+
+        for raw, data in self.log_entries():
+            if task_id and data['task_id'] != task_id:
+                continue
+            if levels and data['level'] not in levels:
+                continue
+            if pattern and not pattern.search(data['message']):
+                continue
+            # TODO: filter by handler step?
+            yield raw, data
+
+    def show(self, task_id=None, levels=None, pattern=None, fmt=DEFAULT_FORMAT):
+        """
+        Print a filtered & re-formatted view of the log to stdout
+
+        :param str task_id: only include log for given task uuid
+        :param bool errors: only include error log lines
+        :param bool warnings: only include warning & error lines
+        :param str pattern: only include log messages matching pattern (regular expression)
+        :param str fmt: output format (fmt.format() applied to dict of LOG_FIELDS extracted from log)
+
+        """
+        for raw, data in self.filtered_entries(task_id=task_id, levels=levels, pattern=pattern):
+            line_out = fmt.format(**data)
+            try:
+                sys.stdout.write(line_out)
+                sys.stdout.flush()
+            except IOError:
+                # this can happen if output is piped to `head` or `less`
+                pass
 
 
 def find_log(input_file):
     """
-    Given the name of an uploaded file, find the log file from the pipeline process that handled it.
+    Given the name of an uploaded file, find the log file(s) from the pipeline process that handled it.
 
     :param str input_file: Name of uploaded file
-    :return: Full path to log file.
+    :return: List of full paths to log files
+
     """
     # TODO: implement find_log
     # Things to try:
-    #   Read LOG_WATCH file and find the file name
     #   Read all process logs in LOGDIR_PROCESS and use pattern match
-    return ''
+
+    # first, if the input file name includes a task_id at the end, remove it
+    filename = os.path.basename(input_file)
+    assert filename, 'No input file name provided!'
+    match = re.match(
+        r"(.+?)([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})?$",
+        filename
+    )
+    filename, task_id = match.groups()
+    task_name_pattern = re.compile(r"task_name='(tasks.\w+)'.*pathname='.*{fn}'".format(fn=filename))
+
+    logfiles = []
+    # read LOG_WATCH file and find the file name
+    with open(LOG_WATCH) as watchlog:
+        for line in watchlog:
+            match = task_name_pattern.search(line)
+            if match:
+                logfiles.append(
+                    os.path.join(LOGDIR_PROCESS, '{}.log'.format(match.group(1)))
+                )
+
+    return None
 
 
 def parse_args():
     """Parse the command line"""
     parser = argparse.ArgumentParser()
-    parser.add_argument('logfile', help='pipeline task log file')
-    parser.add_argument('-t', '--task_id', help='lines for task_id', metavar='ID')
+    parser.add_argument('-l', '--logfile', help='path to pipeline log file')
+    parser.add_argument('-t', '--task_name', help='log for pipeline task')
+    parser.add_argument('-i', '--task_id', help='filter by task_id', metavar='ID')
     parser.add_argument('-e', '--errors', help='error lines only', action='store_true')
     parser.add_argument('-w', '--warnings', help='warning & error lines only', action='store_true')
     parser.add_argument('-p', '--pattern', help='lines matching regex pattern', metavar='REGEX')
@@ -111,10 +150,19 @@ def parse_args():
 
     args = parser.parse_args()
 
-    print('Args: {}\n'.format(args))
+    if not args.logfile:
+        if args.task_name:
+            args.logfile = os.path.join(LOGDIR_PROCESS, 'tasks.{}.log'.format(args.task_name))
+        if args.file:
+            args.logfile = find_log(args.file)
 
-    if args.file:
-        args.logfile = find_log(args.file)
+    args.levels = None
+    if args.errors:
+        args.levels = ('ERROR', 'CRITICAL')
+    if args.warnings:
+        args.levels = ('WARNING', 'ERROR', 'CRITICAL')
+
+    print('Args: {}\n'.format(args))
 
     return args
 
@@ -125,6 +173,6 @@ def parse_args():
     # TODO: filter by file name (parent or child)
 
     lv = LogViewer(args.logfile)
-    lv.show(task_id=args.task_id, errors=args.errors, warnings=args.warnings, pattern=args.pattern)
+    lv.show(task_id=args.task_id, levels=args.levels, pattern=args.pattern)
 
     exit(0)

From 0bb00a6f0a8078c99e035d65b1433b5d94d8d7bd Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Thu, 26 Jul 2018 17:40:12 +1000
Subject: [PATCH 7/8] fixup: replace assert and fix doc strings

---
 aodncore/bin/logview.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
index 899ae558..df4b5d6b 100755
--- a/aodncore/bin/logview.py
+++ b/aodncore/bin/logview.py
@@ -35,8 +35,8 @@ class LogViewer(object):
     """
 
     def __init__(self, logfile):
-        assert logfile, 'No log file specified!'
-        assert os.path.isfile(logfile), '{logfile}: no such file!'.format(logfile=logfile)
+        if not os.path.isfile(logfile):
+            raise ValueError('{logfile}: no such file!'.format(logfile=logfile))
         self.logfile = logfile
 
     def log_entries(self):
@@ -62,8 +62,7 @@ def filtered_entries(self, task_id=None, levels=None, pattern=None):
         Filter the tuples returned by log_entries according to the filters specified.
 
         :param str task_id: only include log for given task uuid
-        :param bool errors: only include error log lines
-        :param bool warnings: only include warning & error lines
+        :param list levels: only include include messages with the given logging levels
         :param str pattern: only include log messages matching pattern (regular expression)
         :return: tuple (raw, data) as for log_entries
 
@@ -86,8 +85,7 @@ def show(self, task_id=None, levels=None, pattern=None, fmt=DEFAULT_FORMAT):
         Print a filtered & re-formatted view of the log to stdout
 
         :param str task_id: only include log for given task uuid
-        :param bool errors: only include error log lines
-        :param bool warnings: only include warning & error lines
+        :param list levels: only include include messages with the given logging levels
         :param str pattern: only include log messages matching pattern (regular expression)
         :param str fmt: output format (fmt.format() applied to dict of LOG_FIELDS extracted from log)
 

From 376946dc62ec0e00bbbf84b5bc3b1e4aa43747b6 Mon Sep 17 00:00:00 2001
From: mhidas <marty.hidas@utas.edu.au>
Date: Wed, 1 Aug 2018 11:27:06 +1000
Subject: [PATCH 8/8] WIP: move LogViewer class into util module and add tests

---
 aodncore/bin/logview.py              | 90 +--------------------------
 aodncore/util/logviewer.py           | 91 ++++++++++++++++++++++++++++
 test_aodncore/util/test_logviewer.py | 22 +++++++
 3 files changed, 114 insertions(+), 89 deletions(-)
 create mode 100644 aodncore/util/logviewer.py
 create mode 100644 test_aodncore/util/test_logviewer.py

diff --git a/aodncore/bin/logview.py b/aodncore/bin/logview.py
index df4b5d6b..848e10ae 100755
--- a/aodncore/bin/logview.py
+++ b/aodncore/bin/logview.py
@@ -5,99 +5,11 @@
 """
 
 import argparse
-from collections import OrderedDict
 import os
 import re
-import sys
-
 
 # location of logs
-LOGDIR_BASE = '/sw/chef/src/tmp/p2_logs'
-LOG_WATCH = LOGDIR_BASE + '/watchservice/pipeline_watchservice-stderr.log'
-LOGDIR_CELERY = LOGDIR_BASE + '/celery'
-LOGDIR_PROCESS = LOGDIR_BASE + '/process'
-
-# regular expressions to match log format and define fields extracted from log
-LOG_FIELDS = OrderedDict([
-    ('time', r"(?P<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+"),
-    ('level', r"(?P<level>[A-Z]+)\s+"),
-    ('task_name', r"tasks.(?P<task_name>\w+)"),
-    ('task_id', r"\[(?P<task_id>[0-9a-f-]+)\]\s+"),
-    ('message', r"(?P<message>.*)")
-])
-INPUT_REGEX = re.compile(''.join(LOG_FIELDS.values()))
-DEFAULT_FORMAT = '{time:20} {level:>9} {message}\n'
-
-
-class LogViewer(object):
-    """
-    Class to parse logs written by pipelines and output various filtered or summary views.
-    """
-
-    def __init__(self, logfile):
-        if not os.path.isfile(logfile):
-            raise ValueError('{logfile}: no such file!'.format(logfile=logfile))
-        self.logfile = logfile
-
-    def log_entries(self):
-        """Parse the log and return a tuple (raw, data) for one log entry at a time, where
-        raw is te full text from the log, and data is a dictionary of extracted fields as
-        per INPUT_REGEX.
-
-        """
-        # TODO: option to read from stdin
-        with open(self.logfile) as log:
-            for line in log:
-                line = line.strip()
-                m = INPUT_REGEX.match(line)
-                if m is None:
-                    # TODO: deal with unformatted lines
-                    continue
-                data = m.groupdict()
-
-                yield line, data
-
-    def filtered_entries(self, task_id=None, levels=None, pattern=None):
-        """
-        Filter the tuples returned by log_entries according to the filters specified.
-
-        :param str task_id: only include log for given task uuid
-        :param list levels: only include include messages with the given logging levels
-        :param str pattern: only include log messages matching pattern (regular expression)
-        :return: tuple (raw, data) as for log_entries
-
-        """
-        if pattern:
-            pattern = re.compile(pattern)
-
-        for raw, data in self.log_entries():
-            if task_id and data['task_id'] != task_id:
-                continue
-            if levels and data['level'] not in levels:
-                continue
-            if pattern and not pattern.search(data['message']):
-                continue
-            # TODO: filter by handler step?
-            yield raw, data
-
-    def show(self, task_id=None, levels=None, pattern=None, fmt=DEFAULT_FORMAT):
-        """
-        Print a filtered & re-formatted view of the log to stdout
-
-        :param str task_id: only include log for given task uuid
-        :param list levels: only include include messages with the given logging levels
-        :param str pattern: only include log messages matching pattern (regular expression)
-        :param str fmt: output format (fmt.format() applied to dict of LOG_FIELDS extracted from log)
-
-        """
-        for raw, data in self.filtered_entries(task_id=task_id, levels=levels, pattern=pattern):
-            line_out = fmt.format(**data)
-            try:
-                sys.stdout.write(line_out)
-                sys.stdout.flush()
-            except IOError:
-                # this can happen if output is piped to `head` or `less`
-                pass
+from aodncore.util.logviewer import LOG_WATCH, LOGDIR_PROCESS, LogViewer
 
 
 def find_log(input_file):
diff --git a/aodncore/util/logviewer.py b/aodncore/util/logviewer.py
new file mode 100644
index 00000000..c1c50cc3
--- /dev/null
+++ b/aodncore/util/logviewer.py
@@ -0,0 +1,91 @@
+import os
+import re
+import sys
+from collections import OrderedDict
+
+LOGDIR_BASE = '/sw/chef/src/tmp/p2_logs'
+LOG_WATCH = LOGDIR_BASE + '/watchservice/pipeline_watchservice-stderr.log'
+LOGDIR_CELERY = LOGDIR_BASE + '/celery'
+LOGDIR_PROCESS = LOGDIR_BASE + '/process'
+
+# regular expressions to match log format and define fields extracted from log
+LOG_FIELDS = OrderedDict([
+    ('time', r"(?P<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s+"),
+    ('level', r"(?P<level>[A-Z]+)\s+"),
+    ('task_name', r"tasks.(?P<task_name>\w+)"),
+    ('task_id', r"\[(?P<task_id>[0-9a-f-]+)\]\s+"),
+    ('message', r"(?P<message>.*)")
+])
+INPUT_REGEX = re.compile(''.join(LOG_FIELDS.values()))
+DEFAULT_FORMAT = '{time:20} {level:>9} {message}\n'
+
+
+class LogViewer(object):
+    """
+    Class to parse logs written by pipelines and output various filtered or summary views.
+    """
+
+    def __init__(self, logfile):
+        if not os.path.isfile(logfile):
+            raise ValueError('{logfile}: no such file!'.format(logfile=logfile))
+        self.logfile = logfile
+
+    def log_entries(self):
+        """Parse the log and return a tuple (raw, data) for one log entry at a time, where
+        raw is te full text from the log, and data is a dictionary of extracted fields as
+        per INPUT_REGEX.
+
+        """
+        # TODO: option to read from stdin
+        with open(self.logfile) as log:
+            for line in log:
+                line = line.strip()
+                m = INPUT_REGEX.match(line)
+                if m is None:
+                    # TODO: deal with unformatted lines
+                    continue
+                data = m.groupdict()
+
+                yield line, data
+
+    def filtered_entries(self, task_id=None, levels=None, pattern=None):
+        """
+        Filter the tuples returned by log_entries according to the filters specified.
+
+        :param str task_id: only include log for given task uuid
+        :param list levels: only include include messages with the given logging levels
+        :param str pattern: only include log messages matching pattern (regular expression)
+        :return: tuple (raw, data) as for log_entries
+
+        """
+        if pattern:
+            pattern = re.compile(pattern)
+
+        for raw, data in self.log_entries():
+            if task_id and data['task_id'] != task_id:
+                continue
+            if levels and data['level'] not in levels:
+                continue
+            if pattern and not pattern.search(data['message']):
+                continue
+            # TODO: filter by handler step?
+            yield raw, data
+
+    def show(self, task_id=None, levels=None, pattern=None, fmt=DEFAULT_FORMAT):
+        """
+        Print a filtered & re-formatted view of the log to stdout
+
+        :param str task_id: only include log for given task uuid
+        :param list levels: only include include messages with the given logging levels
+        :param str pattern: only include log messages matching pattern (regular expression)
+        :param str fmt: output format (fmt.format() applied to dict of LOG_FIELDS extracted from log)
+
+        """
+        for raw, data in self.filtered_entries(task_id=task_id, levels=levels, pattern=pattern):
+            line_out = fmt.format(**data)
+            try:
+                sys.stdout.write(line_out)
+                sys.stdout.flush()
+            except IOError:
+                # this can happen if output is piped to `head` or `less`
+                pass
\ No newline at end of file
diff --git a/test_aodncore/util/test_logviewer.py b/test_aodncore/util/test_logviewer.py
new file mode 100644
index 00000000..ead26fe9
--- /dev/null
+++ b/test_aodncore/util/test_logviewer.py
@@ -0,0 +1,22 @@
+import os
+import unittest
+
+from aodncore.testlib import BaseTestCase
+from aodncore.util.logviewer import LogViewer
+
+from .test_misc import get_nonexistent_path
+
+
+TEST_ROOT = os.path.join(os.path.dirname(__file__))
+LOG_FILE = os.path.join(TEST_ROOT, 'tasks.ANMN_SA.log')
+
+
+class TestLogViewer(BaseTestCase):
+    def test_init(self):
+        lv = LogViewer(LOG_FILE)
+        self.assertEqual(LOG_FILE, lv.logfile)
+        self.assertRaises(ValueError, LogViewer, get_nonexistent_path())
+
+
+if __name__ == '__main__':
+    unittest.main()