run-with-log: new script

Runs a program, captures its stdout/err into a log file with unique
timestamp, and email the log upon error.

Example:

    run-with-log -e [email protected] -p pipeline- -- \
                    pipeline.sh --param1 --param3 [...]

See 'run-with-log -h' for more details.

.gitignore

@@ -36,6 +36,7 @@ scripts/ppsx
 scripts/pss
 scripts/psx
 scripts/rsx
+scripts/run-with-log
 scripts/sort-header
 scripts/sum_file_sizes
 scripts/sumcol

Makefile.am

@@ -30,6 +30,7 @@ bin_SCRIPTS = \
 	scripts/nfs_iostat \
 	scripts/ppsx \
 	scripts/pss \
+	scripts/run-with-log \
 	scripts/sort-header \
 	scripts/sum_file_sizes \
 	scripts/sumcol \

README

@@ -23,6 +23,7 @@ Available scripts
 * ppsx - copy user+hostname+fullpath of file/dir to clipboard.
 * psx - copy fullpath of file/dir to clipboard.
 * rsx - copy rsync-compatible URL of file/dir to clipboard.
+* run-with-log - run a program, log stdout/err to file, email log on errors.
 * sort-header - wrapper for GNU sort, with header line support.
 * sum_file_sizes - sum the size of files.
 * sumcol - sum the values in a column of input file.

configure.ac

@@ -56,6 +56,7 @@ AC_CONFIG_FILES( [
 		  scripts/xtime:scripts/xtime.sh
 		  scripts/xxcat:scripts/xxcat.sh
 		  scripts/create-ssha-passwd:scripts/create-ssha-passwd.py
+		  scripts/run-with-log:scripts/run-with-log.sh
 		  ] )
 
 AC_OUTPUT()

scripts/run-with-log.sh

@@ -0,0 +1,285 @@
+#!/bin/sh
+
+# @autogenerated_warning@
+# @autogenerated_timestamp@
+# @PACKAGE@ @VERSION@
+# @PACKAGE_URL@
+
+COPYRIGHT="
+Copyright (C) 2016  A. Gordon ([email protected])
+License: GPLv3+
+"
+
+
+## Runs a program, saving STDOUT/STDERR into a log, and optionally
+## emailing it on errors.
+
+## TODO: future improvements:
+## 1. Detect existing log files (e.g. with date collosion) and
+##    add a unique sequence to the filename.
+## 2. Don't require "-p PREFIX", use "${COMMAND}-" as default.
+
+set -u
+
+
+die()
+{
+    ## Die BEFORE starting the program (e.g. bad parameters)
+    BASE=$(basename "$0")
+    echo "$BASE: error: $*" >&2
+    exit 1
+}
+
+die_with_log()
+{
+    ## Runtime error, try to send some notification...
+    BASE=$(basename "$0")
+    echo "$BASE: error: $*" >&2
+    if test -n "$email" ; then
+        echo "$BASE: error: $*" \
+            | mail -s "run-with-log: FATAL RUNTIME ERROR" "$email"
+    fi
+    exit 1
+}
+
+check_no_newlines()
+{
+    # Typical Usage:
+    #   check_no_newlines "$VAR" || die "newlines not allowed in \$VAR"
+
+    # Ensure the variable contains a single line.
+    # More than one line is invalid (and will not be detected by grep below).
+    # zero lines means the variable does not contain any newlines.
+    _lc=$(printf "%s" "$1" | LC_ALL=C wc -l) \
+        || die "failed to count lines on '$1'"
+    test "$_lc" -eq 0 && return 0
+    return 1
+}
+
+valid_email()
+{
+    # Typical usage:
+    #   valid_email "$VAR" || die "invalid email: '$VAR'"
+
+    check_no_newlines "$1" || return 1
+
+    # Validate a subset of all possible valid email addresses,
+    # (but a useful enough subset).
+    # Emails without hostnames are OK (local unix users).
+    # Emails with non-FQDN hostnames are OK (local machines).
+    # Some invalid email forms will pass, but they can be used
+    # safely as command-line parameters for the mail program,
+    # which will detect and reject them.
+    printf "%s" "$1" \
+        | LC_ALL=C grep -Eq '^[A-Za-z0-9][-A-Za-z0-9_\.\+]*(@[-A-Za-z0-9_\.]+)?$' \
+        && return 0
+
+    # Default to not valid
+    return 1
+}
+
+
+show_help_and_exit()
+{
+    BASE=$(basename "$0")
+    echo "
+$BASE - Runs a program, saving STDOUT/STDERR to a file,
+        and opptionally emailing the log.
+
+$COPYRIGHT
+Version: @VERSION@
+See: @PACKAGE_URL@
+
+Usage:  $BASE [OPTIONS] -- COMMAND [ARGS]
+
+OPTIONS:
+   -h        - This help screen.
+   -e EMAIL  - Email log to this address (default: $USER).
+   -A        - Always email logs (default: only on COMMAND failure)
+   -L FILE   - Write log to FILE (disables auto filename generation).
+   -p PREFIX - Write log to PREFIX with '-{DATE}.log' suffix appended.
+               -L and -p are mutually exclusive.
+   -n NAME   - Use NAME in email subject line, instead of COMMAND.
+
+
+Example:
+
+    $BASE -e [email protected] -p pipeline- -- \\
+          pipeline.sh --param1 --param3 [...]
+
+The above command runs 'pipeline.sh' and saves STDOUT/STDERR to
+pipeline-{DATE}.log.gz (in the current directory).
+If pipeline.sh exit with non-zero exit code, sends an email
+with the log attached to '[email protected]'.
+The subject of the email will be 'pipeline.sh - {DATE} - ERROR'.
+
+
+"
+    exit 0
+}
+
+
+##
+## Script starts here
+##
+
+## parse parameterse
+show_help=
+email=
+always_send_email=
+logfile_prefix=
+logfile_fixed=
+name=
+while getopts he:AL:n:p: param
+do
+        case $param in
+        A)      always_send_email=y
+                ;;
+        e)      email="$OPTARG"
+                valid_email "$email" || die "invalid email: '$email'"
+                ;;
+        L)      logfile_fixed="$OPTARG"
+                ;;
+        p)      logfile_prefix="$OPTARG"
+                ;;
+        h)      show_help=y
+                ;;
+        n)      name="$OPTARG"
+                ;;
+        ?)      die "unknown option. See -h for help."
+        esac
+done
+[ -n "$show_help" ] && show_help_and_exit;
+
+shift $((OPTIND-1))
+
+test -n "$logfile_fixed" && test -n "$logfile_prefix" \
+    && die "-L and -p are mutually-exclusive. See -h for help."
+test -z "$logfile_fixed" && test -z "$logfile_prefix" \
+    && die "missing -L or -p. See -h for help."
+
+test "$#" -gt 0 || die "missing COMMAND to run. See -h for help."
+COMMAND=$(basename "$1") || die "failed to get basename of '$1'"
+test -z "$name" \
+    && name="$COMMAND" \
+    || name="$name ($COMMAND)"
+
+# If no email specified with -e, send to the current user.
+# This of course requires that the MTA is properly
+# configured on the host.
+if test -z "$email" ; then
+    # FIXME: 'set -u' is defined: will this even work?
+    test -z "$USER" && die "no email specified (-e), and \$USER is empty."
+    email="$USER"
+fi
+
+##
+## Determine log filename
+##
+LOG=
+BEGDATE=$(date +%F-%H%M%S) || die_with_log "failed to get current date"
+
+if test -n "$logfile_fixed" ; then
+    LOG="$logfile_fixed"
+elif test -n "$logfile_prefix" ; then
+    LOG="${logfile_prefix}${BEGDATE}.log"
+fi
+touch "$LOG" || die_with_log "failed to touch log file '$LOG'"
+
+## Get hostname
+hostname=$(hostname) || die "failed to get hostname"
+## 'realpath' is easier, but can't assume it is installed.
+LOG_DISPLAY_NAME=$(
+    cd $(dirname "$LOG") ;
+    b=$(basename "$LOG")
+    echo "$PWD/$b" ) \
+    || die "failed to get directory name for '$LOG'"
+
+##
+## Run the program
+##
+## NOTES:
+## 1. there's a race here: if another process modifies '$LOG'
+##    to prevent writing to it, after the 'touch' above succeeded).
+## 2. If there's a system error (e.g. fork/exec fails),
+##    the error will be printed to our STDERR, not the log.
+##    (and $rc will be >= 126). FIXME: log this errors as well?
+"$@" 1>"$LOG" 2>&1
+
+rc=$?
+
+ENDDATE=$(date +%F-%H%M%S) || die_with_log "failed to get current date"
+
+gzip -f "$LOG" || die_with_log "failed to gzip log file '$LOG'"
+LOG="$LOG.gz"
+test -e "$LOG" || die_with_log "compressed log '$LOG' not found after gzip"
+LOG_DISPLAY_NAME="$LOG_DISPLAY_NAME.gz"
+
+##
+## Send report
+##
+
+msg_status=
+msg_first_line=
+send_email=
+if test "$rc" -eq 0 ; then
+    test -n "$always_send_email" && send_email=y
+    msg_status="OK"
+    msg_first_line="Command '$COMMAND' completed successfully."
+elif test "$rc" -lt 126 ; then
+    send_email=y
+    msg_status="ERROR"
+    msg_first_line="Command '$COMMAND' FAILED - returned error/exit code $rc"
+else
+    send_email=y
+    msg_status="FATAL ERROR"
+    msg_first_line="Command '$COMMAND' FAILED - shell returned error code '$rc'"
+fi
+
+
+
+## Try to grab the last 10 lines from the log.
+## Trim excessive information and invalid characters.
+## Ignore any failures.
+last_log_lines=$(gzip -dc < "$LOG" \
+                          | tail -n10 \
+                          | cut -b1-80 \
+                          | LC_ALL=C tr -dc '[:print:][:space:]' \
+                          | sed 's/^/    /')
+##
+## Build email body and subject
+##
+msg_body="Hello,
+
+$msg_first_line
+
+Command line:
+    $*
+
+Start: $BEGDATE
+End:   $ENDDATE
+
+Host:
+    $hostname
+
+PWD:
+    $PWD
+
+Log:
+    $LOG_DISPLAY_NAME
+
+Last lines from log:
+$last_log_lines
+
+Log attached.
+
+"
+
+msg_subject="$name - $BEGDATE - $msg_status"
+
+if test -n "$send_email" ; then
+    echo "$msg_body" | mail -s "$msg_subject" -a "$LOG" "$email" \
+        || die_with_log "fatal error: failed to send email"
+fi
+
+exit 0