runtest

#!/bin/sh

# Load shared functions, make available to CONFIG_OVERRIDE_SCRIPT_URL.
DOWNLOADREPOS_AS_LIB=1
. ./downloadrepos

# CONFIG_PARAMETERS_SCRIPT_URL can override DEFAULT_PRODUCTION_HOST.
DEFAULT_PRODUCTION_HOST="${DEFAULT_PRODUCTION_HOST:=pavics.ouranos.ca}"

NOTEBOOKS="$1"
if [ -z "$NOTEBOOKS" ]; then
    NOTEBOOKS="notebooks/*.ipynb"
fi

set -x

if [ -n "$FORCE_PAVICS_HOST" ]; then
    # jenkins work-around for envinject incompatibility with pipeline
    echo "FORCE_PAVICS_HOST='$FORCE_PAVICS_HOST'"
    echo "Overriding PAVICS_HOST with value from FORCE_PAVICS_HOST"
    PAVICS_HOST="$FORCE_PAVICS_HOST"
fi

if [ ! -z "$PAVICS_HOST" ]; then
    echo "Will run notebooks against $PAVICS_HOST"

    if [ -z "$TEST_NO_USE_PROD_DATA" ]; then
        # * .ncml links will always comes from the production server, not the server
        # under test since we do not perform regex replace for .ncml links.
        #
        # * Any lines marked with TEST_USE_PROD_DATA will also use data from prod
        # and avoid having to replicate data to test servers.
        sed -i "/\(\.ncml\|TEST_USE_PROD_DATA\)/!s/$DEFAULT_PRODUCTION_HOST/$PAVICS_HOST/g" $NOTEBOOKS
    else
        # Change everything to $PAVICS_HOST (system under test).
        sed -i "s/$DEFAULT_PRODUCTION_HOST/$PAVICS_HOST/g" $NOTEBOOKS
    fi

    if [ x"${RUN_TEST_SKIP_DIFF}" != x"true" ]; then
      git diff  # not working for notebooks from other repos
    fi
fi

# suppress "InsecureRequestWarning: Unverified HTTPS request is being made"
# so tests still pass when user want to disable ssl cert verification
export PYTHONWARNINGS="ignore:Unverified HTTPS request"

# Allow full override of ALL configs before running test suite.
if [ -n "$CONFIG_OVERRIDE_SCRIPT_URL" ]; then
    TMP_CONF_OVERRIDE="/tmp/conf_override"
    rm -vf "$TMP_CONF_OVERRIDE"


    if echo "$CONFIG_OVERRIDE_SCRIPT_URL" | grep -e ^http ; then
        # Use tee to log content of override script for traceability.
        curl --silent "$CONFIG_OVERRIDE_SCRIPT_URL" | tee "$TMP_CONF_OVERRIDE"
    else
        # Not starting with http, it's a local file.
        # Local file can be previously created by CONFIG_PARAMETERS_SCRIPT_URL.
        # See example in test-override/jenkins-params-raven-specific-nb.include.sh.
        TMP_CONF_OVERRIDE="$CONFIG_OVERRIDE_SCRIPT_URL"

        # Log content of override script for traceability.
        cat "$TMP_CONF_OVERRIDE"
    fi

    # Source script so it can alter ALL existing variables in the current context.
    if [ -f "$TMP_CONF_OVERRIDE" ]; then
        . "$TMP_CONF_OVERRIDE"
    fi
fi

# CONFIG_OVERRIDE_SCRIPT_URL can override NBVAL_SANITIZE_CFG_FILE.
py.test --rootdir=. --nbval $NOTEBOOKS --nbval-sanitize-with "${NBVAL_SANITIZE_CFG_FILE:=notebooks/output-sanitize.cfg}" $PYTEST_EXTRA_OPTS
EXIT_CODE="$?"

# lowercase SAVE_RESULTING_NOTEBOOK string
SAVE_RESULTING_NOTEBOOK="$(lowercase "$SAVE_RESULTING_NOTEBOOK")"


# save notebooks resulting from the run
# this might not be the same as what py.test have seen since it's another run

# user can manually diff the original with the resulting notebooks this way:
# nbdiff original.ipynb resulting.ipynb.output.ipynb (conda install nbdime)

# work-around as nbval can not save the result of the run
# see https://github.com/computationalmodelling/nbval/issues/112

BUILDOUT_DIR="buildout"  # hardcode in Jenkinsfile, can not be override.

mkdir -p "$BUILDOUT_DIR/"
for nb in $NOTEBOOKS; do
    filename="$(choose_artifact_filename "$nb")"
    filename="$(echo "$filename" | sed "s/.ipynb$//")"  # remove .ipynb ext
    if [ -e "${BUILDOUT_DIR}/${filename}.ipynb" ]; then
        # prevent name clash
        filename="${filename}_$(date '+%s')"
    fi

    # Save original notebooks that we sed replace the PAVICS_HOST.
    dir_filename="$(dirname "${filename}")"
    if [ -n "${dir_filename}" ]; then
        mkdir -p "${BUILDOUT_DIR}/${dir_filename}"
    fi
    cp "$nb" "$BUILDOUT_DIR/${filename}.ipynb"

    if [ x"$SAVE_RESULTING_NOTEBOOK" = xtrue ]; then
        # Timeout must not be more than 240s (4 mins). Default in Jenkinsfile.
        # Tutorial notebooks should be fast so user do not lose patience waiting
        # for them to run.  If more than 4 mins, in addition to simplifying the
        # notebook, should also check machine performance.
        jupyter nbconvert --to notebook --execute \
            --ExecutePreprocessor.timeout=${SAVE_RESULTING_NOTEBOOK_TIMEOUT:=240} --allow-errors \
            --output-dir "${BUILDOUT_DIR}" --output "${filename}.output.ipynb" "$nb"
    fi
done


# Post-processing steps override in CONFIG_OVERRIDE_SCRIPT_URL.
post_runtest


# exit with return code from py.test
exit $EXIT_CODE