[Issue 47] End to end integration tests (WIP)

.travis.yml

@@ -18,8 +18,8 @@ env:
   global:
     - GRADLE_OPTS="-Xms128m"
     - ORG_GRADLE_PROJECT_logOutput=true
-    - secure: "EPL076YzDvzRIThkh5pbpgd/59vONUsncWg7uESuY/7P7ZVnksUFoE20/vQNo40OTgukkjgnTymb0uJ9jWFpQOnNw+LzdvhC4BtPSAg4SlEuIsOhj6qPdvYe8XhGZj0Z0yGcP7dwu18pgaNSTJ1iVRjPXlysgYOwl0JF/XcL7yzi486JTmmaFlsKUz9sikO00JUwml32DmyUCMsdgtfzKx4CvSepsFjgDlACErul+/kKButS74a9SzaBfLwxI1TFjni9JEMZGcK9pNP6Mi6//1kZcG2W0GWkmAqPo73VvBSaSR/zso/5mJNSsBv7f4eOwIh/Nkv74g4xmVds6wHxCQC77SedGpvVcXnmGVdXHL17sCOenuc1R70K2Qq00hMaDz/2kG497Estd96kpVUav680p8N8WPjEbHqfDY3k+/8MmBce8NWkPGUy4XdF4eJxLUJ8GEXRzR2v9JDY9A2gcNmSC6rwf0VedbopQ/fE5k8CKgvt9eZQM2nJE5ZzObakVssvYNNIexK0NprH0hQMLt68sLBAN7H8M57wbZSUEZD3e52SesK8h0xZIYty1leaUx/G/yo1wxOGNC4fGqyjPYNf04zDv/gnK11eeJBOu+t9T07PVn7XWBg0CdoX4XY+1Rlp8yWvQINXWwYSMhUu+xU1KyCp67qTD7MadZ/OjLE="
-    - secure: "CuLbIV4oqk9W1mNdy9Dz/YmOW/638ufJf+B83lxCky6l+Us6p5sGAyv+GZUWtYKpExFah3+lC6Fp8DOlQOsSpNz50bSEFzOWRKDyLui2btS06wXjPhp5M+dBZgPolTrrxb6BM7gGcA0sPkpFbDufblF7E/wmMiBUePsCnHiP3mGlR+Zxk/i44R3Ooy4ScoPGH8kB6i5S0YEF26WV+qDOv3/qRuMAQpCqWhovIvdNps/aZwvYYYvYLrx3sj/+menay/D23i1W2r4w4hdF2oLekkT+qY22dajTKkE+RpoIKDC1YYIz22jnSUN68DRQipsXD/S3IfK2HacvveopdDytHHy+RfYGvuToMnj7rwXf3ycSTLKYK0/kN4HVsNl4Nq1b9CuxZP5YETU51o0bX1BRw8H5cmGhrzZfmcouYEc7KMl51S+3ksP3fuM5ZH2iej3LVVk5sT19sDwo/xcbXZg7Pv2sCMJdnf9YSfhpXIiqC3ZNIA5sWJLQUJtccUCvlu76/6K8y+34Gz4YJXGYZHNQl+0g+b17dnhgJEG2hDV0cFlu7TJqVwFRijbRpRMitQng5+/cBYCiKPYxfg/iaqFxNfJoJXPe2QaYxkt72FzybtCQvNjlFAzf/KRs41jU9o+ZFBaD4X4jmdPRUouwtGP6k7QxnRcnrhiUHncbewcnhE8="
+#    - secure: "EPL076YzDvzRIThkh5pbpgd/59vONUsncWg7uESuY/7P7ZVnksUFoE20/vQNo40OTgukkjgnTymb0uJ9jWFpQOnNw+LzdvhC4BtPSAg4SlEuIsOhj6qPdvYe8XhGZj0Z0yGcP7dwu18pgaNSTJ1iVRjPXlysgYOwl0JF/XcL7yzi486JTmmaFlsKUz9sikO00JUwml32DmyUCMsdgtfzKx4CvSepsFjgDlACErul+/kKButS74a9SzaBfLwxI1TFjni9JEMZGcK9pNP6Mi6//1kZcG2W0GWkmAqPo73VvBSaSR/zso/5mJNSsBv7f4eOwIh/Nkv74g4xmVds6wHxCQC77SedGpvVcXnmGVdXHL17sCOenuc1R70K2Qq00hMaDz/2kG497Estd96kpVUav680p8N8WPjEbHqfDY3k+/8MmBce8NWkPGUy4XdF4eJxLUJ8GEXRzR2v9JDY9A2gcNmSC6rwf0VedbopQ/fE5k8CKgvt9eZQM2nJE5ZzObakVssvYNNIexK0NprH0hQMLt68sLBAN7H8M57wbZSUEZD3e52SesK8h0xZIYty1leaUx/G/yo1wxOGNC4fGqyjPYNf04zDv/gnK11eeJBOu+t9T07PVn7XWBg0CdoX4XY+1Rlp8yWvQINXWwYSMhUu+xU1KyCp67qTD7MadZ/OjLE="
+#    - secure: "CuLbIV4oqk9W1mNdy9Dz/YmOW/638ufJf+B83lxCky6l+Us6p5sGAyv+GZUWtYKpExFah3+lC6Fp8DOlQOsSpNz50bSEFzOWRKDyLui2btS06wXjPhp5M+dBZgPolTrrxb6BM7gGcA0sPkpFbDufblF7E/wmMiBUePsCnHiP3mGlR+Zxk/i44R3Ooy4ScoPGH8kB6i5S0YEF26WV+qDOv3/qRuMAQpCqWhovIvdNps/aZwvYYYvYLrx3sj/+menay/D23i1W2r4w4hdF2oLekkT+qY22dajTKkE+RpoIKDC1YYIz22jnSUN68DRQipsXD/S3IfK2HacvveopdDytHHy+RfYGvuToMnj7rwXf3ycSTLKYK0/kN4HVsNl4Nq1b9CuxZP5YETU51o0bX1BRw8H5cmGhrzZfmcouYEc7KMl51S+3ksP3fuM5ZH2iej3LVVk5sT19sDwo/xcbXZg7Pv2sCMJdnf9YSfhpXIiqC3ZNIA5sWJLQUJtccUCvlu76/6K8y+34Gz4YJXGYZHNQl+0g+b17dnhgJEG2hDV0cFlu7TJqVwFRijbRpRMitQng5+/cBYCiKPYxfg/iaqFxNfJoJXPe2QaYxkt72FzybtCQvNjlFAzf/KRs41jU9o+ZFBaD4X4jmdPRUouwtGP6k7QxnRcnrhiUHncbewcnhE8="
 
 sudo: required
 
@@ -34,23 +34,28 @@ cache:
 
 stages:
   - name: build
-  - name: snapshot
-    if: branch = master AND NOT (type = pull_request)
+#  - name: snapshot
+#    if: branch = master AND NOT (type = pull_request)
 
 jobs:
   include:
     - stage: build
-      script: ./gradlew clean assemble
-    - stage: snapshot
-      script: ./gradlew publishToRepo -PpublishUrl=jcenterSnapshot -PpublishUsername=$BINTRAY_USER -PpublishPassword=$BINTRAY_KEY
+      script:
+      - ./gradlew clean assemble
+      - ./gradlew install
+      - ./src/test/integration/run-integration-test.sh
+
+#    - stage: snapshot
+#      script: ./gradlew publishToRepo -PpublishUrl=jcenterSnapshot -PpublishUsername=$BINTRAY_USER -PpublishPassword=$BINTRAY_KEY
 
 notifications:
-  slack:
-    matrix:
-      secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM=
-    rooms:
-      secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM=
+#  slack:
+#    matrix:
+#      secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM=
+#    rooms:
+#      secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM=
   email:
-    - [email protected]
-    - [email protected]
-    - [email protected]
+#    - [email protected]
+#    - [email protected]
+#    - [email protected]
+    - [email protected]

src/test/integration/run-integration-test.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+#
+# Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+
+# Download and install flink
+set -veux
+FLINK_VERSION=${FLINK_VERSION:-1.4.2}
+PRAVEGA_VERSION_PREFIX=${PRAVEGA_VERSION_PREFIX:-0.3.0}
+FLINK_PORTAL_PORT=${FLINK_PORTAL_PORT:-8081}
+PRAVEGA_REST_PORT=${PRAVEGA_REST_PORT:-9091}
+PRAVEGA_CONTROLLER_PORT=${PRAVEGA_CONTROLLER_PORT:-9090}
+SCALA_VERSION=${SCALA_VERSION:-2.11}
+WAIT_RETRIES=${WAIT_RETRIES:-48}
+WAIT_SLEEP=${WAIT_SLEEP:-5}
+HTTP_OK=200
+
+WORK_DIR=$PWD
+FLINK_DIR=$HOME/flink-${FLINK_VERSION}
+FLINK_BINARY=flink-${FLINK_VERSION}-bin-hadoop28-scala_${SCALA_VERSION}.tgz
+
+trap cleanup EXIT
+trap "exit 1" SIGTERM SIGHUP SIGINT SIGQUIT
+cleanup() {
+    # clean up flink connector artifacts as $HOME/.m2 may be cacached.
+    #ls -l $HOME/.m2/repository/io/pravega/pravega-connectors-flink_2.11/
+    rm -rf $HOME/.m2/repository/io/pravega/pravega-connectors-flink_2.11/*
+
+    # clean up flink logs
+    #ls -l $FLINK_DIR/log/*
+    rm -f $FLINK_DIR/log/*
+}
+
+wait_for_service() {
+    url=$1
+    count=0
+    until [ "$(curl -s -o /dev/null -w ''%{http_code}'' $url)" = "$HTTP_OK" ]; do
+        if [ $count -ge ${WAIT_RETRIES} ]; then
+            exit 1;
+        fi
+        count=$(($count+1))
+        sleep ${WAIT_SLEEP}
+    done
+}
+
+# clean up flink log directory that hosts job execution results
+rm -f $FLINK_DIR/log/*
+
+# Download flink
+cd $HOME
+wget --no-check-certificate https://archive.apache.org/dist/flink/flink-${FLINK_VERSION}/${FLINK_BINARY}
+tar zxvf $FLINK_BINARY
+rm -f ${FLINK_BINARY}*
+
+# Increase job slots, then start flink cluster
+sed -i '/taskmanager.numberOfTaskSlots/c\taskmanager.numberOfTaskSlots: 5' ${FLINK_DIR}/conf/flink-conf.yaml
+${FLINK_DIR}/bin/start-cluster.sh 
+
+# wait for Flink cluster to start
+wait_for_service http://localhost:${FLINK_PORTAL_PORT}
+
+# Download and install Pravega
+# For the time being, use the Pravega submodule in Flink connector.
+# Eventually use a stable Pravega build that is compatible with Flink connector
+cd ${WORK_DIR}/pravega
+#./gradlew startstandalone 2>&1 | tee /tmp/pravega.log &
+./gradlew startstandalone > /dev/null 2>&1 &
+
+# wait for Pravega to start
+wait_for_service http://localhost:${PRAVEGA_REST_PORT}/v1/scopes
+
+cd ${WORK_DIR}
+# flink connector artifact version
+commit_id=$(git log --format="%h" -n 1)
+count=$(git rev-list --count HEAD)
+version=${PRAVEGA_VERSION_PREFIX}-${count}.${commit_id}-SNAPSHOT
+
+
+# Compile and run sample Flink application
+cd ${WORK_DIR}
+git clone https://github.com//pravega/pravega-samples
+cd ${WORK_DIR}/pravega-samples
+git checkout develop
+sed -i '/connectorVersion/c\connectorVersion='${version}'' gradle.properties
+./gradlew :flink-examples:installDist
+
+# start ExactlyOnceWriter
+${FLINK_DIR}/bin/flink run -c io.pravega.examples.flink.primer.process.ExactlyOnceWriter flink-examples/build/install/pravega-flink-examples/lib/pravega-flink-examples-0.3.0-SNAPSHOT-all.jar --controller tcp://localhost:${PRAVEGA_CONTROLLER_PORT} --scope myscope --stream mystream --exactlyonce true
+
+# start ExactlyOnceChecker
+${FLINK_DIR}/bin/flink run -d -c io.pravega.examples.flink.primer.process.ExactlyOnceChecker flink-examples/build/install/pravega-flink-examples/lib/pravega-flink-examples-0.3.0-SNAPSHOT-all.jar --controller tcp://localhost:${PRAVEGA_CONTROLLER_PORT} --scope myscope --stream mystream 
+
+${FLINK_DIR}/bin/flink list
+job_id=`${FLINK_DIR}/bin/flink list | grep ExactlyOnceChecker | awk '{print $4}'`
+count=0
+ls -l ${FLINK_DIR}/log
+until grep -q "EXACTLY_ONCE" ${FLINK_DIR}/log/*.out; do
+    if [ $count -ge 24 ]; then
+        ${FLINK_DIR}/bin/flink cancel $job_id
+        exit 1
+    fi
+    count=$(($count+1))
+    sleep ${WAIT_SLEEP}
+done
+
+# successful. stop the checker job.
+${FLINK_DIR}/bin/flink stop $job_id