From 9a9e3a906ba85fc89c4366be8853cc9d0f6713bc Mon Sep 17 00:00:00 2001 From: Lida He Date: Thu, 31 May 2018 18:26:23 -0400 Subject: [PATCH 1/2] first cut of end to end integration test Signed-off-by: Lida He --- .travis.yml | 35 +++--- src/test/integration/run-integration-test.sh | 116 +++++++++++++++++++ 2 files changed, 136 insertions(+), 15 deletions(-) create mode 100755 src/test/integration/run-integration-test.sh diff --git a/.travis.yml b/.travis.yml index 8563a394..a6d42e7a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,8 +18,8 @@ env: global: - GRADLE_OPTS="-Xms128m" - ORG_GRADLE_PROJECT_logOutput=true - - secure: "EPL076YzDvzRIThkh5pbpgd/59vONUsncWg7uESuY/7P7ZVnksUFoE20/vQNo40OTgukkjgnTymb0uJ9jWFpQOnNw+LzdvhC4BtPSAg4SlEuIsOhj6qPdvYe8XhGZj0Z0yGcP7dwu18pgaNSTJ1iVRjPXlysgYOwl0JF/XcL7yzi486JTmmaFlsKUz9sikO00JUwml32DmyUCMsdgtfzKx4CvSepsFjgDlACErul+/kKButS74a9SzaBfLwxI1TFjni9JEMZGcK9pNP6Mi6//1kZcG2W0GWkmAqPo73VvBSaSR/zso/5mJNSsBv7f4eOwIh/Nkv74g4xmVds6wHxCQC77SedGpvVcXnmGVdXHL17sCOenuc1R70K2Qq00hMaDz/2kG497Estd96kpVUav680p8N8WPjEbHqfDY3k+/8MmBce8NWkPGUy4XdF4eJxLUJ8GEXRzR2v9JDY9A2gcNmSC6rwf0VedbopQ/fE5k8CKgvt9eZQM2nJE5ZzObakVssvYNNIexK0NprH0hQMLt68sLBAN7H8M57wbZSUEZD3e52SesK8h0xZIYty1leaUx/G/yo1wxOGNC4fGqyjPYNf04zDv/gnK11eeJBOu+t9T07PVn7XWBg0CdoX4XY+1Rlp8yWvQINXWwYSMhUu+xU1KyCp67qTD7MadZ/OjLE=" - - secure: "CuLbIV4oqk9W1mNdy9Dz/YmOW/638ufJf+B83lxCky6l+Us6p5sGAyv+GZUWtYKpExFah3+lC6Fp8DOlQOsSpNz50bSEFzOWRKDyLui2btS06wXjPhp5M+dBZgPolTrrxb6BM7gGcA0sPkpFbDufblF7E/wmMiBUePsCnHiP3mGlR+Zxk/i44R3Ooy4ScoPGH8kB6i5S0YEF26WV+qDOv3/qRuMAQpCqWhovIvdNps/aZwvYYYvYLrx3sj/+menay/D23i1W2r4w4hdF2oLekkT+qY22dajTKkE+RpoIKDC1YYIz22jnSUN68DRQipsXD/S3IfK2HacvveopdDytHHy+RfYGvuToMnj7rwXf3ycSTLKYK0/kN4HVsNl4Nq1b9CuxZP5YETU51o0bX1BRw8H5cmGhrzZfmcouYEc7KMl51S+3ksP3fuM5ZH2iej3LVVk5sT19sDwo/xcbXZg7Pv2sCMJdnf9YSfhpXIiqC3ZNIA5sWJLQUJtccUCvlu76/6K8y+34Gz4YJXGYZHNQl+0g+b17dnhgJEG2hDV0cFlu7TJqVwFRijbRpRMitQng5+/cBYCiKPYxfg/iaqFxNfJoJXPe2QaYxkt72FzybtCQvNjlFAzf/KRs41jU9o+ZFBaD4X4jmdPRUouwtGP6k7QxnRcnrhiUHncbewcnhE8=" +# - secure: "EPL076YzDvzRIThkh5pbpgd/59vONUsncWg7uESuY/7P7ZVnksUFoE20/vQNo40OTgukkjgnTymb0uJ9jWFpQOnNw+LzdvhC4BtPSAg4SlEuIsOhj6qPdvYe8XhGZj0Z0yGcP7dwu18pgaNSTJ1iVRjPXlysgYOwl0JF/XcL7yzi486JTmmaFlsKUz9sikO00JUwml32DmyUCMsdgtfzKx4CvSepsFjgDlACErul+/kKButS74a9SzaBfLwxI1TFjni9JEMZGcK9pNP6Mi6//1kZcG2W0GWkmAqPo73VvBSaSR/zso/5mJNSsBv7f4eOwIh/Nkv74g4xmVds6wHxCQC77SedGpvVcXnmGVdXHL17sCOenuc1R70K2Qq00hMaDz/2kG497Estd96kpVUav680p8N8WPjEbHqfDY3k+/8MmBce8NWkPGUy4XdF4eJxLUJ8GEXRzR2v9JDY9A2gcNmSC6rwf0VedbopQ/fE5k8CKgvt9eZQM2nJE5ZzObakVssvYNNIexK0NprH0hQMLt68sLBAN7H8M57wbZSUEZD3e52SesK8h0xZIYty1leaUx/G/yo1wxOGNC4fGqyjPYNf04zDv/gnK11eeJBOu+t9T07PVn7XWBg0CdoX4XY+1Rlp8yWvQINXWwYSMhUu+xU1KyCp67qTD7MadZ/OjLE=" +# - secure: "CuLbIV4oqk9W1mNdy9Dz/YmOW/638ufJf+B83lxCky6l+Us6p5sGAyv+GZUWtYKpExFah3+lC6Fp8DOlQOsSpNz50bSEFzOWRKDyLui2btS06wXjPhp5M+dBZgPolTrrxb6BM7gGcA0sPkpFbDufblF7E/wmMiBUePsCnHiP3mGlR+Zxk/i44R3Ooy4ScoPGH8kB6i5S0YEF26WV+qDOv3/qRuMAQpCqWhovIvdNps/aZwvYYYvYLrx3sj/+menay/D23i1W2r4w4hdF2oLekkT+qY22dajTKkE+RpoIKDC1YYIz22jnSUN68DRQipsXD/S3IfK2HacvveopdDytHHy+RfYGvuToMnj7rwXf3ycSTLKYK0/kN4HVsNl4Nq1b9CuxZP5YETU51o0bX1BRw8H5cmGhrzZfmcouYEc7KMl51S+3ksP3fuM5ZH2iej3LVVk5sT19sDwo/xcbXZg7Pv2sCMJdnf9YSfhpXIiqC3ZNIA5sWJLQUJtccUCvlu76/6K8y+34Gz4YJXGYZHNQl+0g+b17dnhgJEG2hDV0cFlu7TJqVwFRijbRpRMitQng5+/cBYCiKPYxfg/iaqFxNfJoJXPe2QaYxkt72FzybtCQvNjlFAzf/KRs41jU9o+ZFBaD4X4jmdPRUouwtGP6k7QxnRcnrhiUHncbewcnhE8=" sudo: required @@ -34,23 +34,28 @@ cache: stages: - name: build - - name: snapshot - if: branch = master AND NOT (type = pull_request) +# - name: snapshot +# if: branch = master AND NOT (type = pull_request) jobs: include: - stage: build - script: ./gradlew clean assemble - - stage: snapshot - script: ./gradlew publishToRepo -PpublishUrl=jcenterSnapshot -PpublishUsername=$BINTRAY_USER -PpublishPassword=$BINTRAY_KEY + script: + - ./gradlew clean assemble + - ./gradlew install + - ./src/test/integration/run-integration-test.sh + +# - stage: snapshot +# script: ./gradlew publishToRepo -PpublishUrl=jcenterSnapshot -PpublishUsername=$BINTRAY_USER -PpublishPassword=$BINTRAY_KEY notifications: - slack: - matrix: - secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM= - rooms: - secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM= +# slack: +# matrix: +# secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM= +# rooms: +# secure: Gv0RJx1Sa/y5fmvLNwY+2ivfWZYCM0ekrr6UAHqsegnid6P/DFZrSrfSpwvcVh2OVNH8DHLV0BoiuDJ7amtl1eMDMXz5/lLz8tFWFKaHv4yDSadm8ILY/KnYUoP4IRuM3NyKQmBrmZB9Or5KFXboG6ex6UkgbuYy0Zyl6syEe168Iw8hlCRx26Jei7/y+8eE2MIGFh09TLRZ/944YbULum9H3KQLYv8nFdPc7GmR5AK461fnwZ7iYjb7MXkCctE5Vml3p9+2Qliv1ZJqNsQeKmSFW6IhiP6pNZ1V8VJEWMQmX/nBr9745l/N+CoLQz9ajLonlxn9xHdWms4TEu1ynFk6uxEJjlcpXcvcEaKhqAKcTMl0GMMRab2m+/Vt3S/VutJnVXQmnhZGT9glLFQHwcdHNqM/LEbXtyisB7zmGImUQpF2InCwO25IXug5gv64IfOHGMzL56yNIhbRgBY9Ud4Tux+pmkV5ZxJiBkul7/FiHQX7tQLUrzQosD0oyCOmaWD7kmbt15A0TOkLgup4HE+sSS1ASwisa7J2+HsbI3Upy3rNVKuIJP0L4KSTn4HSlDlMLLcWM+nz/YCEfuwSRXJTIstotNYHdsLUZAZSYAX7ejpeiuBRed4a4AlCROeKbKKwCcSvqCOjmCaPTpwJAGeJByOXLL2hfQzpDMKCIKM= email: - - tom.kaitchuck@dell.com - - bhargav.gulavani@dell.com - - Flavio.Junqueira@emc.com +# - tom.kaitchuck@dell.com +# - bhargav.gulavani@dell.com +# - Flavio.Junqueira@emc.com + - lida.he@dell.com diff --git a/src/test/integration/run-integration-test.sh b/src/test/integration/run-integration-test.sh new file mode 100755 index 00000000..c2dca018 --- /dev/null +++ b/src/test/integration/run-integration-test.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# +# Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# + +# Download and install flink +set -veux +FLINK_VERSION=${FLINK_VERSION:-1.4.2} +PRAVEGA_VERSION_PREFIX=${PRAVEGA_VERSION_PREFIX:-0.3.0} +FLINK_PORTAL_PORT=${FLINK_PORTAL_PORT:-8081} +PRAVEGA_REST_PORT=${PRAVEGA_REST_PORT:-9091} +PRAVEGA_CONTROLLER_PORT=${PRAVEGA_CONTROLLER_PORT:-9090} +SCALA_VERSION=${SCALA_VERSION:-2.11} +WAIT_RETRIES=${WAIT_RETRIES:-48} +WAIT_SLEEP=${WAIT_SLEEP:-5} +HTTP_OK=200 + +WORK_DIR=$PWD +FLINK_DIR=$HOME/flink/flink-${FLINK_VERSION} +FLINK_BINARY=flink-${FLINK_VERSION}-bin-hadoop28-scala_${SCALA_VERSION}.tgz + +trap cleanup EXIT +trap "exit 1" SIGTERM SIGHUP SIGINT SIGQUIT +cleanup() { + # clean up flink connector artifacts as $HOME/.m2 may be cacached. + #ls -l $HOME/.m2/repository/io/pravega/pravega-connectors-flink_2.11/ + rm -rf $HOME/.m2/repository/io/pravega/pravega-connectors-flink_2.11/* + + # clean up flink logs + #ls -l $FLINK_DIR/log/* + rm -f $FLINK_DIR/log/* +} + +wait_for_service() { + url=$1 + count=0 + until [ "$(curl -s -o /dev/null -w ''%{http_code}'' $url)" = "$HTTP_OK" ]; do + if [ $count -ge ${WAIT_RETRIES} ]; then + exit 1; + fi + count=$(($count+1)) + sleep ${WAIT_SLEEP} + done +} + +# clean up flink log directory that hosts job execution results +rm -f $FLINK_DIR/log/* + +# Download flink +cd $HOME/flink +wget --no-check-certificate https://archive.apache.org/dist/flink/flink-${FLINK_VERSION}/${FLINK_BINARY} +tar zxvf $FLINK_BINARY +rm -f ${FLINK_BINARY}* + +# Increase job slots, then start flink cluster +sed -i '/taskmanager.numberOfTaskSlots/c\taskmanager.numberOfTaskSlots: 5' ${FLINK_DIR}/conf/flink-conf.yaml +${FLINK_DIR}/bin/start-cluster.sh + +# wait for Flink cluster to start +wait_for_service http://localhost:${FLINK_PORTAL_PORT} + +# Download and install Pravega +# For the time being, use the Pravega submodule in Flink connector. +# Eventually use a stable Pravega build that is compatible with Flink connector +cd ${WORK_DIR}/pravega +#./gradlew startstandalone 2>&1 | tee /tmp/pravega.log & +./gradlew startstandalone > /dev/null 2>&1 & + +# wait for Pravega to start +wait_for_service http://localhost:${PRAVEGA_REST_PORT}/v1/scopes + +cd ${WORK_DIR} +# flink connector artifact version +commit_id=$(git log --format="%h" -n 1) +count=$(git rev-list --count HEAD) +version=${PRAVEGA_VERSION_PREFIX}-${count}.${commit_id}-SNAPSHOT + + +# Compile and run sample Flink application +cd ${WORK_DIR} +git clone https://github.com//pravega/pravega-samples +cd ${WORK_DIR}/pravega-samples +git checkout develop +sed -i '/connectorVersion/c\connectorVersion='${version}'' gradle.properties +./gradlew :flink-examples:installDist + +# start ExactlyOnceWriter +${FLINK_DIR}/bin/flink run -c io.pravega.examples.flink.primer.process.ExactlyOnceWriter flink-examples/build/install/pravega-flink-examples/lib/pravega-flink-examples-0.3.0-SNAPSHOT-all.jar --controller tcp://localhost:${PRAVEGA_CONTROLLER_PORT} --scope myscope --stream mystream --exactlyonce true + +# start ExactlyOnceChecker +${FLINK_DIR}/bin/flink run -d -c io.pravega.examples.flink.primer.process.ExactlyOnceChecker flink-examples/build/install/pravega-flink-examples/lib/pravega-flink-examples-0.3.0-SNAPSHOT-all.jar --controller tcp://localhost:${PRAVEGA_CONTROLLER_PORT} --scope myscope --stream mystream + +# wait for 5 second to for job to register +sleep 2 + +${FLINK_DIR}/bin/flink list +job_id=`${FLINK_DIR}/bin/flink list | grep ExactlyOnceChecker | awk '{print $4}'` +count=0 +ls -l ${FLINK_DIR}/log +until grep -q "EXACTLY_ONCE" ${FLINK_DIR}/log/*.out; do + if [ $count -ge 24 ]; then + ${FLINK_DIR}/bin/flink cancel $job_id + exit 1 + fi + count=$(($count+1)) + sleep ${WAIT_SLEEP} +done + +# successful. stop the checker job. +${FLINK_DIR}/bin/flink stop $job_id From 75e2e4229239d54bdef819d899236fc703dd7562 Mon Sep 17 00:00:00 2001 From: Lida He Date: Thu, 31 May 2018 18:36:39 -0400 Subject: [PATCH 2/2] bug fix Signed-off-by: Lida He --- src/test/integration/run-integration-test.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/test/integration/run-integration-test.sh b/src/test/integration/run-integration-test.sh index c2dca018..a5b41b21 100755 --- a/src/test/integration/run-integration-test.sh +++ b/src/test/integration/run-integration-test.sh @@ -22,7 +22,7 @@ WAIT_SLEEP=${WAIT_SLEEP:-5} HTTP_OK=200 WORK_DIR=$PWD -FLINK_DIR=$HOME/flink/flink-${FLINK_VERSION} +FLINK_DIR=$HOME/flink-${FLINK_VERSION} FLINK_BINARY=flink-${FLINK_VERSION}-bin-hadoop28-scala_${SCALA_VERSION}.tgz trap cleanup EXIT @@ -53,7 +53,7 @@ wait_for_service() { rm -f $FLINK_DIR/log/* # Download flink -cd $HOME/flink +cd $HOME wget --no-check-certificate https://archive.apache.org/dist/flink/flink-${FLINK_VERSION}/${FLINK_BINARY} tar zxvf $FLINK_BINARY rm -f ${FLINK_BINARY}* @@ -96,9 +96,6 @@ ${FLINK_DIR}/bin/flink run -c io.pravega.examples.flink.primer.process.ExactlyOn # start ExactlyOnceChecker ${FLINK_DIR}/bin/flink run -d -c io.pravega.examples.flink.primer.process.ExactlyOnceChecker flink-examples/build/install/pravega-flink-examples/lib/pravega-flink-examples-0.3.0-SNAPSHOT-all.jar --controller tcp://localhost:${PRAVEGA_CONTROLLER_PORT} --scope myscope --stream mystream -# wait for 5 second to for job to register -sleep 2 - ${FLINK_DIR}/bin/flink list job_id=`${FLINK_DIR}/bin/flink list | grep ExactlyOnceChecker | awk '{print $4}'` count=0