.travis.yml

language: java
dist: trusty
group: travis_lts
git:
  depth: 9999999
  lfs_skip_smudge: true
jdk:
- openjdk8
env:
  matrix:
  - SCALA_VERSION=2.11 TEST_TYPE=cloud UPLOAD=true TESTS_REQUIRE_GCLOUD=true
  - SCALA_VERSION=2.11 TEST_TYPE=integration TEST_DOCKER=true TEST_VERBOSITY=minimal
  - SCALA_VERSION=2.11 TEST_TYPE=unit TEST_DOCKER=true TEST_VERBOSITY=minimal
  - SCALA_VERSION=2.11 TEST_TYPE=variantcalling TEST_DOCKER=true TEST_VERBOSITY=minimal
  - SCALA_VERSION=2.11 TEST_TYPE=python TEST_DOCKER=true TEST_VERBOSITY=minimal
  - SCALA_VERSION=2.11 TEST_TYPE=R TEST_DOCKER=true TEST_VERBOSITY=minimal
  - SCALA_VERSION=2.11 RUN_CNV_GERMLINE_COHORT_WDL=true TESTS_REQUIRE_GCLOUD=true
  - SCALA_VERSION=2.11 RUN_CNV_GERMLINE_CASE_WDL=true TESTS_REQUIRE_GCLOUD=true
  - SCALA_VERSION=2.11 RUN_CNV_SOMATIC_WDL=true TESTS_REQUIRE_GCLOUD=true
  - SCALA_VERSION=2.11 RUN_M2_WDL=true TESTS_REQUIRE_GCLOUD=true
  - SCALA_VERSION=2.11 RUN_CNN_WDL=true TESTS_REQUIRE_GCLOUD=true
  global:
  #gradle needs this
  - TERM=dumb
  #limit gradle jvm memory and disable daemon
  - GRADLE_OPTS="-Xmx2048m -Dorg.gradle.daemon=false"
  #google cloud stuff
  - export CLOUDSDK_CORE_DISABLE_PROMPTS=1
  - export GCLOUD_HOME=$HOME/gcloud/google-cloud-sdk/bin
  - export CLOUDSDK_PYTHON_SITEPACKAGES=1
  - export GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/servicekey.json
  #hellbender specific variables
  - HELLBENDER_TEST_INPUTS=gs://hellbender/test/resources/
  - HELLBENDER_TEST_STAGING=gs://hellbender-test-logs/staging/
  - HELLBENDER_TEST_LOGS=/hellbender-test-logs/build_reports/
  - HELLBENDER_TEST_PROJECT=broad-dsde-dev
  - HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=servicekey.json
  #Work around for https://github.com/googleapis/google-cloud-java/issues/5884
  - export GOOGLE_CLOUD_PROJECT=$HELLBENDER_TEST_PROJECT
  #for uploading artifacts
  - ARTIFACTORY_USERNAME=gatkci
  #artifactory password
  - secure: "E0LWXgX3aWSE/DWHXXDx4vrAq4uX6vKg402wToaZ5otbHQ/UP0H7/FA5jomavAXoC46oMVHZcEltZ5OVhuJ0NW8yYxUCecJ1D/YvVQmnfFABcV/qLM+k4e2rYQOKVw/pejB2gG8XdTA+XE2WyTeENbmIkputS8f1ndKWCmZxuuk="
  # For cromwell jar download
  - CROMWELL_JAR=$HOME/cromwell-36.jar
matrix:
  fast_finish: true
  include:
     - jdk: oraclejdk8
       env: SCALA_VERSION=2.11 TEST_TYPE=integration TEST_VERBOSITY=minimal TEST_REQUIRE_GCLOUD=true
     - jdk: openjdk11
       env: SCALA_VERSION=2.12 TEST_TYPE=integration TEST_VERBOSITY=minimal
     - jdk: openjdk11
       env: SCALA_VERSION=2.12 TEST_TYPE=unit TEST_VERBOSITY=minimal
     - jdk: openjdk11
       env: SCALA_VERSION=2.12 TEST_TYPE=cloud TESTS_REQUIRE_GCLOUD=true TEST_VERBOSITY=minimal
before_cache:
  - rm -f  $HOME/.gradle/caches/modules-2/modules-2.lock
  - rm -fr $HOME/.gradle/caches/*/plugin-resolution/
cache:
  directories:
    - $HOME/.gradle/caches/
    - $HOME/.gradle/wrapper/
    - $HOME/.m2
    - $HOME/gcloud/
    - $HOME/site-library
before_install:
  #skip push builds if there's an associated PR because we don't look at them anway
- if [[ ${TRAVIS_EVENT_TYPE} == push ]];
  then
    PULL_REQUESTS=$( scripts/travis/check_for_pull_request );
    if [[ $( grep -c "commits" <<< ${PULL_REQUESTS} ) -gt 0 ]];
    then
      travis_terminate 0;
    fi;
  fi;
- REPORT_PATH=${TRAVIS_BRANCH}_${TRAVIS_JOB_NUMBER};
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
    echo "Test report will be written to  https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
  fi;
- if [[ ${TRAVIS_SECURE_ENV_VARS} == true ]]; then
    if [[ ${TESTS_REQUIRE_GCLOUD} == true ]]; then
      scripts/travis/install_and_authenticate_to_gcloud.sh;
    fi;
    openssl aes-256-cbc -K $encrypted_4823d58debd3_key -iv $encrypted_4823d58debd3_iv -in resources_for_CI/github_deploy_key.enc -out ~/.ssh/id_rsa -d;
    chmod 600 ~/.ssh/id_rsa;
    echo "Testing github authentication:";
    ssh -T git@github.com;
    echo "Done testing github authentication";
  fi;
# Download Cromwell jar -- if you change the version, please change the CROMWELL_JAR env variable above, too.
- if [[ $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
    wget -O $CROMWELL_JAR https://github.com/broadinstitute/cromwell/releases/download/36/cromwell-36.jar;
  fi;
# Download git lfs files
- git lfs
- git lfs install
- git lfs fetch
- git lfs checkout
# Disable services enabled by default to free a bit of memory
# http://docs.travis-ci.com/user/database-setup/#MySQL
- sudo /etc/init.d/mysql stop
- sudo /etc/init.d/postgresql stop
install:
- if [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
    echo "Can't run cloud tests without keys so don't bother building";
  elif [[ $TEST_DOCKER == true ]]; then
    echo "Skip the install because we're doing a docker build";
  else
    ./gradlew assemble;
    ./gradlew installDist;
    if [[ $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
      echo "building a shadow jar for the wdl";
      ./gradlew shadowJar;
    elif [[ $TEST_TYPE == cloud ]]; then
      echo "building a spark jar for the dataproc tests";
      ./gradlew sparkjar;
    fi;
  fi;
script:
# run a basic sanity check to be sure that gatk doesn't explode
# run tests
- if [[ $TRAVIS_EVENT_TYPE == cron ]]; then
    echo "Not running any tests for nightly builds";
  elif [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
    echo "Can't run cloud tests without keys so don't run tests";
  elif [[ $RUN_CNV_GERMLINE_COHORT_WDL == true ]]; then
    echo "Running CNV germline cohort workflow";
    travis_wait 60 bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh COHORT;
  elif [[ $RUN_CNV_GERMLINE_CASE_WDL == true ]]; then
    echo "Running CNV germline case workflow";
    travis_wait 60 bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh CASE;
  elif [[ $RUN_CNV_SOMATIC_WDL == true ]]; then
    echo "Running CNV somatic workflows";
    bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;
  elif [[ $RUN_M2_WDL == true ]]; then
    echo "Deleting some unused files before running M2 WDL...";
    rm -Rf src/test/resources/large/VQSR;
    echo "Running M2 WDL";
    sudo bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
  elif [[ $RUN_CNN_WDL == true ]]; then
    echo "Running CNN WDL";
    sudo bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;
  elif [[ $TEST_DOCKER == true ]]; then
    echo "Building docker image and running appropriate tests..." ;
    if [ ${TRAVIS_PULL_REQUEST} != false ]; then
      sudo bash build_docker.sh  -e FETCH_HEAD -s -u -t ${TRAVIS_PULL_REQUEST};
      DOCKER_TAG=FETCH_HEAD;
    else
      echo ${TRAVIS_COMMIT};
      sudo bash build_docker.sh  -e ${TRAVIS_COMMIT} -s -u;
      DOCKER_TAG=$TRAVIS_COMMIT;
    fi;
    sudo docker images;
    echo ${TEST_TYPE};
    sudo mkdir -p build/reports/;
    sudo chmod -R a+w build/reports/;
    cp scripts/docker/dockertest.gradle .;
    sudo docker run -v $(pwd):/gatkCloneMountPoint:cached -v $(pwd)/testJars:/jars:cached --rm -e "scala.version=${SCALA_VERSION}" -e "TEST_VERBOSITY=minimal" -e "TEST_TYPE=${TEST_TYPE}" -t broadinstitute/gatk:${DOCKER_TAG} bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh;
    TEST_EXIT_VALUE=$?;
    sudo mkdir build/reports/tests/test && sudo cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test && sudo rm -r build/reports/tests/testOnPackagedReleaseJar;
    $( exit ${TEST_EXIT_VALUE} );
  else
    ./gatk PrintReads -I src/test/resources/NA12878.chr17_69k_70k.dictFix.bam -O output.bam;
    travis_wait 50 ./gradlew -Dscala.version=${SCALA_VERSION} jacocoTestReport;
  fi;
# This creates and uploads the gatk zip file to the nightly build bucket, only keeping the 10 newest entries
# This also constructs the Docker image and uploads it to https://cloud.docker.com/u/broadinstitute/repository/docker/broadinstitute/gatk-nightly/
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_EVENT_TYPE == cron && $UPLOAD == true ]]; then
  docker login --username=$DOCKER_SERVICE_LOGIN --password=$DOCKER_SERVICE_PASS;
  $GCLOUD_HOME/gcloud components -q update gsutil;
  gsutil ls -l gs://gatk-nightly-builds | grep gatk | sort -r -k 2 | grep -o '\S\+$' | tail -n +11 | xargs -I {} gsutil rm {};
  ./gradlew bundle;
  ZIP_FILE="$(ls build/ | grep '^gatk.*.zip$' | grep -iv python)";
  echo "Uploading zip to gs://gatk-nightly-builds/";
  $GCLOUD_HOME/gsutil -m cp build/$ZIP_FILE gs://gatk-nightly-builds/"$(date +%Y-%m-%d)"-$ZIP_FILE;

  bash build_docker.sh  -e ${TRAVIS_COMMIT} -s -u;
  DOCKER_TAG=$(docker images -q | head -n 1);
  DATE=$(date +%Y-%m-%d);
  DOCKER_NIGHTLY_TAG=broadinstitute/gatk-nightly:$DATE-$(git describe)-NIGHTLY-SNAPSHOT;

  echo "Pushing Nightly Docker image";
  docker tag $DOCKER_TAG $DOCKER_NIGHTLY_TAG;
  docker push $DOCKER_NIGHTLY_TAG;
  fi;

after_success:
# - bash <(curl -s https://codecov.io/bash)
- if [[ $TRAVIS_BRANCH == master && $UPLOAD == true ]]; then ./gradlew uploadArchives; fi;
# if this is actually a commit to master and not a pull request build into master, then publish master-snapshot
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_PULL_REQUEST == false && $UPLOAD == true ]]; then
    git tag master;
    ./gradlew uploadArchives; 
  fi;
after_failure:
- dmesg | tail -100
- export FAILED=true
after_script:
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
    if [[ ${TESTS_REQUIRE_GCLOUD} != "true" ]]; then
      scripts/travis/install_and_authenticate_to_gcloud.sh;
    fi;
    $GCLOUD_HOME/gsutil -m cp -z html -z js -z xml -z css -r build/reports/tests gs:/$HELLBENDER_TEST_LOGS$REPORT_PATH/;
    echo "See the test report at https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
    if [[ ${FAILED} == true && ${TRAVIS_PULL_REQUEST} != false && -d build/reports/tests ]]; then
      pip install --user PyGithub;
      python scripts/travis/Reporter.py https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html;
    fi
  fi;