From 37654aa9b0e123f36ac9efdba75e05d5f26b8042 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Tue, 26 Mar 2019 15:40:38 -0700 Subject: [PATCH] Add Python 3.7 tests to nightly cron CI run (#7261) ### Problem We now claim support for both Python 3.6 and Python 3.7. A couple of times, we've been hit by issues specific to Python 3.7. So, we want to have CI to catch these issues. This will close https://github.com/pantsbuild/pants/issues/7162. ### Solution Add new shards to run with Py37 during the cron stage every night. This requires getting Py37 onto the shards. We extend our current use of Pyenv to also install Python 3.7 where relevant. Note we cannot get Python 3.7 onto the Centos6 Build Engine shard, as the OpenSSL shard is too outdated. Instead, we simply use Ubuntu Xenial to bootstrap the Py37 PEX. This is okay to do because we do not build any wheels with this shard, and only use it for internal testing. See https://github.com/pantsbuild/pants/issues/7421. ### Result Cron job now runs with both Python 2.7 and 3.7. See https://travis-ci.org/pantsbuild/pants/builds/509675034 for what the new nightly run will look like. Daily CI is the same as before. --- .travis.yml | 306 +++++++++++++++++- build-support/bin/ci.sh | 12 +- build-support/travis/travis.yml.mustache | 158 ++++++++- .../tensorflow_custom_op/test_zero_out_op.py | 4 + 4 files changed, 464 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 88c1278c80f..8f07eeb787c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,6 +27,7 @@ env: - BOOTSTRAPPED_PEX_KEY_PREFIX=${TRAVIS_BUILD_NUMBER}/${TRAVIS_BUILD_ID}/pants.pex - BOOTSTRAPPED_PEX_URL_PREFIX=s3://${BOOTSTRAPPED_PEX_BUCKET}/${BOOTSTRAPPED_PEX_KEY_PREFIX} - PYENV_PY36_VERSION=3.6.8 + - PYENV_PY37_VERSION=3.7.2 - PYENV_ROOT="${PYENV_ROOT:-${HOME}/.pants_pyenv}" - PYENV_BIN="${PYENV_ROOT}/bin/pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" @@ -119,9 +120,15 @@ pyenv_setup: &pyenv_setup > pyenv_install_py36: &pyenv_install_py36 > "${PYENV_BIN}" install "${PYENV_PY36_VERSION}" +pyenv_install_py37: &pyenv_install_py37 > + "${PYENV_BIN}" install "${PYENV_PY37_VERSION}" + pyenv_global_py36: &pyenv_global_py36 > "${PYENV_BIN}" global "${PYENV_PY36_VERSION}" +pyenv_global_py37: &pyenv_global_py37 > + "${PYENV_BIN}" global "${PYENV_PY37_VERSION}" + # ------------------------------------------------------------------------- # AWS # ------------------------------------------------------------------------- @@ -168,7 +175,11 @@ py27_linux_config: &py27_linux_config py36_linux_config: &py36_linux_config <<: *base_linux_config - python: &python3_version "3.6" + python: "3.6" + +py37_linux_config: &py37_linux_config + <<: *base_linux_config + python: "3.7" base_linux_test_config: &base_linux_test_config <<: *base_linux_config @@ -195,6 +206,13 @@ py36_linux_test_config: &py36_linux_test_config env: - &py36_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux +py37_linux_test_config: &py37_linux_test_config + <<: *py37_linux_config + <<: *base_linux_test_config + stage: *test_cron + env: + - &py37_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux + base_osx_config: &base_osx_config os: osx language: generic @@ -225,6 +243,25 @@ py36_osx_config: &py36_osx_config - *pyenv_install_py36 - *pyenv_global_py36 +py37_osx_config: &py37_osx_config + <<: *base_osx_config + addons: + brew: + packages: &py37_osx_config_brew_packages + - openssl + env: + - &py37_osx_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + before_install: + - curl -L https://github.com/stedolan/jq/releases/download/jq-1.5/jq-osx-amd64 -o /usr/local/bin/jq + - chmod 755 /usr/local/bin/jq + - ./build-support/bin/install_aws_cli_for_ci.sh + - *pyenv_setup + - *pyenv_install_py37 + - *pyenv_global_py37 + base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config before_script: @@ -244,13 +281,23 @@ py36_osx_test_config: &py36_osx_test_config <<: *base_osx_test_config stage: *test env: - # Must duplicate py36_osx_config's env because it cannot be merged into a new anchor - &py36_osx_test_config_env > PATH="/usr/local/opt/openssl/bin:$PATH" LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.osx +py37_osx_test_config: &py37_osx_test_config + <<: *py37_osx_config + <<: *base_osx_test_config + stage: *test_cron + env: + - &py37_osx_test_config_env > + PATH="/usr/local/opt/openssl/bin:$PATH" + LDFLAGS="-L/usr/local/opt/openssl/lib" + CPPFLAGS="-I/usr/local/opt/openssl/include" + BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.osx + linux_with_fuse: &linux_with_fuse before_install: - PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin":$PATH @@ -325,6 +372,21 @@ py36_linux_build_engine: &py36_linux_build_engine - CACHE_NAME=linuxpexbuild.py36 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux +# NB: we do not use Docker for Py37 because Centos6 has too outdated of OpenSSL. +# See https://github.com/pantsbuild/pants/issues/7421. We will want to change this +# shard back to Docker once we have a Centos7 base image. +py37_linux_build_engine: &py37_linux_build_engine + <<: *py37_linux_config + <<: *native_engine_cache_config + stage: *bootstrap_cron + name: "Build Linux native engine and pants.pex (Py3.7 PEX)" + env: + - CACHE_NAME=linuxpexbuild.py37 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux + script: + - ./build-support/bin/ci.sh -7b + - *aws_deploy_pants_pex + base_osx_build_engine: &base_osx_build_engine <<: *native_engine_cache_config stage: *bootstrap @@ -361,6 +423,19 @@ py36_osx_build_engine: &py36_osx_build_engine - ./build-support/bin/ci.sh -b - *aws_deploy_pants_pex +py37_osx_build_engine: &py37_osx_build_engine + <<: *py37_osx_config + <<: *base_osx_build_engine + stage: *bootstrap_cron + name: "Build OSX native engine and pants.pex (Py3.7 PEX)" + env: + - *py37_osx_config_env + - CACHE_NAME=osxpexbuild.py37 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.osx + script: + - ./build-support/bin/ci.sh -7b + - *aws_deploy_pants_pex + # ------------------------------------------------------------------------- # Lint # ------------------------------------------------------------------------- @@ -384,6 +459,15 @@ py36_lint: &py36_lint script: - ./build-support/bin/travis-ci.sh -fmrt +py37_lint: &py37_lint + <<: *py37_linux_test_config + name: "Self-checks and lint (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py37 + script: + - ./build-support/bin/travis-ci.sh -fmrt7 + # ------------------------------------------------------------------------- # Rust lints # ------------------------------------------------------------------------- @@ -399,7 +483,7 @@ linux_rust_clippy: &linux_rust_clippy sudo: required stage: *test language: python - python: *python3_version + python: "3.6" before_script: - ulimit -c unlimited - ulimit -n 8192 @@ -579,7 +663,7 @@ linux_rust_tests: &linux_rust_tests dist: xenial sudo: required language: python - python: *python3_version + python: "3.6" script: - ./build-support/bin/travis-ci.sh -e @@ -597,8 +681,8 @@ osx_rust_tests: &osx_rust_tests - osxfuse script: # N.B. We run this with Python 2 because this osx_image does not have - # Python 3.6 in its environment. We do not care which Python version - # we use and do not want to incur the cost of using pyenv to get 3.6. + # Python 3.6 or 3.7 in its environment. We do not care which Python version + # we use and do not want to incur the cost of using pyenv to get 3.6 or 3.7. - ./build-support/bin/travis-ci.sh -e2 # ------------------------------------------------------------------------- @@ -630,6 +714,14 @@ py36_osx_10_12_sanity_check: &py36_osx_10_12_sanity_check - *py36_osx_test_config_env - CACHE_NAME=macos10.12sanity.py36 +py37_osx_10_12_sanity_check: &py37_osx_10_12_sanity_check + <<: *py37_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Py3.7 PEX)" + env: + - *py37_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py37 + base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check <<: *base_osx_sanity_check osx_image: xcode10.1 @@ -650,6 +742,14 @@ py36_osx_10_13_sanity_check: &py36_osx_10_13_sanity_check - *py36_osx_test_config_env - CACHE_NAME=macos10.13sanity.py36 +py37_osx_10_13_sanity_check: &py37_osx_10_13_sanity_check + <<: *py37_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Py3.7 PEX)" + env: + - *py37_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py37 + # ------------------------------------------------------------------------- # Platform specific tests # ------------------------------------------------------------------------- @@ -672,6 +772,15 @@ py36_osx_platform_tests: &py36_osx_platform_tests script: - ./build-support/bin/travis-ci.sh -z +py37_osx_platform_tests: &py37_osx_platform_tests + <<: *py37_osx_test_config + name: "OSX platform-specific tests (Py3.7 PEX)" + env: + - *py37_osx_test_config_env + - CACHE_NAME=macosplatformtests.py37 + script: + - ./build-support/bin/travis-ci.sh -z7 + # ------------------------------------------------------------------------- # JVM tests # ------------------------------------------------------------------------- @@ -699,6 +808,16 @@ py36_jvm_tests: &py36_jvm_tests script: - ./build-support/bin/travis-ci.sh -j +py37_jvm_tests: &py37_jvm_tests + <<: *py37_linux_test_config + <<: *base_jvm_tests + name: "JVM tests (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxjvmtests.py37 + script: + - ./build-support/bin/travis-ci.sh -j7 + # ------------------------------------------------------------------------- # Deploy # ------------------------------------------------------------------------- @@ -792,14 +911,17 @@ matrix: - <<: *py27_linux_build_engine stage: *bootstrap_cron - <<: *py36_linux_build_engine + - <<: *py37_linux_build_engine - <<: *py27_osx_build_engine - <<: *py27_osx_build_engine stage: *bootstrap_cron - <<: *py36_osx_build_engine + - <<: *py37_osx_build_engine - <<: *py27_lint - <<: *py36_lint + - <<: *py37_lint - <<: *linux_rust_clippy - <<: *cargo_audit @@ -821,6 +943,14 @@ matrix: script: - ./build-support/bin/travis-ci.sh -lp + - <<: *py37_linux_test_config + name: "Unit tests for pants and pants-plugins (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxunittests.py37 + script: + - ./build-support/bin/travis-ci.sh -7lp + - <<: *py27_linux_build_wheels_ucs2 - <<: *py27_linux_build_wheels_ucs4 - <<: *py36_linux_build_wheels @@ -981,6 +1111,158 @@ matrix: script: - ./build-support/bin/travis-ci.sh -c -i 18/19 + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 0 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard0 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 0/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 1 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard1 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 1/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 2 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard2 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 2/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 3 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard3 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 3/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 4 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard4 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 4/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 5 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard5 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 5/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 6 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard6 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 6/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 7 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard7 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 7/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 8 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard8 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 8/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 9 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard9 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 9/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 10 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard10 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 10/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 11 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard11 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 11/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 12 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard12 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 12/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 13 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard13 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 13/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 14 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard14 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 14/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 15 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard15 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 15/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 16 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard16 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 16/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 17 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard17 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 17/19 + + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard 18 (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard18 + script: + - ./build-support/bin/travis-ci.sh -c7 -i 18/19 + - <<: *py27_linux_test_config name: "Blacklisted integration tests for pants - shard 0 (Py2.7 PEX w/ Py3.6 constraints)" stage: *test @@ -1172,17 +1454,29 @@ matrix: script: - ./build-support/bin/travis-ci.sh -n + - <<: *py37_linux_test_config + name: "Python contrib tests (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxcontribtests.py37 + script: + - ./build-support/bin/travis-ci.sh -7n + - <<: *py27_osx_10_12_sanity_check - <<: *py36_osx_10_12_sanity_check + - <<: *py37_osx_10_12_sanity_check - <<: *py27_osx_10_13_sanity_check - <<: *py36_osx_10_13_sanity_check + - <<: *py37_osx_10_13_sanity_check - <<: *py27_osx_platform_tests - <<: *py36_osx_platform_tests + - <<: *py37_osx_platform_tests - <<: *py27_jvm_tests - <<: *py36_jvm_tests + - <<: *py37_jvm_tests - <<: *py27_deploy_stable_multiplatform_pex - <<: *py36_deploy_stable_multiplatform_pex diff --git a/build-support/bin/ci.sh b/build-support/bin/ci.sh index 98efa1feb07..3bcc889dd12 100755 --- a/build-support/bin/ci.sh +++ b/build-support/bin/ci.sh @@ -16,6 +16,7 @@ Runs commons tests for local or hosted CI. Usage: $0 (-h|-2fxbkmrjlpuneycitzsw) -h print out this help message -2 Run using Python 2.7 (defaults to using Python 3.6). + -7 Run using Python 3.7 (defaults to using Python 3.6). -f run python code formatting checks -x run bootstrap clean-all (assume bootstrapping from a fresh clone) @@ -59,10 +60,11 @@ python_unit_shard="0/1" python_contrib_shard="0/1" python_intg_shard="0/1" -while getopts "h2fxbmrjlpeasu:ny:ci:tzw" opt; do +while getopts "h27fxbmrjlpeasu:ny:ci:tzw" opt; do case ${opt} in h) usage ;; 2) python_two="true" ;; + 7) python_three_seven="true" ;; f) run_pre_commit_checks="true" ;; x) run_bootstrap_clean="true" ;; b) run_bootstrap="true" ;; @@ -109,10 +111,12 @@ export PANTS_DEV=1 # these constraints for us already. However, we must set the values here because in # non-bootstrap shards we run CI using `./pants.pex` instead of the scripts `./pants` # and `./pants2`, so those scripts cannot set the relevant environment variables. -if [[ "${python_two:-false}" == "false" ]]; then - py_major_minor="3.6" -else +if [[ "${python_two:-false}" == "true" ]]; then py_major_minor="2.7" +elif [[ "${python_three_seven:-false}" == "true" ]]; then + py_major_minor="3.7" +else + py_major_minor="3.6" fi export PY="${PY:-python${py_major_minor}}" diff --git a/build-support/travis/travis.yml.mustache b/build-support/travis/travis.yml.mustache index 659d6220e59..bec7b237758 100644 --- a/build-support/travis/travis.yml.mustache +++ b/build-support/travis/travis.yml.mustache @@ -20,6 +20,7 @@ env: - BOOTSTRAPPED_PEX_KEY_PREFIX=${TRAVIS_BUILD_NUMBER}/${TRAVIS_BUILD_ID}/pants.pex - BOOTSTRAPPED_PEX_URL_PREFIX=s3://${BOOTSTRAPPED_PEX_BUCKET}/${BOOTSTRAPPED_PEX_KEY_PREFIX} - PYENV_PY36_VERSION=3.6.8 + - PYENV_PY37_VERSION=3.7.2 - PYENV_ROOT="${PYENV_ROOT:-${HOME}/.pants_pyenv}" - PYENV_BIN="${PYENV_ROOT}/bin/pyenv" - PATH="${PYENV_ROOT}/shims:${PATH}" @@ -112,9 +113,15 @@ pyenv_setup: &pyenv_setup > pyenv_install_py36: &pyenv_install_py36 > "${PYENV_BIN}" install "${PYENV_PY36_VERSION}" +pyenv_install_py37: &pyenv_install_py37 > + "${PYENV_BIN}" install "${PYENV_PY37_VERSION}" + pyenv_global_py36: &pyenv_global_py36 > "${PYENV_BIN}" global "${PYENV_PY36_VERSION}" +pyenv_global_py37: &pyenv_global_py37 > + "${PYENV_BIN}" global "${PYENV_PY37_VERSION}" + # ------------------------------------------------------------------------- # AWS # ------------------------------------------------------------------------- @@ -161,7 +168,11 @@ py27_linux_config: &py27_linux_config py36_linux_config: &py36_linux_config <<: *base_linux_config - python: &python3_version "3.6" + python: "3.6" + +py37_linux_config: &py37_linux_config + <<: *base_linux_config + python: "3.7" base_linux_test_config: &base_linux_test_config <<: *base_linux_config @@ -185,6 +196,13 @@ py36_linux_test_config: &py36_linux_test_config env: - &py36_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux +py37_linux_test_config: &py37_linux_test_config + <<: *py37_linux_config + <<: *base_linux_test_config + stage: *test_cron + env: + - &py37_linux_test_config_env BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux + base_osx_config: &base_osx_config os: osx language: generic @@ -209,6 +227,21 @@ py36_osx_config: &py36_osx_config - *pyenv_install_py36 - *pyenv_global_py36 +py37_osx_config: &py37_osx_config + <<: *base_osx_config + addons: + brew: + packages: &py37_osx_config_brew_packages + - openssl + env: + - &py37_osx_config_env > + {{>env_osx_with_pyenv}} + before_install: + {{>before_install_osx}} + - *pyenv_setup + - *pyenv_install_py37 + - *pyenv_global_py37 + base_osx_test_config: &base_osx_test_config <<: *pants_run_cache_config before_script: @@ -228,11 +261,19 @@ py36_osx_test_config: &py36_osx_test_config <<: *base_osx_test_config stage: *test env: - # Must duplicate py36_osx_config's env because it cannot be merged into a new anchor - &py36_osx_test_config_env > {{>env_osx_with_pyenv}} BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.osx +py37_osx_test_config: &py37_osx_test_config + <<: *py37_osx_config + <<: *base_osx_test_config + stage: *test_cron + env: + - &py37_osx_test_config_env > + {{>env_osx_with_pyenv}} + BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.osx + linux_with_fuse: &linux_with_fuse before_install: {{>before_install_linux}} @@ -293,6 +334,21 @@ py36_linux_build_engine: &py36_linux_build_engine - CACHE_NAME=linuxpexbuild.py36 - BOOTSTRAPPED_PEX_KEY_SUFFIX=py36.linux +# NB: we do not use Docker for Py37 because Centos6 has too outdated of OpenSSL. +# See https://github.com/pantsbuild/pants/issues/7421. We will want to change this +# shard back to Docker once we have a Centos7 base image. +py37_linux_build_engine: &py37_linux_build_engine + <<: *py37_linux_config + <<: *native_engine_cache_config + stage: *bootstrap_cron + name: "Build Linux native engine and pants.pex (Py3.7 PEX)" + env: + - CACHE_NAME=linuxpexbuild.py37 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.linux + script: + - ./build-support/bin/ci.sh -7b + - *aws_deploy_pants_pex + base_osx_build_engine: &base_osx_build_engine <<: *native_engine_cache_config stage: *bootstrap @@ -329,6 +385,19 @@ py36_osx_build_engine: &py36_osx_build_engine - ./build-support/bin/ci.sh -b - *aws_deploy_pants_pex +py37_osx_build_engine: &py37_osx_build_engine + <<: *py37_osx_config + <<: *base_osx_build_engine + stage: *bootstrap_cron + name: "Build OSX native engine and pants.pex (Py3.7 PEX)" + env: + - *py37_osx_config_env + - CACHE_NAME=osxpexbuild.py37 + - BOOTSTRAPPED_PEX_KEY_SUFFIX=py37.osx + script: + - ./build-support/bin/ci.sh -7b + - *aws_deploy_pants_pex + # ------------------------------------------------------------------------- # Lint # ------------------------------------------------------------------------- @@ -352,6 +421,15 @@ py36_lint: &py36_lint script: - ./build-support/bin/travis-ci.sh -fmrt +py37_lint: &py37_lint + <<: *py37_linux_test_config + name: "Self-checks and lint (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxselfchecks.py37 + script: + - ./build-support/bin/travis-ci.sh -fmrt7 + # ------------------------------------------------------------------------- # Rust lints # ------------------------------------------------------------------------- @@ -367,7 +445,7 @@ linux_rust_clippy: &linux_rust_clippy sudo: required stage: *test language: python - python: *python3_version + python: "3.6" before_script: - ulimit -c unlimited - ulimit -n 8192 @@ -532,7 +610,7 @@ linux_rust_tests: &linux_rust_tests dist: xenial sudo: required language: python - python: *python3_version + python: "3.6" script: - ./build-support/bin/travis-ci.sh -e @@ -550,8 +628,8 @@ osx_rust_tests: &osx_rust_tests - osxfuse script: # N.B. We run this with Python 2 because this osx_image does not have - # Python 3.6 in its environment. We do not care which Python version - # we use and do not want to incur the cost of using pyenv to get 3.6. + # Python 3.6 or 3.7 in its environment. We do not care which Python version + # we use and do not want to incur the cost of using pyenv to get 3.6 or 3.7. - ./build-support/bin/travis-ci.sh -e2 # ------------------------------------------------------------------------- @@ -583,6 +661,14 @@ py36_osx_10_12_sanity_check: &py36_osx_10_12_sanity_check - *py36_osx_test_config_env - CACHE_NAME=macos10.12sanity.py36 +py37_osx_10_12_sanity_check: &py37_osx_10_12_sanity_check + <<: *py37_osx_test_config + <<: *base_osx_10_12_sanity_check + name: "OSX 10.12 sanity check (Py3.7 PEX)" + env: + - *py37_osx_test_config_env + - CACHE_NAME=macos10.12sanity.py37 + base_osx_10_13_sanity_check: &base_osx_10_13_sanity_check <<: *base_osx_sanity_check osx_image: xcode10.1 @@ -603,6 +689,14 @@ py36_osx_10_13_sanity_check: &py36_osx_10_13_sanity_check - *py36_osx_test_config_env - CACHE_NAME=macos10.13sanity.py36 +py37_osx_10_13_sanity_check: &py37_osx_10_13_sanity_check + <<: *py37_osx_test_config + <<: *base_osx_10_13_sanity_check + name: "OSX 10.13 sanity check (Py3.7 PEX)" + env: + - *py37_osx_test_config_env + - CACHE_NAME=macos10.13sanity.py37 + # ------------------------------------------------------------------------- # Platform specific tests # ------------------------------------------------------------------------- @@ -625,6 +719,15 @@ py36_osx_platform_tests: &py36_osx_platform_tests script: - ./build-support/bin/travis-ci.sh -z +py37_osx_platform_tests: &py37_osx_platform_tests + <<: *py37_osx_test_config + name: "OSX platform-specific tests (Py3.7 PEX)" + env: + - *py37_osx_test_config_env + - CACHE_NAME=macosplatformtests.py37 + script: + - ./build-support/bin/travis-ci.sh -z7 + # ------------------------------------------------------------------------- # JVM tests # ------------------------------------------------------------------------- @@ -652,6 +755,16 @@ py36_jvm_tests: &py36_jvm_tests script: - ./build-support/bin/travis-ci.sh -j +py37_jvm_tests: &py37_jvm_tests + <<: *py37_linux_test_config + <<: *base_jvm_tests + name: "JVM tests (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxjvmtests.py37 + script: + - ./build-support/bin/travis-ci.sh -j7 + # ------------------------------------------------------------------------- # Deploy # ------------------------------------------------------------------------- @@ -745,14 +858,17 @@ matrix: - <<: *py27_linux_build_engine stage: *bootstrap_cron - <<: *py36_linux_build_engine + - <<: *py37_linux_build_engine - <<: *py27_osx_build_engine - <<: *py27_osx_build_engine stage: *bootstrap_cron - <<: *py36_osx_build_engine + - <<: *py37_osx_build_engine - <<: *py27_lint - <<: *py36_lint + - <<: *py37_lint - <<: *linux_rust_clippy - <<: *cargo_audit @@ -774,6 +890,14 @@ matrix: script: - ./build-support/bin/travis-ci.sh -lp + - <<: *py37_linux_test_config + name: "Unit tests for pants and pants-plugins (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxunittests.py37 + script: + - ./build-support/bin/travis-ci.sh -7lp + - <<: *py27_linux_build_wheels_ucs2 - <<: *py27_linux_build_wheels_ucs4 - <<: *py36_linux_build_wheels @@ -791,6 +915,16 @@ matrix: script: - ./build-support/bin/travis-ci.sh -c -i {{.}}/{{py3_integration_shards_length}} +{{/py3_integration_shards}} +{{#py3_integration_shards}} + - <<: *py37_linux_test_config + name: "Integration tests for pants - shard {{.}} (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=integrationshard{{.}} + script: + - ./build-support/bin/travis-ci.sh -c7 -i {{.}}/{{py3_integration_shards_length}} + {{/py3_integration_shards}} {{#py2_blacklist_integration_shards}} - <<: *py27_linux_test_config @@ -835,17 +969,29 @@ matrix: script: - ./build-support/bin/travis-ci.sh -n + - <<: *py37_linux_test_config + name: "Python contrib tests (Py3.7 PEX)" + env: + - *py37_linux_test_config_env + - CACHE_NAME=linuxcontribtests.py37 + script: + - ./build-support/bin/travis-ci.sh -7n + - <<: *py27_osx_10_12_sanity_check - <<: *py36_osx_10_12_sanity_check + - <<: *py37_osx_10_12_sanity_check - <<: *py27_osx_10_13_sanity_check - <<: *py36_osx_10_13_sanity_check + - <<: *py37_osx_10_13_sanity_check - <<: *py27_osx_platform_tests - <<: *py36_osx_platform_tests + - <<: *py37_osx_platform_tests - <<: *py27_jvm_tests - <<: *py36_jvm_tests + - <<: *py37_jvm_tests - <<: *py27_deploy_stable_multiplatform_pex - <<: *py36_deploy_stable_multiplatform_pex diff --git a/examples/tests/python/example_test/tensorflow_custom_op/test_zero_out_op.py b/examples/tests/python/example_test/tensorflow_custom_op/test_zero_out_op.py index 1806f84df7e..2bd22168b03 100644 --- a/examples/tests/python/example_test/tensorflow_custom_op/test_zero_out_op.py +++ b/examples/tests/python/example_test/tensorflow_custom_op/test_zero_out_op.py @@ -4,6 +4,9 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import sys +import unittest + import tensorflow as tf from example.tensorflow_custom_op.zero_out_custom_op import zero_out_module @@ -12,6 +15,7 @@ # This code is from the guide in https://www.tensorflow.org/guide/extend/op. class ZeroOutTest(tf.test.TestCase): + @unittest.skipIf(sys.version_info[0:2] == (3, 7), "See https://github.com/pantsbuild/pants/issues/7417.") def test_zero_out(self): with self.test_session(): result = zero_out_module().zero_out([5, 4, 3, 2, 1])