Add support for running with Python 3.6 and 3.7 (#32)

### Problem
Now that we're releasing Pants with Python 3 wheels (pantsbuild/pants#7197, overall migration tracked by pantsbuild/pants#6062), we need to add a way for every day Pants users to consume the upgrade.

See #30 for more context, including the design goals.

### Solution
Parse the `pants_runtime_python_version` from `pants.ini` to determine which Python interpreter to use. If it's missing, default to Python 2.7.

This requires modifying the venv folders to support multiple Python versions. Rather than using the naming scheme `${pants_version}`, we now consistently use `${pants_version}_py{py_major_minor}`, regardless of if the user is using `pants_runtime_python_version`.

It also requires upgrading virtualenv to avoid a deprecation warning when running with Python 3.

Finally, we add tests to ensure this all works.

### Result
Running `./pants` like normal for the first time will bootstrap Pants again and create the new folder `1.15.dev3_py{27,36,37}`.

Changing the `$PYTHON` env var or the `pants_runtime_python_version` in `pants.ini` will change which Python version is used to run Pants.

When changing interpreters for the first time, it will bootstrap Pants again. After that, you can change without issue.

.travis.yml

@@ -13,8 +13,18 @@ script:
   # Linux Precise and Trusty cannot install Python 3.7 due to an outdated OpenSSL,
   # so we set this variable to allow skipping Python 3.7 on these shards.
   - python37_present="$(if which python3.7 >/dev/null; then echo 'true'; else echo 'false'; fi)"
-  - ./ci.py --pants-version unspecified
-  - ./ci.py --pants-version config
+  - ./ci.py --pants-version unspecified --python-version unspecified
+  # TODO: these entries will not work until 1.15.0 is cut, as ./pants uses 1.14.0 currently
+  # when pants_version is unspecified. Add this test back once 1.15.0 is released.
+  # - ./ci.py --pants-version unspecified --python-version 2.7
+  # - ./ci.py --pants-version unspecified --python-version 3.6
+  # - if [[ "${python37_present}" = 'true' ]]; then
+  #    ./ci.py --pants-version unspecified --python-version 3.7; fi
+  - ./ci.py --pants-version config --python-version unspecified
+  - ./ci.py --pants-version config --python-version 2.7
+  - ./ci.py --pants-version config --python-version 3.6
+  - if [[ "${python37_present}" = 'true' ]]; then
+      ./ci.py --pants-version config --python-version 3.7; fi
 
 osx_setup: &osx_setup
   os: osx
@@ -69,8 +79,15 @@ matrix:
       # tests once https://github.com/pantsbuild/pants/issues/6714 and
       # https://github.com/pantsbuild/pants/issues/7323 are resolved.
       script:
-        - ./ci.py --pants-version unspecified --skip-pantsd-tests
-        - ./ci.py --pants-version config --skip-pantsd-tests
+        - ./ci.py --pants-version unspecified --python-version unspecified --skip-pantsd-tests
+        # TODO: add these once 1.15.0 is released.
+        # - ./ci.py --pants-version unspecified --python-version 2.7 --skip-pantsd-tests
+        # - ./ci.py --pants-version unspecified --python-version 3.6 --skip-pantsd-tests
+        # - ./ci.py --pants-version unspecified --python-version 3.7 --skip-pantsd-tests
+        - ./ci.py --pants-version config --python-version unspecified --skip-pantsd-tests
+        - ./ci.py --pants-version config --python-version 2.7 --skip-pantsd-tests
+        - ./ci.py --pants-version config --python-version 3.6 --skip-pantsd-tests
+        - ./ci.py --pants-version config --python-version 3.7 --skip-pantsd-tests
 
     - name: "OSX 10.13 - High Sierra"
       <<: *osx_setup

ci.py

@@ -22,9 +22,23 @@ def __str__(self):
       return self.value
 
 
+class PythonVersion(Enum):
+  unspecified = "unspecified"
+  py27 = "2.7"
+  py36 = "3.6"
+  py37 = "3.7"
+
+  def __str__(self):
+    return self.value
+
+
 def main() -> None:
   args = create_parser().parse_args()
-  run_tests(test_pants_version=args.pants_version, skip_pantsd_tests=args.skip_pantsd_tests)
+  run_tests(
+    test_pants_version=args.pants_version,
+    test_python_version=args.python_version,
+    skip_pantsd_tests=args.skip_pantsd_tests
+  )
 
 
 def create_parser() -> argparse.ArgumentParser:
@@ -37,43 +51,59 @@ def create_parser() -> argparse.ArgumentParser:
       required=True,
       help="Pants version to configure ./pants to use."
   )
+  parser.add_argument(
+      "--python-version",
+      action="store",
+      type=PythonVersion,
+      choices=list(PythonVersion),
+      required=True,
+      help="Python version to configure ./pants to use."
+  )
   parser.add_argument("--skip-pantsd-tests", action="store_true")
   return parser
 
 
-def run_tests(*, test_pants_version: PantsVersion, skip_pantsd_tests: bool) -> None:
+def run_tests(*, test_pants_version: PantsVersion, test_python_version: PythonVersion, skip_pantsd_tests: bool) -> None:
   version_command = ["./pants", "--version"]
   list_command = ["./pants", "list", "::"]
   env_with_pantsd = {**os.environ, "PANTS_ENABLE_PANTSD": "True"}
   with setup_pants_version(test_pants_version):
-    subprocess.run(version_command, check=True)
-    subprocess.run(list_command, check=True)
-    if not skip_pantsd_tests:
-      subprocess.run(version_command, env=env_with_pantsd, check=True)
-      subprocess.run(list_command, env=env_with_pantsd, check=True)
+    with setup_python_version(test_python_version):
+      subprocess.run(version_command, check=True)
+      subprocess.run(list_command, check=True)
+      if not skip_pantsd_tests:
+        subprocess.run(version_command, env=env_with_pantsd, check=True)
+        subprocess.run(list_command, env=env_with_pantsd, check=True)
 
 
 @contextmanager
 def setup_pants_version(test_pants_version: PantsVersion):
   """Modify pants.ini to allow the pants version to be unspecified or keep what was originally there."""
-  original_config = read_config()
   updated_config = read_config()
   config_entry = "pants_version"
   if test_pants_version == PantsVersion.unspecified:
     updated_config.remove_option(GLOBAL_SECTION, config_entry)
     # NB: We also remove plugins as they refer to the pants_version.
     updated_config.remove_option(GLOBAL_SECTION, "plugins")
-    write_config(updated_config)
   elif test_pants_version == PantsVersion.config:
-    if config_entry not in original_config[GLOBAL_SECTION]:
+    if config_entry not in updated_config[GLOBAL_SECTION]:
       raise ValueError("You requested to use the pants_version from pants.ini for this test, but pants.ini "
                        "does not include a pants_version!")
-  try:
+  with temporarily_rewrite_config(updated_config):
+    yield
+
+
+@contextmanager
+def setup_python_version(test_python_version: PythonVersion):
+  """Modify pants.ini to allow the Python version to be unspecified or change to what was requested."""
+  updated_config = read_config()
+  config_entry = "pants_runtime_python_version"
+  if test_python_version == PythonVersion.unspecified:
+    updated_config.remove_option(GLOBAL_SECTION, config_entry)
+  else:
+    updated_config[GLOBAL_SECTION][config_entry] = test_python_version.value
+  with temporarily_rewrite_config(updated_config):
     yield
-  except subprocess.CalledProcessError:
-    raise
-  finally:
-    write_config(original_config)
 
 
 def read_config() -> configparser.ConfigParser:
@@ -87,5 +117,15 @@ def write_config(config: configparser.ConfigParser) -> None:
     config.write(f)
 
 
+@contextmanager
+def temporarily_rewrite_config(updated_config: configparser.ConfigParser) -> None:
+  original_config = read_config()
+  write_config(updated_config)
+  try:
+    yield
+  finally:
+    write_config(original_config)
+
+
 if __name__ == "__main__":
   main()

pants

@@ -13,12 +13,12 @@
 
 set -eou pipefail
 
-PYTHON_BIN_NAME="${PYTHON:-python2.7}"
+PYTHON_BIN_NAME="${PYTHON:-unspecified}"
 
 PANTS_HOME="${PANTS_HOME:-${XDG_CACHE_HOME:-$HOME/.cache}/pants/setup}"
 PANTS_BOOTSTRAP="${PANTS_HOME}/bootstrap-$(uname -s)-$(uname -m)"
 
-VENV_VERSION=${VENV_VERSION:-16.0.0}
+VENV_VERSION=${VENV_VERSION:-16.4.3}
 
 VENV_PACKAGE=virtualenv-${VENV_VERSION}
 VENV_TARBALL=${VENV_PACKAGE}.tar.gz
@@ -44,6 +44,13 @@ function tempdir {
   mktemp -d "$1"/pants.XXXXXX
 }
 
+function get_exe_path_or_die {
+  exe="$1"
+  if ! which "${exe}"; then
+    die "Could not find ${exe}. Please ensure ${exe} is on your PATH."
+  fi
+}
+
 function get_pants_ini_config_value {
   config_key="$1"
   valid_delimiters="[:=]"
@@ -52,8 +59,18 @@ function get_pants_ini_config_value {
   sed -ne "/${prefix}/ s#${prefix}##p" pants.ini
 }
 
+function get_python_major_minor_version {
+  python_exe="$1"
+  "$python_exe" <<EOF
+import sys
+major_minor_version = ''.join(str(version_num) for version_num in sys.version_info[0:2])
+print(major_minor_version)
+EOF
+}
+
 # The high-level flow:
-# 1.) Resolve the Python interpreter, defaulting to `python2.7`.
+# 1.) Resolve the Python interpreter, first reading from the env var $PYTHON,
+#     then reading from pants.ini, then defaulting to `python2.7`.
 # 2.) Grab pants version from pants.ini or default to latest.
 # 3.) Check for a venv via a naming/path convention and execute if found.
 # 4.) Otherwise create venv and re-exec self.
@@ -62,11 +79,16 @@ function get_pants_ini_config_value {
 # are installed and up to date.
 
 function determine_pants_runtime_python_version {
-  if which "${PYTHON_BIN_NAME}" >/dev/null; then
-    which "${PYTHON_BIN_NAME}"
+  if [[ "${PYTHON_BIN_NAME}" != 'unspecified' ]]; then
+    python_bin_name="${PYTHON_BIN_NAME}"
   else
-    die "Could not find ${PYTHON_BIN_NAME}. Please ensure ${PYTHON_BIN_NAME} is on your PATH."
+    interpreter_version="$(get_pants_ini_config_value 'pants_runtime_python_version')"
+    if [[ -z "${interpreter_version}" ]]; then
+      interpreter_version='2.7'
+    fi
+    python_bin_name="python${interpreter_version}"
   fi
+  get_exe_path_or_die "${python_bin_name}"
 }
 
 # TODO(John Sirois): GC race loser tmp dirs leftover from bootstrap_XXX
@@ -97,7 +119,8 @@ function bootstrap_pants {
     pants_version="unspecified"
   fi
 
-  target_folder_name="${pants_version}"
+  python_major_minor_version="$(get_python_major_minor_version "${python}")"
+  target_folder_name="${pants_version}_py${python_major_minor_version}"
 
   if [[ ! -d "${PANTS_BOOTSTRAP}/${target_folder_name}" ]]; then
     (

pants.ini

@@ -1,5 +1,6 @@
 [GLOBAL]
-pants_version : 1.15.0.dev3
+pants_version : 1.15.0.dev4
+pants_runtime_python_version : 3.6
 plugins : [
 	'pantsbuild.pants.contrib.go==%(pants_version)s',
 	]