Create Python3 virtualenv by default

- Allow specifying Python to use to create virtualenv using GALAXY_PYTHON
  environment variable.
- Integrate Python support docs from https://galaxyproject.org/admin/python/
- Update virtualenv installation to use v16.7.9

.circleci/config.yml

@@ -97,14 +97,14 @@ jobs:
       - *restore_repo_cache
       - *install_tox
       - run: tox -e py27-unit
-  py27_docstring:
+  py35_docstring:
     docker:
-      - image: circleci/python:2.7
+      - image: circleci/python:3.5
     <<: *set_workdir
     steps:
       - *restore_repo_cache
       - *install_tox
-      - run: tox -e py27-lint_docstring_include_list
+      - run: tox -e py35-lint_docstring_include_list
   py27_first_startup:
     docker:
       - image: circleci/python:2.7
@@ -200,7 +200,7 @@ workflows:
           <<: *requires_get_code
       - py27_first_startup:
           <<: *requires_get_code
-      - py27_docstring:
+      - py35_docstring:
           <<: *requires_get_code
       - py35_lint:
           <<: *requires_get_code

doc/source/admin/framework_dependencies.rst

@@ -181,7 +181,7 @@ Galaxy can create a virtualenv using the adapted virtualenv package. Once a vali
         3. Start galaxy using ``sh run.sh`` or execute ``sh scripts/common_startup.sh``.
 
 
-    A Conda environment named ``_galaxy_`` will be created using python 2 and the appropriate virtualenv package will be installed into this environment.
+    A Conda environment named ``_galaxy_`` will be created and the appropriate virtualenv package will be installed into this environment.
     Using this environment a ``.venv`` is initialized. This is a one-time setup, and all other activation and dependency
     management happens exactly as if a system Python was used for creating ``.venv``.
 

doc/source/admin/index.rst

@@ -6,19 +6,20 @@ This documentation is in the midst of being ported and unified based on resource
 .. toctree::
    :maxdepth: 2
 
+   python
+   framework_dependencies
    config
    config_logging
    production
-   scaling
    nginx
    apache
+   scaling
    cluster
    jobs
-   tool_panel
    authentication
+   tool_panel
    dependency_resolvers
    conda_faq
-   framework_dependencies
-   options
    reports
    useful_scripts
+   options

doc/source/admin/python.md

@@ -0,0 +1,37 @@
+# Supported Python versions
+
+Galaxy's core functionality is currently supported on Python **2.7** (although
+deprecated) and **>=3.5** .
+
+If Galaxy complains about the version of Python you are using, check that
+`python --version` reports a supported version. If this is not the case:
+
+1. Completely remove the virtualenv used by Galaxy, e.g. with:
+   `rm -rf /path/to/galaxy/.venv`
+
+2. Let Galaxy know the path of the correct version of Python.
+
+    - If you are using Galaxy >= 20.05, just execute:
+      `export GALAXY_PYTHON=/path/to/python`
+    - If instead you are using an older version of Galaxy, you can manipulate
+      your shell's `$PATH` variable to place the correct version first. This can
+      be done for just Python by creating a new directory, adding a symbolic
+      link to python in there, and putting that directory at the front of
+      `$PATH`:
+
+        ```sh
+        % mkdir ~/galaxy-python
+        % ln -s /path/to/python ~/galaxy-python/python
+        % export PATH=~/galaxy-python:$PATH
+        ```
+
+3. Start Galaxy again.
+
+N.B. If you have compiled your own Python interpreter from source, please ensure
+that the `ssl`, `sqlite3`, `curses` and `bz2` modules were built and can be
+imported after installation. These "extra" modules are built at the end of the
+compilation process and are required by the Galaxy framework. If building on
+Linux, you may need to install the appropriate `-dev` packages for OpenSSL and
+Bzip2. You may also need to build Python with shared libraries
+(`--enable-shared`).
+

run_tests.sh

@@ -1,4 +1,3 @@
-
 #!/bin/sh
 
 cd "$(dirname "$0")"
@@ -313,7 +312,7 @@ then
     echo "Docker version:"
     docker --version
     echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..."
-    name=$(python -c 'import re; import uuid; print re.sub("-","",str(uuid.uuid4()))')
+    name=$(python -c 'import re; import uuid; print re.sub("-", "", str(uuid.uuid4()))')
     # Create a cache dir for pip, so it has the right owner
     DOCKER_PIP_CACHE_DIR="$HOME"/.cache/docker_galaxy_pip
     mkdir -p "$DOCKER_PIP_CACHE_DIR"

scripts/check_python.py

@@ -6,20 +6,28 @@
 
 import sys
 
-version_string = '.'.join(str(_) for _ in sys.version_info[:3])
-
-msg = """ERROR: Your Python version is: %s
-Galaxy is currently supported on Python 2.7 and >=3.5 . To run Galaxy, please
-install a supported Python version. If a supported version is already
-installed but is not your default, https://galaxyproject.org/admin/python/
-contains instructions on how to force Galaxy to use a different version.""" % version_string
-
 
 def check_python():
-    if sys.version_info[:2] == (2, 7) or sys.version_info[:2] >= (3, 5):
+    if sys.version_info[:2] >= (3, 5):
         # supported
         return
+    elif sys.version_info[:2] == (2, 7):
+        msg = """\
+Galaxy support for Python 2.7 is deprecated, please consider moving to
+Python >= 3.5 .
+https://docs.galaxyproject.org/en/latest/admin/python.html contains instructions
+on how to force Galaxy to use a different version."""
+        print(msg, file=sys.stderr)
+        return
     else:
+        version_string = '.'.join(str(_) for _ in sys.version_info[:3])
+        msg = """\
+ERROR: Your Python version is: %s
+Galaxy is currently supported on Python 2.7 (although deprecated) and >=3.5 .
+To run Galaxy, please install a supported Python version.
+If a supported version is already installed but is not your default,
+https://docs.galaxyproject.org/en/latest/admin/python.html contains instructions
+on how to force Galaxy to use a different version.""" % version_string
         print(msg, file=sys.stderr)
         raise Exception(msg)
 

scripts/common_startup.sh

@@ -121,7 +121,7 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
                     echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV"
                     echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an"
                     echo "existing environment before starting Galaxy."
-                    $CONDA_EXE create --yes --override-channels --channel conda-forge --channel defaults --name "$GALAXY_CONDA_ENV" 'python=2.7' 'pip>=9' 'virtualenv>=16'
+                    $CONDA_EXE create --yes --override-channels --channel conda-forge --channel defaults --name "$GALAXY_CONDA_ENV" 'python=3.6' 'pip>=9' 'virtualenv>=16'
                     unset __CONDA_INFO
                 fi
                 conda_activate
@@ -133,13 +133,20 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
             echo "Creating Python virtual environment for Galaxy: $GALAXY_VIRTUAL_ENV"
             echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_VIRTUAL_ENV to an"
             echo "existing environment before starting Galaxy."
-            python ./scripts/check_python.py || exit 1
+            if [ -z "$GALAXY_PYTHON" ]; then
+                if command -v python3 >/dev/null; then
+                    GALAXY_PYTHON=python3
+                else
+                    GALAXY_PYTHON=python
+                fi
+            fi
+            "$GALAXY_PYTHON" ./scripts/check_python.py || exit 1
             if command -v virtualenv >/dev/null; then
-                virtualenv -p "$(command -v python)" "$GALAXY_VIRTUAL_ENV"
+                virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV"
             else
-                vvers=16.1.0
+                vvers=16.7.9
                 vurl="https://files.pythonhosted.org/packages/source/v/virtualenv/virtualenv-${vvers}.tar.gz"
-                vsha=f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208
+                vsha=0d62c70883c0342d59c11d0ddac0d954d0431321a41ab20851facf2b222598f3
                 vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX)
                 vsrc="$vtmp/$(basename $vurl)"
                 # SSL certificates are not checked to prevent problems with messed
@@ -151,16 +158,16 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
                 elif command -v wget >/dev/null; then
                     wget --no-check-certificate -O "$vsrc" "$vurl"
                 else
-                    python -c "try:
+                    "$GALAXY_PYTHON" -c "try:
     from urllib import urlretrieve
 except:
     from urllib.request import urlretrieve
 urlretrieve('$vurl', '$vsrc')"
                 fi
                 echo "Verifying $vsrc checksum is $vsha"
-                python -c "import hashlib; assert hashlib.sha256(open('$vsrc', 'rb').read()).hexdigest() == '$vsha', '$vsrc: invalid checksum'"
+                "$GALAXY_PYTHON" -c "import hashlib; assert hashlib.sha256(open('$vsrc', 'rb').read()).hexdigest() == '$vsha', '$vsrc: invalid checksum'"
                 tar zxf "$vsrc" -C "$vtmp"
-                python "$vtmp/virtualenv-$vvers/src/virtualenv.py" "$GALAXY_VIRTUAL_ENV"
+                "$GALAXY_PYTHON" "$vtmp/virtualenv-$vvers/virtualenv.py" "$GALAXY_VIRTUAL_ENV"
                 rm -rf "$vtmp"
             fi
         fi

tox.ini

@@ -1,17 +1,21 @@
 [tox]
 # envlist is the list of environments that are tested when `tox` is run without any option
 # hyphens in an environment name are used to delimit factors
-envlist = check_py3_compatibility, py{27,35}-first_startup, py{27,35}-lint, py27-lint_docstring_include_list, py{27,35}-unit, test_galaxy_packages, validate_test_tools
+envlist = check_py3_compatibility, py{27,35}-first_startup, py{27,35}-lint, py35-lint_docstring_include_list, py{27,35}-unit, test_galaxy_packages, validate_test_tools
 skipsdist = True
 
 [testenv]
 commands =
     first_startup: bash .ci/first_startup.sh
     lint: bash .ci/flake8_wrapper.sh
+    lint_docstring: bash .ci/flake8_wrapper_docstrings.sh --exclude
+    lint_docstring_include_list: bash .ci/flake8_wrapper_docstrings.sh --include
+
     unit: bash run_tests.sh -u
 whitelist_externals = bash
 passenv = CI
 setenv =
+    first_startup: GALAXY_PYTHON=python
     py{35,36,37}-first_startup: GALAXY_VIRTUAL_ENV=.venv3
     unit: GALAXY_VIRTUAL_ENV={envdir}
     unit: GALAXY_ENABLE_BETA_COMPRESSED_GENBANK_SNIFFING=1
@@ -34,12 +38,6 @@ whitelist_externals = make
 [testenv:mako_count]
 commands = bash .ci/check_mako.sh
 
-[testenv:py27-lint_docstring]
-commands = bash .ci/flake8_wrapper_docstrings.sh --exclude
-
-[testenv:py27-lint_docstring_include_list]
-commands = bash .ci/flake8_wrapper_docstrings.sh --include
-
 [testenv:test_galaxy_packages]
 commands = bash packages/test.sh