From 437f51c2a605702c69359a036d2c0809c2e8592a Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 7 Feb 2020 20:01:42 +0000 Subject: [PATCH 1/2] Create Python3 virtualenv by default - Allow specifying Python to use to create virtualenv using GALAXY_PYTHON environment variable. - Integrate Python support docs from https://galaxyproject.org/admin/python/ - Update virtualenv installation to use v16.7.9 --- .circleci/config.yml | 12 +++---- doc/source/admin/framework_dependencies.rst | 2 +- doc/source/admin/index.rst | 9 ++--- doc/source/admin/python.md | 37 +++++++++++++++++++++ run_tests.sh | 3 +- scripts/check_python.py | 26 ++++++++++----- scripts/common_startup.sh | 23 ++++++++----- tox.ini | 12 +++---- 8 files changed, 85 insertions(+), 39 deletions(-) create mode 100644 doc/source/admin/python.md diff --git a/.circleci/config.yml b/.circleci/config.yml index 833e897606c3..7904c41fac37 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -97,24 +97,20 @@ jobs: - *restore_repo_cache - *install_tox - run: tox -e py27-unit - py27_docstring: + py35_docstring: docker: - - image: circleci/python:2.7 + - image: circleci/python:3.5 <<: *set_workdir steps: - *restore_repo_cache - *install_tox - - run: tox -e py27-lint_docstring_include_list + - run: tox -e py35-lint_docstring_include_list py27_first_startup: docker: - image: circleci/python:2.7 <<: *set_workdir steps: - *restore_repo_cache - - run: sh scripts/common_startup.sh - - run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0127.sqlite - - run: mv db_gx_rev_0127.sqlite database/universe.sqlite - - run: sh manage_db.sh -c ./config/galaxy.yml.sample upgrade - *install_tox - run: tox -e py27-first_startup py35_lint: @@ -200,7 +196,7 @@ workflows: <<: *requires_get_code - py27_first_startup: <<: *requires_get_code - - py27_docstring: + - py35_docstring: <<: *requires_get_code - py35_lint: <<: *requires_get_code diff --git a/doc/source/admin/framework_dependencies.rst b/doc/source/admin/framework_dependencies.rst index fa1c0e9f8f66..8d7bb7aafcb6 100644 --- a/doc/source/admin/framework_dependencies.rst +++ b/doc/source/admin/framework_dependencies.rst @@ -181,7 +181,7 @@ Galaxy can create a virtualenv using the adapted virtualenv package. Once a vali 3. Start galaxy using ``sh run.sh`` or execute ``sh scripts/common_startup.sh``. - A Conda environment named ``_galaxy_`` will be created using python 2 and the appropriate virtualenv package will be installed into this environment. + A Conda environment named ``_galaxy_`` will be created and the appropriate virtualenv package will be installed into this environment. Using this environment a ``.venv`` is initialized. This is a one-time setup, and all other activation and dependency management happens exactly as if a system Python was used for creating ``.venv``. diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index e667df8cabe1..690774a1e662 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -6,19 +6,20 @@ This documentation is in the midst of being ported and unified based on resource .. toctree:: :maxdepth: 2 + python + framework_dependencies config config_logging production - scaling nginx apache + scaling cluster jobs - tool_panel authentication + tool_panel dependency_resolvers conda_faq - framework_dependencies - options reports useful_scripts + options diff --git a/doc/source/admin/python.md b/doc/source/admin/python.md new file mode 100644 index 000000000000..018bff5319ab --- /dev/null +++ b/doc/source/admin/python.md @@ -0,0 +1,37 @@ +# Supported Python versions + +Galaxy's core functionality is currently supported on Python **2.7** (although +deprecated) and **>=3.5** . + +If Galaxy complains about the version of Python you are using, check that +`python --version` reports a supported version. If this is not the case: + +1. Completely remove the virtualenv used by Galaxy, e.g. with: + `rm -rf /path/to/galaxy/.venv` + +2. Let Galaxy know the path of the correct version of Python. + + - If you are using Galaxy >= 20.05, just execute: + `export GALAXY_PYTHON=/path/to/python` + - If instead you are using an older version of Galaxy, you can manipulate + your shell's `$PATH` variable to place the correct version first. This can + be done for just Python by creating a new directory, adding a symbolic + link to python in there, and putting that directory at the front of + `$PATH`: + + ```sh + % mkdir ~/galaxy-python + % ln -s /path/to/python ~/galaxy-python/python + % export PATH=~/galaxy-python:$PATH + ``` + +3. Start Galaxy again. + +N.B. If you have compiled your own Python interpreter from source, please ensure +that the `ssl`, `sqlite3`, `curses` and `bz2` modules were built and can be +imported after installation. These "extra" modules are built at the end of the +compilation process and are required by the Galaxy framework. If building on +Linux, you may need to install the appropriate `-dev` packages for OpenSSL and +Bzip2. You may also need to build Python with shared libraries +(`--enable-shared`). + diff --git a/run_tests.sh b/run_tests.sh index 77c7871043c8..7e6bdb798f2b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -1,4 +1,3 @@ - #!/bin/sh cd "$(dirname "$0")" @@ -313,7 +312,7 @@ then echo "Docker version:" docker --version echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." - name=$(python -c 'import re; import uuid; print re.sub("-","",str(uuid.uuid4()))') + name=$(python -c 'import re; import uuid; print re.sub("-", "", str(uuid.uuid4()))') # Create a cache dir for pip, so it has the right owner DOCKER_PIP_CACHE_DIR="$HOME"/.cache/docker_galaxy_pip mkdir -p "$DOCKER_PIP_CACHE_DIR" diff --git a/scripts/check_python.py b/scripts/check_python.py index 2a5175e29e65..adccedf27b0b 100644 --- a/scripts/check_python.py +++ b/scripts/check_python.py @@ -6,20 +6,28 @@ import sys -version_string = '.'.join(str(_) for _ in sys.version_info[:3]) - -msg = """ERROR: Your Python version is: %s -Galaxy is currently supported on Python 2.7 and >=3.5 . To run Galaxy, please -install a supported Python version. If a supported version is already -installed but is not your default, https://galaxyproject.org/admin/python/ -contains instructions on how to force Galaxy to use a different version.""" % version_string - def check_python(): - if sys.version_info[:2] == (2, 7) or sys.version_info[:2] >= (3, 5): + if sys.version_info[:2] >= (3, 5): # supported return + elif sys.version_info[:2] == (2, 7): + msg = """\ +Galaxy support for Python 2.7 is deprecated, please consider moving to +Python >= 3.5 . +https://docs.galaxyproject.org/en/latest/admin/python.html contains instructions +on how to force Galaxy to use a different version.""" + print(msg, file=sys.stderr) + return else: + version_string = '.'.join(str(_) for _ in sys.version_info[:3]) + msg = """\ +ERROR: Your Python version is: %s +Galaxy is currently supported on Python 2.7 (although deprecated) and >=3.5 . +To run Galaxy, please install a supported Python version. +If a supported version is already installed but is not your default, +https://docs.galaxyproject.org/en/latest/admin/python.html contains instructions +on how to force Galaxy to use a different version.""" % version_string print(msg, file=sys.stderr) raise Exception(msg) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index df3656e9cf41..51a574de2c1f 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -121,7 +121,7 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV" echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an" echo "existing environment before starting Galaxy." - $CONDA_EXE create --yes --override-channels --channel conda-forge --channel defaults --name "$GALAXY_CONDA_ENV" 'python=2.7' 'pip>=9' 'virtualenv>=16' + $CONDA_EXE create --yes --override-channels --channel conda-forge --channel defaults --name "$GALAXY_CONDA_ENV" 'python=3.6' 'pip>=9' 'virtualenv>=16' unset __CONDA_INFO fi conda_activate @@ -133,13 +133,20 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then echo "Creating Python virtual environment for Galaxy: $GALAXY_VIRTUAL_ENV" echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_VIRTUAL_ENV to an" echo "existing environment before starting Galaxy." - python ./scripts/check_python.py || exit 1 + if [ -z "$GALAXY_PYTHON" ]; then + if command -v python3 >/dev/null; then + GALAXY_PYTHON=python3 + else + GALAXY_PYTHON=python + fi + fi + "$GALAXY_PYTHON" ./scripts/check_python.py || exit 1 if command -v virtualenv >/dev/null; then - virtualenv -p "$(command -v python)" "$GALAXY_VIRTUAL_ENV" + virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV" else - vvers=16.1.0 + vvers=16.7.9 vurl="https://files.pythonhosted.org/packages/source/v/virtualenv/virtualenv-${vvers}.tar.gz" - vsha=f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208 + vsha=0d62c70883c0342d59c11d0ddac0d954d0431321a41ab20851facf2b222598f3 vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX) vsrc="$vtmp/$(basename $vurl)" # SSL certificates are not checked to prevent problems with messed @@ -151,16 +158,16 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then elif command -v wget >/dev/null; then wget --no-check-certificate -O "$vsrc" "$vurl" else - python -c "try: + "$GALAXY_PYTHON" -c "try: from urllib import urlretrieve except: from urllib.request import urlretrieve urlretrieve('$vurl', '$vsrc')" fi echo "Verifying $vsrc checksum is $vsha" - python -c "import hashlib; assert hashlib.sha256(open('$vsrc', 'rb').read()).hexdigest() == '$vsha', '$vsrc: invalid checksum'" + "$GALAXY_PYTHON" -c "import hashlib; assert hashlib.sha256(open('$vsrc', 'rb').read()).hexdigest() == '$vsha', '$vsrc: invalid checksum'" tar zxf "$vsrc" -C "$vtmp" - python "$vtmp/virtualenv-$vvers/src/virtualenv.py" "$GALAXY_VIRTUAL_ENV" + "$GALAXY_PYTHON" "$vtmp/virtualenv-$vvers/virtualenv.py" "$GALAXY_VIRTUAL_ENV" rm -rf "$vtmp" fi fi diff --git a/tox.ini b/tox.ini index 6db822ade1a5..ebc9eef4a898 100644 --- a/tox.ini +++ b/tox.ini @@ -1,17 +1,21 @@ [tox] # envlist is the list of environments that are tested when `tox` is run without any option # hyphens in an environment name are used to delimit factors -envlist = check_py3_compatibility, py{27,35}-first_startup, py{27,35}-lint, py27-lint_docstring_include_list, py{27,35}-unit, test_galaxy_packages, validate_test_tools +envlist = check_py3_compatibility, py{27,35}-first_startup, py{27,35}-lint, py35-lint_docstring_include_list, py{27,35}-unit, test_galaxy_packages, validate_test_tools skipsdist = True [testenv] commands = first_startup: bash .ci/first_startup.sh lint: bash .ci/flake8_wrapper.sh + lint_docstring: bash .ci/flake8_wrapper_docstrings.sh --exclude + lint_docstring_include_list: bash .ci/flake8_wrapper_docstrings.sh --include + unit: bash run_tests.sh -u whitelist_externals = bash passenv = CI setenv = + first_startup: GALAXY_PYTHON=python py{35,36,37}-first_startup: GALAXY_VIRTUAL_ENV=.venv3 unit: GALAXY_VIRTUAL_ENV={envdir} unit: GALAXY_ENABLE_BETA_COMPRESSED_GENBANK_SNIFFING=1 @@ -34,12 +38,6 @@ whitelist_externals = make [testenv:mako_count] commands = bash .ci/check_mako.sh -[testenv:py27-lint_docstring] -commands = bash .ci/flake8_wrapper_docstrings.sh --exclude - -[testenv:py27-lint_docstring_include_list] -commands = bash .ci/flake8_wrapper_docstrings.sh --include - [testenv:test_galaxy_packages] commands = bash packages/test.sh From e209777066c658e69249e8abd5f3c9c0ae9a38c4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sun, 9 Feb 2020 16:03:28 +0000 Subject: [PATCH 2/2] Python3: decode the output of ``subprocess.check_output()`` --- doc/source/conf.versioning.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/conf.versioning.py b/doc/source/conf.versioning.py index 65a85d7fd2dd..8b112f7afcad 100644 --- a/doc/source/conf.versioning.py +++ b/doc/source/conf.versioning.py @@ -43,7 +43,8 @@ _target_ver = TARGET_GIT_BRANCH[len('release_'):] # Use tags to determine versions - a stable version will have a branch before it's released, but not a tag. -for _tag in reversed(check_output(('git', 'tag')).splitlines()): +tags = check_output(('git', 'tag')).decode().splitlines() +for _tag in reversed(tags): if _tag.startswith('v') and _tag.count('.') == 1: # this version is released _ver = _tag[1:]