diff --git a/.azure-pipelines/createEnv.py b/.azure-pipelines/createEnv.py new file mode 100755 index 0000000000000..9cb904f8bf315 --- /dev/null +++ b/.azure-pipelines/createEnv.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +import subprocess +import os +import sys + +def parseCommonSH(): + """Parse common.sh in the current working directory, returning a dictionary + version of its key=value pairs""" + d = {} + for line in open('common.sh'): + cols = line.strip().split('=') + if len(cols) == 2: + d[cols[0]] = cols[1] + return d + + +if not os.path.exists('common.sh'): + sys.exit("Can't find common.sh!") + +common = parseCommonSH() +CMD = ['conda', 'create', '-n', 'bioconda', '-y', '-c', 'conda-forge', '-c', 'bioconda', 'bioconda-utils={}'.format(common['BIOCONDA_UTILS_TAG'].strip('v'))] +if sys.platform != 'linux': + CMD.append('conda-forge-ci-setup=2.6.0') + +subprocess.check_call(CMD) diff --git a/azure-pipeline.yml b/azure-pipeline.yml new file mode 100644 index 0000000000000..eea03b68cfa18 --- /dev/null +++ b/azure-pipeline.yml @@ -0,0 +1,193 @@ +pr: + autoCancel: true +trigger: + branches: + exclude: + - master + +stages: +- stage: 'lint' + jobs: + - job: 'lint' + pool: + vmImage: 'ubuntu-latest' + strategy: + maxParallel: 8 + + steps: + - script: | + git remote add upstream https://github.com/bioconda/bioconda-recipes + git fetch upstream master + source=`echo $(Build.SourceBranch) | sed -e "s#^refs/pull/#pull/#"` + git checkout ${source#"refs/heads/"} + + - bash: echo "##vso[task.prependpath]$CONDA/bin" + displayName: Add conda to PATH + + - bash: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh + displayName: Fetch common.sh + + - task: Cache@2 + inputs: + path: "/usr/share/miniconda/envs/bioconda" + key: '"$(Agent.OS)" | **/common.sh' + cacheHitVar: CACHE_RESTORED + displayName: Restore cache + + - script: .azure-pipelines/createEnv.py + condition: ne(variables.CACHE_RESTORED, 'true') + displayName: Install bioconda-utils + + - bash: | + source activate bioconda + set -e + source=`echo $(Build.SourceBranch) | sed -e "s#^refs/pull/#pull/#"` + echo "source is $source" + bioconda-utils lint --loglevel debug --full-report --git-range upstream/master "$source" + displayName: Lint + +- stage: 'test_linux' + jobs: + - job: 'test_linux' + pool: + vmImage: 'ubuntu-latest' + strategy: + maxParallel: 8 + + steps: + - script: | + git remote add upstream https://github.com/bioconda/bioconda-recipes + git fetch upstream master + git branch + source=$(Build.SourceBranch) + git checkout ${source#"refs/heads/"} + + - bash: echo "##vso[task.prependpath]$CONDA/bin" + displayName: Add conda to PATH + + - bash: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh + displayName: Fetch common.sh + + - task: Cache@2 + inputs: + path: "/usr/share/miniconda/envs/bioconda" + key: '"$(Agent.OS)" | **/common.sh' + cacheHitVar: CACHE_RESTORED + displayName: Restore cache + + - script: .azure-pipelines/createEnv.py + condition: ne(variables.CACHE_RESTORED, 'true') + displayName: Install bioconda-utils + + - bash: | + set -e + + rm -f "${HOME}/.condarc" + # Activate our Conda installation as "base". + . /usr/share/miniconda/envs/bioconda/etc/profile.d/conda.sh + conda activate base + + conda config --system --add channels defaults + conda config --system --add channels bioconda + conda config --system --add channels conda-forge + docker pull quay.io/dpryan79/mulled_container:latest + mkdir -p ${CONDA_PREFIX}/conda-bld + + bioconda-utils build recipes config.yml \ + --docker --mulled-test \ + --git-range upstream/master HEAD + docker rmi quay.io/dpryan79/mulled_container:latest + displayName: Test + + - bash: | + ( + mkdir -p /tmp/artifacts/packages + cd /usr/share/miniconda/envs/bioconda/conda-bld || exit 0 + find -name .cache | xargs rm -rf || true + for n in index.html channeldata.json linux-64 osx-64 noarch; do + cp -rv $n /tmp/artifacts/packages || true + done + if command -V docker >/dev/null; then + mkdir -p /tmp/artifacts/images + cd /tmp/artifacts/images + docker image ls --format='{{.Repository}}:{{.Tag}}' | \ + { grep biocontainers || true ; } | \ + xargs -n1 -P4 bash -c ' + test -n "${1+x}" || exit 0 + echo "Start compressing docker image ${1} ..." + docker save "${1}" | gzip -c > "${1##*/}.tar.gz" + echo "Finished compressing docker image ${1} ." + ' -- + fi + ) || true + displayName: Prepare artifacts + + - publish: /tmp/artifacts + artifact: LinuxArtifacts + +- stage: 'test_osx' + jobs: + - job: 'test_osx' + pool: + vmImage: 'macOS-latest' + strategy: + maxParallel: 4 + + steps: + - script: | + git remote add upstream https://github.com/bioconda/bioconda-recipes + git fetch upstream master + source=$(Build.SourceBranch) + git checkout ${source#"refs/heads/"} + + - bash: echo "##vso[task.prependpath]$CONDA/bin" + displayName: Add conda to PATH + + - bash: sudo chown -R $USER $CONDA + displayName: Take ownership of conda installation + + - bash: | + wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh + displayName: Fetch common.sh + + - task: Cache@2 + inputs: + path: "/usr/local/miniconda/envs/bioconda" + key: '"$(Agent.OS)" | **/common.sh' + cacheHitVar: CACHE_RESTORED + displayName: Restore cache + + - script: .azure-pipelines/createEnv.py + condition: ne(variables.CACHE_RESTORED, 'true') + displayName: Install bioconda-utils + + - bash: | + set -e + + . /usr/local/miniconda/envs/bioconda/etc/profile.d/conda.sh + conda activate base + + conda config --system --add channels defaults + conda config --system --add channels bioconda + conda config --system --add channels conda-forge + mkdir -p ${CONDA_PREFIX}/conda-bld + + bioconda-utils build recipes config.yml \ + --git-range upstream/master HEAD + displayName: Test + + - bash: | + ( + mkdir -p /tmp/artifacts/packages + cd /usr/local/miniconda/envs/bioconda/conda-bld || exit 0 + find -name .cache | xargs rm -rf || true + for n in index.html channeldata.json linux-64 osx-64 noarch; do + cp -rv $n /tmp/artifacts/packages || true + done + ) || true + displayName: Prepare artifacts + + - publish: /tmp/artifacts + artifact: OSXArtifacts