Start migration to azure for CI (bioconda#31689)

* Start migration to azure for CI * Update meta.yaml * Update meta.yaml * typo * OK * right, not github actions * bash * ok * typo * needs channels of course * strip v from version * kick it old school * Try tweaking checkout * change checkout * try again * Pull master from upstream too * where is the conda env * Update conda path for caching * ensure failed lint fails the pipeline * Good, failing lints fail the CI * Linux Test * ok * ok * sequential stages * Should fail * implicit jobs * reformat * reformat * debug * umm * ah * try again * oops * right have to make the directory * channels * debug * debug * debug * so python is problematic * maybe py2 works * maybe noarch * ok * pin down conda version * wtf * debug * env * try conda init * wtf is the shell? * right, that won't take immediate effect * move up conda init bash * more testing * foo * oh * add OSX to azure * chown * test artifacts, only one python * debug * oops * debug artifacts * more improvements * clean up * osx * oops * artifacts work * Update meta.yaml * Update azure-pipeline.yml * change checkout * verbose * more testing * right, HEAD is a problem * probably just need to modify the environment variable * maybe * can we avoid using HEAD? * Update azure-pipeline.yml * ok * Update meta.yaml Co-authored-by: Devon Ryan <[email protected]>
jscaber · Dec 19, 2021 · 696aece · 696aece
1 parent 0154db3
commit 696aece
Show file tree

Hide file tree

Showing 2 changed files with 218 additions and 0 deletions.
diff --git a/.azure-pipelines/createEnv.py b/.azure-pipelines/createEnv.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+import subprocess
+import os
+import sys
+
+def parseCommonSH():
+    """Parse common.sh in the current working directory, returning a dictionary
+       version of its key=value pairs"""
+    d = {}
+    for line in open('common.sh'):
+        cols = line.strip().split('=')
+        if len(cols) == 2:
+            d[cols[0]] = cols[1]
+    return d
+
+
+if not os.path.exists('common.sh'):
+    sys.exit("Can't find common.sh!")
+
+common = parseCommonSH()
+CMD = ['conda', 'create', '-n', 'bioconda', '-y', '-c', 'conda-forge', '-c', 'bioconda', 'bioconda-utils={}'.format(common['BIOCONDA_UTILS_TAG'].strip('v'))]
+if sys.platform != 'linux':
+    CMD.append('conda-forge-ci-setup=2.6.0')
+
+subprocess.check_call(CMD)
diff --git a/azure-pipeline.yml b/azure-pipeline.yml
@@ -0,0 +1,193 @@
+pr:
+  autoCancel: true
+trigger:
+  branches:
+    exclude:
+      - master
+
+stages:
+- stage: 'lint'
+  jobs:
+  - job: 'lint'  
+    pool:
+      vmImage: 'ubuntu-latest'
+    strategy:
+      maxParallel: 8
+
+    steps:
+    - script: |
+        git remote add upstream https://github.com/bioconda/bioconda-recipes
+        git fetch upstream master
+        source=`echo $(Build.SourceBranch) | sed -e "s#^refs/pull/#pull/#"`
+        git checkout ${source#"refs/heads/"}
+
+    - bash: echo "##vso[task.prependpath]$CONDA/bin"
+      displayName: Add conda to PATH
+
+    - bash: |
+        wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh
+      displayName: Fetch common.sh
+
+    - task: Cache@2
+      inputs:
+        path: "/usr/share/miniconda/envs/bioconda"
+        key: '"$(Agent.OS)" | **/common.sh'
+        cacheHitVar: CACHE_RESTORED
+      displayName: Restore cache
+
+    - script: .azure-pipelines/createEnv.py
+      condition: ne(variables.CACHE_RESTORED, 'true')
+      displayName: Install bioconda-utils
+
+    - bash: |
+          source activate bioconda
+          set -e
+          source=`echo $(Build.SourceBranch) | sed -e "s#^refs/pull/#pull/#"`
+          echo "source is $source"
+          bioconda-utils lint --loglevel debug --full-report --git-range upstream/master "$source"
+      displayName: Lint
+
+- stage: 'test_linux'
+  jobs:
+  - job: 'test_linux'
+    pool:
+      vmImage: 'ubuntu-latest'
+    strategy:
+      maxParallel: 8
+
+    steps:
+    - script: |
+        git remote add upstream https://github.com/bioconda/bioconda-recipes
+        git fetch upstream master
+        git branch
+        source=$(Build.SourceBranch)
+        git checkout ${source#"refs/heads/"}
+
+    - bash: echo "##vso[task.prependpath]$CONDA/bin"
+      displayName: Add conda to PATH
+
+    - bash: |
+        wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh
+      displayName: Fetch common.sh
+
+    - task: Cache@2
+      inputs:
+        path: "/usr/share/miniconda/envs/bioconda"
+        key: '"$(Agent.OS)" | **/common.sh'
+        cacheHitVar: CACHE_RESTORED
+      displayName: Restore cache
+
+    - script: .azure-pipelines/createEnv.py
+      condition: ne(variables.CACHE_RESTORED, 'true')
+      displayName: Install bioconda-utils
+
+    - bash: |
+        set -e
+
+        rm -f "${HOME}/.condarc"
+        # Activate our Conda installation as "base".
+        . /usr/share/miniconda/envs/bioconda/etc/profile.d/conda.sh
+        conda activate base
+
+        conda config --system --add channels defaults
+        conda config --system --add channels bioconda
+        conda config --system --add channels conda-forge
+        docker pull quay.io/dpryan79/mulled_container:latest
+        mkdir -p ${CONDA_PREFIX}/conda-bld
+
+        bioconda-utils build recipes config.yml \
+            --docker --mulled-test \
+            --git-range upstream/master HEAD
+        docker rmi quay.io/dpryan79/mulled_container:latest
+      displayName: Test
+
+    - bash: |
+        (
+        mkdir -p /tmp/artifacts/packages
+        cd /usr/share/miniconda/envs/bioconda/conda-bld || exit 0
+        find -name .cache | xargs rm -rf || true
+        for n in index.html channeldata.json linux-64 osx-64 noarch; do
+          cp -rv $n /tmp/artifacts/packages || true
+        done
+        if command -V docker >/dev/null; then
+          mkdir -p /tmp/artifacts/images
+          cd /tmp/artifacts/images
+          docker image ls --format='{{.Repository}}:{{.Tag}}' | \
+            { grep biocontainers || true ; } | \
+            xargs -n1 -P4 bash -c '
+              test -n "${1+x}" || exit 0
+              echo "Start compressing docker image ${1} ..."
+              docker save "${1}" | gzip -c > "${1##*/}.tar.gz"
+              echo "Finished compressing docker image ${1} ."
+              ' --
+        fi
+        ) || true
+      displayName: Prepare artifacts
+
+    - publish: /tmp/artifacts
+      artifact: LinuxArtifacts
+
+- stage: 'test_osx'
+  jobs:
+  - job: 'test_osx'  
+    pool:
+      vmImage: 'macOS-latest'
+    strategy:
+      maxParallel: 4
+
+    steps:
+    - script: |
+        git remote add upstream https://github.com/bioconda/bioconda-recipes
+        git fetch upstream master
+        source=$(Build.SourceBranch)
+        git checkout ${source#"refs/heads/"}
+
+    - bash: echo "##vso[task.prependpath]$CONDA/bin"
+      displayName: Add conda to PATH
+
+    - bash: sudo chown -R $USER $CONDA
+      displayName: Take ownership of conda installation
+
+    - bash: |
+        wget https://raw.githubusercontent.com/bioconda/bioconda-common/master/common.sh
+      displayName: Fetch common.sh
+
+    - task: Cache@2
+      inputs:
+        path: "/usr/local/miniconda/envs/bioconda"
+        key: '"$(Agent.OS)" | **/common.sh'
+        cacheHitVar: CACHE_RESTORED
+      displayName: Restore cache
+
+    - script: .azure-pipelines/createEnv.py
+      condition: ne(variables.CACHE_RESTORED, 'true')
+      displayName: Install bioconda-utils
+
+    - bash: |
+        set -e
+
+        . /usr/local/miniconda/envs/bioconda/etc/profile.d/conda.sh
+        conda activate base
+
+        conda config --system --add channels defaults
+        conda config --system --add channels bioconda
+        conda config --system --add channels conda-forge
+        mkdir -p ${CONDA_PREFIX}/conda-bld
+
+        bioconda-utils build recipes config.yml \
+            --git-range upstream/master HEAD
+      displayName: Test
+
+    - bash: |
+        (
+        mkdir -p /tmp/artifacts/packages
+        cd /usr/local/miniconda/envs/bioconda/conda-bld || exit 0
+        find -name .cache | xargs rm -rf || true
+        for n in index.html channeldata.json linux-64 osx-64 noarch; do
+          cp -rv $n /tmp/artifacts/packages || true
+        done
+        ) || true
+      displayName: Prepare artifacts
+
+    - publish: /tmp/artifacts
+      artifact: OSXArtifacts