[ENH] Add singularity and docker to CI tests (#30)

.circleci/config.yml

@@ -86,6 +86,118 @@ jobs:
           paths:
             - miniconda
 
+  get_singularity:
+    machine:
+      image: ubuntu-1604:202004-01
+    steps:
+      - restore_cache:
+          keys:
+            - singularity-v3-{{ .Branch }}-{{ .Revision }}
+
+      - run:
+          name: Download Singularity
+          command: |
+            if [ ! -f /usr/local/bin/singularity ]
+            then
+              sudo apt-get update && sudo apt-get -y install build-essential \
+                libssl-dev \
+                uuid-dev \
+                libgpgme11-dev \
+                squashfs-tools \
+                libseccomp-dev \
+                pkg-config \
+                wget
+              export GOPATH=$HOME/go
+              mkdir -p $GOPATH
+              go get -u github.com/golang/dep/cmd/dep
+              export VERSION=3.6.4
+              wget https://github.com/sylabs/singularity/releases/download/v${VERSION}/singularity-${VERSION}.tar.gz
+              tar -xzf singularity-${VERSION}.tar.gz
+              cd ./singularity
+              ./mconfig && \
+                make -j2 -C ./builddir
+              cd $HOME
+              tar cfz singularity.tar.gz project/singularity/
+              tar cfz go.tar.gz go/
+            fi
+
+      - save_cache:
+         key: singularity-v3-{{ .Branch }}-{{ .Revision }}-{{ epoch }}
+         paths:
+            - /home/circleci/singularity.tar.gz
+            - /home/circleci/go.tar.gz
+
+
+  install_and_test_singularity:
+    machine:
+      image: ubuntu-1604:202004-01
+    working_directory: /home/circleci/src/BOnD
+    steps:
+      - checkout:
+          path: /home/circleci/src/BOnD
+
+      - attach_workspace:
+          at: /tmp
+
+      - restore_cache:
+          keys:
+            - docker-v1-{{ .Branch }}-{{ .Revision }}
+
+      - restore_cache:
+          keys:
+            - singularity-v3-{{ .Branch }}-{{ .Revision }}
+      - run:
+          name: Load Docker image layer cache
+          no_output_timeout: 30m
+          command: |
+            docker info
+            set +o pipefail
+            if [ -f /tmp/cache/docker.tar.gz ]; then
+              sudo apt update && sudo apt -y install pigz
+              pigz -d --stdout /tmp/cache/docker.tar.gz | docker load
+              docker images
+            fi
+
+      - run:
+          name: Install BOnD
+          command: |
+            export PATH=/tmp/miniconda/bin:$PATH
+            source activate bond
+            pip install .
+
+      - run:
+          name: Install Singularity
+          command: |
+              export GOPATH=$HOME/go
+              cd /home/circleci
+              tar xfz go.tar.gz
+              tar xfz singularity.tar.gz
+              cd project/singularity/builddir
+              sudo make install
+
+      - run:
+          name: Build singularity image
+          no_output_timeout: 30m
+          command: |
+              singularity build \
+                /home/circleci/bond-latest.sif \
+                docker-daemon://pennlinc/bond:latest
+
+      - run:
+          name: Test singularity
+          command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "CircleCI Test"
+            export PATH=/tmp/miniconda/bin:$PATH
+            source activate bond
+            mkdir -p /tmp/bids /tmp/group_testing
+            cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/singularity
+            bond-group \
+              /tmp/bids/singularity \
+              /tmp/group_testing/direct \
+              --container /home/circleci/bond-latest.sif
+
+
 
   install_and_test:
     machine:
@@ -119,14 +231,21 @@ jobs:
           command: |
             export PATH=/tmp/miniconda/bin:$PATH
             source activate bond
-            pip install .[all]
+            pip install .
 
       - run:
-          name: Run PyTest
+          name: Test Docker integration
           command: |
+            git config --global user.email "[email protected]"
+            git config --global user.name "CircleCI Test"
             export PATH=/tmp/miniconda/bin:$PATH
             source activate bond
-            py.test -sv tests
+            mkdir -p /tmp/bids /tmp/group_testing
+            cp -r /home/circleci/src/BOnD/bond/testdata/complete /tmp/bids/docker
+            bond-group \
+              /tmp/bids/docker \
+              /tmp/group_testing/docker \
+              --container pennlinc/bond:latest
 
   build_docs:
     docker:
@@ -146,7 +265,7 @@ jobs:
           name: Check Python version and upgrade pip
           command: |
             python --version
-            sudo python -m pip install -U pip 
+            sudo python -m pip install -U pip
             sudo pip install flake8
 
       - run:
@@ -157,7 +276,7 @@ jobs:
       - run:
           name: Install BOnD.
           command: sudo python -m pip install ".[doc]" --no-cache-dir --progress-bar off
-    
+
       - run:
           name: Build documentation
           no_output_timeout: 45m
@@ -352,6 +471,30 @@ workflows:
             tags:
               only: /.*/
 
+      - get_singularity:
+          requires:
+            - build
+          filters:
+            branches:
+              ignore:
+                - /tests?\/.*/
+                - /docker\/.*/
+            tags:
+              only: /.*/
+
+      - install_and_test_singularity:
+          requires:
+            - build
+            - setup_conda
+            - get_singularity
+          filters:
+            branches:
+              ignore:
+                - /tests?\/.*/
+                - /docker\/.*/
+            tags:
+              only: /.*/
+
       - build_docs:
           filters:
             branches:
@@ -365,6 +508,7 @@ workflows:
           requires:
             - build_docs
             - install_and_test
+            - install_and_test_singularity
           filters:
             branches:
               only: master

.gitignore

@@ -135,3 +135,4 @@ data/*
 .DS_Store
 notebooks/testdata
 .vscode/settings.json
+notebooks/test1

Dockerfile

@@ -1,44 +1,35 @@
-FROM python:3
-
-# install conda
-RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh && \
-    bash Miniconda3-4.5.12-Linux-x86_64.sh -b -p /usr/local/miniconda && \
-    rm Miniconda3-4.5.12-Linux-x86_64.sh
-
-ENV PATH=/usr/local/miniconda/bin:$PATH
-
-# activate conda environment
-RUN echo "source activate base" > ~/.bashrc
-
-RUN which conda
+FROM ubuntu:bionic-20200921
 
 # get the validator branch skip_session_check
 RUN apt-get update && \
-    apt-get install -y git
+    apt-get install -y --no-install-recommends \
+    curl ca-certificates && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# USE CONDA FOR INSTALLING NPM
-RUN conda install nodejs
+# Installing Neurodebian and nodejs packages
+COPY neurodebian.gpg /usr/local/etc/neurodebian.gpg
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
+    curl -sSL "http://neuro.debian.net/lists/$( lsb_release -c | cut -f2 ).us-ca.full" >> /etc/apt/sources.list.d/neurodebian.sources.list && \
+    apt-key add /usr/local/etc/neurodebian.gpg && \
+    (apt-key adv --refresh-keys --keyserver hkp://ha.pool.sks-keyservers.net 0xA5D32F012649A5A9 || true)
 
-RUN npm --version
+# get the validator branch skip_session_check
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    datalad nodejs python3 python3-pip python3-setuptools && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-RUN npm install -g yarn
 
-RUN mkdir -p /home/validator && \
+RUN npm install -g yarn && \
+   mkdir -p /home/validator && \
     cd /home/validator && \
-    git clone -b skip_session_checks --single-branch https://github.com/bids-standard/bids-validator.git
-
-
-RUN ls /home/validator/bids-validator
-RUN cd /home/validator/bids-validator && \
+    git clone -b skip_session_checks \
+        --single-branch https://github.com/bids-standard/bids-validator.git  && \
+    cd /home/validator/bids-validator && \
     yarn && \
-    npm install -g bids-validator
-
-RUN which bids-validator
-
-# prepare env
-COPY requirements.txt ./
-RUN pip install --no-cache-dir -r requirements.txt
+    cd bids-validator && npm install -g
 
-COPY . .
+COPY . /src/BOnD
+RUN pip3 install --no-cache-dir "/src/BOnD"
 
-ENTRYPOINT [ "bids-validator"]
+ENTRYPOINT [ "/bin/bash"]