From de00fef72644f49e4877f5ec60bcd06218b777bc Mon Sep 17 00:00:00 2001 From: Shashank Bansal Date: Tue, 21 Sep 2021 11:04:25 -0700 Subject: [PATCH] replace circleci config --- .circleci/config.yml | 530 +------------------------------------------ 1 file changed, 4 insertions(+), 526 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8ba541d..a4908677 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,536 +1,14 @@ version: 2.1 jobs: - build: - docker: - - image: docker:18.01.0-ce-git - working_directory: /tmp/src/fitlins - steps: - - run: - name: Install parallel gzip and Python 3 - command: | - apk add --no-cache pigz python3 - - checkout - - setup_remote_docker - - run: - name: Fetch latest release image - no_output_timeout: 10m - command: | - docker pull poldracklab/fitlins:latest - - run: - name: Build image - no_output_timeout: 30m - command: | - THISVERSION=$( python3 -c 'import fitlins; print(fitlins.__version__)' ) - echo "Building version ${CIRCLE_TAG:-$THISVERSION}" - docker build \ - --cache-from=poldracklab/fitlins \ - --rm=false \ - --tag poldracklab/fitlins:latest \ - --build-arg BUILD_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" \ - --build-arg VCS_REF="$(git rev-parse --short HEAD)" \ - --build-arg VERSION="${CIRCLE_TAG:-$THISVERSION}" . - - run: - name: Verify executable can be run - command: docker run --rm poldracklab/fitlins:latest --help - - run: - name: Check version - command: | - THISVERSION=$( python3 -c 'import fitlins; print(fitlins.__version__)' ) - THISVERSION="${CIRCLE_TAG:-$THISVERSION}" - DOCKERVERSION=$(docker run --rm poldracklab/fitlins:latest --version \ - | tail -n 1 | sed -e 's/.*fit/fit/' -e 's/[\r\n]//g') - echo "$THISVERSION" - echo "$DOCKERVERSION" - test "$DOCKERVERSION" = "fitlins v$THISVERSION" - - run: - name: Docker save - no_output_timeout: 40m - command: | - mkdir -p /tmp/cache - docker save poldracklab/fitlins:latest \ - | pigz -2 -p 3 > /tmp/cache/docker.tar.gz - - persist_to_workspace: - root: /tmp - paths: - - src/fitlins - - save_cache: - key: docker-v3-{{ .Branch }}-{{ .Revision }}-{{ epoch }} - paths: - - /tmp/cache/docker.tar.gz - - cache_test_data: - docker: - - image: markiewicz/datalad:0.0.1 - working_directory: /tmp/data - steps: - - checkout: - path: /tmp/src - - restore_cache: - keys: - - ds003-v3-{{ .Branch }}- - - ds003-v3-master- - - ds003-v3- - - run: - name: Set git ID - command: | - git config --global user.name 'FitLins CircleCI User' - git config --global user.email 'crn.poldracklab@gmail.com' - - run: - name: Install fMRIPrep derivatives of ds000003 and reference data - command: | - datalad install -r -s https://gin.g-node.org/shashankbansal56/fitlins_tests - datalad update --merge fitlins_tests - datalad get fitlins_tests/ds003/fmriprep/sub-0{1,2,3}/func/*_space-MNI152NLin2009cAsym_desc-*.nii.gz \ - fitlins_tests/ds003/fmriprep/sub-0{1,2,3}/func/*_desc-confounds_*.tsv \ - fitlins_tests/ds003/fmriprep/dataset_description.json \ - fitlins_tests/ds003/fmriprep/sub-*/*/*.json - datalad get -r fitlins_tests/ds003/nistats_smooth/ fitlins_tests/ds003/afni_smooth/ fitlins_tests/ds003/afni_blurto/ fitlins_tests/ds003/nistats_blurto/ - - save_cache: - key: ds003-v3-{{ .Branch }}-{{ .BuildNum }} - paths: - - /tmp/data - - build_docs: - docker: - - image: python:3.7.4 - working_directory: /tmp/src/fitlins - steps: - - run: - name: Check Python version and upgrade pip - command: | - python --version - python -m pip install -U pip setuptools - - run: - name: Install pandoc - command: | - apt-get update - apt-get install -y pandoc - - checkout - - run: - name: Install FitLins - command: | - python -m pip install ".[docs]" - - run: - name: Build docs - command: | - make html - - store_artifacts: - path: /tmp/src/fitlins/docs/_build/html - - run_pytest: - machine: - image: circleci/classic:201711-01 - working_directory: /tmp/src/fitlins - steps: - - attach_workspace: - at: /tmp - - restore_cache: - keys: - - docker-v3-{{ .Branch }}-{{ .Revision }} - - run: - name: Load Docker image layer cache - no_output_timeout: 30m - command: | - docker info - set +o pipefail - if [ -f /tmp/cache/docker.tar.gz ]; then - sudo apt update && sudo apt -y install pigz - pigz -d --stdout /tmp/cache/docker.tar.gz | docker load - docker images - fi - - run: - name: Run pytest - no_output_timeout: 2h - command: | - mkdir -p /tmp/pytestcov - chmod 777 /tmp/pytestcov - docker run --rm \ - -v /tmp/pytestcov:/scratch \ - -e COVERAGE_FILE=/scratch/.coverage.pytest \ - --entrypoint="/neurodocker/startup.sh" \ - poldracklab/fitlins:latest \ - pytest --cov=fitlins --cov-report xml:/scratch/cov_pytest.xml \ - --cov-config /src/fitlins/docker/multiproc.coveragerc \ - --ignore-glob=/src/fitlins/fitlins/tests/* /src/fitlins - - run: - name: Submit pytest coverage - command: | - bash <(curl --retry 5 -s https://codecov.io/bash) \ - -f /tmp/pytestcov/cov_pytest.xml \ - -F pytest \ - -e CIRCLE_JOB - - test_ds003: - parameters: - test_name: - type: string - machine: - image: circleci/classic:201711-01 - working_directory: /tmp/src/fitlins - steps: - - attach_workspace: - at: /tmp - - restore_cache: - keys: - - docker-v3-{{ .Branch }}-{{ .Revision }} - - restore_cache: - keys: - - ds003-v3-{{ .Branch }}- - - ds003-v3-master- - - ds003-v3- - - run: - name: Load Docker image layer cache - no_output_timeout: 30m - command: | - docker info - set +o pipefail - if [ -f /tmp/cache/docker.tar.gz ]; then - sudo apt update && sudo apt -y install pigz - pigz -d --stdout /tmp/cache/docker.tar.gz | docker load - docker images - fi - - run: - name: Run FitLins - no_output_timeout: 4h - command: | - mkdir -p /tmp/ds003/work /tmp/ds003/derivatives - chmod 777 /tmp/ds003/work /tmp/ds003/derivatives - export CONDA_PREFIX=/opt/miniconda-latest/envs/neuro - docker run --rm -v /tmp/data/fitlins_tests:/data:ro \ - -v /tmp/ds003/derivatives:/out \ - -v /tmp/ds003/work:/scratch \ - -v /tmp/src/fitlins:/src/fitlins \ - -e COVERAGE_FILE=/scratch/.coverage_<< parameters.test_name >> \ - --entrypoint="/neurodocker/startup.sh" \ - poldracklab/fitlins:latest \ - pytest --cov=fitlins \ - --cov-config /src/fitlins/docker/multiproc.coveragerc \ - --cov-report xml:/scratch/.coverage_pytest_<< parameters.test_name >>.xml \ - $CONDA_PREFIX/lib/python3.6/site-packages/fitlins/tests \ - --fitlins-path=$CONDA_PREFIX/bin/fitlins \ - --bids-dir=/data/ds003/fmriprep/sourcedata \ - --output-dir=/out \ - --derivatives=/data/ds003/fmriprep \ - --model=/src/fitlins/examples/models/ds000003/models/model-001_smdl.json \ - --work-dir=/scratch \ - --test-name=<< parameters.test_name >> \ - --database-path=/out/ds003_database \ - --reference-dir=/data/ds003 - - run: - name: Combine coverage and submit - command: | - bash <(curl --retry 5 -s https://codecov.io/bash) \ - -f /tmp/ds003/work/.coverage_pytest_<< parameters.test_name >>.xml \ - -F << parameters.test_name >> \ - -e CIRCLE_JOB - - store_artifacts: - path: /tmp/ds003 - - test_database_file: - machine: - image: circleci/classic:201711-01 - working_directory: /tmp/src/fitlins - steps: - - attach_workspace: - at: /tmp - - restore_cache: - keys: - - docker-v3-{{ .Branch }}-{{ .Revision }} - - restore_cache: - keys: - - ds003-v3-{{ .Branch }}- - - ds003-v3-master- - - ds003-v3- - - run: - name: Load Docker image layer cache - no_output_timeout: 30m - command: | - docker info - set +o pipefail - if [ -f /tmp/cache/docker.tar.gz ]; then - sudo apt update && sudo apt -y install pigz - pigz -d --stdout /tmp/cache/docker.tar.gz | docker load - docker images - fi - - run: - name: Run FitLins Database-file - no_output_timeout: 2h - command: | - mkdir -p /tmp/ds003/work /tmp/ds003/derivatives - chmod 777 /tmp/ds003/work /tmp/ds003/derivatives - export CONDA_PREFIX=/opt/miniconda-latest/envs/neuro - docker run --rm -v /tmp/data/fitlins_tests/ds003/fmriprep:/data:ro \ - -v /tmp/ds003/derivatives:/out \ - -v /tmp/ds003/work:/scratch \ - -v /tmp/data/fitlins_tests/ds003/models:/models \ - -e COVERAGE_FILE=/scratch/.coverage.database-file-new \ - --entrypoint="/neurodocker/startup.sh" \ - poldracklab/fitlins:latest \ - coverage run \ - --rcfile /src/fitlins/docker/multiproc.coveragerc \ - $CONDA_PREFIX/bin/fitlins /data/sourcedata /out dataset \ - -d /data -m /models/model-001_smdl.json -w /scratch \ - --participant-label 01 02 03 --space MNI152NLin2009cAsym \ - --n-cpus 2 --database-path /out/pybidsdb -vv - - run: - name: Run FitLins Database-file_exists - no_output_timeout: 2h - command: | - export CONDA_PREFIX=/opt/miniconda-latest/envs/neuro - docker run --rm -v /tmp/data/fitlins_tests/ds003/fmriprep:/data:ro \ - -v /tmp/ds003/derivatives:/out \ - -v /tmp/ds003/work:/scratch \ - -v /tmp/data/fitlins_tests/ds003/models:/models \ - -e COVERAGE_FILE=/scratch/.coverage.database-file-exists \ - --entrypoint="/neurodocker/startup.sh" \ - poldracklab/fitlins:latest \ - coverage run \ - --rcfile /src/fitlins/docker/multiproc.coveragerc \ - $CONDA_PREFIX/bin/fitlins /data/sourcedata /out dataset \ - -d /data -m /models/model-001_smdl.json -w /scratch \ - --participant-label 01 02 03 --space MNI152NLin2009cAsym \ - --n-cpus 2 --database-path /out/pybidsdb -vv - - run: - name: Combine coverage and submit - command: | - # combine coverage - docker run --rm \ - -v /tmp/ds003/work:/scratch \ - -w /scratch \ - --entrypoint="/neurodocker/startup.sh" \ - poldracklab/fitlins:latest \ - bash -c "coverage combine /scratch/.coverage* \ - && coverage xml" - # submit coverage - bash <(curl --retry 5 -s https://codecov.io/bash) \ - -f /tmp/ds003/work/coverage.xml \ - -F database-file \ - -e CIRCLE_JOB - - store_artifacts: - path: /tmp/ds003 - - test_packaging: - docker: - - image: python:3.7.4 - working_directory: /tmp/src/fitlins - steps: - - run: - name: Check Python version and upgrade pip - command: | - python --version - python -m pip install -U pip setuptools virtualenv twine - - checkout - - run: - name: Build FitLins - command: | - virtualenv --python=python /tmp/venv/build - source /tmp/venv/build/bin/activate - pip install --upgrade "pip>=19.1" setuptools wheel - python setup.py sdist - pip wheel --no-deps -w dist/ . - - store_artifacts: - path: /tmp/src/fitlins/dist - - run: - name: Check sdist distribution - command: | - THISVERSION=$( python -c 'import fitlins; print(fitlins.__version__)' ) - THISVERSION="${CIRCLE_TAG:-$THISVERSION}" - twine check dist/fitlins*.tar.gz - virtualenv --python=python /tmp/venv/sdist - source /tmp/venv/sdist/bin/activate - pip install --upgrade "pip>=19.1" setuptools wheel - pip install dist/fitlins*.tar.gz - which fitlins | grep sdist\\/bin - INSTALLED_VERSION=$(fitlins --version) - echo "VERSION: \"$THISVERSION\"" - echo "INSTALLED: \"$INSTALLED_VERSION\"" - test "$INSTALLED_VERSION" = "fitlins v$THISVERSION" - - run: - name: Check wheel distribution - command: | - THISVERSION=$( python -c 'import fitlins; print(fitlins.__version__)' ) - THISVERSION="${CIRCLE_TAG:-$THISVERSION}" - twine check dist/fitlins*.whl - virtualenv --python=python /tmp/venv/wheel - source /tmp/venv/wheel/bin/activate - pip install --upgrade "pip>=19.1" setuptools wheel - pip install dist/fitlins*.whl - which fitlins | grep wheel\\/bin - INSTALLED_VERSION=$(fitlins --version) - echo "VERSION: \"$THISVERSION\"" - echo "INSTALLED: \"$INSTALLED_VERSION\"" - test "$INSTALLED_VERSION" = "fitlins v$THISVERSION" - - deploy_pypi: - docker: - - image: python:3.7.4 - working_directory: /tmp/src/fitlins - steps: - - run: - name: Check Python version and upgrade pip - command: | - python --version - python -m pip install -U pip setuptools virtualenv twine - - checkout - - run: - name: Build FitLins - command: | - virtualenv --python=python /tmp/venv/build - source /tmp/venv/build/bin/activate - pip install --upgrade "pip>=19.1" setuptools wheel - python setup.py sdist - pip wheel --no-deps -w dist/ . - - store_artifacts: - path: /tmp/src/fitlins/dist - - run: - name: Upload packages to PyPI - command: | - twine check dist/fitlins* - twine upload dist/fitlins* - - deploy_master: - machine: - image: circleci/classic:201711-01 - working_directory: /tmp/src/fitlins - steps: - - restore_cache: - keys: - - docker-v3-{{ .Branch }}-{{ .Revision }} - - run: - name: Load Docker image layer cache - no_output_timeout: 30m - command: | - docker info - set +o pipefail - if [ -f /tmp/cache/docker.tar.gz ]; then - zcat /tmp/cache/docker.tar.gz | docker load - docker images - fi - - run: - name: Deploy to Docker Hub - no_output_timeout: 40m - command: | - if [[ -n "$DOCKER_ACCESS_TOKEN" ]]; then - docker login -u $DOCKER_USER -p $DOCKER_ACCESS_TOKEN - docker tag poldracklab/fitlins poldracklab/fitlins:master - docker push poldracklab/fitlins:master - fi - - deploy_docker: - machine: - image: circleci/classic:201711-01 - working_directory: /tmp/src/fitlins - steps: - - restore_cache: - keys: - - docker-v3-{{ .Branch }}-{{ .Revision }} - - run: - name: Load Docker image layer cache - no_output_timeout: 30m - command: | - docker info - set +o pipefail - if [ -f /tmp/cache/docker.tar.gz ]; then - zcat /tmp/cache/docker.tar.gz | docker load - docker images - fi - - run: - name: Deploy to Docker Hub - no_output_timeout: 40m - command: | - if [[ -n "$DOCKER_ACCESS_TOKEN" ]]; then - docker login -u $DOCKER_USER -p $DOCKER_ACCESS_TOKEN - docker push poldracklab/fitlins:latest - docker tag poldracklab/fitlins poldracklab/fitlins:$CIRCLE_TAG - docker push poldracklab/fitlins:$CIRCLE_TAG - fi - - deployable: + nocircle: docker: - image: busybox:latest steps: - - run: echo Deploying! + - run: echo Not running Circle on this branch! workflows: version: 2 - fitlins_tests: + nocircle: jobs: - - build: - filters: - branches: - ignore: /doc\/.*/ - tags: - only: /.*/ - - cache_test_data: - filters: - branches: - ignore: /doc\/.*/ - tags: - only: /.*/ - - build_docs: - filters: - tags: - only: /.*/ - - run_pytest: - requires: - - build - filters: - tags: - only: /.*/ - - test_ds003: - name: test_ds003_<< matrix.test_name >> - matrix: - parameters: - test_name: ["afni_smooth", "nistats_smooth", "afni_blurto", "nistats_blurto"] - requires: - - build - - cache_test_data - filters: - tags: - only: /.*/ - - test_database_file: - requires: - - build - - cache_test_data - filters: - tags: - only: /.*/ - - test_packaging: - filters: - tags: - only: /.*/ - - deploy_master: - requires: - - test_ds003_nistats_smooth - - test_ds003_nistats_blurto - - test_ds003_afni_smooth - - test_ds003_afni_blurto - filters: - branches: - only: master - - deploy_docker: - requires: - - test_ds003_nistats_smooth - - test_ds003_nistats_blurto - - test_ds003_afni_smooth - - test_ds003_afni_blurto - - build_docs - filters: - branches: - ignore: /.*/ - tags: - only: /.*/ - - deploy_pypi: - requires: - - test_ds003_nistats_smooth - - test_ds003_nistats_blurto - - test_ds003_afni_smooth - - test_ds003_afni_blurto - - build_docs - - test_packaging - filters: - branches: - ignore: /.*/ - tags: - only: /.*/ + - nocircle