From 5c411d1c5b98ea061c0c931a3df7d90f78f0d514 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 7 Apr 2021 01:12:13 +0100 Subject: [PATCH] 1956 - docker image building pipelines (#1957) * refactor docker building Signed-off-by: Wenqi Li --- .dockerignore | 1 - .github/workflows/docker.yml | 112 +++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 44 ++++++++++++- .github/workflows/setupapp.yml | 43 ------------- Dockerfile | 3 +- 5 files changed, 154 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/docker.yml diff --git a/.dockerignore b/.dockerignore index 262da4d0dd..4e1161bfb2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,6 @@ docs/ .coverage/ coverage.xml .readthedocs.yml -*.md *.toml !README.md diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..2745d4169f --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,112 @@ +name: docker +# versioning: compute a static version file +# local_docker: use the version file to build docker images +# docker_test_latest: test the latest internal docker image (has flake) +# docker_test_dockerhub: test the latest dockerhub release (no flake) +on: + # master only docker deployment and quick tests + push: + branches: + - master + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + versioning: + # compute versioning file from python setup.py + # upload as artifact + # (also used in release.yml) + if: github.repository == 'Project-MONAI/MONAI' + container: + image: localhost:5000/local_monai:latest + runs-on: [self-hosted, linux, x64, build_only] + steps: + - uses: actions/checkout@v2 + # full history so that we can git describe + with: + ref: master + fetch-depth: 0 + - shell: bash + run: | + git describe + python setup.py build + cat build/lib/monai/_version.py + - name: Upload version + uses: actions/upload-artifact@v2 + with: + name: _version.py + path: build/lib/monai/_version.py + - name: Clean up directory + shell: bash + run: | + ls -al + rm -rf {*,.[^.]*} + + local_docker: + # builds two versions: local_monai:latest and local_monai:dockerhub + # latest: used for local tests + # dockerhub: release, no flake package + if: github.repository == 'Project-MONAI/MONAI' + needs: versioning + runs-on: [self-hosted, linux, x64, build_only] + steps: + - uses: actions/checkout@v2 + with: + ref: master + - name: Download version + uses: actions/download-artifact@v2 + with: + name: _version.py + - name: docker_build + shell: bash + run: | + # get tag info for versioning + cat _version.py + mv _version.py monai/ + # build and run original docker image for local registry + docker build -t localhost:5000/local_monai:latest -f Dockerfile . + docker push localhost:5000/local_monai:latest + # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com + sed -i '/flake/d' requirements-dev.txt + docker build -t projectmonai/monai:latest -f Dockerfile . + # also push as tag "dockerhub" to local registry + docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub + docker push localhost:5000/local_monai:dockerhub + # distribute as always w/ tag "latest" to hub.docker.com + echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin + docker push projectmonai/monai:latest + docker logout + + docker_test_latest: + if: github.repository == 'Project-MONAI/MONAI' + needs: local_docker + container: + image: localhost:5000/local_monai:latest + runs-on: [self-hosted, linux, x64, common] + steps: + - name: Import + run: | + python -c 'import monai; monai.config.print_config()' + cd /opt/monai + ls -al + ngc --version + python -m tests.min_tests + env: + QUICKTEST: True + + docker_test_dockerhub: + if: github.repository == 'Project-MONAI/MONAI' + needs: local_docker + container: + image: localhost:5000/local_monai:dockerhub + runs-on: [self-hosted, linux, x64, common] + steps: + - name: Import + run: | + python -c 'import monai; monai.config.print_config()' + cd /opt/monai + ls -al + ngc --version + python -m tests.min_tests + env: + QUICKTEST: True diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f36abc9fcf..00e28ecd52 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -83,14 +83,49 @@ jobs: password: ${{ secrets.TEST_PYPI }} repository_url: https://test.pypi.org/legacy/ - release_docker: + versioning: + # compute versioning file from python setup.py + # upload as artifact + # (also used in docker.yml) if: github.repository == 'Project-MONAI/MONAI' needs: packaging - runs-on: [ self-hosted, linux, x64, build_only ] + container: + image: localhost:5000/local_monai:latest + runs-on: [self-hosted, linux, x64, build_only] steps: - uses: actions/checkout@v2 + # full history so that we can git describe with: ref: master + fetch-depth: 0 + - shell: bash + run: | + git describe + python setup.py build + cat build/lib/monai/_version.py + - name: Upload version + uses: actions/upload-artifact@v2 + with: + name: _version.py + path: build/lib/monai/_version.py + - name: Clean up directory + shell: bash + run: | + ls -al + rm -rf {*,.[^.]*} + + release_tag_docker: + if: github.repository == 'Project-MONAI/MONAI' + needs: versioning + runs-on: [self-hosted, linux, x64, build_only] + steps: + - uses: actions/checkout@v2 + with: + ref: master + - name: Download version + uses: actions/download-artifact@v2 + with: + name: _version.py - name: Set tag id: versioning run: echo ::set-output name=tag::${GITHUB_REF#refs/*/} @@ -99,12 +134,15 @@ jobs: RELEASE_VERSION: ${{ steps.versioning.outputs.tag }} run: | echo "$RELEASE_VERSION" + cat _version.py - if: startsWith(github.ref, 'refs/tags/') name: build with the tag env: RELEASE_VERSION: ${{ steps.versioning.outputs.tag }} + shell: bash run: | - git fetch --depth=1 origin +refs/tags/*:refs/tags/* + # get tag info for versioning + mv _version.py monai/ # remove flake package as it is not needed on hub.docker.com sed -i '/flake/d' requirements-dev.txt docker build -t projectmonai/monai:"$RELEASE_VERSION" -f Dockerfile . diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml index 450be403a0..dc65141fe8 100644 --- a/.github/workflows/setupapp.yml +++ b/.github/workflows/setupapp.yml @@ -148,46 +148,3 @@ jobs: python -m tests.min_tests env: QUICKTEST: True - - local_docker: - if: github.repository == 'Project-MONAI/MONAI' - runs-on: [self-hosted, linux, x64, build_only] - # we only push built container if it is built from master branch - steps: - - uses: actions/checkout@v2 - with: - ref: master - - name: docker_build - run: | - # get tag info for versioning - git fetch --depth=1 origin +refs/tags/*:refs/tags/* - # build and run original docker image for local registry - docker build -t localhost:5000/local_monai:latest -f Dockerfile . - docker push localhost:5000/local_monai:latest - # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com - sed -i '/flake/d' requirements-dev.txt - docker build -t projectmonai/monai:latest -f Dockerfile . - # also push as tag "dockerhub" to local registry - docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub - docker push localhost:5000/local_monai:dockerhub - # distribute as always w/ tag "latest" to hub.docker.com - echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin - docker push projectmonai/monai:latest - docker logout - - docker: - if: github.repository == 'Project-MONAI/MONAI' - needs: local_docker - container: - image: localhost:5000/local_monai:latest - runs-on: [self-hosted, linux, x64, common] - steps: - - name: Import - run: | - python -c 'import monai; monai.config.print_config()' - cd /opt/monai - ls -al - ngc --version - python -m tests.min_tests - env: - QUICKTEST: True diff --git a/Dockerfile b/Dockerfile index 57ea567869..23be9ae1c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,10 +30,9 @@ RUN cp /tmp/requirements.txt /tmp/req.bak \ # please specify exact files and folders to be copied -- else, basically always, the Docker build process cannot cache # this or anything below it and always will build from at most here; one file change leads to no caching from here on... -COPY LICENSE setup.py setup.cfg versioneer.py runtests.sh .gitignore .gitattributes README.md MANIFEST.in ./ +COPY LICENSE CHANGELOG.md CODE_OF_CONDUCT.md CONTRIBUTING.md README.md versioneer.py setup.py setup.cfg runtests.sh MANIFEST.in ./ COPY tests ./tests COPY monai ./monai -COPY .git ./.git RUN BUILD_MONAI=1 FORCE_CUDA=1 python setup.py develop \ && rm -rf build __pycache__