1956 - docker image building pipelines (Project-MONAI#1957)

* refactor docker building Signed-off-by: Wenqi Li <[email protected]>
gagandaroach · Apr 7, 2021 · 5c411d1 · 5c411d1
1 parent 81cfbf0
commit 5c411d1
Show file tree

Hide file tree

Showing 5 changed files with 154 additions and 49 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -8,7 +8,6 @@ docs/
 .coverage/
 coverage.xml
 .readthedocs.yml
-*.md
 *.toml
 
 !README.md

diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -0,0 +1,112 @@
+name: docker
+# versioning: compute a static version file
+# local_docker: use the version file to build docker images
+# docker_test_latest: test the latest internal docker image (has flake)
+# docker_test_dockerhub: test the latest dockerhub release (no flake)
+on:
+  # master only docker deployment and quick tests
+  push:
+    branches:
+      - master
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  versioning:
+    # compute versioning file from python setup.py
+    # upload as artifact
+    # (also used in release.yml)
+    if: github.repository == 'Project-MONAI/MONAI'
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+      - uses: actions/checkout@v2
+        # full history so that we can git describe
+        with:
+          ref: master
+          fetch-depth: 0
+      - shell: bash
+        run: |
+          git describe
+          python setup.py build
+          cat build/lib/monai/_version.py
+      - name: Upload version
+        uses: actions/upload-artifact@v2
+        with:
+          name: _version.py
+          path: build/lib/monai/_version.py
+      - name: Clean up directory
+        shell: bash
+        run: |
+          ls -al
+          rm -rf {*,.[^.]*}
+
+  local_docker:
+    # builds two versions: local_monai:latest and local_monai:dockerhub
+    # latest: used for local tests
+    # dockerhub: release, no flake package
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: versioning
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        ref: master
+    - name: Download version
+      uses: actions/download-artifact@v2
+      with:
+        name: _version.py
+    - name: docker_build
+      shell: bash
+      run: |
+        # get tag info for versioning
+        cat _version.py
+        mv _version.py monai/
+        # build and run original docker image for local registry
+        docker build -t localhost:5000/local_monai:latest -f Dockerfile .
+        docker push localhost:5000/local_monai:latest
+        # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com
+        sed -i '/flake/d' requirements-dev.txt
+        docker build -t projectmonai/monai:latest -f Dockerfile .
+        # also push as tag "dockerhub" to local registry
+        docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub
+        docker push localhost:5000/local_monai:dockerhub
+        # distribute as always w/ tag "latest" to hub.docker.com
+        echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin
+        docker push projectmonai/monai:latest
+        docker logout
+
+  docker_test_latest:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: local_docker
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - name: Import
+      run: |
+        python -c 'import monai; monai.config.print_config()'
+        cd /opt/monai
+        ls -al
+        ngc --version
+        python -m tests.min_tests
+      env:
+        QUICKTEST: True
+
+  docker_test_dockerhub:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: local_docker
+    container:
+      image: localhost:5000/local_monai:dockerhub
+    runs-on: [self-hosted, linux, x64, common]
+    steps:
+    - name: Import
+      run: |
+        python -c 'import monai; monai.config.print_config()'
+        cd /opt/monai
+        ls -al
+        ngc --version
+        python -m tests.min_tests
+      env:
+        QUICKTEST: True
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -83,14 +83,49 @@ jobs:
         password: ${{ secrets.TEST_PYPI }}
         repository_url: https://test.pypi.org/legacy/
 
-  release_docker:
+  versioning:
+    # compute versioning file from python setup.py
+    # upload as artifact
+    # (also used in docker.yml)
     if: github.repository == 'Project-MONAI/MONAI'
     needs: packaging
-    runs-on: [ self-hosted, linux, x64, build_only ]
+    container:
+      image: localhost:5000/local_monai:latest
+    runs-on: [self-hosted, linux, x64, build_only]
     steps:
       - uses: actions/checkout@v2
+        # full history so that we can git describe
         with:
           ref: master
+          fetch-depth: 0
+      - shell: bash
+        run: |
+          git describe
+          python setup.py build
+          cat build/lib/monai/_version.py
+      - name: Upload version
+        uses: actions/upload-artifact@v2
+        with:
+          name: _version.py
+          path: build/lib/monai/_version.py
+      - name: Clean up directory
+        shell: bash
+        run: |
+          ls -al
+          rm -rf {*,.[^.]*}
+
+  release_tag_docker:
+    if: github.repository == 'Project-MONAI/MONAI'
+    needs: versioning
+    runs-on: [self-hosted, linux, x64, build_only]
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: master
+      - name: Download version
+        uses: actions/download-artifact@v2
+        with:
+          name: _version.py
       - name: Set tag
         id: versioning
         run: echo ::set-output name=tag::${GITHUB_REF#refs/*/}
@@ -99,12 +134,15 @@ jobs:
           RELEASE_VERSION: ${{ steps.versioning.outputs.tag }}
         run: |
           echo "$RELEASE_VERSION"
+          cat _version.py
       - if: startsWith(github.ref, 'refs/tags/')
         name: build with the tag
         env:
           RELEASE_VERSION: ${{ steps.versioning.outputs.tag }}
+        shell: bash
         run: |
-          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
+          # get tag info for versioning
+          mv _version.py monai/
           # remove flake package as it is not needed on hub.docker.com
           sed -i '/flake/d' requirements-dev.txt
           docker build -t projectmonai/monai:"$RELEASE_VERSION" -f Dockerfile .

diff --git a/.github/workflows/setupapp.yml b/.github/workflows/setupapp.yml
@@ -148,46 +148,3 @@ jobs:
         python -m tests.min_tests
       env:
         QUICKTEST: True
-
-  local_docker:
-    if: github.repository == 'Project-MONAI/MONAI'
-    runs-on: [self-hosted, linux, x64, build_only]
-    # we only push built container if it is built from master branch
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        ref: master
-    - name: docker_build
-      run: |
-        # get tag info for versioning
-        git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-        # build and run original docker image for local registry
-        docker build -t localhost:5000/local_monai:latest -f Dockerfile .
-        docker push localhost:5000/local_monai:latest
-        # build once more w/ tag "latest": remove flake package as it is not needed on hub.docker.com
-        sed -i '/flake/d' requirements-dev.txt
-        docker build -t projectmonai/monai:latest -f Dockerfile .
-        # also push as tag "dockerhub" to local registry
-        docker image tag projectmonai/monai:latest localhost:5000/local_monai:dockerhub
-        docker push localhost:5000/local_monai:dockerhub
-        # distribute as always w/ tag "latest" to hub.docker.com
-        echo "${{ secrets.DOCKER_PW }}" | docker login -u projectmonai --password-stdin
-        docker push projectmonai/monai:latest
-        docker logout
-
-  docker:
-    if: github.repository == 'Project-MONAI/MONAI'
-    needs: local_docker
-    container:
-      image: localhost:5000/local_monai:latest
-    runs-on: [self-hosted, linux, x64, common]
-    steps:
-    - name: Import
-      run: |
-        python -c 'import monai; monai.config.print_config()'
-        cd /opt/monai
-        ls -al
-        ngc --version
-        python -m tests.min_tests
-      env:
-        QUICKTEST: True
diff --git a/Dockerfile b/Dockerfile
@@ -30,10 +30,9 @@ RUN cp /tmp/requirements.txt /tmp/req.bak \
 # please specify exact files and folders to be copied -- else, basically always, the Docker build process cannot cache
 # this or anything below it and always will build from at most here; one file change leads to no caching from here on...
 
-COPY LICENSE setup.py setup.cfg versioneer.py runtests.sh .gitignore .gitattributes README.md MANIFEST.in ./
+COPY LICENSE CHANGELOG.md CODE_OF_CONDUCT.md CONTRIBUTING.md README.md versioneer.py setup.py setup.cfg runtests.sh MANIFEST.in ./
 COPY tests ./tests
 COPY monai ./monai
-COPY .git ./.git
 RUN BUILD_MONAI=1 FORCE_CUDA=1 python setup.py develop \
   && rm -rf build __pycache__