Feat/improve cicd part5 (#2583)

* update deploy github pages * update release workflow * run docker build test * fix bug * add bump type * test * use python image * update release workflow
unit8co · Nov 3, 2024 · ae04750 · ae04750
1 parent a7837c7
commit ae04750
Show file tree

Hide file tree

Showing 3 changed files with 91 additions and 108 deletions.
diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml
@@ -7,49 +7,54 @@ jobs:
   deploy-docs:
     runs-on: ubuntu-latest
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.9"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-      - name: "3. Install pandoc"
+      # use `uv` to retrieve the latest dependency versions
+      - name: "Compile Dependency Versions"
         run: |
-          sudo apt-get install -y pandoc
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv pip compile requirements/dev-all.txt > requirements-latest.txt
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "4. Cache gradle distribution"
-        uses: actions/cache@v2
+      # only restore cache but do not upload
+      - name: "Restore cached python environment"
+        uses: actions/cache/restore@v4
+        id: pythonenv-cache
         with:
-          path: ~/.gradle/wrapper/dists
-          key: release-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt', 'requirements-latest.txt') }}
 
-      - name: "4.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: release-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Install pandoc"
+        run: |
+          sudo apt-get install -y pandoc
 
-      - name: "5. Setup pip"
+      - name: "Setup Pip"
         run: |
-          ./gradlew setupPip
+          python -m pip install --upgrade pip
 
-      - name: "6. Attach cache for pip"
-        uses: actions/cache@v1
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: release-${{ runner.os }}-pip-${{ hashFiles('requirements/core.txt', 'requirements/release.txt') }}
-          restore-keys: |
-            release-${{ runner.os }}-pip-
+      - name: "Install Dependencies"
+        run: |
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt
+
+      - name: "Install libomp (for LightGBM)"
+        run: |
+          ./.github/scripts/libomp-${{ runner.os }}.sh
+
+      - name: "Install Locally"
+        run: |
+          pip install .
 
-      - name: "7. Build docs"
+      - name: "Build docs"
         run: |
-          ./gradlew buildDocs
+          make --directory ./docs build-all-docs
 
-      - name: "8. Publish documentation to gh-pages"
+      - name: "Publish documentation to gh-pages"
         uses: s0/[email protected]
         env:
           REPO: self

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -12,46 +12,37 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "1. Clone repository"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           token: ${{ secrets.RELEASE_WORKFLOW_TOKEN_NEW_FINE_GRAINED }}
           fetch-depth: '1'
 
       - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-      - name: "3. Update pip"
+      - name: "Setup Pip"
         run: |
           python -m pip install --upgrade pip
 
-      - name: "4. Attach cache for pip"
-        uses: actions/cache@v1
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: release-${{ runner.os }}-pip-${{ hashFiles('requirements/release.txt') }}
-          restore-keys: |
-            release-${{ runner.os }}-pip-
-
-      - name: "5. Install release dependencies"
+      - name: "Install release dependencies"
         run: |
           pip install -q -r requirements/release.txt
 
-      - name: "6. Determine next version"
+      - name: "Determine next version"
         uses: hrzn/github-tag-action@master
         id: bump_dry
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           DRY_RUN: true
           BUMP_TYPE: ${{ github.event.inputs.bump_type}}
 
-      - name: "7. Bump version"
+      - name: "Bump version"
         run: |
           bump2version --new-version ${{ steps.bump_dry.outputs.new_tag }} patch
 
-      - name: "8. Commit new version"
+      - name: "Commit new version"
         uses: stefanzweifel/[email protected]
         with:
           commit_message: Release ${{ steps.bump_dry.outputs.new_tag }}
@@ -60,7 +51,7 @@ jobs:
           commit_user_name: Unit8 Bot
           commit_user_email: [email protected]
 
-      - name: "9. Publish new tag"
+      - name: "Publish new tag"
         uses: hrzn/github-tag-action@master
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -77,13 +68,12 @@ jobs:
           draft: false
           body_path: .github/RELEASE_TEMPLATE/release_body.md
 
-
   deploy-docker:
     needs: [release]
     runs-on: ubuntu-latest
     steps:
       - name: "1. Clone repository"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: "2. Determine current version"
         uses: hrzn/github-tag-action@master
@@ -93,83 +83,76 @@ jobs:
           DRY_RUN: true
           BUMP_TYPE: ${{ github.event.inputs.bump_type}}
 
-      - name: "3. Login to docker hub"
-        run: docker login -u $DOCKER_HUB_USER -p $DOCKER_HUB_TOKEN
-        env:
-          DOCKER_HUB_USER: ${{ secrets.DOCKER_HUB_USER }}
-          DOCKER_HUB_TOKEN: ${{ secrets.DOCKER_HUB_TOKEN }}
+      - name: "Set up QEMU"
+        uses: docker/setup-qemu-action@v3
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "4. Cache gradle distribution"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/wrapper/dists
-          key: release-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+      - name: "Set up Docker Buildx"
+        uses: docker/setup-buildx-action@v3
 
-      - name: "4.1 Cache gradle packages"
-        uses: actions/cache@v2
+      - name: "Login to Docker Hub"
+        uses: docker/login-action@v3
         with:
-          path: ~/.gradle/caches
-          key: release-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
-
-      #check build.gradle file for explanation of next steps
-      - name: "5. Publish image with tag corresponding to current version"
-        run: |
-          ./gradlew dockerPushVersion -P version=${{ steps.bump_dry.outputs.tag }}
-
-      - name: "6. Publish image with tag 'latest' if not hotfix"
-        if: ${{ !contains(github.event.head_commit.message, '#hotfix') }}
-        run: |
-          ./gradlew dockerPushLatest
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
+      - name: "Build and push"
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: unit8/darts:${{ steps.bump_dry.outputs.tag }}, unit8/darts:latest
 
   deploy-docs:
     runs-on: ubuntu-latest
     needs: [release]
     steps:
-      - name: "1. Clone repository"
-        uses: actions/checkout@v2
+      - name: "Clone repository"
+        uses: actions/checkout@v4
 
-      - name: "2. Set up Python 3.9"
-        uses: actions/setup-python@v1
+      - name: "Set up Python 3.9"
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
 
-      - name: "3. Install pandoc"
+      # use `uv` to retrieve the latest dependency versions
+      - name: "Compile Dependency Versions"
         run: |
-          sudo apt-get install -y pandoc
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          uv pip compile requirements/dev-all.txt > requirements-latest.txt
 
-        # downloading gradle multiple times in parallel can yield to connection errors
-      - name: "4. Cache gradle distribution"
-        uses: actions/cache@v2
+      # only restore cache but do not upload
+      - name: "Restore cached python environment"
+        uses: actions/cache/restore@v4
+        id: pythonenv-cache
         with:
-          path: ~/.gradle/wrapper/dists
-          key: release-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }}
+          path: ${{ env.pythonLocation }}
+          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements/*.txt', 'requirements-latest.txt') }}
 
-      - name: "4.1 Cache gradle packages"
-        uses: actions/cache@v2
-        with:
-          path: ~/.gradle/caches
-          key: release-${{ runner.os }}-gradle-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties', 'build.gradle') }}
+      - name: "Install pandoc"
+        run: |
+          sudo apt-get install -y pandoc
 
-      - name: "5. Setup pip"
+      - name: "Setup Pip"
         run: |
-          ./gradlew setupPip
+          python -m pip install --upgrade pip
 
-      - name: "6. Attach cache for pip"
-        uses: actions/cache@v1
-        id: cache
-        with:
-          path: ~/.cache/pip
-          key: release-${{ runner.os }}-pip-${{ hashFiles('requirements/core.txt', 'requirements/release.txt') }}
-          restore-keys: |
-            release-${{ runner.os }}-pip-
+      - name: "Install Dependencies"
+        run: |
+          # install latest dependencies (potentially updating cached dependencies)
+          pip install -U -r requirements/dev-all.txt
+
+      - name: "Install libomp (for LightGBM)"
+        run: |
+          ./.github/scripts/libomp-${{ runner.os }}.sh
+
+      - name: "Install Locally"
+        run: |
+          pip install .
 
-      - name: "7. Build docs"
+      - name: "Build docs"
         run: |
-          ./gradlew buildDocs
+          make --directory ./docs build-all-docs
 
-      - name: "8. Publish documentation to gh-pages"
+      - name: "Publish documentation to gh-pages"
         uses: s0/[email protected]
         env:
           REPO: self

diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,4 @@
-FROM ubuntu:latest
-
-# setup packages
-RUN apt-get update -y
-RUN apt-get install -y python3 python-is-python3 python3-pip default-jre
-RUN pip install --upgrade pip
+FROM python:3.10
 
 # install python requirements before copying the rest of the files
 # this way we can cache the requirements and not have to reinstall them