ci: release pypi and refactor release image (#316)

* add ghcr.io

* pypi release ci

* remove fixed version for mnist example

* refactor release image

* fix pypi action

.github/workflows/release.yml

@@ -1,19 +1,72 @@
-name: Release by Tag CI
+name: Release Docker Image and Pypi Package
 
 on:
-  push:
-    tags:
-      - 'v*.*.*'
+  release:
+    types: [published]
 
 jobs:
+  pypi-release:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup python
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.7"
+          architecture: "x64"
+
+      - name: Get pip cache
+        id: pip-cache-path
+        run: echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache pip dependencies
+        uses: actions/cache@v3
+        id: pip-cache
+        with:
+          path: ${{ steps.pip-cache-path.outputs.dir }}
+          key: ${{ runner.os }}-codestyle-${{ hashFiles('client/requirements-dev.txt')}}
 
-  build:
+      - name: Install dependencies
+        working-directory: ./client
+        run: |
+          make install-req
+          make install-dev-req
 
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+        with:
+          strip_v: true
+
+      - name: Build Python Package
+        working-directory: ./client
+        env:
+          PYPI_RELEASE_VERSION: ${{ steps.tag.outputs.tag }}
+        run: make build-wheel
+
+      - name: Publish a Python distribution to PyPI
+        uses: pypa/[email protected]
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          verbose: true
+          packages_dir: client/dist/
+          verify_metadata: false
+
+  image-release:
     runs-on: ubuntu-latest
 
+    needs:
+      - pypi-release
+
     steps:
       - uses: actions/checkout@v3
-      # UI build
+
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -38,6 +91,7 @@ jobs:
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
           restore-keys: |
             ${{ runner.os }}-yarn-
+
       - name: Install project dependencies
         working-directory: ./console
         if: steps.yarn-cache.outputs.cache-hit != 'true' # Over here!
@@ -55,39 +109,45 @@ jobs:
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
-      - uses: actions/checkout@v3
+
       - name: Set up JDK 11
         uses: actions/setup-java@v3
         with:
-          java-version: '11'
-          distribution: 'adopt'
-          cache: 'maven'
+          java-version: "11"
+          distribution: "adopt"
+          cache: "maven"
           server-id: starwhale # Value of the distributionManagement/repository/id field of the pom.xml
+
       - name: Build with Maven
         working-directory: ./server
         run: make build-package
 
-      # docker-hub login
-      - name: Log in to Docker Hub
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
         with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1
         with:
-          # Optionally strip `v` prefix
           strip_v: true
-      # docker image build
-      - name: Docker image build and push
+
+      - name: Build and push docker image
         working-directory: ./docker
         env:
-          RELEASE_VERSION: v${{ steps.tag.outputs.tag }}
+          VERSION_FORMAT: ${{ steps.tag.outputs.tag }}
+          PYPI_RELEASE_VERSION: ${{ steps.tag.outputs.tag }}
         run: |
-          make build-base VERSION_FORMAT=$RELEASE_VERSION
-          make build-starwhale VERSION_FORMAT=$RELEASE_VERSION
-          make build-taskset VERSION_FORMAT=$RELEASE_VERSION
-          make build-server-when-ui-and-jar-built VERSION_FORMAT=$RELEASE_VERSION
-          make release
-      # todo cli build and upload
+          make build-starwhale
+          make release-starwhale
+          make build-server
+          make release-server

client/Makefile

@@ -13,7 +13,7 @@ build-wheel: check clean
 upload-pypi:
 	twine upload dist/*
 
-build-dev-wheel:
+install-dev-wheel:
 	python3 -m pip install -e .
 
 install-dev-req:

client/requirements-dev.txt

@@ -13,4 +13,6 @@ pytest >= 6.0
 testfixtures >= 6.10.3
 pytest-env >= 0.6.0
 pytest-xdist >= 2.0
-pytest-cov>=3.0.0
+pytest-cov>=3.0.0
+setuptools>=62.0.0
+wheel >= 0.35.0

client/setup.py

@@ -1,3 +1,4 @@
+import os
 from setuptools import setup, find_packages
 
 install_requires = [
@@ -21,11 +22,22 @@
 ]
 
 
+def _format_version() -> str:
+    _v = os.environ.get("PYPI_RELEASE_VERSION", "0.1.0.alpha1")
+    _v = _v.lstrip("v").replace("-", ".")
+    _vs = _v.split(".", 3)
+    if len(_vs) == 4:
+        _vs[-1] = _vs[-1].replace(".", "")
+        return ".".join(_vs)
+    else:
+        return _v
+
+
 setup(
     name="starwhale",
     author="Starwhale Team",
     author_email="[email protected]",
-    version="0.1.0.dev15",
+    version=_format_version(),
     description="MLOps Platform",
     keywords="MLOps AI",
     url="https://github.com/star-whale/starwhale",

docker/Makefile

@@ -1,52 +1,73 @@
 GIT_REVISION = $(shell git rev-parse --short HEAD)
-DATE = $(shell date "+%Y%m%d%H%M%S")
+DATE = $(shell date "+%Y%m%d")
 BASE_VERSION = 0.1.0
-VERSION_FORMAT = ${BASE_VERSION}-nightly-${DATE}-${GIT_REVISION}
-SW_VERSION := 0.1.0.dev15
+RELEASE_VERSION = ${BASE_VERSION}-nightly-${DATE}-${GIT_REVISION}
+PYPI_RELEASE_VERSION = 0.1.0.dev15
 
-BASE_IMAGE :=starwhaleai/base:${VERSION_FORMAT}
-NODEJS_IMAGE :=starwhaleai/nodejs:${VERSION_FORMAT}
-SW_IMAGE := starwhaleai/starwhale:${VERSION_FORMAT}
-SW_LATEST_IMAGE := starwhaleai/starwhale:latest
-SW_TASKSET_IMAGE := starwhaleai/taskset:${VERSION_FORMAT}
-SW_SERVER_IMAGE := starwhaleai/server:${VERSION_FORMAT}
+DOCKER_HUB_REPO := starwhaleai
+GHCR_IO_REPO := ghcr.io/star-whale
+
+# Please update base/nodejs/taskset image version by manual, DOT NOT USE RELEASE TAG.
+# These images versions are slow to release.
+FIXED_VERSION_BASE_IMAGE := 0.1.1
+FIXED_VERSION_NODEJS_IMAGE := 0.1.1
+FIXED_VERSION_TASKSET_IMAGE := 0.1.1
+
+DH_BASE_IMAGE := ${DOCKER_HUB_REPO}/base:${FIXED_VERSION_BASE_IMAGE}
+DH_NODEJS_IMAGE := ${DOCKER_HUB_REPO}/nodejs:${FIXED_VERSION_NODEJS_IMAGE}
 
 DOCKER_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
 ROOT_DIR := $(dir $(abspath $(DOCKER_DIR)))
 
+define build-image
+	docker build --build-arg BASE_IMAGE=$(DH_BASE_IMAGE) --build-arg SW_VERSION=$(PYPI_RELEASE_VERSION) -f Dockerfile.$(1) -t ${DOCKER_HUB_REPO}/$(1):$(2) .
+	docker tag ${DOCKER_HUB_REPO}/$(1):$(2) ${DOCKER_HUB_REPO}/$(1):latest
+	docker tag ${DOCKER_HUB_REPO}/$(1):$(2) ${GHCR_IO_REPO}/$(1):$(2)
+	docker tag ${DOCKER_HUB_REPO}/$(1):$(2) ${GHCR_IO_REPO}/$(1):latest
+endef
+
+define push-image
+	docker push ${DOCKER_HUB_REPO}/$(1):$(2)
+	docker push ${DOCKER_HUB_REPO}/$(1):latest
+	docker push ${GHCR_IO_REPO}/$(1):latest
+	docker push ${GHCR_IO_REPO}/$(1):$(2)
+endef
+
 build-base:
-	docker build -f Dockerfile.base -t $(BASE_IMAGE) .
+	$(call build-image,base,${FIXED_VERSION_BASE_IMAGE})
+
+release-base:
+	$(call push-image,base,${FIXED_VERSION_BASE_IMAGE})
 
 build-starwhale:
-	docker build --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg SW_VERSION=$(SW_VERSION) -f Dockerfile.sw -t $(SW_IMAGE) .
-	docker tag $(SW_IMAGE) $(SW_LATEST_IMAGE)
+	$(call build-image,starwhale,${RELEASE_VERSION})
 
-release:
-	docker push $(BASE_IMAGE)
-	docker push $(SW_IMAGE)
-	docker push $(SW_LATEST_IMAGE)
-	docker push $(SW_TASKSET_IMAGE)
-	docker push starwhaleai/server:${VERSION_FORMAT}
+release-starwhale:
+	$(call push-image,starwhale,${RELEASE_VERSION})
 
 build-taskset:
-	docker build --build-arg BASE_IMAGE=$(BASE_IMAGE) -f Dockerfile.taskset -t $(SW_TASKSET_IMAGE) .
+	$(call build-image,taskset,${FIXED_VERSION_TASKSET_IMAGE})
 
-# stable:starwhaleai/nodejs:0.1.0-nightly-20220505190936
-build-nodejs:
-	docker build -t ${NODEJS_IMAGE} -f Dockerfile.nodejs .
+release-taskset:
+	$(call push-image,taskset,${FIXED_VERSION_TASKSET_IMAGE})
 
-build-ui:
-	docker run --rm -it -v $(ROOT_DIR)console:/app -w /app starwhaleai/nodejs:0.1.0-nightly-20220505190936 yarn install;
-	docker run --rm -it -v $(ROOT_DIR)console:/app -w /app starwhaleai/nodejs:0.1.0-nightly-20220505190936 yarn build;
+build-server:
+	$(call build-image,server,${RELEASE_VERSION})
 
-build-jar-with-ui: build-ui build-jar
+build-server-all: build-console build-jar
+	$(call build-image,server,${RELEASE_VERSION})
 
-build-jar:
-	docker volume create --name maven-repo; docker run --rm -it -v maven-repo:/root/.m2 -v $(ROOT_DIR):/app -w /app maven:3.8.5-openjdk-11 mvn clean package -f server/pom.xml -DskipTests;
+release-server:
+	$(call push-image,server,${RELEASE_VERSION})
+
+build-nodejs:
+	$(call build-image,nodejs,${FIXED_VERSION_NODEJS_IMAGE})
 
-build-server: build-jar-with-ui
-	docker build -f Dockerfile.server -t $(SW_SERVER_IMAGE) .
+release-nodejs:
+	$(call push-image,nodejs,${FIXED_VERSION_NODEJS_IMAGE})
 
-# used when release
-build-server-when-ui-and-jar-built:
-	docker build -f Dockerfile.server -t $(SW_SERVER_IMAGE) .
+build-console:
+	docker run --rm -it -v $(ROOT_DIR)console:/app -w /app ${DH_NODEJS_IMAGE} bash -c "yarn install && yarn build"
+
+build-jar:
+	docker volume create --name maven-repo; docker run --rm -it -v maven-repo:/root/.m2 -v $(ROOT_DIR):/app -w /app maven:3.8.5-openjdk-11 mvn clean package -f server/pom.xml -DskipTests;

example/mnist/requirements.txt

@@ -3,4 +3,4 @@ torch
 torchvision
 Pillow
 scikit-learn
-starwhale==0.1.0.dev15
+starwhale