From 89bf425ae1e531e1f115066adb1b329414302450 Mon Sep 17 00:00:00 2001 From: tianwei Date: Fri, 13 May 2022 13:55:53 +0800 Subject: [PATCH] ci: release pypi and refactor release image (#316) * add ghcr.io * pypi release ci * remove fixed version for mnist example * refactor release image * fix pypi action --- .github/workflows/release.yml | 110 ++++++++++++++---- client/Makefile | 2 +- client/requirements-dev.txt | 4 +- client/setup.py | 14 ++- .../{Dockerfile.sw => Dockerfile.starwhale} | 0 docker/Makefile | 87 ++++++++------ example/mnist/requirements.txt | 2 +- 7 files changed, 157 insertions(+), 62 deletions(-) rename docker/{Dockerfile.sw => Dockerfile.starwhale} (100%) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe62b73ce7..5fa5ee1389 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,19 +1,72 @@ -name: Release by Tag CI +name: Release Docker Image and Pypi Package on: - push: - tags: - - 'v*.*.*' + release: + types: [published] jobs: + pypi-release: + runs-on: ubuntu-latest + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v3 + + - name: Setup python + uses: actions/setup-python@v3 + with: + python-version: "3.7" + architecture: "x64" + + - name: Get pip cache + id: pip-cache-path + run: echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip dependencies + uses: actions/cache@v3 + id: pip-cache + with: + path: ${{ steps.pip-cache-path.outputs.dir }} + key: ${{ runner.os }}-codestyle-${{ hashFiles('client/requirements-dev.txt')}} - build: + - name: Install dependencies + working-directory: ./client + run: | + make install-req + make install-dev-req + - name: Get tag + id: tag + uses: dawidd6/action-get-tag@v1 + with: + strip_v: true + + - name: Build Python Package + working-directory: ./client + env: + PYPI_RELEASE_VERSION: ${{ steps.tag.outputs.tag }} + run: make build-wheel + + - name: Publish a Python distribution to PyPI + uses: pypa/gh-action-pypi-publish@v1.5.0 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} + verbose: true + packages_dir: client/dist/ + verify_metadata: false + + image-release: runs-on: ubuntu-latest + needs: + - pypi-release + steps: - uses: actions/checkout@v3 - # UI build + - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v3 with: @@ -38,6 +91,7 @@ jobs: key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-yarn- + - name: Install project dependencies working-directory: ./console if: steps.yarn-cache.outputs.cache-hit != 'true' # Over here! @@ -55,39 +109,45 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - uses: actions/checkout@v3 + - name: Set up JDK 11 uses: actions/setup-java@v3 with: - java-version: '11' - distribution: 'adopt' - cache: 'maven' + java-version: "11" + distribution: "adopt" + cache: "maven" server-id: starwhale # Value of the distributionManagement/repository/id field of the pom.xml + - name: Build with Maven working-directory: ./server run: make build-package - # docker-hub login - - name: Log in to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + - name: Login to Docker Hub + uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Get tag id: tag uses: dawidd6/action-get-tag@v1 with: - # Optionally strip `v` prefix strip_v: true - # docker image build - - name: Docker image build and push + + - name: Build and push docker image working-directory: ./docker env: - RELEASE_VERSION: v${{ steps.tag.outputs.tag }} + VERSION_FORMAT: ${{ steps.tag.outputs.tag }} + PYPI_RELEASE_VERSION: ${{ steps.tag.outputs.tag }} run: | - make build-base VERSION_FORMAT=$RELEASE_VERSION - make build-starwhale VERSION_FORMAT=$RELEASE_VERSION - make build-taskset VERSION_FORMAT=$RELEASE_VERSION - make build-server-when-ui-and-jar-built VERSION_FORMAT=$RELEASE_VERSION - make release - # todo cli build and upload + make build-starwhale + make release-starwhale + make build-server + make release-server diff --git a/client/Makefile b/client/Makefile index 8da4f706c4..dadf5f8e3e 100644 --- a/client/Makefile +++ b/client/Makefile @@ -13,7 +13,7 @@ build-wheel: check clean upload-pypi: twine upload dist/* -build-dev-wheel: +install-dev-wheel: python3 -m pip install -e . install-dev-req: diff --git a/client/requirements-dev.txt b/client/requirements-dev.txt index 03608056d5..babb75ecb6 100644 --- a/client/requirements-dev.txt +++ b/client/requirements-dev.txt @@ -13,4 +13,6 @@ pytest >= 6.0 testfixtures >= 6.10.3 pytest-env >= 0.6.0 pytest-xdist >= 2.0 -pytest-cov>=3.0.0 \ No newline at end of file +pytest-cov>=3.0.0 +setuptools>=62.0.0 +wheel >= 0.35.0 \ No newline at end of file diff --git a/client/setup.py b/client/setup.py index c0114051a4..dd0fcd0558 100644 --- a/client/setup.py +++ b/client/setup.py @@ -1,3 +1,4 @@ +import os from setuptools import setup, find_packages install_requires = [ @@ -21,11 +22,22 @@ ] +def _format_version() -> str: + _v = os.environ.get("PYPI_RELEASE_VERSION", "0.1.0.alpha1") + _v = _v.lstrip("v").replace("-", ".") + _vs = _v.split(".", 3) + if len(_vs) == 4: + _vs[-1] = _vs[-1].replace(".", "") + return ".".join(_vs) + else: + return _v + + setup( name="starwhale", author="Starwhale Team", author_email="developer@starwhale.ai", - version="0.1.0.dev15", + version=_format_version(), description="MLOps Platform", keywords="MLOps AI", url="https://github.com/star-whale/starwhale", diff --git a/docker/Dockerfile.sw b/docker/Dockerfile.starwhale similarity index 100% rename from docker/Dockerfile.sw rename to docker/Dockerfile.starwhale diff --git a/docker/Makefile b/docker/Makefile index 7f23f70bf2..7bfb0c7323 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,52 +1,73 @@ GIT_REVISION = $(shell git rev-parse --short HEAD) -DATE = $(shell date "+%Y%m%d%H%M%S") +DATE = $(shell date "+%Y%m%d") BASE_VERSION = 0.1.0 -VERSION_FORMAT = ${BASE_VERSION}-nightly-${DATE}-${GIT_REVISION} -SW_VERSION := 0.1.0.dev15 +RELEASE_VERSION = ${BASE_VERSION}-nightly-${DATE}-${GIT_REVISION} +PYPI_RELEASE_VERSION = 0.1.0.dev15 -BASE_IMAGE :=starwhaleai/base:${VERSION_FORMAT} -NODEJS_IMAGE :=starwhaleai/nodejs:${VERSION_FORMAT} -SW_IMAGE := starwhaleai/starwhale:${VERSION_FORMAT} -SW_LATEST_IMAGE := starwhaleai/starwhale:latest -SW_TASKSET_IMAGE := starwhaleai/taskset:${VERSION_FORMAT} -SW_SERVER_IMAGE := starwhaleai/server:${VERSION_FORMAT} +DOCKER_HUB_REPO := starwhaleai +GHCR_IO_REPO := ghcr.io/star-whale + +# Please update base/nodejs/taskset image version by manual, DOT NOT USE RELEASE TAG. +# These images versions are slow to release. +FIXED_VERSION_BASE_IMAGE := 0.1.1 +FIXED_VERSION_NODEJS_IMAGE := 0.1.1 +FIXED_VERSION_TASKSET_IMAGE := 0.1.1 + +DH_BASE_IMAGE := ${DOCKER_HUB_REPO}/base:${FIXED_VERSION_BASE_IMAGE} +DH_NODEJS_IMAGE := ${DOCKER_HUB_REPO}/nodejs:${FIXED_VERSION_NODEJS_IMAGE} DOCKER_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) ROOT_DIR := $(dir $(abspath $(DOCKER_DIR))) +define build-image + docker build --build-arg BASE_IMAGE=$(DH_BASE_IMAGE) --build-arg SW_VERSION=$(PYPI_RELEASE_VERSION) -f Dockerfile.$(1) -t ${DOCKER_HUB_REPO}/$(1):$(2) . + docker tag ${DOCKER_HUB_REPO}/$(1):$(2) ${DOCKER_HUB_REPO}/$(1):latest + docker tag ${DOCKER_HUB_REPO}/$(1):$(2) ${GHCR_IO_REPO}/$(1):$(2) + docker tag ${DOCKER_HUB_REPO}/$(1):$(2) ${GHCR_IO_REPO}/$(1):latest +endef + +define push-image + docker push ${DOCKER_HUB_REPO}/$(1):$(2) + docker push ${DOCKER_HUB_REPO}/$(1):latest + docker push ${GHCR_IO_REPO}/$(1):latest + docker push ${GHCR_IO_REPO}/$(1):$(2) +endef + build-base: - docker build -f Dockerfile.base -t $(BASE_IMAGE) . + $(call build-image,base,${FIXED_VERSION_BASE_IMAGE}) + +release-base: + $(call push-image,base,${FIXED_VERSION_BASE_IMAGE}) build-starwhale: - docker build --build-arg BASE_IMAGE=$(BASE_IMAGE) --build-arg SW_VERSION=$(SW_VERSION) -f Dockerfile.sw -t $(SW_IMAGE) . - docker tag $(SW_IMAGE) $(SW_LATEST_IMAGE) + $(call build-image,starwhale,${RELEASE_VERSION}) -release: - docker push $(BASE_IMAGE) - docker push $(SW_IMAGE) - docker push $(SW_LATEST_IMAGE) - docker push $(SW_TASKSET_IMAGE) - docker push starwhaleai/server:${VERSION_FORMAT} +release-starwhale: + $(call push-image,starwhale,${RELEASE_VERSION}) build-taskset: - docker build --build-arg BASE_IMAGE=$(BASE_IMAGE) -f Dockerfile.taskset -t $(SW_TASKSET_IMAGE) . + $(call build-image,taskset,${FIXED_VERSION_TASKSET_IMAGE}) -# stable:starwhaleai/nodejs:0.1.0-nightly-20220505190936 -build-nodejs: - docker build -t ${NODEJS_IMAGE} -f Dockerfile.nodejs . +release-taskset: + $(call push-image,taskset,${FIXED_VERSION_TASKSET_IMAGE}) -build-ui: - docker run --rm -it -v $(ROOT_DIR)console:/app -w /app starwhaleai/nodejs:0.1.0-nightly-20220505190936 yarn install; - docker run --rm -it -v $(ROOT_DIR)console:/app -w /app starwhaleai/nodejs:0.1.0-nightly-20220505190936 yarn build; +build-server: + $(call build-image,server,${RELEASE_VERSION}) -build-jar-with-ui: build-ui build-jar +build-server-all: build-console build-jar + $(call build-image,server,${RELEASE_VERSION}) -build-jar: - docker volume create --name maven-repo; docker run --rm -it -v maven-repo:/root/.m2 -v $(ROOT_DIR):/app -w /app maven:3.8.5-openjdk-11 mvn clean package -f server/pom.xml -DskipTests; +release-server: + $(call push-image,server,${RELEASE_VERSION}) + +build-nodejs: + $(call build-image,nodejs,${FIXED_VERSION_NODEJS_IMAGE}) -build-server: build-jar-with-ui - docker build -f Dockerfile.server -t $(SW_SERVER_IMAGE) . +release-nodejs: + $(call push-image,nodejs,${FIXED_VERSION_NODEJS_IMAGE}) -# used when release -build-server-when-ui-and-jar-built: - docker build -f Dockerfile.server -t $(SW_SERVER_IMAGE) . \ No newline at end of file +build-console: + docker run --rm -it -v $(ROOT_DIR)console:/app -w /app ${DH_NODEJS_IMAGE} bash -c "yarn install && yarn build" + +build-jar: + docker volume create --name maven-repo; docker run --rm -it -v maven-repo:/root/.m2 -v $(ROOT_DIR):/app -w /app maven:3.8.5-openjdk-11 mvn clean package -f server/pom.xml -DskipTests; \ No newline at end of file diff --git a/example/mnist/requirements.txt b/example/mnist/requirements.txt index ebf1f66f7d..d77cbdc2f0 100644 --- a/example/mnist/requirements.txt +++ b/example/mnist/requirements.txt @@ -3,4 +3,4 @@ torch torchvision Pillow scikit-learn -starwhale==0.1.0.dev15 \ No newline at end of file +starwhale \ No newline at end of file