From 07d42f59de7af9b94f83cad6dd8d80cb8dd3c1a2 Mon Sep 17 00:00:00 2001 From: Jerry Shao Date: Wed, 6 Nov 2024 20:32:56 +0800 Subject: [PATCH] [MINOR] improve(build): Improve the release script (#5340) ### What changes were proposed in this pull request? * Make Python package built and publish automatically. * Change the Python doc generation command. * Remove the Python package from SVN. ### Why are the changes needed? To meet the new code changes in 0.7. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual verification. --- dev/release/do-release.sh | 2 +- dev/release/release-build.sh | 57 +++++++++++++++++++++++------------- dev/release/release-util.sh | 2 +- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/dev/release/do-release.sh b/dev/release/do-release.sh index 8f0fc7f08d9..cc2825a8b37 100755 --- a/dev/release/do-release.sh +++ b/dev/release/do-release.sh @@ -33,7 +33,7 @@ done DRY_RUN=${DRY_RUN:-0} export DRY_RUN -cmds=("git" "gpg" "svn" "twine" "shasum" "sha1sum" "jq") +cmds=("git" "gpg" "svn" "twine" "shasum" "sha1sum" "jq" "make") for cmd in "${cmds[@]}"; do if ! command -v $cmd &> /dev/null; then echo "$cmd is required to run this script." diff --git a/dev/release/release-build.sh b/dev/release/release-build.sh index 73271119772..45617c76e37 100755 --- a/dev/release/release-build.sh +++ b/dev/release/release-build.sh @@ -122,21 +122,24 @@ if [[ "$1" == "finalize" ]]; then echo "git tag v$RELEASE_VERSION created" fi + PYGRAVITINO_VERSION="${RELEASE_VERSION/-incubating}" + git clone "https://$ASF_USERNAME:$ENCODED_ASF_PASSWORD@$ASF_GRAVITINO_REPO" -b "v$RELEASE_VERSION" + cd gravitino + $GRADLE :clients:client-python:distribution -x test + cd .. + cp gravitino/clients/client-python/dist/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz . + echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour \ + --output apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc \ + --detach-sig apache_gravitino-$PYGRAVITINO_VERSION.tar.gz + # upload to PyPi. - # todo: uncomment below codes if possible, it will download Gravitino Python binary from the dev directory echo "Uploading Gravitino to PyPi" - # svn co --depth=empty "$RELEASE_STAGING_LOCATION/$RELEASE_TAG" svn-gravitino - # cd svn-gravitino - PYGRAVITINO_VERSION=`echo "$RELEASE_VERSION" | sed -e "s/-/./" -e "s/preview/dev/"` - # svn update "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz" - # svn update "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc" twine upload -u __token__ -p $PYPI_API_TOKEN \ --repository-url https://upload.pypi.org/legacy/ \ "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz" \ "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc" - # cd .. - # rm -rf svn-gravitino echo "Python Gravitino package uploaded" + rm -fr gravitino # Moves the binaries from dev directory to release directory. echo "Moving Gravitino binaries to the release directory" @@ -171,6 +174,12 @@ if [ -z "$PYGRAVITINO_VERSION"]; then PYGRAVITINO_VERSION=$(cat clients/client-python/setup.py | grep "version=" | awk -F"\"" '{print $2}') fi +if [[ "$PYGRAVITINO_VERSION" == *"dev"* ]]; then + RC_PYGRAVITINO_VERSION="${PYGRAVITINO_VERSION}" +else + RC_PYGRAVITINO_VERSION="${PYGRAVITINO_VERSION}rc${RC_COUNT}" +fi + # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins # machines by explicitly calling /usr/sbin/lsof. LSOF=lsof @@ -194,6 +203,8 @@ if [[ "$1" == "package" ]]; then rm -f gravitino-$GRAVITINO_VERSION-src/LICENSE.bin rm -f gravitino-$GRAVITINO_VERSION-src/NOTICE.bin + rm -f gravitino-$GRAVITINO_VERSION-src/LICENSE.rest + rm -f gravitino-$GRAVITINO_VERSION-src/NOTICE.rest rm -f gravitino-$GRAVITINO_VERSION-src/web/LICENSE.bin rm -f gravitino-$GRAVITINO_VERSION-src/web/NOTICE.bin @@ -212,6 +223,7 @@ if [[ "$1" == "package" ]]; then echo "Creating distribution" + sed -i".tmp3" 's/ version=.*$/ version="'"$RC_PYGRAVITINO_VERSION"'",/g' clients/client-python/setup.py $GRADLE assembleDistribution -x test $GRADLE :clients:client-python:distribution -x test cd .. @@ -238,30 +250,33 @@ if [[ "$1" == "package" ]]; then shasum -a 512 gravitino-trino-connector-$GRAVITINO_VERSION.tar.gz > gravitino-trino-connector-$GRAVITINO_VERSION.tar.gz.sha512 echo "Copying and signing Gravitino Python client binary distribution" - cp gravitino-$GRAVITINO_VERSION-bin/clients/client-python/dist/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz . + cp gravitino-$GRAVITINO_VERSION-bin/clients/client-python/dist/apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz . echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour \ - --output apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc \ - --detach-sig apache_gravitino-$PYGRAVITINO_VERSION.tar.gz - shasum -a 512 apache_gravitino-$PYGRAVITINO_VERSION.tar.gz > apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.sha512 + --output apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz.asc \ + --detach-sig apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz + shasum -a 512 apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz > apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz.sha512 } make_binary_release rm -rf gravitino-$GRAVITINO_VERSION-bin/ if ! is_dry_run; then + if [[ -z "$PYPI_API_TOKEN" ]]; then + error 'The environment variable PYPI_API_TOKEN is not set. Exiting.' + fi + + echo "Uploading Gravitino Python package $RC_RC_PYGRAVITINO_VERSION to PyPi" + twine upload -u __token__ -p $PYPI_API_TOKEN \ + --repository-url https://upload.pypi.org/legacy/ \ + "apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz" \ + "apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz.asc" + svn co --depth=empty $RELEASE_STAGING_LOCATION svn-gravitino rm -rf "svn-gravitino/${DEST_DIR_NAME}" mkdir -p "svn-gravitino/${DEST_DIR_NAME}" echo "Copying release tarballs" cp gravitino-* "svn-gravitino/${DEST_DIR_NAME}/" - cp apache_gravitino-* "svn-gravitino/${DEST_DIR_NAME}/" - # remove python client tarball - # todo: remove this when python version supports include '-incubating' or the project is graduated - rm "svn-gravitino/${DEST_DIR_NAME}/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz" - rm "svn-gravitino/${DEST_DIR_NAME}/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc" - rm "svn-gravitino/${DEST_DIR_NAME}/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.sha512" - svn add "svn-gravitino/${DEST_DIR_NAME}" cd svn-gravitino @@ -279,11 +294,11 @@ if [[ "$1" == "docs" ]]; then cd gravitino-$GRAVITINO_VERSION-docs echo "Building Gravitino Java and Python docs" $GRADLE :clients:client-java:build -x test - $GRADLE :clients:client-python:pydoc + $GRADLE :clients:client-python:doc cd .. cp -r gravitino-$GRAVITINO_VERSION-docs/clients/client-java/build/docs/javadoc gravitino-$GRAVITINO_VERSION-javadoc - cp -r gravitino-$GRAVITINO_VERSION-docs/clients/client-python/docs gravitino-$PYGRAVITINO_VERSION-pydoc + cp -r gravitino-$GRAVITINO_VERSION-docs/clients/client-python/docs/build/html gravitino-$PYGRAVITINO_VERSION-pydoc rm -fr gravitino-$GRAVITINO_VERSION-docs fi diff --git a/dev/release/release-util.sh b/dev/release/release-util.sh index ca2c69be205..851f61a5d02 100755 --- a/dev/release/release-util.sh +++ b/dev/release/release-util.sh @@ -111,7 +111,6 @@ function get_release_info { # - If not, need to check whether the previous version has been already released or not. # - If it has, then we're building rc1 of the current version. # - If it has not, we're building the next RC of the previous version. - local RC_COUNT if [ $REV != 0 ]; then local PREV_REL_REV=$((REV - 1)) local PREV_REL_TAG="v${SHORT_VERSION}.${PREV_REL_REV}-incubating" @@ -134,6 +133,7 @@ function get_release_info { export RELEASE_VERSION=$(read_config "Release" "$RELEASE_VERSION") RC_COUNT=$(read_config "RC #" "$RC_COUNT") + export RC_COUNT # Check if the RC already exists, and if re-creating the RC, skip tag creation. RELEASE_TAG="v${RELEASE_VERSION}-rc${RC_COUNT}"