[MINOR] improve(build): Improve the release script (#5340)

### What changes were proposed in this pull request? * Make Python package built and publish automatically. * Change the Python doc generation command. * Remove the Python package from SVN. ### Why are the changes needed? To meet the new code changes in 0.7. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Manual verification.
apache · Nov 6, 2024 · 07d42f5 · 07d42f5
1 parent 6776402
commit 07d42f5
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 23 deletions.
diff --git a/dev/release/do-release.sh b/dev/release/do-release.sh
@@ -33,7 +33,7 @@ done
 DRY_RUN=${DRY_RUN:-0}
 export DRY_RUN
 
-cmds=("git" "gpg" "svn" "twine" "shasum" "sha1sum" "jq")
+cmds=("git" "gpg" "svn" "twine" "shasum" "sha1sum" "jq" "make")
 for cmd in "${cmds[@]}"; do
   if ! command -v $cmd &> /dev/null; then
     echo "$cmd is required to run this script."

diff --git a/dev/release/release-build.sh b/dev/release/release-build.sh
@@ -122,21 +122,24 @@ if [[ "$1" == "finalize" ]]; then
     echo "git tag v$RELEASE_VERSION created"
   fi
 
+  PYGRAVITINO_VERSION="${RELEASE_VERSION/-incubating}"
+  git clone "https://$ASF_USERNAME:$ENCODED_ASF_PASSWORD@$ASF_GRAVITINO_REPO" -b "v$RELEASE_VERSION"
+  cd gravitino
+  $GRADLE :clients:client-python:distribution -x test
+  cd ..
+  cp gravitino/clients/client-python/dist/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz .
+  echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour \
+    --output apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc \
+    --detach-sig apache_gravitino-$PYGRAVITINO_VERSION.tar.gz
+
   # upload to PyPi.
-  # todo: uncomment below codes if possible, it will download Gravitino Python binary from the dev directory
   echo "Uploading Gravitino to PyPi"
-  # svn co --depth=empty "$RELEASE_STAGING_LOCATION/$RELEASE_TAG" svn-gravitino
-  # cd svn-gravitino
-  PYGRAVITINO_VERSION=`echo "$RELEASE_VERSION" |  sed -e "s/-/./" -e "s/preview/dev/"`
-  # svn update "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz"
-  # svn update "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc"
   twine upload -u __token__  -p $PYPI_API_TOKEN \
     --repository-url https://upload.pypi.org/legacy/ \
     "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz" \
     "apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc"
-  # cd ..
-  # rm -rf svn-gravitino
   echo "Python Gravitino package uploaded"
+  rm -fr gravitino
 
   # Moves the binaries from dev directory to release directory.
   echo "Moving Gravitino binaries to the release directory"
@@ -171,6 +174,12 @@ if [ -z "$PYGRAVITINO_VERSION"]; then
   PYGRAVITINO_VERSION=$(cat clients/client-python/setup.py | grep "version=" | awk -F"\"" '{print $2}')
 fi
 
+if [[ "$PYGRAVITINO_VERSION" == *"dev"* ]]; then
+  RC_PYGRAVITINO_VERSION="${PYGRAVITINO_VERSION}"
+else
+  RC_PYGRAVITINO_VERSION="${PYGRAVITINO_VERSION}rc${RC_COUNT}"
+fi
+
 # This is a band-aid fix to avoid the failure of Maven nightly snapshot in some Jenkins
 # machines by explicitly calling /usr/sbin/lsof.
 LSOF=lsof
@@ -194,6 +203,8 @@ if [[ "$1" == "package" ]]; then
 
   rm -f gravitino-$GRAVITINO_VERSION-src/LICENSE.bin
   rm -f gravitino-$GRAVITINO_VERSION-src/NOTICE.bin
+  rm -f gravitino-$GRAVITINO_VERSION-src/LICENSE.rest
+  rm -f gravitino-$GRAVITINO_VERSION-src/NOTICE.rest
   rm -f gravitino-$GRAVITINO_VERSION-src/web/LICENSE.bin
   rm -f gravitino-$GRAVITINO_VERSION-src/web/NOTICE.bin
 
@@ -212,6 +223,7 @@ if [[ "$1" == "package" ]]; then
 
     echo "Creating distribution"
 
+    sed -i".tmp3" 's/    version=.*$/    version="'"$RC_PYGRAVITINO_VERSION"'",/g' clients/client-python/setup.py
     $GRADLE assembleDistribution -x test
     $GRADLE :clients:client-python:distribution -x test
     cd ..
@@ -238,30 +250,33 @@ if [[ "$1" == "package" ]]; then
     shasum -a 512 gravitino-trino-connector-$GRAVITINO_VERSION.tar.gz > gravitino-trino-connector-$GRAVITINO_VERSION.tar.gz.sha512
 
     echo "Copying and signing Gravitino Python client binary distribution"
-    cp gravitino-$GRAVITINO_VERSION-bin/clients/client-python/dist/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz .
+    cp gravitino-$GRAVITINO_VERSION-bin/clients/client-python/dist/apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz .
     echo $GPG_PASSPHRASE | $GPG --passphrase-fd 0 --armour \
-      --output apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc \
-      --detach-sig apache_gravitino-$PYGRAVITINO_VERSION.tar.gz
-    shasum -a 512 apache_gravitino-$PYGRAVITINO_VERSION.tar.gz > apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.sha512
+      --output apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz.asc \
+      --detach-sig apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz
+    shasum -a 512 apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz > apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz.sha512
   }
 
   make_binary_release
   rm -rf gravitino-$GRAVITINO_VERSION-bin/
 
   if ! is_dry_run; then
+    if [[ -z "$PYPI_API_TOKEN" ]]; then
+      error 'The environment variable PYPI_API_TOKEN is not set. Exiting.'
+    fi
+
+    echo "Uploading Gravitino Python package $RC_RC_PYGRAVITINO_VERSION to PyPi"
+    twine upload -u __token__  -p $PYPI_API_TOKEN \
+      --repository-url https://upload.pypi.org/legacy/ \
+      "apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz" \
+      "apache_gravitino-$RC_PYGRAVITINO_VERSION.tar.gz.asc"
+
     svn co --depth=empty $RELEASE_STAGING_LOCATION svn-gravitino
     rm -rf "svn-gravitino/${DEST_DIR_NAME}"
     mkdir -p "svn-gravitino/${DEST_DIR_NAME}"
 
     echo "Copying release tarballs"
     cp gravitino-* "svn-gravitino/${DEST_DIR_NAME}/"
-    cp apache_gravitino-* "svn-gravitino/${DEST_DIR_NAME}/"
-    # remove python client tarball
-    # todo: remove this when python version supports include '-incubating' or the project is graduated
-    rm "svn-gravitino/${DEST_DIR_NAME}/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz"
-    rm "svn-gravitino/${DEST_DIR_NAME}/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.asc"
-    rm "svn-gravitino/${DEST_DIR_NAME}/apache_gravitino-$PYGRAVITINO_VERSION.tar.gz.sha512"
-
     svn add "svn-gravitino/${DEST_DIR_NAME}"
 
     cd svn-gravitino
@@ -279,11 +294,11 @@ if [[ "$1" == "docs" ]]; then
   cd gravitino-$GRAVITINO_VERSION-docs
   echo "Building Gravitino Java and Python docs"
   $GRADLE :clients:client-java:build -x test
-  $GRADLE :clients:client-python:pydoc
+  $GRADLE :clients:client-python:doc
   cd ..
 
   cp -r gravitino-$GRAVITINO_VERSION-docs/clients/client-java/build/docs/javadoc gravitino-$GRAVITINO_VERSION-javadoc
-  cp -r gravitino-$GRAVITINO_VERSION-docs/clients/client-python/docs gravitino-$PYGRAVITINO_VERSION-pydoc
+  cp -r gravitino-$GRAVITINO_VERSION-docs/clients/client-python/docs/build/html gravitino-$PYGRAVITINO_VERSION-pydoc
 
   rm -fr gravitino-$GRAVITINO_VERSION-docs
 fi

diff --git a/dev/release/release-util.sh b/dev/release/release-util.sh
@@ -111,7 +111,6 @@ function get_release_info {
   # - If not, need to check whether the previous version has been already released or not.
   #   - If it has, then we're building rc1 of the current version.
   #   - If it has not, we're building the next RC of the previous version.
-  local RC_COUNT
   if [ $REV != 0 ]; then
     local PREV_REL_REV=$((REV - 1))
     local PREV_REL_TAG="v${SHORT_VERSION}.${PREV_REL_REV}-incubating"
@@ -134,6 +133,7 @@ function get_release_info {
   export RELEASE_VERSION=$(read_config "Release" "$RELEASE_VERSION")
 
   RC_COUNT=$(read_config "RC #" "$RC_COUNT")
+  export RC_COUNT
 
   # Check if the RC already exists, and if re-creating the RC, skip tag creation.
   RELEASE_TAG="v${RELEASE_VERSION}-rc${RC_COUNT}"