diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b68dd303841d8..a0b71fcff02e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -747,13 +747,7 @@ publish-rustdoc: image: node:16 variables: GIT_DEPTH: 100 - # --- Following variables are for rustdocs deployment --- - # Space separated values of branches and tags to generate rustdocs - RUSTDOCS_DEPLOY_REFS: "master monthly-2021-09+1 monthly-2021-08 v3.0.0" - # Location of the docs index template - INDEX_TPL: ".maintain/docs-index-tpl.ejs" - # Where the `/latest` symbolic link links to. One of the $RUSTDOCS_DEPLOY_REFS value. - LATEST: "monthly-2021-09+1" + RUSTDOCS_DEPLOY_REFS: "master" rules: - if: $CI_PIPELINE_SOURCE == "pipeline" when: never @@ -783,34 +777,18 @@ publish-rustdoc: - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - git fetch origin gh-pages - # Install `ejs` and generate index.html based on RUSTDOCS_DEPLOY_REFS - - yarn global add ejs - - 'ejs ${INDEX_TPL} -i "{\"deploy_refs\":\"${RUSTDOCS_DEPLOY_REFS}\",\"repo_name\":\"${CI_PROJECT_NAME}\",\"latest\":\"${LATEST}\"}" > /tmp/index.html' # Save README and docs - cp -r ./crate-docs/ /tmp/doc/ - cp README.md /tmp/doc/ - git checkout gh-pages - # Remove directories no longer necessary, as specified in $RUSTDOCS_DEPLOY_REFS. - # Also ensure $RUSTDOCS_DEPLOY_REFS is not just empty spaces. - # Even though this block spans multiple lines, they are concatenated to run as a single line - # command, so note for the semi-colons in the inner-most code block. - - if [[ ! -z ${RUSTDOCS_DEPLOY_REFS// } ]]; then - for FILE in *; do - if [[ ! " $RUSTDOCS_DEPLOY_REFS " =~ " $FILE " ]]; then - echo "Removing ${FILE}..."; - rm -rf $FILE; - fi - done - fi - # Move the index page & built back - - mv -f /tmp/index.html . + # Install `index-tpl-crud` and generate index.html based on RUSTDOCS_DEPLOY_REFS + - which index-tpl-crud &> /dev/null || yarn global add @jimmychu0807/index-tpl-crud + - index-tpl-crud upsert ./index.html ${CI_COMMIT_REF_NAME} # Ensure the destination dir doesn't exist. - rm -rf ${CI_COMMIT_REF_NAME} - mv -f /tmp/doc ${CI_COMMIT_REF_NAME} - # Add the symlink - - '[[ -e "$LATEST" ]] && ln -sf "${LATEST}" latest' # Upload files - - git add --all --force + - git add --all # `git commit` has an exit code of > 0 if there is nothing to commit. # This causes GitLab to exit immediately and marks this job failed. # We don't want to mark the entire job failed if there's nothing to diff --git a/.maintain/rustdocs-release.sh b/.maintain/rustdocs-release.sh new file mode 100755 index 0000000000000..8a250849ee7a6 --- /dev/null +++ b/.maintain/rustdocs-release.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash +# set -x + +# This script manages the deployment of Substrate rustdocs to https://paritytech.github.io/substrate/. +# - With `deploy` sub-command, it will checkout the passed-in branch/tag ref, build the rustdocs +# locally (this takes some time), update the `index.html` index page, and push it to remote +# `gh-pages` branch. So users running this command need to have write access to the remote +# `gh-pages` branch. This sub-command depends on [@jimmychu0807/index-tpl-crud](https://www.npmjs.com/package/@jimmychu0807/index-tpl-crud) +# to update the DOM of index.html page. +# - With `remove` sub-command, it will remove the deployed rustdocs from `gh-pages`, and update the +# index.html page as necessary. It may remove the `latest` symbolic link. +# +# Examples: +# # Showing help text +# rustdocs-release.sh -h +# +# # Deploy rustdocs of `monthly-2021-10` tag +# rustdocs-release.sh deploy monthly-2021-10 +# +# # In addition to the above, the `latest` symlink will point to this version of rustdocs +# rustdocs-release.sh deploy -l monthly-2021-10 +# +# # Remove the rustdocs of `monthly-2021-10` from `gh-pages`. +# rustdocs-release.sh remove monthly-2021-10 +# +# Dependencies: +# - @jimmychu0807/index-tpl-crud - https://www.npmjs.com/package/@jimmychu0807/index-tpl-crud +# + +# Script setting +# The git repo http URL +REMOTE_REPO="https://github.com/paritytech/substrate.git" +TMP_PREFIX="/tmp" # tmp location that the built doc is copied to. +DOC_INDEX_PAGE="sc_service/index.html" + +# Set to `true` if using cargo `nightly` toolchain to build the doc. +# Set to `false` to use the default cargo toolchain. This is preferred if you want to build +# the rustdocs with a pinned nightly version set to your default toolchain. +CARGO_NIGHTLY=false + +# Set the git remote name. Most of the time the default is `origin`. +GIT_REMOTE="origin" +LATEST=false + +# Setting the help text +declare -A HELP_TXT +HELP_TXT["deploy"]=$(cat <<-EOH +Build and deploy the rustdocs of the specified branch/tag to \`gh-pages\` branch. + + usage: $0 deploy [-l] + example: $0 deploy -l monthly-2021-10 + + options: + -l The \`latest\` path will be sym'linked to this rustdocs version +EOH +) + +HELP_TXT["remove"]=$(cat <<-EOH +Remove the rustdocs of the specified version from \`gh-pages\` branch. + + usage: $0 remove + example: $0 remove monthly-2021-10 +EOH +) + +set_and_check_rustdoc_ref() { + [[ -z "$1" ]] && { + echo -e "git branch_ref is not specified.\n" + echo "${HELP_TXT[$2]}" + exit 1 + } + BUILD_RUSTDOC_REF=$1 +} + +check_local_change() { + # Check there is no local changes before proceeding + [[ -n $(git status --porcelain) ]] \ + && echo "Local changes exist, please either discard or commit them as this command will change the current checkout branch." \ + && exit 1 +} + +build_rustdocs() { + # Build the docs + time cargo $($CARGO_NIGHTLY && echo "+nightly") doc --workspace --all-features --verbose \ + || { echo "Generate $1 rustdocs failed" && exit 1; } + rm -f target/doc/.lock + + # Moving the built doc to the tmp location + mv target/doc "${2}" + [[ -n "${DOC_INDEX_PAGE}" ]] \ + && echo "" > "${2}/index.html" +} + +upsert_index_page() { + # Check if `index-tpl-crud` exists + which index-tpl-crud &> /dev/null || yarn global add @jimmychu0807/index-tpl-crud + index-tpl-crud upsert $($1 && echo "-l") ./index.html "$2" +} + +rm_index_page() { + which index-tpl-crud &> /dev/null || yarn global add @jimmychu0807/index-tpl-crud + index-tpl-crud rm ./index.html "$1" +} + +git_add_commit_push() { + git add --all + git commit -m "$1" || echo "Nothing to commit" + git push "${GIT_REMOTE}" gh-pages --force +} + +import_gh_key() { + [[ -n $GITHUB_SSH_PRIV_KEY ]] && { + eval $(ssh-agent) + ssh-add - <<< $GITHUB_SSH_PRIV_KEY + } + + # Adding github.com as known_hosts + ssh-keygen -F github.com &>/dev/null || { + [[ -e ~/.ssh ]] || mkdir ~/.ssh + [[ -e ~/.ssh/known_hosts ]] || touch ~/.ssh/known_hosts + ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + } +} + +deploy_main() { + check_local_change + import_gh_key + + CURRENT_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + TMP_PROJECT_PATH="${TMP_PREFIX}/${PROJECT_NAME}" + DOC_PATH="${TMP_PROJECT_PATH}/${BUILD_RUSTDOC_REF}" + + # Build the tmp project path + rm -rf "${TMP_PROJECT_PATH}" && mkdir "${TMP_PROJECT_PATH}" + + # Copy .gitignore file to tmp + [[ -e "${PROJECT_PATH}/.gitignore" ]] && cp "${PROJECT_PATH}/.gitignore" "${TMP_PROJECT_PATH}" + + git fetch --all + git checkout -f ${BUILD_RUSTDOC_REF} || { echo "Checkout \`${BUILD_RUSTDOC_REF}\` error." && exit 1; } + build_rustdocs "${BUILD_RUSTDOC_REF}" "${DOC_PATH}" + + # git checkout `gh-pages` branch + git fetch "${GIT_REMOTE}" gh-pages + git checkout gh-pages + # Move the built back + [[ -e "${TMP_PROJECT_PATH}/.gitignore" ]] && cp -f "${TMP_PROJECT_PATH}/.gitignore" . + # Ensure the destination dir doesn't exist under current path. + rm -rf "${BUILD_RUSTDOC_REF}" + mv -f "${DOC_PATH}" "${BUILD_RUSTDOC_REF}" + + upsert_index_page $LATEST "${BUILD_RUSTDOC_REF}" + # Add the latest symlink + $LATEST && rm -rf latest && ln -sf "${BUILD_RUSTDOC_REF}" latest + + git_add_commit_push "___Deployed rustdocs of ${BUILD_RUSTDOC_REF}___" + # Clean up + # Remove the tmp asset created + rm -rf "${TMP_PROJECT_PATH}" + # Resume back previous checkout branch. + git checkout -f "$CURRENT_GIT_BRANCH" +} + +remove_main() { + check_local_change + import_gh_key + + CURRENT_GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + + # git checkout `gh-pages` branch + git fetch "${GIT_REMOTE}" gh-pages + git checkout gh-pages + + rm -rf "${BUILD_RUSTDOC_REF}" + rm_index_page "${BUILD_RUSTDOC_REF}" + # check if the destination of `latest` exists and rmove if not. + [[ -e "latest" ]] || rm latest + + git_add_commit_push "___Removed rustdocs of ${BUILD_RUSTDOC_REF}___" + + # Resume back previous checkout branch. + git checkout -f "$CURRENT_GIT_BRANCH" +} + +# ---- The script execution entry point starts here ---- + +# Arguments handling +SUBCMD=$1 +[[ $SUBCMD == "deploy" || $SUBCMD == "remove" ]] \ + || { echo "Please specify a subcommand of \`deploy\` or \`remove\`" && exit 1 ;} +shift + +# After removing the subcommand, there could only be 1 or 2 parameters afterward +[[ $# -lt 1 || $# -gt 2 ]] && { + echo "${HELP_TXT[${SUBCMD}]}" + exit 1 +} + +# Parsing options and argument for `deploy` subcommand +[[ $SUBCMD == "deploy" ]] && { + while getopts :lh opt; do + case $opt in + l) + LATEST=true + ;; + h) + echo "${HELP_TXT[$SUBCMD]}" + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac + done +} +# Parsing options and argument for `remove` subcommand +[[ $SUBCMD == "remove" ]] && { + while getopts :h opt; do + case $opt in + h) + echo "${HELP_TXT[$SUBCMD]}" + exit 0 + ;; + \?) + echo "Invalid option: -$OPTARG" >&2 + exit 1 + ;; + esac + done +} + +shift $(($OPTIND - 1)) +set_and_check_rustdoc_ref ${1:-''} $SUBCMD + +SCRIPT=$(realpath $0) +SCRIPT_PATH=$(dirname $SCRIPT) +PROJECT_PATH=$(dirname ${SCRIPT_PATH}) +PROJECT_NAME=$(basename "$PROJECT_PATH") + +pushd "${PROJECT_PATH}" &>/dev/null +[[ $SUBCMD == "deploy" ]] && deploy_main +[[ $SUBCMD == "remove" ]] && remove_main +popd &>/dev/null diff --git a/docs/rustdocs-release.md b/docs/rustdocs-release.md new file mode 100644 index 0000000000000..5c7e2db40a2cd --- /dev/null +++ b/docs/rustdocs-release.md @@ -0,0 +1,20 @@ +# Rustdocs Release Process + +There is [a script in place](../.maintain/rustdocs-release.sh) to manage the deployment of Substrate rustdocs at +https://paritytech.github.io/substrate, which is pushing the rustdocs file in `gh-pages` branch of +https://github.com/paritytech/substrate. + +The documentation at the top of the `rustdocs-release.sh` explains most of the mechanics of the script. + +Manage the rustdocs deployment with one of the following commands. + +```bash +# Deploy rustdocs of `monthly-2021-10` tag +.maintain/rustdocs-release.sh deploy monthly-2021-10 + +# In addition to the above, the `latest` symlink will point to this version of rustdocs +.maintain/rustdocs-release.sh deploy -l monthly-2021-10 + +# Remove the rustdocs of `monthly-2021-10` from `gh-pages`. +.maintain/rustdocs-release.sh remove monthly-2021-10 +```