From 99e6f4dd1146171c8bf8c9d50acc6e9379e0215a Mon Sep 17 00:00:00 2001 From: Sam Darwin Date: Tue, 5 Nov 2024 17:27:14 -0700 Subject: [PATCH] antora jenkinsfile (#4) --- docs/jenkins-details.md | 2 + jenkinsfiles/antora_libraries_1 | 360 +++++++++++++++++++++ jenkinsfiles/standard_libraries_1 | 2 +- scripts/boostorg_buffers_prebuild.sh | 4 + scripts/boostorg_url_prebuild.sh | 4 + scripts/cppalliance_http_proto_prebuild.sh | 4 + 6 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 jenkinsfiles/antora_libraries_1 create mode 100755 scripts/boostorg_buffers_prebuild.sh create mode 100755 scripts/boostorg_url_prebuild.sh create mode 100755 scripts/cppalliance_http_proto_prebuild.sh diff --git a/docs/jenkins-details.md b/docs/jenkins-details.md index c8670cd..78ae3a9 100644 --- a/docs/jenkins-details.md +++ b/docs/jenkins-details.md @@ -11,6 +11,8 @@ Install Jenkins - https://www.jenkins.io/doc/book/installing/ Initially a n2-standard-2 on GCE, 100GB disk. That may change. +Manually create a jenkins:jenkins user with uid and gid of 150, so those values are predictable for use in containers. + Install the Jenkins package: ``` diff --git a/jenkinsfiles/antora_libraries_1 b/jenkinsfiles/antora_libraries_1 new file mode 100644 index 0000000..cd62edc --- /dev/null +++ b/jenkinsfiles/antora_libraries_1 @@ -0,0 +1,360 @@ +pipeline { + + // agent { + // node { + // label 'jenkinspool1' + // } + // } + + agent { + docker { + image 'cppalliance/boost_superproject_build:24.04-v2' + args '-v /var/lib/jenkins/jenkinsfilecache:/jenkinsfilecache' + // label 'jenkinspool1' + } + } + + stages { + stage('Preclean Workspace') { + steps { + sh """#!/bin/bash + set -xe + ls -al || true + rm -rf * .* + ls -al || true + """ + checkout scm + } + } + + stage('Set Variables') { + steps { + sh '''#!/bin/bash -xe + echo "" > jenkinsjobinfo.sh + chmod 777 jenkinsjobinfo.sh + REPONAME=$(basename -s .git "$(git config --get remote.origin.url)") + # REPONAME=$(basename `git rev-parse --show-toplevel`) + DNSREPONAME=$(echo $REPONAME | tr '_' '-') + ORGANIZATION=$(basename $(dirname "${GIT_URL}")) + echo "export JENKINS_CI_REPO=cppalliance/jenkins-ci" >> jenkinsjobinfo.sh + echo "export JENKINS_CI_REPO_BRANCH=master" >> jenkinsjobinfo.sh + # echo "export JENKINS_CI_REPO=sdarwin/jenkins-ci" >> jenkinsjobinfo.sh + # echo "export JENKINS_CI_REPO_BRANCH=testing" >> jenkinsjobinfo.sh + echo "export PRTEST=prtest" >> jenkinsjobinfo.sh + echo "export REPONAME=${REPONAME}" >> jenkinsjobinfo.sh + echo "export DNSREPONAME=${DNSREPONAME}" >> jenkinsjobinfo.sh + echo "export ORGANIZATION=${ORGANIZATION}" >> jenkinsjobinfo.sh + echo "export ONLY_BUILD_ON_DOCS_MODIFICATION=false" >> jenkinsjobinfo.sh + echo "export PATH_TO_DOCS=doc/build/site" >> jenkinsjobinfo.sh + ''' + } + } + + stage('Diagnostics') { + steps { + sh '''#!/bin/bash + set -x + # not set -e. errors may occur in diagnostics + cat jenkinsjobinfo.sh + . jenkinsjobinfo.sh + ls -al + cat /etc/os-release + pwd + env + whoami + touch $(date "+%A-%B-%d-%T-%y") + mount | grep ^/dev/ | grep -v /etc | awk '{print \$3}' + git branch + git branch -avv + true + ''' + } + } + + stage('Prebuild script') { + when { + anyOf{ + branch 'develop' + branch 'master' + expression { env.CHANGE_ID != null } + } + } + steps { + sh '''#!/bin/bash + set -xe + . jenkinsjobinfo.sh + curl -f -o jenkins_prebuild_script.sh https://raw.githubusercontent.com/${JENKINS_CI_REPO}/${JENKINS_CI_REPO_BRANCH}/scripts/${ORGANIZATION}_${REPONAME}_prebuild.sh || true + if [ -f jenkins_prebuild_script.sh ]; then + chmod 755 jenkins_prebuild_script.sh + ./jenkins_prebuild_script.sh + fi + ''' + } + } + + stage('Check if docs were modified') { + when { + anyOf{ + expression { env.CHANGE_ID != null } + } + } + steps { + sh '''#!/bin/bash + set -xe + . jenkinsjobinfo.sh + + if [ "$ONLY_BUILD_ON_DOCS_MODIFICATION" == "true" ]; then + echo "Starting check to see if docs have been updated." + git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*' + git fetch origin develop + mergebase=$(git merge-base HEAD remotes/origin/develop) + counter=0 + for i in $(git diff --name-only HEAD $mergebase) + do + echo "file is $i" + if [[ $i =~ ^doc/ ]]; then + counter=$((counter+1)) + fi + done + + if [ "$counter" -eq "0" ]; then + echo "No docs found. Exiting." + # exit 1 + echo "export AUTOCANCEL=true" >> jenkinsjobinfo.sh + else + echo "Found $counter docs. Proceeding." + fi + else + echo "Not checking if docs were updated. Always build." + fi + ''' + } + } + + stage('Determine if the job should exit') { + when { + anyOf{ + expression { env.CHANGE_ID != null } + } + } + environment { + // See https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables + AUTOCANCEL = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${AUTOCANCEL}"' + )}""" + } + steps { + script { + if (env.AUTOCANCEL == "true") { + currentBuild.result = 'ABORTED' + error("Aborting the build.") + } + } + } + } + + // To skip this step, and actually all steps, adjust the job's Filter regex in the Jenkins UI + // (develop|master|PR-.*) will build all branches + // (PR-.*) will build pull requests. Etc. + stage('Build docs') { + when { + anyOf{ + branch 'develop' + branch 'master' + expression { env.CHANGE_ID != null } + } + } + + steps { + sh '''#!/bin/bash + set -xe + . jenkinsjobinfo.sh + export pythonvirtenvpath=/opt/venvboostdocs + if [ -f ${pythonvirtenvpath}/bin/activate ]; then + source ${pythonvirtenvpath}/bin/activate + fi + + # Is there a custom build script? Often not. But let's check: + + curl -f -o jenkins_build_script.sh https://raw.githubusercontent.com/${JENKINS_CI_REPO}/${JENKINS_CI_REPO_BRANCH}/scripts/${ORGANIZATION}_${REPONAME}_build.sh || true + if [ -f jenkins_build_script.sh ]; then + chmod 755 jenkins_build_script.sh + ./jenkins_build_script.sh + exit 0 + fi + + # Otherwise, proceed using standard build steps: + + curl -s -S --retry 10 -L -o linuxdocs.sh https://github.com/boostorg/release-tools/raw/develop/build_docs/linuxdocs.sh + chmod 755 linuxdocs.sh + + # 2024-11 Antora requires a full copy of boostorg/boost which is 2GB. Not ideal. Let us provide that directory here. + # Since some implementations of antora/mrdocs seem to be using boost-root, symlink the filecache to the boost-root, + # so it will use the cache no-matter-what (hopefully). + savedworkspacedir=$(pwd) + cd /jenkinsfilecache + if [[ "${JOB_BASE_NAME}" =~ PR ]]; then + CACHE_FOLDER="PR" + BOOST_BRANCH="develop" + else + CACHE_FOLDER=${JOB_BASE_NAME} + BOOST_BRANCH=${BRANCH_NAME} + fi + mkdir -p ${REPONAME}/${CACHE_FOLDER} + cd ${REPONAME}/${CACHE_FOLDER} + if [ ! -d boost ]; then + git clone https://github.com/boostorg/boost -b ${BOOST_BRANCH} + fi + cd boost + git checkout ${BOOST_BRANCH} + git pull + git submodule update --init + FILECACHE_PATH=$(pwd) + cd libs + if [ ! -d buffers ]; then + git clone https://github.com/cppalliance/buffers -b ${BOOST_BRANCH} + else + cd buffers + git checkout -b ${BOOST_BRANCH} || true + git pull || true + cd .. + fi + cd .. + cd $savedworkspacedir + ln -s ${FILECACHE_PATH} boost-root || true + export BOOST_SRC_DIR=$(pwd)/boost-root + + # 2024-11 Antora requires a full copy of boostorg/boost which is 2GB. Not ideal. Let us provide that directory here. + + ./linuxdocs.sh --debug --boostrootsubdir + + # Another issue, if boost-root is now a symlink, it disappears when the docker container is gone + # Copy the docs "locally" + cp -rp boost-root/libs/${REPONAME}/doc/* doc/ + ''' + } + } + + stage('Postbuild script') { + when { + anyOf{ + branch 'develop' + branch 'master' + expression { env.CHANGE_ID != null } + } + } + steps { + sh '''#!/bin/bash + set -xe + + . jenkinsjobinfo.sh + curl -f -o jenkins_postbuild_script.sh https://raw.githubusercontent.com/${JENKINS_CI_REPO}/${JENKINS_CI_REPO_BRANCH}/scripts/${ORGANIZATION}_${REPONAME}_postbuild.sh || true + if [ -f jenkins_postbuild_script.sh ]; then + chmod 755 jenkins_postbuild_script.sh + ./jenkins_postbuild_script.sh + fi + ''' + } + } + + stage('Main branches: Upload to S3') { + when { + anyOf{ + branch 'develop' + branch 'master' + } + } + + environment { + // See https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables + REPONAME = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${REPONAME}"' + )}""" + DNSREPONAME = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${DNSREPONAME}"' + )}""" + PATH_TO_DOCS = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${PATH_TO_DOCS}"' + )}""" + } + + steps { + withAWS(region:'us-east-1', credentials: 'cppalliance-bot-aws-user') { + s3Upload(bucket:"cppalliance-websites", path:"${BRANCH_NAME}.${DNSREPONAME}.cpp.al", workingDir: "${PATH_TO_DOCS}", includePathPattern:"**") + } + } + } + + + stage('Pull requests: Upload to S3') { + when { + anyOf{ + expression { env.CHANGE_ID != null } + } + } + + environment { + // See https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables + REPONAME = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${REPONAME}"' + )}""" + DNSREPONAME = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${DNSREPONAME}"' + )}""" + PRTEST = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${PRTEST}"' + )}""" + PATH_TO_DOCS = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${PATH_TO_DOCS}"' + )}""" + DIFF2HTML = """${sh( + returnStdout: true, + script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${DIFF2HTML}"' + )}""" + + } + + steps { + withAWS(region:'us-east-1', credentials: 'cppalliance-bot-aws-user') { + s3Upload(bucket:"cppalliance-previews", path:"${DNSREPONAME}/${CHANGE_ID}/", workingDir: "${PATH_TO_DOCS}", includePathPattern:"**") + } + script { + + commenttext = "An automated preview of the documentation is available at [https://${env.CHANGE_ID}.${env.DNSREPONAME}.${env.PRTEST}.cppalliance.org/index.html](https://${env.CHANGE_ID}.${env.DNSREPONAME}.${env.PRTEST}.cppalliance.org/index.html)" + + pullRequest.comment(commenttext) + + } + } + } + + stage('Post Diagnostics') { + steps { + sh '''#!/bin/bash + set -x + # not set -e. errors may occur in diagnostics + cat jenkinsjobinfo.sh + . jenkinsjobinfo.sh + ls -al + cat /etc/os-release + pwd + env + whoami + touch $(date "+%A-%B-%d-%T-%y") + mount | grep ^/dev/ | grep -v /etc | awk '{print \$3}' + git branch + git branch -avv + true + ''' + } + } + } +} diff --git a/jenkinsfiles/standard_libraries_1 b/jenkinsfiles/standard_libraries_1 index 9b2e9d9..5bea258 100644 --- a/jenkinsfiles/standard_libraries_1 +++ b/jenkinsfiles/standard_libraries_1 @@ -18,7 +18,7 @@ pipeline { steps { sh """#!/bin/bash set -xe - rm -rf * + rm -rf * .* """ checkout scm } diff --git a/scripts/boostorg_buffers_prebuild.sh b/scripts/boostorg_buffers_prebuild.sh new file mode 100755 index 0000000..4f66ca7 --- /dev/null +++ b/scripts/boostorg_buffers_prebuild.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -xe +echo "export PRTEST=prtest2" >> jenkinsjobinfo.sh diff --git a/scripts/boostorg_url_prebuild.sh b/scripts/boostorg_url_prebuild.sh new file mode 100755 index 0000000..4f66ca7 --- /dev/null +++ b/scripts/boostorg_url_prebuild.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -xe +echo "export PRTEST=prtest2" >> jenkinsjobinfo.sh diff --git a/scripts/cppalliance_http_proto_prebuild.sh b/scripts/cppalliance_http_proto_prebuild.sh new file mode 100755 index 0000000..a431d92 --- /dev/null +++ b/scripts/cppalliance_http_proto_prebuild.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +set -xe +echo "export PRTEST=prtest" >> jenkinsjobinfo.sh