antora jenkinsfile (#4)

docs/jenkins-details.md

@@ -11,6 +11,8 @@ Install Jenkins - https://www.jenkins.io/doc/book/installing/
 
 Initially a n2-standard-2 on GCE, 100GB disk. That may change.   
 
+Manually create a jenkins:jenkins user with uid and gid of 150, so those values are predictable for use in containers.
+
 Install the Jenkins package:  
 
 ```

jenkinsfiles/antora_libraries_1

@@ -0,0 +1,360 @@
+pipeline {
+
+    // agent {
+    //     node {
+    //         label 'jenkinspool1'
+    //     }
+    // }
+
+    agent {
+        docker {
+          image 'cppalliance/boost_superproject_build:24.04-v2'
+          args '-v /var/lib/jenkins/jenkinsfilecache:/jenkinsfilecache'
+          // label 'jenkinspool1'
+          }
+    }
+
+    stages {
+        stage('Preclean Workspace') {
+            steps {
+                sh """#!/bin/bash
+                set -xe
+                ls -al || true
+                rm -rf * .*
+                ls -al || true
+                """
+                checkout scm
+            }
+        }
+
+        stage('Set Variables') {
+            steps {
+                sh '''#!/bin/bash -xe
+                echo "" > jenkinsjobinfo.sh
+                chmod 777 jenkinsjobinfo.sh
+                REPONAME=$(basename -s .git "$(git config --get remote.origin.url)")
+                # REPONAME=$(basename `git rev-parse --show-toplevel`)
+                DNSREPONAME=$(echo $REPONAME | tr '_' '-')
+                ORGANIZATION=$(basename $(dirname "${GIT_URL}"))
+                echo "export JENKINS_CI_REPO=cppalliance/jenkins-ci" >> jenkinsjobinfo.sh
+                echo "export JENKINS_CI_REPO_BRANCH=master" >> jenkinsjobinfo.sh
+                # echo "export JENKINS_CI_REPO=sdarwin/jenkins-ci" >> jenkinsjobinfo.sh
+                # echo "export JENKINS_CI_REPO_BRANCH=testing" >> jenkinsjobinfo.sh
+                echo "export PRTEST=prtest" >> jenkinsjobinfo.sh
+                echo "export REPONAME=${REPONAME}" >> jenkinsjobinfo.sh
+                echo "export DNSREPONAME=${DNSREPONAME}" >> jenkinsjobinfo.sh
+                echo "export ORGANIZATION=${ORGANIZATION}" >> jenkinsjobinfo.sh
+                echo "export ONLY_BUILD_ON_DOCS_MODIFICATION=false" >> jenkinsjobinfo.sh
+                echo "export PATH_TO_DOCS=doc/build/site" >> jenkinsjobinfo.sh
+                '''
+            }
+        }
+
+        stage('Diagnostics') {
+            steps {
+                sh '''#!/bin/bash
+                set -x
+                # not set -e. errors may occur in diagnostics
+                cat jenkinsjobinfo.sh
+                . jenkinsjobinfo.sh
+                ls -al
+                cat /etc/os-release
+                pwd
+                env 
+                whoami
+                touch $(date "+%A-%B-%d-%T-%y")
+                mount | grep ^/dev/ | grep -v /etc | awk '{print \$3}'
+                git branch
+                git branch -avv
+                true
+                '''
+            }
+        }
+
+        stage('Prebuild script') {
+            when {
+                anyOf{
+                    branch 'develop'
+                    branch 'master'
+                    expression { env.CHANGE_ID != null }
+                }
+           } 
+            steps {
+                sh '''#!/bin/bash
+                set -xe
+                . jenkinsjobinfo.sh
+                curl -f -o jenkins_prebuild_script.sh https://raw.githubusercontent.com/${JENKINS_CI_REPO}/${JENKINS_CI_REPO_BRANCH}/scripts/${ORGANIZATION}_${REPONAME}_prebuild.sh || true
+                if [ -f jenkins_prebuild_script.sh ]; then
+                    chmod 755 jenkins_prebuild_script.sh
+                    ./jenkins_prebuild_script.sh
+                fi
+                '''
+            }
+         }
+
+        stage('Check if docs were modified') {
+            when {
+                anyOf{
+                    expression { env.CHANGE_ID != null }
+                }
+           }
+            steps {
+                sh '''#!/bin/bash
+                set -xe
+                . jenkinsjobinfo.sh
+
+                if [ "$ONLY_BUILD_ON_DOCS_MODIFICATION" == "true" ]; then
+                    echo "Starting check to see if docs have been updated."
+                    git config remote.origin.fetch '+refs/heads/*:refs/remotes/origin/*'
+                    git fetch origin develop
+                    mergebase=$(git merge-base HEAD remotes/origin/develop)
+                    counter=0
+                    for i in $(git diff --name-only HEAD $mergebase)
+                    do
+                      echo "file is $i"
+                      if [[ $i =~ ^doc/ ]]; then
+                        counter=$((counter+1))
+                      fi
+                    done
+
+                    if [ "$counter" -eq "0" ]; then
+                      echo "No docs found. Exiting."
+                      # exit 1
+                      echo "export AUTOCANCEL=true" >> jenkinsjobinfo.sh
+                    else
+                      echo "Found $counter docs. Proceeding."
+                    fi
+                 else
+                     echo "Not checking if docs were updated. Always build."
+                 fi
+                '''
+            }
+         }
+
+        stage('Determine if the job should exit') {
+            when {
+                anyOf{
+                    expression { env.CHANGE_ID != null }
+                }
+            }
+             environment {
+                 // See https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables
+                  AUTOCANCEL = """${sh(
+                  returnStdout: true,
+                  script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${AUTOCANCEL}"'
+              )}"""
+             }
+              steps {
+                  script {
+                      if (env.AUTOCANCEL == "true") {
+                         currentBuild.result = 'ABORTED'
+                         error("Aborting the build.")
+                       }
+                  }
+              }
+         }
+
+        // To skip this step, and actually all steps, adjust the job's Filter regex in the Jenkins UI
+        // (develop|master|PR-.*) will build all branches
+        // (PR-.*) will build pull requests. Etc.
+        stage('Build docs') {
+            when {
+                anyOf{
+                    branch 'develop'
+                    branch 'master'
+                    expression { env.CHANGE_ID != null }
+                }
+            } 
+
+            steps {
+                sh '''#!/bin/bash
+                set -xe
+                . jenkinsjobinfo.sh
+                export pythonvirtenvpath=/opt/venvboostdocs
+                if [ -f ${pythonvirtenvpath}/bin/activate ]; then
+                    source ${pythonvirtenvpath}/bin/activate
+                fi
+
+                # Is there a custom build script? Often not. But let's check:
+
+                curl -f -o jenkins_build_script.sh https://raw.githubusercontent.com/${JENKINS_CI_REPO}/${JENKINS_CI_REPO_BRANCH}/scripts/${ORGANIZATION}_${REPONAME}_build.sh || true
+                if [ -f jenkins_build_script.sh ]; then
+                    chmod 755 jenkins_build_script.sh
+                    ./jenkins_build_script.sh
+                    exit 0
+                fi
+
+                # Otherwise, proceed using standard build steps:
+
+                curl -s -S --retry 10 -L -o linuxdocs.sh https://github.com/boostorg/release-tools/raw/develop/build_docs/linuxdocs.sh
+                chmod 755 linuxdocs.sh
+
+                # 2024-11 Antora requires a full copy of boostorg/boost which is 2GB. Not ideal. Let us provide that directory here.
+                # Since some implementations of antora/mrdocs seem to be using boost-root, symlink the filecache to the boost-root,
+		# so it will use the cache no-matter-what (hopefully).
+                savedworkspacedir=$(pwd)
+                cd /jenkinsfilecache
+                if [[ "${JOB_BASE_NAME}" =~ PR ]]; then
+                    CACHE_FOLDER="PR"
+                    BOOST_BRANCH="develop"
+                else
+                    CACHE_FOLDER=${JOB_BASE_NAME}
+                    BOOST_BRANCH=${BRANCH_NAME}
+                fi
+                mkdir -p ${REPONAME}/${CACHE_FOLDER}
+                cd ${REPONAME}/${CACHE_FOLDER}
+                if [ ! -d boost ]; then
+                    git clone https://github.com/boostorg/boost -b ${BOOST_BRANCH}
+                fi
+                cd boost
+                git checkout ${BOOST_BRANCH}
+                git pull
+                git submodule update --init
+                FILECACHE_PATH=$(pwd)
+                cd libs
+                if [ ! -d buffers ]; then
+                   git clone https://github.com/cppalliance/buffers -b ${BOOST_BRANCH}
+                else
+                   cd buffers
+                   git checkout -b ${BOOST_BRANCH} || true
+                   git pull || true
+                   cd ..
+                fi
+                cd ..
+                cd $savedworkspacedir
+                ln -s ${FILECACHE_PATH} boost-root || true
+                export BOOST_SRC_DIR=$(pwd)/boost-root
+
+                # </done> 2024-11 Antora requires a full copy of boostorg/boost which is 2GB. Not ideal. Let us provide that directory here.
+
+                ./linuxdocs.sh --debug --boostrootsubdir
+
+		# Another issue, if boost-root is now a symlink, it disappears when the docker container is gone  
+                # Copy the docs "locally"
+                cp -rp boost-root/libs/${REPONAME}/doc/* doc/
+                '''
+                }
+            }
+
+        stage('Postbuild script') {
+            when {
+                anyOf{
+                    branch 'develop'
+                    branch 'master'
+                    expression { env.CHANGE_ID != null }
+                }
+           }
+            steps {
+                sh '''#!/bin/bash
+                set -xe
+
+                . jenkinsjobinfo.sh
+                curl -f -o jenkins_postbuild_script.sh https://raw.githubusercontent.com/${JENKINS_CI_REPO}/${JENKINS_CI_REPO_BRANCH}/scripts/${ORGANIZATION}_${REPONAME}_postbuild.sh || true
+                if [ -f jenkins_postbuild_script.sh ]; then
+                    chmod 755 jenkins_postbuild_script.sh
+                    ./jenkins_postbuild_script.sh
+                fi
+                '''
+            }
+         }
+
+        stage('Main branches: Upload to S3') {
+            when {
+                anyOf{
+                    branch 'develop'
+                    branch 'master'
+                }
+            }
+
+            environment {
+                // See https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables
+                 REPONAME = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${REPONAME}"'
+             )}"""
+                 DNSREPONAME = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${DNSREPONAME}"'
+             )}"""
+                 PATH_TO_DOCS = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${PATH_TO_DOCS}"'
+             )}"""
+            }
+
+            steps {
+                withAWS(region:'us-east-1', credentials: 'cppalliance-bot-aws-user') {
+                    s3Upload(bucket:"cppalliance-websites", path:"${BRANCH_NAME}.${DNSREPONAME}.cpp.al", workingDir: "${PATH_TO_DOCS}", includePathPattern:"**")
+                }
+            }
+        }
+
+
+        stage('Pull requests: Upload to S3') {
+            when { 
+                anyOf{
+                    expression { env.CHANGE_ID != null }
+                }
+            }
+
+            environment {
+                // See https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables
+                 REPONAME = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${REPONAME}"'
+             )}"""
+                 DNSREPONAME = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${DNSREPONAME}"'
+             )}"""
+                 PRTEST = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${PRTEST}"'
+             )}"""
+                 PATH_TO_DOCS = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${PATH_TO_DOCS}"'
+             )}"""
+                 DIFF2HTML = """${sh(
+                 returnStdout: true,
+                 script: '#!/bin/bash \n' + 'source jenkinsjobinfo.sh; echo -n "${DIFF2HTML}"'
+             )}"""
+
+            }
+
+            steps {
+                withAWS(region:'us-east-1', credentials: 'cppalliance-bot-aws-user') {
+                    s3Upload(bucket:"cppalliance-previews", path:"${DNSREPONAME}/${CHANGE_ID}/", workingDir: "${PATH_TO_DOCS}", includePathPattern:"**")
+                }
+                script {
+
+                    commenttext = "An automated preview of the documentation is available at [https://${env.CHANGE_ID}.${env.DNSREPONAME}.${env.PRTEST}.cppalliance.org/index.html](https://${env.CHANGE_ID}.${env.DNSREPONAME}.${env.PRTEST}.cppalliance.org/index.html)"
+
+                    pullRequest.comment(commenttext)
+
+                }
+            }
+        }
+
+        stage('Post Diagnostics') {
+            steps {
+                sh '''#!/bin/bash
+                set -x
+                # not set -e. errors may occur in diagnostics
+                cat jenkinsjobinfo.sh
+                . jenkinsjobinfo.sh
+                ls -al
+                cat /etc/os-release
+                pwd
+                env
+                whoami
+                touch $(date "+%A-%B-%d-%T-%y") 
+                mount | grep ^/dev/ | grep -v /etc | awk '{print \$3}' 
+                git branch
+                git branch -avv
+                true
+                '''
+            }
+        }
+    }
+}

jenkinsfiles/standard_libraries_1

@@ -18,7 +18,7 @@ pipeline {
             steps {
                 sh """#!/bin/bash
                 set -xe
-                rm -rf *
+                rm -rf * .*
                 """
                 checkout scm
             }

scripts/boostorg_buffers_prebuild.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -xe
+echo "export PRTEST=prtest2" >> jenkinsjobinfo.sh

scripts/boostorg_url_prebuild.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -xe
+echo "export PRTEST=prtest2" >> jenkinsjobinfo.sh

scripts/cppalliance_http_proto_prebuild.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -xe
+echo "export PRTEST=prtest" >> jenkinsjobinfo.sh