kie-issues#821: Setup weekly cloud jobs (#1177)

* Setup weekly cloud jobs

* Update agent label to use the utility

.ci/jenkins/Jenkinsfile.weekly

@@ -79,6 +79,19 @@ pipeline {
                 }
             }
         }
+
+        stage('Start Cloud weekly') {
+            when {
+                expression { !params.SKIP_CLOUD_WEEKLY }
+            }
+            steps {
+                script {
+                    def buildParams = getDefaultBuildParams()
+                    addSkipTestsParam(buildParams)
+                    build(job: './0-kogito-weekly-cloud', wait: false, parameters: buildParams, propagate: false)
+                }
+            }
+        }
     }
     post {
         unsuccessful {

.ci/jenkins/Jenkinsfile.weekly.cloud

@@ -0,0 +1,260 @@
+import org.jenkinsci.plugins.workflow.libs.Library
+
+@Library('jenkins-pipeline-shared-libraries')_
+
+// Deploy jobs
+IMAGES_DEPLOY = 'kogito-images.weekly-deploy'
+SEVERLESS_OPERATOR_DEPLOY = 'kogito-serverless-operator.weekly-deploy'
+
+// Map of executed jobs
+// See https://javadoc.jenkins.io/plugin/workflow-support/org/jenkinsci/plugins/workflow/support/steps/build/RunWrapper.html
+// for more options on built job entity
+JOBS = [:]
+
+FAILED_STAGES = [:]
+UNSTABLE_STAGES = [:]
+
+defaultImageParamsPrefix = 'IMAGE'
+
+// Should be multibranch pipeline
+pipeline {
+    agent {
+        label util.getLabel('ubuntu')
+    }
+
+    options {
+        timeout(time: 1380, unit: 'MINUTES')
+    }
+
+    // parameters {
+    // For parameters, check into ./dsl/jobs.groovy file
+    // }
+
+    environment {
+        // Some generated env is also defined into ./dsl/jobs.groovy file
+
+        KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}")
+
+        IMAGE_NAME_WEEKLY_SUFFIX = 'nightly'
+
+        // Use branch name in weekly tag as we may have parallel main and release branch builds
+        WEEKLY_TAG = """${getBuildBranch()}-${getCurrentDate()}"""
+    }
+
+    stages {
+        stage('Initialize') {
+            steps {
+                script {
+                    echo "weekly tag is ${env.WEEKLY_TAG}"
+
+                    currentBuild.displayName = env.WEEKLY_TAG
+                }
+            }
+        }
+
+        stage('Build & Deploy Images') {
+            when {
+                expression { return isImagesDeploy() }
+            }
+            steps {
+                script {
+                    def buildParams = getDefaultBuildParams()
+                    addImageBuildParams(buildParams, env.WEEKLY_TAG)
+                    addDeployImageWithLatestTagParam(buildParams)
+
+                    // For building
+                    addAppsParam(buildParams)
+
+                    // For testing
+                    addSkipTestsParam(buildParams)
+                    addExamplesParam(buildParams)
+
+                    buildJob(IMAGES_DEPLOY, buildParams)
+                }
+            }
+            post {
+                failure {
+                    addFailedStage(IMAGES_DEPLOY)
+                }
+            }
+        }
+
+        stage('Build & Deploy Serverless Operator') {
+            when {
+                expression { return isOperatorDeploy() }
+            }
+            steps {
+                script {
+                    def buildParams = getDefaultBuildParams()
+                    addSkipTestsParam(buildParams)
+                    addImageBuildParams(buildParams, env.WEEKLY_TAG)
+                    addDeployImageWithLatestTagParam(buildParams)
+
+                    buildJob(SEVERLESS_OPERATOR_DEPLOY, buildParams)
+                }
+            }
+            post {
+                failure {
+                    addFailedStage(SEVERLESS_OPERATOR_DEPLOY)
+                }
+            }
+        }
+    }
+    post {
+        unsuccessful {
+            sendPipelineErrorNotification()
+        }
+    }
+}
+
+def buildJob(String jobName, List buildParams, String jobKey = jobName) {
+    echo "[${jobKey}] Build ${jobName} with params ${buildParams}"
+
+    def job = build(job: "${jobName}", wait: true, parameters: buildParams, propagate: false)
+    JOBS[jobKey] = job
+
+    // Set Unstable if job did not succeed
+    if (!isJobSucceeded(jobKey)) {
+        addUnstableStage(jobKey)
+        unstable("Job ${jobName} finished with result ${job.result}")
+    }
+    return job
+}
+
+def getJob(String jobKey) {
+    return JOBS[jobKey]
+}
+
+String getJobUrl(String jobKey) {
+    echo "getJobUrl for ${jobKey}"
+    return getJob(jobKey)?.absoluteUrl ?: ''
+}
+
+boolean isJobSucceeded(String jobKey) {
+    return getJob(jobKey)?.result == 'SUCCESS'
+}
+
+boolean isJobUnstable(String jobKey) {
+    return getJob(jobKey)?.result == 'UNSTABLE'
+}
+
+void addFailedStage(String jobKey = '') {
+    FAILED_STAGES.put("${env.STAGE_NAME}", jobKey)
+}
+void addUnstableStage(String jobKey = '') {
+    UNSTABLE_STAGES.put("${env.STAGE_NAME}", jobKey)
+}
+
+void sendPipelineErrorNotification() {
+    String bodyMsg = "Kogito Cloud weekly job #${env.BUILD_NUMBER} was: ${currentBuild.currentResult}"
+
+    paramsStr = ''
+    if (params.SKIP_TESTS) {
+        paramsStr += '\n- Tests skipped'
+    }
+    if (params.SKIP_IMAGES) {
+        paramsStr += '\n- Images skipped'
+    }
+    if (params.SKIP_OPERATOR) {
+        paramsStr += '\n- Operator skipped'
+    }
+    bodyMsg += paramsStr ? "\n\nConfiguration:${paramsStr}" : '\n'
+
+    if (FAILED_STAGES.size() > 0) {
+        bodyMsg += '\nFailed stages: \n- '
+        bodyMsg += FAILED_STAGES.collect { "${it.key} => ${getJobUrl(it.value)}" }.join('\n- ')
+    }
+    bodyMsg += '\n'
+    if (UNSTABLE_STAGES.size() > 0) {
+        bodyMsg += '\nUnstable stages: \n- '
+        bodyMsg += UNSTABLE_STAGES.collect { "${it.key} => ${getJobUrl(it.value)}" }.join('\n- ')
+    }
+    bodyMsg += '\n'
+    bodyMsg += "\nPlease look here: ${env.BUILD_URL}"
+    emailext body: bodyMsg, subject: "[${getBuildBranch()}][d] Full Pipeline",
+                to: env.KOGITO_CI_EMAIL_TO
+}
+
+List getDefaultBuildParams(String buildBranchName = '', String key = '') {
+    buildBranchName = buildBranchName ?: getBuildBranch()
+    List params = []
+    addStringParam(params, 'DISPLAY_NAME', "${key ? "${key}-" : ''}${env.WEEKLY_TAG}")
+    addStringParam(params, 'GIT_CHECKOUT_DATETIME', getCheckoutDatetime())
+    addBooleanParam(params, 'SEND_NOTIFICATION', true)
+
+    return params
+}
+
+void addSkipTestsParam(buildParams) {
+    addBooleanParam(buildParams, 'SKIP_TESTS', params.SKIP_TESTS)
+}
+
+void addSkipIntegrationTestsParam(buildParams) {
+    addBooleanParam(buildParams, 'SKIP_INTEGRATION_TESTS', params.SKIP_TESTS)
+}
+
+void addAppsParam(buildParams) {
+    addStringParam(buildParams, 'APPS_REF', "${getBuildBranch()}")
+    addStringParam(buildParams, 'APPS_URI', "https://github.com/${getGitAuthor()}/incubator-kie-kogito-apps")
+}
+
+void addExamplesParam(buildParams) {
+    addStringParam(buildParams, 'EXAMPLES_URI', "https://github.com/${getGitAuthor()}/incubator-kie-kogito-examples")
+    addStringParam(buildParams, 'EXAMPLES_REF', "nightly-${getBuildBranch()}")
+}
+
+void addImageBuildParams(List buildParams, String tag, String paramsPrefix = defaultImageParamsPrefix, String extraSuffix = '') {
+    addStringParam(buildParams, constructKey(paramsPrefix, 'REGISTRY_CREDENTIALS'), env.IMAGE_REGISTRY_CREDENTIALS)
+    addStringParam(buildParams, constructKey(paramsPrefix, 'REGISTRY'), env.IMAGE_REGISTRY)
+    addStringParam(buildParams, constructKey(paramsPrefix, 'NAMESPACE'), env.IMAGE_NAMESPACE)
+    addStringParam(buildParams, constructKey(paramsPrefix, 'NAME_SUFFIX'), (extraSuffix ? "${extraSuffix}-" : '') + env.IMAGE_NAME_WEEKLY_SUFFIX)
+    addStringParam(buildParams, constructKey(paramsPrefix, 'TAG'), tag)
+}
+
+void addDeployImageWithLatestTagParam(buildParams) {
+    addBooleanParam(buildParams, 'DEPLOY_WITH_LATEST_TAG', isDeployImagesLatestTag())
+}
+
+void addStringParam(List params, String key, String value) {
+    params.add(string(name: key, value: value))
+}
+
+void addBooleanParam(List params, String key, boolean value) {
+    params.add(booleanParam(name: key, value: value))
+}
+
+String constructKey(String prefix, String paramId) {
+    return prefix ? "${prefix}_${paramId}" : paramId
+}
+
+String getBuildBranch() {
+    return env.GIT_BRANCH_NAME
+}
+
+String getGitAuthor() {
+    return env.GIT_AUTHOR
+}
+
+String getGitAuthorCredsId() {
+    return env.GIT_AUTHOR_CREDS_ID
+}
+
+boolean isDeployImagesLatestTag() {
+    return getBuildBranch() == env.BRANCH_FOR_LATEST
+}
+
+boolean isImagesDeploy() {
+    return !params.SKIP_IMAGES
+}
+
+boolean isOperatorDeploy() {
+    return !params.SKIP_OPERATOR
+}
+
+String getCurrentDate() {
+    return sh(returnStdout: true, script: 'date -u "+%Y-%m-%d"').trim()
+}
+
+String getCheckoutDatetime() {
+    return params.GIT_CHECKOUT_DATETIME
+}

.ci/jenkins/dsl/jobs.groovy

@@ -63,6 +63,9 @@ if (isMainStream()) {
 
 // Weekly
 setupWeeklyJob()
+if (isMainStream()) {
+    setupWeeklyCloudJob()
+}
 
 // Release
 setupReleaseArtifactsJob()
@@ -188,6 +191,38 @@ void setupWeeklyJob() {
     KogitoJobTemplate.createPipelineJob(this, jobParams)?.with {
         parameters {
             booleanParam('SKIP_TESTS', false, 'Skip all tests')
+            booleanParam('SKIP_CLOUD_WEEKLY', !isMainStream(), 'Skip cloud weekly execution')
+        }
+    }
+}
+
+void setupWeeklyCloudJob() {
+    def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-weekly-cloud', JobType.OTHER, "${jenkins_path}/Jenkinsfile.weekly.cloud", 'Kogito Weekly')
+    jobParams.env.putAll([
+        JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}",
+
+        GIT_BRANCH_NAME: "${GIT_BRANCH}",
+        GIT_AUTHOR: "${GIT_AUTHOR_NAME}",
+        GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}",
+
+        IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS}",
+        IMAGE_REGISTRY: "${CLOUD_IMAGE_REGISTRY}",
+        IMAGE_NAMESPACE: "${CLOUD_IMAGE_NAMESPACE}",
+        BRANCH_FOR_LATEST: "${CLOUD_IMAGE_LATEST_GIT_BRANCH}",
+
+        MAVEN_SETTINGS_CONFIG_FILE_ID: "${MAVEN_SETTINGS_FILE_ID}",
+        ARTIFACTS_REPOSITORY: "${MAVEN_ARTIFACTS_REPOSITORY}",
+    ])
+    KogitoJobTemplate.createPipelineJob(this, jobParams)?.with {
+        parameters {
+            booleanParam('SKIP_TESTS', false, 'Skip all tests')
+
+            booleanParam('SKIP_IMAGES', false, 'To skip Images Deployment')
+            booleanParam('SKIP_OPERATOR', false, 'To skip Operator Deployment')
+
+            stringParam('GIT_CHECKOUT_DATETIME', '', 'Git checkout date and time - (Y-m-d H:i)')
+
+            booleanParam('USE_TEMP_OPENSHIFT_REGISTRY', false, 'If enabled, use Openshift registry to push temporary images')
         }
     }
 }