From 954496946dcd0a13231554bbebe58655c3b99839 Mon Sep 17 00:00:00 2001 From: Rodrigo Antunes Date: Wed, 31 Jan 2024 10:54:22 -0300 Subject: [PATCH 1/2] Setup weekly cloud jobs --- .ci/jenkins/Jenkinsfile.weekly | 13 ++ .ci/jenkins/Jenkinsfile.weekly.cloud | 260 +++++++++++++++++++++++++++ .ci/jenkins/dsl/jobs.groovy | 35 ++++ 3 files changed, 308 insertions(+) create mode 100644 .ci/jenkins/Jenkinsfile.weekly.cloud diff --git a/.ci/jenkins/Jenkinsfile.weekly b/.ci/jenkins/Jenkinsfile.weekly index 1e4d2c534..04abb86b2 100644 --- a/.ci/jenkins/Jenkinsfile.weekly +++ b/.ci/jenkins/Jenkinsfile.weekly @@ -79,6 +79,19 @@ pipeline { } } } + + stage('Start Cloud weekly') { + when { + expression { !params.SKIP_CLOUD_WEEKLY } + } + steps { + script { + def buildParams = getDefaultBuildParams() + addSkipTestsParam(buildParams) + build(job: './0-kogito-weekly-cloud', wait: false, parameters: buildParams, propagate: false) + } + } + } } post { unsuccessful { diff --git a/.ci/jenkins/Jenkinsfile.weekly.cloud b/.ci/jenkins/Jenkinsfile.weekly.cloud new file mode 100644 index 000000000..2970b8489 --- /dev/null +++ b/.ci/jenkins/Jenkinsfile.weekly.cloud @@ -0,0 +1,260 @@ +import org.jenkinsci.plugins.workflow.libs.Library + +@Library('jenkins-pipeline-shared-libraries')_ + +// Deploy jobs +IMAGES_DEPLOY = 'kogito-images.weekly-deploy' +SEVERLESS_OPERATOR_DEPLOY = 'kogito-serverless-operator.weekly-deploy' + +// Map of executed jobs +// See https://javadoc.jenkins.io/plugin/workflow-support/org/jenkinsci/plugins/workflow/support/steps/build/RunWrapper.html +// for more options on built job entity +JOBS = [:] + +FAILED_STAGES = [:] +UNSTABLE_STAGES = [:] + +defaultImageParamsPrefix = 'IMAGE' + +// Should be multibranch pipeline +pipeline { + agent { + label 'ubuntu' + } + + options { + timeout(time: 1380, unit: 'MINUTES') + } + + // parameters { + // For parameters, check into ./dsl/jobs.groovy file + // } + + environment { + // Some generated env is also defined into ./dsl/jobs.groovy file + + KOGITO_CI_EMAIL_TO = credentials("${JENKINS_EMAIL_CREDS_ID}") + + IMAGE_NAME_WEEKLY_SUFFIX = 'nightly' + + // Use branch name in weekly tag as we may have parallel main and release branch builds + WEEKLY_TAG = """${getBuildBranch()}-${getCurrentDate()}""" + } + + stages { + stage('Initialize') { + steps { + script { + echo "weekly tag is ${env.WEEKLY_TAG}" + + currentBuild.displayName = env.WEEKLY_TAG + } + } + } + + stage('Build & Deploy Images') { + when { + expression { return isImagesDeploy() } + } + steps { + script { + def buildParams = getDefaultBuildParams() + addImageBuildParams(buildParams, env.WEEKLY_TAG) + addDeployImageWithLatestTagParam(buildParams) + + // For building + addAppsParam(buildParams) + + // For testing + addSkipTestsParam(buildParams) + addExamplesParam(buildParams) + + buildJob(IMAGES_DEPLOY, buildParams) + } + } + post { + failure { + addFailedStage(IMAGES_DEPLOY) + } + } + } + + stage('Build & Deploy Serverless Operator') { + when { + expression { return isOperatorDeploy() } + } + steps { + script { + def buildParams = getDefaultBuildParams() + addSkipTestsParam(buildParams) + addImageBuildParams(buildParams, env.WEEKLY_TAG) + addDeployImageWithLatestTagParam(buildParams) + + buildJob(SEVERLESS_OPERATOR_DEPLOY, buildParams) + } + } + post { + failure { + addFailedStage(SEVERLESS_OPERATOR_DEPLOY) + } + } + } + } + post { + unsuccessful { + sendPipelineErrorNotification() + } + } +} + +def buildJob(String jobName, List buildParams, String jobKey = jobName) { + echo "[${jobKey}] Build ${jobName} with params ${buildParams}" + + def job = build(job: "${jobName}", wait: true, parameters: buildParams, propagate: false) + JOBS[jobKey] = job + + // Set Unstable if job did not succeed + if (!isJobSucceeded(jobKey)) { + addUnstableStage(jobKey) + unstable("Job ${jobName} finished with result ${job.result}") + } + return job +} + +def getJob(String jobKey) { + return JOBS[jobKey] +} + +String getJobUrl(String jobKey) { + echo "getJobUrl for ${jobKey}" + return getJob(jobKey)?.absoluteUrl ?: '' +} + +boolean isJobSucceeded(String jobKey) { + return getJob(jobKey)?.result == 'SUCCESS' +} + +boolean isJobUnstable(String jobKey) { + return getJob(jobKey)?.result == 'UNSTABLE' +} + +void addFailedStage(String jobKey = '') { + FAILED_STAGES.put("${env.STAGE_NAME}", jobKey) +} +void addUnstableStage(String jobKey = '') { + UNSTABLE_STAGES.put("${env.STAGE_NAME}", jobKey) +} + +void sendPipelineErrorNotification() { + String bodyMsg = "Kogito Cloud weekly job #${env.BUILD_NUMBER} was: ${currentBuild.currentResult}" + + paramsStr = '' + if (params.SKIP_TESTS) { + paramsStr += '\n- Tests skipped' + } + if (params.SKIP_IMAGES) { + paramsStr += '\n- Images skipped' + } + if (params.SKIP_OPERATOR) { + paramsStr += '\n- Operator skipped' + } + bodyMsg += paramsStr ? "\n\nConfiguration:${paramsStr}" : '\n' + + if (FAILED_STAGES.size() > 0) { + bodyMsg += '\nFailed stages: \n- ' + bodyMsg += FAILED_STAGES.collect { "${it.key} => ${getJobUrl(it.value)}" }.join('\n- ') + } + bodyMsg += '\n' + if (UNSTABLE_STAGES.size() > 0) { + bodyMsg += '\nUnstable stages: \n- ' + bodyMsg += UNSTABLE_STAGES.collect { "${it.key} => ${getJobUrl(it.value)}" }.join('\n- ') + } + bodyMsg += '\n' + bodyMsg += "\nPlease look here: ${env.BUILD_URL}" + emailext body: bodyMsg, subject: "[${getBuildBranch()}][d] Full Pipeline", + to: env.KOGITO_CI_EMAIL_TO +} + +List getDefaultBuildParams(String buildBranchName = '', String key = '') { + buildBranchName = buildBranchName ?: getBuildBranch() + List params = [] + addStringParam(params, 'DISPLAY_NAME', "${key ? "${key}-" : ''}${env.WEEKLY_TAG}") + addStringParam(params, 'GIT_CHECKOUT_DATETIME', getCheckoutDatetime()) + addBooleanParam(params, 'SEND_NOTIFICATION', true) + + return params +} + +void addSkipTestsParam(buildParams) { + addBooleanParam(buildParams, 'SKIP_TESTS', params.SKIP_TESTS) +} + +void addSkipIntegrationTestsParam(buildParams) { + addBooleanParam(buildParams, 'SKIP_INTEGRATION_TESTS', params.SKIP_TESTS) +} + +void addAppsParam(buildParams) { + addStringParam(buildParams, 'APPS_REF', "${getBuildBranch()}") + addStringParam(buildParams, 'APPS_URI', "https://github.com/${getGitAuthor()}/incubator-kie-kogito-apps") +} + +void addExamplesParam(buildParams) { + addStringParam(buildParams, 'EXAMPLES_URI', "https://github.com/${getGitAuthor()}/incubator-kie-kogito-examples") + addStringParam(buildParams, 'EXAMPLES_REF', "nightly-${getBuildBranch()}") +} + +void addImageBuildParams(List buildParams, String tag, String paramsPrefix = defaultImageParamsPrefix, String extraSuffix = '') { + addStringParam(buildParams, constructKey(paramsPrefix, 'REGISTRY_CREDENTIALS'), env.IMAGE_REGISTRY_CREDENTIALS) + addStringParam(buildParams, constructKey(paramsPrefix, 'REGISTRY'), env.IMAGE_REGISTRY) + addStringParam(buildParams, constructKey(paramsPrefix, 'NAMESPACE'), env.IMAGE_NAMESPACE) + addStringParam(buildParams, constructKey(paramsPrefix, 'NAME_SUFFIX'), (extraSuffix ? "${extraSuffix}-" : '') + env.IMAGE_NAME_WEEKLY_SUFFIX) + addStringParam(buildParams, constructKey(paramsPrefix, 'TAG'), tag) +} + +void addDeployImageWithLatestTagParam(buildParams) { + addBooleanParam(buildParams, 'DEPLOY_WITH_LATEST_TAG', isDeployImagesLatestTag()) +} + +void addStringParam(List params, String key, String value) { + params.add(string(name: key, value: value)) +} + +void addBooleanParam(List params, String key, boolean value) { + params.add(booleanParam(name: key, value: value)) +} + +String constructKey(String prefix, String paramId) { + return prefix ? "${prefix}_${paramId}" : paramId +} + +String getBuildBranch() { + return env.GIT_BRANCH_NAME +} + +String getGitAuthor() { + return env.GIT_AUTHOR +} + +String getGitAuthorCredsId() { + return env.GIT_AUTHOR_CREDS_ID +} + +boolean isDeployImagesLatestTag() { + return getBuildBranch() == env.BRANCH_FOR_LATEST +} + +boolean isImagesDeploy() { + return !params.SKIP_IMAGES +} + +boolean isOperatorDeploy() { + return !params.SKIP_OPERATOR +} + +String getCurrentDate() { + return sh(returnStdout: true, script: 'date -u "+%Y-%m-%d"').trim() +} + +String getCheckoutDatetime() { + return params.GIT_CHECKOUT_DATETIME +} diff --git a/.ci/jenkins/dsl/jobs.groovy b/.ci/jenkins/dsl/jobs.groovy index 73e59e1a9..65a257c0e 100644 --- a/.ci/jenkins/dsl/jobs.groovy +++ b/.ci/jenkins/dsl/jobs.groovy @@ -63,6 +63,9 @@ if (isMainStream()) { // Weekly setupWeeklyJob() +if (isMainStream()) { + setupWeeklyCloudJob() +} // Release setupReleaseArtifactsJob() @@ -188,6 +191,38 @@ void setupWeeklyJob() { KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { parameters { booleanParam('SKIP_TESTS', false, 'Skip all tests') + booleanParam('SKIP_CLOUD_WEEKLY', !isMainStream(), 'Skip cloud weekly execution') + } + } +} + +void setupWeeklyCloudJob() { + def jobParams = JobParamsUtils.getBasicJobParams(this, '0-kogito-weekly-cloud', JobType.OTHER, "${jenkins_path}/Jenkinsfile.weekly.cloud", 'Kogito Weekly') + jobParams.env.putAll([ + JENKINS_EMAIL_CREDS_ID: "${JENKINS_EMAIL_CREDS_ID}", + + GIT_BRANCH_NAME: "${GIT_BRANCH}", + GIT_AUTHOR: "${GIT_AUTHOR_NAME}", + GIT_AUTHOR_CREDS_ID: "${GIT_AUTHOR_CREDENTIALS_ID}", + + IMAGE_REGISTRY_CREDENTIALS: "${CLOUD_IMAGE_REGISTRY_CREDENTIALS}", + IMAGE_REGISTRY: "${CLOUD_IMAGE_REGISTRY}", + IMAGE_NAMESPACE: "${CLOUD_IMAGE_NAMESPACE}", + BRANCH_FOR_LATEST: "${CLOUD_IMAGE_LATEST_GIT_BRANCH}", + + MAVEN_SETTINGS_CONFIG_FILE_ID: "${MAVEN_SETTINGS_FILE_ID}", + ARTIFACTS_REPOSITORY: "${MAVEN_ARTIFACTS_REPOSITORY}", + ]) + KogitoJobTemplate.createPipelineJob(this, jobParams)?.with { + parameters { + booleanParam('SKIP_TESTS', false, 'Skip all tests') + + booleanParam('SKIP_IMAGES', false, 'To skip Images Deployment') + booleanParam('SKIP_OPERATOR', false, 'To skip Operator Deployment') + + stringParam('GIT_CHECKOUT_DATETIME', '', 'Git checkout date and time - (Y-m-d H:i)') + + booleanParam('USE_TEMP_OPENSHIFT_REGISTRY', false, 'If enabled, use Openshift registry to push temporary images') } } } From cd7cb5c0588e8888c14a9da19213b754b8a0ce20 Mon Sep 17 00:00:00 2001 From: Rodrigo Antunes Date: Wed, 7 Feb 2024 08:03:43 -0300 Subject: [PATCH 2/2] Update agent label to use the utility --- .ci/jenkins/Jenkinsfile.weekly.cloud | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/jenkins/Jenkinsfile.weekly.cloud b/.ci/jenkins/Jenkinsfile.weekly.cloud index 2970b8489..beeb2ef9e 100644 --- a/.ci/jenkins/Jenkinsfile.weekly.cloud +++ b/.ci/jenkins/Jenkinsfile.weekly.cloud @@ -19,7 +19,7 @@ defaultImageParamsPrefix = 'IMAGE' // Should be multibranch pipeline pipeline { agent { - label 'ubuntu' + label util.getLabel('ubuntu') } options {