From cf9d355415291a6e36b1ad87b5e24d381afef70b Mon Sep 17 00:00:00 2001 From: Kapil Powar Date: Wed, 25 Oct 2023 16:31:08 +0530 Subject: [PATCH] Automate API check for updates Automate API check for updates Check if update is available in Artifactory for any JDK version --- buildenv/jenkins/API_check | 128 +++++++++++++++++++++++ jck/jck-semiauto-updater/update-check.sh | 82 +++++++++++++++ 2 files changed, 210 insertions(+) create mode 100644 buildenv/jenkins/API_check create mode 100755 jck/jck-semiauto-updater/update-check.sh diff --git a/buildenv/jenkins/API_check b/buildenv/jenkins/API_check new file mode 100644 index 0000000000..acb700281f --- /dev/null +++ b/buildenv/jenkins/API_check @@ -0,0 +1,128 @@ +#!groovy + +timestamps{ + stage('Setup') { + def CHECK_LABEL = params.CHECK_LABEL ?: "ci.role.test&&hw.arch.x86&&sw.os.linux" + println "CHECK_LABEL: ${CHECK_LABEL}" + node(CHECK_LABEL) { + //echo ("clone the SCM GIT repo") + try { + def gitConfig = scm.getUserRemoteConfigs().get(0) + timeout(time: 1, unit: 'HOURS') { + forceCleanWS() + } + checkout scm: [$class: 'GitSCM', + branches: [[name: "${scm.branches[0].name}"]], + extensions: [ + [$class: 'CleanBeforeCheckout'], + [$class: 'CloneOption'], + [$class: 'RelativeTargetDirectory', relativeTargetDir: 'aqa-tests']], + userRemoteConfigs: [[url: "${gitConfig.getUrl()}"]] + ] + automateAPICheck() + } catch (Exception e) { + // build result may not be updated correctly at the moment (see https://issues.jenkins.io/browse/JENKINS-56402) + // if there is an exception, set currentBuild.result to ABORTED/FAILURE + if (e.toString().contains("The job is completed without further action")){ + println("JCKUpdater script returned exit code 2, as no new update available. The job is completed without further action.") + currentBuild.result = "SUCCESS" + } else if (e.toString().contains("FlowInterruptedException")) { + currentBuild.result = 'ABORTED' + println("Exception: " + e.toString()) + } else { + currentBuild.result = 'FAILURE' + println("Exception: " + e.toString()) + } + } + } + } +} +def forceCleanWS() { + try { + cleanWs disableDeferredWipeout: true, deleteDirs: true + } catch (Exception e) { + echo 'Exception: ' + e.toString() + //cleanWs has issue to delete workspace that contains non-ASCII filename in TKG output https://issues.jenkins.io/browse/JENKINS-33478 + //cannot delete workspace directly. Otherwise, Jenkins job will abort due to missing workspace + sh "rm -rf ${env.WORKSPACE}/aqa-tests/TKG" + // call cleanWs() again + cleanWs disableDeferredWipeout: true, deleteDirs: true + } +} + +def automateAPICheck() { + echo ("checking if new JCK material is available .....") + def ARTIFACTORY_API_URL_OPTION = params.ARTIFACTORY_API_URL ? "-au ${params.ARTIFACTORY_API_URL}": "" + stage('API_CHECK') { + withCredentials([ + usernamePassword(credentialsId: "${params.ARTIFACTORY_CREDENTIALS}", + usernameVariable: 'ARTIFACTORY_USER', passwordVariable: 'ARTIFACTORY_CREDENTIALS') + ]){ + // def currentDirectory = sh(script: 'pwd', returnStdout: true).trim() + dir("${WORKSPACE}/aqa-tests/jck/jck-semiauto-updater"){ + def SCRIPT ="./update-check.sh ${ARTIFACTORY_API_URL_OPTION} -at \"\$ARTIFACTORY_CREDENTIALS\" " + def tempOutputFile = "temp_output.txt" + def EXITCODE = sh(script: "${SCRIPT} > ${tempOutputFile}", returnStatus: true) + + if (EXITCODE == 0) { + def scriptOutput = readFile(tempOutputFile).trim() + def jdkList = [:] + testJobs = [:] + echo ("Automate API script output:") + echo scriptOutput + + jdkList= getJDKList(scriptOutput) + jdkList.each { key, value -> + println("JDK number: $key") + def CHILD_JOB_NAME="JCK_Sync" + def BUILD_URL="${JENKINS_URL}job/${CHILD_JOB_NAME}/" + def CHILD_PARAMS = [] + def JCK_GIT_REPO = "git@github.ibm.com:runtimes/JCK${key}-unzipped.git" + params.each { param -> + def variable = param.value.toString() + if (param.key == "JDK_VERSION") { + variable = "${key}" + CHILD_PARAMS << string(name: param.key, value: variable) + } else if (param.key == "JCK_GIT_REPO") { + variable = "${JCK_GIT_REPO}" + CHILD_PARAMS << string(name: param.key, value: variable) + } else { + if (variable == "true" || variable == "false") { + CHILD_PARAMS << booleanParam(name: param.key, value: variable.toBoolean()) + } else { + CHILD_PARAMS << string(name: param.key, value: variable) + } + } + } + //def BUILD_INFO = build job: CHILD_JOB_NAME, parameters: CHILD_PARAMS + testJobs["JCK_Sync${key}"] = { + build job: CHILD_JOB_NAME, parameters: CHILD_PARAMS + } + } + parallel testJobs + } else { + error ("script failed with exit code ${EXITCODE}") + } + } + } + } +} + +def getJDKList(scriptOutput) +{ + def pattern = /JDK=(.+)/ + def matcher = (scriptOutput =~ pattern) + def stringWithoutJDK + if (matcher.find()) { + stringWithoutJDK = matcher.group(1).trim() + } + def commaSeparatedValues = stringWithoutJDK?.split(',') + def keyValueMap = [:] + commaSeparatedValues?.each { value -> + def spaceSeparatedValues = value.trim().split(/\s+/) + def key = spaceSeparatedValues[0] + def remainingPart = spaceSeparatedValues[1..-1].join(' ') + keyValueMap[key] = remainingPart + } + return keyValueMap +} \ No newline at end of file diff --git a/jck/jck-semiauto-updater/update-check.sh b/jck/jck-semiauto-updater/update-check.sh new file mode 100755 index 0000000000..8ad3e7cd7b --- /dev/null +++ b/jck/jck-semiauto-updater/update-check.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +# File to store the previous state +PREVIOUS_STATE_FILE="/home/jenkins/previous_state.txt" + +function list_folders_depth { + local folder_url="$1" + local depth="$2" + if [ "$depth" -eq 0 ]; then + return + fi + + local children=$(curl -X GET -ks -H X-JFrog-Art-Api:${ARTIFACTORY_TOKEN} $folder_url | grep -o '"uri" : "[^"]*"' | awk -F': "' '{print $2}' | sed 's/"$//' | grep -v "$folder_url$") + for child in $children; do + if [ "$depth" -eq 1 ]; then + echo "$folder_url$child" + else + local child_url="${folder_url}${child}" + list_folders_depth "$child_url" "$((depth - 1))" + fi + done +} + +function compare_states { + + # Read the previous state from the file, or create an empty file if it doesn't exist + if [ -e "$PREVIOUS_STATE_FILE" ]; then + PREVIOUS_STATE=$(cat "$PREVIOUS_STATE_FILE") + else + PREVIOUS_STATE="" + fi + + # Retrieve the current state of the repository + CURRENT_STATE=$(list_folders_depth "$ARTIFACTORY_API_URL" 3) + # Calculate the difference between the initial state and current state + added_folders=$(comm -13 <(echo "$PREVIOUS_STATE" | sort) <(echo "$CURRENT_STATE" | sort)) + + if [ -n "$added_folders" ]; then + + JDK_LIST=() + IFS=$'\n' + for folder_entry in $added_folders; do + JDK=$(echo "$folder_entry" | awk -F'/' '{print $(NF-2)}') + UPDATE=$(echo "$folder_entry" | awk -F'/' '{print $(NF)}') + JDK_LIST+=("$JDK $UPDATE") + done + jdk_values=$(IFS=,; echo "${JDK_LIST[*]}") + echo "Update available for JDK= $jdk_values" + else + echo "No new folders detected." + fi + + # Store the current state as the new previous state + echo "$CURRENT_STATE" > "$PREVIOUS_STATE_FILE" +} + +parseCommandLineArgs() +{ + while [ $# -gt 0 ] && [[ ."$1" = .-* ]] ; do + opt="$1"; + shift; + case "$opt" in + + "--artifactory_token" | "-at") + ARTIFACTORY_TOKEN="$1"; shift;; + + "--artifactory_url" | "-au") + ARTIFACTORY_API_URL="$1"; shift;; + + *) echo >&2 "Invalid option: ${opt}"; echo "This option was unrecognized."; usage; exit 1; + esac + done +} + +parseCommandLineArgs "$@" + +if [ "$ARTIFACTORY_TOKEN" != "" ] && [ "$ARTIFACTORY_API_URL" != "" ] ; then + compare_states +else + echo "Please provide missing arguments" + exit 1 +fi \ No newline at end of file