Data Prepper release fixes (opensearch-project#1803)

* Provide the full path to the sign artifacts commands. Signed-off-by: David Venable <[email protected]> * Provide root access in order to perform the Docker image copy. Signed-off-by: David Venable <[email protected]> * Use the new Jenkins image for the Data Prepper release job. Signed-off-by: David Venable <[email protected]> * Updated the Data Prepper tests per new changes. Signed-off-by: David Venable <[email protected]> * Delete the entire workspace after the build. Signed-off-by: David Venable <[email protected]> * Added parenthesis Signed-off-by: David Venable <[email protected]> * Added a one hour timeout. Signed-off-by: David Venable <[email protected]> * Update the latest tag on the Data Prepper Docker image. Signed-off-by: David Venable <[email protected]> * Fixed the Data Prepper release unit tests. Signed-off-by: David Venable <[email protected]> * Removed the Maven tool to use the Maven installed on the Docker image. Signed-off-by: David Venable <[email protected]> * Change the Maven signature type. Signed-off-by: David Venable <[email protected]> * Fixed the Data Prepper unit tests based on the working build. Signed-off-by: David Venable <[email protected]>
naveenpajjuri · Mar 23, 2022 · 656e612 · 656e612
1 parent 254a30d
commit 656e612
Show file tree

Hide file tree

Showing 4 changed files with 33 additions and 15 deletions.
diff --git a/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile b/jenkins/data-prepper/release-data-prepper-all-artifacts.jenkinsfile
@@ -1,11 +1,15 @@
 lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
 
 pipeline {
+    options {
+        timeout(time: 1, unit: 'HOURS')
+    }
     agent {
         docker {
             label 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
-            image 'opensearchstaging/ci-runner:ci-runner-centos7-v1'
+            image 'opensearchstaging/ci-runner:ci-runner-ubuntu2004-data-prepper-pipeline-v1'
             alwaysPull true
+            args '-u root'
         }
     }
     parameters {
@@ -43,7 +47,7 @@ pipeline {
                 script {
                     signatureType = '.sig'
                     signArtifacts(
-                            artifactPath: 'archive',
+                            artifactPath: "${WORKSPACE}/archive",
                             sigtype: signatureType,
                             platform: 'linux'
                     )
@@ -53,10 +57,9 @@ pipeline {
         stage('Sign Maven Artifacts') {
             steps {
                 script {
-                    signatureType = '.sig'
                     signArtifacts(
-                            artifactPath: 'maven',
-                            sigtype: signatureType,
+                            artifactPath: "${WORKSPACE}/maven",
+                            type: 'maven',
                             platform: 'linux'
                     )
                 }
@@ -80,13 +83,16 @@ pipeline {
                         destinationType: 'docker',
                         destinationCredentialIdentifier: 'jenkins-staging-docker-prod-token'
                     )
+                    copyContainer(
+                        sourceImagePath: "${DATA_PREPPER_STAGING_CONTAINER_REPOSITORY}/data-prepper:${VERSION}-${DATA_PREPPER_BUILD_NUMBER}",
+                        destinationImagePath: "opensearchproject/data-prepper:latest",
+                        destinationType: 'docker',
+                        destinationCredentialIdentifier: 'jenkins-staging-docker-prod-token'
+                    )
                 }
             }
         }
         stage('Upload Artifacts to Sonatype') {
-            tools {
-                maven "maven-3.8.2"
-            }
             environment {
                 REPO_URL = "https://aws.oss.sonatype.org/"
                 STAGING_PROFILE_ID = "${SONATYPE_STAGING_PROFILE_ID}"
@@ -105,6 +111,7 @@ pipeline {
     post() {
         always {
             script {
+                sh("rm -rf $WORKSPACE/*")
                 postCleanup()
             }
         }

diff --git a/tests/jenkins/TestDataPrepperReleaseArtifacts.groovy b/tests/jenkins/TestDataPrepperReleaseArtifacts.groovy
@@ -25,7 +25,7 @@ class TestDataPrepperReleaseArtifacts extends BuildPipelineTest {
 
         String sourceImageRepository = 'http://public.ecr.aws/data-prepper-container-repository'
 
-        this.registerLibTester(new SignArtifactsLibTester( '.sig', 'linux',  'archive', null, null))
+        this.registerLibTester(new SignArtifactsLibTester( '.sig', 'linux', "${workspace}/archive", null, null))
 
         this.registerLibTester(new CopyContainerLibTester("${sourceImageRepository}/data-prepper:${version}-997908",
                 "opensearchproject/data-prepper:${version}",

diff --git a/...nkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt b/...nkinsjob-regression-files/data-prepper/release-data-prepper-all-artifacts.jenkinsfile.txt
@@ -2,7 +2,8 @@
       release-data-prepper-all-artifacts.legacySCM(groovy.lang.Closure)
       release-data-prepper-all-artifacts.library({identifier=jenkins@20211123, retriever=null})
       release-data-prepper-all-artifacts.pipeline(groovy.lang.Closure)
-         release-data-prepper-all-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-centos7-v1, reuseNode:false, stages:[:], args:, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
+         release-data-prepper-all-artifacts.timeout({time=1, unit=HOURS})
+         release-data-prepper-all-artifacts.echo(Executing on agent [docker:[image:opensearchstaging/ci-runner:ci-runner-ubuntu2004-data-prepper-pipeline-v1, reuseNode:false, stages:[:], args:-u root, alwaysPull:true, containerPerStageRoot:false, label:Jenkins-Agent-al2-x64-c54xlarge-Docker-Host]])
          release-data-prepper-all-artifacts.stage(Download Archives, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
                release-data-prepper-all-artifacts.dir(archive, groovy.lang.Closure)
@@ -39,7 +40,7 @@
                   release-data-prepper-all-artifacts.sh(curl -sSL http://staging-artifacts.cloudfront.net/0.22.1/997908/maven/org/opensearch/dataprepper/data-prepper-api/0.22.1/data-prepper-api-0.22.1.module.sha512 -o org/opensearch/dataprepper/data-prepper-api/0.22.1/data-prepper-api-0.22.1.module.sha512)
          release-data-prepper-all-artifacts.stage(Sign Archives, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
-               release-data-prepper-all-artifacts.signArtifacts({artifactPath=archive, sigtype=.sig, platform=linux})
+               release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/archive, sigtype=.sig, platform=linux})
                   signArtifacts.fileExists(/tmp/workspace/sign.sh)
                   signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
                   signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
@@ -53,11 +54,11 @@
             export UNSIGNED_BUCKET=signer_client_unsigned_bucket
             export SIGNED_BUCKET=signer_client_signed_bucket
 
-            /tmp/workspace/sign.sh archive --sigtype=.sig --platform=linux
+            /tmp/workspace/sign.sh /tmp/workspace/archive --sigtype=.sig --platform=linux
         )
          release-data-prepper-all-artifacts.stage(Sign Maven Artifacts, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
-               release-data-prepper-all-artifacts.signArtifacts({artifactPath=maven, sigtype=.sig, platform=linux})
+               release-data-prepper-all-artifacts.signArtifacts({artifactPath=/tmp/workspace/maven, type=maven, platform=linux})
                   signArtifacts.fileExists(/tmp/workspace/sign.sh)
                   signArtifacts.git({url=https://github.com/opensearch-project/opensearch-build.git, branch=main})
                   signArtifacts.sh(curl -sSL https://artifacts.opensearch.org/publickeys/opensearch.pgp | gpg --import -)
@@ -71,7 +72,7 @@
             export UNSIGNED_BUCKET=signer_client_unsigned_bucket
             export SIGNED_BUCKET=signer_client_signed_bucket
 
-            /tmp/workspace/sign.sh maven --sigtype=.sig --platform=linux
+            /tmp/workspace/sign.sh /tmp/workspace/maven --type=maven --platform=linux
         )
          release-data-prepper-all-artifacts.stage(Release Archives to Production Distribution Bucket, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
@@ -88,11 +89,21 @@
                 docker login -u DOCKER_USERNAME -p DOCKER_PASSWORD
                 gcrane cp http://public.ecr.aws/data-prepper-container-repository/data-prepper:0.22.1-997908 opensearchproject/data-prepper:0.22.1
             )
+               release-data-prepper-all-artifacts.copyContainer({sourceImagePath=http://public.ecr.aws/data-prepper-container-repository/data-prepper:0.22.1-997908, destinationImagePath=opensearchproject/data-prepper:latest, destinationType=docker, destinationCredentialIdentifier=jenkins-staging-docker-prod-token})
+                  copyContainer.sh({script=test -f /usr/local/bin/gcrane && echo '1' || echo '0' , returnStdout=true})
+                  copyContainer.sh(docker logout)
+                  copyContainer.usernamePassword({credentialsId=jenkins-staging-docker-prod-token, usernameVariable=DOCKER_USERNAME, passwordVariable=DOCKER_PASSWORD})
+                  copyContainer.withCredentials([[DOCKER_USERNAME, DOCKER_PASSWORD]], groovy.lang.Closure)
+                     copyContainer.sh(
+                docker login -u DOCKER_USERNAME -p DOCKER_PASSWORD
+                gcrane cp http://public.ecr.aws/data-prepper-container-repository/data-prepper:0.22.1-997908 opensearchproject/data-prepper:latest
+            )
          release-data-prepper-all-artifacts.stage(Upload Artifacts to Sonatype, groovy.lang.Closure)
             release-data-prepper-all-artifacts.script(groovy.lang.Closure)
                release-data-prepper-all-artifacts.usernamePassword({credentialsId=Sonatype, usernameVariable=SONATYPE_USERNAME, passwordVariable=SONATYPE_PASSWORD})
                release-data-prepper-all-artifacts.withCredentials([[SONATYPE_USERNAME, SONATYPE_PASSWORD]], groovy.lang.Closure)
                   release-data-prepper-all-artifacts.sh($WORKSPACE/publish/stage-maven-release.sh $WORKSPACE/maven)
          release-data-prepper-all-artifacts.script(groovy.lang.Closure)
+            release-data-prepper-all-artifacts.sh(rm -rf /tmp/workspace/*)
             release-data-prepper-all-artifacts.postCleanup()
                postCleanup.cleanWs({disableDeferredWipeout=true, deleteDirs=true})
diff --git a/tests/jenkins/lib-testers/SignArtifactsLibTester.groovy b/tests/jenkins/lib-testers/SignArtifactsLibTester.groovy
@@ -34,7 +34,7 @@ class SignArtifactsLibTester extends LibFunctionTester {
         assertThat(call.args.platform.first(), notNullValue())
         if(call.args.artifactPath.first().toString().endsWith(".yml")){
             assertThat(call.args.type.first(), notNullValue())
-        } else {
+        } else if(call.args.type.first() != 'maven'){
             assertThat(call.args.sigtype.first(), notNullValue())
         }
     }