Add yum repo for RPM of OS and OSD in staging (opensearch-project#1960)

* add createrepo for rpm Signed-off-by: Tianle Huang <[email protected]> * move to assemble Signed-off-by: Tianle Huang <[email protected]> * typo Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]> * add repo file Signed-off-by: Tianle Huang <[email protected]> * add logs Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]> * add file path Signed-off-by: Tianle Huang <[email protected]> * create new groovy Signed-off-by: Tianle Huang <[email protected]> * fix param Signed-off-by: Tianle Huang <[email protected]> * address comments Signed-off-by: Tianle Huang <[email protected]> * fix name Signed-off-by: Tianle Huang <[email protected]> * temp change for 1.3.0 testing Signed-off-by: Tianle Huang <[email protected]> * update osd 2.0 for testing Signed-off-by: Tianle Huang <[email protected]> * temp stop upload Signed-off-by: Tianle Huang <[email protected]> * update name Signed-off-by: Tianle Huang <[email protected]> * restore temp changes Signed-off-by: Tianle Huang <[email protected]> * add unit tests Signed-off-by: Tianle Huang <[email protected]> * add more unit tests Signed-off-by: Tianle Huang <[email protected]> * more Signed-off-by: Tianle Huang <[email protected]> * fix Signed-off-by: Tianle Huang <[email protected]> * address comment Signed-off-by: Tianle Huang <[email protected]> * Add test txt Signed-off-by: Peter Zhu <[email protected]> Co-authored-by: Peter Zhu <[email protected]>
naveenpajjuri · Apr 14, 2022 · 8876571 · 8876571
1 parent 826325d
commit 8876571
Show file tree

Hide file tree

Showing 13 changed files with 2,762 additions and 0 deletions.
diff --git a/tests/data/opensearch-build-1.3.0-rpm.yml b/tests/data/opensearch-build-1.3.0-rpm.yml
diff --git a/tests/jenkins/TestAssembleManifest.groovy b/tests/jenkins/TestAssembleManifest.groovy
@@ -0,0 +1,35 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package jenkins.tests
+
+import org.junit.*
+
+class TestAssembleManifest extends BuildPipelineTest {
+
+    @Test
+    void testAssembleManifest_rpm() {
+        this.registerLibTester(new AssembleManifestLibTester('tests/data/opensearch-build-1.3.0-rpm.yml'))
+
+        this.registerLibTester(new BuildYumRepoTester(
+            'tests/data/opensearch-build-1.3.0-rpm.yml',
+            'https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64'
+        ))
+
+        super.setUp()
+
+        super.testPipeline('tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile')
+    }
+
+    @Test
+    void testAssembleManifest_tar() {
+        this.registerLibTester(new AssembleManifestLibTester('tests/data/opensearch-build-1.3.0.yml'))
+        super.setUp()
+
+        super.testPipeline('tests/jenkins/jobs/AssembleManifest_tar_Jenkinsfile')
+    }
+}
diff --git a/tests/jenkins/TestBuildYumRepo.groovy b/tests/jenkins/TestBuildYumRepo.groovy
@@ -0,0 +1,29 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+package jenkins.tests
+
+import org.junit.*
+
+class TestBuildYumRepo extends BuildPipelineTest {
+
+    @Before
+    void setUp() {
+        this.registerLibTester(new BuildYumRepoTester(
+            'tests/data/opensearch-build-1.3.0.yml',
+            'https://ci.opensearch.org/ci/dbc/test/1.3.0/9/linux/x64'
+        ))
+
+        super.setUp()
+    }
+
+    @Test
+    void testBuildYumRepo() {
+        super.testPipeline('tests/jenkins/jobs/BuildYumRepo_Jenkinsfile')
+    }
+
+}
diff --git a/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile b/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile
@@ -0,0 +1,16 @@
+lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm))
+
+pipeline {
+    agent none
+    stages {
+        stage('assembleManifest rpm') {
+            steps {
+                script {
+                    assembleManifest(
+                        buildManifest: 'tests/data/opensearch-build-1.3.0-rpm.yml'
+                    )
+                }
+            }
+        }
+    }
+}
diff --git a/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleManifest_rpm_Jenkinsfile.txt
@@ -0,0 +1,28 @@
+   AssembleManifest_rpm_Jenkinsfile.run()
+      AssembleManifest_rpm_Jenkinsfile.legacySCM(groovy.lang.Closure)
+      AssembleManifest_rpm_Jenkinsfile.library({identifier=jenkins@20211123, retriever=null})
+      AssembleManifest_rpm_Jenkinsfile.pipeline(groovy.lang.Closure)
+         AssembleManifest_rpm_Jenkinsfile.echo(Executing on agent [label:none])
+         AssembleManifest_rpm_Jenkinsfile.stage(assembleManifest rpm, groovy.lang.Closure)
+            AssembleManifest_rpm_Jenkinsfile.script(groovy.lang.Closure)
+               AssembleManifest_rpm_Jenkinsfile.assembleManifest({buildManifest=tests/data/opensearch-build-1.3.0-rpm.yml})
+                  assembleManifest.legacySCM(groovy.lang.Closure)
+                  assembleManifest.library({identifier=jenkins@20211123, retriever=null})
+                  assembleManifest.readYaml({file=tests/data/opensearch-build-1.3.0-rpm.yml})
+                  BuildManifest.asBoolean()
+                  BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 123)
+                  assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.3.0-rpm.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64)
+                  assembleManifest.buildYumRepo({baseUrl=https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64, buildManifest=tests/data/opensearch-build-1.3.0-rpm.yml})
+                     buildYumRepo.legacySCM(groovy.lang.Closure)
+                     buildYumRepo.library({identifier=jenkins@20211123, retriever=null})
+                     buildYumRepo.readYaml({file=tests/data/opensearch-build-1.3.0-rpm.yml})
+                     BuildManifest.asBoolean()
+                     buildYumRepo.echo(Create yum repo metadata and repo file tests/data/opensearch-build-1.3.0-rpm.yml)
+                     buildYumRepo.sh(createrepo "rpm/dist/opensearch")
+                     buildYumRepo.writeFile({file=rpm/dist/opensearch/opensearch-1.3.0.staging.repo, text=[opensearch-1.3.0-123-staging]
+name=OpenSearch 1.3.0 123 Staging
+baseurl=https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64/rpm/dist/opensearch/
+enabled=1
+gpgcheck=0
+autorefresh=1
+type=rpm-md})
diff --git a/tests/jenkins/jobs/AssembleManifest_tar_Jenkinsfile b/tests/jenkins/jobs/AssembleManifest_tar_Jenkinsfile
@@ -0,0 +1,16 @@
+lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm))
+
+pipeline {
+    agent none
+    stages {
+        stage('assembleManifest tar') {
+            steps {
+                script {
+                    assembleManifest(
+                        buildManifest: 'tests/data/opensearch-build-1.3.0.yml'
+                    )
+                }
+            }
+        }
+    }
+}
diff --git a/tests/jenkins/jobs/AssembleManifest_tar_Jenkinsfile.txt b/tests/jenkins/jobs/AssembleManifest_tar_Jenkinsfile.txt
@@ -0,0 +1,14 @@
+   AssembleManifest_tar_Jenkinsfile.run()
+      AssembleManifest_tar_Jenkinsfile.legacySCM(groovy.lang.Closure)
+      AssembleManifest_tar_Jenkinsfile.library({identifier=jenkins@20211123, retriever=null})
+      AssembleManifest_tar_Jenkinsfile.pipeline(groovy.lang.Closure)
+         AssembleManifest_tar_Jenkinsfile.echo(Executing on agent [label:none])
+         AssembleManifest_tar_Jenkinsfile.stage(assembleManifest tar, groovy.lang.Closure)
+            AssembleManifest_tar_Jenkinsfile.script(groovy.lang.Closure)
+               AssembleManifest_tar_Jenkinsfile.assembleManifest({buildManifest=tests/data/opensearch-build-1.3.0.yml})
+                  assembleManifest.legacySCM(groovy.lang.Closure)
+                  assembleManifest.library({identifier=jenkins@20211123, retriever=null})
+                  assembleManifest.readYaml({file=tests/data/opensearch-build-1.3.0.yml})
+                  BuildManifest.asBoolean()
+                  BuildManifest.getArtifactRootUrlWithoutDistribution(https://ci.opensearch.org/dbc, vars-build, 123)
+                  assembleManifest.sh(./assemble.sh "tests/data/opensearch-build-1.3.0.yml" --base-url https://ci.opensearch.org/dbc/vars-build/1.3.0/123/linux/x64)
diff --git a/tests/jenkins/jobs/BuildYumRepo_Jenkinsfile b/tests/jenkins/jobs/BuildYumRepo_Jenkinsfile
@@ -0,0 +1,17 @@
+lib = library(identifier: "jenkins@20211123", retriever: legacySCM(scm))
+
+pipeline {
+    agent none
+    stages {
+        stage('buildYumRepo') {
+            steps {
+                script {
+                    buildYumRepo(
+                        buildManifest: 'tests/data/opensearch-build-1.3.0.yml',
+                        baseUrl: 'https://ci.opensearch.org/ci/dbc/test/1.3.0/9/linux/x64'
+                    )
+                }
+            }
+        }
+    }
+}
diff --git a/tests/jenkins/jobs/BuildYumRepo_Jenkinsfile.txt b/tests/jenkins/jobs/BuildYumRepo_Jenkinsfile.txt
@@ -0,0 +1,21 @@
+   BuildYumRepo_Jenkinsfile.run()
+      BuildYumRepo_Jenkinsfile.legacySCM(groovy.lang.Closure)
+      BuildYumRepo_Jenkinsfile.library({identifier=jenkins@20211123, retriever=null})
+      BuildYumRepo_Jenkinsfile.pipeline(groovy.lang.Closure)
+         BuildYumRepo_Jenkinsfile.echo(Executing on agent [label:none])
+         BuildYumRepo_Jenkinsfile.stage(buildYumRepo, groovy.lang.Closure)
+            BuildYumRepo_Jenkinsfile.script(groovy.lang.Closure)
+               BuildYumRepo_Jenkinsfile.buildYumRepo({buildManifest=tests/data/opensearch-build-1.3.0.yml, baseUrl=https://ci.opensearch.org/ci/dbc/test/1.3.0/9/linux/x64})
+                  buildYumRepo.legacySCM(groovy.lang.Closure)
+                  buildYumRepo.library({identifier=jenkins@20211123, retriever=null})
+                  buildYumRepo.readYaml({file=tests/data/opensearch-build-1.3.0.yml})
+                  BuildManifest.asBoolean()
+                  buildYumRepo.echo(Create yum repo metadata and repo file tests/data/opensearch-build-1.3.0.yml)
+                  buildYumRepo.sh(createrepo "rpm/dist/opensearch")
+                  buildYumRepo.writeFile({file=rpm/dist/opensearch/opensearch-1.3.0.staging.repo, text=[opensearch-1.3.0-123-staging]
+name=OpenSearch 1.3.0 123 Staging
+baseurl=https://ci.opensearch.org/ci/dbc/test/1.3.0/9/linux/x64/rpm/dist/opensearch/
+enabled=1
+gpgcheck=0
+autorefresh=1
+type=rpm-md})
diff --git a/tests/jenkins/lib-testers/AssembleManifestLibTester.groovy b/tests/jenkins/lib-testers/AssembleManifestLibTester.groovy
@@ -0,0 +1,31 @@
+import static org.hamcrest.CoreMatchers.notNullValue
+import static org.hamcrest.MatcherAssert.assertThat
+
+class AssembleManifestLibTester extends LibFunctionTester {
+
+    private String buildManifest
+
+    public AssembleManifestLibTester(buildManifest) {
+        this.buildManifest = buildManifest
+    }
+
+    void configure(helper, binding) {
+
+        binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc')
+        binding.setVariable('JOB_NAME', 'vars-build')
+        binding.setVariable('BUILD_NUMBER', '123')
+    }
+
+    void parameterInvariantsAssertions(call) {
+        assertThat(call.args.buildManifest.first(), notNullValue())
+    }
+
+    boolean expectedParametersMatcher(call) {
+        return call.args.buildManifest.first().toString().equals(this.buildManifest)
+    }
+
+    String libFunctionName() {
+        return 'assembleManifest'
+    }
+
+}
diff --git a/tests/jenkins/lib-testers/BuildYumRepoTester.groovy b/tests/jenkins/lib-testers/BuildYumRepoTester.groovy
@@ -0,0 +1,34 @@
+import static org.hamcrest.CoreMatchers.notNullValue
+import static org.hamcrest.MatcherAssert.assertThat
+
+class BuildYumRepoTester extends LibFunctionTester {
+
+    private String buildManifest
+    private String baseUrl
+
+    public BuildYumRepoTester(buildManifest, baseUrl) {
+        this.buildManifest = buildManifest
+        this.baseUrl = baseUrl
+    }
+
+    void configure(helper, binding) {
+        binding.setVariable('BUILD_NUMBER', '123')
+
+        helper.registerAllowedMethod("writeFile", [Map])
+    }
+
+    void parameterInvariantsAssertions(call) {
+        assertThat(call.args.buildManifest.first(), notNullValue())
+        assertThat(call.args.baseUrl.first(), notNullValue())
+    }
+
+    boolean expectedParametersMatcher(call) {
+        return call.args.buildManifest.first().toString().equals(this.buildManifest) &&
+            call.args.baseUrl.first().toString().equals(this.baseUrl)
+    }
+
+    String libFunctionName() {
+        return 'buildYumRepo'
+    }
+
+}
diff --git a/vars/assembleManifest.groovy b/vars/assembleManifest.groovy
@@ -7,4 +7,11 @@ void call(Map args = [:]) {
         "\"${args.buildManifest}\"",
         "--base-url ${baseUrl}"
     ].join(' '))
+
+    if (buildManifest.build.distribution == 'rpm') {
+        buildYumRepo(
+            baseUrl: baseUrl,
+            buildManifest: args.buildManifest
+        )
+    }
 }
diff --git a/vars/buildYumRepo.groovy b/vars/buildYumRepo.groovy
@@ -0,0 +1,28 @@
+void call(Map args = [:]) {
+    lib = library(identifier: 'jenkins@20211123', retriever: legacySCM(scm))
+
+    def buildManifest = lib.jenkins.BuildManifest.new(readYaml(file: args.buildManifest))
+    echo "Create yum repo metadata and repo file ${args.buildManifest}"
+
+    def filename = buildManifest.build.getFilename()
+    def name = buildManifest.build.name
+    def version = buildManifest.build.version
+    def repoFilePath = "rpm/dist/${filename}"
+
+    sh([
+        'createrepo',
+        "\"${repoFilePath}\"",
+    ].join(' '))
+
+    def repoFileContent = [
+        "[${filename}-${version}-${BUILD_NUMBER}-staging]",
+        "name=${name} ${version} ${BUILD_NUMBER} Staging",
+        "baseurl=${args.baseUrl}/${repoFilePath}/",
+        "enabled=1",
+        "gpgcheck=0",
+        "autorefresh=1",
+        "type=rpm-md"
+    ].join('\n')
+
+    writeFile file: "${repoFilePath}/${filename}-${version}.staging.repo", text: repoFileContent
+}