Merge pull request #1182 from opendevstack/feature/fix-release-manage…

…r-rollout-image-listing

fix release manager rollout image listing when using Helm

CHANGELOG.md

@@ -1,7 +1,9 @@
 # Changelog
 
 ## Unreleased
+=======
 * Update TRC reference document with SSDS and TCP ([#1189](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1189))
+* Helm Deployment Strategy handle race condition when rollout strategy promoting previous version image ([#1182](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1182))
 
 ### Added
 * add devcontainer setup ([#1172](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1172))

src/org/ods/component/HelmDeploymentStrategy.groovy

@@ -1,5 +1,6 @@
 package org.ods.component
 
+import com.cloudbees.groovy.cps.NonCPS
 import groovy.json.JsonOutput
 import groovy.transform.TypeChecked
 import groovy.transform.TypeCheckingMode
@@ -195,14 +196,107 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy {
                         'helmAdditionalFlags': options.helmAdditionalFlags,
                     ])
                 rolloutData["${resourceKind}/${resourceName}"] = podData
+
+                // We need to find the pod that was created as a result of the deployment.
+                // The previous pod may still be alive when we use a rollout strategy.
+                // We can tell one from the other using their creation timestamp,
+                // being the most recent the one we are interested in.
+                def latestPods = getLatestPods(podData)
+                // While very unlikely, it may happen that there is more than one pod with the same timestamp.
+                // Note that timestamp resolution is seconds.
+                // If that happens, we are unable to know which is the correct pod.
+                // However, it doesn't matter which pod is the right one, if they all have the same images.
+                def sameImages = haveSameImages(latestPods)
+                if (!sameImages) {
+                    throw new RuntimeException(
+                        "Unable to determine the most recent Pod. " +
+                        "Multiple pods running with the same latest creation timestamp " +
+                        "and different images found for ${resourceName}"
+                    )
+                }
                 // TODO: Once the orchestration pipeline can deal with multiple replicas,
                 // update this to store multiple pod artifacts.
                 // TODO: Potential conflict if resourceName is duplicated between
                 // Deployment and DeploymentConfig resource.
-                context.addDeploymentToArtifactURIs(resourceName, podData[0]?.toMap())
+                context.addDeploymentToArtifactURIs(resourceName, latestPods[0]?.toMap())
             }
         }
-        rolloutData
+        return rolloutData
+    }
+
+    /**
+     * Returns the pods with the latest creation timestamp.
+     * Note that the resolution of this timestamp is seconds and there may be more than one pod with the same
+     * latest timestamp.
+     *
+     * @param pods the pods over which to find the latest ones.
+     * @return a list with all the pods sharing the same, latest timestamp.
+     */
+    @NonCPS
+    private static List getLatestPods(Iterable pods) {
+        return maxElements(pods) { it.podMetaDataCreationTimestamp }
     }
 
+    /**
+     * Checks whether all the given pods contain the same images, ignoring order and multiplicity.
+     *
+     * @param pods the pods to check for image equality.
+     * @return true if all the pods have the same images or false otherwise.
+     */
+    @NonCPS
+    private static boolean haveSameImages(Iterable pods) {
+        return areEqual(pods) { a, b ->
+            def imagesA = a.containers.values() as Set
+            def imagesB = b.containers.values() as Set
+            return imagesA == imagesB
+        }
+    }
+
+    /**
+     * Selects the items in the iterable which when passed as a parameter to the supplied closure
+     * return the maximum value. A null return value represents the least possible return value,
+     * so any item for which the supplied closure returns null, won't be selected (unless all items return null).
+     * The return list contains all the elements that returned the maximum value.
+     *
+     * @param iterable the iterable over which to search for maximum values.
+     * @param getValue a closure returning the value that corresponds to each element.
+     * @return the list of all the elements for which the closure returns the maximum value.
+     */
+    @NonCPS
+    private static List maxElements(Iterable iterable, Closure getValue) {
+        if (!iterable) {
+            return [] // Return an empty list if the iterable is null or empty
+        }
+
+        // Find the maximum value using the closure
+        def maxValue = iterable.collect(getValue).max()
+
+        // Find all elements with the maximum value
+        return iterable.findAll { getValue(it) == maxValue }
+    }
+
+    /**
+     * Checks whether all the elements in the given iterable are deemed as equal by the given closure.
+     *
+     * @param iterable the iterable over which to check for element equality.
+     * @param equals a closure that checks two elements for equality.
+     * @return true if all the elements are equal or false otherwise.
+     */
+    @NonCPS
+    private static boolean areEqual(Iterable iterable, Closure equals) {
+        def equal = true
+        if (iterable) {
+            def first = true
+            def base = null
+            iterable.each {
+                if (first) {
+                    base = it
+                    first = false
+                } else if (!equals(base, it)) {
+                    equal = false
+                }
+            }
+        }
+        return equal
+    }
 }

src/org/ods/util/PodData.groovy

@@ -17,6 +17,8 @@ class PodData {
 
     // podMetaDataCreationTimestamp equals .metadata.creationTimestamp.
     // Example: 2020-11-02T10:57:35Z
+    // We can use String to compare timestamps in this case,
+    // because ISO 8601 timestamps are designed to be sortable as strings.
     String podMetaDataCreationTimestamp
 
     // deploymentId is the name of the pod manager, such as the ReplicaSet or

test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy

@@ -83,4 +83,133 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase {
 
         assert expectedDeploymentMeans == actualDeploymentMeans
     }
+
+    def "rollout: check deploymentMean when multiple pods then accept only latest"() {
+        given:
+
+        def expectedDeploymentMeans = [
+            "builds": [:],
+            "deployments": [
+                "bar-deploymentMean": [
+                    "type": "helm",
+                    "selector": "app=foo-bar",
+                    "chartDir": "chart",
+                    "helmReleaseName": "bar",
+                    "helmEnvBasedValuesFiles": [],
+                    "helmValuesFiles": ["values.yaml"],
+                    "helmValues": [:],
+                    "helmDefaultFlags": ["--install", "--atomic"],
+                    "helmAdditionalFlags": []
+                ],
+                "bar":[
+                    "podName": null,
+                    "podNamespace": null,
+                    "podMetaDataCreationTimestamp": "2024-12-12T20:10:47Z",
+                    "deploymentId": "bar-124",
+                    "podNode": null,
+                    "podIp": null,
+                    "podStatus": null,
+                    "podStartupTimeStamp": null,
+                    "containers": [
+                        "containerA": "imageAnew",
+                        "containerB": "imageBnew",
+                    ],
+                ]
+            ]
+        ]
+        def config = [:]
+
+        def ctxData = contextData + [environment: 'dev', targetProject: 'foo-dev', openshiftRolloutTimeoutRetries: 5, chartDir: 'chart']
+        IContext context = new Context(null, ctxData, logger)
+        OpenShiftService openShiftService = Mock(OpenShiftService.class)
+        openShiftService.checkForPodData(*_) >> [
+            new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:46Z", containers: ["containerA": "imageAold", "containerB": "imageBold"]]),
+            new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:47Z", containers: ["containerA": "imageAnew", "containerB": "imageBnew"]]),
+            new PodData([deploymentId: "${contextData.componentId}-123", podMetaDataCreationTimestamp: "2024-11-11T20:10:46Z"])
+        ]
+        ServiceRegistry.instance.add(OpenShiftService, openShiftService)
+
+        JenkinsService jenkinsService = Stub(JenkinsService.class)
+        jenkinsService.maybeWithPrivateKeyCredentials(*_) >> { args -> args[1]('/tmp/file') }
+        ServiceRegistry.instance.add(JenkinsService, jenkinsService)
+
+        HelmDeploymentStrategy strategy = Spy(HelmDeploymentStrategy, constructorArgs: [null, context, config, openShiftService, jenkinsService, logger])
+
+        when:
+        def deploymentResources = [Deployment: ['bar']]
+        def rolloutData = strategy.getRolloutData(deploymentResources)
+        def actualDeploymentMeans = context.getBuildArtifactURIs()
+
+
+        then:
+        printCallStack()
+        assertJobStatusSuccess()
+
+        assert expectedDeploymentMeans == actualDeploymentMeans
+    }
+
+    def "rollout: check deploymentMean when multiple pods with same timestamp but different image then pipeline fails"() {
+        given:
+
+        def expectedDeploymentMeans = [
+            "builds": [:],
+            "deployments": [
+                "bar-deploymentMean": [
+                    "type": "helm",
+                    "selector": "app=foo-bar",
+                    "chartDir": "chart",
+                    "helmReleaseName": "bar",
+                    "helmEnvBasedValuesFiles": [],
+                    "helmValuesFiles": ["values.yaml"],
+                    "helmValues": [:],
+                    "helmDefaultFlags": ["--install", "--atomic"],
+                    "helmAdditionalFlags": []
+                ],
+                "bar":[
+                    "podName": null,
+                    "podNamespace": null,
+                    "podMetaDataCreationTimestamp": "2024-12-12T20:10:47Z",
+                    "deploymentId": "bar-124",
+                    "podNode": null,
+                    "podIp": null,
+                    "podStatus": null,
+                    "podStartupTimeStamp": null,
+                    "containers": [
+                        "containerA": "imageAnew",
+                        "containerB": "imageBnew",
+                    ],
+                ]
+            ]
+        ]
+        def config = [:]
+
+        def ctxData = contextData + [environment: 'dev', targetProject: 'foo-dev', openshiftRolloutTimeoutRetries: 5, chartDir: 'chart']
+        IContext context = new Context(null, ctxData, logger)
+        OpenShiftService openShiftService = Mock(OpenShiftService.class)
+        openShiftService.checkForPodData(*_) >> [
+            new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:47Z", containers: ["containerA": "imageAnew", "containerB": "imageBnew"]]),
+            new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:47Z", containers: ["containerA": "imageAold", "containerB": "imageBold"]]),
+        ]
+        ServiceRegistry.instance.add(OpenShiftService, openShiftService)
+
+        JenkinsService jenkinsService = Stub(JenkinsService.class)
+        jenkinsService.maybeWithPrivateKeyCredentials(*_) >> { args -> args[1]('/tmp/file') }
+        ServiceRegistry.instance.add(JenkinsService, jenkinsService)
+
+        HelmDeploymentStrategy strategy = Spy(HelmDeploymentStrategy, constructorArgs: [null, context, config, openShiftService, jenkinsService, logger])
+
+        when:
+        def deploymentResources = [Deployment: ['bar']]
+        def rolloutData = strategy.getRolloutData(deploymentResources)
+        def actualDeploymentMeans = context.getBuildArtifactURIs()
+
+
+        then:
+        printCallStack()
+        def e = thrown(RuntimeException)
+
+        assert e.message == "Unable to determine the most recent Pod. Multiple pods running with the same latest creation timestamp and different images found for bar"
+
+    }
+
 }