diff --git a/CHANGELOG.md b/CHANGELOG.md index 194685036..968c74216 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # Changelog ## Unreleased +======= * Update TRC reference document with SSDS and TCP ([#1189](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1189)) +* Helm Deployment Strategy handle race condition when rollout strategy promoting previous version image ([#1182](https://github.com/opendevstack/ods-jenkins-shared-library/pull/1182)) ### Added * add devcontainer setup ([#1172](https://github.com/opendevstack/ods-jenkins-shared-library/issues/1172)) diff --git a/src/org/ods/component/HelmDeploymentStrategy.groovy b/src/org/ods/component/HelmDeploymentStrategy.groovy index 908cc14fb..a6fd31a8e 100644 --- a/src/org/ods/component/HelmDeploymentStrategy.groovy +++ b/src/org/ods/component/HelmDeploymentStrategy.groovy @@ -1,5 +1,6 @@ package org.ods.component +import com.cloudbees.groovy.cps.NonCPS import groovy.json.JsonOutput import groovy.transform.TypeChecked import groovy.transform.TypeCheckingMode @@ -195,14 +196,107 @@ class HelmDeploymentStrategy extends AbstractDeploymentStrategy { 'helmAdditionalFlags': options.helmAdditionalFlags, ]) rolloutData["${resourceKind}/${resourceName}"] = podData + + // We need to find the pod that was created as a result of the deployment. + // The previous pod may still be alive when we use a rollout strategy. + // We can tell one from the other using their creation timestamp, + // being the most recent the one we are interested in. + def latestPods = getLatestPods(podData) + // While very unlikely, it may happen that there is more than one pod with the same timestamp. + // Note that timestamp resolution is seconds. + // If that happens, we are unable to know which is the correct pod. + // However, it doesn't matter which pod is the right one, if they all have the same images. + def sameImages = haveSameImages(latestPods) + if (!sameImages) { + throw new RuntimeException( + "Unable to determine the most recent Pod. " + + "Multiple pods running with the same latest creation timestamp " + + "and different images found for ${resourceName}" + ) + } // TODO: Once the orchestration pipeline can deal with multiple replicas, // update this to store multiple pod artifacts. // TODO: Potential conflict if resourceName is duplicated between // Deployment and DeploymentConfig resource. - context.addDeploymentToArtifactURIs(resourceName, podData[0]?.toMap()) + context.addDeploymentToArtifactURIs(resourceName, latestPods[0]?.toMap()) } } - rolloutData + return rolloutData + } + + /** + * Returns the pods with the latest creation timestamp. + * Note that the resolution of this timestamp is seconds and there may be more than one pod with the same + * latest timestamp. + * + * @param pods the pods over which to find the latest ones. + * @return a list with all the pods sharing the same, latest timestamp. + */ + @NonCPS + private static List getLatestPods(Iterable pods) { + return maxElements(pods) { it.podMetaDataCreationTimestamp } } + /** + * Checks whether all the given pods contain the same images, ignoring order and multiplicity. + * + * @param pods the pods to check for image equality. + * @return true if all the pods have the same images or false otherwise. + */ + @NonCPS + private static boolean haveSameImages(Iterable pods) { + return areEqual(pods) { a, b -> + def imagesA = a.containers.values() as Set + def imagesB = b.containers.values() as Set + return imagesA == imagesB + } + } + + /** + * Selects the items in the iterable which when passed as a parameter to the supplied closure + * return the maximum value. A null return value represents the least possible return value, + * so any item for which the supplied closure returns null, won't be selected (unless all items return null). + * The return list contains all the elements that returned the maximum value. + * + * @param iterable the iterable over which to search for maximum values. + * @param getValue a closure returning the value that corresponds to each element. + * @return the list of all the elements for which the closure returns the maximum value. + */ + @NonCPS + private static List maxElements(Iterable iterable, Closure getValue) { + if (!iterable) { + return [] // Return an empty list if the iterable is null or empty + } + + // Find the maximum value using the closure + def maxValue = iterable.collect(getValue).max() + + // Find all elements with the maximum value + return iterable.findAll { getValue(it) == maxValue } + } + + /** + * Checks whether all the elements in the given iterable are deemed as equal by the given closure. + * + * @param iterable the iterable over which to check for element equality. + * @param equals a closure that checks two elements for equality. + * @return true if all the elements are equal or false otherwise. + */ + @NonCPS + private static boolean areEqual(Iterable iterable, Closure equals) { + def equal = true + if (iterable) { + def first = true + def base = null + iterable.each { + if (first) { + base = it + first = false + } else if (!equals(base, it)) { + equal = false + } + } + } + return equal + } } diff --git a/src/org/ods/util/PodData.groovy b/src/org/ods/util/PodData.groovy index dd26b8903..2ef4e7479 100644 --- a/src/org/ods/util/PodData.groovy +++ b/src/org/ods/util/PodData.groovy @@ -17,6 +17,8 @@ class PodData { // podMetaDataCreationTimestamp equals .metadata.creationTimestamp. // Example: 2020-11-02T10:57:35Z + // We can use String to compare timestamps in this case, + // because ISO 8601 timestamps are designed to be sortable as strings. String podMetaDataCreationTimestamp // deploymentId is the name of the pod manager, such as the ReplicaSet or diff --git a/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy b/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy index 81edfd3b5..2d8d10510 100644 --- a/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy +++ b/test/groovy/org/ods/component/HelmDeploymentStrategySpec.groovy @@ -83,4 +83,133 @@ class HelmDeploymentStrategySpec extends PipelineSpockTestBase { assert expectedDeploymentMeans == actualDeploymentMeans } + + def "rollout: check deploymentMean when multiple pods then accept only latest"() { + given: + + def expectedDeploymentMeans = [ + "builds": [:], + "deployments": [ + "bar-deploymentMean": [ + "type": "helm", + "selector": "app=foo-bar", + "chartDir": "chart", + "helmReleaseName": "bar", + "helmEnvBasedValuesFiles": [], + "helmValuesFiles": ["values.yaml"], + "helmValues": [:], + "helmDefaultFlags": ["--install", "--atomic"], + "helmAdditionalFlags": [] + ], + "bar":[ + "podName": null, + "podNamespace": null, + "podMetaDataCreationTimestamp": "2024-12-12T20:10:47Z", + "deploymentId": "bar-124", + "podNode": null, + "podIp": null, + "podStatus": null, + "podStartupTimeStamp": null, + "containers": [ + "containerA": "imageAnew", + "containerB": "imageBnew", + ], + ] + ] + ] + def config = [:] + + def ctxData = contextData + [environment: 'dev', targetProject: 'foo-dev', openshiftRolloutTimeoutRetries: 5, chartDir: 'chart'] + IContext context = new Context(null, ctxData, logger) + OpenShiftService openShiftService = Mock(OpenShiftService.class) + openShiftService.checkForPodData(*_) >> [ + new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:46Z", containers: ["containerA": "imageAold", "containerB": "imageBold"]]), + new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:47Z", containers: ["containerA": "imageAnew", "containerB": "imageBnew"]]), + new PodData([deploymentId: "${contextData.componentId}-123", podMetaDataCreationTimestamp: "2024-11-11T20:10:46Z"]) + ] + ServiceRegistry.instance.add(OpenShiftService, openShiftService) + + JenkinsService jenkinsService = Stub(JenkinsService.class) + jenkinsService.maybeWithPrivateKeyCredentials(*_) >> { args -> args[1]('/tmp/file') } + ServiceRegistry.instance.add(JenkinsService, jenkinsService) + + HelmDeploymentStrategy strategy = Spy(HelmDeploymentStrategy, constructorArgs: [null, context, config, openShiftService, jenkinsService, logger]) + + when: + def deploymentResources = [Deployment: ['bar']] + def rolloutData = strategy.getRolloutData(deploymentResources) + def actualDeploymentMeans = context.getBuildArtifactURIs() + + + then: + printCallStack() + assertJobStatusSuccess() + + assert expectedDeploymentMeans == actualDeploymentMeans + } + + def "rollout: check deploymentMean when multiple pods with same timestamp but different image then pipeline fails"() { + given: + + def expectedDeploymentMeans = [ + "builds": [:], + "deployments": [ + "bar-deploymentMean": [ + "type": "helm", + "selector": "app=foo-bar", + "chartDir": "chart", + "helmReleaseName": "bar", + "helmEnvBasedValuesFiles": [], + "helmValuesFiles": ["values.yaml"], + "helmValues": [:], + "helmDefaultFlags": ["--install", "--atomic"], + "helmAdditionalFlags": [] + ], + "bar":[ + "podName": null, + "podNamespace": null, + "podMetaDataCreationTimestamp": "2024-12-12T20:10:47Z", + "deploymentId": "bar-124", + "podNode": null, + "podIp": null, + "podStatus": null, + "podStartupTimeStamp": null, + "containers": [ + "containerA": "imageAnew", + "containerB": "imageBnew", + ], + ] + ] + ] + def config = [:] + + def ctxData = contextData + [environment: 'dev', targetProject: 'foo-dev', openshiftRolloutTimeoutRetries: 5, chartDir: 'chart'] + IContext context = new Context(null, ctxData, logger) + OpenShiftService openShiftService = Mock(OpenShiftService.class) + openShiftService.checkForPodData(*_) >> [ + new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:47Z", containers: ["containerA": "imageAnew", "containerB": "imageBnew"]]), + new PodData([deploymentId: "${contextData.componentId}-124", podMetaDataCreationTimestamp: "2024-12-12T20:10:47Z", containers: ["containerA": "imageAold", "containerB": "imageBold"]]), + ] + ServiceRegistry.instance.add(OpenShiftService, openShiftService) + + JenkinsService jenkinsService = Stub(JenkinsService.class) + jenkinsService.maybeWithPrivateKeyCredentials(*_) >> { args -> args[1]('/tmp/file') } + ServiceRegistry.instance.add(JenkinsService, jenkinsService) + + HelmDeploymentStrategy strategy = Spy(HelmDeploymentStrategy, constructorArgs: [null, context, config, openShiftService, jenkinsService, logger]) + + when: + def deploymentResources = [Deployment: ['bar']] + def rolloutData = strategy.getRolloutData(deploymentResources) + def actualDeploymentMeans = context.getBuildArtifactURIs() + + + then: + printCallStack() + def e = thrown(RuntimeException) + + assert e.message == "Unable to determine the most recent Pod. Multiple pods running with the same latest creation timestamp and different images found for bar" + + } + }