Fix hub template watches

This fixes the case where multiple managed clusters with the same policy
with a hub template don't get updated when the referenced object in the
hub template is changed.

This additionally handles clean up properly when template resolution
fails.

Relates:
https://issues.redhat.com/browse/ACM-8928

Signed-off-by: mprahl <[email protected]>
(cherry picked from commit f81d226)

controllers/propagator/propagation.go

@@ -257,6 +257,16 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
 	}
 
 	var templateResult templates.TemplateResult
+	var cacheCleanUp templates.CacheCleanUpFunc
+
+	defer func() {
+		if cacheCleanUp != nil {
+			err := cacheCleanUp()
+			if err != nil {
+				log.Error(err, "Failed to perform the cache clean up after template resolution")
+			}
+		}
+	}()
 
 	// A policy can have multiple policy templates within it, iterate and process each
 	for _, policyT := range replicatedPlc.Spec.PolicyTemplates {
@@ -330,6 +340,10 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
 			policyT.ObjectDefinition.Raw, templateContext, &templateResolverOptions,
 		)
 
+		if templateResult.CacheCleanUp != nil {
+			cacheCleanUp = templateResult.CacheCleanUp
+		}
+
 		if tplErr != nil {
 			log.Error(tplErr, "Failed to resolve templates")
 
@@ -415,13 +429,6 @@ func (r *ReplicatedPolicyReconciler) processTemplates(
 		}
 	}
 
-	if templateResult.CacheCleanUp != nil {
-		err := templateResult.CacheCleanUp()
-		if err != nil {
-			return fmt.Errorf("%w%w", ErrRetryable, err)
-		}
-	}
-
 	log.V(1).Info("Successfully processed templates")
 
 	return nil

go.mod

@@ -13,8 +13,8 @@ require (
 	github.com/prometheus/client_golang v1.17.0
 	github.com/spf13/pflag v1.0.5
 	github.com/stolostron/go-log-utils v0.1.2
-	github.com/stolostron/go-template-utils/v4 v4.0.0
-	github.com/stolostron/kubernetes-dependency-watches v0.5.1
+	github.com/stolostron/go-template-utils/v4 v4.0.1-0.20231212190701-4dc096ec1b40
+	github.com/stolostron/kubernetes-dependency-watches v0.5.2-0.20231212185913-628ab39622b8
 	k8s.io/api v0.27.7
 	k8s.io/apimachinery v0.27.7
 	k8s.io/client-go v0.27.7

go.sum

@@ -193,10 +193,10 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stolostron/go-log-utils v0.1.2 h1:7l1aJWvBqU2+DUyimcslT5SJpdygVY/clRDmX5sO29c=
 github.com/stolostron/go-log-utils v0.1.2/go.mod h1:8zrB8UJmp1rXhv3Ck9bBl5SpNfKk3SApeElbo96YRtQ=
-github.com/stolostron/go-template-utils/v4 v4.0.0 h1:gvSfhXIoymo5Ql9MH/ofTTOtBVkaUBq8HokCGR4xkkc=
-github.com/stolostron/go-template-utils/v4 v4.0.0/go.mod h1:svIOPUJpG/ObRn3WwZMBGMEMsBgKH8LVfhsaIArgAAQ=
-github.com/stolostron/kubernetes-dependency-watches v0.5.1 h1:NZ9N5/VWtwKcawgg4TGI4A5+weSkLrXZMjU7w91xfvU=
-github.com/stolostron/kubernetes-dependency-watches v0.5.1/go.mod h1:8vRsL1GGBw0jjCwP8CH8d30NVNYKhUy0rdBSQZ2znx8=
+github.com/stolostron/go-template-utils/v4 v4.0.1-0.20231212190701-4dc096ec1b40 h1:7j8DANWGJzpgTLl0xdNvjVDK2kdqcaYwBoyJ9JH4thw=
+github.com/stolostron/go-template-utils/v4 v4.0.1-0.20231212190701-4dc096ec1b40/go.mod h1:HtsbCTMS4oFBQpy8le4/Td1YTsHfsh1aP8uRMcquaHI=
+github.com/stolostron/kubernetes-dependency-watches v0.5.2-0.20231212185913-628ab39622b8 h1:fA4m/qD8S/l4jSyf2W/eG1iCSnokRXfkoKde4Ohy1f8=
+github.com/stolostron/kubernetes-dependency-watches v0.5.2-0.20231212185913-628ab39622b8/go.mod h1:8vRsL1GGBw0jjCwP8CH8d30NVNYKhUy0rdBSQZ2znx8=
 github.com/stolostron/multicloud-operators-subscription v1.2.4-0-20211122-7277a37.0.20231027053049-af37552e5602 h1:yTNmyt3R/76cG8ikLKG046PJcmhXpn8dbbigXYpmu4s=
 github.com/stolostron/multicloud-operators-subscription v1.2.4-0-20211122-7277a37.0.20231027053049-af37552e5602/go.mod h1:+p7x8NNLtrlfv8ZDjomRYAoIpHUd4R4XMo3jYwZWWmc=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=