From 6726f703bfa1977568e5f2061881f65ead8b4a33 Mon Sep 17 00:00:00 2001 From: Joao Marcal Date: Thu, 5 Sep 2024 12:13:55 +0100 Subject: [PATCH] mcoa: add support for MCO installing CLO --- go.mod | 1 + go.sum | 2 + .../config/rbac/mco_role.yaml | 12 +++ .../multiclusterobservability_controller.go | 49 ++++++++- operators/multiclusterobservability/main.go | 4 + .../kustomization.yaml | 4 + .../cluster-logging-operator/namespace.yaml | 5 + .../operator_group.yaml | 9 ++ .../subscription.yaml | 11 +++ .../addon_deployment_config.yaml | 5 +- .../pkg/rendering/renderer.go | 24 +++++ .../pkg/rendering/renderer_clo.go | 99 +++++++++++++++++++ .../pkg/rendering/renderer_mcoa.go | 3 + .../pkg/rendering/templates/templates.go | 16 +++ 14 files changed, 239 insertions(+), 5 deletions(-) create mode 100644 operators/multiclusterobservability/manifests/base/cluster-logging-operator/kustomization.yaml create mode 100644 operators/multiclusterobservability/manifests/base/cluster-logging-operator/namespace.yaml create mode 100644 operators/multiclusterobservability/manifests/base/cluster-logging-operator/operator_group.yaml create mode 100644 operators/multiclusterobservability/manifests/base/cluster-logging-operator/subscription.yaml create mode 100644 operators/multiclusterobservability/pkg/rendering/renderer_clo.go diff --git a/go.mod b/go.mod index f410ba1db..e5eefed31 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/openshift/cluster-monitoring-operator v0.1.1-0.20240628115213-cd0d275afa06 github.com/openshift/hypershift/api v0.0.0-20240627155356-f85c65d962aa github.com/openshift/library-go v0.0.0-20240621150525-4bb4238aef81 + github.com/operator-framework/api v0.24.0 github.com/prometheus-community/prom-label-proxy v0.8.1-0.20240127162815-c1195f9aabc0 github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.74.0 github.com/prometheus-operator/prometheus-operator/pkg/client v0.74.0 diff --git a/go.sum b/go.sum index bf2d5773c..ee31c931d 100644 --- a/go.sum +++ b/go.sum @@ -895,6 +895,8 @@ github.com/openzipkin-contrib/zipkin-go-opentracing v0.4.5/go.mod h1:/wsWhb9smxS github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= +github.com/operator-framework/api v0.24.0 h1:fHynWEzuY/YhUTlsK9hd+QQ0bZcFakxCTdaZbFaVXbc= +github.com/operator-framework/api v0.24.0/go.mod h1:EXKrka63NyQDDpWZ+DGTDEliNV0xRq6UMZRoUPhilVM= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= diff --git a/operators/multiclusterobservability/config/rbac/mco_role.yaml b/operators/multiclusterobservability/config/rbac/mco_role.yaml index e95636115..a572482e1 100644 --- a/operators/multiclusterobservability/config/rbac/mco_role.yaml +++ b/operators/multiclusterobservability/config/rbac/mco_role.yaml @@ -381,3 +381,15 @@ rules: - get - list - watch +- apiGroups: + - operators.coreos.com + resources: + - operatorgroups + - subscriptions + verbs: + - get + - list + - watch + - create + - patch + - delete \ No newline at end of file diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go index ab621998e..e5407e9fe 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go @@ -17,13 +17,17 @@ import ( imagev1client "github.com/openshift/client-go/image/clientset/versioned/typed/image/v1" operatorconfig "github.com/stolostron/multicluster-observability-operator/operators/pkg/config" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" "github.com/go-logr/logr" routev1 "github.com/openshift/api/route/v1" + operatorsv1 "github.com/operator-framework/api/pkg/operators/v1" + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" mchv1 "github.com/stolostron/multiclusterhub-operator/api/v1" observatoriumv1alpha1 "github.com/stolostron/observatorium-operator/api/v1alpha1" + "golang.org/x/exp/slices" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -255,8 +259,18 @@ func (r *MultiClusterObservabilityReconciler) Reconcile(ctx context.Context, req } instance.Spec.StorageConfig.StorageClass = storageClassSelected + renderCLO, err := renderCLO(r.Client, instance) + if err != nil { + return ctrl.Result{}, err + } + // Render the templates with a specified CR - renderer := rendering.NewMCORenderer(instance, r.Client, r.ImageClient) + renderOptions := &rendering.RenderOptions{ + RenderCLO: renderCLO, + } + renderer := rendering.NewMCORenderer(instance, r.Client, r.ImageClient). + WithRenderOptions(renderOptions) + toDeploy, err := renderer.Render() if err != nil { reqLogger.Error(err, "Failed to render multiClusterMonitoring templates") @@ -472,6 +486,10 @@ func (r *MultiClusterObservabilityReconciler) SetupWithManager(mgr ctrl.Manager) Owns(&addonv1alpha1.AddOnDeploymentConfig{}). // Watch for changes to secondary ClusterManagementAddOn CR and requeue the owner MultiClusterObservability Owns(&addonv1alpha1.ClusterManagementAddOn{}). + // Watch for changes to secondary Subscription CR and requeue the owner MultiClusterObservability + Owns(&operatorsv1alpha1.Subscription{}). + // Watch for changes to secondary OperatorGroup CR and requeue the owner MultiClusterObservability + Owns(&operatorsv1.OperatorGroup{}). // Watch the configmap for thanos-ruler-custom-rules update Watches(&corev1.ConfigMap{}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(cmPred)). // Watch the secret for deleting event of alertmanager-config @@ -967,3 +985,32 @@ func (r *MultiClusterObservabilityReconciler) deleteServiceMonitorInOpenshiftMon } return nil } + +func renderCLO(kubeClient client.Client, instance *mcov1beta2.MultiClusterObservability) (bool, error) { + // MCO should not install CLO if capabilities is not enabled + if instance.Spec.Capabilities == nil { + return false, nil + } + + crd := &apiextensionsv1.CustomResourceDefinition{} + key := client.ObjectKey{Name: "clusterlogforwarders.logging.openshift.io"} + if err := kubeClient.Get(context.TODO(), key, crd); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + + // TODO @JoaoBraveCoding: maybe we should check if the namespace also exists + + subscription := operatorsv1alpha1.Subscription{} + key = client.ObjectKey{Name: "cluster-logging", Namespace: "openshift-logging"} + if err := kubeClient.Get(context.TODO(), key, &subscription); err != nil { + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + } + + return true, nil +} diff --git a/operators/multiclusterobservability/main.go b/operators/multiclusterobservability/main.go index 58defaa3d..b0b4745b0 100644 --- a/operators/multiclusterobservability/main.go +++ b/operators/multiclusterobservability/main.go @@ -25,6 +25,8 @@ import ( oauthv1 "github.com/openshift/api/oauth/v1" operatorv1 "github.com/openshift/api/operator/v1" routev1 "github.com/openshift/api/route/v1" + operatorsv1 "github.com/operator-framework/api/pkg/operators/v1" + operatorsv1alpha1 "github.com/operator-framework/api/pkg/operators/v1alpha1" prometheusv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -72,6 +74,8 @@ func init() { utilruntime.Must(observatoriumAPIs.AddToScheme(scheme)) utilruntime.Must(prometheusv1.AddToScheme(scheme)) utilruntime.Must(addonv1alpha1.AddToScheme(scheme)) + utilruntime.Must(operatorsv1.AddToScheme(scheme)) + utilruntime.Must(operatorsv1alpha1.AddToScheme(scheme)) utilruntime.Must(imagev1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/operators/multiclusterobservability/manifests/base/cluster-logging-operator/kustomization.yaml b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/kustomization.yaml new file mode 100644 index 000000000..123cdca0c --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- namespace.yaml +- operator_group.yaml +- subscription.yaml diff --git a/operators/multiclusterobservability/manifests/base/cluster-logging-operator/namespace.yaml b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/namespace.yaml new file mode 100644 index 000000000..fff442e83 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/namespace.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-logging +spec: {} \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/cluster-logging-operator/operator_group.yaml b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/operator_group.yaml new file mode 100644 index 000000000..c9fa90b64 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/operator_group.yaml @@ -0,0 +1,9 @@ +apiVersion: operators.coreos.com/v1 +kind: OperatorGroup +metadata: + name: openshift-logging + namespace: openshift-logging + annotations: + olm.providedAPIs: ClusterLogForwarder.v1.logging.openshift.io,ClusterLogging.v1.logging.openshift.io +spec: + upgradeStrategy: Default \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/cluster-logging-operator/subscription.yaml b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/subscription.yaml new file mode 100644 index 000000000..1ee24beb9 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/cluster-logging-operator/subscription.yaml @@ -0,0 +1,11 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + name: cluster-logging + namespace: openshift-logging +spec: + channel: stable-5.9 + installPlanApproval: Automatic + name: cluster-logging + source: redhat-operators + sourceNamespace: openshift-marketplace \ No newline at end of file diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml index fe8d773e1..6f1878bfe 100644 --- a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml @@ -4,7 +4,4 @@ metadata: name: multicluster-observability-addon namespace: open-cluster-management-observability spec: - customizedVariables: - # Operator Subscription Channels - - name: openshiftLoggingChannel - value: stable-5.9 + customizedVariables: [] diff --git a/operators/multiclusterobservability/pkg/rendering/renderer.go b/operators/multiclusterobservability/pkg/rendering/renderer.go index b66cf28cf..3906bb36a 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer.go @@ -23,15 +23,21 @@ import ( var log = logf.Log.WithName("renderer") +type RenderOptions struct { + RenderCLO bool +} + type MCORenderer struct { kubeClient client.Client imageClient *imagev1client.ImageV1Client renderer *rendererutil.Renderer cr *obv1beta2.MultiClusterObservability + options *RenderOptions renderGrafanaFns map[string]rendererutil.RenderFn renderAlertManagerFns map[string]rendererutil.RenderFn renderThanosFns map[string]rendererutil.RenderFn renderProxyFns map[string]rendererutil.RenderFn + renderCLOFns map[string]rendererutil.RenderFn renderMCOAFns map[string]rendererutil.RenderFn } @@ -46,10 +52,16 @@ func NewMCORenderer(multipleClusterMonitoring *obv1beta2.MultiClusterObservabili mcoRenderer.newAlertManagerRenderer() mcoRenderer.newThanosRenderer() mcoRenderer.newProxyRenderer() + mcoRenderer.newCLORenderer() mcoRenderer.newMCOARenderer() return mcoRenderer } +func (r *MCORenderer) WithRenderOptions(options *RenderOptions) *MCORenderer { + r.options = options + return r +} + func (r *MCORenderer) Render() ([]*unstructured.Unstructured, error) { // load and render generic templates genericTemplates, err := templates.GetOrLoadGenericTemplates(templatesutil.GetTemplateRenderer()) @@ -109,6 +121,18 @@ func (r *MCORenderer) Render() ([]*unstructured.Unstructured, error) { } resources = append(resources, proxyResources...) + // load and render cluster-logging-operator templates + if r.options != nil && r.options.RenderCLO { + cloTemplates, err := templates.GetOrLoadCLOTemplates(templatesutil.GetTemplateRenderer()) + if err != nil { + return nil, err + } + cloResources, err := r.renderCLOTemplates(cloTemplates, namespace, labels) + if err != nil { + return nil, err + } + resources = append(resources, cloResources...) + } // load and render multicluster-observability-addon templates mcoaTemplates, err := templates.GetOrLoadMCOATemplates(templatesutil.GetTemplateRenderer()) if err != nil { diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_clo.go b/operators/multiclusterobservability/pkg/rendering/renderer_clo.go new file mode 100644 index 000000000..93ef9f2d3 --- /dev/null +++ b/operators/multiclusterobservability/pkg/rendering/renderer_clo.go @@ -0,0 +1,99 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package rendering + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/kustomize/api/resource" + + rendererutil "github.com/stolostron/multicluster-observability-operator/operators/pkg/rendering" +) + +const ( + cloSubscriptionChannel = "stable-5.9" +) + +func (r *MCORenderer) newCLORenderer() { + r.renderCLOFns = map[string]rendererutil.RenderFn{ + "Namespace": r.renderer.RenderNamespace, + "Subscription": r.renderSubscription, + "OperatorGroup": r.renderOperatorGroup, + } +} + +func (r *MCORenderer) renderSubscription( + res *resource.Resource, + namespace string, + labels map[string]string, +) (*unstructured.Unstructured, error) { + m, err := res.Map() + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{Object: m} + + cLabels := u.GetLabels() + if cLabels == nil { + cLabels = make(map[string]string) + } + for k, v := range labels { + cLabels[k] = v + } + u.SetLabels(cLabels) + + return u, nil +} +func (r *MCORenderer) renderOperatorGroup( + res *resource.Resource, + namespace string, + labels map[string]string, +) (*unstructured.Unstructured, error) { + m, err := res.Map() + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{Object: m} + + cLabels := u.GetLabels() + if cLabels == nil { + cLabels = make(map[string]string) + } + for k, v := range labels { + cLabels[k] = v + } + u.SetLabels(cLabels) + + return u, nil +} + +func (r *MCORenderer) renderCLOTemplates( + templates []*resource.Resource, + namespace string, + labels map[string]string, +) ([]*unstructured.Unstructured, error) { + uobjs := []*unstructured.Unstructured{} + for _, template := range templates { + render, ok := r.renderMCOAFns[template.GetKind()] + if !ok { + m, err := template.Map() + if err != nil { + return []*unstructured.Unstructured{}, err + } + uobjs = append(uobjs, &unstructured.Unstructured{Object: m}) + continue + } + uobj, err := render(template.DeepCopy(), namespace, labels) + if err != nil { + return []*unstructured.Unstructured{}, err + } + if uobj == nil { + continue + } + uobjs = append(uobjs, uobj) + + } + + return uobjs, nil +} diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go b/operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go index 48f5446d6..df4715fdf 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go @@ -26,6 +26,7 @@ const ( nameUserWorkloadLogsCollection = "userWorkloadLogsCollection" nameUserWorkloadTracesCollection = "userWorkloadTracesCollection" nameUserWorkloadInstrumentation = "userWorkloadInstrumentation" + nameOpenshiftLoggingChannel = "openshiftLoggingChannel" // AODC CustomizedVariable Values clfV1 = "clusterlogforwarders.v1.logging.openshift.io" @@ -188,6 +189,8 @@ func (r *MCORenderer) renderAddonDeploymentConfig( ) } + appendCustomVar(aodc, nameOpenshiftLoggingChannel, cloSubscriptionChannel) + if cs.Platform != nil { if cs.Platform.Logs.Collection.Enabled { appendCustomVar(aodc, namePlatformLogsCollection, clfV1) diff --git a/operators/multiclusterobservability/pkg/rendering/templates/templates.go b/operators/multiclusterobservability/pkg/rendering/templates/templates.go index 67ee900bc..b4bfd544c 100644 --- a/operators/multiclusterobservability/pkg/rendering/templates/templates.go +++ b/operators/multiclusterobservability/pkg/rendering/templates/templates.go @@ -21,6 +21,7 @@ var ( proxyTemplates []*resource.Resource endpointObservabilityTemplates []*resource.Resource prometheusTemplates []*resource.Resource + cloTemplates []*resource.Resource mcoaTemplates []*resource.Resource ) @@ -105,6 +106,21 @@ func GetOrLoadProxyTemplates(r *templates.TemplateRenderer) ([]*resource.Resourc return proxyTemplates, nil } +// GetOrLoadCLOTemplates reads the cluster-logging-operator manifests. +func GetOrLoadCLOTemplates(r *templates.TemplateRenderer) ([]*resource.Resource, error) { + if len(cloTemplates) > 0 { + return cloTemplates, nil + } + + basePath := path.Join(r.GetTemplatesPath(), "base") + + // add mcoa templates + if err := r.AddTemplateFromPath(path.Join(basePath, "cluster-logging-operator"), &cloTemplates); err != nil { + return mcoaTemplates, err + } + return cloTemplates, nil +} + // GetOrLoadMCOATemplates reads the multicluster-observability-addon manifests. func GetOrLoadMCOATemplates(r *templates.TemplateRenderer) ([]*resource.Resource, error) { if len(mcoaTemplates) > 0 {