From 0a3c387c94d154024a7a4a730256746512cd23bc Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Sat, 13 Jan 2024 19:43:03 -0800 Subject: [PATCH] Make cluster-proxy work with multicluster-controlplane Signed-off-by: Tamal Saha --- .gitignore | 2 +- Makefile | 2 +- .../templates/manager-deployment.yaml | 2 +- charts/cluster-proxy/values.yaml | 8 +-- cmd/addon-manager/main.go | 64 +++++++++++++++++-- pkg/proxyagent/agent/agent.go | 3 +- .../templates/addon-agent-deployment.yaml | 4 +- .../managedproxyconfiguration_controller.go | 51 ++++++++++++--- pkg/proxyserver/controllers/manifests.go | 30 ++++----- 9 files changed, 127 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 42130a32..a67b8dbe 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ *.out # Dependency directories (remove the comment below to include it) -# vendor/ +vendor/ /apiserver.local.config /bin diff --git a/Makefile b/Makefile index 7d44bdb4..a198fa75 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ # Image URL to use all building/pushing image targets -IMG ?= controller:latest IMAGE_REGISTRY_NAME ?= quay.io/open-cluster-management IMAGE_NAME = cluster-proxy IMAGE_TAG ?= latest +IMG ?= $(IMAGE_REGISTRY_NAME)/$(IMAGE_NAME):$(IMAGE_TAG) E2E_TEST_CLUSTER_NAME ?= loopback # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false" diff --git a/charts/cluster-proxy/templates/manager-deployment.yaml b/charts/cluster-proxy/templates/manager-deployment.yaml index cafcc1b8..cf0a0650 100644 --- a/charts/cluster-proxy/templates/manager-deployment.yaml +++ b/charts/cluster-proxy/templates/manager-deployment.yaml @@ -17,7 +17,7 @@ spec: containers: - name: manager image: {{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag | default (print "v" .Chart.Version) }} - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: - /manager args: diff --git a/charts/cluster-proxy/values.yaml b/charts/cluster-proxy/values.yaml index cf619840..4220e7f5 100644 --- a/charts/cluster-proxy/values.yaml +++ b/charts/cluster-proxy/values.yaml @@ -1,19 +1,19 @@ # Image registry -registry: quay.io/open-cluster-management +registry: ghcr.io/kluster-manager # Image of the cluster-gateway instances image: cluster-proxy # Image tag -tag: +tag: latest # Number of replicas replicas: 1 spokeAddonNamespace: "open-cluster-management-cluster-proxy" -proxyServerImage: quay.io/open-cluster-management/cluster-proxy -proxyAgentImage: quay.io/open-cluster-management/cluster-proxy +proxyServerImage: ghcr.io/kluster-manager/cluster-proxy +proxyAgentImage: ghcr.io/kluster-manager/cluster-proxy proxyServer: entrypointLoadBalancer: false diff --git a/cmd/addon-manager/main.go b/cmd/addon-manager/main.go index 1e06694a..1dae4751 100644 --- a/cmd/addon-manager/main.go +++ b/cmd/addon-manager/main.go @@ -31,6 +31,8 @@ import ( "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" "open-cluster-management.io/addon-framework/pkg/addonmanager" @@ -46,6 +48,8 @@ import ( "open-cluster-management.io/cluster-proxy/pkg/proxyserver/controllers" "open-cluster-management.io/cluster-proxy/pkg/proxyserver/operator/authentication/selfsigned" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/healthz" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" //+kubebuilder:scaffold:imports @@ -72,6 +76,7 @@ func main() { var signerSecretNamespace, signerSecretName string var agentInstallAll bool var enableKubeApiProxy bool + var mcKubeconfig string logger := klogr.New() klog.SetOutput(os.Stdout) @@ -95,13 +100,30 @@ func main() { "Configure the install strategy of agent on managed clusters. "+ "Enabling this will automatically install agent on all managed cluster.") flag.BoolVar(&enableKubeApiProxy, "enable-kube-api-proxy", true, "Enable proxy to agent kube-apiserver") + flag.StringVar(&mcKubeconfig, "multicluster-kubeconfig", "", + "The path to multicluster-controlplane kubeconfig") flag.Parse() // pipe controller-runtime logs to klog ctrl.SetLogger(logger) - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + var mcConfig, hostConfig *rest.Config + + if mcKubeconfig != "" { + var err error + mcConfig, err = clientcmd.BuildConfigFromFlags("", mcKubeconfig) + if err != nil { + setupLog.Error(err, "unable to build multicluster rest config") + os.Exit(1) + } + hostConfig = ctrl.GetConfigOrDie() + } else { + hostConfig = ctrl.GetConfigOrDie() + mcConfig = hostConfig + } + + mgr, err := ctrl.NewManager(mcConfig, ctrl.Options{ Scheme: scheme, Metrics: metricsserver.Options{BindAddress: metricsAddr}, HealthProbeBindAddress: probeAddr, @@ -119,6 +141,12 @@ func main() { os.Exit(1) } + hostClient, err := kubernetes.NewForConfig(hostConfig) + if err != nil { + setupLog.Error(err, "unable to set up host kubernetes native client") + os.Exit(1) + } + nativeClient, err := kubernetes.NewForConfig(mgr.GetConfig()) if err != nil { setupLog.Error(err, "unable to set up kubernetes native client") @@ -147,22 +175,30 @@ func main() { } informerFactory := externalversions.NewSharedInformerFactory(client, 0) - nativeInformer := informers.NewSharedInformerFactoryWithOptions(nativeClient, 0) + hostInformer := informers.NewSharedInformerFactoryWithOptions(hostClient, 0, informers.WithNamespace(signerSecretNamespace)) // loading self-signer selfSigner, err := selfsigned.NewSelfSignerFromSecretOrGenerate( - nativeClient, signerSecretNamespace, signerSecretName) + hostClient, signerSecretNamespace, signerSecretName) if err != nil { setupLog.Error(err, "failed loading self-signer") os.Exit(1) } + hostKubeClient, err := newHostClient(hostConfig) + if err != nil { + setupLog.Error(err, "failed create host KubeClient") + os.Exit(1) + } + if err := controllers.RegisterClusterManagementAddonReconciler( mgr, selfSigner, - nativeClient, - nativeInformer.Core().V1().Secrets(), + hostKubeClient, + hostClient, + hostInformer.Core().V1().Secrets(), supportsV1CSR, + mcKubeconfig != "", ); err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterManagementAddonReconciler") os.Exit(1) @@ -195,6 +231,7 @@ func main() { supportsV1CSR, mgr.GetClient(), nativeClient, + hostClient, agentInstallAll, enableKubeApiProxy, addonClient, @@ -212,7 +249,7 @@ func main() { ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler()) defer cancel() go informerFactory.Start(ctx.Done()) - go nativeInformer.Start(ctx.Done()) + go hostInformer.Start(ctx.Done()) go func() { if err := addonManager.Start(ctx); err != nil { setupLog.Error(err, "unable to start addon manager") @@ -225,3 +262,18 @@ func main() { os.Exit(1) } } + +func newHostClient(hostConfig *rest.Config) (client.Client, error) { + hc, err := rest.HTTPClientFor(hostConfig) + if err != nil { + return nil, err + } + mapper, err := apiutil.NewDynamicRESTMapper(hostConfig, hc) + if err != nil { + return nil, err + } + return client.New(hostConfig, client.Options{ + Scheme: clientgoscheme.Scheme, + Mapper: mapper, + }) +} diff --git a/pkg/proxyagent/agent/agent.go b/pkg/proxyagent/agent/agent.go index 4be2d37e..537b0aa0 100644 --- a/pkg/proxyagent/agent/agent.go +++ b/pkg/proxyagent/agent/agent.go @@ -53,6 +53,7 @@ func NewAgentAddon( v1CSRSupported bool, runtimeClient client.Client, nativeClient kubernetes.Interface, + hostClient kubernetes.Interface, agentInstallAll bool, enableKubeApiProxy bool, addonClient addonclient.Interface) (agent.AgentAddon, error) { @@ -134,7 +135,7 @@ func NewAgentAddon( utils.AddOnDeploymentConfigGVR, ). WithGetValuesFuncs( - GetClusterProxyValueFunc(runtimeClient, nativeClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy), + GetClusterProxyValueFunc(runtimeClient, hostClient, signerNamespace, caCertData, v1CSRSupported, enableKubeApiProxy), addonfactory.GetAddOnDeploymentConfigValues( utils.NewAddOnDeploymentConfigGetter(addonClient), toAgentAddOnChartValues(caCertData), diff --git a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml index 7eb49da1..e81cea49 100644 --- a/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml +++ b/pkg/proxyagent/agent/manifests/charts/addon-agent/templates/addon-agent-deployment.yaml @@ -38,7 +38,7 @@ spec: containers: - name: proxy-agent image: {{ .Values.proxyAgentImage }} - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: - /proxy-agent args: @@ -90,7 +90,7 @@ spec: cpu: "300m" - name: addon-agent image: {{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }} - imagePullPolicy: IfNotPresent + imagePullPolicy: Always command: - /agent args: diff --git a/pkg/proxyserver/controllers/managedproxyconfiguration_controller.go b/pkg/proxyserver/controllers/managedproxyconfiguration_controller.go index 1d605f57..a57fed3f 100644 --- a/pkg/proxyserver/controllers/managedproxyconfiguration_controller.go +++ b/pkg/proxyserver/controllers/managedproxyconfiguration_controller.go @@ -18,6 +18,7 @@ import ( "github.com/openshift/library-go/pkg/operator/events" "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -47,12 +48,15 @@ var log = ctrl.Log.WithName("ClusterManagementAddonReconciler") func RegisterClusterManagementAddonReconciler( mgr manager.Manager, selfSigner selfsigned.SelfSigner, + hostClient client.Client, nativeClient kubernetes.Interface, secretInformer informercorev1.SecretInformer, supportsV1CSR bool, + mcMode bool, ) error { r := &ManagedProxyConfigurationReconciler{ Client: mgr.GetClient(), + HostClient: hostClient, SelfSigner: selfSigner, CAPair: selfSigner.CA(), newCertRotatorFunc: func(namespace, name string, sans ...string) selfsigned.CertRotation { @@ -72,12 +76,14 @@ func RegisterClusterManagementAddonReconciler( EventRecorder: events.NewInMemoryRecorder("ClusterManagementAddonReconciler"), supportsV1CSR: supportsV1CSR, + mcMode: mcMode, } return r.SetupWithManager(mgr) } type ManagedProxyConfigurationReconciler struct { - client.Client + Client client.Client + HostClient client.Client SelfSigner selfsigned.SelfSigner CAPair *crypto.CA SecretLister corev1listers.SecretLister @@ -88,6 +94,7 @@ type ManagedProxyConfigurationReconciler struct { newCertRotatorFunc func(namespace, name string, sans ...string) selfsigned.CertRotation supportsV1CSR bool + mcMode bool } func (c *ManagedProxyConfigurationReconciler) SetupWithManager(mgr ctrl.Manager) error { @@ -171,12 +178,40 @@ func (c *ManagedProxyConfigurationReconciler) deployProxyServer(config *proxyv1a newProxyServerRole(config), newProxyServerRoleBinding(config), } + if c.mcMode { + var manager appsv1.Deployment + key := client.ObjectKey{ + Namespace: config.Spec.ProxyServer.Namespace, + Name: "cluster-proxy-addon-manager", + } + if err := c.HostClient.Get(context.Background(), key, &manager); err == nil { + ownerRef := metav1.NewControllerRef(&manager, schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }) + for i, resource := range resources { + resource.SetOwnerReferences([]metav1.OwnerReference{ + *ownerRef, + }) + resources[i] = resource + } + } + } else { + for i, resource := range resources { + resource.SetOwnerReferences([]metav1.OwnerReference{ + newOwnerReference(config), + }) + resources[i] = resource + } + } + anyCreated := false createdKinds := sets.NewString() anyUpdated := false updatedKinds := sets.NewString() for _, resource := range resources { - gvks, _, err := c.Scheme().ObjectKinds(resource) + gvks, _, err := c.HostClient.Scheme().ObjectKinds(resource) if err != nil { return false, err } @@ -224,7 +259,7 @@ func (c *ManagedProxyConfigurationReconciler) ensure(incomingGeneration int64, g // create if it doesn't exist current := &unstructured.Unstructured{} current.SetGroupVersionKind(gvk) - if err := c.Client.Get( + if err := c.HostClient.Get( context.TODO(), types.NamespacedName{ Namespace: resource.GetNamespace(), @@ -239,7 +274,7 @@ func (c *ManagedProxyConfigurationReconciler) ensure(incomingGeneration int64, g ) } // if not found, then create - if err := c.Client.Create(context.TODO(), resource); err != nil { + if err := c.HostClient.Create(context.TODO(), resource); err != nil { if !apierrors.IsAlreadyExists(err) { return false, false, errors.Wrapf(err, "failed to create resource kind: %s, namespace: %s, name %s", @@ -270,7 +305,7 @@ func (c *ManagedProxyConfigurationReconciler) ensure(incomingGeneration int64, g // update if generation bumped if !created && int(incomingGeneration) > currentGeneration { resource.SetResourceVersion(current.GetResourceVersion()) - if err := c.Client.Update(context.TODO(), resource); err != nil { + if err := c.HostClient.Update(context.TODO(), resource); err != nil { if apierrors.IsConflict(err) { return c.ensure(incomingGeneration, gvk, resource) } @@ -353,7 +388,7 @@ func (c *ManagedProxyConfigurationReconciler) ensureEntrypoint(config *proxyv1al }, }, } - if err := c.Client.Create(context.TODO(), proxyService); err != nil { + if err := c.HostClient.Create(context.TODO(), proxyService); err != nil { if !apierrors.IsAlreadyExists(err) { return "", errors.Wrapf(err, "failed to ensure entrypoint service for proxy-server") } @@ -451,11 +486,11 @@ func (c *ManagedProxyConfigurationReconciler) ensureBasicResources(config *proxy } func (c *ManagedProxyConfigurationReconciler) ensureNamespace(config *proxyv1alpha1.ManagedProxyConfiguration) error { - if err := c.Client.Get(context.TODO(), types.NamespacedName{ + if err := c.HostClient.Get(context.TODO(), types.NamespacedName{ Name: config.Spec.ProxyServer.Namespace, }, &corev1.Namespace{}); err != nil { if apierrors.IsNotFound(err) { - if err := c.Client.Create(context.TODO(), &corev1.Namespace{ + if err := c.HostClient.Create(context.TODO(), &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: config.Spec.ProxyServer.Namespace, }, diff --git a/pkg/proxyserver/controllers/manifests.go b/pkg/proxyserver/controllers/manifests.go index 39fb5db9..9f0ad325 100644 --- a/pkg/proxyserver/controllers/manifests.go +++ b/pkg/proxyserver/controllers/manifests.go @@ -29,9 +29,9 @@ func newServiceAccount(config *proxyv1alpha1.ManagedProxyConfiguration) *corev1. ObjectMeta: metav1.ObjectMeta{ Namespace: config.Spec.ProxyServer.Namespace, Name: common.AddonName, - OwnerReferences: []metav1.OwnerReference{ - newOwnerReference(config), - }, + //OwnerReferences: []metav1.OwnerReference{ + // newOwnerReference(config), + //}, }, } } @@ -41,9 +41,9 @@ func newProxySecret(config *proxyv1alpha1.ManagedProxyConfiguration, caData []by ObjectMeta: metav1.ObjectMeta{ Namespace: config.Spec.ProxyServer.Namespace, Name: signerSecretName, - OwnerReferences: []metav1.OwnerReference{ - newOwnerReference(config), - }, + //OwnerReferences: []metav1.OwnerReference{ + // newOwnerReference(config), + //}, }, Data: map[string][]byte{ "ca.crt": caData, @@ -55,9 +55,9 @@ func newProxyService(config *proxyv1alpha1.ManagedProxyConfiguration) *corev1.Se ObjectMeta: metav1.ObjectMeta{ Namespace: config.Spec.ProxyServer.Namespace, Name: config.Spec.ProxyServer.InClusterServiceName, - OwnerReferences: []metav1.OwnerReference{ - newOwnerReference(config), - }, + //OwnerReferences: []metav1.OwnerReference{ + // newOwnerReference(config), + //}, }, Spec: corev1.ServiceSpec{ Selector: map[string]string{ @@ -83,9 +83,9 @@ func newProxyServerDeployment(config *proxyv1alpha1.ManagedProxyConfiguration) * ObjectMeta: metav1.ObjectMeta{ Namespace: config.Spec.ProxyServer.Namespace, Name: config.Name, - OwnerReferences: []metav1.OwnerReference{ - newOwnerReference(config), - }, + //OwnerReferences: []metav1.OwnerReference{ + // newOwnerReference(config), + //}, Labels: map[string]string{ common.AnnotationKeyConfigurationGeneration: strconv.Itoa(int(config.Generation)), }, @@ -193,9 +193,9 @@ func newProxyServerRole(config *proxyv1alpha1.ManagedProxyConfiguration) *rbacv1 ObjectMeta: metav1.ObjectMeta{ Namespace: config.Spec.ProxyServer.Namespace, Name: "cluster-proxy-addon-agent:portforward", - OwnerReferences: []metav1.OwnerReference{ - newOwnerReference(config), - }, + //OwnerReferences: []metav1.OwnerReference{ + // newOwnerReference(config), + //}, }, Rules: []rbacv1.PolicyRule{ {