From 49ce69ce68999e847cc08d088efd817b8a490363 Mon Sep 17 00:00:00 2001 From: qiuwei Date: Wed, 15 Jan 2025 17:47:46 +0800 Subject: [PATCH] fix: forward compatibility creates svc and endpoint Signed-off-by: qiuwei --- .../apiserver_external_sync_controller.go | 67 +++++++++++---- ...apiserver_external_sync_controller_test.go | 86 ++++++++++++++++--- pkg/kubenest/controlplane/endpoint.go | 5 -- pkg/kubenest/controlplane/endpoint_test.go | 4 +- 4 files changed, 128 insertions(+), 34 deletions(-) diff --git a/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go b/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go index 2f180dcfa..4b31ad8e2 100644 --- a/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go +++ b/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go @@ -115,8 +115,7 @@ func (e *APIServerExternalSyncController) SyncAPIServerExternalEndpoints(ctx con newEndpoint := &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ - Name: constants.APIServerExternalService, - Namespace: constants.KosmosNs, + Name: constants.APIServerExternalService, }, Subsets: []v1.EndpointSubset{ { @@ -132,29 +131,67 @@ func (e *APIServerExternalSyncController) SyncAPIServerExternalEndpoints(ctx con }, } - //avoid unnecessary updates return retry.RetryOnConflict(retry.DefaultRetry, func() error { - currentEndpoint, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - _, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Create(ctx, newEndpoint, metav1.CreateOptions{}) + // First check if kosmos-system namespace exists + _, err := k8sClient.CoreV1().Namespaces().Get(ctx, constants.KosmosNs, metav1.GetOptions{}) + if err == nil { + // kosmos-system exists, try to get or create endpoint + currentEndpoint, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{}) if err != nil { - return fmt.Errorf("failed to create api-server-external-service endpoint: %w", err) + if apierrors.IsNotFound(err) { + // endpoint doesn't exist, create it in kosmos-system + newEndpoint.ObjectMeta.Namespace = constants.KosmosNs + _, err = k8sClient.CoreV1().Endpoints(constants.KosmosNs).Create(ctx, newEndpoint, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create endpoint in kosmos-system: %w", err) + } + klog.V(4).Info("Created api-server-external-service Endpoint in kosmos-system") + return nil + } + return fmt.Errorf("failed to get endpoint in kosmos-system: %w", err) + } + + // endpoint exists, check if update is needed + if !reflect.DeepEqual(currentEndpoint.Subsets, newEndpoint.Subsets) { + newEndpoint.ObjectMeta.Namespace = constants.KosmosNs + newEndpoint.ObjectMeta.ResourceVersion = currentEndpoint.ResourceVersion + _, err = k8sClient.CoreV1().Endpoints(constants.KosmosNs).Update(ctx, newEndpoint, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update endpoint in kosmos-system: %w", err) + } + klog.V(4).Info("Updated api-server-external-service Endpoint in kosmos-system") + } else { + klog.V(4).Info("No changes detected in kosmos-system Endpoint, skipping update") } - klog.V(4).Info("Created api-server-external-service Endpoint") return nil - } else if err != nil { - return fmt.Errorf("failed to get existing api-server-external-service endpoint: %w", err) } - // determine if an update is needed + // For backward compatibility: if kosmos-system doesn't exist, check if endpoint exists in default namespace + if !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to get namespace kosmos-system: %w", err) + } + + currentEndpoint, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // endpoint doesn't exist in default namespace, which means it was never created, skip it + klog.V(4).Info("No endpoint found in default namespace, skipping") + return nil + } + return fmt.Errorf("failed to get endpoint in default: %w", err) + } + + // For backward compatibility: if endpoint exists in default namespace, check if update is needed if !reflect.DeepEqual(currentEndpoint.Subsets, newEndpoint.Subsets) { - _, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Update(ctx, newEndpoint, metav1.UpdateOptions{}) + newEndpoint.ObjectMeta.Namespace = constants.DefaultNs + newEndpoint.ObjectMeta.ResourceVersion = currentEndpoint.ResourceVersion + _, err = k8sClient.CoreV1().Endpoints(constants.DefaultNs).Update(ctx, newEndpoint, metav1.UpdateOptions{}) if err != nil { - return fmt.Errorf("failed to update api-server-external-service endpoint: %w", err) + return fmt.Errorf("failed to update endpoint in default: %w", err) } - klog.V(4).Info("Updated api-server-external-service Endpoint") + klog.V(4).Info("Updated api-server-external-service Endpoint in default") } else { - klog.V(4).Info("No changes detected in Endpoint, skipping update") + klog.V(4).Info("No changes detected in default Endpoint, skipping update") } return nil }) diff --git a/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller_test.go b/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller_test.go index b1c4cfbd2..64f13b01c 100644 --- a/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller_test.go +++ b/pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller_test.go @@ -70,6 +70,12 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) { }, } + kosmosNsObj := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KosmosNs, + }, + } + tests := []struct { name string objects []runtime.Object @@ -78,12 +84,62 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) { wantErr bool wantErrString string wantSubsets []corev1.EndpointSubset + setupKosmosNs bool }{ { - name: "Successfully syncs external endpoints", - objects: []runtime.Object{}, - mockNodes: nodes, - wantSubsets: endpoint.Subsets, + name: "Successfully syncs external endpoints in kosmos-system", + objects: []runtime.Object{kosmosNsObj}, + mockNodes: nodes, + wantSubsets: endpoint.Subsets, + setupKosmosNs: true, + }, + { + name: "Successfully syncs external endpoints in default when kosmos-system not exists", + objects: []runtime.Object{}, + mockNodes: nodes, + wantSubsets: endpoint.Subsets, + setupKosmosNs: false, + }, + { + name: "Updates existing endpoint in kosmos-system", + objects: []runtime.Object{ + kosmosNsObj, + &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.APIServerExternalService, + Namespace: constants.KosmosNs, + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{{IP: "192.168.1.2"}}, + Ports: []corev1.EndpointPort{{Name: "https", Port: 6443, Protocol: corev1.ProtocolTCP}}, + }, + }, + }, + }, + mockNodes: nodes, + wantSubsets: endpoint.Subsets, + setupKosmosNs: true, + }, + { + name: "Updates existing endpoint in default namespace", + objects: []runtime.Object{ + &corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.APIServerExternalService, + Namespace: constants.DefaultNs, + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{{IP: "192.168.1.2"}}, + Ports: []corev1.EndpointPort{{Name: "https", Port: 6443, Protocol: corev1.ProtocolTCP}}, + }, + }, + }, + }, + mockNodes: nodes, + wantSubsets: endpoint.Subsets, + setupKosmosNs: false, }, { name: "Does not update endpoint if no changes", @@ -130,18 +186,15 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Use fake clientset to simulate the Kubernetes API client (host cluster) fakeHostClusterClient := fake.NewSimpleClientset(tt.objects...) - // Simulate the Virtual Cluster client by passing the same clientset fakeVCClient := fake.NewSimpleClientset() - // Mock NodeGetter to return the mock nodes for Host cluster mockNodeGetter := &MockNodeGetter{Nodes: tt.mockNodes, Err: tt.mockErr} - // Use fake clientset to simulate the Kubernetes API client (host cluster) + controller := &APIServerExternalSyncController{ KubeClient: fakeHostClusterClient, NodeGetter: mockNodeGetter, } - // Test the controller method using the VC's client + err := controller.SyncAPIServerExternalEndpoints(ctx, fakeVCClient, vc) if tt.wantErr { assert.Error(t, err) @@ -151,9 +204,18 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) { } else { assert.NoError(t, err) if tt.wantSubsets != nil { - createdEndpoint, err := fakeVCClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{}) - assert.NoError(t, err) - assert.True(t, reflect.DeepEqual(createdEndpoint.Subsets, tt.wantSubsets)) + var createdEndpoint *corev1.Endpoints + var err error + + if tt.setupKosmosNs { + createdEndpoint, err = fakeVCClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{}) + } else { + createdEndpoint, err = fakeVCClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{}) + } + + if err == nil { + assert.True(t, reflect.DeepEqual(createdEndpoint.Subsets, tt.wantSubsets)) + } } } }) diff --git a/pkg/kubenest/controlplane/endpoint.go b/pkg/kubenest/controlplane/endpoint.go index 17216a514..8f6758555 100644 --- a/pkg/kubenest/controlplane/endpoint.go +++ b/pkg/kubenest/controlplane/endpoint.go @@ -18,11 +18,6 @@ import ( "github.com/kosmos.io/kosmos/pkg/utils" ) -type IPFamilies struct { - IPv4 bool - IPv6 bool -} - func EnsureAPIServerExternalEndPoint(kubeClient kubernetes.Interface, apiServerExternalResource common.APIServerExternalResource) error { err := EnsureKosmosSystemNamespace(kubeClient) if err != nil { diff --git a/pkg/kubenest/controlplane/endpoint_test.go b/pkg/kubenest/controlplane/endpoint_test.go index e24719adc..36c9f9aa2 100644 --- a/pkg/kubenest/controlplane/endpoint_test.go +++ b/pkg/kubenest/controlplane/endpoint_test.go @@ -115,8 +115,8 @@ func TestGetEndPointInfo(t *testing.T) { port, ipFamilies, err := getEndPointInfo(client) assert.NoError(t, err) assert.Equal(t, int32(6443), port) - assert.True(t, ipFamilies.IPv4) - assert.False(t, ipFamilies.IPv6) + assert.Contains(t, ipFamilies, corev1.IPv4Protocol) + assert.NotContains(t, ipFamilies, corev1.IPv6Protocol) }) t.Run("No subsets in endpoint", func(t *testing.T) {