Skip to content

Commit

Permalink
fix: forward compatibility creates svc and endpoint
Browse files Browse the repository at this point in the history 
Signed-off-by: qiuwei <[email protected]>
  • Loading branch information
@qiuwei68
qiuwei68 committed Jan 16, 2025
1 parent 1e990ec commit 49ce69c
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 34 deletions.
67 changes: 52 additions & 15 deletions pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,7 @@ func (e *APIServerExternalSyncController) SyncAPIServerExternalEndpoints(ctx con

newEndpoint := &v1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: constants.APIServerExternalService,
Namespace: constants.KosmosNs,
Name: constants.APIServerExternalService,
},
Subsets: []v1.EndpointSubset{
{
Expand All @@ -132,29 +131,67 @@ func (e *APIServerExternalSyncController) SyncAPIServerExternalEndpoints(ctx con
},
}

//avoid unnecessary updates
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
currentEndpoint, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{})
if apierrors.IsNotFound(err) {
_, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Create(ctx, newEndpoint, metav1.CreateOptions{})
// First check if kosmos-system namespace exists
_, err := k8sClient.CoreV1().Namespaces().Get(ctx, constants.KosmosNs, metav1.GetOptions{})
if err == nil {
// kosmos-system exists, try to get or create endpoint
currentEndpoint, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("failed to create api-server-external-service endpoint: %w", err)
if apierrors.IsNotFound(err) {
// endpoint doesn't exist, create it in kosmos-system
newEndpoint.ObjectMeta.Namespace = constants.KosmosNs
_, err = k8sClient.CoreV1().Endpoints(constants.KosmosNs).Create(ctx, newEndpoint, metav1.CreateOptions{})
if err != nil {
return fmt.Errorf("failed to create endpoint in kosmos-system: %w", err)
}
klog.V(4).Info("Created api-server-external-service Endpoint in kosmos-system")
return nil
}
return fmt.Errorf("failed to get endpoint in kosmos-system: %w", err)
}

// endpoint exists, check if update is needed
if !reflect.DeepEqual(currentEndpoint.Subsets, newEndpoint.Subsets) {
newEndpoint.ObjectMeta.Namespace = constants.KosmosNs
newEndpoint.ObjectMeta.ResourceVersion = currentEndpoint.ResourceVersion
_, err = k8sClient.CoreV1().Endpoints(constants.KosmosNs).Update(ctx, newEndpoint, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("failed to update endpoint in kosmos-system: %w", err)
}
klog.V(4).Info("Updated api-server-external-service Endpoint in kosmos-system")
} else {
klog.V(4).Info("No changes detected in kosmos-system Endpoint, skipping update")
}
klog.V(4).Info("Created api-server-external-service Endpoint")
return nil
} else if err != nil {
return fmt.Errorf("failed to get existing api-server-external-service endpoint: %w", err)
}

// determine if an update is needed
// For backward compatibility: if kosmos-system doesn't exist, check if endpoint exists in default namespace
if !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to get namespace kosmos-system: %w", err)
}

currentEndpoint, err := k8sClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{})
if err != nil {
if apierrors.IsNotFound(err) {
// endpoint doesn't exist in default namespace, which means it was never created, skip it
klog.V(4).Info("No endpoint found in default namespace, skipping")
return nil
}
return fmt.Errorf("failed to get endpoint in default: %w", err)
}

// For backward compatibility: if endpoint exists in default namespace, check if update is needed
if !reflect.DeepEqual(currentEndpoint.Subsets, newEndpoint.Subsets) {
_, err := k8sClient.CoreV1().Endpoints(constants.KosmosNs).Update(ctx, newEndpoint, metav1.UpdateOptions{})
newEndpoint.ObjectMeta.Namespace = constants.DefaultNs
newEndpoint.ObjectMeta.ResourceVersion = currentEndpoint.ResourceVersion
_, err = k8sClient.CoreV1().Endpoints(constants.DefaultNs).Update(ctx, newEndpoint, metav1.UpdateOptions{})
if err != nil {
return fmt.Errorf("failed to update api-server-external-service endpoint: %w", err)
return fmt.Errorf("failed to update endpoint in default: %w", err)
}
klog.V(4).Info("Updated api-server-external-service Endpoint")
klog.V(4).Info("Updated api-server-external-service Endpoint in default")
} else {
klog.V(4).Info("No changes detected in Endpoint, skipping update")
klog.V(4).Info("No changes detected in default Endpoint, skipping update")
}
return nil
})
Expand Down
86 changes: 74 additions & 12 deletions pkg/kubenest/controller/endpoints.sync.controller/apiserver_external_sync_controller_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) {
},
}

kosmosNsObj := &corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KosmosNs,
},
}

tests := []struct {
name string
objects []runtime.Object
Expand All @@ -78,12 +84,62 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) {
wantErr bool
wantErrString string
wantSubsets []corev1.EndpointSubset
setupKosmosNs bool
}{
{
name: "Successfully syncs external endpoints",
objects: []runtime.Object{},
mockNodes: nodes,
wantSubsets: endpoint.Subsets,
name: "Successfully syncs external endpoints in kosmos-system",
objects: []runtime.Object{kosmosNsObj},
mockNodes: nodes,
wantSubsets: endpoint.Subsets,
setupKosmosNs: true,
},
{
name: "Successfully syncs external endpoints in default when kosmos-system not exists",
objects: []runtime.Object{},
mockNodes: nodes,
wantSubsets: endpoint.Subsets,
setupKosmosNs: false,
},
{
name: "Updates existing endpoint in kosmos-system",
objects: []runtime.Object{
kosmosNsObj,
&corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: constants.APIServerExternalService,
Namespace: constants.KosmosNs,
},
Subsets: []corev1.EndpointSubset{
{
Addresses: []corev1.EndpointAddress{{IP: "192.168.1.2"}},
Ports: []corev1.EndpointPort{{Name: "https", Port: 6443, Protocol: corev1.ProtocolTCP}},
},
},
},
},
mockNodes: nodes,
wantSubsets: endpoint.Subsets,
setupKosmosNs: true,
},
{
name: "Updates existing endpoint in default namespace",
objects: []runtime.Object{
&corev1.Endpoints{
ObjectMeta: metav1.ObjectMeta{
Name: constants.APIServerExternalService,
Namespace: constants.DefaultNs,
},
Subsets: []corev1.EndpointSubset{
{
Addresses: []corev1.EndpointAddress{{IP: "192.168.1.2"}},
Ports: []corev1.EndpointPort{{Name: "https", Port: 6443, Protocol: corev1.ProtocolTCP}},
},
},
},
},
mockNodes: nodes,
wantSubsets: endpoint.Subsets,
setupKosmosNs: false,
},
{
name: "Does not update endpoint if no changes",
Expand Down Expand Up @@ -130,18 +186,15 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Use fake clientset to simulate the Kubernetes API client (host cluster)
fakeHostClusterClient := fake.NewSimpleClientset(tt.objects...)
// Simulate the Virtual Cluster client by passing the same clientset
fakeVCClient := fake.NewSimpleClientset()
// Mock NodeGetter to return the mock nodes for Host cluster
mockNodeGetter := &MockNodeGetter{Nodes: tt.mockNodes, Err: tt.mockErr}
// Use fake clientset to simulate the Kubernetes API client (host cluster)

controller := &APIServerExternalSyncController{
KubeClient: fakeHostClusterClient,
NodeGetter: mockNodeGetter,
}
// Test the controller method using the VC's client

err := controller.SyncAPIServerExternalEndpoints(ctx, fakeVCClient, vc)
if tt.wantErr {
assert.Error(t, err)
Expand All @@ -151,9 +204,18 @@ func TestSyncAPIServerExternalEndpoints(t *testing.T) {
} else {
assert.NoError(t, err)
if tt.wantSubsets != nil {
createdEndpoint, err := fakeVCClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{})
assert.NoError(t, err)
assert.True(t, reflect.DeepEqual(createdEndpoint.Subsets, tt.wantSubsets))
var createdEndpoint *corev1.Endpoints
var err error

if tt.setupKosmosNs {
createdEndpoint, err = fakeVCClient.CoreV1().Endpoints(constants.KosmosNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{})
} else {
createdEndpoint, err = fakeVCClient.CoreV1().Endpoints(constants.DefaultNs).Get(ctx, constants.APIServerExternalService, metav1.GetOptions{})
}

if err == nil {
assert.True(t, reflect.DeepEqual(createdEndpoint.Subsets, tt.wantSubsets))
}
}
}
})
Expand Down
5 changes: 0 additions & 5 deletions pkg/kubenest/controlplane/endpoint.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,6 @@ import (
"github.com/kosmos.io/kosmos/pkg/utils"
)

type IPFamilies struct {
IPv4 bool
IPv6 bool
}

func EnsureAPIServerExternalEndPoint(kubeClient kubernetes.Interface, apiServerExternalResource common.APIServerExternalResource) error {
err := EnsureKosmosSystemNamespace(kubeClient)
if err != nil {
Expand Down
4 changes: 2 additions & 2 deletions pkg/kubenest/controlplane/endpoint_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ func TestGetEndPointInfo(t *testing.T) {
port, ipFamilies, err := getEndPointInfo(client)
assert.NoError(t, err)
assert.Equal(t, int32(6443), port)
assert.True(t, ipFamilies.IPv4)
assert.False(t, ipFamilies.IPv6)
assert.Contains(t, ipFamilies, corev1.IPv4Protocol)
assert.NotContains(t, ipFamilies, corev1.IPv6Protocol)
})

t.Run("No subsets in endpoint", func(t *testing.T) {
Expand Down

0 comments on commit 49ce69c

Please sign in to comment.