Merge pull request kosmos-io#537 from qiuwei68/feature_hostports

feat: add admin-config-clusterip secret use cluster ip and port

pkg/kubenest/controlplane/apiserver.go

@@ -34,7 +34,7 @@ func DeleteVirtualClusterAPIServer(client clientset.Interface, name, namespace s
 	}
 	err := manager.ReleaseHostPort(name)
 	if err != nil {
-		klog.Errorf("Error releasing host port for cluster %s: %v", name, err)
+		klog.Warningf("Error releasing host port for cluster %s: %v", name, err)
 	}
 	return nil
 }

pkg/kubenest/controlplane/component.go

@@ -123,7 +123,7 @@ func getKubeControllerManagerManifest(name, namespace string) (*appsv1.Deploymen
 		ImageRepository:           imageRepository,
 		Version:                   imageVersion,
 		VirtualClusterCertsSecret: fmt.Sprintf("%s-%s", name, "cert"),
-		KubeconfigSecret:          fmt.Sprintf("%s-%s", name, "admin-config"),
+		KubeconfigSecret:          fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
 		ServiceSubnet:             constants.ApiServerServiceSubnet,
 		Replicas:                  constants.KubeControllerReplicas,
 	})
@@ -170,7 +170,7 @@ func getVirtualClusterSchedulerManifest(name, namespace string) (*appsv1.Deploym
 		SystemNamespace:  constants.SystemNs,
 		ImageRepository:  imageRepository,
 		Version:          imageVersion,
-		KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config"),
+		KubeconfigSecret: fmt.Sprintf("%s-%s", name, "admin-config-clusterip"),
 		Replicas:         constants.VirtualClusterSchedulerReplicas,
 	})
 	if err != nil {

pkg/kubenest/controlplane/service.go

@@ -50,7 +50,7 @@ func DeleteVirtualClusterService(client clientset.Interface, name, namespace str
 
 	err := manager.ReleaseHostPort(name)
 	if err != nil {
-		klog.Errorf("Error releasing host port for cluster %s: %v", name, err)
+		klog.Warningf("Error releasing host port for cluster %s: %v", name, err)
 	}
 
 	klog.V(2).Infof("Successfully uninstalled service for virtualcluster %s", name)

pkg/kubenest/tasks/upload.go

@@ -24,6 +24,11 @@ var (
 	VirtualClusterControllerLabel = labels.Set{constants.VirtualClusterLabelKeyName: constants.VirtualClusterController}
 )
 
+type PortInfo struct {
+	NodePort      int32
+	ClusterIPPort int32
+}
+
 func NewUploadCertsTask() workflow.Task {
 	return workflow.Task{
 		Name:        "Upload-Certs",
@@ -165,19 +170,32 @@ func runUploadAdminKubeconfig(r workflow.RunData) error {
 		return errors.New("UploadAdminKubeconfig task invoked with an invalid data struct")
 	}
 
-	var endpoint string
+	var controlplaneIpEndpoint, clusterIPEndpoint string
 	service, err := data.RemoteClient().CoreV1().Services(data.GetNamespace()).Get(context.TODO(), fmt.Sprintf("%s-%s", data.GetName(), "apiserver"), metav1.GetOptions{})
 	if err != nil {
 		return err
 	}
-	nodePort := getNodePortFromAPIServerService(service)
-	endpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), nodePort)
-	kubeconfig, err := buildKubeConfigFromSpec(data, endpoint)
+	portInfo := getPortInfoFromAPIServerService(service)
+	// controlplane address + nodePort
+	controlplaneIpEndpoint = fmt.Sprintf("https://%s:%d", data.ControlplaneAddress(), portInfo.NodePort)
+	controlplaneIpKubeconfig, err := buildKubeConfigFromSpec(data, controlplaneIpEndpoint)
+	if err != nil {
+		return err
+	}
+
+	//clusterIP address + clusterIPPort
+	clusterIPEndpoint = fmt.Sprintf("https://%s:%d", service.Spec.ClusterIP, portInfo.ClusterIPPort)
+	clusterIPKubeconfig, err := buildKubeConfigFromSpec(data, clusterIPEndpoint)
+	if err != nil {
+		return err
+	}
+
+	controlplaneIpConfigBytes, err := clientcmd.Write(*controlplaneIpKubeconfig)
 	if err != nil {
 		return err
 	}
 
-	configBytes, err := clientcmd.Write(*kubeconfig)
+	clusterIPConfigBytes, err := clientcmd.Write(*clusterIPKubeconfig)
 	if err != nil {
 		return err
 	}
@@ -188,28 +206,41 @@ func runUploadAdminKubeconfig(r workflow.RunData) error {
 			Name:      fmt.Sprintf("%s-%s", data.GetName(), "admin-config"),
 			Labels:    VirtualClusterControllerLabel,
 		},
-		Data: map[string][]byte{"kubeconfig": configBytes},
+		Data: map[string][]byte{"kubeconfig": controlplaneIpConfigBytes},
 	})
 	if err != nil {
 		return fmt.Errorf("failed to create secret of kubeconfig, err: %w", err)
 	}
 
-	klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secret of virtual cluster apiserver kubeconfig", "virtual cluster", klog.KObj(data))
+	err = createOrUpdateSecret(data.RemoteClient(), &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: data.GetNamespace(),
+			Name:      fmt.Sprintf("%s-%s", data.GetName(), "admin-config-clusterip"),
+			Labels:    VirtualClusterControllerLabel,
+		},
+		Data: map[string][]byte{"kubeconfig": clusterIPConfigBytes},
+	})
+	if err != nil {
+		return fmt.Errorf("failed to create secret of kubeconfig-clusterip, err: %w", err)
+	}
+
+	klog.V(2).InfoS("[UploadAdminKubeconfig] Successfully created secrets of virtual cluster apiserver kubeconfig", "virtual cluster", klog.KObj(data))
 	return nil
 }
 
-func getNodePortFromAPIServerService(service *corev1.Service) int32 {
-	var nodePort int32
+func getPortInfoFromAPIServerService(service *corev1.Service) PortInfo {
+	var portInfo PortInfo
 	if service.Spec.Type == corev1.ServiceTypeNodePort {
 		for _, port := range service.Spec.Ports {
 			if port.Name != constants.APIServerSVCPortName {
 				continue
 			}
-			nodePort = port.NodePort
+			portInfo.NodePort = port.NodePort
+			portInfo.ClusterIPPort = port.Port
 		}
 	}
 
-	return nodePort
+	return portInfo
 }
 
 func buildKubeConfigFromSpec(data InitData, serverURL string) (*clientcmdapi.Config, error) {
@@ -272,6 +303,7 @@ func deleteSecrets(r workflow.RunData) error {
 		fmt.Sprintf("%s-%s", data.GetName(), "cert"),
 		fmt.Sprintf("%s-%s", data.GetName(), "etcd-cert"),
 		fmt.Sprintf("%s-%s", data.GetName(), "admin-config"),
+		fmt.Sprintf("%s-%s", data.GetName(), "admin-config-clusterip"),
 	}
 	for _, secret := range secrets {
 		err := data.RemoteClient().CoreV1().Secrets(data.GetNamespace()).Delete(context.TODO(), secret, metav1.DeleteOptions{})