Merge pull request kosmos-io#503 from zhouhaoA1/feature_components_de…

…ploy

fix virtualcluster deploy yaml and virtualcluster controller bug

cmd/kubenest/operator/app/operator.go

@@ -98,6 +98,7 @@ func run(ctx context.Context, opts *options.Options) error {
 		EventRecorder:   mgr.GetEventRecorderFor(constants.InitControllerName),
 		HostPortManager: hostPortManager,
 		RootClientSet:   hostKubeClient,
+		KosmosClient:    kosmosClient,
 	}
 	if err = VirtualClusterInitController.SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("error starting %s: %v", constants.InitControllerName, err)

deploy/virtual-cluster-operator.yml

@@ -276,6 +276,28 @@ spec:
       # Enter the name of the node where the virtual cluster operator is deployed
       nodeName: nodeName
       serviceAccountName: virtual-cluster-operator
+      tolerations:
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchExpressions:
+                    - key: app
+                      operator: In
+                      values:
+                        - virtual-cluster-operator
+                topologyKey: kubernetes.io/hostname
       containers:
       - name: virtual-cluster-operator
         # Change a valid image address

pkg/kubenest/constants/constant.go

@@ -98,6 +98,7 @@ const (
 	// nolint
 	HostCoreDnsComponents    = "host-core-dns-components"
 	VirtualCoreDnsComponents = "virtual-core-dns-components"
+	PrometheusRuleManifest   = "prometheus-rules"
 )
 
 type Action string

pkg/kubenest/controller/virtualcluster_init_controller.go

@@ -27,6 +27,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
 	"github.com/kosmos.io/kosmos/pkg/apis/kosmos/v1alpha1"
+	"github.com/kosmos.io/kosmos/pkg/generated/clientset/versioned"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/constants"
 	vcnodecontroller "github.com/kosmos.io/kosmos/pkg/kubenest/controller/virtualcluster.node.controller"
 	"github.com/kosmos.io/kosmos/pkg/kubenest/util"
@@ -38,6 +39,7 @@ type VirtualClusterInitController struct {
 	EventRecorder   record.EventRecorder
 	HostPortManager *vcnodecontroller.HostPortManager
 	RootClientSet   kubernetes.Interface
+	KosmosClient    versioned.Interface
 	lock            sync.Mutex
 }
 
@@ -104,7 +106,15 @@ func (c *VirtualClusterInitController) Reconcile(ctx context.Context, request re
 		if err != nil {
 			return reconcile.Result{RequeueAfter: RequeueTime}, errors.Wrapf(err, "Error update virtualcluster %s status", updatedCluster.Name)
 		}
-
+		//get latest virtualcluster
+		if err = c.Get(ctx, request.NamespacedName, originalCluster); err != nil {
+			if apierrors.IsNotFound(err) {
+				klog.Warningf("Virtualcluster %s is not found, previous status %s. This should not happen normally", updatedCluster.Name, updatedCluster.Status.Phase)
+				return reconcile.Result{}, nil
+			}
+			return reconcile.Result{RequeueAfter: RequeueTime}, nil
+		}
+		updatedCluster := originalCluster.DeepCopy()
 		err = c.createVirtualCluster(updatedCluster)
 		if err != nil {
 			klog.Errorf("Failed to create virtualcluster %s. err: %s", updatedCluster.Name, err.Error())
@@ -322,11 +332,9 @@ func (c *VirtualClusterInitController) assignNodesByPolicy(virtualCluster *v1alp
 		for _, index := range newAssignNodes {
 			updated := globalNodes[index].DeepCopy()
 			updated.Spec.State = v1alpha1.NodeInUse
-			updated.Status = v1alpha1.GlobalNodeStatus{
-				VirtualCluster: virtualCluster.Name,
-			}
+			updated.Status.VirtualCluster = virtualCluster.Name
 			klog.V(2).Infof("Assign node %s for virtualcluster %s policy %s", updated.Name, virtualCluster.GetName(), policy.LabelSelector.String())
-			if err := c.Client.Patch(context.TODO(), updated, client.MergeFrom(&globalNodes[index])); err != nil {
+			if _, err := c.KosmosClient.KosmosV1alpha1().GlobalNodes().Update(context.TODO(), updated, metav1.UpdateOptions{}); err != nil {
 				return nil, errors.Wrapf(err, "Failed to update globalNode %s to InUse", updated.Name)
 			}
 			globalNodes[index] = *updated

pkg/kubenest/tasks/manifests_components.go

@@ -85,7 +85,7 @@ func applyComponentsManifests(r workflow.RunData) error {
 
 	for _, component := range components {
 		klog.V(2).Infof("Deploy component %s", component.Name)
-		err = applyTemplatedManifests(dynamicClient, component.Path, templatedMapping)
+		err = applyTemplatedManifests(component.Name, dynamicClient, component.Path, templatedMapping)
 		if err != nil {
 			return err
 		}
@@ -117,7 +117,7 @@ func getComponentsConfig(client clientset.Interface) ([]ComponentConfig, error)
 	return components, nil
 }
 
-func applyTemplatedManifests(dynamicClient dynamic.Interface, manifestGlob string, templateMapping map[string]interface{}) error {
+func applyTemplatedManifests(component string, dynamicClient dynamic.Interface, manifestGlob string, templateMapping map[string]interface{}) error {
 	manifests, err := filepath.Glob(manifestGlob)
 	klog.V(2).Infof("Component Manifests %s", manifestGlob)
 	if err != nil {
@@ -133,9 +133,13 @@ func applyTemplatedManifests(dynamicClient dynamic.Interface, manifestGlob strin
 		if err != nil {
 			return errors.Wrapf(err, "Read file %s error", manifest)
 		}
-		templateData, err := util.ParseTemplate(string(bytesData), templateMapping)
-		if err != nil {
-			panic(err)
+		templateData := bytesData
+		// template doesn't suit for prometheus rules, we deploy it directly
+		if component != constants.PrometheusRuleManifest {
+			templateData, err = util.ParseTemplate(string(bytesData), templateMapping)
+			if err != nil {
+				return errors.Wrapf(err, "Parse manifest file %s template error", manifest)
+			}
 		}
 		err = yaml.Unmarshal(templateData, &obj)
 		if err != nil {