title: Add ipsec support

Description: Sync code, adapt to ACK Signed-off-by: GreatLazyMan <[email protected]>
kosmos-io · Nov 29, 2023 · 6d4cdd2 · 6d4cdd2
1 parent 417ab54
commit 6d4cdd2
Show file tree

Hide file tree

Showing 22 changed files with 797 additions and 49 deletions.
diff --git a/cmd/clusterlink/elector/app/elector.go b/cmd/clusterlink/elector/app/elector.go
@@ -126,10 +126,11 @@ func run(ctx context.Context, opts *options.Options) error {
 						err := elector.EnsureGateWayRole()
 						if err != nil {
 							klog.Errorf("set gateway role failure: %v, retry after 10 sec.", err)
-							time.Sleep(10 * time.Second)
+							time.Sleep(3 * time.Second)
 						} else {
-							klog.V(4).Info("ensure gateway role success, recheck after 60 sec.")
-							time.Sleep(60 * time.Second)
+							timeToRecheck := 3 * time.Second
+							klog.V(4).Infof("ensure gateway role success, recheck after %d sec.", int(timeToRecheck))
+							time.Sleep(timeToRecheck)
 						}
 					}
 				}

diff --git a/deploy/crds/kosmos.io_clusternodes.yaml b/deploy/crds/kosmos.io_clusternodes.yaml
@@ -45,6 +45,8 @@ spec:
             properties:
               clusterName:
                 type: string
+              elasticip:
+                type: string
               interfaceName:
                 type: string
               ip:
@@ -63,11 +65,13 @@ spec:
                 type: array
             type: object
           status:
+            properties:
+              nodeStatus:
+                type: string
             type: object
         required:
         - spec
         type: object
     served: true
     storage: true
-    subresources:
-      status: {}
+    subresources: {}
diff --git a/deploy/crds/kosmos.io_clusters.yaml b/deploy/crds/kosmos.io_clusters.yaml
@@ -105,6 +105,12 @@ spec:
                       - nodeName
                       type: object
                     type: array
+                  nodeElasticIPMap:
+                    additionalProperties:
+                      type: string
+                    description: NodeElasticIPMap presents mapping between nodename
+                      in kubernetes and elasticIP
+                    type: object
                   useIPPool:
                     default: false
                     type: boolean

diff --git a/deploy/crds/kosmos.io_nodeconfigs.yaml b/deploy/crds/kosmos.io_nodeconfigs.yaml
@@ -122,6 +122,52 @@ spec:
                   - gw
                   type: object
                 type: array
+              xfrmpolicies:
+                items:
+                  properties:
+                    dir:
+                      type: integer
+                    leftip:
+                      type: string
+                    leftnet:
+                      type: string
+                    reqid:
+                      type: integer
+                    rightip:
+                      type: string
+                    rightnet:
+                      type: string
+                  required:
+                  - dir
+                  - leftip
+                  - leftnet
+                  - reqid
+                  - rightip
+                  - rightnet
+                  type: object
+                type: array
+              xfrmstates:
+                items:
+                  properties:
+                    PSK:
+                      type: string
+                    leftip:
+                      type: string
+                    reqid:
+                      type: integer
+                    rightip:
+                      type: string
+                    spi:
+                      format: int32
+                      type: integer
+                  required:
+                  - PSK
+                  - leftip
+                  - reqid
+                  - rightip
+                  - spi
+                  type: object
+                type: array
             type: object
           status:
             properties:

diff --git a/pkg/apis/kosmos/v1alpha1/cluster_types.go b/pkg/apis/kosmos/v1alpha1/cluster_types.go
@@ -91,6 +91,10 @@ type ClusterLinkOptions struct {
 
 	// +optional
 	GlobalCIDRsMap map[string]string `json:"globalCIDRsMap,omitempty"`
+
+	// NodeElasticIPMap presents mapping between nodename in kubernetes and elasticIP
+	// +optional
+	NodeElasticIPMap map[string]string `json:"nodeElasticIPMap,omitempty"`
 }
 
 type ClusterTreeOptions struct {

diff --git a/pkg/apis/kosmos/v1alpha1/clusternode_types.go b/pkg/apis/kosmos/v1alpha1/clusternode_types.go
@@ -8,7 +8,6 @@ import (
 
 // +genclient
 // +genclient:nonNamespaced
-// +kubebuilder:subresource:status
 // +kubebuilder:resource:scope="Cluster"
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +kubebuilder:printcolumn:name="ROLES",type=string,JSONPath=`.spec.roles`
@@ -33,6 +32,8 @@ type ClusterNodeSpec struct {
 	// +optional
 	IP string `json:"ip,omitempty"`
 	// +optional
+	ElasticIP string `json:"elasticip,omitempty"`
+	// +optional
 	IP6 string `json:"ip6,omitempty"`
 	// +optional
 	Roles []Role `json:"roles,omitempty"`
@@ -41,6 +42,8 @@ type ClusterNodeSpec struct {
 }
 
 type ClusterNodeStatus struct {
+	// +optional
+	NodeStatus string `json:"nodeStatus,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

diff --git a/pkg/apis/kosmos/v1alpha1/constants.go b/pkg/apis/kosmos/v1alpha1/constants.go
@@ -28,3 +28,17 @@ type DeviceType string
 const (
 	VxlanDevice DeviceType = "vxlan"
 )
+
+const (
+	DefaultPSK       string = "bfd6224354977084568832b811226b3d6cff6685"
+	DefaultPSKPreStr        = "WelcometoKosmos"
+	DefaultReqID     int    = 336
+)
+
+type IPSECDirection int
+
+const (
+	IPSECIn  IPSECDirection = 0
+	IPSECOut IPSECDirection = 1
+	IPSECFwd IPSECDirection = 2
+)
diff --git a/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go b/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go
@@ -21,11 +21,13 @@ type NodeConfig struct {
 }
 
 type NodeConfigSpec struct {
-	Devices  []Device   `json:"devices,omitempty"`
-	Routes   []Route    `json:"routes,omitempty"`
-	Iptables []Iptables `json:"iptables,omitempty"`
-	Fdbs     []Fdb      `json:"fdbs,omitempty"`
-	Arps     []Arp      `json:"arps,omitempty"`
+	Devices      []Device     `json:"devices,omitempty"`
+	Routes       []Route      `json:"routes,omitempty"`
+	Iptables     []Iptables   `json:"iptables,omitempty"`
+	Fdbs         []Fdb        `json:"fdbs,omitempty"`
+	Arps         []Arp        `json:"arps,omitempty"`
+	XfrmPolicies []XfrmPolicy `json:"xfrmpolicies,omitempty"`
+	XfrmStates   []XfrmState  `json:"xfrmstates,omitempty"`
 }
 
 type NodeConfigStatus struct {
@@ -101,6 +103,40 @@ func (a *Arp) Compare(v Arp) bool {
 		a.Dev == v.Dev
 }
 
+type XfrmPolicy struct {
+	LeftIP   string `json:"leftip"`
+	LeftNet  string `json:"leftnet"`
+	RightIP  string `json:"rightip"`
+	RightNet string `json:"rightnet"`
+	ReqID    int    `json:"reqid"`
+	Dir      int    `json:"dir"`
+}
+
+func (a *XfrmPolicy) Compare(v XfrmPolicy) bool {
+	return a.LeftIP == v.LeftIP &&
+		a.LeftNet == v.LeftNet &&
+		a.RightNet == v.RightNet &&
+		a.RightIP == v.RightIP &&
+		a.ReqID == v.ReqID &&
+		a.Dir == v.Dir
+}
+
+type XfrmState struct {
+	LeftIP  string `json:"leftip"`
+	RightIP string `json:"rightip"`
+	ReqID   int    `json:"reqid"`
+	SPI     uint32 `json:"spi"`
+	PSK     string `json:"PSK"`
+}
+
+func (a *XfrmState) Compare(v XfrmState) bool {
+	return a.LeftIP == v.LeftIP &&
+		a.RightIP == v.RightIP &&
+		a.ReqID == v.ReqID &&
+		a.PSK == v.PSK &&
+		a.SPI == v.SPI
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 
 type NodeConfigList struct {

diff --git a/pkg/clusterlink/agent-manager/network-manager/network_manager.go b/pkg/clusterlink/agent-manager/network-manager/network_manager.go
@@ -112,6 +112,21 @@ func (e *NetworkManager) Diff(oldConfig, newConfig *clusterlinkv1alpha1.NodeConf
 		createConfig.Routes = createRecord
 		isSame = false
 	}
+	// ipsec:
+	if flag, deleteRecord, createRecord := compareFunc(oldConfig.XfrmPolicies, newConfig.XfrmPolicies, func(a, b clusterlinkv1alpha1.XfrmPolicy) bool {
+		return a.Compare(b)
+	}); !flag {
+		deleteConfig.XfrmPolicies = deleteRecord
+		createConfig.XfrmPolicies = createRecord
+		isSame = false
+	}
+	if flag, deleteRecord, createRecord := compareFunc(oldConfig.XfrmStates, newConfig.XfrmStates, func(a, b clusterlinkv1alpha1.XfrmState) bool {
+		return a.Compare(b)
+	}); !flag {
+		deleteConfig.XfrmStates = deleteRecord
+		createConfig.XfrmStates = createRecord
+		isSame = false
+	}
 	// iptables:
 	if flag, deleteRecord, createRecord := compareFunc(oldConfig.Iptables, newConfig.Iptables, func(a, b clusterlinkv1alpha1.Iptables) bool {
 		return a.Compare(b)
@@ -188,6 +203,18 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
 				errs = errors.Wrap(err, fmt.Sprint(errs))
 			}
 		}
+		if config.XfrmPolicies != nil {
+			if err := e.NetworkInterface.DeleteXfrmPolicies(config.XfrmPolicies); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
+		if config.XfrmStates != nil {
+			if err := e.NetworkInterface.DeleteXfrmStates(config.XfrmStates); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
 	}
 
 	if configDiff.createConfig != nil {
@@ -223,6 +250,18 @@ func (e *NetworkManager) WriteSys(configDiff *ConfigDiff) error {
 				errs = errors.Wrap(err, fmt.Sprint(errs))
 			}
 		}
+		if config.XfrmPolicies != nil {
+			if err := e.NetworkInterface.AddXfrmPolicies(config.XfrmPolicies); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
+		if config.XfrmStates != nil {
+			if err := e.NetworkInterface.AddXfrmStates(config.XfrmStates); err != nil {
+				klog.Warning(err)
+				errs = errors.Wrap(err, fmt.Sprint(errs))
+			}
+		}
 	}
 
 	return errs
@@ -254,11 +293,13 @@ func (e *NetworkManager) UpdateFromChecker() NodeConfigSyncStatus {
 }
 
 func printNodeConfig(data *clusterlinkv1alpha1.NodeConfigSpec) {
-	klog.Infof("device: ", data.Devices)
-	klog.Infof("Arps: ", data.Arps)
-	klog.Infof("Fdbs: ", data.Fdbs)
-	klog.Infof("Iptables: ", data.Iptables)
-	klog.Infof("Routes: ", data.Routes)
+	klog.Infof("device: %v", data.Devices)
+	klog.Infof("Arps: %v", data.Arps)
+	klog.Infof("Fdbs: %v", data.Fdbs)
+	klog.Infof("Iptables: %v", data.Iptables)
+	klog.Infof("Routes: %v", data.Routes)
+	klog.Infof("XfrmPolicys: %v", data.XfrmPolicies)
+	klog.Infof("XfrmStates: %v", data.XfrmStates)
 }
 
 func (e *NetworkManager) UpdateSync() NodeConfigSyncStatus {

diff --git a/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go b/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go
@@ -339,10 +339,6 @@ func (c *Controller) Reconcile(key utils.QueueKey) error {
 	}
 
 	klog.Infof("start reconcile cluster %s", cluster.Name)
-	if cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico {
-		klog.Infof("cluster %s cni type is %s skip reconcile", cluster.Name, cluster.Spec.ClusterLinkOptions.CNI)
-		return nil
-	}
 	for ipPool := range c.globalExtIPPoolSet {
 		if ipPool.cluster == cluster.Name {
 			delete(c.globalExtIPPoolSet, ipPool)
@@ -375,6 +371,10 @@ func (c *Controller) Reconcile(key utils.QueueKey) error {
 		c.globalExtIPPoolSet[extIPPool] = struct{}{}
 	}
 	klog.Infof("now has %d globalIPPools", len(c.globalExtIPPoolSet))
+	if cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico {
+		klog.Infof("cluster %s cni type is %s skip reconcile", cluster.Name, cluster.Spec.ClusterLinkOptions.CNI)
+		return nil
+	}
 	if c.iPPoolClient == nil {
 		if cluster.Name == c.clusterName {
 			ipPoolClient, err := c.createIPPoolClient(cluster)