title: Add ipsec tunnel mode to support cross clusters and elastic ip

Signed-off-by: GreatLazyMan <[email protected]>
kosmos-io · Jan 17, 2024 · d1a6fda · d1a6fda
1 parent 42f2476
commit d1a6fda
Show file tree

Hide file tree

Showing 6 changed files with 45 additions and 33 deletions.
diff --git a/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go b/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go
@@ -104,6 +104,12 @@ func (a *Arp) Compare(v Arp) bool {
 		a.Dev == v.Dev
 }
 
+/*
+Just like linux command:
+
+	ip xfrm policy add src $LeftNet dst $RightNet dir $Dir \
+	                   tmpl src $LeftIP dst $RightIP proto esp reqid $ReqID mode tunnel
+*/
 type XfrmPolicy struct {
 	LeftIP   string `json:"leftip"`
 	LeftNet  string `json:"leftnet"`
@@ -122,6 +128,11 @@ func (a *XfrmPolicy) Compare(v XfrmPolicy) bool {
 		a.Dir == v.Dir
 }
 
+/*
+Just like linux command:
+
+	ip xfrm state add src $LeftIP dst $RightIP proto esp spi $SPI reqid $ReqID mode tunnel aead 'rfc4106(gcm(aes))' $PSK 128
+*/
 type XfrmState struct {
 	LeftIP  string `json:"leftip"`
 	RightIP string `json:"rightip"`

diff --git a/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go b/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go
@@ -339,7 +339,7 @@ func (c *Controller) Reconcile(key utils.QueueKey) error {
 	}
 
 	klog.Infof("start reconcile cluster %s", cluster.Name)
-	if cluster.Name == c.clusterName && cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico {
+	if cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico {
 		klog.Infof("cluster %s cni type is %s skip reconcile", cluster.Name, cluster.Spec.ClusterLinkOptions.CNI)
 		return nil
 	}

diff --git a/pkg/clusterlink/controllers/cluster/cluster_controller.go b/pkg/clusterlink/controllers/cluster/cluster_controller.go
@@ -50,30 +50,6 @@ const (
 	KubeFlannelNetworkConf = "net-conf.json"
 	KubeFlannelIPPool      = "Network"
 	KubeSystemNamespace    = "kube-system"
-	InvalidService         = `
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    kosmos.io/app: coredns
-  name: invalidsvc
-  namespace: {{ .Namespace }}
-spec:
-  clusterIP: 8.8.8.8
-  clusterIPs:
-  - 8.8.8.8
-  ipFamilies:
-  - IPv4
-  ports:
-    - name: dns
-      port: 53
-      protocol: UDP
-      targetPort: 53
-  selector:
-    invalid/app: null
-  sessionAffinity: None
-  type: ClusterIP
-`
 )
 
 type SetClusterPodCIDRFun func(cluster *clusterlinkv1alpha1.Cluster) error
@@ -135,7 +111,9 @@ func (c *Controller) Start(ctx context.Context) error {
 	var podFilterFunc func(pod *corev1.Pod) bool
 	if cluster.Spec.ClusterLinkOptions.UseExternalApiserver {
 		podFilterFunc = func(pod *corev1.Pod) bool {
+			// TODO: find a better way
 			// some k8s, apiserver not a pod in cluster, maybe not a good way
+			// so we choose some kube-system pod and clusterlink-controller-manager itself as filter
 			return pod.Labels["k8s-app"] == "kube-proxy" || pod.Labels["app"] == "clusterlink-controller-manager"
 		}
 	} else {

diff --git a/pkg/clusterlink/controllers/cluster/invalid_manifest_services.go b/pkg/clusterlink/controllers/cluster/invalid_manifest_services.go
@@ -0,0 +1,28 @@
+package cluster
+
+const (
+	InvalidService = `
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    kosmos.io/app: coredns
+  name: invalidsvc
+  namespace: {{ .Namespace }}
+spec:
+  clusterIP: 8.8.8.8
+  clusterIPs:
+  - 8.8.8.8
+  ipFamilies:
+  - IPv4
+  ports:
+    - name: dns
+      port: 53
+      protocol: UDP
+      targetPort: 53
+  selector:
+    invalid/app: null
+  sessionAffinity: None
+  type: ClusterIP
+`
+)
diff --git a/pkg/clusterlink/elector/elector.go b/pkg/clusterlink/elector/elector.go
@@ -81,6 +81,7 @@ func (e *Elector) EnsureGateWayRole() error {
 
 		if needReelect {
 			if !isCurrentNodeWithEIP && len(readyNodes) > 0 {
+				// TODO: now choose first one, find a better way
 				sort.Strings(readyNodes)
 				e.nodeName = readyNodes[0]
 			} else {

diff --git a/pkg/clusterlink/network/iptables/iptables.go b/pkg/clusterlink/network/iptables/iptables.go
@@ -74,15 +74,9 @@ func init() {
 		return
 	}
 	if len(ret_nft) > len(ret_legacy) {
-		klog.Info("use iptables-nft as default iptables")
-		_, err := execInterface.Command("ln", []string{"-sf", "/sbin/xtables-nft-multi", "/sbin/iptables"}...).CombinedOutput()
+		err := os.Setenv("IPTABLES_PATH", "/sbin/xtables-nft-multi")
 		if err != nil {
-			klog.Errorf("%s: %v", errInfo, err)
-			return
-		}
-		_, err = execInterface.Command("ln", []string{"-sf", "/sbin/xtables-nft-multi", "/sbin/ip6tables"}...).CombinedOutput()
-		if err != nil {
-			klog.Errorf("%s: %v", errInfo, err)
+			klog.Errorf("%s, set env error: %v", errInfo, err)
 			return
 		}
 	}