From d1a6fdada6b951d33c1a981748c27781325b46a9 Mon Sep 17 00:00:00 2001 From: GreatLazyMan Date: Wed, 17 Jan 2024 12:15:19 +0800 Subject: [PATCH] title: Add ipsec tunnel mode to support cross clusters and elastic ip Signed-off-by: GreatLazyMan --- pkg/apis/kosmos/v1alpha1/nodeconfig_types.go | 11 ++++++++ .../calicoippool/calicoippool_controller.go | 2 +- .../controllers/cluster/cluster_controller.go | 26 ++--------------- .../cluster/invalid_manifest_services.go | 28 +++++++++++++++++++ pkg/clusterlink/elector/elector.go | 1 + pkg/clusterlink/network/iptables/iptables.go | 10 ++----- 6 files changed, 45 insertions(+), 33 deletions(-) create mode 100644 pkg/clusterlink/controllers/cluster/invalid_manifest_services.go diff --git a/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go b/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go index 46e8b4f33..19020fbef 100644 --- a/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go +++ b/pkg/apis/kosmos/v1alpha1/nodeconfig_types.go @@ -104,6 +104,12 @@ func (a *Arp) Compare(v Arp) bool { a.Dev == v.Dev } +/* +Just like linux command: + + ip xfrm policy add src $LeftNet dst $RightNet dir $Dir \ + tmpl src $LeftIP dst $RightIP proto esp reqid $ReqID mode tunnel +*/ type XfrmPolicy struct { LeftIP string `json:"leftip"` LeftNet string `json:"leftnet"` @@ -122,6 +128,11 @@ func (a *XfrmPolicy) Compare(v XfrmPolicy) bool { a.Dir == v.Dir } +/* +Just like linux command: + + ip xfrm state add src $LeftIP dst $RightIP proto esp spi $SPI reqid $ReqID mode tunnel aead 'rfc4106(gcm(aes))' $PSK 128 +*/ type XfrmState struct { LeftIP string `json:"leftip"` RightIP string `json:"rightip"` diff --git a/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go b/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go index 72f40c749..ba409c794 100644 --- a/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go +++ b/pkg/clusterlink/controllers/calicoippool/calicoippool_controller.go @@ -339,7 +339,7 @@ func (c *Controller) Reconcile(key utils.QueueKey) error { } klog.Infof("start reconcile cluster %s", cluster.Name) - if cluster.Name == c.clusterName && cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico { + if cluster.Spec.ClusterLinkOptions.CNI != utils.CNITypeCalico { klog.Infof("cluster %s cni type is %s skip reconcile", cluster.Name, cluster.Spec.ClusterLinkOptions.CNI) return nil } diff --git a/pkg/clusterlink/controllers/cluster/cluster_controller.go b/pkg/clusterlink/controllers/cluster/cluster_controller.go index d21c2c7b9..592e367cf 100644 --- a/pkg/clusterlink/controllers/cluster/cluster_controller.go +++ b/pkg/clusterlink/controllers/cluster/cluster_controller.go @@ -50,30 +50,6 @@ const ( KubeFlannelNetworkConf = "net-conf.json" KubeFlannelIPPool = "Network" KubeSystemNamespace = "kube-system" - InvalidService = ` -apiVersion: v1 -kind: Service -metadata: - labels: - kosmos.io/app: coredns - name: invalidsvc - namespace: {{ .Namespace }} -spec: - clusterIP: 8.8.8.8 - clusterIPs: - - 8.8.8.8 - ipFamilies: - - IPv4 - ports: - - name: dns - port: 53 - protocol: UDP - targetPort: 53 - selector: - invalid/app: null - sessionAffinity: None - type: ClusterIP -` ) type SetClusterPodCIDRFun func(cluster *clusterlinkv1alpha1.Cluster) error @@ -135,7 +111,9 @@ func (c *Controller) Start(ctx context.Context) error { var podFilterFunc func(pod *corev1.Pod) bool if cluster.Spec.ClusterLinkOptions.UseExternalApiserver { podFilterFunc = func(pod *corev1.Pod) bool { + // TODO: find a better way // some k8s, apiserver not a pod in cluster, maybe not a good way + // so we choose some kube-system pod and clusterlink-controller-manager itself as filter return pod.Labels["k8s-app"] == "kube-proxy" || pod.Labels["app"] == "clusterlink-controller-manager" } } else { diff --git a/pkg/clusterlink/controllers/cluster/invalid_manifest_services.go b/pkg/clusterlink/controllers/cluster/invalid_manifest_services.go new file mode 100644 index 000000000..6e613c7f8 --- /dev/null +++ b/pkg/clusterlink/controllers/cluster/invalid_manifest_services.go @@ -0,0 +1,28 @@ +package cluster + +const ( + InvalidService = ` +apiVersion: v1 +kind: Service +metadata: + labels: + kosmos.io/app: coredns + name: invalidsvc + namespace: {{ .Namespace }} +spec: + clusterIP: 8.8.8.8 + clusterIPs: + - 8.8.8.8 + ipFamilies: + - IPv4 + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + selector: + invalid/app: null + sessionAffinity: None + type: ClusterIP +` +) diff --git a/pkg/clusterlink/elector/elector.go b/pkg/clusterlink/elector/elector.go index 08b88bd54..e1fc4828c 100644 --- a/pkg/clusterlink/elector/elector.go +++ b/pkg/clusterlink/elector/elector.go @@ -81,6 +81,7 @@ func (e *Elector) EnsureGateWayRole() error { if needReelect { if !isCurrentNodeWithEIP && len(readyNodes) > 0 { + // TODO: now choose first one, find a better way sort.Strings(readyNodes) e.nodeName = readyNodes[0] } else { diff --git a/pkg/clusterlink/network/iptables/iptables.go b/pkg/clusterlink/network/iptables/iptables.go index eb53e71ed..f2949f2ab 100644 --- a/pkg/clusterlink/network/iptables/iptables.go +++ b/pkg/clusterlink/network/iptables/iptables.go @@ -74,15 +74,9 @@ func init() { return } if len(ret_nft) > len(ret_legacy) { - klog.Info("use iptables-nft as default iptables") - _, err := execInterface.Command("ln", []string{"-sf", "/sbin/xtables-nft-multi", "/sbin/iptables"}...).CombinedOutput() + err := os.Setenv("IPTABLES_PATH", "/sbin/xtables-nft-multi") if err != nil { - klog.Errorf("%s: %v", errInfo, err) - return - } - _, err = execInterface.Command("ln", []string{"-sf", "/sbin/xtables-nft-multi", "/sbin/ip6tables"}...).CombinedOutput() - if err != nil { - klog.Errorf("%s: %v", errInfo, err) + klog.Errorf("%s, set env error: %v", errInfo, err) return } }