Release cce-network-v2/2.10.1

cce-network-v2/VERSION

@@ -1 +1 @@
-2.10.0
+2.10.1

cce-network-v2/docs/release.md

@@ -2,6 +2,11 @@
 v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如下：
 
 ### 2.10 (2024/03/05)
+### 2.10.1 [20240325]
+1. [BUG] 修复 vpc-route 模式下，重启 operator 可能导致多个节点的 cidr 重复的问题
+2. [BUG] 修复调用 bce sdk 出错时，可能出现的stack overflow，导致operator重启的问题
+3. [Opimize] vpc-eni 增加 mac 地址合法性校验，避免误操作其它网卡
+
 ### 2.10.0 （2024/03/05）
 1. [Feature] VPC-ENI 支持自动获取节点 eni 配额信息，去掉了自定义 ENI 配额的参数。
 2. [Feature] VPC-ENI 支持 ebc 主网卡辅助 IP 模式 
@@ -22,6 +27,10 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如
 3. 新特性: 支持ubuntu 22.04 操作系统，在容器网络环境下，定义 systemd-networkd 的 MacAddressPolicy 为 none。
 4. 新特性：支持 pod 级 Qos
 
+### 2.9.5 [20240325]
+1. [BUG] 修复 vpc-route 模式下，重启 operator 可能导致多个节点的 cidr 重复的问题
+2. [BUG] 修复调用 bce sdk 出错时，可能出现的stack overflow，导致operator重启的问题
+
 ### 2.9.4 [20240305]
 1. [Feature] 支持 BBC 实例通过 Node 上增加 `network.cce.baidubce.com/node-eni-subnet` Anotation 配置指定节点上 ENI 的子网。 
 

cce-network-v2/pkg/bce/api/cloud/error.go

@@ -119,8 +119,7 @@ func IsErrorRouteRuleRepeated(err error) bool {
 }
 
 func IsErrorQuotaLimitExceeded(err error) bool {
-	return ReasonForError(err) == ErrorReasonQuotaLimitExceeded ||
-		IsErrorQuotaLimitExceeded(err)
+	return ReasonForError(err) == ErrorReasonQuotaLimitExceeded
 }
 
 func IsErrorCreateRouteRuleExceededQuota(err error) bool {

cce-network-v2/pkg/bce/bcesync/bcc_primary_eni.go

@@ -2,6 +2,7 @@ package bcesync
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	enisdk "github.com/baidubce/bce-sdk-go/services/eni"
@@ -82,6 +83,10 @@ func (es *remoteBCCPrimarySyncher) statENI(ctx context.Context, eniID string) (*
 		if bcceni.EniId != k8seni.Spec.ENI.ID {
 			continue
 		}
+
+		if bcceni.MacAddress == "" {
+			return nil, errors.New("vpc mac address is empty")
+		}
 		trancelateENI := eni.Eni{
 			Eni: enisdk.Eni{
 				EniId:       bcceni.EniId,

cce-network-v2/pkg/bce/bcesync/eni.go

@@ -377,7 +377,11 @@ func (es *eniSyncher) refreshENI(ctx context.Context, newObj *ccev2.ENI) error {
 		return err
 	}
 
-	if eniCache != nil && eniCache.MacAddress != "" {
+	if eniCache != nil {
+		if eniCache.MacAddress == "" {
+			return errors.New("vpc mac address is empty")
+		}
+
 		newObj.Spec.ENI.ID = eniCache.EniId
 		newObj.Spec.ENI.Name = eniCache.Name
 		newObj.Spec.ENI.MacAddress = eniCache.MacAddress

cce-network-v2/pkg/bce/bcesync/physical_eni.go

@@ -212,7 +212,10 @@ func (es *physicalENISyncer) refreshENI(ctx context.Context, newObj *ccev2.ENI)
 		return err
 	}
 
-	if eniCache != nil && eniCache.MacAddress != "" {
+	if eniCache != nil {
+		if eniCache.MacAddress == "" {
+			return errors.New("vpc mac address is empty")
+		}
 		newObj.Spec.ENI.ID = eniCache.Id
 		newObj.Spec.ENI.Name = eniCache.Name
 		newObj.Spec.ENI.MacAddress = eniCache.MacAddress

cce-network-v2/pkg/ipam/allocator/podcidr/podcidr.go

@@ -24,17 +24,16 @@ import (
 
 	"github.com/sirupsen/logrus"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
 
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/cidr"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/controller"
 	ipPkg "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ip"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ipam"
-	ipamOption "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ipam/option"
 	v2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/lock"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/logging"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/logging/logfields"
-	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/option"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/revert"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/trigger"
 )
@@ -162,6 +161,7 @@ var updateK8sInterval = 15 * time.Second
 type NodesPodCIDRManager struct {
 	k8sReSyncController *controller.Manager
 	k8sReSync           *trigger.Trigger
+	nodeGetter          ipam.NetResourceSetGetterUpdater
 
 	// Lock protects all fields below
 	lock.Mutex
@@ -217,6 +217,7 @@ func NewNodesPodCIDRManager(
 		nodes:                map[string]*nodeCIDRs{},
 		netResourceSetsToK8s: map[string]*netResourceSetK8sOp{},
 		k8sReSyncController:  controller.NewManager(),
+		nodeGetter:           nodeGetter,
 	}
 
 	// Have a trigger so that multiple calls, within a second, to sync with k8s
@@ -369,64 +370,8 @@ func (n *NodesPodCIDRManager) Create(node *v2.NetResourceSet) error {
 func (n *NodesPodCIDRManager) Update(node *v2.NetResourceSet) error {
 	n.Mutex.Lock()
 	defer n.Mutex.Unlock()
-	return n.update(node)
-}
 
-// Needs n.Mutex to be held.
-func (n *NodesPodCIDRManager) update(node *v2.NetResourceSet) error {
-	var (
-		updateStatus, updateSpec bool
-		cn                       *v2.NetResourceSet
-		err                      error
-	)
-	if option.Config.IPAMMode() == ipamOption.IPAMClusterPoolV2 || option.Config.IPAMMode() == ipamOption.IPAMVpcRoute {
-		cn, updateSpec, updateStatus, err = n.allocateNodeV2(node)
-		if err != nil {
-			return err
-		}
-	} else {
-		// FIXME: This code block falls back to the old behavior of clusterpool,
-		// where we only assign one pod CIDR for IPv4 and IPv6. Once v2 becomes
-		// fully backwards compatible with v1, we can remove this else block.
-		var allocated bool
-		cn, allocated, updateStatus, err = n.allocateNode(node)
-		if err != nil {
-			return err
-		}
-		// if allocated is false it means that we were unable to allocate
-		// a CIDR so we need to update the status of the node into k8s.
-		updateStatus = !allocated && updateStatus
-		// ClusterPool v1 never updates both the spec and the status
-		updateSpec = !updateStatus
-	}
-	if cn == nil {
-		// no-op
-		return nil
-	}
-	if updateStatus {
-		// the n.syncNode will never fail because it's only adding elements to a
-		// map.
-		// NodesPodCIDRManager will later on sync the node into k8s by the
-		// controller defined, which keeps retrying to create the node in k8s
-		// until it succeeds.
-
-		// If the resource version is != "" it means the object already exists
-		// in kubernetes so we should perform an update status instead of a create.
-		if cn.GetResourceVersion() != "" {
-			n.syncNode(k8sOpUpdateStatus, cn)
-		} else {
-			n.syncNode(k8sOpCreate, cn)
-		}
-	}
-	if updateSpec {
-		// If the resource version is != "" it means the object already exists
-		// in kubernetes so we should perform an update instead of a create.
-		if cn.GetResourceVersion() != "" {
-			n.syncNode(k8sOpUpdate, cn)
-		} else {
-			n.syncNode(k8sOpCreate, cn)
-		}
-	}
+	n.upsertLocked(node)
 	return nil
 }
 
@@ -455,20 +400,68 @@ func (n *NodesPodCIDRManager) Delete(nodeName string) error {
 func (n *NodesPodCIDRManager) Resync(context.Context, time.Time) {
 	n.Mutex.Lock()
 	if !n.canAllocatePodCIDRs {
+		nrsDatas, err := n.nodeGetter.Lister().List(labels.Everything())
+		if err != nil {
+			log.WithError(err).Fatal("Failed to list NetResourceSet")
+		}
+		for _, nrs := range nrsDatas {
+			n.upsertLocked(nrs)
+		}
+
+		log.Infof("completed to resync %d nrs cidr", len(nrsDatas))
+		// We can now allocate podCIDRs
 		n.canAllocatePodCIDRs = true
 		// Iterate over all nodes that we have kept stored up until Resync
 		// is called as now we are allowed to allocate podCIDRs for nodes
 		// without any podCIDR.
 		for _, cn := range n.nodesToAllocate {
-			n.update(cn)
+			n.upsertLocked(cn)
 		}
 		n.nodesToAllocate = nil
+		log.Infof("completed to allocate new %d nodes cidr", len(n.nodesToAllocate))
 	}
 	n.Mutex.Unlock()
 
 	n.k8sReSync.Trigger()
 }
 
+// Needs n.Mutex to be held.
+func (n *NodesPodCIDRManager) upsertLocked(node *v2.NetResourceSet) {
+	cn, allocated, updateStatus, err := n.allocateNode(node)
+	if err != nil {
+		return
+	}
+	if cn == nil {
+		// no-op
+		return
+	}
+	// if allocated is false it means that we were unable to allocate
+	// a CIDR so we need to update the status of the node into k8s.
+	if !allocated && updateStatus {
+		// the n.syncNode will never fail because it's only adding elements to a
+		// map.
+		// NodesPodCIDRManager will later on sync the node into k8s by the
+		// controller defined, which keeps retrying to create the node in k8s
+		// until it succeeds.
+
+		// If the resource version is != "" it means the object already exists
+		// in kubernetes so we should perform an update status instead of a create.
+		if cn.GetResourceVersion() != "" {
+			n.syncNode(k8sOpUpdateStatus, cn)
+		} else {
+			n.syncNode(k8sOpCreate, cn)
+		}
+		return
+	}
+	// If the resource version is != "" it means the object already exists
+	// in kubernetes so we should perform an update instead of a create.
+	if cn.GetResourceVersion() != "" {
+		n.syncNode(k8sOpUpdate, cn)
+	} else {
+		n.syncNode(k8sOpCreate, cn)
+	}
+}
+
 // AllocateNode allocates the podCIDRs for the given node. Returns a DeepCopied
 // node with the podCIDRs allocated. In case there weren't CIDRs allocated
 // the returned node will be nil.

cce-network-v2/pkg/ipam/allocator/podcidr/podcidr_test.go

@@ -71,7 +71,7 @@ func mustNewTrigger(f func(), minInterval time.Duration) *trigger.Trigger {
 	return t
 }
 
-var defaultIPAMModes = []string{ipamOption.IPAMClusterPool, ipamOption.IPAMClusterPoolV2}
+var defaultIPAMModes = []string{ipamOption.IPAMClusterPool}
 
 func runWithIPAMModes(ipamModes []string, testFunc func(mode string)) {
 	oldIPAMMode := option.Config.IPAM