Release cce-network-v2/2.9.5

cce-network-v2/VERSION

@@ -1 +1 @@
-2.9.4
+2.9.5

cce-network-v2/docs/release.md

@@ -8,6 +8,10 @@ v2 版本新架构，支持VPC-ENI 辅助IP和vpc路由。版本发布历史如
 3. 新特性: 支持ubuntu 22.04 操作系统，在容器网络环境下，定义 systemd-networkd 的 MacAddressPolicy 为 none。
 4. 新特性：支持 pod 级 Qos
 
+### 2.9.5 [20240325]
+1. [BUG] 修复 vpc-route 模式下，重启 operator 可能导致多个节点的 cidr 重复的问题
+2. [BUG] 修复调用 bce sdk 出错时，可能出现的stack overflow，导致operator重启的问题
+
 ### 2.9.4 [20240305]
 1. [Feature] 支持 BBC 实例通过 Node 上增加 `network.cce.baidubce.com/node-eni-subnet` Anotation 配置指定节点上 ENI 的子网。 
 

cce-network-v2/go.mod

@@ -61,7 +61,6 @@ require (
 	github.com/coreos/go-iptables v0.6.0 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
-	github.com/godbus/dbus/v5 v5.0.4 // indirect
 	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1 // indirect

cce-network-v2/go.sum

@@ -255,7 +255,6 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe
 github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ=
 github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0=
 github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw=
-github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=

cce-network-v2/pkg/bce/api/cloud/error.go

@@ -117,8 +117,7 @@ func IsErrorRouteRuleRepeated(err error) bool {
 }
 
 func IsErrorQuotaLimitExceeded(err error) bool {
-	return ReasonForError(err) == ErrorReasonQuotaLimitExceeded ||
-		IsErrorQuotaLimitExceeded(err)
+	return ReasonForError(err) == ErrorReasonQuotaLimitExceeded
 }
 
 func IsErrorCreateRouteRuleExceededQuota(err error) bool {

cce-network-v2/pkg/ipam/allocator/podcidr/podcidr.go

@@ -24,17 +24,16 @@ import (
 
 	"github.com/sirupsen/logrus"
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/labels"
 
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/cidr"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/controller"
 	ipPkg "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ip"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ipam"
-	ipamOption "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/ipam/option"
 	v2 "github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/k8s/apis/cce.baidubce.com/v2"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/lock"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/logging"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/logging/logfields"
-	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/option"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/revert"
 	"github.com/baidubce/baiducloud-cce-cni-driver/cce-network-v2/pkg/trigger"
 )
@@ -162,6 +161,7 @@ var updateK8sInterval = 15 * time.Second
 type NodesPodCIDRManager struct {
 	k8sReSyncController *controller.Manager
 	k8sReSync           *trigger.Trigger
+	nodeGetter          ipam.NetResourceSetGetterUpdater
 
 	// Lock protects all fields below
 	lock.Mutex
@@ -217,6 +217,7 @@ func NewNodesPodCIDRManager(
 		nodes:                map[string]*nodeCIDRs{},
 		netResourceSetsToK8s: map[string]*netResourceSetK8sOp{},
 		k8sReSyncController:  controller.NewManager(),
+		nodeGetter:           nodeGetter,
 	}
 
 	// Have a trigger so that multiple calls, within a second, to sync with k8s
@@ -369,64 +370,8 @@ func (n *NodesPodCIDRManager) Create(node *v2.NetResourceSet) error {
 func (n *NodesPodCIDRManager) Update(node *v2.NetResourceSet) error {
 	n.Mutex.Lock()
 	defer n.Mutex.Unlock()
-	return n.update(node)
-}
 
-// Needs n.Mutex to be held.
-func (n *NodesPodCIDRManager) update(node *v2.NetResourceSet) error {
-	var (
-		updateStatus, updateSpec bool
-		cn                       *v2.NetResourceSet
-		err                      error
-	)
-	if option.Config.IPAMMode() == ipamOption.IPAMClusterPoolV2 || option.Config.IPAMMode() == ipamOption.IPAMVpcRoute {
-		cn, updateSpec, updateStatus, err = n.allocateNodeV2(node)
-		if err != nil {
-			return err
-		}
-	} else {
-		// FIXME: This code block falls back to the old behavior of clusterpool,
-		// where we only assign one pod CIDR for IPv4 and IPv6. Once v2 becomes
-		// fully backwards compatible with v1, we can remove this else block.
-		var allocated bool
-		cn, allocated, updateStatus, err = n.allocateNode(node)
-		if err != nil {
-			return err
-		}
-		// if allocated is false it means that we were unable to allocate
-		// a CIDR so we need to update the status of the node into k8s.
-		updateStatus = !allocated && updateStatus
-		// ClusterPool v1 never updates both the spec and the status
-		updateSpec = !updateStatus
-	}
-	if cn == nil {
-		// no-op
-		return nil
-	}
-	if updateStatus {
-		// the n.syncNode will never fail because it's only adding elements to a
-		// map.
-		// NodesPodCIDRManager will later on sync the node into k8s by the
-		// controller defined, which keeps retrying to create the node in k8s
-		// until it succeeds.
-
-		// If the resource version is != "" it means the object already exists
-		// in kubernetes so we should perform an update status instead of a create.
-		if cn.GetResourceVersion() != "" {
-			n.syncNode(k8sOpUpdateStatus, cn)
-		} else {
-			n.syncNode(k8sOpCreate, cn)
-		}
-	}
-	if updateSpec {
-		// If the resource version is != "" it means the object already exists
-		// in kubernetes so we should perform an update instead of a create.
-		if cn.GetResourceVersion() != "" {
-			n.syncNode(k8sOpUpdate, cn)
-		} else {
-			n.syncNode(k8sOpCreate, cn)
-		}
-	}
+	n.upsertLocked(node)
 	return nil
 }
 
@@ -455,20 +400,68 @@ func (n *NodesPodCIDRManager) Delete(nodeName string) error {
 func (n *NodesPodCIDRManager) Resync(context.Context, time.Time) {
 	n.Mutex.Lock()
 	if !n.canAllocatePodCIDRs {
+		nrsDatas, err := n.nodeGetter.Lister().List(labels.Everything())
+		if err != nil {
+			log.WithError(err).Fatal("Failed to list NetResourceSet")
+		}
+		for _, nrs := range nrsDatas {
+			n.upsertLocked(nrs)
+		}
+
+		log.Infof("completed to resync %d nrs cidr", len(nrsDatas))
+		// We can now allocate podCIDRs
 		n.canAllocatePodCIDRs = true
 		// Iterate over all nodes that we have kept stored up until Resync
 		// is called as now we are allowed to allocate podCIDRs for nodes
 		// without any podCIDR.
 		for _, cn := range n.nodesToAllocate {
-			n.update(cn)
+			n.upsertLocked(cn)
 		}
 		n.nodesToAllocate = nil
+		log.Infof("completed to allocate new %d nodes cidr", len(n.nodesToAllocate))
 	}
 	n.Mutex.Unlock()
 
 	n.k8sReSync.Trigger()
 }
 
+// Needs n.Mutex to be held.
+func (n *NodesPodCIDRManager) upsertLocked(node *v2.NetResourceSet) {
+	cn, allocated, updateStatus, err := n.allocateNode(node)
+	if err != nil {
+		return
+	}
+	if cn == nil {
+		// no-op
+		return
+	}
+	// if allocated is false it means that we were unable to allocate
+	// a CIDR so we need to update the status of the node into k8s.
+	if !allocated && updateStatus {
+		// the n.syncNode will never fail because it's only adding elements to a
+		// map.
+		// NodesPodCIDRManager will later on sync the node into k8s by the
+		// controller defined, which keeps retrying to create the node in k8s
+		// until it succeeds.
+
+		// If the resource version is != "" it means the object already exists
+		// in kubernetes so we should perform an update status instead of a create.
+		if cn.GetResourceVersion() != "" {
+			n.syncNode(k8sOpUpdateStatus, cn)
+		} else {
+			n.syncNode(k8sOpCreate, cn)
+		}
+		return
+	}
+	// If the resource version is != "" it means the object already exists
+	// in kubernetes so we should perform an update instead of a create.
+	if cn.GetResourceVersion() != "" {
+		n.syncNode(k8sOpUpdate, cn)
+	} else {
+		n.syncNode(k8sOpCreate, cn)
+	}
+}
+
 // AllocateNode allocates the podCIDRs for the given node. Returns a DeepCopied
 // node with the podCIDRs allocated. In case there weren't CIDRs allocated
 // the returned node will be nil.

cce-network-v2/pkg/ipam/allocator/podcidr/podcidr_test.go

@@ -71,7 +71,7 @@ func mustNewTrigger(f func(), minInterval time.Duration) *trigger.Trigger {
 	return t
 }
 
-var defaultIPAMModes = []string{ipamOption.IPAMClusterPool, ipamOption.IPAMClusterPoolV2}
+var defaultIPAMModes = []string{ipamOption.IPAMClusterPool}
 
 func runWithIPAMModes(ipamModes []string, testFunc func(mode string)) {
 	oldIPAMMode := option.Config.IPAM