IPAM sends GARPS to updating arp cache table

Signed-off-by: Cyclinder Kuo <[email protected]>

cmd/spiderpool/cmd/command_add.go

@@ -25,6 +25,7 @@ import (
 	"github.com/spidernet-io/spiderpool/api/v1/agent/models"
 	"github.com/spidernet-io/spiderpool/pkg/constant"
 	spiderpoolip "github.com/spidernet-io/spiderpool/pkg/ip"
+	"github.com/spidernet-io/spiderpool/pkg/networking/networking"
 	"github.com/spidernet-io/spiderpool/pkg/openapi"
 )
 
@@ -164,8 +165,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
 	// do ip conflict and gateway detection
 	logger.Sugar().Info("postIpam response",
 		zap.Any("DNS", ipamResponse.Payload.DNS),
-		zap.Any("Routes", ipamResponse.Payload.Routes),
-		zap.Any("Ips", ipamResponse.Payload.Ips))
+		zap.Any("Routes", ipamResponse.Payload.Routes))
 
 	if err = DetectIPConflictAndGatewayReachable(logger, args.IfName, hostNs, netns, ipamResponse.Payload.Ips); err != nil {
 		if errors.Is(err, constant.ErrIPConflict) || errors.Is(err, constant.ErrGatewayUnreachable) {
@@ -179,6 +179,30 @@ func CmdAdd(args *skel.CmdArgs) (err error) {
 		return err
 	}
 
+	// CNI will set the interface to up, and the kernel only sends GARPs/Unsolicited NA when the interface
+	// goes from down to up or when the link-layer address changes on the interfaces. in order to the
+	// kernel send GARPs/Unsolicited NA when the interface goes from down to up.
+	// see https://github.com/spidernet-io/spiderpool/issues/4650
+	var ipRes []net.IP
+	for _, i := range ipamResponse.Payload.Ips {
+		if i.Address != nil && *i.Address != "" {
+			ipa, _, err := net.ParseCIDR(*i.Address)
+			if err != nil {
+				logger.Error(err.Error())
+				continue
+			}
+			ipRes = append(ipRes, ipa)
+		}
+	}
+
+	err = netns.Do(func(netNS ns.NetNS) error {
+		return networking.AnnounceIPs(logger, args.IfName, ipRes)
+	})
+
+	if err != nil {
+		logger.Error(err.Error())
+	}
+
 	// Assemble the result of IPAM request response.
 	result, err := assembleResult(conf.CNIVersion, args.IfName, ipamResponse)
 	if err != nil {

pkg/multuscniconfig/multusconfig_validate.go

@@ -110,14 +110,14 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel
 			return field.Forbidden(cniTypeField, fmt.Sprintf("the cniType %s only supports %s, please remove other CNI configs", *multusConfig.Spec.CniType, macvlanConfigField.String()))
 		}
 
-		if injectRdmaResource {
+		if injectNetworkResource {
 			if err := ValidateRdmaResouce(multusConfig.Name, multusConfig.Namespace, *multusConfig.Spec.MacvlanConfig.RdmaResourceName, multusConfig.Spec.MacvlanConfig.SpiderpoolConfigPools); err != nil {
 				return field.Invalid(macvlanConfigField, *multusConfig.Spec.MacvlanConfig, err.Error())
 			}
 		}
 
 		if injectNetworkResource {
-			if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.MacvlanConfig.RdmaResourceName, multusConfig.Spec.MacvlanConfig.SpiderpoolConfigPools); err != nil {
+			if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, *multusConfig.Spec.MacvlanConfig.RdmaResourceName, multusConfig.Spec.MacvlanConfig.SpiderpoolConfigPools); err != nil {
 				return field.Invalid(macvlanConfigField, *multusConfig.Spec.MacvlanConfig, err.Error())
 			}
 		}
@@ -148,7 +148,7 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel
 		}
 
 		if injectNetworkResource {
-			if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IPVlanConfig.RdmaResourceName, multusConfig.Spec.IPVlanConfig.SpiderpoolConfigPools); err != nil {
+			if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, *multusConfig.Spec.IPVlanConfig.RdmaResourceName, multusConfig.Spec.IPVlanConfig.SpiderpoolConfigPools); err != nil {
 				return field.Invalid(ipvlanConfigField, *multusConfig.Spec.IPVlanConfig, err.Error())
 			}
 		}
@@ -183,13 +183,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel
 				return field.Invalid(sriovConfigField, *multusConfig.Spec.SriovConfig, err.Error())
 			}
 		}
-
-		if injectNetworkResource {
-			if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.SriovConfig.ResourceName, multusConfig.Spec.SriovConfig.SpiderpoolConfigPools); err != nil {
-				return field.Invalid(sriovConfigField, *multusConfig.Spec.SriovConfig, err.Error())
-			}
-		}
-
 	case constant.IBSriovCNI:
 		if multusConfig.Spec.IbSriovConfig == nil {
 			return field.Required(ibsriovConfigField, fmt.Sprintf("no %s specified", ibsriovConfigField.String()))
@@ -208,13 +201,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel
 				return field.Invalid(ibsriovConfigField, *multusConfig.Spec.IbSriovConfig, err.Error())
 			}
 		}
-
-		if injectNetworkResource {
-			if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IbSriovConfig.ResourceName, multusConfig.Spec.IbSriovConfig.SpiderpoolConfigPools); err != nil {
-				return field.Invalid(ibsriovConfigField, *multusConfig.Spec.IbSriovConfig, err.Error())
-			}
-		}
-
 	case constant.IPoIBCNI:
 		if multusConfig.Spec.IpoibConfig == nil {
 			return field.Required(ipoibConfigField, fmt.Sprintf("no %s specified", ipoibConfigField.String()))
@@ -234,12 +220,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel
 			}
 		}
 
-		if injectNetworkResource {
-			if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IpoibConfig.Master, multusConfig.Spec.IpoibConfig.SpiderpoolConfigPools); err != nil {
-				return field.Invalid(ipoibConfigField, *multusConfig.Spec.IpoibConfig, err.Error())
-			}
-		}
-
 	case constant.OvsCNI:
 		if injectRdmaResource || injectNetworkResource {
 			return field.Forbidden(cniTypeField, fmt.Sprintf("the cniType %s does not support RDMA resource or network resource injected", *multusConfig.Spec.CniType))

pkg/networking/networking/packet.go

@@ -18,6 +18,7 @@ import (
 	"syscall"
 	"time"
 
+	"go.uber.org/zap"
 	"golang.org/x/net/icmp"
 	"golang.org/x/net/ipv6"
 	"golang.org/x/sys/unix"
@@ -313,3 +314,39 @@ func ParseIPv6NeighborAdvertisementMsg(n int, buf []byte) (srcIP net.IP, mac net
 	}
 	return
 }
+
+func AnnounceIPs(logger *zap.Logger, iface string, ips []net.IP) error {
+	l, err := netlink.LinkByName(iface)
+	if err != nil {
+		return err
+	}
+
+	for _, addr := range ips {
+		logger.Debug("announcing ip", zap.String("ip", addr.String()), zap.String("interface", iface))
+		if addr.To4() != nil {
+			// send an gratuitous arp to announce the new mac address
+			if err = SendARPReuqest(l, addr, addr); err != nil {
+				logger.Error("failed to send gratuitous arps", zap.Error(err))
+			} else {
+				logger.Info("Send gratuitous arps successfully", zap.String("interface", iface))
+			}
+		} else {
+			ifi, err := net.InterfaceByName(iface)
+			if err != nil {
+				return fmt.Errorf("failed to InterfaceByName %s: %w", iface, err)
+			}
+
+			ndpClient, _, err := ndp.Listen(ifi, ndp.LinkLocal)
+			if err != nil {
+				return fmt.Errorf("failed to init ndp client: %w", err)
+			}
+			defer ndpClient.Close()
+			if err = SendUnsolicitedNeighborAdvertisement(addr, ifi, ndpClient); err != nil {
+				logger.Error("failed to send unsolicited neighbor advertisements", zap.Error(err))
+			} else {
+				logger.Info("Send unsolicited neighbor advertisements successfully", zap.String("interface", iface))
+			}
+		}
+	}
+	return nil
+}

test/e2e/spidermultus/spidermultus_test.go

@@ -805,26 +805,25 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() {
 
 	It("resoucename and ippools config must be both set when spidermutlus with annotation: cni.spidernet.io/network-resource-inject", Label("M00031"), func() {
 		var smcName string = "ann-network-resource" + common.GenerateString(10, true)
-		smc := &spiderpoolv2beta1.SpiderMultusConfig{
+		smc := &v2beta1.SpiderMultusConfig{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      smcName,
 				Namespace: namespace,
 				Annotations: map[string]string{
 					constant.AnnoNetworkResourceInject: "test",
 				},
 			},
-			Spec: spiderpoolv2beta1.MultusCNIConfigSpec{
+			Spec: v2beta1.MultusCNIConfigSpec{
 				CniType: ptr.To(constant.MacvlanCNI),
-				MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{
+				MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{
 					Master:           []string{common.NIC1},
-					EnableRdma:       true,
-					RdmaResourceName: "test",
-					SpiderpoolConfigPools: &spiderpoolv2beta1.SpiderpoolPools{
+					RdmaResourceName: ptr.To("test"),
+					SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{
 						IPv4IPPool: []string{"test"},
 					},
 				},
 				EnableCoordinator: ptr.To(true),
-				CoordinatorConfig: &spiderpoolv2beta1.CoordinatorSpec{
+				CoordinatorConfig: &v2beta1.CoordinatorSpec{
 					PodRPFilter: nil,
 				},
 			},
@@ -836,23 +835,22 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() {
 
 	It("return an err if resoucename is set without ippools config when spidermutlus with annotation: cni.spidernet.io/network-resource-inject", Label("M00032"), func() {
 		var smcName string = "ann-network-resource" + common.GenerateString(10, true)
-		smc := &spiderpoolv2beta1.SpiderMultusConfig{
+		smc := &v2beta1.SpiderMultusConfig{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:      smcName,
 				Namespace: namespace,
 				Annotations: map[string]string{
 					constant.AnnoNetworkResourceInject: "test",
 				},
 			},
-			Spec: spiderpoolv2beta1.MultusCNIConfigSpec{
+			Spec: v2beta1.MultusCNIConfigSpec{
 				CniType: ptr.To(constant.MacvlanCNI),
-				MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{
+				MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{
 					Master:           []string{common.NIC1},
-					EnableRdma:       true,
-					RdmaResourceName: "test",
+					RdmaResourceName: ptr.To("test"),
 				},
 				EnableCoordinator: ptr.To(true),
-				CoordinatorConfig: &spiderpoolv2beta1.CoordinatorSpec{
+				CoordinatorConfig: &v2beta1.CoordinatorSpec{
 					PodRPFilter: nil,
 				},
 			},