From b1055c9362630bb7cd4e0a8d3cfe912312009eba Mon Sep 17 00:00:00 2001 From: Cyclinder Kuo Date: Mon, 24 Feb 2025 16:51:14 +0800 Subject: [PATCH] IPAM sends GARPS to updating arp cache table Signed-off-by: Cyclinder Kuo --- cmd/spiderpool/cmd/command_add.go | 28 +++++++++++++-- pkg/multuscniconfig/multusconfig_validate.go | 32 ----------------- pkg/networking/networking/packet.go | 37 ++++++++++++++++++++ test/e2e/spidermultus/spidermultus_test.go | 24 ++++++------- 4 files changed, 74 insertions(+), 47 deletions(-) diff --git a/cmd/spiderpool/cmd/command_add.go b/cmd/spiderpool/cmd/command_add.go index 7525f9b84..f56010337 100644 --- a/cmd/spiderpool/cmd/command_add.go +++ b/cmd/spiderpool/cmd/command_add.go @@ -25,6 +25,7 @@ import ( "github.com/spidernet-io/spiderpool/api/v1/agent/models" "github.com/spidernet-io/spiderpool/pkg/constant" spiderpoolip "github.com/spidernet-io/spiderpool/pkg/ip" + "github.com/spidernet-io/spiderpool/pkg/networking/networking" "github.com/spidernet-io/spiderpool/pkg/openapi" ) @@ -164,8 +165,7 @@ func CmdAdd(args *skel.CmdArgs) (err error) { // do ip conflict and gateway detection logger.Sugar().Info("postIpam response", zap.Any("DNS", ipamResponse.Payload.DNS), - zap.Any("Routes", ipamResponse.Payload.Routes), - zap.Any("Ips", ipamResponse.Payload.Ips)) + zap.Any("Routes", ipamResponse.Payload.Routes)) if err = DetectIPConflictAndGatewayReachable(logger, args.IfName, hostNs, netns, ipamResponse.Payload.Ips); err != nil { if errors.Is(err, constant.ErrIPConflict) || errors.Is(err, constant.ErrGatewayUnreachable) { @@ -179,6 +179,30 @@ func CmdAdd(args *skel.CmdArgs) (err error) { return err } + // CNI will set the interface to up, and the kernel only sends GARPs/Unsolicited NA when the interface + // goes from down to up or when the link-layer address changes on the interfaces. in order to the + // kernel send GARPs/Unsolicited NA when the interface goes from down to up. + // see https://github.com/spidernet-io/spiderpool/issues/4650 + var ipRes []net.IP + for _, i := range ipamResponse.Payload.Ips { + if i.Address != nil && *i.Address != "" { + ipa, _, err := net.ParseCIDR(*i.Address) + if err != nil { + logger.Error(err.Error()) + continue + } + ipRes = append(ipRes, ipa) + } + } + + err = netns.Do(func(netNS ns.NetNS) error { + return networking.AnnounceIPs(logger, args.IfName, ipRes) + }) + + if err != nil { + logger.Error(err.Error()) + } + // Assemble the result of IPAM request response. result, err := assembleResult(conf.CNIVersion, args.IfName, ipamResponse) if err != nil { diff --git a/pkg/multuscniconfig/multusconfig_validate.go b/pkg/multuscniconfig/multusconfig_validate.go index 4e4918a89..98c7dd52f 100644 --- a/pkg/multuscniconfig/multusconfig_validate.go +++ b/pkg/multuscniconfig/multusconfig_validate.go @@ -116,12 +116,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel } } - if injectNetworkResource { - if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.MacvlanConfig.RdmaResourceName, multusConfig.Spec.MacvlanConfig.SpiderpoolConfigPools); err != nil { - return field.Invalid(macvlanConfigField, *multusConfig.Spec.MacvlanConfig, err.Error()) - } - } - case constant.IPVlanCNI: if multusConfig.Spec.IPVlanConfig == nil { return field.Required(ipvlanConfigField, fmt.Sprintf("no %s specified", ipvlanConfigField.String())) @@ -147,12 +141,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel } } - if injectNetworkResource { - if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IPVlanConfig.RdmaResourceName, multusConfig.Spec.IPVlanConfig.SpiderpoolConfigPools); err != nil { - return field.Invalid(ipvlanConfigField, *multusConfig.Spec.IPVlanConfig, err.Error()) - } - } - case constant.SriovCNI: if multusConfig.Spec.SriovConfig == nil { return field.Required(sriovConfigField, fmt.Sprintf("no %s specified", sriovConfigField.String())) @@ -183,13 +171,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel return field.Invalid(sriovConfigField, *multusConfig.Spec.SriovConfig, err.Error()) } } - - if injectNetworkResource { - if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.SriovConfig.ResourceName, multusConfig.Spec.SriovConfig.SpiderpoolConfigPools); err != nil { - return field.Invalid(sriovConfigField, *multusConfig.Spec.SriovConfig, err.Error()) - } - } - case constant.IBSriovCNI: if multusConfig.Spec.IbSriovConfig == nil { return field.Required(ibsriovConfigField, fmt.Sprintf("no %s specified", ibsriovConfigField.String())) @@ -208,13 +189,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel return field.Invalid(ibsriovConfigField, *multusConfig.Spec.IbSriovConfig, err.Error()) } } - - if injectNetworkResource { - if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IbSriovConfig.ResourceName, multusConfig.Spec.IbSriovConfig.SpiderpoolConfigPools); err != nil { - return field.Invalid(ibsriovConfigField, *multusConfig.Spec.IbSriovConfig, err.Error()) - } - } - case constant.IPoIBCNI: if multusConfig.Spec.IpoibConfig == nil { return field.Required(ipoibConfigField, fmt.Sprintf("no %s specified", ipoibConfigField.String())) @@ -234,12 +208,6 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel } } - if injectNetworkResource { - if err := ValidateNetworkResouce(multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IpoibConfig.Master, multusConfig.Spec.IpoibConfig.SpiderpoolConfigPools); err != nil { - return field.Invalid(ipoibConfigField, *multusConfig.Spec.IpoibConfig, err.Error()) - } - } - case constant.OvsCNI: if injectRdmaResource || injectNetworkResource { return field.Forbidden(cniTypeField, fmt.Sprintf("the cniType %s does not support RDMA resource or network resource injected", *multusConfig.Spec.CniType)) diff --git a/pkg/networking/networking/packet.go b/pkg/networking/networking/packet.go index bdc3de2dd..dd746ca7a 100644 --- a/pkg/networking/networking/packet.go +++ b/pkg/networking/networking/packet.go @@ -18,6 +18,7 @@ import ( "syscall" "time" + "go.uber.org/zap" "golang.org/x/net/icmp" "golang.org/x/net/ipv6" "golang.org/x/sys/unix" @@ -313,3 +314,39 @@ func ParseIPv6NeighborAdvertisementMsg(n int, buf []byte) (srcIP net.IP, mac net } return } + +func AnnounceIPs(logger *zap.Logger, iface string, ips []net.IP) error { + l, err := netlink.LinkByName(iface) + if err != nil { + return err + } + + for _, addr := range ips { + logger.Debug("announcing ip", zap.String("ip", addr.String()), zap.String("interface", iface)) + if addr.To4() != nil { + // send an gratuitous arp to announce the new mac address + if err = SendARPReuqest(l, addr, addr); err != nil { + logger.Error("failed to send gratuitous arps", zap.Error(err)) + } else { + logger.Info("Send gratuitous arps successfully", zap.String("interface", iface)) + } + } else { + ifi, err := net.InterfaceByName(iface) + if err != nil { + return fmt.Errorf("failed to InterfaceByName %s: %w", iface, err) + } + + ndpClient, _, err := ndp.Listen(ifi, ndp.LinkLocal) + if err != nil { + return fmt.Errorf("failed to init ndp client: %w", err) + } + defer ndpClient.Close() + if err = SendUnsolicitedNeighborAdvertisement(addr, ifi, ndpClient); err != nil { + logger.Error("failed to send unsolicited neighbor advertisements", zap.Error(err)) + } else { + logger.Info("Send unsolicited neighbor advertisements successfully", zap.String("interface", iface)) + } + } + } + return nil +} diff --git a/test/e2e/spidermultus/spidermultus_test.go b/test/e2e/spidermultus/spidermultus_test.go index a7d3dce93..3a47d56e8 100644 --- a/test/e2e/spidermultus/spidermultus_test.go +++ b/test/e2e/spidermultus/spidermultus_test.go @@ -805,7 +805,7 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { It("resoucename and ippools config must be both set when spidermutlus with annotation: cni.spidernet.io/network-resource-inject", Label("M00031"), func() { var smcName string = "ann-network-resource" + common.GenerateString(10, true) - smc := &spiderpoolv2beta1.SpiderMultusConfig{ + smc := &v2beta1.SpiderMultusConfig{ ObjectMeta: metav1.ObjectMeta{ Name: smcName, Namespace: namespace, @@ -813,18 +813,17 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { constant.AnnoNetworkResourceInject: "test", }, }, - Spec: spiderpoolv2beta1.MultusCNIConfigSpec{ + Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.MacvlanCNI), - MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{ + MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ Master: []string{common.NIC1}, - EnableRdma: true, - RdmaResourceName: "test", - SpiderpoolConfigPools: &spiderpoolv2beta1.SpiderpoolPools{ + RdmaResourceName: ptr.To("test"), + SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, }, }, EnableCoordinator: ptr.To(true), - CoordinatorConfig: &spiderpoolv2beta1.CoordinatorSpec{ + CoordinatorConfig: &v2beta1.CoordinatorSpec{ PodRPFilter: nil, }, }, @@ -836,7 +835,7 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { It("return an err if resoucename is set without ippools config when spidermutlus with annotation: cni.spidernet.io/network-resource-inject", Label("M00032"), func() { var smcName string = "ann-network-resource" + common.GenerateString(10, true) - smc := &spiderpoolv2beta1.SpiderMultusConfig{ + smc := &v2beta1.SpiderMultusConfig{ ObjectMeta: metav1.ObjectMeta{ Name: smcName, Namespace: namespace, @@ -844,15 +843,14 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { constant.AnnoNetworkResourceInject: "test", }, }, - Spec: spiderpoolv2beta1.MultusCNIConfigSpec{ + Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.MacvlanCNI), - MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{ + MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ Master: []string{common.NIC1}, - EnableRdma: true, - RdmaResourceName: "test", + RdmaResourceName: ptr.To("test"), }, EnableCoordinator: ptr.To(true), - CoordinatorConfig: &spiderpoolv2beta1.CoordinatorSpec{ + CoordinatorConfig: &v2beta1.CoordinatorSpec{ PodRPFilter: nil, }, },